From 53433090a39195d9df7c920d2e4d142f89abae31 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Wed, 12 Sep 2018 14:09:17 -0400 Subject: [PATCH 01/14] pgsql: update the query format --- .../pgsql/migrations/00001_initial_schema.go | 206 +++++++++--------- 1 file changed, 103 insertions(+), 103 deletions(-) diff --git a/database/pgsql/migrations/00001_initial_schema.go b/database/pgsql/migrations/00001_initial_schema.go index e51037ee..2f6638b8 100644 --- a/database/pgsql/migrations/00001_initial_schema.go +++ b/database/pgsql/migrations/00001_initial_schema.go @@ -22,170 +22,170 @@ func init() { Up: migrate.Queries([]string{ // namespaces `CREATE TABLE IF NOT EXISTS namespace ( - id SERIAL PRIMARY KEY, - name TEXT NULL, - version_format TEXT, - UNIQUE (name, version_format));`, + id SERIAL PRIMARY KEY, + name TEXT NULL, + version_format TEXT, + UNIQUE (name, version_format));`, `CREATE INDEX ON namespace(name);`, // features `CREATE TABLE IF NOT EXISTS feature ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - version TEXT NOT NULL, - version_format TEXT NOT NULL, - UNIQUE (name, version, version_format));`, + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + version TEXT NOT NULL, + version_format TEXT NOT NULL, + UNIQUE (name, version, version_format));`, `CREATE INDEX ON feature(name);`, `CREATE TABLE IF NOT EXISTS namespaced_feature ( - id SERIAL PRIMARY KEY, - namespace_id INT REFERENCES namespace, - feature_id INT REFERENCES feature, - UNIQUE (namespace_id, feature_id));`, + id SERIAL PRIMARY KEY, + namespace_id INT REFERENCES namespace, + feature_id INT REFERENCES feature, + UNIQUE (namespace_id, feature_id));`, // layers `CREATE TABLE IF NOT EXISTS layer( - id SERIAL PRIMARY KEY, - hash TEXT NOT NULL UNIQUE);`, + id SERIAL PRIMARY KEY, + hash TEXT NOT NULL UNIQUE);`, `CREATE TABLE IF NOT EXISTS layer_feature ( - id SERIAL PRIMARY KEY, - layer_id INT REFERENCES layer ON DELETE CASCADE, - feature_id INT REFERENCES feature ON DELETE CASCADE, - UNIQUE (layer_id, feature_id));`, + id SERIAL PRIMARY KEY, + layer_id INT REFERENCES layer ON DELETE CASCADE, + feature_id INT REFERENCES feature ON DELETE CASCADE, + UNIQUE (layer_id, feature_id));`, `CREATE INDEX ON layer_feature(layer_id);`, `CREATE TABLE IF NOT EXISTS layer_lister ( - id SERIAL PRIMARY KEY, - layer_id INT REFERENCES layer ON DELETE CASCADE, - lister TEXT NOT NULL, - UNIQUE (layer_id, lister));`, + id SERIAL PRIMARY KEY, + layer_id INT REFERENCES layer ON DELETE CASCADE, + lister TEXT NOT NULL, + UNIQUE (layer_id, lister));`, `CREATE INDEX ON layer_lister(layer_id);`, `CREATE TABLE IF NOT EXISTS layer_detector ( - id SERIAL PRIMARY KEY, - layer_id INT REFERENCES layer ON DELETE CASCADE, - detector TEXT, - UNIQUE (layer_id, detector));`, + id SERIAL PRIMARY KEY, + layer_id INT REFERENCES layer ON DELETE CASCADE, + detector TEXT, + UNIQUE (layer_id, detector));`, `CREATE INDEX ON layer_detector(layer_id);`, `CREATE TABLE IF NOT EXISTS layer_namespace ( - id SERIAL PRIMARY KEY, - layer_id INT REFERENCES layer ON DELETE CASCADE, - namespace_id INT REFERENCES namespace ON DELETE CASCADE, - UNIQUE (layer_id, namespace_id));`, + id SERIAL PRIMARY KEY, + layer_id INT REFERENCES layer ON DELETE CASCADE, + namespace_id INT REFERENCES namespace ON DELETE CASCADE, + UNIQUE (layer_id, namespace_id));`, `CREATE INDEX ON layer_namespace(layer_id);`, // ancestry `CREATE TABLE IF NOT EXISTS ancestry ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL UNIQUE);`, + id SERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE);`, `CREATE TABLE IF NOT EXISTS ancestry_layer ( - id SERIAL PRIMARY KEY, - ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, - ancestry_index INT NOT NULL, - layer_id INT REFERENCES layer ON DELETE RESTRICT, - UNIQUE (ancestry_id, ancestry_index));`, + id SERIAL PRIMARY KEY, + ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, + ancestry_index INT NOT NULL, + layer_id INT REFERENCES layer ON DELETE RESTRICT, + UNIQUE (ancestry_id, ancestry_index));`, `CREATE INDEX ON ancestry_layer(ancestry_id);`, `CREATE TABLE IF NOT EXISTS ancestry_feature( - id SERIAL PRIMARY KEY, - ancestry_layer_id INT REFERENCES ancestry_layer ON DELETE CASCADE, - namespaced_feature_id INT REFERENCES namespaced_feature ON DELETE CASCADE, - UNIQUE (ancestry_layer_id, namespaced_feature_id));`, + id SERIAL PRIMARY KEY, + ancestry_layer_id INT REFERENCES ancestry_layer ON DELETE CASCADE, + namespaced_feature_id INT REFERENCES namespaced_feature ON DELETE CASCADE, + UNIQUE (ancestry_layer_id, namespaced_feature_id));`, `CREATE TABLE IF NOT EXISTS ancestry_lister ( - id SERIAL PRIMARY KEY, - ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, - lister TEXT, - UNIQUE (ancestry_id, lister));`, + id SERIAL PRIMARY KEY, + ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, + lister TEXT, + UNIQUE (ancestry_id, lister));`, `CREATE INDEX ON ancestry_lister(ancestry_id);`, `CREATE TABLE IF NOT EXISTS ancestry_detector ( - id SERIAL PRIMARY KEY, - ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, - detector TEXT, - UNIQUE (ancestry_id, detector));`, + id SERIAL PRIMARY KEY, + ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, + detector TEXT, + UNIQUE (ancestry_id, detector));`, `CREATE INDEX ON ancestry_detector(ancestry_id);`, `CREATE TYPE severity AS ENUM ('Unknown', 'Negligible', 'Low', 'Medium', 'High', 'Critical', 'Defcon1');`, // vulnerability `CREATE TABLE IF NOT EXISTS vulnerability ( - id SERIAL PRIMARY KEY, - namespace_id INT NOT NULL REFERENCES Namespace, - name TEXT NOT NULL, - description TEXT NULL, - link TEXT NULL, - severity severity NOT NULL, - metadata TEXT NULL, - created_at TIMESTAMP WITH TIME ZONE, - deleted_at TIMESTAMP WITH TIME ZONE NULL);`, + id SERIAL PRIMARY KEY, + namespace_id INT NOT NULL REFERENCES Namespace, + name TEXT NOT NULL, + description TEXT NULL, + link TEXT NULL, + severity severity NOT NULL, + metadata TEXT NULL, + created_at TIMESTAMP WITH TIME ZONE, + deleted_at TIMESTAMP WITH TIME ZONE NULL);`, `CREATE INDEX ON vulnerability(namespace_id, name);`, `CREATE INDEX ON vulnerability(namespace_id);`, `CREATE TABLE IF NOT EXISTS vulnerability_affected_feature ( - id SERIAL PRIMARY KEY, - vulnerability_id INT NOT NULL REFERENCES vulnerability ON DELETE CASCADE, - feature_name TEXT NOT NULL, - affected_version TEXT, - fixedin TEXT);`, + id SERIAL PRIMARY KEY, + vulnerability_id INT NOT NULL REFERENCES vulnerability ON DELETE CASCADE, + feature_name TEXT NOT NULL, + affected_version TEXT, + fixedin TEXT);`, `CREATE INDEX ON vulnerability_affected_feature(vulnerability_id, feature_name);`, `CREATE TABLE IF NOT EXISTS vulnerability_affected_namespaced_feature( - id SERIAL PRIMARY KEY, - vulnerability_id INT NOT NULL REFERENCES vulnerability ON DELETE CASCADE, - namespaced_feature_id INT NOT NULL REFERENCES namespaced_feature ON DELETE CASCADE, - added_by INT NOT NULL REFERENCES vulnerability_affected_feature ON DELETE CASCADE, - UNIQUE (vulnerability_id, namespaced_feature_id));`, + id SERIAL PRIMARY KEY, + vulnerability_id INT NOT NULL REFERENCES vulnerability ON DELETE CASCADE, + namespaced_feature_id INT NOT NULL REFERENCES namespaced_feature ON DELETE CASCADE, + added_by INT NOT NULL REFERENCES vulnerability_affected_feature ON DELETE CASCADE, + UNIQUE (vulnerability_id, namespaced_feature_id));`, `CREATE INDEX ON vulnerability_affected_namespaced_feature(namespaced_feature_id);`, `CREATE TABLE IF NOT EXISTS KeyValue ( - id SERIAL PRIMARY KEY, - key TEXT NOT NULL UNIQUE, - value TEXT);`, + id SERIAL PRIMARY KEY, + key TEXT NOT NULL UNIQUE, + value TEXT);`, `CREATE TABLE IF NOT EXISTS Lock ( - id SERIAL PRIMARY KEY, - name VARCHAR(64) NOT NULL UNIQUE, - owner VARCHAR(64) NOT NULL, - until TIMESTAMP WITH TIME ZONE);`, + id SERIAL PRIMARY KEY, + name VARCHAR(64) NOT NULL UNIQUE, + owner VARCHAR(64) NOT NULL, + until TIMESTAMP WITH TIME ZONE);`, `CREATE INDEX ON Lock (owner);`, // Notification `CREATE TABLE IF NOT EXISTS Vulnerability_Notification ( - id SERIAL PRIMARY KEY, - name VARCHAR(64) NOT NULL UNIQUE, - created_at TIMESTAMP WITH TIME ZONE, - notified_at TIMESTAMP WITH TIME ZONE NULL, - deleted_at TIMESTAMP WITH TIME ZONE NULL, - old_vulnerability_id INT NULL REFERENCES Vulnerability ON DELETE CASCADE, - new_vulnerability_id INT NULL REFERENCES Vulnerability ON DELETE CASCADE);`, + id SERIAL PRIMARY KEY, + name VARCHAR(64) NOT NULL UNIQUE, + created_at TIMESTAMP WITH TIME ZONE, + notified_at TIMESTAMP WITH TIME ZONE NULL, + deleted_at TIMESTAMP WITH TIME ZONE NULL, + old_vulnerability_id INT NULL REFERENCES Vulnerability ON DELETE CASCADE, + new_vulnerability_id INT NULL REFERENCES Vulnerability ON DELETE CASCADE);`, `CREATE INDEX ON Vulnerability_Notification (notified_at);`, }), Down: migrate.Queries([]string{ `DROP TABLE IF EXISTS - ancestry, - ancestry_layer, - ancestry_detector, - ancestry_lister, - ancestry_feature, - feature, - namespaced_feature, - keyvalue, - layer, - layer_detector, - layer_feature, - layer_lister, - layer_namespace, - lock, - namespace, - vulnerability, - vulnerability_affected_feature, - vulnerability_affected_namespaced_feature, - vulnerability_notification - CASCADE;`, + ancestry, + ancestry_layer, + ancestry_detector, + ancestry_lister, + ancestry_feature, + feature, + namespaced_feature, + keyvalue, + layer, + layer_detector, + layer_feature, + layer_lister, + layer_namespace, + lock, + namespace, + vulnerability, + vulnerability_affected_feature, + vulnerability_affected_namespaced_feature, + vulnerability_notification + CASCADE;`, `DROP TYPE IF EXISTS severity;`, }), }) From 0609ed964b0673806462a24147e6028da85d8a38 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Wed, 12 Sep 2018 16:15:35 -0400 Subject: [PATCH 02/14] config: removed worker config All processors will now be used to process the layers. --- cmd/clair/config.go | 7 ------- cmd/clair/main.go | 23 +++++------------------ config.yaml.sample | 13 ------------- worker.go | 5 ----- 4 files changed, 5 insertions(+), 43 deletions(-) diff --git a/cmd/clair/config.go b/cmd/clair/config.go index 09a01364..d5d2c402 100644 --- a/cmd/clair/config.go +++ b/cmd/clair/config.go @@ -26,8 +26,6 @@ import ( "github.com/coreos/clair" "github.com/coreos/clair/api" "github.com/coreos/clair/database" - "github.com/coreos/clair/ext/featurefmt" - "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/notification" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/pkg/pagination" @@ -47,7 +45,6 @@ type File struct { type Config struct { Database database.RegistrableComponentConfig Updater *clair.UpdaterConfig - Worker *clair.WorkerConfig Notifier *notification.Config API *api.Config } @@ -62,10 +59,6 @@ func DefaultConfig() Config { EnabledUpdaters: vulnsrc.ListUpdaters(), Interval: 1 * time.Hour, }, - Worker: &clair.WorkerConfig{ - EnabledDetectors: featurens.ListDetectors(), - EnabledListers: featurefmt.ListListers(), - }, API: &api.Config{ HealthAddr: "0.0.0.0:6061", Addr: "0.0.0.0:6060", diff --git a/cmd/clair/main.go b/cmd/clair/main.go index da046351..e329af34 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -112,29 +112,16 @@ func configClairVersion(config *Config) { "Updaters": strings.Join(updaters, ","), }).Info("Clair registered components") - unregDetectors := strutil.CompareStringLists(config.Worker.EnabledDetectors, detectors) - unregListers := strutil.CompareStringLists(config.Worker.EnabledListers, listers) unregUpdaters := strutil.CompareStringLists(config.Updater.EnabledUpdaters, updaters) - if len(unregDetectors) != 0 || len(unregListers) != 0 || len(unregUpdaters) != 0 { + if len(unregUpdaters) != 0 { log.WithFields(log.Fields{ - "Unknown Detectors": strings.Join(unregDetectors, ","), - "Unknown Listers": strings.Join(unregListers, ","), - "Unknown Updaters": strings.Join(unregUpdaters, ","), - "Available Listers": strings.Join(featurefmt.ListListers(), ","), - "Available Detectors": strings.Join(featurens.ListDetectors(), ","), - "Available Updaters": strings.Join(vulnsrc.ListUpdaters(), ","), + "Unknown Updaters": strings.Join(unregUpdaters, ","), + "Available Updaters": strings.Join(vulnsrc.ListUpdaters(), ","), }).Fatal("Unknown or unregistered components are configured") } - // verify the user specified detectors/listers/updaters are implemented. If - // some are not registered, it logs warning and won't use the unregistered - // extensions. - - clair.Processors = database.Processors{ - Detectors: strutil.CompareStringListsInBoth(config.Worker.EnabledDetectors, detectors), - Listers: strutil.CompareStringListsInBoth(config.Worker.EnabledListers, listers), - } - + // All listers and detectors are enabled. + clair.Processors = database.Processors{Detectors: detectors, Listers: listers} clair.EnabledUpdaters = strutil.CompareStringListsInBoth(config.Updater.EnabledUpdaters, updaters) } diff --git a/config.yaml.sample b/config.yaml.sample index cb6fd5d3..1128aa10 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -51,19 +51,6 @@ clair: keyfile: certfile: - worker: - namespace_detectors: - - os-release - - lsb-release - - apt-sources - - alpine-release - - redhat-release - - feature_listers: - - apk - - dpkg - - rpm - updater: # Frequency the database will be updated with vulnerabilities from the default data sources # The value 0 disables the updater entirely. diff --git a/worker.go b/worker.go index 106a4c3e..5890acbb 100644 --- a/worker.go +++ b/worker.go @@ -52,11 +52,6 @@ var ( Processors database.Processors ) -type WorkerConfig struct { - EnabledDetectors []string `yaml:"namespace_detectors"` - EnabledListers []string `yaml:"feature_listers"` -} - // LayerRequest represents all information necessary to download and process a // layer. type LayerRequest struct { From 9f5d1ea4e16793ebd9390673aed34855671b5c24 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Wed, 12 Sep 2018 16:41:45 -0400 Subject: [PATCH 03/14] v3: associate feature and namespace with detector --- api/v3/clairpb/clair.pb.go | 352 +++++++++++++++++++----------- api/v3/clairpb/clair.proto | 48 ++-- api/v3/clairpb/clair.swagger.json | 73 +++++-- api/v3/util.go | 6 +- 4 files changed, 312 insertions(+), 167 deletions(-) diff --git a/api/v3/clairpb/clair.pb.go b/api/v3/clairpb/clair.pb.go index ac441ec0..b6b2184e 100644 --- a/api/v3/clairpb/clair.pb.go +++ b/api/v3/clairpb/clair.pb.go @@ -9,6 +9,8 @@ It is generated from these files: It has these top-level messages: Vulnerability + Detector + Namespace Feature Layer ClairStatus @@ -48,6 +50,30 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +type Detector_Type int32 + +const ( + Detector_DETECTOR_TYPE_INVALID Detector_Type = 0 + Detector_DETECTOR_TYPE_NAMESPACE Detector_Type = 1 + Detector_DETECTOR_TYPE_FEATURE Detector_Type = 2 +) + +var Detector_Type_name = map[int32]string{ + 0: "DETECTOR_TYPE_INVALID", + 1: "DETECTOR_TYPE_NAMESPACE", + 2: "DETECTOR_TYPE_FEATURE", +} +var Detector_Type_value = map[string]int32{ + "DETECTOR_TYPE_INVALID": 0, + "DETECTOR_TYPE_NAMESPACE": 1, + "DETECTOR_TYPE_FEATURE": 2, +} + +func (x Detector_Type) String() string { + return proto.EnumName(Detector_Type_name, int32(x)) +} +func (Detector_Type) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1, 0} } + type Vulnerability struct { // The name of the vulnerability. Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` @@ -130,23 +156,88 @@ func (m *Vulnerability) GetAffectedVersions() []*Feature { return nil } +type Detector struct { + // The name of the detector. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The version of the detector. + Version string `protobuf:"bytes,2,opt,name=version" json:"version,omitempty"` + // The type of the detector. + Type Detector_Type `protobuf:"varint,3,opt,name=type,enum=coreos.clair.Detector_Type" json:"type,omitempty"` +} + +func (m *Detector) Reset() { *m = Detector{} } +func (m *Detector) String() string { return proto.CompactTextString(m) } +func (*Detector) ProtoMessage() {} +func (*Detector) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *Detector) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Detector) GetVersion() string { + if m != nil { + return m.Version + } + return "" +} + +func (m *Detector) GetType() Detector_Type { + if m != nil { + return m.Type + } + return Detector_DETECTOR_TYPE_INVALID +} + +type Namespace struct { + // The name of the namespace. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The detector used to detect the namespace. This only exists when present in + // an Ancestry Feature. + Detector *Detector `protobuf:"bytes,2,opt,name=detector" json:"detector,omitempty"` +} + +func (m *Namespace) Reset() { *m = Namespace{} } +func (m *Namespace) String() string { return proto.CompactTextString(m) } +func (*Namespace) ProtoMessage() {} +func (*Namespace) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } + +func (m *Namespace) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Namespace) GetDetector() *Detector { + if m != nil { + return m.Detector + } + return nil +} + type Feature struct { // The name of the feature. Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - // The name of the namespace in which the feature is detected. - NamespaceName string `protobuf:"bytes,2,opt,name=namespace_name,json=namespaceName" json:"namespace_name,omitempty"` + // The namespace in which the feature is detected. + Namespace *Namespace `protobuf:"bytes,2,opt,name=namespace" json:"namespace,omitempty"` // The specific version of this feature. Version string `protobuf:"bytes,3,opt,name=version" json:"version,omitempty"` // The format used to parse version numbers for the feature. VersionFormat string `protobuf:"bytes,4,opt,name=version_format,json=versionFormat" json:"version_format,omitempty"` + // The detector used to detect this feature. This only exists when present in + // an Ancestry. + Detector *Detector `protobuf:"bytes,5,opt,name=detector" json:"detector,omitempty"` // The list of vulnerabilities that affect the feature. - Vulnerabilities []*Vulnerability `protobuf:"bytes,5,rep,name=vulnerabilities" json:"vulnerabilities,omitempty"` + Vulnerabilities []*Vulnerability `protobuf:"bytes,6,rep,name=vulnerabilities" json:"vulnerabilities,omitempty"` } func (m *Feature) Reset() { *m = Feature{} } func (m *Feature) String() string { return proto.CompactTextString(m) } func (*Feature) ProtoMessage() {} -func (*Feature) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } +func (*Feature) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } func (m *Feature) GetName() string { if m != nil { @@ -155,11 +246,11 @@ func (m *Feature) GetName() string { return "" } -func (m *Feature) GetNamespaceName() string { +func (m *Feature) GetNamespace() *Namespace { if m != nil { - return m.NamespaceName + return m.Namespace } - return "" + return nil } func (m *Feature) GetVersion() string { @@ -176,6 +267,13 @@ func (m *Feature) GetVersionFormat() string { return "" } +func (m *Feature) GetDetector() *Detector { + if m != nil { + return m.Detector + } + return nil +} + func (m *Feature) GetVulnerabilities() []*Vulnerability { if m != nil { return m.Vulnerabilities @@ -191,7 +289,7 @@ type Layer struct { func (m *Layer) Reset() { *m = Layer{} } func (m *Layer) String() string { return proto.CompactTextString(m) } func (*Layer) ProtoMessage() {} -func (*Layer) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } +func (*Layer) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } func (m *Layer) GetHash() string { if m != nil { @@ -201,27 +299,18 @@ func (m *Layer) GetHash() string { } type ClairStatus struct { - // The configured list of feature listers used to scan an ancestry. - Listers []string `protobuf:"bytes,1,rep,name=listers" json:"listers,omitempty"` - // The configured list of namespace detectors used to scan an ancestry. - Detectors []string `protobuf:"bytes,2,rep,name=detectors" json:"detectors,omitempty"` + // The implemented detectors in this Clair instance + Detectors []*Detector `protobuf:"bytes,1,rep,name=detectors" json:"detectors,omitempty"` // The time at which the updater last ran. - LastUpdateTime *google_protobuf.Timestamp `protobuf:"bytes,3,opt,name=last_update_time,json=lastUpdateTime" json:"last_update_time,omitempty"` + LastUpdateTime *google_protobuf.Timestamp `protobuf:"bytes,2,opt,name=last_update_time,json=lastUpdateTime" json:"last_update_time,omitempty"` } func (m *ClairStatus) Reset() { *m = ClairStatus{} } func (m *ClairStatus) String() string { return proto.CompactTextString(m) } func (*ClairStatus) ProtoMessage() {} -func (*ClairStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } +func (*ClairStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } -func (m *ClairStatus) GetListers() []string { - if m != nil { - return m.Listers - } - return nil -} - -func (m *ClairStatus) GetDetectors() []string { +func (m *ClairStatus) GetDetectors() []*Detector { if m != nil { return m.Detectors } @@ -243,7 +332,7 @@ type GetAncestryRequest struct { func (m *GetAncestryRequest) Reset() { *m = GetAncestryRequest{} } func (m *GetAncestryRequest) String() string { return proto.CompactTextString(m) } func (*GetAncestryRequest) ProtoMessage() {} -func (*GetAncestryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } +func (*GetAncestryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } func (m *GetAncestryRequest) GetAncestryName() string { if m != nil { @@ -262,7 +351,7 @@ type GetAncestryResponse struct { func (m *GetAncestryResponse) Reset() { *m = GetAncestryResponse{} } func (m *GetAncestryResponse) String() string { return proto.CompactTextString(m) } func (*GetAncestryResponse) ProtoMessage() {} -func (*GetAncestryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } +func (*GetAncestryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } func (m *GetAncestryResponse) GetAncestry() *GetAncestryResponse_Ancestry { if m != nil { @@ -289,7 +378,7 @@ func (m *GetAncestryResponse_AncestryLayer) Reset() { *m = GetAncestryRe func (m *GetAncestryResponse_AncestryLayer) String() string { return proto.CompactTextString(m) } func (*GetAncestryResponse_AncestryLayer) ProtoMessage() {} func (*GetAncestryResponse_AncestryLayer) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{5, 0} + return fileDescriptor0, []int{7, 0} } func (m *GetAncestryResponse_AncestryLayer) GetLayer() *Layer { @@ -309,18 +398,17 @@ func (m *GetAncestryResponse_AncestryLayer) GetDetectedFeatures() []*Feature { type GetAncestryResponse_Ancestry struct { // The name of the desired ancestry. Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - // The configured list of feature listers used to scan this ancestry. - ScannedListers []string `protobuf:"bytes,4,rep,name=scanned_listers,json=scannedListers" json:"scanned_listers,omitempty"` - // The configured list of namespace detectors used to scan an ancestry. - ScannedDetectors []string `protobuf:"bytes,5,rep,name=scanned_detectors,json=scannedDetectors" json:"scanned_detectors,omitempty"` + // The detectors used to scan this Ancestry. It may not be the current set + // of detectors in clair status. + Detectors []*Detector `protobuf:"bytes,2,rep,name=detectors" json:"detectors,omitempty"` // The list of layers along with detected features in each. - Layers []*GetAncestryResponse_AncestryLayer `protobuf:"bytes,6,rep,name=layers" json:"layers,omitempty"` + Layers []*GetAncestryResponse_AncestryLayer `protobuf:"bytes,3,rep,name=layers" json:"layers,omitempty"` } func (m *GetAncestryResponse_Ancestry) Reset() { *m = GetAncestryResponse_Ancestry{} } func (m *GetAncestryResponse_Ancestry) String() string { return proto.CompactTextString(m) } func (*GetAncestryResponse_Ancestry) ProtoMessage() {} -func (*GetAncestryResponse_Ancestry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5, 1} } +func (*GetAncestryResponse_Ancestry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7, 1} } func (m *GetAncestryResponse_Ancestry) GetName() string { if m != nil { @@ -329,16 +417,9 @@ func (m *GetAncestryResponse_Ancestry) GetName() string { return "" } -func (m *GetAncestryResponse_Ancestry) GetScannedListers() []string { +func (m *GetAncestryResponse_Ancestry) GetDetectors() []*Detector { if m != nil { - return m.ScannedListers - } - return nil -} - -func (m *GetAncestryResponse_Ancestry) GetScannedDetectors() []string { - if m != nil { - return m.ScannedDetectors + return m.Detectors } return nil } @@ -364,7 +445,7 @@ type PostAncestryRequest struct { func (m *PostAncestryRequest) Reset() { *m = PostAncestryRequest{} } func (m *PostAncestryRequest) String() string { return proto.CompactTextString(m) } func (*PostAncestryRequest) ProtoMessage() {} -func (*PostAncestryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } +func (*PostAncestryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } func (m *PostAncestryRequest) GetAncestryName() string { if m != nil { @@ -400,7 +481,7 @@ func (m *PostAncestryRequest_PostLayer) Reset() { *m = PostAncestryReque func (m *PostAncestryRequest_PostLayer) String() string { return proto.CompactTextString(m) } func (*PostAncestryRequest_PostLayer) ProtoMessage() {} func (*PostAncestryRequest_PostLayer) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{6, 0} + return fileDescriptor0, []int{8, 0} } func (m *PostAncestryRequest_PostLayer) GetHash() string { @@ -432,7 +513,7 @@ type PostAncestryResponse struct { func (m *PostAncestryResponse) Reset() { *m = PostAncestryResponse{} } func (m *PostAncestryResponse) String() string { return proto.CompactTextString(m) } func (*PostAncestryResponse) ProtoMessage() {} -func (*PostAncestryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } +func (*PostAncestryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } func (m *PostAncestryResponse) GetStatus() *ClairStatus { if m != nil { @@ -457,7 +538,7 @@ type GetNotificationRequest struct { func (m *GetNotificationRequest) Reset() { *m = GetNotificationRequest{} } func (m *GetNotificationRequest) String() string { return proto.CompactTextString(m) } func (*GetNotificationRequest) ProtoMessage() {} -func (*GetNotificationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } +func (*GetNotificationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } func (m *GetNotificationRequest) GetOldVulnerabilityPage() string { if m != nil { @@ -495,7 +576,7 @@ type GetNotificationResponse struct { func (m *GetNotificationResponse) Reset() { *m = GetNotificationResponse{} } func (m *GetNotificationResponse) String() string { return proto.CompactTextString(m) } func (*GetNotificationResponse) ProtoMessage() {} -func (*GetNotificationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } +func (*GetNotificationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } func (m *GetNotificationResponse) GetNotification() *GetNotificationResponse_Notification { if m != nil { @@ -523,7 +604,7 @@ func (m *GetNotificationResponse_Notification) Reset() { *m = GetNotific func (m *GetNotificationResponse_Notification) String() string { return proto.CompactTextString(m) } func (*GetNotificationResponse_Notification) ProtoMessage() {} func (*GetNotificationResponse_Notification) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{9, 0} + return fileDescriptor0, []int{11, 0} } func (m *GetNotificationResponse_Notification) GetName() string { @@ -585,7 +666,7 @@ type PagedVulnerableAncestries struct { func (m *PagedVulnerableAncestries) Reset() { *m = PagedVulnerableAncestries{} } func (m *PagedVulnerableAncestries) String() string { return proto.CompactTextString(m) } func (*PagedVulnerableAncestries) ProtoMessage() {} -func (*PagedVulnerableAncestries) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } +func (*PagedVulnerableAncestries) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } func (m *PagedVulnerableAncestries) GetCurrentPage() string { if m != nil { @@ -638,7 +719,7 @@ func (m *PagedVulnerableAncestries_IndexedAncestryName) String() string { } func (*PagedVulnerableAncestries_IndexedAncestryName) ProtoMessage() {} func (*PagedVulnerableAncestries_IndexedAncestryName) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{10, 0} + return fileDescriptor0, []int{12, 0} } func (m *PagedVulnerableAncestries_IndexedAncestryName) GetIndex() int32 { @@ -663,7 +744,7 @@ type MarkNotificationAsReadRequest struct { func (m *MarkNotificationAsReadRequest) Reset() { *m = MarkNotificationAsReadRequest{} } func (m *MarkNotificationAsReadRequest) String() string { return proto.CompactTextString(m) } func (*MarkNotificationAsReadRequest) ProtoMessage() {} -func (*MarkNotificationAsReadRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } +func (*MarkNotificationAsReadRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } func (m *MarkNotificationAsReadRequest) GetName() string { if m != nil { @@ -678,7 +759,7 @@ type MarkNotificationAsReadResponse struct { func (m *MarkNotificationAsReadResponse) Reset() { *m = MarkNotificationAsReadResponse{} } func (m *MarkNotificationAsReadResponse) String() string { return proto.CompactTextString(m) } func (*MarkNotificationAsReadResponse) ProtoMessage() {} -func (*MarkNotificationAsReadResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } +func (*MarkNotificationAsReadResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} } type GetStatusRequest struct { } @@ -686,7 +767,7 @@ type GetStatusRequest struct { func (m *GetStatusRequest) Reset() { *m = GetStatusRequest{} } func (m *GetStatusRequest) String() string { return proto.CompactTextString(m) } func (*GetStatusRequest) ProtoMessage() {} -func (*GetStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } +func (*GetStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} } type GetStatusResponse struct { // The status of the current Clair instance. @@ -696,7 +777,7 @@ type GetStatusResponse struct { func (m *GetStatusResponse) Reset() { *m = GetStatusResponse{} } func (m *GetStatusResponse) String() string { return proto.CompactTextString(m) } func (*GetStatusResponse) ProtoMessage() {} -func (*GetStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} } +func (*GetStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} } func (m *GetStatusResponse) GetStatus() *ClairStatus { if m != nil { @@ -707,6 +788,8 @@ func (m *GetStatusResponse) GetStatus() *ClairStatus { func init() { proto.RegisterType((*Vulnerability)(nil), "coreos.clair.Vulnerability") + proto.RegisterType((*Detector)(nil), "coreos.clair.Detector") + proto.RegisterType((*Namespace)(nil), "coreos.clair.Namespace") proto.RegisterType((*Feature)(nil), "coreos.clair.Feature") proto.RegisterType((*Layer)(nil), "coreos.clair.Layer") proto.RegisterType((*ClairStatus)(nil), "coreos.clair.ClairStatus") @@ -726,6 +809,7 @@ func init() { proto.RegisterType((*MarkNotificationAsReadResponse)(nil), "coreos.clair.MarkNotificationAsReadResponse") proto.RegisterType((*GetStatusRequest)(nil), "coreos.clair.GetStatusRequest") proto.RegisterType((*GetStatusResponse)(nil), "coreos.clair.GetStatusResponse") + proto.RegisterEnum("coreos.clair.Detector_Type", Detector_Type_name, Detector_Type_value) } // Reference imports to suppress errors if they are not otherwise used. @@ -1007,83 +1091,89 @@ var _StatusService_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("api/v3/clairpb/clair.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1237 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x57, 0x4b, 0x6f, 0x1b, 0xd5, - 0x17, 0xd7, 0xd8, 0x71, 0x1c, 0x1f, 0xdb, 0x49, 0x7a, 0x93, 0xa6, 0x93, 0x49, 0x1f, 0xc9, 0xfc, - 0xff, 0x55, 0x4b, 0x8b, 0x6c, 0xe1, 0xb2, 0x68, 0xcb, 0x02, 0xa5, 0x8f, 0x84, 0x4a, 0xa5, 0xaa, - 0xa6, 0xd0, 0x05, 0x08, 0x59, 0xd7, 0x33, 0xc7, 0xc9, 0x28, 0xe3, 0x19, 0x33, 0xf7, 0xda, 0x89, - 0x55, 0x95, 0x05, 0x5b, 0x76, 0xb0, 0xe0, 0x33, 0xb0, 0xe1, 0x1b, 0xb0, 0x62, 0xcb, 0x02, 0xc1, - 0x16, 0x76, 0x2c, 0xf8, 0x02, 0xec, 0xd1, 0x7d, 0x4d, 0x66, 0x12, 0xe7, 0xd1, 0xb2, 0xf2, 0x9c, - 0xf7, 0xeb, 0x77, 0xcf, 0x49, 0xc0, 0xa1, 0xc3, 0xb0, 0x3d, 0xbe, 0xd3, 0xf6, 0x23, 0x1a, 0xa6, - 0xc3, 0x9e, 0xfa, 0x6d, 0x0d, 0xd3, 0x84, 0x27, 0xa4, 0xe1, 0x27, 0x29, 0x26, 0xac, 0x25, 0x79, - 0xce, 0xb5, 0x9d, 0x24, 0xd9, 0x89, 0xb0, 0x2d, 0x65, 0xbd, 0x51, 0xbf, 0xcd, 0xc3, 0x01, 0x32, - 0x4e, 0x07, 0x43, 0xa5, 0xee, 0x5c, 0xd6, 0x0a, 0xc2, 0x23, 0x8d, 0xe3, 0x84, 0x53, 0x1e, 0x26, - 0x31, 0x53, 0x52, 0xf7, 0xfb, 0x12, 0x34, 0x5f, 0x8e, 0xa2, 0x18, 0x53, 0xda, 0x0b, 0xa3, 0x90, - 0x4f, 0x08, 0x81, 0x99, 0x98, 0x0e, 0xd0, 0xb6, 0xd6, 0xad, 0x9b, 0x35, 0x4f, 0x7e, 0x93, 0xeb, - 0x30, 0x2f, 0x7e, 0xd9, 0x90, 0xfa, 0xd8, 0x95, 0xd2, 0x92, 0x94, 0x36, 0x33, 0xee, 0x33, 0xa1, - 0xb6, 0x0e, 0xf5, 0x00, 0x99, 0x9f, 0x86, 0x43, 0x11, 0xc2, 0x2e, 0x4b, 0x9d, 0x3c, 0x4b, 0x38, - 0x8f, 0xc2, 0x78, 0xcf, 0x9e, 0x51, 0xce, 0xc5, 0x37, 0x71, 0x60, 0x8e, 0xe1, 0x18, 0xd3, 0x90, - 0x4f, 0xec, 0x8a, 0xe4, 0x67, 0xb4, 0x90, 0x0d, 0x90, 0xd3, 0x80, 0x72, 0x6a, 0xcf, 0x2a, 0x99, - 0xa1, 0xc9, 0x2a, 0xcc, 0xf5, 0xc3, 0x03, 0x0c, 0xba, 0xbd, 0x89, 0x5d, 0x95, 0xb2, 0xaa, 0xa4, - 0x1f, 0x4c, 0xc8, 0x03, 0xb8, 0x40, 0xfb, 0x7d, 0xf4, 0x39, 0x06, 0xdd, 0x31, 0xa6, 0x4c, 0x14, - 0x6c, 0xcf, 0xad, 0x97, 0x6f, 0xd6, 0x3b, 0x17, 0x5b, 0xf9, 0xf6, 0xb5, 0xb6, 0x90, 0xf2, 0x51, - 0x8a, 0xde, 0xa2, 0xd1, 0x7f, 0xa9, 0xd5, 0xdd, 0x5f, 0x2c, 0xa8, 0x6a, 0xe9, 0x7f, 0xe9, 0x89, - 0x0d, 0x55, 0x9d, 0x81, 0xee, 0x87, 0x21, 0x85, 0x03, 0xfd, 0xd9, 0xed, 0x27, 0xe9, 0x80, 0x72, - 0xdd, 0x95, 0xa6, 0xe6, 0x6e, 0x49, 0x26, 0x79, 0x0c, 0x0b, 0xe3, 0xdc, 0x80, 0x42, 0x64, 0x76, - 0x45, 0x56, 0xb2, 0x56, 0xac, 0xa4, 0x30, 0x45, 0xef, 0xa8, 0x8d, 0xbb, 0x06, 0x95, 0xa7, 0x74, - 0x82, 0xa9, 0xa8, 0x65, 0x97, 0xb2, 0x5d, 0x53, 0x8b, 0xf8, 0x76, 0xbf, 0xb1, 0xa0, 0xfe, 0x50, - 0x78, 0x79, 0xc1, 0x29, 0x1f, 0x31, 0x91, 0x74, 0x14, 0x32, 0x8e, 0x29, 0xb3, 0xad, 0xf5, 0xb2, - 0x48, 0x5a, 0x93, 0xe4, 0x32, 0xd4, 0x02, 0xe4, 0xe8, 0xf3, 0x24, 0x65, 0x76, 0x49, 0xca, 0x0e, - 0x19, 0xe4, 0x11, 0x2c, 0x46, 0x94, 0xf1, 0xee, 0x68, 0x18, 0x50, 0x8e, 0x5d, 0x01, 0x45, 0x59, - 0x75, 0xbd, 0xe3, 0xb4, 0x14, 0x0c, 0x5b, 0x06, 0xa7, 0xad, 0x4f, 0x0c, 0x4e, 0xbd, 0x79, 0x61, - 0xf3, 0xa9, 0x34, 0x11, 0x4c, 0xf7, 0x1e, 0x90, 0x6d, 0xe4, 0x9b, 0xb1, 0x8f, 0x8c, 0xa7, 0x13, - 0x0f, 0xbf, 0x1c, 0x21, 0xe3, 0xe4, 0x7f, 0xd0, 0xa4, 0x9a, 0xd5, 0xcd, 0x0d, 0xa3, 0x61, 0x98, - 0xa2, 0xdb, 0xee, 0xaf, 0x65, 0x58, 0x2a, 0xd8, 0xb2, 0x61, 0x12, 0x33, 0x24, 0x5b, 0x30, 0x67, - 0xf4, 0xa4, 0x5d, 0xbd, 0x73, 0xab, 0xd8, 0xbd, 0x29, 0x46, 0xad, 0x8c, 0x91, 0xd9, 0x92, 0xf7, - 0x60, 0x96, 0xc9, 0x16, 0xc9, 0x61, 0xd7, 0x3b, 0xab, 0x45, 0x2f, 0xb9, 0x1e, 0x7a, 0x5a, 0xd1, - 0xf9, 0x0a, 0x9a, 0xc6, 0x91, 0x1a, 0xc0, 0x3b, 0x50, 0x89, 0xc4, 0x87, 0x4e, 0x64, 0xa9, 0xe8, - 0x42, 0xea, 0x78, 0x4a, 0x43, 0xe0, 0x58, 0x35, 0x17, 0x83, 0x6e, 0x5f, 0x61, 0x51, 0x75, 0xfd, - 0x64, 0x1c, 0x1b, 0x7d, 0xcd, 0x60, 0xce, 0x4f, 0x16, 0xcc, 0x99, 0x04, 0xa6, 0x02, 0xf9, 0x06, - 0x2c, 0x30, 0x9f, 0xc6, 0x31, 0x06, 0x5d, 0x33, 0xf4, 0x19, 0x39, 0xd8, 0x79, 0xcd, 0x7e, 0xaa, - 0x67, 0x7f, 0x1b, 0x2e, 0x18, 0xc5, 0x43, 0x0c, 0x54, 0xa4, 0xea, 0xa2, 0x16, 0x3c, 0xca, 0xa0, - 0xb0, 0x0d, 0xb3, 0xb2, 0x06, 0x66, 0xcf, 0xca, 0x7c, 0xdb, 0xe7, 0xef, 0xb7, 0x6a, 0x81, 0x36, - 0x77, 0xff, 0x2c, 0xc1, 0xd2, 0xf3, 0x84, 0xbd, 0x15, 0x1e, 0xc8, 0x0a, 0xcc, 0xea, 0xb7, 0xa5, - 0x1e, 0xa7, 0xa6, 0xc8, 0xc3, 0x2c, 0xbb, 0xb2, 0xcc, 0xee, 0x76, 0x31, 0xbb, 0x29, 0xf1, 0x24, - 0xaf, 0x90, 0x99, 0xf3, 0xb3, 0x05, 0xb5, 0x8c, 0x3b, 0xed, 0x5d, 0x09, 0xde, 0x90, 0xf2, 0x5d, - 0x1d, 0x5c, 0x7e, 0x13, 0x0f, 0xaa, 0xbb, 0x48, 0x83, 0xc3, 0xd8, 0x77, 0xdf, 0x20, 0x76, 0xeb, - 0x23, 0x65, 0xfa, 0x38, 0x16, 0x52, 0xe3, 0xc8, 0xb9, 0x0f, 0x8d, 0xbc, 0x80, 0x2c, 0x42, 0x79, - 0x0f, 0x27, 0x3a, 0x15, 0xf1, 0x49, 0x96, 0xa1, 0x32, 0xa6, 0xd1, 0xc8, 0x2c, 0x29, 0x45, 0xdc, - 0x2f, 0xdd, 0xb5, 0xdc, 0x27, 0xb0, 0x5c, 0x0c, 0xa9, 0x9f, 0xcc, 0x21, 0xd4, 0xad, 0x73, 0x42, - 0xdd, 0xfd, 0xd1, 0x82, 0x95, 0x6d, 0xe4, 0xcf, 0x12, 0x1e, 0xf6, 0x43, 0x5f, 0xde, 0x19, 0x33, - 0xad, 0xf7, 0x61, 0x25, 0x89, 0x82, 0x6e, 0x7e, 0x2b, 0x4d, 0xba, 0x43, 0xba, 0x63, 0xc6, 0xb6, - 0x9c, 0x44, 0x41, 0x61, 0x83, 0x3d, 0xa7, 0x3b, 0x28, 0xac, 0x62, 0xdc, 0x9f, 0x66, 0xa5, 0xca, - 0x58, 0x8e, 0x71, 0xff, 0xb8, 0xd5, 0x32, 0x54, 0xa2, 0x70, 0x10, 0x72, 0xb9, 0x7a, 0x2a, 0x9e, - 0x22, 0x32, 0xe8, 0xcf, 0x1c, 0x42, 0xdf, 0xfd, 0xa3, 0x04, 0x97, 0x8e, 0x25, 0xac, 0xeb, 0x7f, - 0x09, 0x8d, 0x38, 0xc7, 0xd7, 0x5d, 0xe8, 0x1c, 0x83, 0xf1, 0x34, 0xe3, 0x56, 0x81, 0x59, 0xf0, - 0xe3, 0xfc, 0x6d, 0x41, 0x23, 0x2f, 0x9e, 0xfa, 0x26, 0x6d, 0xa8, 0xfa, 0x29, 0x52, 0x8e, 0x81, - 0xae, 0xd4, 0x90, 0xe2, 0x22, 0x2a, 0x77, 0x18, 0xe8, 0x83, 0x92, 0xd1, 0xc2, 0x2a, 0xc0, 0x08, - 0x85, 0x95, 0xaa, 0xd2, 0x90, 0xe4, 0x1e, 0x94, 0x93, 0x28, 0x90, 0xe7, 0xb5, 0xde, 0xb9, 0x71, - 0x04, 0x70, 0x74, 0x07, 0xb3, 0xde, 0x47, 0xa8, 0x81, 0x10, 0x22, 0xf3, 0x84, 0x8d, 0x30, 0x8d, - 0x71, 0x5f, 0x5e, 0xdf, 0x37, 0x31, 0x8d, 0x71, 0xdf, 0xfd, 0xad, 0x04, 0xab, 0x27, 0xaa, 0x90, - 0x0d, 0x68, 0xf8, 0xa3, 0x34, 0xc5, 0x98, 0xe7, 0x81, 0x50, 0xd7, 0x3c, 0x39, 0xc9, 0x35, 0xa8, - 0xc5, 0x78, 0xc0, 0xf3, 0x23, 0x9f, 0x13, 0x8c, 0x53, 0xc6, 0xbc, 0x09, 0xcd, 0x02, 0x5c, 0x64, - 0x27, 0xce, 0x38, 0x96, 0x45, 0x0b, 0xf2, 0x39, 0x00, 0xcd, 0xd2, 0xd4, 0xc7, 0xf6, 0x83, 0x73, - 0x16, 0xde, 0x7a, 0x12, 0x07, 0x78, 0x80, 0xc1, 0x66, 0x6e, 0x0b, 0x79, 0x39, 0x77, 0xce, 0x87, - 0xb0, 0x34, 0x45, 0x45, 0x14, 0x13, 0x0a, 0xb6, 0xec, 0x42, 0xc5, 0x53, 0x44, 0x06, 0x8d, 0x52, - 0x0e, 0xb3, 0x77, 0xe0, 0xca, 0xc7, 0x34, 0xdd, 0xcb, 0x43, 0x68, 0x93, 0x79, 0x48, 0x03, 0xf3, - 0xd4, 0xa6, 0xe0, 0xc9, 0x5d, 0x87, 0xab, 0x27, 0x19, 0x29, 0xc4, 0xba, 0x04, 0x16, 0xb7, 0x91, - 0xeb, 0x07, 0xad, 0x3c, 0xb9, 0x5b, 0x70, 0x21, 0xc7, 0x7b, 0xeb, 0xbd, 0xd0, 0xf9, 0xc7, 0x82, - 0x05, 0x53, 0xed, 0x0b, 0x4c, 0xc7, 0xa1, 0x8f, 0x64, 0x04, 0xf5, 0xdc, 0x0d, 0x20, 0xeb, 0xa7, - 0x9c, 0x07, 0x99, 0x8c, 0xb3, 0x71, 0xe6, 0x01, 0x71, 0x37, 0xbe, 0xfe, 0xfd, 0xaf, 0xef, 0x4a, - 0x6b, 0x64, 0xb5, 0x6d, 0x8e, 0x40, 0xfb, 0x55, 0xe1, 0x46, 0xbc, 0x26, 0x7b, 0xd0, 0xc8, 0x6f, - 0x3b, 0xb2, 0x71, 0xe6, 0xf2, 0x75, 0xdc, 0xd3, 0x54, 0x74, 0xe4, 0x65, 0x19, 0x79, 0xde, 0xad, - 0x65, 0x91, 0xef, 0x5b, 0xb7, 0x3a, 0x3f, 0x94, 0x60, 0x29, 0xdf, 0x72, 0x53, 0xfb, 0x6b, 0x58, - 0x38, 0xb2, 0x38, 0xc8, 0xff, 0xcf, 0xd8, 0x2b, 0x2a, 0x95, 0xeb, 0xe7, 0xda, 0x3e, 0xee, 0x15, - 0x99, 0xcd, 0x25, 0x72, 0xb1, 0x9d, 0xdf, 0x3c, 0xac, 0xfd, 0x4a, 0xf5, 0xe0, 0x5b, 0x0b, 0x56, - 0xa6, 0xa3, 0x81, 0x1c, 0xb9, 0x83, 0xa7, 0x02, 0xcd, 0x79, 0xf7, 0x7c, 0xca, 0xc5, 0xa4, 0x6e, - 0x4d, 0x4f, 0xaa, 0x13, 0x43, 0x53, 0xa1, 0xc6, 0x34, 0xe9, 0x0b, 0xa8, 0x65, 0xe0, 0x23, 0x57, - 0x8f, 0x15, 0x5e, 0x40, 0xaa, 0x73, 0xed, 0x44, 0xb9, 0x8e, 0xbe, 0x20, 0xa3, 0xd7, 0x48, 0xb5, - 0xad, 0x30, 0xf9, 0xe0, 0x2a, 0x2c, 0xf9, 0xc9, 0xa0, 0x68, 0x36, 0xec, 0x7d, 0x56, 0xd5, 0xff, - 0x71, 0xf5, 0x66, 0xe5, 0x1f, 0xaa, 0x77, 0xfe, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x7b, 0xd2, 0x74, - 0xfa, 0x8a, 0x0d, 0x00, 0x00, + // 1334 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4d, 0x6f, 0x1b, 0xc5, + 0x1b, 0xff, 0xaf, 0x13, 0xc7, 0xf6, 0x63, 0x3b, 0x71, 0x27, 0x69, 0xe2, 0x6c, 0xfe, 0x6d, 0x93, + 0x85, 0x8a, 0x52, 0x90, 0x2d, 0xdc, 0x22, 0xb5, 0xe5, 0x80, 0xdc, 0xc4, 0x09, 0x91, 0xda, 0x10, + 0x6d, 0xd2, 0x48, 0x05, 0x21, 0x6b, 0xe2, 0x7d, 0x9c, 0xac, 0xb2, 0xde, 0x5d, 0x76, 0xc7, 0x49, + 0xad, 0xaa, 0x1c, 0xb8, 0x71, 0x43, 0x70, 0xe0, 0xc4, 0x07, 0xe0, 0x82, 0xf8, 0x0e, 0xf0, 0x09, + 0xe0, 0x0a, 0x37, 0x0e, 0x7c, 0x01, 0xee, 0x68, 0x66, 0x67, 0x36, 0xbb, 0xc9, 0x26, 0x71, 0x7b, + 0xf2, 0xcc, 0xf3, 0xfe, 0xf2, 0x9b, 0xe7, 0x59, 0x83, 0x4e, 0x7d, 0xbb, 0x79, 0x7c, 0xaf, 0xd9, + 0x73, 0xa8, 0x1d, 0xf8, 0xfb, 0xd1, 0x6f, 0xc3, 0x0f, 0x3c, 0xe6, 0x91, 0x4a, 0xcf, 0x0b, 0xd0, + 0x0b, 0x1b, 0x82, 0xa6, 0xdf, 0x3a, 0xf0, 0xbc, 0x03, 0x07, 0x9b, 0x82, 0xb7, 0x3f, 0xec, 0x37, + 0x99, 0x3d, 0xc0, 0x90, 0xd1, 0x81, 0x1f, 0x89, 0xeb, 0xff, 0x97, 0x02, 0xdc, 0x22, 0x75, 0x5d, + 0x8f, 0x51, 0x66, 0x7b, 0x6e, 0x18, 0x71, 0x8d, 0x1f, 0x72, 0x50, 0xdd, 0x1b, 0x3a, 0x2e, 0x06, + 0x74, 0xdf, 0x76, 0x6c, 0x36, 0x22, 0x04, 0x26, 0x5d, 0x3a, 0xc0, 0xba, 0xb6, 0xac, 0xdd, 0x29, + 0x99, 0xe2, 0x4c, 0x6e, 0xc3, 0x34, 0xff, 0x0d, 0x7d, 0xda, 0xc3, 0xae, 0xe0, 0xe6, 0x04, 0xb7, + 0x1a, 0x53, 0xb7, 0xb8, 0xd8, 0x32, 0x94, 0x2d, 0x0c, 0x7b, 0x81, 0xed, 0x73, 0x17, 0xf5, 0x09, + 0x21, 0x93, 0x24, 0x71, 0xe3, 0x8e, 0xed, 0x1e, 0xd5, 0x27, 0x23, 0xe3, 0xfc, 0x4c, 0x74, 0x28, + 0x86, 0x78, 0x8c, 0x81, 0xcd, 0x46, 0xf5, 0xbc, 0xa0, 0xc7, 0x77, 0xce, 0x1b, 0x20, 0xa3, 0x16, + 0x65, 0xb4, 0x3e, 0x15, 0xf1, 0xd4, 0x9d, 0x2c, 0x42, 0xb1, 0x6f, 0xbf, 0x40, 0xab, 0xbb, 0x3f, + 0xaa, 0x17, 0x04, 0xaf, 0x20, 0xee, 0x8f, 0x47, 0xe4, 0x31, 0x5c, 0xa3, 0xfd, 0x3e, 0xf6, 0x18, + 0x5a, 0xdd, 0x63, 0x0c, 0x42, 0x9e, 0x70, 0xbd, 0xb8, 0x3c, 0x71, 0xa7, 0xdc, 0xba, 0xde, 0x48, + 0x96, 0xaf, 0xb1, 0x8e, 0x94, 0x0d, 0x03, 0x34, 0x6b, 0x4a, 0x7e, 0x4f, 0x8a, 0x1b, 0xbf, 0x69, + 0x50, 0x5c, 0x43, 0x86, 0x3d, 0xe6, 0x05, 0x99, 0x45, 0xa9, 0x43, 0x41, 0xda, 0x96, 0xd5, 0x50, + 0x57, 0xd2, 0x84, 0x49, 0x36, 0xf2, 0x51, 0x14, 0x60, 0xba, 0xb5, 0x94, 0xf6, 0xa8, 0x6c, 0x36, + 0x76, 0x47, 0x3e, 0x9a, 0x42, 0xd0, 0x78, 0x0e, 0x93, 0xfc, 0x46, 0x16, 0xe1, 0xfa, 0x5a, 0x67, + 0xb7, 0xb3, 0xba, 0xfb, 0xa9, 0xd9, 0xdd, 0x7d, 0xbe, 0xdd, 0xe9, 0x6e, 0x6e, 0xed, 0xb5, 0x9f, + 0x6c, 0xae, 0xd5, 0xfe, 0x47, 0x96, 0x60, 0x21, 0xcd, 0xda, 0x6a, 0x3f, 0xed, 0xec, 0x6c, 0xb7, + 0x57, 0x3b, 0x35, 0xed, 0xbc, 0xde, 0x7a, 0xa7, 0xbd, 0xfb, 0xcc, 0xec, 0xd4, 0x72, 0xc6, 0x0e, + 0x94, 0xb6, 0x54, 0x93, 0x32, 0xd3, 0x68, 0x41, 0xd1, 0x92, 0x21, 0x89, 0x3c, 0xca, 0xad, 0xf9, + 0xec, 0x80, 0xcd, 0x58, 0xce, 0xf8, 0x36, 0x07, 0x05, 0x59, 0xb9, 0x4c, 0x9b, 0x1f, 0x42, 0x29, + 0x46, 0x86, 0x34, 0xba, 0x90, 0x36, 0x1a, 0xc7, 0x64, 0x9e, 0x4a, 0x26, 0x2b, 0x3a, 0x91, 0xae, + 0xe8, 0x6d, 0x98, 0x96, 0xc7, 0x6e, 0xdf, 0x0b, 0x06, 0x94, 0x49, 0x04, 0x55, 0x25, 0x75, 0x5d, + 0x10, 0x53, 0xb9, 0xe4, 0xc7, 0xcb, 0x85, 0x74, 0x60, 0xe6, 0x38, 0xf1, 0x00, 0x6c, 0x0c, 0xeb, + 0x53, 0x02, 0x29, 0x67, 0xfa, 0x96, 0x7a, 0x25, 0xe6, 0x59, 0x1d, 0x63, 0x09, 0xf2, 0x4f, 0xe8, + 0x08, 0x05, 0x54, 0x0e, 0x69, 0x78, 0xa8, 0xea, 0xc1, 0xcf, 0xc6, 0x37, 0x1a, 0x94, 0x57, 0xb9, + 0x95, 0x1d, 0x46, 0xd9, 0x30, 0x24, 0xf7, 0xa1, 0xa4, 0xfc, 0x87, 0x75, 0x4d, 0x78, 0xbb, 0x28, + 0xd0, 0x53, 0x41, 0xb2, 0x06, 0x35, 0x87, 0x86, 0xac, 0x3b, 0xf4, 0x2d, 0xca, 0xb0, 0xcb, 0x1f, + 0xba, 0x2c, 0xae, 0xde, 0x88, 0x1e, 0x79, 0x43, 0x4d, 0x81, 0xc6, 0xae, 0x9a, 0x02, 0xe6, 0x34, + 0xd7, 0x79, 0x26, 0x54, 0x38, 0xd1, 0x78, 0x08, 0x64, 0x03, 0x59, 0xdb, 0xed, 0x61, 0xc8, 0x82, + 0x91, 0x89, 0x5f, 0x0e, 0x31, 0x64, 0xe4, 0x2d, 0xa8, 0x52, 0x49, 0xea, 0x26, 0xda, 0x59, 0x51, + 0x44, 0xde, 0x2f, 0xe3, 0x97, 0x09, 0x98, 0x4d, 0xe9, 0x86, 0xbe, 0xe7, 0x86, 0x48, 0xd6, 0xa1, + 0xa8, 0xe4, 0x84, 0x5e, 0xb9, 0x75, 0x37, 0x9d, 0x4d, 0x86, 0x52, 0x23, 0x26, 0xc4, 0xba, 0xe4, + 0x03, 0x98, 0x0a, 0x45, 0x81, 0x64, 0x5a, 0x8b, 0x69, 0x2b, 0x89, 0x0a, 0x9a, 0x52, 0x50, 0xff, + 0x0a, 0xaa, 0xca, 0x50, 0x54, 0xfe, 0x77, 0x21, 0xef, 0xf0, 0x83, 0x0c, 0x64, 0x36, 0x6d, 0x42, + 0xc8, 0x98, 0x91, 0x04, 0x9f, 0x12, 0x51, 0x71, 0xd1, 0xea, 0xf6, 0x23, 0x34, 0x73, 0xcf, 0x97, + 0x4d, 0x09, 0x25, 0x2f, 0x09, 0xa1, 0xfe, 0xa3, 0x06, 0x45, 0x15, 0x40, 0xe6, 0x53, 0x48, 0xb5, + 0x3a, 0x37, 0x6e, 0xab, 0x37, 0x60, 0x4a, 0xc4, 0x18, 0xd6, 0x27, 0x84, 0x4a, 0x73, 0xfc, 0x7a, + 0x46, 0x29, 0x4a, 0x75, 0xe3, 0xaf, 0x1c, 0xcc, 0x6e, 0x7b, 0xe1, 0x1b, 0xf5, 0x9b, 0xcc, 0xc3, + 0x94, 0x7c, 0x6d, 0xd1, 0x80, 0x93, 0x37, 0xb2, 0x7a, 0x26, 0xba, 0xf7, 0xd2, 0xd1, 0x65, 0xf8, + 0x13, 0xb4, 0x54, 0x64, 0xfa, 0xaf, 0x1a, 0x94, 0x62, 0x6a, 0xd6, 0xab, 0xe1, 0x34, 0x9f, 0xb2, + 0x43, 0xe9, 0x5c, 0x9c, 0x89, 0x09, 0x85, 0x43, 0xa4, 0xd6, 0xa9, 0xef, 0x07, 0xaf, 0xe1, 0xbb, + 0xf1, 0x49, 0xa4, 0xda, 0x71, 0x39, 0x57, 0x19, 0xd2, 0x1f, 0x41, 0x25, 0xc9, 0x20, 0x35, 0x98, + 0x38, 0xc2, 0x91, 0x0c, 0x85, 0x1f, 0xc9, 0x1c, 0xe4, 0x8f, 0xa9, 0x33, 0x54, 0x6b, 0x2f, 0xba, + 0x3c, 0xca, 0x3d, 0xd0, 0x8c, 0x4d, 0x98, 0x4b, 0xbb, 0x94, 0x4f, 0xe2, 0x14, 0xca, 0xda, 0x98, + 0x50, 0x36, 0x7e, 0xd6, 0x60, 0x7e, 0x03, 0xd9, 0x96, 0xc7, 0xec, 0xbe, 0xdd, 0x13, 0x5b, 0x5a, + 0x75, 0xeb, 0x3e, 0xcc, 0x7b, 0x8e, 0xd5, 0x4d, 0xce, 0x9c, 0x51, 0xd7, 0xa7, 0x07, 0xaa, 0x6d, + 0x73, 0x9e, 0x63, 0xa5, 0xe6, 0xd3, 0x36, 0x3d, 0xe0, 0xd0, 0x9b, 0x77, 0xf1, 0x24, 0x4b, 0x2b, + 0x4a, 0x63, 0xce, 0xc5, 0x93, 0xf3, 0x5a, 0x73, 0x90, 0x77, 0xec, 0x81, 0xcd, 0xc4, 0x08, 0xce, + 0x9b, 0xd1, 0x25, 0x86, 0xf6, 0xe4, 0x29, 0xb4, 0x8d, 0x3f, 0x73, 0xb0, 0x70, 0x2e, 0x60, 0x99, + 0xff, 0x1e, 0x54, 0xdc, 0x04, 0x5d, 0x56, 0xa1, 0x75, 0x0e, 0xc6, 0x59, 0xca, 0x8d, 0x14, 0x31, + 0x65, 0x47, 0xff, 0x47, 0x83, 0x4a, 0x92, 0x7d, 0xd1, 0x66, 0xee, 0x05, 0x48, 0x19, 0x5a, 0x6a, + 0x33, 0xcb, 0x2b, 0xff, 0x9e, 0x88, 0xcc, 0xa1, 0x25, 0x57, 0x4c, 0x7c, 0xe7, 0x5a, 0x16, 0x3a, + 0xc8, 0xb5, 0xa2, 0x2c, 0xd5, 0x95, 0x3c, 0x84, 0x09, 0xcf, 0xb1, 0xe4, 0x46, 0x79, 0xe7, 0x0c, + 0xe0, 0xe8, 0x01, 0xc6, 0xb5, 0x77, 0x50, 0x02, 0xc1, 0xc6, 0xd0, 0xe4, 0x3a, 0x5c, 0xd5, 0xc5, + 0x13, 0xf1, 0xed, 0xf2, 0x3a, 0xaa, 0x2e, 0x9e, 0x18, 0xbf, 0xe7, 0x60, 0xf1, 0x42, 0x11, 0xb2, + 0x02, 0x95, 0xde, 0x30, 0x08, 0xd0, 0x65, 0x49, 0x20, 0x94, 0x25, 0x4d, 0x74, 0x72, 0x09, 0x4a, + 0x2e, 0xbe, 0x60, 0xc9, 0x96, 0x17, 0x39, 0xe1, 0x92, 0x36, 0xb7, 0xa1, 0x9a, 0x82, 0x8b, 0xa8, + 0xc4, 0x15, 0xab, 0x30, 0xad, 0x41, 0x3e, 0x07, 0xa0, 0x71, 0x98, 0xf5, 0xbc, 0x78, 0xa4, 0x1f, + 0x8d, 0x99, 0x78, 0x63, 0xd3, 0xb5, 0xf0, 0x05, 0x5a, 0xed, 0xc4, 0x14, 0x32, 0x13, 0xe6, 0xf4, + 0x8f, 0x61, 0x36, 0x43, 0x84, 0x27, 0x63, 0x73, 0xb2, 0xa8, 0x42, 0xde, 0x8c, 0x2e, 0x31, 0x34, + 0x72, 0x09, 0xcc, 0xde, 0x83, 0x1b, 0x4f, 0x69, 0x70, 0x94, 0x84, 0x50, 0x3b, 0x34, 0x91, 0x5a, + 0xea, 0xa9, 0x65, 0xe0, 0xc9, 0x58, 0x86, 0x9b, 0x17, 0x29, 0x45, 0x88, 0x35, 0x08, 0xd4, 0x36, + 0x90, 0xc9, 0x07, 0x1d, 0x59, 0x32, 0xd6, 0xe1, 0x5a, 0x82, 0xf6, 0xc6, 0x73, 0xa1, 0xf5, 0xaf, + 0x06, 0x33, 0x2a, 0xdb, 0x1d, 0x0c, 0x8e, 0xed, 0x1e, 0x92, 0x21, 0x94, 0x13, 0x3b, 0x80, 0x2c, + 0x5f, 0xb2, 0x1e, 0x44, 0x30, 0xfa, 0xca, 0x95, 0x0b, 0xc4, 0x58, 0xf9, 0xfa, 0x8f, 0xbf, 0xbf, + 0xcf, 0x2d, 0x91, 0xc5, 0xa6, 0x5a, 0x02, 0xcd, 0x97, 0xa9, 0x1d, 0xf1, 0x8a, 0x1c, 0x41, 0x25, + 0x39, 0xed, 0xc8, 0xca, 0x95, 0xc3, 0x57, 0x37, 0x2e, 0x13, 0x91, 0x9e, 0xe7, 0x84, 0xe7, 0x69, + 0xa3, 0x14, 0x7b, 0x7e, 0xa4, 0xdd, 0x6d, 0xfd, 0x94, 0x83, 0xd9, 0x64, 0xc9, 0x55, 0xee, 0xaf, + 0x60, 0xe6, 0xcc, 0xe0, 0x20, 0x6f, 0x5f, 0x31, 0x57, 0xa2, 0x50, 0x6e, 0x8f, 0x35, 0x7d, 0x8c, + 0x1b, 0x22, 0x9a, 0x05, 0x72, 0xbd, 0x99, 0x9c, 0x3c, 0x61, 0xf3, 0x65, 0x54, 0x83, 0xef, 0x34, + 0x98, 0xcf, 0x46, 0x03, 0x39, 0xb3, 0x07, 0x2f, 0x05, 0x9a, 0xfe, 0xfe, 0x78, 0xc2, 0xe9, 0xa0, + 0xee, 0x66, 0x07, 0xd5, 0x72, 0xa1, 0x1a, 0xa1, 0x46, 0x15, 0xe9, 0x0b, 0x28, 0xc5, 0xe0, 0x23, + 0x37, 0xcf, 0x25, 0x9e, 0x42, 0xaa, 0x7e, 0xeb, 0x42, 0xbe, 0xf4, 0x3e, 0x23, 0xbc, 0x97, 0x48, + 0xa1, 0x19, 0x61, 0xf2, 0xf1, 0x4d, 0x98, 0xed, 0x79, 0x83, 0xb4, 0x9a, 0xbf, 0xff, 0x59, 0x41, + 0xfe, 0x5f, 0xdd, 0x9f, 0x12, 0x1f, 0xa2, 0xf7, 0xfe, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x9f, 0xd1, + 0xa1, 0x1f, 0xc8, 0x0e, 0x00, 0x00, } diff --git a/api/v3/clairpb/clair.proto b/api/v3/clairpb/clair.proto index c0e3ff8a..f8545cff 100644 --- a/api/v3/clairpb/clair.proto +++ b/api/v3/clairpb/clair.proto @@ -44,17 +44,42 @@ message Vulnerability { repeated Feature affected_versions = 8; } +message Detector { + enum Type { + DETECTOR_TYPE_INVALID = 0; + DETECTOR_TYPE_NAMESPACE = 1; + DETECTOR_TYPE_FEATURE = 2; + } + // The name of the detector. + string name = 1; + // The version of the detector. + string version = 2; + // The type of the detector. + Type type = 3; +} + +message Namespace { + // The name of the namespace. + string name = 1; + // The detector used to detect the namespace. This only exists when present in + // an Ancestry Feature. + Detector detector = 2; +} + message Feature { // The name of the feature. string name = 1; - // The name of the namespace in which the feature is detected. - string namespace_name = 2; + // The namespace in which the feature is detected. + Namespace namespace = 2; // The specific version of this feature. string version = 3; // The format used to parse version numbers for the feature. string version_format = 4; + // The detector used to detect this feature. This only exists when present in + // an Ancestry. + Detector detector = 5; // The list of vulnerabilities that affect the feature. - repeated Vulnerability vulnerabilities = 5; + repeated Vulnerability vulnerabilities = 6; } message Layer { @@ -77,12 +102,10 @@ service AncestryService { } message ClairStatus { - // The configured list of feature listers used to scan an ancestry. - repeated string listers = 1; - // The configured list of namespace detectors used to scan an ancestry. - repeated string detectors = 2; + // The implemented detectors in this Clair instance + repeated Detector detectors = 1; // The time at which the updater last ran. - google.protobuf.Timestamp last_update_time = 3; + google.protobuf.Timestamp last_update_time = 2; } message GetAncestryRequest { @@ -100,12 +123,11 @@ message GetAncestryResponse { message Ancestry { // The name of the desired ancestry. string name = 1; - // The configured list of feature listers used to scan this ancestry. - repeated string scanned_listers = 4; - // The configured list of namespace detectors used to scan an ancestry. - repeated string scanned_detectors = 5; + // The detectors used to scan this Ancestry. It may not be the current set + // of detectors in clair status. + repeated Detector detectors = 2; // The list of layers along with detected features in each. - repeated AncestryLayer layers = 6; + repeated AncestryLayer layers = 3; } // The ancestry requested. Ancestry ancestry = 1; diff --git a/api/v3/clairpb/clair.swagger.json b/api/v3/clairpb/clair.swagger.json index 32e00e53..a967092f 100644 --- a/api/v3/clairpb/clair.swagger.json +++ b/api/v3/clairpb/clair.swagger.json @@ -163,19 +163,12 @@ "type": "string", "description": "The name of the desired ancestry." }, - "scanned_listers": { - "type": "array", - "items": { - "type": "string" - }, - "description": "The configured list of feature listers used to scan this ancestry." - }, - "scanned_detectors": { + "detectors": { "type": "array", "items": { - "type": "string" + "$ref": "#/definitions/clairDetector" }, - "description": "The configured list of namespace detectors used to scan an ancestry." + "description": "The detectors used to scan this Ancestry. It may not be the current set\nof detectors in clair status." }, "layers": { "type": "array", @@ -268,19 +261,12 @@ "clairClairStatus": { "type": "object", "properties": { - "listers": { - "type": "array", - "items": { - "type": "string" - }, - "description": "The configured list of feature listers used to scan an ancestry." - }, "detectors": { "type": "array", "items": { - "type": "string" + "$ref": "#/definitions/clairDetector" }, - "description": "The configured list of namespace detectors used to scan an ancestry." + "title": "The implemented detectors in this Clair instance" }, "last_update_time": { "type": "string", @@ -289,6 +275,32 @@ } } }, + "clairDetector": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the detector." + }, + "version": { + "type": "string", + "description": "The version of the detector." + }, + "type": { + "$ref": "#/definitions/clairDetectorType", + "description": "The type of the detector." + } + } + }, + "clairDetectorType": { + "type": "string", + "enum": [ + "DETECTOR_TYPE_INVALID", + "DETECTOR_TYPE_NAMESPACE", + "DETECTOR_TYPE_FEATURE" + ], + "default": "DETECTOR_TYPE_INVALID" + }, "clairFeature": { "type": "object", "properties": { @@ -296,9 +308,9 @@ "type": "string", "description": "The name of the feature." }, - "namespace_name": { - "type": "string", - "description": "The name of the namespace in which the feature is detected." + "namespace": { + "$ref": "#/definitions/clairNamespace", + "description": "The namespace in which the feature is detected." }, "version": { "type": "string", @@ -308,6 +320,10 @@ "type": "string", "description": "The format used to parse version numbers for the feature." }, + "detector": { + "$ref": "#/definitions/clairDetector", + "description": "The detector used to detect this feature. This only exists when present in\nan Ancestry." + }, "vulnerabilities": { "type": "array", "items": { @@ -360,6 +376,19 @@ "clairMarkNotificationAsReadResponse": { "type": "object" }, + "clairNamespace": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the namespace." + }, + "detector": { + "$ref": "#/definitions/clairDetector", + "description": "The detector used to detect the namespace. This only exists when present in\nan Ancestry Feature." + } + } + }, "clairPagedVulnerableAncestries": { "type": "object", "properties": { diff --git a/api/v3/util.go b/api/v3/util.go index 6b6baa0a..d70390d4 100644 --- a/api/v3/util.go +++ b/api/v3/util.go @@ -50,7 +50,11 @@ func GetPbAncestryLayer(session database.Session, layer database.AncestryLayer) return nil, status.Error(codes.Internal, err.Error()) } - for _, feature := range features { + // NOTE(sidac): It's quite inefficient, but the easiest way to implement + // this feature for now, we should refactor the implementation if there's + // any performance issue. It's expected that the number of features is less + // than 1000. + for _, feature := range affectedFeatures { if !feature.Valid { return nil, status.Error(codes.Internal, "ancestry feature is not found") } From db2db8bbe8a17e10c9fb365196f88d552e70e91d Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Mon, 8 Oct 2018 10:42:17 -0400 Subject: [PATCH 04/14] database: Update database model and interface for detectors All detected features and namespaces under the context of Layer and Ancestry will now have the detectors associated, so that the API can provide the detection information to the Client. --- database/database.go | 23 ++++--- database/detector.go | 144 +++++++++++++++++++++++++++++++++++++++++++ database/mock.go | 14 ++++- database/models.go | 128 +++++++++++++++++++++++++++++++------- 4 files changed, 276 insertions(+), 33 deletions(-) create mode 100644 database/detector.go diff --git a/database/database.go b/database/database.go index c77effac..f760061f 100644 --- a/database/database.go +++ b/database/database.go @@ -33,6 +33,14 @@ var ( // fails (i.e. when an entity which is supposed to be unique is detected // twice) ErrInconsistent = errors.New("database: inconsistent database") + + // ErrInvalidParameters is an error that occurs when the parameters are not valid. + ErrInvalidParameters = errors.New("database: parameters are not valid") + + // ErrMissingEntities is an error that occurs when an associated immutable + // entity doesn't exist in the database. This error can indicate a wrong + // implementation or corrupted database. + ErrMissingEntities = errors.New("database: associated immutable entities are missing in the database") ) // RegistrableComponentConfig is a configuration block that can be used to @@ -99,6 +107,9 @@ type Session interface { // namespaced features. If the ancestry is not found, return false. FindAncestry(name string) (ancestry Ancestry, found bool, err error) + // PersistDetector inserts a slice of detectors if not in the database. + PersistDetectors(detectors []Detector) error + // PersistFeatures inserts a set of features if not in the database. PersistFeatures(features []Feature) error @@ -120,12 +131,10 @@ type Session interface { // PersistNamespaces inserts a set of namespaces if not in the database. PersistNamespaces([]Namespace) error - // PersistLayer persists a layer's content in the database. The given - // namespaces and features can be partial content of this layer. + // PersistLayer appends a layer's content in the database. // - // The layer, namespaces and features are expected to be already existing - // in the database. - PersistLayer(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error + // If any feature, namespace, or detector is not in the database, it returns not found error. + PersistLayer(hash string, features []LayerFeature, namespaces []LayerNamespace, detectedBy []Detector) error // FindLayer returns a layer with all detected features and // namespaces. @@ -157,8 +166,8 @@ type Session interface { // affected ancestries affected by old or new vulnerability. // // Because the number of affected ancestries maybe large, they are paginated - // and their pages are specified by the paination token, which, if empty, are - // always considered first page. + // and their pages are specified by the pagination token, which should be + // considered first page when it's empty. FindVulnerabilityNotification(name string, limit int, oldVulnerabilityPage pagination.Token, newVulnerabilityPage pagination.Token) (noti VulnerabilityNotificationWithVulnerable, found bool, err error) // MarkNotificationAsRead marks a Notification as notified now, assuming diff --git a/database/detector.go b/database/detector.go new file mode 100644 index 00000000..e1295340 --- /dev/null +++ b/database/detector.go @@ -0,0 +1,144 @@ +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "database/sql/driver" + "errors" + "fmt" + "strings" +) + +const ( + // NamespaceDetectorType is a type of detector that extracts the namespaces. + NamespaceDetectorType DetectorType = "namespace" + // FeatureDetectorType is a type of detector that extracts the features. + FeatureDetectorType DetectorType = "feature" +) + +// DetectorTypes contains all detector types. +var ( + DetectorTypes = []DetectorType{ + NamespaceDetectorType, + FeatureDetectorType, + } + // ErrFailedToParseDetectorType is the error returned when a detector type could + // not be parsed from a string. + ErrFailedToParseDetectorType = errors.New("failed to parse DetectorType from input") + // ErrInvalidDetector is the error returned when a detector from database has + // invalid name or version or type. + ErrInvalidDetector = errors.New("the detector has invalid metadata") +) + +// DetectorType is the type of a detector. +type DetectorType string + +// Value implements the database/sql/driver.Valuer interface. +func (s DetectorType) Value() (driver.Value, error) { + return string(s), nil +} + +// Scan implements the database/sql.Scanner interface. +func (s *DetectorType) Scan(value interface{}) error { + val, ok := value.([]byte) + if !ok { + return errors.New("could not scan a Severity from a non-string input") + } + + var err error + *s, err = NewDetectorType(string(val)) + if err != nil { + return err + } + + return nil +} + +// NewDetectorType attempts to parse a string into a standard DetectorType +// value. +func NewDetectorType(s string) (DetectorType, error) { + for _, ss := range DetectorTypes { + if strings.EqualFold(s, string(ss)) { + return ss, nil + } + } + + return "", ErrFailedToParseDetectorType +} + +// Valid checks if a detector type is defined. +func (s DetectorType) Valid() bool { + for _, t := range DetectorTypes { + if s == t { + return true + } + } + + return false +} + +// Detector is an versioned Clair extension. +type Detector struct { + // Name of an extension should be non-empty and uniquely identifies the + // extension. + Name string + // Version of an extension should be non-empty. + Version string + // DType is the type of the extension and should be one of the types in + // DetectorTypes. + DType DetectorType +} + +// Valid checks if all fields in the detector satisfies the spec. +func (d Detector) Valid() bool { + if d.Name == "" || d.Version == "" || !d.DType.Valid() { + return false + } + + return true +} + +// String returns a unique string representation of the detector. +func (d Detector) String() string { + return fmt.Sprintf("%s:%s", d.Name, d.Version) +} + +// NewNamespaceDetector returns a new namespace detector. +func NewNamespaceDetector(name, version string) Detector { + return Detector{ + Name: name, + Version: version, + DType: NamespaceDetectorType, + } +} + +// NewFeatureDetector returns a new feature detector. +func NewFeatureDetector(name, version string) Detector { + return Detector{ + Name: name, + Version: version, + DType: FeatureDetectorType, + } +} + +// SerializeDetectors returns the string representation of given detectors. +func SerializeDetectors(detectors []Detector) []string { + strDetectors := []string{} + for _, d := range detectors { + strDetectors = append(strDetectors, d.String()) + } + + return strDetectors +} diff --git a/database/mock.go b/database/mock.go index 9995bc49..b95d1b4d 100644 --- a/database/mock.go +++ b/database/mock.go @@ -30,9 +30,10 @@ type MockSession struct { FctFindAffectedNamespacedFeatures func(features []NamespacedFeature) ([]NullableAffectedNamespacedFeature, error) FctPersistNamespaces func([]Namespace) error FctPersistFeatures func([]Feature) error + FctPersistDetectors func(detectors []Detector) error FctPersistNamespacedFeatures func([]NamespacedFeature) error FctCacheAffectedNamespacedFeatures func([]NamespacedFeature) error - FctPersistLayer func(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error + FctPersistLayer func(hash string, features []LayerFeature, namespaces []LayerNamespace, by []Detector) error FctFindLayer func(name string) (Layer, bool, error) FctInsertVulnerabilities func([]VulnerabilityWithAffected) error FctFindVulnerabilities func([]VulnerabilityID) ([]NullableVulnerability, error) @@ -85,6 +86,13 @@ func (ms *MockSession) FindAffectedNamespacedFeatures(features []NamespacedFeatu panic("required mock function not implemented") } +func (ms *MockSession) PersistDetectors(detectors []Detector) error { + if ms.FctPersistDetectors != nil { + return ms.FctPersistDetectors(detectors) + } + panic("required mock function not implemented") +} + func (ms *MockSession) PersistNamespaces(namespaces []Namespace) error { if ms.FctPersistNamespaces != nil { return ms.FctPersistNamespaces(namespaces) @@ -113,9 +121,9 @@ func (ms *MockSession) CacheAffectedNamespacedFeatures(namespacedFeatures []Name panic("required mock function not implemented") } -func (ms *MockSession) PersistLayer(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error { +func (ms *MockSession) PersistLayer(hash string, features []LayerFeature, namespaces []LayerNamespace, detectors []Detector) error { if ms.FctPersistLayer != nil { - return ms.FctPersistLayer(hash, namespaces, features, processedBy) + return ms.FctPersistLayer(hash, features, namespaces, detectors) } panic("required mock function not implemented") } diff --git a/database/models.go b/database/models.go index b0157b90..4c1d0905 100644 --- a/database/models.go +++ b/database/models.go @@ -22,47 +22,129 @@ import ( "github.com/coreos/clair/pkg/pagination" ) -// Processors are extentions to scan a layer's content. -type Processors struct { - Listers []string - Detectors []string -} - // Ancestry is a manifest that keeps all layers in an image in order. type Ancestry struct { + // Name is a globally unique value for a set of layers. This is often the + // sha256 digest of an OCI/Docker manifest. Name string - // ProcessedBy contains the processors that are used when computing the + // By contains the processors that are used when computing the // content of this ancestry. - ProcessedBy Processors + By []Detector // Layers should be ordered and i_th layer is the parent of i+1_th layer in // the slice. Layers []AncestryLayer } +// Valid checks if the ancestry is compliant to spec. +func (a *Ancestry) Valid() bool { + if a == nil { + return false + } + + if a.Name == "" { + return false + } + + for _, d := range a.By { + if !d.Valid() { + return false + } + } + + for _, l := range a.Layers { + if !l.Valid() { + return false + } + } + + return true +} + // AncestryLayer is a layer with all detected namespaced features. type AncestryLayer struct { - LayerMetadata - - // DetectedFeatures are the features introduced by this layer when it was + // Hash is the sha-256 tarsum on the layer's blob content. + Hash string + // Features are the features introduced by this layer when it was // processed. - DetectedFeatures []NamespacedFeature + Features []AncestryFeature } -// LayerMetadata contains the metadata of a layer. -type LayerMetadata struct { - // Hash is content hash of the layer. - Hash string - // ProcessedBy contains the processors that processed this layer. - ProcessedBy Processors +// Valid checks if the Ancestry Layer is compliant to the spec. +func (l *AncestryLayer) Valid() bool { + if l == nil { + return false + } + + if l.Hash == "" { + return false + } + + return true +} + +// GetFeatures returns the Ancestry's features. +func (l *AncestryLayer) GetFeatures() []NamespacedFeature { + nsf := make([]NamespacedFeature, 0, len(l.Features)) + for _, f := range l.Features { + nsf = append(nsf, f.NamespacedFeature) + } + + return nsf +} + +// AncestryFeature is a namespaced feature with the detectors used to +// find this feature. +type AncestryFeature struct { + NamespacedFeature + + // FeatureBy is the detector that detected the feature. + FeatureBy Detector + // NamespaceBy is the detector that detected the namespace. + NamespaceBy Detector } -// Layer is a layer with its detected namespaces and features by -// ProcessedBy. +// Layer is a layer with all the detected features and namespaces. type Layer struct { - LayerMetadata + // Hash is the sha-256 tarsum on the layer's blob content. + Hash string + // By contains a list of detectors scanned this Layer. + By []Detector + Namespaces []LayerNamespace + Features []LayerFeature +} + +func (l *Layer) GetFeatures() []Feature { + features := make([]Feature, 0, len(l.Features)) + for _, f := range l.Features { + features = append(features, f.Feature) + } + + return features +} + +func (l *Layer) GetNamespaces() []Namespace { + namespaces := make([]Namespace, 0, len(l.Namespaces)) + for _, ns := range l.Namespaces { + namespaces = append(namespaces, ns.Namespace) + } + + return namespaces +} + +// LayerNamespace is a namespace with detection information. +type LayerNamespace struct { + Namespace + + // By is the detector found the namespace. + By Detector +} + +// LayerFeature is a feature with detection information. +type LayerFeature struct { + Feature - Namespaces []Namespace - Features []Feature + // By is the detector found the feature. + By Detector } // Namespace is the contextual information around features. From dca2d4e597ba837b6f96f3b3e32e23f6b843f9ab Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Thu, 13 Sep 2018 17:06:46 -0400 Subject: [PATCH 05/14] pgsql: Add detector to database schema 'detector' table is added to store the metadata of detectors. 'layer_feature', 'layer_namespace', and 'ancestry_feature' tables are modified to store the detection relationship between the feature/namespace with the detector. --- .../pgsql/migrations/00001_initial_schema.go | 178 +++++++++++------- database/pgsql/migrations/util.go | 30 +++ 2 files changed, 141 insertions(+), 67 deletions(-) create mode 100644 database/pgsql/migrations/util.go diff --git a/database/pgsql/migrations/00001_initial_schema.go b/database/pgsql/migrations/00001_initial_schema.go index 2f6638b8..69bd4081 100644 --- a/database/pgsql/migrations/00001_initial_schema.go +++ b/database/pgsql/migrations/00001_initial_schema.go @@ -14,12 +14,11 @@ package migrations -import "github.com/remind101/migrate" - -func init() { - RegisterMigration(migrate.Migration{ - ID: 1, - Up: migrate.Queries([]string{ +var ( + // entities are the basic building blocks to relate the vulnerabilities with + // the ancestry. + entities = MigrationQuery{ + Up: []string{ // namespaces `CREATE TABLE IF NOT EXISTS namespace ( id SERIAL PRIMARY KEY, @@ -42,40 +41,72 @@ func init() { namespace_id INT REFERENCES namespace, feature_id INT REFERENCES feature, UNIQUE (namespace_id, feature_id));`, + }, + Down: []string{ + `DROP TABLE IF EXISTS namespace, feature, namespaced_feature CASCADE;`, + }, + } + // detector is analysis extensions used by the worker. + detector = MigrationQuery{ + Up: []string{ + // Detector Type + `CREATE TYPE detector_type AS ENUM ('namespace', 'feature');`, + + // Detector + `CREATE TABLE IF NOT EXISTS detector ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + version TEXT NOT NULL, + dtype detector_type NOT NULL, + UNIQUE (name, version, dtype));`, + }, + Down: []string{ + `DROP TABLE IF EXISTS detector CASCADE;`, + `DROP TYPE IF EXISTS detector_type;`, + }, + } + + // layer contains all metadata and scanned features and namespaces. + layer = MigrationQuery{ + Up: []string{ // layers `CREATE TABLE IF NOT EXISTS layer( id SERIAL PRIMARY KEY, hash TEXT NOT NULL UNIQUE);`, + `CREATE TABLE IF NOT EXISTS layer_detector( + id SERIAL PRIMARY KEY, + layer_id INT REFERENCES layer ON DELETE CASCADE, + detector_id INT REFERENCES detector ON DELETE CASCADE, + UNIQUE(layer_id, detector_id));`, + `CREATE INDEX ON layer_detector(layer_id);`, + `CREATE TABLE IF NOT EXISTS layer_feature ( id SERIAL PRIMARY KEY, layer_id INT REFERENCES layer ON DELETE CASCADE, feature_id INT REFERENCES feature ON DELETE CASCADE, + detector_id INT REFERENCES detector ON DELETE CASCADE, UNIQUE (layer_id, feature_id));`, `CREATE INDEX ON layer_feature(layer_id);`, - `CREATE TABLE IF NOT EXISTS layer_lister ( - id SERIAL PRIMARY KEY, - layer_id INT REFERENCES layer ON DELETE CASCADE, - lister TEXT NOT NULL, - UNIQUE (layer_id, lister));`, - `CREATE INDEX ON layer_lister(layer_id);`, - - `CREATE TABLE IF NOT EXISTS layer_detector ( - id SERIAL PRIMARY KEY, - layer_id INT REFERENCES layer ON DELETE CASCADE, - detector TEXT, - UNIQUE (layer_id, detector));`, - `CREATE INDEX ON layer_detector(layer_id);`, - `CREATE TABLE IF NOT EXISTS layer_namespace ( id SERIAL PRIMARY KEY, layer_id INT REFERENCES layer ON DELETE CASCADE, namespace_id INT REFERENCES namespace ON DELETE CASCADE, + detector_id INT REFERENCES detector ON DELETE CASCADE, UNIQUE (layer_id, namespace_id));`, `CREATE INDEX ON layer_namespace(layer_id);`, - + }, + Down: []string{ + `DROP TABLE IF EXISTS layer, layer_detector, layer_feature, layer_namespace CASCADE;`, + }, + } + + // ancestry contains all meta information around scanned manifest and its + // layers. + ancestry = MigrationQuery{ + Up: []string{ // ancestry `CREATE TABLE IF NOT EXISTS ancestry ( id SERIAL PRIMARY KEY, @@ -93,28 +124,31 @@ func init() { id SERIAL PRIMARY KEY, ancestry_layer_id INT REFERENCES ancestry_layer ON DELETE CASCADE, namespaced_feature_id INT REFERENCES namespaced_feature ON DELETE CASCADE, + feature_detector_id INT REFERENCES detector ON DELETE CASCADE, + namespace_detector_id INT REFERENCES detector ON DELETE CASCADE, UNIQUE (ancestry_layer_id, namespaced_feature_id));`, - `CREATE TABLE IF NOT EXISTS ancestry_lister ( - id SERIAL PRIMARY KEY, - ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, - lister TEXT, - UNIQUE (ancestry_id, lister));`, - `CREATE INDEX ON ancestry_lister(ancestry_id);`, - - `CREATE TABLE IF NOT EXISTS ancestry_detector ( + `CREATE TABLE IF NOT EXISTS ancestry_detector( id SERIAL PRIMARY KEY, - ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, - detector TEXT, - UNIQUE (ancestry_id, detector));`, + ancestry_id INT REFERENCES layer ON DELETE CASCADE, + detector_id INT REFERENCES detector ON DELETE CASCADE, + UNIQUE(ancestry_id, detector_id));`, `CREATE INDEX ON ancestry_detector(ancestry_id);`, - + }, + Down: []string{ + `DROP TABLE IF EXISTS ancestry, ancestry_layer, ancestry_feature, ancestry_detector CASCADE;`, + }, + } + + // vulnerability contains the metadata and vulnerability affecting relation. + vulnerability = MigrationQuery{ + Up: []string{ `CREATE TYPE severity AS ENUM ('Unknown', 'Negligible', 'Low', 'Medium', 'High', 'Critical', 'Defcon1');`, // vulnerability `CREATE TABLE IF NOT EXISTS vulnerability ( id SERIAL PRIMARY KEY, - namespace_id INT NOT NULL REFERENCES Namespace, + namespace_id INT REFERENCES Namespace, name TEXT NOT NULL, description TEXT NULL, link TEXT NULL, @@ -127,7 +161,7 @@ func init() { `CREATE TABLE IF NOT EXISTS vulnerability_affected_feature ( id SERIAL PRIMARY KEY, - vulnerability_id INT NOT NULL REFERENCES vulnerability ON DELETE CASCADE, + vulnerability_id INT REFERENCES vulnerability ON DELETE CASCADE, feature_name TEXT NOT NULL, affected_version TEXT, fixedin TEXT);`, @@ -135,12 +169,22 @@ func init() { `CREATE TABLE IF NOT EXISTS vulnerability_affected_namespaced_feature( id SERIAL PRIMARY KEY, - vulnerability_id INT NOT NULL REFERENCES vulnerability ON DELETE CASCADE, - namespaced_feature_id INT NOT NULL REFERENCES namespaced_feature ON DELETE CASCADE, - added_by INT NOT NULL REFERENCES vulnerability_affected_feature ON DELETE CASCADE, + vulnerability_id INT REFERENCES vulnerability ON DELETE CASCADE, + namespaced_feature_id INT REFERENCES namespaced_feature ON DELETE CASCADE, + added_by INT REFERENCES vulnerability_affected_feature ON DELETE CASCADE, UNIQUE (vulnerability_id, namespaced_feature_id));`, `CREATE INDEX ON vulnerability_affected_namespaced_feature(namespaced_feature_id);`, - + }, + Down: []string{ + `DROP TYPE IF EXISTS severity;`, + `DROP TABLE IF EXISTS vulnerability, vulnerability_affected_feature, vulnerability_affected_namespaced_feature CASCADE;`, + }, + } + + // updaterLock is the lock to be used by updater to prevent multiple + // updaters running on the same vulnerability source. + updaterLock = MigrationQuery{ + Up: []string{ `CREATE TABLE IF NOT EXISTS KeyValue ( id SERIAL PRIMARY KEY, key TEXT NOT NULL UNIQUE, @@ -152,8 +196,16 @@ func init() { owner VARCHAR(64) NOT NULL, until TIMESTAMP WITH TIME ZONE);`, `CREATE INDEX ON Lock (owner);`, - - // Notification + }, + Down: []string{ + `DROP TABLE IF EXISTS KeyValue, Lock CASCADE;`, + }, + } + + // notification is the vulnerability notification spawned by the + // vulnerability changes. + notification = MigrationQuery{ + Up: []string{ `CREATE TABLE IF NOT EXISTS Vulnerability_Notification ( id SERIAL PRIMARY KEY, name VARCHAR(64) NOT NULL UNIQUE, @@ -163,30 +215,22 @@ func init() { old_vulnerability_id INT NULL REFERENCES Vulnerability ON DELETE CASCADE, new_vulnerability_id INT NULL REFERENCES Vulnerability ON DELETE CASCADE);`, `CREATE INDEX ON Vulnerability_Notification (notified_at);`, - }), - Down: migrate.Queries([]string{ - `DROP TABLE IF EXISTS - ancestry, - ancestry_layer, - ancestry_detector, - ancestry_lister, - ancestry_feature, - feature, - namespaced_feature, - keyvalue, - layer, - layer_detector, - layer_feature, - layer_lister, - layer_namespace, - lock, - namespace, - vulnerability, - vulnerability_affected_feature, - vulnerability_affected_namespaced_feature, - vulnerability_notification - CASCADE;`, - `DROP TYPE IF EXISTS severity;`, - }), - }) + }, + Down: []string{ + `DROP TABLE IF EXISTS Vulnerability_Notification CASCADE;`, + }, + } +) + +func init() { + RegisterMigration(NewSimpleMigration(1, + []MigrationQuery{ + entities, + detector, + layer, + ancestry, + vulnerability, + updaterLock, + notification, + })) } diff --git a/database/pgsql/migrations/util.go b/database/pgsql/migrations/util.go new file mode 100644 index 00000000..8b8898e0 --- /dev/null +++ b/database/pgsql/migrations/util.go @@ -0,0 +1,30 @@ +package migrations + +import "github.com/remind101/migrate" + +// MigrationQuery contains the Up migration and Down migration in Plain strings. +type MigrationQuery struct { + Up []string + Down []string +} + +// ConcatMigrationQueries concats migration queries in the give order. +func ConcatMigrationQueries(qs []MigrationQuery) MigrationQuery { + r := MigrationQuery{} + for _, q := range qs { + r.Up = append(r.Up, q.Up...) + r.Down = append(r.Down, q.Down...) + } + return r +} + +// NewSimpleMigration returns a simple migration plan with all provided +// migration queries concatted in order. +func NewSimpleMigration(id int, qs []MigrationQuery) migrate.Migration { + q := ConcatMigrationQueries(qs) + return migrate.Migration{ + ID: id, + Up: migrate.Queries(q.Up), + Down: migrate.Queries(q.Down), + } +} From 34d0e516e0792ca2d06299a1262e5676d4145f80 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Wed, 19 Sep 2018 13:53:30 -0400 Subject: [PATCH 06/14] vendor: Add golang-set dependency Golang-set library is added to make it easier to support set operations. --- glide.lock | 6 +- glide.yaml | 2 + .../github.com/deckarep/golang-set/.gitignore | 22 + .../deckarep/golang-set/.travis.yml | 11 + vendor/github.com/deckarep/golang-set/LICENSE | 22 + .../github.com/deckarep/golang-set/README.md | 95 ++ .../deckarep/golang-set/bench_test.go | 674 +++++++++ .../deckarep/golang-set/iterator.go | 58 + .../golang-set/iterator_example_test.go | 32 + vendor/github.com/deckarep/golang-set/set.go | 217 +++ .../deckarep/golang-set/set_test.go | 1200 +++++++++++++++++ .../deckarep/golang-set/threadsafe.go | 283 ++++ .../deckarep/golang-set/threadsafe_test.go | 524 +++++++ .../deckarep/golang-set/threadunsafe.go | 337 +++++ 14 files changed, 3481 insertions(+), 2 deletions(-) create mode 100644 vendor/github.com/deckarep/golang-set/.gitignore create mode 100644 vendor/github.com/deckarep/golang-set/.travis.yml create mode 100644 vendor/github.com/deckarep/golang-set/LICENSE create mode 100644 vendor/github.com/deckarep/golang-set/README.md create mode 100644 vendor/github.com/deckarep/golang-set/bench_test.go create mode 100644 vendor/github.com/deckarep/golang-set/iterator.go create mode 100644 vendor/github.com/deckarep/golang-set/iterator_example_test.go create mode 100644 vendor/github.com/deckarep/golang-set/set.go create mode 100644 vendor/github.com/deckarep/golang-set/set_test.go create mode 100644 vendor/github.com/deckarep/golang-set/threadsafe.go create mode 100644 vendor/github.com/deckarep/golang-set/threadsafe_test.go create mode 100644 vendor/github.com/deckarep/golang-set/threadunsafe.go diff --git a/glide.lock b/glide.lock index 73e38f49..b87b6d83 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 9bf7ad53b92119a17068b8724d9b406a7ca84b5bfcd0baba44b08c696a538b14 -updated: 2018-09-06T15:58:19.234504-04:00 +hash: 3fd0e471868863d6ef4cd32bbcdc9b3d061911a15b458e7edd26cfba4faa62db +updated: 2018-09-17T13:13:44.344244-04:00 imports: - name: github.com/beorn7/perks version: 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 @@ -15,6 +15,8 @@ imports: version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 subpackages: - spew +- name: github.com/deckarep/golang-set + version: cbaa98ba5575e67703b32b4b19f73c91f3c4159e - name: github.com/fernet/fernet-go version: 1b2437bc582b3cfbb341ee5a29f8ef5b42912ff2 - name: github.com/golang/protobuf diff --git a/glide.yaml b/glide.yaml index e5956e18..3da576b1 100644 --- a/glide.yaml +++ b/glide.yaml @@ -28,3 +28,5 @@ import: - assert - package: gopkg.in/yaml.v2 - package: github.com/cockroachdb/cmux +- package: github.com/deckarep/golang-set + version: ^1.7.1 diff --git a/vendor/github.com/deckarep/golang-set/.gitignore b/vendor/github.com/deckarep/golang-set/.gitignore new file mode 100644 index 00000000..00268614 --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/.gitignore @@ -0,0 +1,22 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe diff --git a/vendor/github.com/deckarep/golang-set/.travis.yml b/vendor/github.com/deckarep/golang-set/.travis.yml new file mode 100644 index 00000000..c760d24d --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/.travis.yml @@ -0,0 +1,11 @@ +language: go + +go: + - 1.8 + - 1.9 + - tip + +script: + - go test -race ./... + - go test -bench=. + diff --git a/vendor/github.com/deckarep/golang-set/LICENSE b/vendor/github.com/deckarep/golang-set/LICENSE new file mode 100644 index 00000000..b5768f89 --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/LICENSE @@ -0,0 +1,22 @@ +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/deckarep/golang-set/README.md b/vendor/github.com/deckarep/golang-set/README.md new file mode 100644 index 00000000..c3b50b2c --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/README.md @@ -0,0 +1,95 @@ +[![Build Status](https://travis-ci.org/deckarep/golang-set.svg?branch=master)](https://travis-ci.org/deckarep/golang-set) +[![Go Report Card](https://goreportcard.com/badge/github.com/deckarep/golang-set)](https://goreportcard.com/report/github.com/deckarep/golang-set) +[![GoDoc](https://godoc.org/github.com/deckarep/golang-set?status.svg)](http://godoc.org/github.com/deckarep/golang-set) + +## golang-set + + +The missing set collection for the Go language. Until Go has sets built-in...use this. + +Coming from Python one of the things I miss is the superbly wonderful set collection. This is my attempt to mimic the primary features of the set from Python. +You can of course argue that there is no need for a set in Go, otherwise the creators would have added one to the standard library. To those I say simply ignore this repository +and carry-on and to the rest that find this useful please contribute in helping me make it better by: + +* Helping to make more idiomatic improvements to the code. +* Helping to increase the performance of it. ~~(So far, no attempt has been made, but since it uses a map internally, I expect it to be mostly performant.)~~ +* Helping to make the unit-tests more robust and kick-ass. +* Helping to fill in the [documentation.](http://godoc.org/github.com/deckarep/golang-set) +* Simply offering feedback and suggestions. (Positive, constructive feedback is appreciated.) + +I have to give some credit for helping seed the idea with this post on [stackoverflow.](http://programmers.stackexchange.com/questions/177428/sets-data-structure-in-golang) + +*Update* - as of 3/9/2014, you can use a compile-time generic version of this package in the [gen](http://clipperhouse.github.io/gen/) framework. This framework allows you to use the golang-set in a completely generic and type-safe way by allowing you to generate a supporting .go file based on your custom types. + +## Features (as of 9/22/2014) + +* a CartesianProduct() method has been added with unit-tests: [Read more about the cartesian product](http://en.wikipedia.org/wiki/Cartesian_product) + +## Features (as of 9/15/2014) + +* a PowerSet() method has been added with unit-tests: [Read more about the Power set](http://en.wikipedia.org/wiki/Power_set) + +## Features (as of 4/22/2014) + +* One common interface to both implementations +* Two set implementations to choose from + * a thread-safe implementation designed for concurrent use + * a non-thread-safe implementation designed for performance +* 75 benchmarks for both implementations +* 35 unit tests for both implementations +* 14 concurrent tests for the thread-safe implementation + + + +Please see the unit test file for additional usage examples. The Python set documentation will also do a better job than I can of explaining how a set typically [works.](http://docs.python.org/2/library/sets.html) Please keep in mind +however that the Python set is a built-in type and supports additional features and syntax that make it awesome. + +## Examples but not exhaustive: + +```go +requiredClasses := mapset.NewSet() +requiredClasses.Add("Cooking") +requiredClasses.Add("English") +requiredClasses.Add("Math") +requiredClasses.Add("Biology") + +scienceSlice := []interface{}{"Biology", "Chemistry"} +scienceClasses := mapset.NewSetFromSlice(scienceSlice) + +electiveClasses := mapset.NewSet() +electiveClasses.Add("Welding") +electiveClasses.Add("Music") +electiveClasses.Add("Automotive") + +bonusClasses := mapset.NewSet() +bonusClasses.Add("Go Programming") +bonusClasses.Add("Python Programming") + +//Show me all the available classes I can take +allClasses := requiredClasses.Union(scienceClasses).Union(electiveClasses).Union(bonusClasses) +fmt.Println(allClasses) //Set{Cooking, English, Math, Chemistry, Welding, Biology, Music, Automotive, Go Programming, Python Programming} + + +//Is cooking considered a science class? +fmt.Println(scienceClasses.Contains("Cooking")) //false + +//Show me all classes that are not science classes, since I hate science. +fmt.Println(allClasses.Difference(scienceClasses)) //Set{Music, Automotive, Go Programming, Python Programming, Cooking, English, Math, Welding} + +//Which science classes are also required classes? +fmt.Println(scienceClasses.Intersect(requiredClasses)) //Set{Biology} + +//How many bonus classes do you offer? +fmt.Println(bonusClasses.Cardinality()) //2 + +//Do you have the following classes? Welding, Automotive and English? +fmt.Println(allClasses.IsSuperset(mapset.NewSetFromSlice([]interface{}{"Welding", "Automotive", "English"}))) //true +``` + +Thanks! + +-Ralph + +[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/deckarep/golang-set/trend.png)](https://bitdeli.com/free "Bitdeli Badge") + +[![Analytics](https://ga-beacon.appspot.com/UA-42584447-2/deckarep/golang-set)](https://github.com/igrigorik/ga-beacon) diff --git a/vendor/github.com/deckarep/golang-set/bench_test.go b/vendor/github.com/deckarep/golang-set/bench_test.go new file mode 100644 index 00000000..f893d101 --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/bench_test.go @@ -0,0 +1,674 @@ +package mapset + +import ( + "math/rand" + "testing" +) + +func nrand(n int) []int { + i := make([]int, n) + for ind := range i { + i[ind] = rand.Int() + } + return i +} + +func toInterfaces(i []int) []interface{} { + ifs := make([]interface{}, len(i)) + for ind, v := range i { + ifs[ind] = v + } + return ifs +} + +func benchAdd(b *testing.B, s Set) { + nums := nrand(b.N) + b.ResetTimer() + for _, v := range nums { + s.Add(v) + } +} + +func BenchmarkAddSafe(b *testing.B) { + benchAdd(b, NewSet()) +} + +func BenchmarkAddUnsafe(b *testing.B) { + benchAdd(b, NewThreadUnsafeSet()) +} + +func benchRemove(b *testing.B, s Set) { + nums := nrand(b.N) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for _, v := range nums { + s.Remove(v) + } +} + +func BenchmarkRemoveSafe(b *testing.B) { + benchRemove(b, NewSet()) +} + +func BenchmarkRemoveUnsafe(b *testing.B) { + benchRemove(b, NewThreadUnsafeSet()) +} + +func benchCardinality(b *testing.B, s Set) { + for i := 0; i < b.N; i++ { + s.Cardinality() + } +} + +func BenchmarkCardinalitySafe(b *testing.B) { + benchCardinality(b, NewSet()) +} + +func BenchmarkCardinalityUnsafe(b *testing.B) { + benchCardinality(b, NewThreadUnsafeSet()) +} + +func benchClear(b *testing.B, s Set) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Clear() + } +} + +func BenchmarkClearSafe(b *testing.B) { + benchClear(b, NewSet()) +} + +func BenchmarkClearUnsafe(b *testing.B) { + benchClear(b, NewThreadUnsafeSet()) +} + +func benchClone(b *testing.B, n int, s Set) { + nums := toInterfaces(nrand(n)) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Clone() + } +} + +func BenchmarkClone1Safe(b *testing.B) { + benchClone(b, 1, NewSet()) +} + +func BenchmarkClone1Unsafe(b *testing.B) { + benchClone(b, 1, NewThreadUnsafeSet()) +} + +func BenchmarkClone10Safe(b *testing.B) { + benchClone(b, 10, NewSet()) +} + +func BenchmarkClone10Unsafe(b *testing.B) { + benchClone(b, 10, NewThreadUnsafeSet()) +} + +func BenchmarkClone100Safe(b *testing.B) { + benchClone(b, 100, NewSet()) +} + +func BenchmarkClone100Unsafe(b *testing.B) { + benchClone(b, 100, NewThreadUnsafeSet()) +} + +func benchContains(b *testing.B, n int, s Set) { + nums := toInterfaces(nrand(n)) + for _, v := range nums { + s.Add(v) + } + + nums[n-1] = -1 // Definitely not in s + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Contains(nums...) + } +} + +func BenchmarkContains1Safe(b *testing.B) { + benchContains(b, 1, NewSet()) +} + +func BenchmarkContains1Unsafe(b *testing.B) { + benchContains(b, 1, NewThreadUnsafeSet()) +} + +func BenchmarkContains10Safe(b *testing.B) { + benchContains(b, 10, NewSet()) +} + +func BenchmarkContains10Unsafe(b *testing.B) { + benchContains(b, 10, NewThreadUnsafeSet()) +} + +func BenchmarkContains100Safe(b *testing.B) { + benchContains(b, 100, NewSet()) +} + +func BenchmarkContains100Unsafe(b *testing.B) { + benchContains(b, 100, NewThreadUnsafeSet()) +} + +func benchEqual(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Equal(t) + } +} + +func BenchmarkEqual1Safe(b *testing.B) { + benchEqual(b, 1, NewSet(), NewSet()) +} + +func BenchmarkEqual1Unsafe(b *testing.B) { + benchEqual(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkEqual10Safe(b *testing.B) { + benchEqual(b, 10, NewSet(), NewSet()) +} + +func BenchmarkEqual10Unsafe(b *testing.B) { + benchEqual(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkEqual100Safe(b *testing.B) { + benchEqual(b, 100, NewSet(), NewSet()) +} + +func BenchmarkEqual100Unsafe(b *testing.B) { + benchEqual(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchDifference(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + } + for _, v := range nums[:n/2] { + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Difference(t) + } +} + +func benchIsSubset(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.IsSubset(t) + } +} + +func BenchmarkIsSubset1Safe(b *testing.B) { + benchIsSubset(b, 1, NewSet(), NewSet()) +} + +func BenchmarkIsSubset1Unsafe(b *testing.B) { + benchIsSubset(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsSubset10Safe(b *testing.B) { + benchIsSubset(b, 10, NewSet(), NewSet()) +} + +func BenchmarkIsSubset10Unsafe(b *testing.B) { + benchIsSubset(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsSubset100Safe(b *testing.B) { + benchIsSubset(b, 100, NewSet(), NewSet()) +} + +func BenchmarkIsSubset100Unsafe(b *testing.B) { + benchIsSubset(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchIsSuperset(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.IsSuperset(t) + } +} + +func BenchmarkIsSuperset1Safe(b *testing.B) { + benchIsSuperset(b, 1, NewSet(), NewSet()) +} + +func BenchmarkIsSuperset1Unsafe(b *testing.B) { + benchIsSuperset(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsSuperset10Safe(b *testing.B) { + benchIsSuperset(b, 10, NewSet(), NewSet()) +} + +func BenchmarkIsSuperset10Unsafe(b *testing.B) { + benchIsSuperset(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsSuperset100Safe(b *testing.B) { + benchIsSuperset(b, 100, NewSet(), NewSet()) +} + +func BenchmarkIsSuperset100Unsafe(b *testing.B) { + benchIsSuperset(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchIsProperSubset(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.IsProperSubset(t) + } +} + +func BenchmarkIsProperSubset1Safe(b *testing.B) { + benchIsProperSubset(b, 1, NewSet(), NewSet()) +} + +func BenchmarkIsProperSubset1Unsafe(b *testing.B) { + benchIsProperSubset(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsProperSubset10Safe(b *testing.B) { + benchIsProperSubset(b, 10, NewSet(), NewSet()) +} + +func BenchmarkIsProperSubset10Unsafe(b *testing.B) { + benchIsProperSubset(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsProperSubset100Safe(b *testing.B) { + benchIsProperSubset(b, 100, NewSet(), NewSet()) +} + +func BenchmarkIsProperSubset100Unsafe(b *testing.B) { + benchIsProperSubset(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchIsProperSuperset(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.IsProperSuperset(t) + } +} + +func BenchmarkIsProperSuperset1Safe(b *testing.B) { + benchIsProperSuperset(b, 1, NewSet(), NewSet()) +} + +func BenchmarkIsProperSuperset1Unsafe(b *testing.B) { + benchIsProperSuperset(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsProperSuperset10Safe(b *testing.B) { + benchIsProperSuperset(b, 10, NewSet(), NewSet()) +} + +func BenchmarkIsProperSuperset10Unsafe(b *testing.B) { + benchIsProperSuperset(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsProperSuperset100Safe(b *testing.B) { + benchIsProperSuperset(b, 100, NewSet(), NewSet()) +} + +func BenchmarkIsProperSuperset100Unsafe(b *testing.B) { + benchIsProperSuperset(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkDifference1Safe(b *testing.B) { + benchDifference(b, 1, NewSet(), NewSet()) +} + +func BenchmarkDifference1Unsafe(b *testing.B) { + benchDifference(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkDifference10Safe(b *testing.B) { + benchDifference(b, 10, NewSet(), NewSet()) +} + +func BenchmarkDifference10Unsafe(b *testing.B) { + benchDifference(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkDifference100Safe(b *testing.B) { + benchDifference(b, 100, NewSet(), NewSet()) +} + +func BenchmarkDifference100Unsafe(b *testing.B) { + benchDifference(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchIntersect(b *testing.B, n int, s, t Set) { + nums := nrand(int(float64(n) * float64(1.5))) + for _, v := range nums[:n] { + s.Add(v) + } + for _, v := range nums[n/2:] { + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Intersect(t) + } +} + +func BenchmarkIntersect1Safe(b *testing.B) { + benchIntersect(b, 1, NewSet(), NewSet()) +} + +func BenchmarkIntersect1Unsafe(b *testing.B) { + benchIntersect(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIntersect10Safe(b *testing.B) { + benchIntersect(b, 10, NewSet(), NewSet()) +} + +func BenchmarkIntersect10Unsafe(b *testing.B) { + benchIntersect(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIntersect100Safe(b *testing.B) { + benchIntersect(b, 100, NewSet(), NewSet()) +} + +func BenchmarkIntersect100Unsafe(b *testing.B) { + benchIntersect(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchSymmetricDifference(b *testing.B, n int, s, t Set) { + nums := nrand(int(float64(n) * float64(1.5))) + for _, v := range nums[:n] { + s.Add(v) + } + for _, v := range nums[n/2:] { + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.SymmetricDifference(t) + } +} + +func BenchmarkSymmetricDifference1Safe(b *testing.B) { + benchSymmetricDifference(b, 1, NewSet(), NewSet()) +} + +func BenchmarkSymmetricDifference1Unsafe(b *testing.B) { + benchSymmetricDifference(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkSymmetricDifference10Safe(b *testing.B) { + benchSymmetricDifference(b, 10, NewSet(), NewSet()) +} + +func BenchmarkSymmetricDifference10Unsafe(b *testing.B) { + benchSymmetricDifference(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkSymmetricDifference100Safe(b *testing.B) { + benchSymmetricDifference(b, 100, NewSet(), NewSet()) +} + +func BenchmarkSymmetricDifference100Unsafe(b *testing.B) { + benchSymmetricDifference(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchUnion(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums[:n/2] { + s.Add(v) + } + for _, v := range nums[n/2:] { + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Union(t) + } +} + +func BenchmarkUnion1Safe(b *testing.B) { + benchUnion(b, 1, NewSet(), NewSet()) +} + +func BenchmarkUnion1Unsafe(b *testing.B) { + benchUnion(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkUnion10Safe(b *testing.B) { + benchUnion(b, 10, NewSet(), NewSet()) +} + +func BenchmarkUnion10Unsafe(b *testing.B) { + benchUnion(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkUnion100Safe(b *testing.B) { + benchUnion(b, 100, NewSet(), NewSet()) +} + +func BenchmarkUnion100Unsafe(b *testing.B) { + benchUnion(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchEach(b *testing.B, n int, s Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Each(func(elem interface{}) bool { + return false + }) + } +} + +func BenchmarkEach1Safe(b *testing.B) { + benchEach(b, 1, NewSet()) +} + +func BenchmarkEach1Unsafe(b *testing.B) { + benchEach(b, 1, NewThreadUnsafeSet()) +} + +func BenchmarkEach10Safe(b *testing.B) { + benchEach(b, 10, NewSet()) +} + +func BenchmarkEach10Unsafe(b *testing.B) { + benchEach(b, 10, NewThreadUnsafeSet()) +} + +func BenchmarkEach100Safe(b *testing.B) { + benchEach(b, 100, NewSet()) +} + +func BenchmarkEach100Unsafe(b *testing.B) { + benchEach(b, 100, NewThreadUnsafeSet()) +} + +func benchIter(b *testing.B, n int, s Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + c := s.Iter() + for range c { + + } + } +} + +func BenchmarkIter1Safe(b *testing.B) { + benchIter(b, 1, NewSet()) +} + +func BenchmarkIter1Unsafe(b *testing.B) { + benchIter(b, 1, NewThreadUnsafeSet()) +} + +func BenchmarkIter10Safe(b *testing.B) { + benchIter(b, 10, NewSet()) +} + +func BenchmarkIter10Unsafe(b *testing.B) { + benchIter(b, 10, NewThreadUnsafeSet()) +} + +func BenchmarkIter100Safe(b *testing.B) { + benchIter(b, 100, NewSet()) +} + +func BenchmarkIter100Unsafe(b *testing.B) { + benchIter(b, 100, NewThreadUnsafeSet()) +} + +func benchIterator(b *testing.B, n int, s Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + c := s.Iterator().C + for range c { + + } + } +} + +func BenchmarkIterator1Safe(b *testing.B) { + benchIterator(b, 1, NewSet()) +} + +func BenchmarkIterator1Unsafe(b *testing.B) { + benchIterator(b, 1, NewThreadUnsafeSet()) +} + +func BenchmarkIterator10Safe(b *testing.B) { + benchIterator(b, 10, NewSet()) +} + +func BenchmarkIterator10Unsafe(b *testing.B) { + benchIterator(b, 10, NewThreadUnsafeSet()) +} + +func BenchmarkIterator100Safe(b *testing.B) { + benchIterator(b, 100, NewSet()) +} + +func BenchmarkIterator100Unsafe(b *testing.B) { + benchIterator(b, 100, NewThreadUnsafeSet()) +} + +func benchString(b *testing.B, n int, s Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = s.String() + } +} + +func BenchmarkString1Safe(b *testing.B) { + benchString(b, 1, NewSet()) +} + +func BenchmarkString1Unsafe(b *testing.B) { + benchString(b, 1, NewThreadUnsafeSet()) +} + +func BenchmarkString10Safe(b *testing.B) { + benchString(b, 10, NewSet()) +} + +func BenchmarkString10Unsafe(b *testing.B) { + benchString(b, 10, NewThreadUnsafeSet()) +} + +func BenchmarkString100Safe(b *testing.B) { + benchString(b, 100, NewSet()) +} + +func BenchmarkString100Unsafe(b *testing.B) { + benchString(b, 100, NewThreadUnsafeSet()) +} + +func benchToSlice(b *testing.B, s Set) { + nums := nrand(b.N) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.ToSlice() + } +} + +func BenchmarkToSliceSafe(b *testing.B) { + benchToSlice(b, NewSet()) +} + +func BenchmarkToSliceUnsafe(b *testing.B) { + benchToSlice(b, NewThreadUnsafeSet()) +} diff --git a/vendor/github.com/deckarep/golang-set/iterator.go b/vendor/github.com/deckarep/golang-set/iterator.go new file mode 100644 index 00000000..9dfecade --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/iterator.go @@ -0,0 +1,58 @@ +/* +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package mapset + +// Iterator defines an iterator over a Set, its C channel can be used to range over the Set's +// elements. +type Iterator struct { + C <-chan interface{} + stop chan struct{} +} + +// Stop stops the Iterator, no further elements will be received on C, C will be closed. +func (i *Iterator) Stop() { + // Allows for Stop() to be called multiple times + // (close() panics when called on already closed channel) + defer func() { + recover() + }() + + close(i.stop) + + // Exhaust any remaining elements. + for range i.C { + } +} + +// newIterator returns a new Iterator instance together with its item and stop channels. +func newIterator() (*Iterator, chan<- interface{}, <-chan struct{}) { + itemChan := make(chan interface{}) + stopChan := make(chan struct{}) + return &Iterator{ + C: itemChan, + stop: stopChan, + }, itemChan, stopChan +} diff --git a/vendor/github.com/deckarep/golang-set/iterator_example_test.go b/vendor/github.com/deckarep/golang-set/iterator_example_test.go new file mode 100644 index 00000000..fc4235d7 --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/iterator_example_test.go @@ -0,0 +1,32 @@ +package mapset + +import ( + "fmt" +) + +type YourType struct { + Name string +} + +func ExampleIterator() { + set := NewSetFromSlice([]interface{}{ + &YourType{Name: "Alise"}, + &YourType{Name: "Bob"}, + &YourType{Name: "John"}, + &YourType{Name: "Nick"}, + }) + + var found *YourType + it := set.Iterator() + + for elem := range it.C { + if elem.(*YourType).Name == "John" { + found = elem.(*YourType) + it.Stop() + } + } + + fmt.Printf("Found %+v\n", found) + + // Output: Found &{Name:John} +} diff --git a/vendor/github.com/deckarep/golang-set/set.go b/vendor/github.com/deckarep/golang-set/set.go new file mode 100644 index 00000000..29eb2e5a --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/set.go @@ -0,0 +1,217 @@ +/* +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Package mapset implements a simple and generic set collection. +// Items stored within it are unordered and unique. It supports +// typical set operations: membership testing, intersection, union, +// difference, symmetric difference and cloning. +// +// Package mapset provides two implementations of the Set +// interface. The default implementation is safe for concurrent +// access, but a non-thread-safe implementation is also provided for +// programs that can benefit from the slight speed improvement and +// that can enforce mutual exclusion through other means. +package mapset + +// Set is the primary interface provided by the mapset package. It +// represents an unordered set of data and a large number of +// operations that can be applied to that set. +type Set interface { + // Adds an element to the set. Returns whether + // the item was added. + Add(i interface{}) bool + + // Returns the number of elements in the set. + Cardinality() int + + // Removes all elements from the set, leaving + // the empty set. + Clear() + + // Returns a clone of the set using the same + // implementation, duplicating all keys. + Clone() Set + + // Returns whether the given items + // are all in the set. + Contains(i ...interface{}) bool + + // Returns the difference between this set + // and other. The returned set will contain + // all elements of this set that are not also + // elements of other. + // + // Note that the argument to Difference + // must be of the same type as the receiver + // of the method. Otherwise, Difference will + // panic. + Difference(other Set) Set + + // Determines if two sets are equal to each + // other. If they have the same cardinality + // and contain the same elements, they are + // considered equal. The order in which + // the elements were added is irrelevant. + // + // Note that the argument to Equal must be + // of the same type as the receiver of the + // method. Otherwise, Equal will panic. + Equal(other Set) bool + + // Returns a new set containing only the elements + // that exist only in both sets. + // + // Note that the argument to Intersect + // must be of the same type as the receiver + // of the method. Otherwise, Intersect will + // panic. + Intersect(other Set) Set + + // Determines if every element in this set is in + // the other set but the two sets are not equal. + // + // Note that the argument to IsProperSubset + // must be of the same type as the receiver + // of the method. Otherwise, IsProperSubset + // will panic. + IsProperSubset(other Set) bool + + // Determines if every element in the other set + // is in this set but the two sets are not + // equal. + // + // Note that the argument to IsSuperset + // must be of the same type as the receiver + // of the method. Otherwise, IsSuperset will + // panic. + IsProperSuperset(other Set) bool + + // Determines if every element in this set is in + // the other set. + // + // Note that the argument to IsSubset + // must be of the same type as the receiver + // of the method. Otherwise, IsSubset will + // panic. + IsSubset(other Set) bool + + // Determines if every element in the other set + // is in this set. + // + // Note that the argument to IsSuperset + // must be of the same type as the receiver + // of the method. Otherwise, IsSuperset will + // panic. + IsSuperset(other Set) bool + + // Iterates over elements and executes the passed func against each element. + // If passed func returns true, stop iteration at the time. + Each(func(interface{}) bool) + + // Returns a channel of elements that you can + // range over. + Iter() <-chan interface{} + + // Returns an Iterator object that you can + // use to range over the set. + Iterator() *Iterator + + // Remove a single element from the set. + Remove(i interface{}) + + // Provides a convenient string representation + // of the current state of the set. + String() string + + // Returns a new set with all elements which are + // in either this set or the other set but not in both. + // + // Note that the argument to SymmetricDifference + // must be of the same type as the receiver + // of the method. Otherwise, SymmetricDifference + // will panic. + SymmetricDifference(other Set) Set + + // Returns a new set with all elements in both sets. + // + // Note that the argument to Union must be of the + + // same type as the receiver of the method. + // Otherwise, IsSuperset will panic. + Union(other Set) Set + + // Pop removes and returns an arbitrary item from the set. + Pop() interface{} + + // Returns all subsets of a given set (Power Set). + PowerSet() Set + + // Returns the Cartesian Product of two sets. + CartesianProduct(other Set) Set + + // Returns the members of the set as a slice. + ToSlice() []interface{} +} + +// NewSet creates and returns a reference to an empty set. Operations +// on the resulting set are thread-safe. +func NewSet(s ...interface{}) Set { + set := newThreadSafeSet() + for _, item := range s { + set.Add(item) + } + return &set +} + +// NewSetWith creates and returns a new set with the given elements. +// Operations on the resulting set are thread-safe. +func NewSetWith(elts ...interface{}) Set { + return NewSetFromSlice(elts) +} + +// NewSetFromSlice creates and returns a reference to a set from an +// existing slice. Operations on the resulting set are thread-safe. +func NewSetFromSlice(s []interface{}) Set { + a := NewSet(s...) + return a +} + +// NewThreadUnsafeSet creates and returns a reference to an empty set. +// Operations on the resulting set are not thread-safe. +func NewThreadUnsafeSet() Set { + set := newThreadUnsafeSet() + return &set +} + +// NewThreadUnsafeSetFromSlice creates and returns a reference to a +// set from an existing slice. Operations on the resulting set are +// not thread-safe. +func NewThreadUnsafeSetFromSlice(s []interface{}) Set { + a := NewThreadUnsafeSet() + for _, item := range s { + a.Add(item) + } + return a +} diff --git a/vendor/github.com/deckarep/golang-set/set_test.go b/vendor/github.com/deckarep/golang-set/set_test.go new file mode 100644 index 00000000..6cdf5833 --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/set_test.go @@ -0,0 +1,1200 @@ +/* +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package mapset + +import "testing" + +func makeSet(ints []int) Set { + set := NewSet() + for _, i := range ints { + set.Add(i) + } + return set +} + +func makeUnsafeSet(ints []int) Set { + set := NewThreadUnsafeSet() + for _, i := range ints { + set.Add(i) + } + return set +} + +func assertEqual(a, b Set, t *testing.T) { + if !a.Equal(b) { + t.Errorf("%v != %v\n", a, b) + } +} + +func Test_NewSet(t *testing.T) { + a := NewSet() + if a.Cardinality() != 0 { + t.Error("NewSet should start out as an empty set") + } + + assertEqual(NewSetFromSlice([]interface{}{}), NewSet(), t) + assertEqual(NewSetFromSlice([]interface{}{1}), NewSet(1), t) + assertEqual(NewSetFromSlice([]interface{}{1, 2}), NewSet(1, 2), t) + assertEqual(NewSetFromSlice([]interface{}{"a"}), NewSet("a"), t) + assertEqual(NewSetFromSlice([]interface{}{"a", "b"}), NewSet("a", "b"), t) +} + +func Test_NewUnsafeSet(t *testing.T) { + a := NewThreadUnsafeSet() + + if a.Cardinality() != 0 { + t.Error("NewSet should start out as an empty set") + } +} + +func Test_AddSet(t *testing.T) { + a := makeSet([]int{1, 2, 3}) + + if a.Cardinality() != 3 { + t.Error("AddSet does not have a size of 3 even though 3 items were added to a new set") + } +} + +func Test_AddUnsafeSet(t *testing.T) { + a := makeUnsafeSet([]int{1, 2, 3}) + + if a.Cardinality() != 3 { + t.Error("AddSet does not have a size of 3 even though 3 items were added to a new set") + } +} + +func Test_AddSetNoDuplicate(t *testing.T) { + a := makeSet([]int{7, 5, 3, 7}) + + if a.Cardinality() != 3 { + t.Error("AddSetNoDuplicate set should have 3 elements since 7 is a duplicate") + } + + if !(a.Contains(7) && a.Contains(5) && a.Contains(3)) { + t.Error("AddSetNoDuplicate set should have a 7, 5, and 3 in it.") + } +} + +func Test_AddUnsafeSetNoDuplicate(t *testing.T) { + a := makeUnsafeSet([]int{7, 5, 3, 7}) + + if a.Cardinality() != 3 { + t.Error("AddSetNoDuplicate set should have 3 elements since 7 is a duplicate") + } + + if !(a.Contains(7) && a.Contains(5) && a.Contains(3)) { + t.Error("AddSetNoDuplicate set should have a 7, 5, and 3 in it.") + } +} + +func Test_RemoveSet(t *testing.T) { + a := makeSet([]int{6, 3, 1}) + + a.Remove(3) + + if a.Cardinality() != 2 { + t.Error("RemoveSet should only have 2 items in the set") + } + + if !(a.Contains(6) && a.Contains(1)) { + t.Error("RemoveSet should have only items 6 and 1 in the set") + } + + a.Remove(6) + a.Remove(1) + + if a.Cardinality() != 0 { + t.Error("RemoveSet should be an empty set after removing 6 and 1") + } +} + +func Test_RemoveUnsafeSet(t *testing.T) { + a := makeUnsafeSet([]int{6, 3, 1}) + + a.Remove(3) + + if a.Cardinality() != 2 { + t.Error("RemoveSet should only have 2 items in the set") + } + + if !(a.Contains(6) && a.Contains(1)) { + t.Error("RemoveSet should have only items 6 and 1 in the set") + } + + a.Remove(6) + a.Remove(1) + + if a.Cardinality() != 0 { + t.Error("RemoveSet should be an empty set after removing 6 and 1") + } +} + +func Test_ContainsSet(t *testing.T) { + a := NewSet() + + a.Add(71) + + if !a.Contains(71) { + t.Error("ContainsSet should contain 71") + } + + a.Remove(71) + + if a.Contains(71) { + t.Error("ContainsSet should not contain 71") + } + + a.Add(13) + a.Add(7) + a.Add(1) + + if !(a.Contains(13) && a.Contains(7) && a.Contains(1)) { + t.Error("ContainsSet should contain 13, 7, 1") + } +} + +func Test_ContainsUnsafeSet(t *testing.T) { + a := NewThreadUnsafeSet() + + a.Add(71) + + if !a.Contains(71) { + t.Error("ContainsSet should contain 71") + } + + a.Remove(71) + + if a.Contains(71) { + t.Error("ContainsSet should not contain 71") + } + + a.Add(13) + a.Add(7) + a.Add(1) + + if !(a.Contains(13) && a.Contains(7) && a.Contains(1)) { + t.Error("ContainsSet should contain 13, 7, 1") + } +} + +func Test_ContainsMultipleSet(t *testing.T) { + a := makeSet([]int{8, 6, 7, 5, 3, 0, 9}) + + if !a.Contains(8, 6, 7, 5, 3, 0, 9) { + t.Error("ContainsAll should contain Jenny's phone number") + } + + if a.Contains(8, 6, 11, 5, 3, 0, 9) { + t.Error("ContainsAll should not have all of these numbers") + } +} + +func Test_ContainsMultipleUnsafeSet(t *testing.T) { + a := makeUnsafeSet([]int{8, 6, 7, 5, 3, 0, 9}) + + if !a.Contains(8, 6, 7, 5, 3, 0, 9) { + t.Error("ContainsAll should contain Jenny's phone number") + } + + if a.Contains(8, 6, 11, 5, 3, 0, 9) { + t.Error("ContainsAll should not have all of these numbers") + } +} + +func Test_ClearSet(t *testing.T) { + a := makeSet([]int{2, 5, 9, 10}) + + a.Clear() + + if a.Cardinality() != 0 { + t.Error("ClearSet should be an empty set") + } +} + +func Test_ClearUnsafeSet(t *testing.T) { + a := makeUnsafeSet([]int{2, 5, 9, 10}) + + a.Clear() + + if a.Cardinality() != 0 { + t.Error("ClearSet should be an empty set") + } +} + +func Test_CardinalitySet(t *testing.T) { + a := NewSet() + + if a.Cardinality() != 0 { + t.Error("set should be an empty set") + } + + a.Add(1) + + if a.Cardinality() != 1 { + t.Error("set should have a size of 1") + } + + a.Remove(1) + + if a.Cardinality() != 0 { + t.Error("set should be an empty set") + } + + a.Add(9) + + if a.Cardinality() != 1 { + t.Error("set should have a size of 1") + } + + a.Clear() + + if a.Cardinality() != 0 { + t.Error("set should have a size of 1") + } +} + +func Test_CardinalityUnsafeSet(t *testing.T) { + a := NewThreadUnsafeSet() + + if a.Cardinality() != 0 { + t.Error("set should be an empty set") + } + + a.Add(1) + + if a.Cardinality() != 1 { + t.Error("set should have a size of 1") + } + + a.Remove(1) + + if a.Cardinality() != 0 { + t.Error("set should be an empty set") + } + + a.Add(9) + + if a.Cardinality() != 1 { + t.Error("set should have a size of 1") + } + + a.Clear() + + if a.Cardinality() != 0 { + t.Error("set should have a size of 1") + } +} + +func Test_SetIsSubset(t *testing.T) { + a := makeSet([]int{1, 2, 3, 5, 7}) + + b := NewSet() + b.Add(3) + b.Add(5) + b.Add(7) + + if !b.IsSubset(a) { + t.Error("set b should be a subset of set a") + } + + b.Add(72) + + if b.IsSubset(a) { + t.Error("set b should not be a subset of set a because it contains 72 which is not in the set of a") + } +} + +func Test_SetIsProperSubset(t *testing.T) { + a := makeSet([]int{1, 2, 3, 5, 7}) + b := makeSet([]int{7, 5, 3, 2, 1}) + + if !a.IsSubset(b) { + t.Error("set a should be a subset of set b") + } + if a.IsProperSubset(b) { + t.Error("set a should not be a proper subset of set b (they're equal)") + } + + b.Add(72) + + if !a.IsSubset(b) { + t.Error("set a should be a subset of set b") + } + if !a.IsProperSubset(b) { + t.Error("set a should be a proper subset of set b") + } +} + +func Test_UnsafeSetIsSubset(t *testing.T) { + a := makeUnsafeSet([]int{1, 2, 3, 5, 7}) + + b := NewThreadUnsafeSet() + b.Add(3) + b.Add(5) + b.Add(7) + + if !b.IsSubset(a) { + t.Error("set b should be a subset of set a") + } + + b.Add(72) + + if b.IsSubset(a) { + t.Error("set b should not be a subset of set a because it contains 72 which is not in the set of a") + } +} + +func Test_UnsafeSetIsProperSubset(t *testing.T) { + a := makeUnsafeSet([]int{1, 2, 3, 5, 7}) + b := NewThreadUnsafeSet() + b.Add(7) + b.Add(1) + b.Add(5) + b.Add(3) + b.Add(2) + + if !a.IsSubset(b) { + t.Error("set a should be a subset of set b") + } + if a.IsProperSubset(b) { + t.Error("set a should not be a proper subset of set b (they're equal)") + } + + b.Add(72) + + if !a.IsSubset(b) { + t.Error("set a should be a subset of set b") + } + if !a.IsProperSubset(b) { + t.Error("set a should be a proper subset of set b because set b has 72") + } +} + +func Test_SetIsSuperset(t *testing.T) { + a := NewSet() + a.Add(9) + a.Add(5) + a.Add(2) + a.Add(1) + a.Add(11) + + b := NewSet() + b.Add(5) + b.Add(2) + b.Add(11) + + if !a.IsSuperset(b) { + t.Error("set a should be a superset of set b") + } + + b.Add(42) + + if a.IsSuperset(b) { + t.Error("set a should not be a superset of set b because set b has a 42") + } +} + +func Test_SetIsProperSuperset(t *testing.T) { + a := NewSet() + a.Add(5) + a.Add(2) + a.Add(11) + + b := NewSet() + b.Add(2) + b.Add(5) + b.Add(11) + + if !a.IsSuperset(b) { + t.Error("set a should be a superset of set b") + } + if a.IsProperSuperset(b) { + t.Error("set a should not be a proper superset of set b (they're equal)") + } + + a.Add(9) + + if !a.IsSuperset(b) { + t.Error("set a should be a superset of set b") + } + if !a.IsProperSuperset(b) { + t.Error("set a not be a proper superset of set b because set a has a 9") + } + + b.Add(42) + + if a.IsSuperset(b) { + t.Error("set a should not be a superset of set b because set b has a 42") + } + if a.IsProperSuperset(b) { + t.Error("set a should not be a proper superset of set b because set b has a 42") + } +} + +func Test_UnsafeSetIsSuperset(t *testing.T) { + a := NewThreadUnsafeSet() + a.Add(9) + a.Add(5) + a.Add(2) + a.Add(1) + a.Add(11) + + b := NewThreadUnsafeSet() + b.Add(5) + b.Add(2) + b.Add(11) + + if !a.IsSuperset(b) { + t.Error("set a should be a superset of set b") + } + + b.Add(42) + + if a.IsSuperset(b) { + t.Error("set a should not be a superset of set b because set a has a 42") + } +} + +func Test_UnsafeSetIsProperSuperset(t *testing.T) { + a := NewThreadUnsafeSet() + a.Add(5) + a.Add(2) + a.Add(11) + + b := NewThreadUnsafeSet() + b.Add(2) + b.Add(5) + b.Add(11) + + if !a.IsSuperset(b) { + t.Error("set a should be a superset of set b") + } + if a.IsProperSuperset(b) { + t.Error("set a should not be a proper superset of set b (they're equal)") + } + + a.Add(9) + + if !a.IsSuperset(b) { + t.Error("set a should be a superset of set b") + } + if !a.IsProperSuperset(b) { + t.Error("set a not be a proper superset of set b because set a has a 9") + } + + b.Add(42) + + if a.IsSuperset(b) { + t.Error("set a should not be a superset of set b because set b has a 42") + } + if a.IsProperSuperset(b) { + t.Error("set a should not be a proper superset of set b because set b has a 42") + } +} + +func Test_SetUnion(t *testing.T) { + a := NewSet() + + b := NewSet() + b.Add(1) + b.Add(2) + b.Add(3) + b.Add(4) + b.Add(5) + + c := a.Union(b) + + if c.Cardinality() != 5 { + t.Error("set c is unioned with an empty set and therefore should have 5 elements in it") + } + + d := NewSet() + d.Add(10) + d.Add(14) + d.Add(0) + + e := c.Union(d) + if e.Cardinality() != 8 { + t.Error("set e should should have 8 elements in it after being unioned with set c to d") + } + + f := NewSet() + f.Add(14) + f.Add(3) + + g := f.Union(e) + if g.Cardinality() != 8 { + t.Error("set g should still have 8 elements in it after being unioned with set f that has duplicates") + } +} + +func Test_UnsafeSetUnion(t *testing.T) { + a := NewThreadUnsafeSet() + + b := NewThreadUnsafeSet() + b.Add(1) + b.Add(2) + b.Add(3) + b.Add(4) + b.Add(5) + + c := a.Union(b) + + if c.Cardinality() != 5 { + t.Error("set c is unioned with an empty set and therefore should have 5 elements in it") + } + + d := NewThreadUnsafeSet() + d.Add(10) + d.Add(14) + d.Add(0) + + e := c.Union(d) + if e.Cardinality() != 8 { + t.Error("set e should should have 8 elements in it after being unioned with set c to d") + } + + f := NewThreadUnsafeSet() + f.Add(14) + f.Add(3) + + g := f.Union(e) + if g.Cardinality() != 8 { + t.Error("set g should still have 8 elements in it after being unioned with set f that has duplicates") + } +} + +func Test_SetIntersect(t *testing.T) { + a := NewSet() + a.Add(1) + a.Add(3) + a.Add(5) + + b := NewSet() + a.Add(2) + a.Add(4) + a.Add(6) + + c := a.Intersect(b) + + if c.Cardinality() != 0 { + t.Error("set c should be the empty set because there is no common items to intersect") + } + + a.Add(10) + b.Add(10) + + d := a.Intersect(b) + + if !(d.Cardinality() == 1 && d.Contains(10)) { + t.Error("set d should have a size of 1 and contain the item 10") + } +} + +func Test_UnsafeSetIntersect(t *testing.T) { + a := NewThreadUnsafeSet() + a.Add(1) + a.Add(3) + a.Add(5) + + b := NewThreadUnsafeSet() + a.Add(2) + a.Add(4) + a.Add(6) + + c := a.Intersect(b) + + if c.Cardinality() != 0 { + t.Error("set c should be the empty set because there is no common items to intersect") + } + + a.Add(10) + b.Add(10) + + d := a.Intersect(b) + + if !(d.Cardinality() == 1 && d.Contains(10)) { + t.Error("set d should have a size of 1 and contain the item 10") + } +} + +func Test_SetDifference(t *testing.T) { + a := NewSet() + a.Add(1) + a.Add(2) + a.Add(3) + + b := NewSet() + b.Add(1) + b.Add(3) + b.Add(4) + b.Add(5) + b.Add(6) + b.Add(99) + + c := a.Difference(b) + + if !(c.Cardinality() == 1 && c.Contains(2)) { + t.Error("the difference of set a to b is the set of 1 item: 2") + } +} + +func Test_UnsafeSetDifference(t *testing.T) { + a := NewThreadUnsafeSet() + a.Add(1) + a.Add(2) + a.Add(3) + + b := NewThreadUnsafeSet() + b.Add(1) + b.Add(3) + b.Add(4) + b.Add(5) + b.Add(6) + b.Add(99) + + c := a.Difference(b) + + if !(c.Cardinality() == 1 && c.Contains(2)) { + t.Error("the difference of set a to b is the set of 1 item: 2") + } +} + +func Test_SetSymmetricDifference(t *testing.T) { + a := NewSet() + a.Add(1) + a.Add(2) + a.Add(3) + a.Add(45) + + b := NewSet() + b.Add(1) + b.Add(3) + b.Add(4) + b.Add(5) + b.Add(6) + b.Add(99) + + c := a.SymmetricDifference(b) + + if !(c.Cardinality() == 6 && c.Contains(2) && c.Contains(45) && c.Contains(4) && c.Contains(5) && c.Contains(6) && c.Contains(99)) { + t.Error("the symmetric difference of set a to b is the set of 6 items: 2, 45, 4, 5, 6, 99") + } +} + +func Test_UnsafeSetSymmetricDifference(t *testing.T) { + a := NewThreadUnsafeSet() + a.Add(1) + a.Add(2) + a.Add(3) + a.Add(45) + + b := NewThreadUnsafeSet() + b.Add(1) + b.Add(3) + b.Add(4) + b.Add(5) + b.Add(6) + b.Add(99) + + c := a.SymmetricDifference(b) + + if !(c.Cardinality() == 6 && c.Contains(2) && c.Contains(45) && c.Contains(4) && c.Contains(5) && c.Contains(6) && c.Contains(99)) { + t.Error("the symmetric difference of set a to b is the set of 6 items: 2, 45, 4, 5, 6, 99") + } +} + +func Test_SetEqual(t *testing.T) { + a := NewSet() + b := NewSet() + + if !a.Equal(b) { + t.Error("Both a and b are empty sets, and should be equal") + } + + a.Add(10) + + if a.Equal(b) { + t.Error("a should not be equal to b because b is empty and a has item 1 in it") + } + + b.Add(10) + + if !a.Equal(b) { + t.Error("a is now equal again to b because both have the item 10 in them") + } + + b.Add(8) + b.Add(3) + b.Add(47) + + if a.Equal(b) { + t.Error("b has 3 more elements in it so therefore should not be equal to a") + } + + a.Add(8) + a.Add(3) + a.Add(47) + + if !a.Equal(b) { + t.Error("a and b should be equal with the same number of elements") + } +} + +func Test_UnsafeSetEqual(t *testing.T) { + a := NewThreadUnsafeSet() + b := NewThreadUnsafeSet() + + if !a.Equal(b) { + t.Error("Both a and b are empty sets, and should be equal") + } + + a.Add(10) + + if a.Equal(b) { + t.Error("a should not be equal to b because b is empty and a has item 1 in it") + } + + b.Add(10) + + if !a.Equal(b) { + t.Error("a is now equal again to b because both have the item 10 in them") + } + + b.Add(8) + b.Add(3) + b.Add(47) + + if a.Equal(b) { + t.Error("b has 3 more elements in it so therefore should not be equal to a") + } + + a.Add(8) + a.Add(3) + a.Add(47) + + if !a.Equal(b) { + t.Error("a and b should be equal with the same number of elements") + } +} + +func Test_SetClone(t *testing.T) { + a := NewSet() + a.Add(1) + a.Add(2) + + b := a.Clone() + + if !a.Equal(b) { + t.Error("Clones should be equal") + } + + a.Add(3) + if a.Equal(b) { + t.Error("a contains one more element, they should not be equal") + } + + c := a.Clone() + c.Remove(1) + + if a.Equal(c) { + t.Error("C contains one element less, they should not be equal") + } +} + +func Test_UnsafeSetClone(t *testing.T) { + a := NewThreadUnsafeSet() + a.Add(1) + a.Add(2) + + b := a.Clone() + + if !a.Equal(b) { + t.Error("Clones should be equal") + } + + a.Add(3) + if a.Equal(b) { + t.Error("a contains one more element, they should not be equal") + } + + c := a.Clone() + c.Remove(1) + + if a.Equal(c) { + t.Error("C contains one element less, they should not be equal") + } +} + +func Test_Each(t *testing.T) { + a := NewSet() + + a.Add("Z") + a.Add("Y") + a.Add("X") + a.Add("W") + + b := NewSet() + a.Each(func(elem interface{}) bool { + b.Add(elem) + return false + }) + + if !a.Equal(b) { + t.Error("The sets are not equal after iterating (Each) through the first set") + } + + var count int + a.Each(func(elem interface{}) bool { + if count == 2 { + return true + } + count++ + return false + }) + if count != 2 { + t.Error("Iteration should stop on the way") + } +} + +func Test_Iter(t *testing.T) { + a := NewSet() + + a.Add("Z") + a.Add("Y") + a.Add("X") + a.Add("W") + + b := NewSet() + for val := range a.Iter() { + b.Add(val) + } + + if !a.Equal(b) { + t.Error("The sets are not equal after iterating (Iter) through the first set") + } +} + +func Test_UnsafeIter(t *testing.T) { + a := NewThreadUnsafeSet() + + a.Add("Z") + a.Add("Y") + a.Add("X") + a.Add("W") + + b := NewThreadUnsafeSet() + for val := range a.Iter() { + b.Add(val) + } + + if !a.Equal(b) { + t.Error("The sets are not equal after iterating (Iter) through the first set") + } +} + +func Test_Iterator(t *testing.T) { + a := NewSet() + + a.Add("Z") + a.Add("Y") + a.Add("X") + a.Add("W") + + b := NewSet() + for val := range a.Iterator().C { + b.Add(val) + } + + if !a.Equal(b) { + t.Error("The sets are not equal after iterating (Iterator) through the first set") + } +} + +func Test_UnsafeIterator(t *testing.T) { + a := NewThreadUnsafeSet() + + a.Add("Z") + a.Add("Y") + a.Add("X") + a.Add("W") + + b := NewThreadUnsafeSet() + for val := range a.Iterator().C { + b.Add(val) + } + + if !a.Equal(b) { + t.Error("The sets are not equal after iterating (Iterator) through the first set") + } +} + +func Test_IteratorStop(t *testing.T) { + a := NewSet() + + a.Add("Z") + a.Add("Y") + a.Add("X") + a.Add("W") + + it := a.Iterator() + it.Stop() + for range it.C { + t.Error("The iterating (Iterator) did not stop after Stop() has been called") + } +} + +func Test_PopSafe(t *testing.T) { + a := NewSet() + + a.Add("a") + a.Add("b") + a.Add("c") + a.Add("d") + + captureSet := NewSet() + captureSet.Add(a.Pop()) + captureSet.Add(a.Pop()) + captureSet.Add(a.Pop()) + captureSet.Add(a.Pop()) + finalNil := a.Pop() + + if captureSet.Cardinality() != 4 { + t.Error("unexpected captureSet cardinality; should be 4") + } + + if a.Cardinality() != 0 { + t.Error("unepxected a cardinality; should be zero") + } + + if !captureSet.Contains("c", "a", "d", "b") { + t.Error("unexpected result set; should be a,b,c,d (any order is fine") + } + + if finalNil != nil { + t.Error("when original set is empty, further pops should result in nil") + } +} + +func Test_PopUnsafe(t *testing.T) { + a := NewThreadUnsafeSet() + + a.Add("a") + a.Add("b") + a.Add("c") + a.Add("d") + + captureSet := NewThreadUnsafeSet() + captureSet.Add(a.Pop()) + captureSet.Add(a.Pop()) + captureSet.Add(a.Pop()) + captureSet.Add(a.Pop()) + finalNil := a.Pop() + + if captureSet.Cardinality() != 4 { + t.Error("unexpected captureSet cardinality; should be 4") + } + + if a.Cardinality() != 0 { + t.Error("unepxected a cardinality; should be zero") + } + + if !captureSet.Contains("c", "a", "d", "b") { + t.Error("unexpected result set; should be a,b,c,d (any order is fine") + } + + if finalNil != nil { + t.Error("when original set is empty, further pops should result in nil") + } +} + +func Test_PowerSet(t *testing.T) { + a := NewThreadUnsafeSet() + + a.Add(1) + a.Add("delta") + a.Add("chi") + a.Add(4) + + b := a.PowerSet() + if b.Cardinality() != 16 { + t.Error("unexpected PowerSet cardinality") + } +} + +func Test_PowerSetThreadSafe(t *testing.T) { + set := NewSet().PowerSet() + _, setIsThreadSafe := set.(*threadSafeSet) + if !setIsThreadSafe { + t.Error("result of PowerSet should be thread safe") + } + + subset := set.Pop() + _, subsetIsThreadSafe := subset.(*threadSafeSet) + if !subsetIsThreadSafe { + t.Error("subsets in PowerSet result should be thread safe") + } +} + +func Test_EmptySetProperties(t *testing.T) { + empty := NewSet() + + a := NewSet() + a.Add(1) + a.Add("foo") + a.Add("bar") + + b := NewSet() + b.Add("one") + b.Add("two") + b.Add(3) + b.Add(4) + + if !empty.IsSubset(a) || !empty.IsSubset(b) { + t.Error("The empty set is supposed to be a subset of all sets") + } + + if !a.IsSuperset(empty) || !b.IsSuperset(empty) { + t.Error("All sets are supposed to be a superset of the empty set") + } + + if !empty.IsSubset(empty) || !empty.IsSuperset(empty) { + t.Error("The empty set is supposed to be a subset and a superset of itself") + } + + c := a.Union(empty) + if !c.Equal(a) { + t.Error("The union of any set with the empty set is supposed to be equal to itself") + } + + c = a.Intersect(empty) + if !c.Equal(empty) { + t.Error("The intesection of any set with the empty set is supposed to be the empty set") + } + + c = a.CartesianProduct(empty) + if c.Cardinality() != 0 { + t.Error("Cartesian product of any set and the empty set must be the empty set") + } + + if empty.Cardinality() != 0 { + t.Error("Cardinality of the empty set is supposed to be zero") + } + + c = empty.PowerSet() + if c.Cardinality() != 1 { + t.Error("Cardinality of the power set of the empty set is supposed to be one { {} }") + } +} + +func Test_CartesianProduct(t *testing.T) { + a := NewThreadUnsafeSet() + b := NewThreadUnsafeSet() + empty := NewThreadUnsafeSet() + + a.Add(1) + a.Add(2) + a.Add(3) + + b.Add("one") + b.Add("two") + b.Add("three") + b.Add("alpha") + b.Add("gamma") + + c := a.CartesianProduct(b) + d := b.CartesianProduct(a) + + if c.Cardinality() != d.Cardinality() { + t.Error("Cardinality of AxB must be equal to BxA") + } + + if c.Cardinality() != (a.Cardinality() * b.Cardinality()) { + t.Error("Unexpected cardinality for cartesian product set") + } + + c = a.CartesianProduct(empty) + d = empty.CartesianProduct(b) + + if c.Cardinality() != 0 || d.Cardinality() != 0 { + t.Error("Cartesian product of any set and the empty set Ax0 || 0xA must be the empty set") + } +} + +func Test_ToSliceUnthreadsafe(t *testing.T) { + s := makeUnsafeSet([]int{1, 2, 3}) + setAsSlice := s.ToSlice() + if len(setAsSlice) != s.Cardinality() { + t.Errorf("Set length is incorrect: %v", len(setAsSlice)) + } + + for _, i := range setAsSlice { + if !s.Contains(i) { + t.Errorf("Set is missing element: %v", i) + } + } +} + +func Test_Example(t *testing.T) { + /* + requiredClasses := NewSet() + requiredClasses.Add("Cooking") + requiredClasses.Add("English") + requiredClasses.Add("Math") + requiredClasses.Add("Biology") + + scienceSlice := []interface{}{"Biology", "Chemistry"} + scienceClasses := NewSetFromSlice(scienceSlice) + + electiveClasses := NewSet() + electiveClasses.Add("Welding") + electiveClasses.Add("Music") + electiveClasses.Add("Automotive") + + bonusClasses := NewSet() + bonusClasses.Add("Go Programming") + bonusClasses.Add("Python Programming") + + //Show me all the available classes I can take + allClasses := requiredClasses.Union(scienceClasses).Union(electiveClasses).Union(bonusClasses) + fmt.Println(allClasses) //Set{English, Chemistry, Automotive, Cooking, Math, Biology, Welding, Music, Go Programming} + + //Is cooking considered a science class? + fmt.Println(scienceClasses.Contains("Cooking")) //false + + //Show me all classes that are not science classes, since I hate science. + fmt.Println(allClasses.Difference(scienceClasses)) //Set{English, Automotive, Cooking, Math, Welding, Music, Go Programming} + + //Which science classes are also required classes? + fmt.Println(scienceClasses.Intersect(requiredClasses)) //Set{Biology} + + //How many bonus classes do you offer? + fmt.Println(bonusClasses.Cardinality()) //2 + + //Do you have the following classes? Welding, Automotive and English? + fmt.Println(allClasses.ContainsAll("Welding", "Automotive", "English")) + */ +} diff --git a/vendor/github.com/deckarep/golang-set/threadsafe.go b/vendor/github.com/deckarep/golang-set/threadsafe.go new file mode 100644 index 00000000..269b4ab0 --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/threadsafe.go @@ -0,0 +1,283 @@ +/* +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package mapset + +import "sync" + +type threadSafeSet struct { + s threadUnsafeSet + sync.RWMutex +} + +func newThreadSafeSet() threadSafeSet { + return threadSafeSet{s: newThreadUnsafeSet()} +} + +func (set *threadSafeSet) Add(i interface{}) bool { + set.Lock() + ret := set.s.Add(i) + set.Unlock() + return ret +} + +func (set *threadSafeSet) Contains(i ...interface{}) bool { + set.RLock() + ret := set.s.Contains(i...) + set.RUnlock() + return ret +} + +func (set *threadSafeSet) IsSubset(other Set) bool { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + ret := set.s.IsSubset(&o.s) + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) IsProperSubset(other Set) bool { + o := other.(*threadSafeSet) + + set.RLock() + defer set.RUnlock() + o.RLock() + defer o.RUnlock() + + return set.s.IsProperSubset(&o.s) +} + +func (set *threadSafeSet) IsSuperset(other Set) bool { + return other.IsSubset(set) +} + +func (set *threadSafeSet) IsProperSuperset(other Set) bool { + return other.IsProperSubset(set) +} + +func (set *threadSafeSet) Union(other Set) Set { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + unsafeUnion := set.s.Union(&o.s).(*threadUnsafeSet) + ret := &threadSafeSet{s: *unsafeUnion} + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) Intersect(other Set) Set { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + unsafeIntersection := set.s.Intersect(&o.s).(*threadUnsafeSet) + ret := &threadSafeSet{s: *unsafeIntersection} + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) Difference(other Set) Set { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + unsafeDifference := set.s.Difference(&o.s).(*threadUnsafeSet) + ret := &threadSafeSet{s: *unsafeDifference} + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) SymmetricDifference(other Set) Set { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + unsafeDifference := set.s.SymmetricDifference(&o.s).(*threadUnsafeSet) + ret := &threadSafeSet{s: *unsafeDifference} + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) Clear() { + set.Lock() + set.s = newThreadUnsafeSet() + set.Unlock() +} + +func (set *threadSafeSet) Remove(i interface{}) { + set.Lock() + delete(set.s, i) + set.Unlock() +} + +func (set *threadSafeSet) Cardinality() int { + set.RLock() + defer set.RUnlock() + return len(set.s) +} + +func (set *threadSafeSet) Each(cb func(interface{}) bool) { + set.RLock() + for elem := range set.s { + if cb(elem) { + break + } + } + set.RUnlock() +} + +func (set *threadSafeSet) Iter() <-chan interface{} { + ch := make(chan interface{}) + go func() { + set.RLock() + + for elem := range set.s { + ch <- elem + } + close(ch) + set.RUnlock() + }() + + return ch +} + +func (set *threadSafeSet) Iterator() *Iterator { + iterator, ch, stopCh := newIterator() + + go func() { + set.RLock() + L: + for elem := range set.s { + select { + case <-stopCh: + break L + case ch <- elem: + } + } + close(ch) + set.RUnlock() + }() + + return iterator +} + +func (set *threadSafeSet) Equal(other Set) bool { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + ret := set.s.Equal(&o.s) + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) Clone() Set { + set.RLock() + + unsafeClone := set.s.Clone().(*threadUnsafeSet) + ret := &threadSafeSet{s: *unsafeClone} + set.RUnlock() + return ret +} + +func (set *threadSafeSet) String() string { + set.RLock() + ret := set.s.String() + set.RUnlock() + return ret +} + +func (set *threadSafeSet) PowerSet() Set { + set.RLock() + unsafePowerSet := set.s.PowerSet().(*threadUnsafeSet) + set.RUnlock() + + ret := &threadSafeSet{s: newThreadUnsafeSet()} + for subset := range unsafePowerSet.Iter() { + unsafeSubset := subset.(*threadUnsafeSet) + ret.Add(&threadSafeSet{s: *unsafeSubset}) + } + return ret +} + +func (set *threadSafeSet) Pop() interface{} { + set.Lock() + defer set.Unlock() + return set.s.Pop() +} + +func (set *threadSafeSet) CartesianProduct(other Set) Set { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + unsafeCartProduct := set.s.CartesianProduct(&o.s).(*threadUnsafeSet) + ret := &threadSafeSet{s: *unsafeCartProduct} + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) ToSlice() []interface{} { + keys := make([]interface{}, 0, set.Cardinality()) + set.RLock() + for elem := range set.s { + keys = append(keys, elem) + } + set.RUnlock() + return keys +} + +func (set *threadSafeSet) MarshalJSON() ([]byte, error) { + set.RLock() + b, err := set.s.MarshalJSON() + set.RUnlock() + + return b, err +} + +func (set *threadSafeSet) UnmarshalJSON(p []byte) error { + set.RLock() + err := set.s.UnmarshalJSON(p) + set.RUnlock() + + return err +} diff --git a/vendor/github.com/deckarep/golang-set/threadsafe_test.go b/vendor/github.com/deckarep/golang-set/threadsafe_test.go new file mode 100644 index 00000000..5c32fcbd --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/threadsafe_test.go @@ -0,0 +1,524 @@ +/* +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package mapset + +import ( + "encoding/json" + "math/rand" + "runtime" + "sync" + "sync/atomic" + "testing" +) + +const N = 1000 + +func Test_AddConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + + var wg sync.WaitGroup + wg.Add(len(ints)) + for i := 0; i < len(ints); i++ { + go func(i int) { + s.Add(i) + wg.Done() + }(i) + } + + wg.Wait() + for _, i := range ints { + if !s.Contains(i) { + t.Errorf("Set is missing element: %v", i) + } + } +} + +func Test_CardinalityConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + elems := s.Cardinality() + for i := 0; i < N; i++ { + newElems := s.Cardinality() + if newElems < elems { + t.Errorf("Cardinality shrunk from %v to %v", elems, newElems) + } + } + wg.Done() + }() + + for i := 0; i < N; i++ { + s.Add(rand.Int()) + } + wg.Wait() +} + +func Test_ClearConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + + var wg sync.WaitGroup + wg.Add(len(ints)) + for i := 0; i < len(ints); i++ { + go func() { + s.Clear() + wg.Done() + }() + go func(i int) { + s.Add(i) + }(i) + } + + wg.Wait() +} + +func Test_CloneConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + + for _, v := range ints { + s.Add(v) + } + + var wg sync.WaitGroup + wg.Add(len(ints)) + for i := range ints { + go func(i int) { + s.Remove(i) + wg.Done() + }(i) + } + + s.Clone() +} + +func Test_ContainsConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + interfaces := make([]interface{}, 0) + for _, v := range ints { + s.Add(v) + interfaces = append(interfaces, v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.Contains(interfaces...) + wg.Done() + }() + } + wg.Wait() +} + +func Test_DifferenceConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.Difference(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_EqualConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.Equal(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_IntersectConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.Intersect(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_IsSubsetConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.IsSubset(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_IsProperSubsetConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.IsProperSubset(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_IsSupersetConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.IsSuperset(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_IsProperSupersetConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.IsProperSuperset(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_EachConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + concurrent := 10 + + s := NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + } + + var count int64 + wg := new(sync.WaitGroup) + wg.Add(concurrent) + for n := 0; n < concurrent; n++ { + go func() { + defer wg.Done() + s.Each(func(elem interface{}) bool { + atomic.AddInt64(&count, 1) + return false + }) + }() + } + wg.Wait() + + if count != int64(N*concurrent) { + t.Errorf("%v != %v", count, int64(N*concurrent)) + } +} + +func Test_IterConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + } + + cs := make([]<-chan interface{}, 0) + for range ints { + cs = append(cs, s.Iter()) + } + + c := make(chan interface{}) + go func() { + for n := 0; n < len(ints)*N; { + for _, d := range cs { + select { + case <-d: + n++ + c <- nil + default: + } + } + } + close(c) + }() + + for range c { + } +} + +func Test_RemoveConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + } + + var wg sync.WaitGroup + wg.Add(len(ints)) + for _, v := range ints { + go func(i int) { + s.Remove(i) + wg.Done() + }(v) + } + wg.Wait() + + if s.Cardinality() != 0 { + t.Errorf("Expected cardinality 0; got %v", s.Cardinality()) + } +} + +func Test_StringConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + } + + var wg sync.WaitGroup + wg.Add(len(ints)) + for range ints { + go func() { + _ = s.String() + wg.Done() + }() + } + wg.Wait() +} + +func Test_SymmetricDifferenceConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.SymmetricDifference(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_ToSlice(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + + var wg sync.WaitGroup + wg.Add(len(ints)) + for i := 0; i < len(ints); i++ { + go func(i int) { + s.Add(i) + wg.Done() + }(i) + } + + wg.Wait() + setAsSlice := s.ToSlice() + if len(setAsSlice) != s.Cardinality() { + t.Errorf("Set length is incorrect: %v", len(setAsSlice)) + } + + for _, i := range setAsSlice { + if !s.Contains(i) { + t.Errorf("Set is missing element: %v", i) + } + } +} + +// Test_ToSliceDeadlock - fixes issue: https://github.com/deckarep/golang-set/issues/36 +// This code reveals the deadlock however it doesn't happen consistently. +func Test_ToSliceDeadlock(t *testing.T) { + runtime.GOMAXPROCS(2) + + var wg sync.WaitGroup + set := NewSet() + workers := 10 + wg.Add(workers) + for i := 1; i <= workers; i++ { + go func() { + for j := 0; j < 1000; j++ { + set.Add(1) + set.ToSlice() + } + wg.Done() + }() + } + wg.Wait() +} + +func Test_UnmarshalJSON(t *testing.T) { + s := []byte(`["test", 1, 2, 3, ["4,5,6"]]`) + expected := NewSetFromSlice( + []interface{}{ + json.Number("1"), + json.Number("2"), + json.Number("3"), + "test", + }, + ) + actual := NewSet() + err := json.Unmarshal(s, actual) + if err != nil { + t.Errorf("Error should be nil: %v", err) + } + + if !expected.Equal(actual) { + t.Errorf("Expected no difference, got: %v", expected.Difference(actual)) + } +} + +func Test_MarshalJSON(t *testing.T) { + expected := NewSetFromSlice( + []interface{}{ + json.Number("1"), + "test", + }, + ) + + b, err := json.Marshal( + NewSetFromSlice( + []interface{}{ + 1, + "test", + }, + ), + ) + if err != nil { + t.Errorf("Error should be nil: %v", err) + } + + actual := NewSet() + err = json.Unmarshal(b, actual) + if err != nil { + t.Errorf("Error should be nil: %v", err) + } + + if !expected.Equal(actual) { + t.Errorf("Expected no difference, got: %v", expected.Difference(actual)) + } +} diff --git a/vendor/github.com/deckarep/golang-set/threadunsafe.go b/vendor/github.com/deckarep/golang-set/threadunsafe.go new file mode 100644 index 00000000..10bdd46f --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/threadunsafe.go @@ -0,0 +1,337 @@ +/* +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package mapset + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strings" +) + +type threadUnsafeSet map[interface{}]struct{} + +// An OrderedPair represents a 2-tuple of values. +type OrderedPair struct { + First interface{} + Second interface{} +} + +func newThreadUnsafeSet() threadUnsafeSet { + return make(threadUnsafeSet) +} + +// Equal says whether two 2-tuples contain the same values in the same order. +func (pair *OrderedPair) Equal(other OrderedPair) bool { + if pair.First == other.First && + pair.Second == other.Second { + return true + } + + return false +} + +func (set *threadUnsafeSet) Add(i interface{}) bool { + _, found := (*set)[i] + if found { + return false //False if it existed already + } + + (*set)[i] = struct{}{} + return true +} + +func (set *threadUnsafeSet) Contains(i ...interface{}) bool { + for _, val := range i { + if _, ok := (*set)[val]; !ok { + return false + } + } + return true +} + +func (set *threadUnsafeSet) IsSubset(other Set) bool { + _ = other.(*threadUnsafeSet) + for elem := range *set { + if !other.Contains(elem) { + return false + } + } + return true +} + +func (set *threadUnsafeSet) IsProperSubset(other Set) bool { + return set.IsSubset(other) && !set.Equal(other) +} + +func (set *threadUnsafeSet) IsSuperset(other Set) bool { + return other.IsSubset(set) +} + +func (set *threadUnsafeSet) IsProperSuperset(other Set) bool { + return set.IsSuperset(other) && !set.Equal(other) +} + +func (set *threadUnsafeSet) Union(other Set) Set { + o := other.(*threadUnsafeSet) + + unionedSet := newThreadUnsafeSet() + + for elem := range *set { + unionedSet.Add(elem) + } + for elem := range *o { + unionedSet.Add(elem) + } + return &unionedSet +} + +func (set *threadUnsafeSet) Intersect(other Set) Set { + o := other.(*threadUnsafeSet) + + intersection := newThreadUnsafeSet() + // loop over smaller set + if set.Cardinality() < other.Cardinality() { + for elem := range *set { + if other.Contains(elem) { + intersection.Add(elem) + } + } + } else { + for elem := range *o { + if set.Contains(elem) { + intersection.Add(elem) + } + } + } + return &intersection +} + +func (set *threadUnsafeSet) Difference(other Set) Set { + _ = other.(*threadUnsafeSet) + + difference := newThreadUnsafeSet() + for elem := range *set { + if !other.Contains(elem) { + difference.Add(elem) + } + } + return &difference +} + +func (set *threadUnsafeSet) SymmetricDifference(other Set) Set { + _ = other.(*threadUnsafeSet) + + aDiff := set.Difference(other) + bDiff := other.Difference(set) + return aDiff.Union(bDiff) +} + +func (set *threadUnsafeSet) Clear() { + *set = newThreadUnsafeSet() +} + +func (set *threadUnsafeSet) Remove(i interface{}) { + delete(*set, i) +} + +func (set *threadUnsafeSet) Cardinality() int { + return len(*set) +} + +func (set *threadUnsafeSet) Each(cb func(interface{}) bool) { + for elem := range *set { + if cb(elem) { + break + } + } +} + +func (set *threadUnsafeSet) Iter() <-chan interface{} { + ch := make(chan interface{}) + go func() { + for elem := range *set { + ch <- elem + } + close(ch) + }() + + return ch +} + +func (set *threadUnsafeSet) Iterator() *Iterator { + iterator, ch, stopCh := newIterator() + + go func() { + L: + for elem := range *set { + select { + case <-stopCh: + break L + case ch <- elem: + } + } + close(ch) + }() + + return iterator +} + +func (set *threadUnsafeSet) Equal(other Set) bool { + _ = other.(*threadUnsafeSet) + + if set.Cardinality() != other.Cardinality() { + return false + } + for elem := range *set { + if !other.Contains(elem) { + return false + } + } + return true +} + +func (set *threadUnsafeSet) Clone() Set { + clonedSet := newThreadUnsafeSet() + for elem := range *set { + clonedSet.Add(elem) + } + return &clonedSet +} + +func (set *threadUnsafeSet) String() string { + items := make([]string, 0, len(*set)) + + for elem := range *set { + items = append(items, fmt.Sprintf("%v", elem)) + } + return fmt.Sprintf("Set{%s}", strings.Join(items, ", ")) +} + +// String outputs a 2-tuple in the form "(A, B)". +func (pair OrderedPair) String() string { + return fmt.Sprintf("(%v, %v)", pair.First, pair.Second) +} + +func (set *threadUnsafeSet) Pop() interface{} { + for item := range *set { + delete(*set, item) + return item + } + return nil +} + +func (set *threadUnsafeSet) PowerSet() Set { + powSet := NewThreadUnsafeSet() + nullset := newThreadUnsafeSet() + powSet.Add(&nullset) + + for es := range *set { + u := newThreadUnsafeSet() + j := powSet.Iter() + for er := range j { + p := newThreadUnsafeSet() + if reflect.TypeOf(er).Name() == "" { + k := er.(*threadUnsafeSet) + for ek := range *(k) { + p.Add(ek) + } + } else { + p.Add(er) + } + p.Add(es) + u.Add(&p) + } + + powSet = powSet.Union(&u) + } + + return powSet +} + +func (set *threadUnsafeSet) CartesianProduct(other Set) Set { + o := other.(*threadUnsafeSet) + cartProduct := NewThreadUnsafeSet() + + for i := range *set { + for j := range *o { + elem := OrderedPair{First: i, Second: j} + cartProduct.Add(elem) + } + } + + return cartProduct +} + +func (set *threadUnsafeSet) ToSlice() []interface{} { + keys := make([]interface{}, 0, set.Cardinality()) + for elem := range *set { + keys = append(keys, elem) + } + + return keys +} + +// MarshalJSON creates a JSON array from the set, it marshals all elements +func (set *threadUnsafeSet) MarshalJSON() ([]byte, error) { + items := make([]string, 0, set.Cardinality()) + + for elem := range *set { + b, err := json.Marshal(elem) + if err != nil { + return nil, err + } + + items = append(items, string(b)) + } + + return []byte(fmt.Sprintf("[%s]", strings.Join(items, ","))), nil +} + +// UnmarshalJSON recreates a set from a JSON array, it only decodes +// primitive types. Numbers are decoded as json.Number. +func (set *threadUnsafeSet) UnmarshalJSON(b []byte) error { + var i []interface{} + + d := json.NewDecoder(bytes.NewReader(b)) + d.UseNumber() + err := d.Decode(&i) + if err != nil { + return err + } + + for _, v := range i { + switch t := v.(type) { + case []interface{}, map[string]interface{}: + continue + default: + set.Add(t) + } + } + + return nil +} From 53bf19aecfcccb367bc359a2dd6d7320fa4e4855 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Wed, 19 Sep 2018 14:31:15 -0400 Subject: [PATCH 07/14] ext: Lister and Detector returns detector info with detected content 1. Every Lister and Detector are versioned 2. detected content, are returned in a map with detector info as the key --- ext/featurefmt/apk/apk.go | 2 +- ext/featurefmt/dpkg/dpkg.go | 2 +- ext/featurefmt/driver.go | 71 +++++++++++------- ext/featurefmt/rpm/rpm.go | 2 +- ext/featurens/alpinerelease/alpinerelease.go | 2 +- ext/featurens/aptsources/aptsources.go | 2 +- ext/featurens/driver.go | 75 ++++++++++++-------- ext/featurens/driver_test.go | 63 +++++++--------- ext/featurens/lsbrelease/lsbrelease.go | 2 +- ext/featurens/osrelease/osrelease.go | 2 +- ext/featurens/redhatrelease/redhatrelease.go | 2 +- ext/imagefmt/driver.go | 9 ++- 12 files changed, 130 insertions(+), 104 deletions(-) diff --git a/ext/featurefmt/apk/apk.go b/ext/featurefmt/apk/apk.go index 195c8920..eda25cd4 100644 --- a/ext/featurefmt/apk/apk.go +++ b/ext/featurefmt/apk/apk.go @@ -29,7 +29,7 @@ import ( ) func init() { - featurefmt.RegisterLister("apk", dpkg.ParserName, &lister{}) + featurefmt.RegisterLister("apk", "1.0", &lister{}) } type lister struct{} diff --git a/ext/featurefmt/dpkg/dpkg.go b/ext/featurefmt/dpkg/dpkg.go index 6b987cf3..0ac30a90 100644 --- a/ext/featurefmt/dpkg/dpkg.go +++ b/ext/featurefmt/dpkg/dpkg.go @@ -37,7 +37,7 @@ var ( type lister struct{} func init() { - featurefmt.RegisterLister("dpkg", dpkg.ParserName, &lister{}) + featurefmt.RegisterLister("dpkg", "1.0", &lister{}) } func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) { diff --git a/ext/featurefmt/driver.go b/ext/featurefmt/driver.go index 0f48b0e7..6ade140b 100644 --- a/ext/featurefmt/driver.go +++ b/ext/featurefmt/driver.go @@ -31,9 +31,8 @@ import ( ) var ( - listersM sync.RWMutex - listers = make(map[string]Lister) - versionfmtListerName = make(map[string][]string) + listersM sync.RWMutex + listers = make(map[string]lister) ) // Lister represents an ability to list the features present in an image layer. @@ -48,13 +47,19 @@ type Lister interface { RequiredFilenames() []string } +type lister struct { + Lister + + info database.Detector +} + // RegisterLister makes a Lister available by the provided name. // // If called twice with the same name, the name is blank, or if the provided // Lister is nil, this function panics. -func RegisterLister(name string, versionfmt string, l Lister) { - if name == "" { - panic("featurefmt: could not register a Lister with an empty name") +func RegisterLister(name string, version string, l Lister) { + if name == "" || version == "" { + panic("featurefmt: could not register a Lister with an empty name or version") } if l == nil { panic("featurefmt: could not register a nil Lister") @@ -67,51 +72,65 @@ func RegisterLister(name string, versionfmt string, l Lister) { panic("featurefmt: RegisterLister called twice for " + name) } - listers[name] = l - versionfmtListerName[versionfmt] = append(versionfmtListerName[versionfmt], name) + listers[name] = lister{l, database.NewFeatureDetector(name, version)} } // ListFeatures produces the list of Features in an image layer using // every registered Lister. -func ListFeatures(files tarutil.FilesMap, listerNames []string) ([]database.Feature, error) { +func ListFeatures(files tarutil.FilesMap, toUse []database.Detector) ([]database.LayerFeature, error) { listersM.RLock() defer listersM.RUnlock() - var totalFeatures []database.Feature + features := []database.LayerFeature{} + for _, d := range toUse { + // Only use the detector with the same type + if d.DType != database.FeatureDetectorType { + continue + } - for _, name := range listerNames { - if lister, ok := listers[name]; ok { - features, err := lister.ListFeatures(files) + if lister, ok := listers[d.Name]; ok { + fs, err := lister.ListFeatures(files) if err != nil { - return []database.Feature{}, err + return nil, err } - totalFeatures = append(totalFeatures, features...) + + for _, f := range fs { + features = append(features, database.LayerFeature{ + Feature: f, + By: lister.info, + }) + } + } else { - log.WithField("Name", name).Warn("Unknown Lister") + log.WithField("Name", d).Fatal("unknown feature detector") } } - return totalFeatures, nil + return features, nil } -// RequiredFilenames returns the total list of files required for all -// registered Listers. -func RequiredFilenames(listerNames []string) (files []string) { +// RequiredFilenames returns all files required by the give extensions. Any +// extension metadata that has non feature-detector type will be skipped. +func RequiredFilenames(toUse []database.Detector) (files []string) { listersM.RLock() defer listersM.RUnlock() - for _, lister := range listers { - files = append(files, lister.RequiredFilenames()...) + for _, d := range toUse { + if d.DType != database.FeatureDetectorType { + continue + } + + files = append(files, listers[d.Name].RequiredFilenames()...) } return } // ListListers returns the names of all the registered feature listers. -func ListListers() []string { - r := []string{} - for name := range listers { - r = append(r, name) +func ListListers() []database.Detector { + r := []database.Detector{} + for _, d := range listers { + r = append(r, d.info) } return r } diff --git a/ext/featurefmt/rpm/rpm.go b/ext/featurefmt/rpm/rpm.go index 5a0e1fa1..be582384 100644 --- a/ext/featurefmt/rpm/rpm.go +++ b/ext/featurefmt/rpm/rpm.go @@ -35,7 +35,7 @@ import ( type lister struct{} func init() { - featurefmt.RegisterLister("rpm", rpm.ParserName, &lister{}) + featurefmt.RegisterLister("rpm", "1.0", &lister{}) } func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) { diff --git a/ext/featurens/alpinerelease/alpinerelease.go b/ext/featurens/alpinerelease/alpinerelease.go index fafe5c9f..8fe6f808 100644 --- a/ext/featurens/alpinerelease/alpinerelease.go +++ b/ext/featurens/alpinerelease/alpinerelease.go @@ -36,7 +36,7 @@ const ( var versionRegexp = regexp.MustCompile(`^(\d)+\.(\d)+\.(\d)+$`) func init() { - featurens.RegisterDetector("alpine-release", &detector{}) + featurens.RegisterDetector("alpine-release", "1.0", &detector{}) } type detector struct{} diff --git a/ext/featurens/aptsources/aptsources.go b/ext/featurens/aptsources/aptsources.go index c43818e7..287073f4 100644 --- a/ext/featurens/aptsources/aptsources.go +++ b/ext/featurens/aptsources/aptsources.go @@ -32,7 +32,7 @@ import ( type detector struct{} func init() { - featurens.RegisterDetector("apt-sources", &detector{}) + featurens.RegisterDetector("apt-sources", "1.0", &detector{}) } func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { diff --git a/ext/featurens/driver.go b/ext/featurens/driver.go index b7e0ad37..781ba5d4 100644 --- a/ext/featurens/driver.go +++ b/ext/featurens/driver.go @@ -29,7 +29,7 @@ import ( var ( detectorsM sync.RWMutex - detectors = make(map[string]Detector) + detectors = make(map[string]detector) ) // Detector represents an ability to detect a namespace used for organizing @@ -46,13 +46,19 @@ type Detector interface { RequiredFilenames() []string } +type detector struct { + Detector + + info database.Detector +} + // RegisterDetector makes a detector available by the provided name. // // If called twice with the same name, the name is blank, or if the provided // Detector is nil, this function panics. -func RegisterDetector(name string, d Detector) { - if name == "" { - panic("namespace: could not register a Detector with an empty name") +func RegisterDetector(name string, version string, d Detector) { + if name == "" || version == "" { + panic("namespace: could not register a Detector with an empty name or version") } if d == nil { panic("namespace: could not register a nil Detector") @@ -61,60 +67,69 @@ func RegisterDetector(name string, d Detector) { detectorsM.Lock() defer detectorsM.Unlock() - if _, dup := detectors[name]; dup { + if _, ok := detectors[name]; ok { panic("namespace: RegisterDetector called twice for " + name) } - detectors[name] = d + detectors[name] = detector{d, database.NewNamespaceDetector(name, version)} } -// Detect iterators through all registered Detectors and returns all non-nil detected namespaces -func Detect(files tarutil.FilesMap, detectorNames []string) ([]database.Namespace, error) { +// Detect uses detectors specified to retrieve the detect result. +func Detect(files tarutil.FilesMap, toUse []database.Detector) ([]database.LayerNamespace, error) { detectorsM.RLock() defer detectorsM.RUnlock() - namespaces := map[string]*database.Namespace{} - for _, name := range detectorNames { - if detector, ok := detectors[name]; ok { + + namespaces := []database.LayerNamespace{} + for _, d := range toUse { + // Only use the detector with the same type + if d.DType != database.NamespaceDetectorType { + continue + } + + if detector, ok := detectors[d.Name]; ok { namespace, err := detector.Detect(files) if err != nil { - log.WithError(err).WithField("name", name).Warning("failed while attempting to detect namespace") + log.WithError(err).WithField("detector", d).Warning("failed while attempting to detect namespace") return nil, err } if namespace != nil { - log.WithFields(log.Fields{"name": name, "namespace": namespace.Name}).Debug("detected namespace") - namespaces[namespace.Name] = namespace + log.WithFields(log.Fields{"detector": d, "namespace": namespace.Name}).Debug("detected namespace") + namespaces = append(namespaces, database.LayerNamespace{ + Namespace: *namespace, + By: detector.info, + }) } } else { - log.WithField("Name", name).Warn("Unknown namespace detector") + log.WithField("detector", d).Fatal("unknown namespace detector") } } - nslist := []database.Namespace{} - for _, ns := range namespaces { - nslist = append(nslist, *ns) - } - return nslist, nil + return namespaces, nil } -// RequiredFilenames returns the total list of files required for all -// registered Detectors. -func RequiredFilenames(detectorNames []string) (files []string) { +// RequiredFilenames returns all files required by the give extensions. Any +// extension metadata that has non namespace-detector type will be skipped. +func RequiredFilenames(toUse []database.Detector) (files []string) { detectorsM.RLock() defer detectorsM.RUnlock() - for _, detector := range detectors { - files = append(files, detector.RequiredFilenames()...) + for _, d := range toUse { + if d.DType != database.NamespaceDetectorType { + continue + } + + files = append(files, detectors[d.Name].RequiredFilenames()...) } return } -// ListDetectors returns the names of all registered namespace detectors. -func ListDetectors() []string { - r := []string{} - for name := range detectors { - r = append(r, name) +// ListDetectors returns the info of all registered namespace detectors. +func ListDetectors() []database.Detector { + r := make([]database.Detector, 0, len(detectors)) + for _, d := range detectors { + r = append(r, d.info) } return r } diff --git a/ext/featurens/driver_test.go b/ext/featurens/driver_test.go index 0d46b0b6..1218a852 100644 --- a/ext/featurens/driver_test.go +++ b/ext/featurens/driver_test.go @@ -8,6 +8,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/pkg/tarutil" + "github.com/coreos/clair/pkg/testutil" _ "github.com/coreos/clair/ext/featurens/alpinerelease" _ "github.com/coreos/clair/ext/featurens/aptsources" @@ -16,40 +17,14 @@ import ( _ "github.com/coreos/clair/ext/featurens/redhatrelease" ) -type MultipleNamespaceTestData struct { - Files tarutil.FilesMap - ExpectedNamespaces []database.Namespace -} - -func assertnsNameEqual(t *testing.T, nslist_expected, nslist []database.Namespace) { - assert.Equal(t, len(nslist_expected), len(nslist)) - expected := map[string]struct{}{} - input := map[string]struct{}{} - // compare the two sets - for i := range nslist_expected { - expected[nslist_expected[i].Name] = struct{}{} - input[nslist[i].Name] = struct{}{} - } - assert.Equal(t, expected, input) -} - -func testMultipleNamespace(t *testing.T, testData []MultipleNamespaceTestData) { - for _, td := range testData { - nslist, err := featurens.Detect(td.Files, featurens.ListDetectors()) - assert.Nil(t, err) - assertnsNameEqual(t, td.ExpectedNamespaces, nslist) - } -} - -func TestMultipleNamespaceDetector(t *testing.T) { - testData := []MultipleNamespaceTestData{ - { - ExpectedNamespaces: []database.Namespace{ - {Name: "debian:8", VersionFormat: "dpkg"}, - {Name: "alpine:v3.3", VersionFormat: "dpkg"}, - }, - Files: tarutil.FilesMap{ - "etc/os-release": []byte(` +var namespaceDetectorTests = []struct { + in tarutil.FilesMap + out []database.LayerNamespace + err string +}{ + { + in: tarutil.FilesMap{ + "etc/os-release": []byte(` PRETTY_NAME="Debian GNU/Linux 8 (jessie)" NAME="Debian GNU/Linux" VERSION_ID="8" @@ -58,9 +33,23 @@ ID=debian HOME_URL="http://www.debian.org/" SUPPORT_URL="http://www.debian.org/support/" BUG_REPORT_URL="https://bugs.debian.org/"`), - "etc/alpine-release": []byte(`3.3.4`), - }, + "etc/alpine-release": []byte(`3.3.4`), + }, + out: []database.LayerNamespace{ + {database.Namespace{"debian:8", "dpkg"}, database.NewNamespaceDetector("os-release", "1.0")}, + {database.Namespace{"alpine:v3.3", "dpkg"}, database.NewNamespaceDetector("alpine-release", "1.0")}, }, + }, +} + +func TestNamespaceDetector(t *testing.T) { + for _, test := range namespaceDetectorTests { + out, err := featurens.Detect(test.in, featurens.ListDetectors()) + if test.err != "" { + assert.EqualError(t, err, test.err) + return + } + + testutil.AssertLayerNamespacesEqual(t, test.out, out) } - testMultipleNamespace(t, testData) } diff --git a/ext/featurens/lsbrelease/lsbrelease.go b/ext/featurens/lsbrelease/lsbrelease.go index d883215f..de528327 100644 --- a/ext/featurens/lsbrelease/lsbrelease.go +++ b/ext/featurens/lsbrelease/lsbrelease.go @@ -38,7 +38,7 @@ var ( type detector struct{} func init() { - featurens.RegisterDetector("lsb-release", &detector{}) + featurens.RegisterDetector("lsb-release", "1.0", &detector{}) } func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { diff --git a/ext/featurens/osrelease/osrelease.go b/ext/featurens/osrelease/osrelease.go index 1139739d..8e736743 100644 --- a/ext/featurens/osrelease/osrelease.go +++ b/ext/featurens/osrelease/osrelease.go @@ -45,7 +45,7 @@ var ( type detector struct{} func init() { - featurens.RegisterDetector("os-release", &detector{}) + featurens.RegisterDetector("os-release", "1.0", &detector{}) } func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { diff --git a/ext/featurens/redhatrelease/redhatrelease.go b/ext/featurens/redhatrelease/redhatrelease.go index 0dabc3fa..a8d27081 100644 --- a/ext/featurens/redhatrelease/redhatrelease.go +++ b/ext/featurens/redhatrelease/redhatrelease.go @@ -38,7 +38,7 @@ var ( type detector struct{} func init() { - featurens.RegisterDetector("redhat-release", &detector{}) + featurens.RegisterDetector("redhat-release", "1.0", &detector{}) } func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { diff --git a/ext/imagefmt/driver.go b/ext/imagefmt/driver.go index 46724401..9838efec 100644 --- a/ext/imagefmt/driver.go +++ b/ext/imagefmt/driver.go @@ -33,6 +33,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/strutil" "github.com/coreos/clair/pkg/tarutil" ) @@ -106,7 +107,7 @@ func UnregisterExtractor(name string) { func Extract(format, path string, headers map[string]string, toExtract []string) (tarutil.FilesMap, error) { var layerReader io.ReadCloser if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { - // Create a new HTTP request object. + log.WithField("path", strutil.CleanURL(path)).Debug("start downloading layer blob...") request, err := http.NewRequest("GET", path, nil) if err != nil { return nil, ErrCouldNotFindLayer @@ -127,21 +128,23 @@ func Extract(format, path string, headers map[string]string, toExtract []string) client := &http.Client{Transport: tr} r, err := client.Do(request) if err != nil { - log.WithError(err).Warning("could not download layer") + log.WithError(err).Error("could not download layer") return nil, ErrCouldNotFindLayer } // Fail if we don't receive a 2xx HTTP status code. if math.Floor(float64(r.StatusCode/100)) != 2 { - log.WithField("status code", r.StatusCode).Warning("could not download layer: expected 2XX") + log.WithError(ErrCouldNotFindLayer).WithField("status code", r.StatusCode).Error("could not download layer: expected 2XX") return nil, ErrCouldNotFindLayer } layerReader = r.Body } else { + log.WithField("path", strutil.CleanURL(path)).Debug("start reading layer blob from local file system...") var err error layerReader, err = os.Open(path) if err != nil { + log.WithError(ErrCouldNotFindLayer).Error("could not open layer") return nil, ErrCouldNotFindLayer } } From 9c49d9dc5591d62a86632881af8d7a7f15fbf25e Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Wed, 19 Sep 2018 15:38:07 -0400 Subject: [PATCH 08/14] pgsql: Move queries to corresponding files Aggregate queries in their corresponding files instead of having the single file for every queries because the database is more complicated. --- database/pgsql/ancestry.go | 32 +++++ database/pgsql/feature.go | 38 ++++++ database/pgsql/keyvalue.go | 9 ++ database/pgsql/layer.go | 34 +++-- database/pgsql/lock.go | 8 ++ database/pgsql/namespace.go | 4 + database/pgsql/notification.go | 43 ++++++ database/pgsql/queries.go | 229 -------------------------------- database/pgsql/vulnerability.go | 67 ++++++++++ 9 files changed, 226 insertions(+), 238 deletions(-) diff --git a/database/pgsql/ancestry.go b/database/pgsql/ancestry.go index 495d299f..36d8fcde 100644 --- a/database/pgsql/ancestry.go +++ b/database/pgsql/ancestry.go @@ -10,6 +10,38 @@ import ( "github.com/coreos/clair/pkg/commonerr" ) +const ( + insertAncestry = ` + INSERT INTO ancestry (name) VALUES ($1) RETURNING id` + + searchAncestryLayer = ` + SELECT layer.hash, layer.id, ancestry_layer.ancestry_index + FROM layer, ancestry_layer + WHERE ancestry_layer.ancestry_id = $1 + AND ancestry_layer.layer_id = layer.id + ORDER BY ancestry_layer.ancestry_index ASC` + + searchAncestryFeatures = ` + SELECT namespace.name, namespace.version_format, feature.name, feature.version, feature.version_format, ancestry_layer.ancestry_index + FROM namespace, feature, namespaced_feature, ancestry_layer, ancestry_feature + WHERE ancestry_layer.ancestry_id = $1 + AND ancestry_feature.ancestry_layer_id = ancestry_layer.id + AND ancestry_feature.namespaced_feature_id = namespaced_feature.id + AND namespaced_feature.feature_id = feature.id + AND namespaced_feature.namespace_id = namespace.id` + + searchAncestry = `SELECT id FROM ancestry WHERE name = $1` + removeAncestry = `DELETE FROM ancestry WHERE name = $1` + insertAncestryLayer = ` + INSERT INTO ancestry_layer (ancestry_id, ancestry_index, layer_id) VALUES + ($1, $2, (SELECT layer.id FROM layer WHERE hash = $3 LIMIT 1)) + RETURNING id` + insertAncestryLayerFeature = ` + INSERT INTO ancestry_feature + (ancestry_layer_id, namespaced_feature_id, feature_detector_id, namespace_detector_id) VALUES + ($1, $2, $3, $4)` +) + type ancestryLayerWithID struct { database.AncestryLayer diff --git a/database/pgsql/feature.go b/database/pgsql/feature.go index 81ef857d..e1c0781c 100644 --- a/database/pgsql/feature.go +++ b/database/pgsql/feature.go @@ -27,6 +27,44 @@ import ( "github.com/coreos/clair/pkg/commonerr" ) +const ( + // feature.go + soiNamespacedFeature = ` + WITH new_feature_ns AS ( + INSERT INTO namespaced_feature(feature_id, namespace_id) + SELECT CAST ($1 AS INTEGER), CAST ($2 AS INTEGER) + WHERE NOT EXISTS ( SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2) + RETURNING id + ) + SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2 + UNION + SELECT id FROM new_feature_ns` + + searchPotentialAffectingVulneraibilities = ` + SELECT nf.id, v.id, vaf.affected_version, vaf.id + FROM vulnerability_affected_feature AS vaf, vulnerability AS v, + namespaced_feature AS nf, feature AS f + WHERE nf.id = ANY($1) + AND nf.feature_id = f.id + AND nf.namespace_id = v.namespace_id + AND vaf.feature_name = f.name + AND vaf.vulnerability_id = v.id + AND v.deleted_at IS NULL` + + searchNamespacedFeaturesVulnerabilities = ` + SELECT vanf.namespaced_feature_id, v.name, v.description, v.link, + v.severity, v.metadata, vaf.fixedin, n.name, n.version_format + FROM vulnerability_affected_namespaced_feature AS vanf, + Vulnerability AS v, + vulnerability_affected_feature AS vaf, + namespace AS n + WHERE vanf.namespaced_feature_id = ANY($1) + AND vaf.id = vanf.added_by + AND v.id = vanf.vulnerability_id + AND n.id = v.namespace_id + AND v.deleted_at IS NULL` +) + var ( errFeatureNotFound = errors.New("Feature not found") ) diff --git a/database/pgsql/keyvalue.go b/database/pgsql/keyvalue.go index 1f85fab5..9c985279 100644 --- a/database/pgsql/keyvalue.go +++ b/database/pgsql/keyvalue.go @@ -23,6 +23,15 @@ import ( "github.com/coreos/clair/pkg/commonerr" ) +const ( + searchKeyValue = `SELECT value FROM KeyValue WHERE key = $1` + upsertKeyValue = ` + INSERT INTO KeyValue(key, value) + VALUES ($1, $2) + ON CONFLICT ON CONSTRAINT keyvalue_key_key + DO UPDATE SET key=$1, value=$2` +) + func (tx *pgSession) UpdateKeyValue(key, value string) (err error) { if key == "" || value == "" { log.Warning("could not insert a flag which has an empty name or value") diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go index 8a35a144..e474164e 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer.go @@ -22,15 +22,31 @@ import ( "github.com/coreos/clair/pkg/commonerr" ) -func (tx *pgSession) FindLayer(hash string) (database.Layer, bool, error) { - var ( - layer database.Layer - layerID int64 - ok bool - err error - ) - - layer.LayerMetadata, layerID, ok, err = tx.findLayer(hash) +const ( + soiLayer = ` + WITH new_layer AS ( + INSERT INTO layer (hash) + SELECT CAST ($1 AS VARCHAR) + WHERE NOT EXISTS (SELECT id FROM layer WHERE hash = $1) + RETURNING id + ) + SELECT id FROM new_Layer + UNION + SELECT id FROM layer WHERE hash = $1` + + searchLayerFeatures = ` + SELECT feature_id, detector_id + FROM layer_feature + WHERE layer_id = $1` + + searchLayerNamespaces = ` + SELECT namespace.Name, namespace.version_format + FROM namespace, layer_namespace + WHERE layer_namespace.layer_id = $1 + AND layer_namespace.namespace_id = namespace.id` + + searchLayer = `SELECT id FROM layer WHERE hash = $1` +) if err != nil { return layer, false, err } diff --git a/database/pgsql/lock.go b/database/pgsql/lock.go index c8918ebc..b7e859e6 100644 --- a/database/pgsql/lock.go +++ b/database/pgsql/lock.go @@ -23,6 +23,14 @@ import ( "github.com/coreos/clair/pkg/commonerr" ) +const ( + soiLock = `INSERT INTO lock(name, owner, until) VALUES ($1, $2, $3)` + searchLock = `SELECT owner, until FROM Lock WHERE name = $1` + updateLock = `UPDATE Lock SET until = $3 WHERE name = $1 AND owner = $2` + removeLock = `DELETE FROM Lock WHERE name = $1 AND owner = $2` + removeLockExpired = `DELETE FROM LOCK WHERE until < CURRENT_TIMESTAMP` +) + var ( errLockNotFound = errors.New("lock is not in database") ) diff --git a/database/pgsql/namespace.go b/database/pgsql/namespace.go index 1a78f837..bd0dae34 100644 --- a/database/pgsql/namespace.go +++ b/database/pgsql/namespace.go @@ -23,6 +23,10 @@ import ( "github.com/coreos/clair/pkg/commonerr" ) +const ( + searchNamespaceID = `SELECT id FROM Namespace WHERE name = $1 AND version_format = $2` +) + var ( errNamespaceNotFound = errors.New("Requested Namespace is not in database") ) diff --git a/database/pgsql/notification.go b/database/pgsql/notification.go index 4ddf042f..44eff64b 100644 --- a/database/pgsql/notification.go +++ b/database/pgsql/notification.go @@ -26,6 +26,49 @@ import ( "github.com/coreos/clair/pkg/pagination" ) +const ( + // notification.go + insertNotification = ` + INSERT INTO Vulnerability_Notification(name, created_at, old_vulnerability_id, new_vulnerability_id) + VALUES ($1, $2, $3, $4)` + + updatedNotificationAsRead = ` + UPDATE Vulnerability_Notification + SET notified_at = CURRENT_TIMESTAMP + WHERE name = $1` + + removeNotification = ` + UPDATE Vulnerability_Notification + SET deleted_at = CURRENT_TIMESTAMP + WHERE name = $1 AND deleted_at IS NULL` + + searchNotificationAvailable = ` + SELECT name, created_at, notified_at, deleted_at + FROM Vulnerability_Notification + WHERE (notified_at IS NULL OR notified_at < $1) + AND deleted_at IS NULL + AND name NOT IN (SELECT name FROM Lock) + ORDER BY Random() + LIMIT 1` + + searchNotification = ` + SELECT created_at, notified_at, deleted_at, old_vulnerability_id, new_vulnerability_id + FROM Vulnerability_Notification + WHERE name = $1` + + searchNotificationVulnerableAncestry = ` + SELECT DISTINCT ON (a.id) + a.id, a.name + FROM vulnerability_affected_namespaced_feature AS vanf, + ancestry_layer AS al, ancestry_feature AS af + WHERE vanf.vulnerability_id = $1 + AND al.ancestry_id >= $2 + AND al.id = af.ancestry_layer_id + AND af.namespaced_feature_id = vanf.namespaced_feature_id + ORDER BY a.id ASC + LIMIT $3;` +) + var ( errNotificationNotFound = errors.New("requested notification is not found") ) diff --git a/database/pgsql/queries.go b/database/pgsql/queries.go index fa137620..ad7cfc44 100644 --- a/database/pgsql/queries.go +++ b/database/pgsql/queries.go @@ -21,235 +21,6 @@ import ( "github.com/lib/pq" ) -const ( - lockVulnerabilityAffects = `LOCK vulnerability_affected_namespaced_feature IN SHARE ROW EXCLUSIVE MODE` - - // keyvalue.go - searchKeyValue = `SELECT value FROM KeyValue WHERE key = $1` - upsertKeyValue = ` - INSERT INTO KeyValue(key, value) - VALUES ($1, $2) - ON CONFLICT ON CONSTRAINT keyvalue_key_key - DO UPDATE SET key=$1, value=$2` - - // namespace.go - - searchNamespaceID = `SELECT id FROM Namespace WHERE name = $1 AND version_format = $2` - - // feature.go - soiNamespacedFeature = ` - WITH new_feature_ns AS ( - INSERT INTO namespaced_feature(feature_id, namespace_id) - SELECT CAST ($1 AS INTEGER), CAST ($2 AS INTEGER) - WHERE NOT EXISTS ( SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2) - RETURNING id - ) - SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2 - UNION - SELECT id FROM new_feature_ns` - - searchPotentialAffectingVulneraibilities = ` - SELECT nf.id, v.id, vaf.affected_version, vaf.id - FROM vulnerability_affected_feature AS vaf, vulnerability AS v, - namespaced_feature AS nf, feature AS f - WHERE nf.id = ANY($1) - AND nf.feature_id = f.id - AND nf.namespace_id = v.namespace_id - AND vaf.feature_name = f.name - AND vaf.vulnerability_id = v.id - AND v.deleted_at IS NULL` - - searchNamespacedFeaturesVulnerabilities = ` - SELECT vanf.namespaced_feature_id, v.name, v.description, v.link, - v.severity, v.metadata, vaf.fixedin, n.name, n.version_format - FROM vulnerability_affected_namespaced_feature AS vanf, - Vulnerability AS v, - vulnerability_affected_feature AS vaf, - namespace AS n - WHERE vanf.namespaced_feature_id = ANY($1) - AND vaf.id = vanf.added_by - AND v.id = vanf.vulnerability_id - AND n.id = v.namespace_id - AND v.deleted_at IS NULL` - - // layer.go - soiLayer = ` - WITH new_layer AS ( - INSERT INTO layer (hash) - SELECT CAST ($1 AS VARCHAR) - WHERE NOT EXISTS (SELECT id FROM layer WHERE hash = $1) - RETURNING id - ) - SELECT id FROM new_Layer - UNION - SELECT id FROM layer WHERE hash = $1` - - searchLayerFeatures = ` - SELECT feature.Name, feature.Version, feature.version_format - FROM feature, layer_feature - WHERE layer_feature.layer_id = $1 - AND layer_feature.feature_id = feature.id` - - searchLayerNamespaces = ` - SELECT namespace.Name, namespace.version_format - FROM namespace, layer_namespace - WHERE layer_namespace.layer_id = $1 - AND layer_namespace.namespace_id = namespace.id` - - searchLayer = `SELECT id FROM layer WHERE hash = $1` - searchLayerDetectors = `SELECT detector FROM layer_detector WHERE layer_id = $1` - searchLayerListers = `SELECT lister FROM layer_lister WHERE layer_id = $1` - - // lock.go - soiLock = `INSERT INTO lock(name, owner, until) VALUES ($1, $2, $3)` - - searchLock = `SELECT owner, until FROM Lock WHERE name = $1` - updateLock = `UPDATE Lock SET until = $3 WHERE name = $1 AND owner = $2` - removeLock = `DELETE FROM Lock WHERE name = $1 AND owner = $2` - removeLockExpired = `DELETE FROM LOCK WHERE until < CURRENT_TIMESTAMP` - - // vulnerability.go - searchVulnerability = ` - SELECT v.id, v.description, v.link, v.severity, v.metadata, n.version_format - FROM vulnerability AS v, namespace AS n - WHERE v.namespace_id = n.id - AND v.name = $1 - AND n.name = $2 - AND v.deleted_at IS NULL - ` - - insertVulnerabilityAffected = ` - INSERT INTO vulnerability_affected_feature(vulnerability_id, feature_name, affected_version, fixedin) - VALUES ($1, $2, $3, $4) - RETURNING ID - ` - - searchVulnerabilityAffected = ` - SELECT vulnerability_id, feature_name, affected_version, fixedin - FROM vulnerability_affected_feature - WHERE vulnerability_id = ANY($1) - ` - - searchVulnerabilityByID = ` - SELECT v.name, v.description, v.link, v.severity, v.metadata, n.name, n.version_format - FROM vulnerability AS v, namespace AS n - WHERE v.namespace_id = n.id - AND v.id = $1` - - searchVulnerabilityPotentialAffected = ` - WITH req AS ( - SELECT vaf.id AS vaf_id, n.id AS n_id, vaf.feature_name AS name, v.id AS vulnerability_id - FROM vulnerability_affected_feature AS vaf, - vulnerability AS v, - namespace AS n - WHERE vaf.vulnerability_id = ANY($1) - AND v.id = vaf.vulnerability_id - AND n.id = v.namespace_id - ) - SELECT req.vulnerability_id, nf.id, f.version, req.vaf_id AS added_by - FROM feature AS f, namespaced_feature AS nf, req - WHERE f.name = req.name - AND nf.namespace_id = req.n_id - AND nf.feature_id = f.id` - - insertVulnerabilityAffectedNamespacedFeature = ` - INSERT INTO vulnerability_affected_namespaced_feature(vulnerability_id, namespaced_feature_id, added_by) - VALUES ($1, $2, $3)` - - insertVulnerability = ` - WITH ns AS ( - SELECT id FROM namespace WHERE name = $6 AND version_format = $7 - ) - INSERT INTO Vulnerability(namespace_id, name, description, link, severity, metadata, created_at) - VALUES((SELECT id FROM ns), $1, $2, $3, $4, $5, CURRENT_TIMESTAMP) - RETURNING id` - - removeVulnerability = ` - UPDATE Vulnerability - SET deleted_at = CURRENT_TIMESTAMP - WHERE namespace_id = (SELECT id FROM Namespace WHERE name = $1) - AND name = $2 - AND deleted_at IS NULL - RETURNING id` - - // notification.go - insertNotification = ` - INSERT INTO Vulnerability_Notification(name, created_at, old_vulnerability_id, new_vulnerability_id) - VALUES ($1, $2, $3, $4)` - - updatedNotificationAsRead = ` - UPDATE Vulnerability_Notification - SET notified_at = CURRENT_TIMESTAMP - WHERE name = $1` - - removeNotification = ` - UPDATE Vulnerability_Notification - SET deleted_at = CURRENT_TIMESTAMP - WHERE name = $1 AND deleted_at IS NULL` - - searchNotificationAvailable = ` - SELECT name, created_at, notified_at, deleted_at - FROM Vulnerability_Notification - WHERE (notified_at IS NULL OR notified_at < $1) - AND deleted_at IS NULL - AND name NOT IN (SELECT name FROM Lock) - ORDER BY Random() - LIMIT 1` - - searchNotification = ` - SELECT created_at, notified_at, deleted_at, old_vulnerability_id, new_vulnerability_id - FROM Vulnerability_Notification - WHERE name = $1` - - searchNotificationVulnerableAncestry = ` - SELECT DISTINCT ON (a.id) - a.id, a.name - FROM vulnerability_affected_namespaced_feature AS vanf, - ancestry_layer AS al, ancestry_feature AS af - WHERE vanf.vulnerability_id = $1 - AND al.ancestry_id >= $2 - AND al.id = af.ancestry_layer_id - AND af.namespaced_feature_id = vanf.namespaced_feature_id - ORDER BY a.id ASC - LIMIT $3;` - - // ancestry.go - persistAncestryLister = ` - INSERT INTO ancestry_lister (ancestry_id, lister) - SELECT CAST ($1 AS INTEGER), CAST ($2 AS TEXT) - WHERE NOT EXISTS (SELECT id FROM ancestry_lister WHERE ancestry_id = $1 AND lister = $2) ON CONFLICT DO NOTHING` - - persistAncestryDetector = ` - INSERT INTO ancestry_detector (ancestry_id, detector) - SELECT CAST ($1 AS INTEGER), CAST ($2 AS TEXT) - WHERE NOT EXISTS (SELECT id FROM ancestry_detector WHERE ancestry_id = $1 AND detector = $2) ON CONFLICT DO NOTHING` - - insertAncestry = `INSERT INTO ancestry (name) VALUES ($1) RETURNING id` - - searchAncestryLayer = ` - SELECT layer.hash, layer.id, ancestry_layer.ancestry_index - FROM layer, ancestry_layer - WHERE ancestry_layer.ancestry_id = $1 - AND ancestry_layer.layer_id = layer.id - ORDER BY ancestry_layer.ancestry_index ASC` - - searchAncestryFeatures = ` - SELECT namespace.name, namespace.version_format, feature.name, feature.version, feature.version_format, ancestry_layer.ancestry_index - FROM namespace, feature, namespaced_feature, ancestry_layer, ancestry_feature - WHERE ancestry_layer.ancestry_id = $1 - AND ancestry_feature.ancestry_layer_id = ancestry_layer.id - AND ancestry_feature.namespaced_feature_id = namespaced_feature.id - AND namespaced_feature.feature_id = feature.id - AND namespaced_feature.namespace_id = namespace.id` - - searchAncestry = `SELECT id FROM ancestry WHERE name = $1` - searchAncestryDetectors = `SELECT detector FROM ancestry_detector WHERE ancestry_id = $1` - searchAncestryListers = `SELECT lister FROM ancestry_lister WHERE ancestry_id = $1` - removeAncestry = `DELETE FROM ancestry WHERE name = $1` - insertAncestryLayer = `INSERT INTO ancestry_layer(ancestry_id, ancestry_index, layer_id) VALUES($1,$2, (SELECT layer.id FROM layer WHERE hash = $3 LIMIT 1)) RETURNING id` - insertAncestryLayerFeature = `INSERT INTO ancestry_feature(ancestry_layer_id, namespaced_feature_id) VALUES ($1, $2)` -) - // NOTE(Sida): Every search query can only have count less than postgres set // stack depth. IN will be resolved to nested OR_s and the parser might exceed // stack depth. TODO(Sida): Generate different queries for different count: if diff --git a/database/pgsql/vulnerability.go b/database/pgsql/vulnerability.go index fb483cfe..93518a87 100644 --- a/database/pgsql/vulnerability.go +++ b/database/pgsql/vulnerability.go @@ -26,6 +26,73 @@ import ( "github.com/coreos/clair/ext/versionfmt" ) +const ( + lockVulnerabilityAffects = `LOCK vulnerability_affected_namespaced_feature IN SHARE ROW EXCLUSIVE MODE` + + searchVulnerability = ` + SELECT v.id, v.description, v.link, v.severity, v.metadata, n.version_format + FROM vulnerability AS v, namespace AS n + WHERE v.namespace_id = n.id + AND v.name = $1 + AND n.name = $2 + AND v.deleted_at IS NULL + ` + + insertVulnerabilityAffected = ` + INSERT INTO vulnerability_affected_feature(vulnerability_id, feature_name, affected_version, fixedin) + VALUES ($1, $2, $3, $4) + RETURNING ID + ` + + searchVulnerabilityAffected = ` + SELECT vulnerability_id, feature_name, affected_version, fixedin + FROM vulnerability_affected_feature + WHERE vulnerability_id = ANY($1) + ` + + searchVulnerabilityByID = ` + SELECT v.name, v.description, v.link, v.severity, v.metadata, n.name, n.version_format + FROM vulnerability AS v, namespace AS n + WHERE v.namespace_id = n.id + AND v.id = $1` + + searchVulnerabilityPotentialAffected = ` + WITH req AS ( + SELECT vaf.id AS vaf_id, n.id AS n_id, vaf.feature_name AS name, v.id AS vulnerability_id + FROM vulnerability_affected_feature AS vaf, + vulnerability AS v, + namespace AS n + WHERE vaf.vulnerability_id = ANY($1) + AND v.id = vaf.vulnerability_id + AND n.id = v.namespace_id + ) + SELECT req.vulnerability_id, nf.id, f.version, req.vaf_id AS added_by + FROM feature AS f, namespaced_feature AS nf, req + WHERE f.name = req.name + AND nf.namespace_id = req.n_id + AND nf.feature_id = f.id` + + insertVulnerabilityAffectedNamespacedFeature = ` + INSERT INTO vulnerability_affected_namespaced_feature(vulnerability_id, namespaced_feature_id, added_by) + VALUES ($1, $2, $3)` + + insertVulnerability = ` + WITH ns AS ( + SELECT id FROM namespace WHERE name = $6 AND version_format = $7 + ) + INSERT INTO Vulnerability(namespace_id, name, description, link, severity, metadata, created_at) + VALUES((SELECT id FROM ns), $1, $2, $3, $4, $5, CURRENT_TIMESTAMP) + RETURNING id` + + removeVulnerability = ` + UPDATE Vulnerability + SET deleted_at = CURRENT_TIMESTAMP + WHERE namespace_id = (SELECT id FROM Namespace WHERE name = $1) + AND name = $2 + AND deleted_at IS NULL + RETURNING id` +) + var ( errVulnerabilityNotFound = errors.New("vulnerability is not in database") ) From 48427e9b8808f86929ffb905952395c91644f04e Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Thu, 20 Sep 2018 15:39:10 -0400 Subject: [PATCH 09/14] api: Add detectors for RPC Change the V3 implementation to accommondate the detectors. --- api/v3/clairpb/convert.go | 40 ++++++++++++++++++++++++++++-------- api/v3/rpc.go | 5 ++--- api/v3/util.go | 43 +++++++++++++++++++++------------------ 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/api/v3/clairpb/convert.go b/api/v3/clairpb/convert.go index 8414ed92..da4e9e33 100644 --- a/api/v3/clairpb/convert.go +++ b/api/v3/clairpb/convert.go @@ -22,6 +22,13 @@ import ( "github.com/coreos/clair/ext/versionfmt" ) +// DatabaseDetectorTypeMapping maps the database detector type to the integer +// enum proto. +var DatabaseDetectorTypeMapping = map[database.DetectorType]Detector_Type{ + database.NamespaceDetectorType: Detector_Type(1), + database.FeatureDetectorType: Detector_Type(2), +} + // PagedVulnerableAncestriesFromDatabaseModel converts database // PagedVulnerableAncestries to api PagedVulnerableAncestries and assigns // indexes to ancestries. @@ -122,23 +129,38 @@ func VulnerabilityWithFixedInFromDatabaseModel(dbVuln database.VulnerabilityWith return vuln, nil } -// LayerFromDatabaseModel converts database layer to api layer. -func LayerFromDatabaseModel(dbLayer database.LayerMetadata) *Layer { - layer := Layer{Hash: dbLayer.Hash} - return &layer -} - // NamespacedFeatureFromDatabaseModel converts database namespacedFeature to api Feature. -func NamespacedFeatureFromDatabaseModel(feature database.NamespacedFeature) *Feature { +func NamespacedFeatureFromDatabaseModel(feature database.AncestryFeature) *Feature { version := feature.Feature.Version if version == versionfmt.MaxVersion { version = "None" } return &Feature{ - Name: feature.Feature.Name, - NamespaceName: feature.Namespace.Name, + Name: feature.Feature.Name, + Namespace: &Namespace{ + Name: feature.Namespace.Name, + Detector: DetectorFromDatabaseModel(feature.NamespaceBy), + }, VersionFormat: feature.Namespace.VersionFormat, Version: version, + Detector: DetectorFromDatabaseModel(feature.FeatureBy), } } + +func DetectorFromDatabaseModel(detector database.Detector) *Detector { + return &Detector{ + Name: detector.Name, + Version: detector.Version, + Type: DatabaseDetectorTypeMapping[detector.DType], + } +} + +func DetectorsFromDatabaseModel(dbDetectors []database.Detector) []*Detector { + detectors := make([]*Detector, 0, len(dbDetectors)) + for _, d := range dbDetectors { + detectors = append(detectors, DetectorFromDatabaseModel(d)) + } + + return detectors +} diff --git a/api/v3/rpc.go b/api/v3/rpc.go index 830bde9b..51502c2c 100644 --- a/api/v3/rpc.go +++ b/api/v3/rpc.go @@ -129,9 +129,8 @@ func (s *AncestryServer) GetAncestry(ctx context.Context, req *pb.GetAncestryReq } pbAncestry := &pb.GetAncestryResponse_Ancestry{ - Name: ancestry.Name, - ScannedDetectors: ancestry.ProcessedBy.Detectors, - ScannedListers: ancestry.ProcessedBy.Listers, + Name: ancestry.Name, + Detectors: pb.DetectorsFromDatabaseModel(ancestry.By), } for _, layer := range ancestry.Layers { diff --git a/api/v3/util.go b/api/v3/util.go index d70390d4..392a147e 100644 --- a/api/v3/util.go +++ b/api/v3/util.go @@ -13,8 +13,7 @@ import ( // protobuf struct. func GetClairStatus(store database.Datastore) (*pb.ClairStatus, error) { status := &pb.ClairStatus{ - Listers: clair.Processors.Listers, - Detectors: clair.Processors.Detectors, + Detectors: pb.DetectorsFromDatabaseModel(clair.EnabledDetectors), } t, firstUpdate, err := clair.GetLastUpdateTime(store) @@ -34,19 +33,16 @@ func GetClairStatus(store database.Datastore) (*pb.ClairStatus, error) { // GetPbAncestryLayer retrieves an ancestry layer with vulnerabilities and // features in an ancestry based on the provided database layer. -func GetPbAncestryLayer(session database.Session, layer database.AncestryLayer) (*pb.GetAncestryResponse_AncestryLayer, error) { +func GetPbAncestryLayer(tx database.Session, layer database.AncestryLayer) (*pb.GetAncestryResponse_AncestryLayer, error) { pbLayer := &pb.GetAncestryResponse_AncestryLayer{ Layer: &pb.Layer{ Hash: layer.Hash, }, } - var ( - features []database.NullableAffectedNamespacedFeature - err error - ) - - if features, err = session.FindAffectedNamespacedFeatures(layer.DetectedFeatures); err != nil { + features := layer.GetFeatures() + affectedFeatures, err := tx.FindAffectedNamespacedFeatures(features) + if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -59,20 +55,27 @@ func GetPbAncestryLayer(session database.Session, layer database.AncestryLayer) return nil, status.Error(codes.Internal, "ancestry feature is not found") } - var ( - pbFeature = pb.NamespacedFeatureFromDatabaseModel(feature.NamespacedFeature) - pbVuln *pb.Vulnerability - err error - ) - for _, vuln := range feature.AffectedBy { - if pbVuln, err = pb.VulnerabilityWithFixedInFromDatabaseModel(vuln); err != nil { - return nil, status.Error(codes.Internal, err.Error()) + for _, detectedFeature := range layer.Features { + if detectedFeature.NamespacedFeature != feature.NamespacedFeature { + continue } - pbFeature.Vulnerabilities = append(pbFeature.Vulnerabilities, pbVuln) - } + var ( + pbFeature = pb.NamespacedFeatureFromDatabaseModel(detectedFeature) + pbVuln *pb.Vulnerability + err error + ) + + for _, vuln := range feature.AffectedBy { + if pbVuln, err = pb.VulnerabilityWithFixedInFromDatabaseModel(vuln); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } - pbLayer.DetectedFeatures = append(pbLayer.DetectedFeatures, pbFeature) + pbFeature.Vulnerabilities = append(pbFeature.Vulnerabilities, pbVuln) + } + + pbLayer.DetectedFeatures = append(pbLayer.DetectedFeatures, pbFeature) + } } return pbLayer, nil From 028324014ba3b7111e4e4533d6a8d4d99bb1fd72 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Thu, 20 Sep 2018 15:57:06 -0400 Subject: [PATCH 10/14] clair: Implement worker detector support The worker is changed to accommodate the new database model and API. Worker is refactored to move the database query helper functions to pkg. --- cmd/clair/main.go | 44 ++- pkg/dbutil/dbutil.go | 288 ++++++++++++++++++ pkg/strutil/strutil.go | 60 ++-- pkg/strutil/strutil_test.go | 4 +- updater.go | 4 +- worker.go | 587 ++++++++++++++---------------------- worker_test.go | 461 +++++++++++++--------------- 7 files changed, 770 insertions(+), 678 deletions(-) create mode 100644 pkg/dbutil/dbutil.go diff --git a/cmd/clair/main.go b/cmd/clair/main.go index e329af34..e28a30d5 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -102,27 +102,13 @@ func stopCPUProfiling(f *os.File) { } func configClairVersion(config *Config) { - listers := featurefmt.ListListers() - detectors := featurens.ListDetectors() - updaters := vulnsrc.ListUpdaters() + clair.EnabledDetectors = append(featurefmt.ListListers(), featurens.ListDetectors()...) + clair.EnabledUpdaters = strutil.Intersect(config.Updater.EnabledUpdaters, vulnsrc.ListUpdaters()) log.WithFields(log.Fields{ - "Listers": strings.Join(listers, ","), - "Detectors": strings.Join(detectors, ","), - "Updaters": strings.Join(updaters, ","), - }).Info("Clair registered components") - - unregUpdaters := strutil.CompareStringLists(config.Updater.EnabledUpdaters, updaters) - if len(unregUpdaters) != 0 { - log.WithFields(log.Fields{ - "Unknown Updaters": strings.Join(unregUpdaters, ","), - "Available Updaters": strings.Join(vulnsrc.ListUpdaters(), ","), - }).Fatal("Unknown or unregistered components are configured") - } - - // All listers and detectors are enabled. - clair.Processors = database.Processors{Detectors: detectors, Listers: listers} - clair.EnabledUpdaters = strutil.CompareStringListsInBoth(config.Updater.EnabledUpdaters, updaters) + "Detectors": database.SerializeDetectors(clair.EnabledDetectors), + "Updaters": clair.EnabledUpdaters, + }).Info("enabled Clair extensions") } // Boot starts Clair instance with the provided config. @@ -147,6 +133,7 @@ func Boot(config *Config) { defer db.Close() + clair.InitWorker(db) // Start notifier st.Begin() go clair.RunNotifier(config.Notifier, db, st) @@ -167,6 +154,18 @@ func Boot(config *Config) { st.Stop() } +// Initialize logging system +func configureLogger(flagLogLevel *string) { + logLevel, err := log.ParseLevel(strings.ToUpper(*flagLogLevel)) + if err != nil { + log.WithError(err).Error("failed to set logger parser level") + } + + log.SetLevel(logLevel) + log.SetOutput(os.Stdout) + log.SetFormatter(&formatter.JSONExtendedFormatter{ShowLn: true}) +} + func main() { // Parse command-line arguments flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) @@ -176,6 +175,7 @@ func main() { flagInsecureTLS := flag.Bool("insecure-tls", false, "Disable TLS server's certificate chain and hostname verification when pulling layers.") flag.Parse() + configureLogger(flagLogLevel) // Check for dependencies. for _, bin := range BinaryDependencies { _, err := exec.LookPath(bin) @@ -184,12 +184,6 @@ func main() { } } - // Initialize logging system - logLevel, err := log.ParseLevel(strings.ToUpper(*flagLogLevel)) - log.SetLevel(logLevel) - log.SetOutput(os.Stdout) - log.SetFormatter(&formatter.JSONExtendedFormatter{ShowLn: true}) - config, err := LoadConfig(*flagConfigPath) if err != nil { log.WithError(err).Fatal("failed to load configuration") diff --git a/pkg/dbutil/dbutil.go b/pkg/dbutil/dbutil.go new file mode 100644 index 00000000..57334298 --- /dev/null +++ b/pkg/dbutil/dbutil.go @@ -0,0 +1,288 @@ +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dbutil + +import ( + "github.com/deckarep/golang-set" + + "github.com/coreos/clair/database" +) + +// DeduplicateNamespaces deduplicates a list of namespaces. +func DeduplicateNamespaces(namespaces ...database.Namespace) []database.Namespace { + nsSet := mapset.NewSet() + for _, ns := range namespaces { + nsSet.Add(ns) + } + + result := make([]database.Namespace, 0, nsSet.Cardinality()) + for ns := range nsSet.Iter() { + result = append(result, ns.(database.Namespace)) + } + + return result +} + +// DeduplicateFeatures deduplicates a list of list of features. +func DeduplicateFeatures(features ...database.Feature) []database.Feature { + fSet := mapset.NewSet() + for _, f := range features { + fSet.Add(f) + } + + result := make([]database.Feature, 0, fSet.Cardinality()) + for f := range fSet.Iter() { + result = append(result, f.(database.Feature)) + } + + return result +} + +// PersistPartialLayer wraps session PersistLayer function with begin and +// commit. +func PersistPartialLayer(datastore database.Datastore, layer *database.Layer) error { + tx, err := datastore.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := tx.PersistLayer(layer.Hash, layer.Features, layer.Namespaces, layer.By); err != nil { + return err + } + + return tx.Commit() +} + +// PersistFeatures wraps session PersistFeatures function with begin and commit. +func PersistFeatures(datastore database.Datastore, features []database.Feature) error { + tx, err := datastore.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := tx.PersistFeatures(features); err != nil { + return err + } + return tx.Commit() +} + +// PersistNamespaces wraps session PersistNamespaces function with begin and +// commit. +func PersistNamespaces(datastore database.Datastore, namespaces []database.Namespace) error { + tx, err := datastore.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := tx.PersistNamespaces(namespaces); err != nil { + return err + } + + return tx.Commit() +} + +// FindAncestry wraps session FindAncestry function with begin and rollback. +func FindAncestry(datastore database.Datastore, name string) (database.Ancestry, bool, error) { + tx, err := datastore.Begin() + defer tx.Rollback() + + if err != nil { + return database.Ancestry{}, false, err + } + + return tx.FindAncestry(name) +} + +// FindLayer wraps session FindLayer function with begin and rollback. +func FindLayer(datastore database.Datastore, hash string) (layer database.Layer, ok bool, err error) { + var tx database.Session + if tx, err = datastore.Begin(); err != nil { + return + } + + defer tx.Rollback() + layer, ok, err = tx.FindLayer(hash) + return +} + +// DeduplicateNamespacedFeatures returns a copy of all unique features in the +// input. +func DeduplicateNamespacedFeatures(features []database.NamespacedFeature) []database.NamespacedFeature { + nsSet := mapset.NewSet() + for _, ns := range features { + nsSet.Add(ns) + } + + result := make([]database.NamespacedFeature, 0, nsSet.Cardinality()) + for ns := range nsSet.Iter() { + result = append(result, ns.(database.NamespacedFeature)) + } + + return result +} + +// GetAncestryFeatures returns a list of unique namespaced features in the +// ancestry. +func GetAncestryFeatures(ancestry database.Ancestry) []database.NamespacedFeature { + features := []database.NamespacedFeature{} + for _, layer := range ancestry.Layers { + features = append(features, layer.GetFeatures()...) + } + + return DeduplicateNamespacedFeatures(features) +} + +// UpsertAncestry wraps session UpsertAncestry function with begin and commit. +func UpsertAncestry(datastore database.Datastore, ancestry database.Ancestry) error { + tx, err := datastore.Begin() + if err != nil { + return err + } + + if err = tx.UpsertAncestry(ancestry); err != nil { + tx.Rollback() + return err + } + + if err = tx.Commit(); err != nil { + return err + } + + return nil +} + +// PersistNamespacedFeatures wraps session PersistNamespacedFeatures function +// with begin and commit. +func PersistNamespacedFeatures(datastore database.Datastore, features []database.NamespacedFeature) error { + tx, err := datastore.Begin() + if err != nil { + return err + } + + if err := tx.PersistNamespacedFeatures(features); err != nil { + tx.Rollback() + return err + } + + if err := tx.Commit(); err != nil { + return err + } + + return nil +} + +// CacheRelatedVulnerability wraps session CacheAffectedNamespacedFeatures +// function with begin and commit. +func CacheRelatedVulnerability(datastore database.Datastore, features []database.NamespacedFeature) error { + tx, err := datastore.Begin() + if err != nil { + return err + } + + if err := tx.CacheAffectedNamespacedFeatures(features); err != nil { + tx.Rollback() + return err + } + + return tx.Commit() +} + +// IntersectDetectors returns the detectors in both d1 and d2. +func IntersectDetectors(d1 []database.Detector, d2 []database.Detector) []database.Detector { + d1Set := mapset.NewSet() + for _, d := range d1 { + d1Set.Add(d) + } + + d2Set := mapset.NewSet() + for _, d := range d2 { + d2Set.Add(d) + } + + inter := d1Set.Intersect(d2Set) + result := make([]database.Detector, 0, inter.Cardinality()) + for d := range inter.Iter() { + result = append(result, d.(database.Detector)) + } + + return result +} + +// DiffDetectors returns the detectors belongs to d1 but not d2 +func DiffDetectors(d1 []database.Detector, d2 []database.Detector) []database.Detector { + d1Set := mapset.NewSet() + for _, d := range d1 { + d1Set.Add(d) + } + + d2Set := mapset.NewSet() + for _, d := range d2 { + d2Set.Add(d) + } + + diff := d1Set.Difference(d2Set) + result := make([]database.Detector, 0, diff.Cardinality()) + for d := range diff.Iter() { + result = append(result, d.(database.Detector)) + } + + return result +} + +// MergeLayers merges all content in new layer to l, where the content is +// updated. +func MergeLayers(l *database.Layer, new *database.Layer) *database.Layer { + featureSet := mapset.NewSet() + namespaceSet := mapset.NewSet() + bySet := mapset.NewSet() + + for _, f := range l.Features { + featureSet.Add(f) + } + + for _, ns := range l.Namespaces { + namespaceSet.Add(ns) + } + + for _, d := range l.By { + bySet.Add(d) + } + + for _, feature := range new.Features { + if !featureSet.Contains(feature) { + l.Features = append(l.Features, feature) + featureSet.Add(feature) + } + } + + for _, namespace := range new.Namespaces { + if !namespaceSet.Contains(namespace) { + l.Namespaces = append(l.Namespaces, namespace) + namespaceSet.Add(namespace) + } + } + + for _, detector := range new.By { + if !bySet.Contains(detector) { + l.By = append(l.By, detector) + bySet.Add(detector) + } + } + + return l +} diff --git a/pkg/strutil/strutil.go b/pkg/strutil/strutil.go index a8d04f21..bfd8dc01 100644 --- a/pkg/strutil/strutil.go +++ b/pkg/strutil/strutil.go @@ -14,42 +14,46 @@ package strutil -// CompareStringLists returns the strings that are present in X but not in Y. -func CompareStringLists(X, Y []string) []string { - m := make(map[string]bool) +import ( + "regexp" - for _, y := range Y { - m[y] = true - } + set "github.com/deckarep/golang-set" +) + +var urlParametersRegexp = regexp.MustCompile(`(\?|\&)([^=]+)\=([^ &]+)`) - diff := []string{} +func convertToSet(X []string) set.Set { + s := set.NewSet() for _, x := range X { - if m[x] { - continue - } + s.Add(x) + } + return s +} - diff = append(diff, x) - m[x] = true +func setToStringSlice(s set.Set) []string { + strs := make([]string, 0, s.Cardinality()) + for _, str := range s.ToSlice() { + strs = append(strs, str.(string)) } - return diff + return strs } -// CompareStringListsInBoth returns the strings that are present in both X and Y. -func CompareStringListsInBoth(X, Y []string) []string { - m := make(map[string]struct{}) - - for _, y := range Y { - m[y] = struct{}{} - } +// Difference returns the strings that are present in X but not in Y. +func Difference(X, Y []string) []string { + x := convertToSet(X) + y := convertToSet(Y) + return setToStringSlice(x.Difference(y)) +} - diff := []string{} - for _, x := range X { - if _, e := m[x]; e { - diff = append(diff, x) - delete(m, x) - } - } +// Intersect returns the strings that are present in both X and Y. +func Intersect(X, Y []string) []string { + x := convertToSet(X) + y := convertToSet(Y) + return setToStringSlice(x.Intersect(y)) +} - return diff +// CleanURL removes all parameters from an URL. +func CleanURL(str string) string { + return urlParametersRegexp.ReplaceAllString(str, "") } diff --git a/pkg/strutil/strutil_test.go b/pkg/strutil/strutil_test.go index 4cbf1e90..2e81856c 100644 --- a/pkg/strutil/strutil_test.go +++ b/pkg/strutil/strutil_test.go @@ -21,12 +21,12 @@ import ( ) func TestStringComparison(t *testing.T) { - cmp := CompareStringLists([]string{"a", "b", "b", "a"}, []string{"a", "c"}) + cmp := Difference([]string{"a", "b", "b", "a"}, []string{"a", "c"}) assert.Len(t, cmp, 1) assert.NotContains(t, cmp, "a") assert.Contains(t, cmp, "b") - cmp = CompareStringListsInBoth([]string{"a", "a", "b", "c"}, []string{"a", "c", "c"}) + cmp = Intersect([]string{"a", "a", "b", "c"}, []string{"a", "c", "c"}) assert.Len(t, cmp, 2) assert.NotContains(t, cmp, "b") assert.Contains(t, cmp, "a") diff --git a/updater.go b/updater.go index 792e068b..d8184178 100644 --- a/updater.go +++ b/updater.go @@ -21,6 +21,8 @@ import ( "sync" "time" + "github.com/coreos/clair/pkg/dbutil" + "github.com/pborman/uuid" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -208,7 +210,7 @@ func update(datastore database.Datastore, firstUpdate bool) { namespaces = append(namespaces, ns) } - if err := persistNamespaces(datastore, namespaces); err != nil { + if err := dbutil.PersistNamespaces(datastore, namespaces); err != nil { log.WithError(err).Error("Unable to insert namespaces") return } diff --git a/worker.go b/worker.go index 5890acbb..16f6a3e6 100644 --- a/worker.go +++ b/worker.go @@ -16,9 +16,9 @@ package clair import ( "errors" - "regexp" "sync" + "github.com/deckarep/golang-set" log "github.com/sirupsen/logrus" "github.com/coreos/clair/database" @@ -26,11 +26,9 @@ import ( "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/imagefmt" "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/dbutil" "github.com/coreos/clair/pkg/strutil" -) - -const ( - logLayerName = "layer" + "github.com/coreos/clair/pkg/tarutil" ) var ( @@ -38,18 +36,8 @@ var ( // manager is not supported. ErrUnsupported = commonerr.NewBadRequestError("worker: OS and/or package manager are not supported") - // ErrParentUnknown is the error that should be raised when a parent layer - // has yet to be processed for the current layer. - ErrParentUnknown = commonerr.NewBadRequestError("worker: parent layer is unknown, it must be processed first") - - urlParametersRegexp = regexp.MustCompile(`(\?|\&)([^=]+)\=([^ &]+)`) - - // Processors contain the names of namespace detectors and feature listers - // enabled in this instance of Clair. - // - // Processors are initialized during booting and configured in the - // configuration file. - Processors database.Processors + // EnabledDetectors are detectors to be used to scan the layers. + EnabledDetectors []database.Detector ) // LayerRequest represents all information necessary to download and process a @@ -60,296 +48,176 @@ type LayerRequest struct { Headers map[string]string } -// partialLayer stores layer's content detected by `processedBy` processors. -type partialLayer struct { - hash string - processedBy database.Processors - namespaces []database.Namespace - features []database.Feature - - err error +type processResult struct { + existingLayer *database.Layer + newLayerContent *database.Layer + err error } -// processRequest stores parameters used for processing layers. +// processRequest stores parameters used for processing a layer. type processRequest struct { - request LayerRequest - // notProcessedBy represents a set of processors used to process the - // request. - notProcessedBy database.Processors + LayerRequest + + existingLayer *database.Layer + detectors []database.Detector } -// cleanURL removes all parameters from an URL. -func cleanURL(str string) string { - return urlParametersRegexp.ReplaceAllString(str, "") +type introducedFeature struct { + feature database.AncestryFeature + layerIndex int } -// processLayers in parallel processes a set of requests for unique set of layers +// processRequests in parallel processes a set of requests for unique set of layers // and returns sets of unique namespaces, features and layers to be inserted // into the database. -func processRequests(imageFormat string, toDetect []processRequest) ([]database.Namespace, []database.Feature, map[string]partialLayer, error) { +func processRequests(imageFormat string, toDetect map[string]*processRequest) (map[string]*processResult, error) { wg := &sync.WaitGroup{} wg.Add(len(toDetect)) - results := make([]partialLayer, len(toDetect)) + + results := map[string]*processResult{} + for i := range toDetect { + results[i] = nil + } + for i := range toDetect { - go func(req *processRequest, res *partialLayer) { - res.hash = req.request.Hash - res.processedBy = req.notProcessedBy - res.namespaces, res.features, res.err = detectContent(imageFormat, req.request.Hash, req.request.Path, req.request.Headers, req.notProcessedBy) + result := processResult{} + results[i] = &result + go func(req *processRequest, res *processResult) { + *res = *detectContent(imageFormat, req) wg.Done() - }(&toDetect[i], &results[i]) + }(toDetect[i], &result) } - wg.Wait() - distinctNS := map[database.Namespace]struct{}{} - distinctF := map[database.Feature]struct{}{} + wg.Wait() errs := []error{} for _, r := range results { errs = append(errs, r.err) } if err := commonerr.CombineErrors(errs...); err != nil { - return nil, nil, nil, err - } - - updates := map[string]partialLayer{} - for _, r := range results { - for _, ns := range r.namespaces { - distinctNS[ns] = struct{}{} - } - - for _, f := range r.features { - distinctF[f] = struct{}{} - } - - if _, ok := updates[r.hash]; !ok { - updates[r.hash] = r - } else { - return nil, nil, nil, errors.New("Duplicated updates is not allowed") - } - } - - namespaces := make([]database.Namespace, 0, len(distinctNS)) - features := make([]database.Feature, 0, len(distinctF)) - - for ns := range distinctNS { - namespaces = append(namespaces, ns) + return nil, err } - for f := range distinctF { - features = append(features, f) - } - return namespaces, features, updates, nil + return results, nil } -func getLayer(datastore database.Datastore, req LayerRequest) (layer database.Layer, preq *processRequest, err error) { - var ( - tx database.Session - ok bool - ) - - if tx, err = datastore.Begin(); err != nil { - return - } - - defer tx.Rollback() - - if layer, ok, err = tx.FindLayer(req.Hash); err != nil { +func getProcessRequest(datastore database.Datastore, req LayerRequest) (preq *processRequest, err error) { + layer, ok, err := dbutil.FindLayer(datastore, req.Hash) + if err != nil { return } if !ok { - layer = database.Layer{ - LayerMetadata: database.LayerMetadata{ - Hash: req.Hash, - }, - } - + log.WithField("layer", req.Hash).Debug("found no existing layer in database") preq = &processRequest{ - request: req, - notProcessedBy: Processors, + LayerRequest: req, + existingLayer: &database.Layer{Hash: req.Hash}, + detectors: EnabledDetectors, } } else { - notProcessed := getNotProcessedBy(layer.ProcessedBy) - if !(len(notProcessed.Detectors) == 0 && len(notProcessed.Listers) == 0 && ok) { - preq = &processRequest{ - request: req, - notProcessedBy: notProcessed, - } + log.WithFields(log.Fields{ + "layer": layer.Hash, + "detectors": layer.By, + "feature count": len(layer.Features), + "namespace count": len(layer.Namespaces), + }).Debug("found existing layer in database") + + preq = &processRequest{ + LayerRequest: req, + existingLayer: &layer, + detectors: dbutil.DiffDetectors(EnabledDetectors, layer.By), } } return } -// processLayers processes a set of post layer requests, stores layers and -// returns an ordered list of processed layers with detected features and -// namespaces. -func processLayers(datastore database.Datastore, imageFormat string, requests []LayerRequest) ([]database.Layer, error) { - toDetect := []processRequest{} - layers := map[string]database.Layer{} - for _, req := range requests { - if _, ok := layers[req.Hash]; ok { - continue - } - layer, preq, err := getLayer(datastore, req) - if err != nil { - return nil, err - } - layers[req.Hash] = layer - if preq != nil { - toDetect = append(toDetect, *preq) - } - } - - namespaces, features, partialLayers, err := processRequests(imageFormat, toDetect) - if err != nil { - return nil, err +func persistProcessResult(datastore database.Datastore, results map[string]*processResult) error { + features := []database.Feature{} + namespaces := []database.Namespace{} + for _, r := range results { + features = append(features, r.newLayerContent.GetFeatures()...) + namespaces = append(namespaces, r.newLayerContent.GetNamespaces()...) } - // Store partial results. - if err := persistNamespaces(datastore, namespaces); err != nil { - return nil, err + features = dbutil.DeduplicateFeatures(features...) + namespaces = dbutil.DeduplicateNamespaces(namespaces...) + if err := dbutil.PersistNamespaces(datastore, namespaces); err != nil { + return err } - if err := persistFeatures(datastore, features); err != nil { - return nil, err + if err := dbutil.PersistFeatures(datastore, features); err != nil { + return err } - for _, layer := range partialLayers { - if err := persistPartialLayer(datastore, layer); err != nil { - return nil, err - } - - log.WithFields(log.Fields{ - "Hash": layer.hash, - "namespace count": len(layer.namespaces), - "feature count": len(layer.features), - "namespace detectors": layer.processedBy.Detectors, - "feature listers": layer.processedBy.Listers, - }).Debug("saved layer") - } - - // NOTE(Sida): The full layers are computed using partially - // processed layers in current database session. If any other instances of - // Clair are changing some layers in this set of layers, it might generate - // different results especially when the other Clair is with different - // processors. - completeLayers := []database.Layer{} - for _, req := range requests { - if partialLayer, ok := partialLayers[req.Hash]; ok { - completeLayers = append(completeLayers, combineLayers(layers[req.Hash], partialLayer)) - } else { - completeLayers = append(completeLayers, layers[req.Hash]) + for _, layer := range results { + if err := dbutil.PersistPartialLayer(datastore, layer.newLayerContent); err != nil { + return err } } - return completeLayers, nil + return nil } -func persistPartialLayer(datastore database.Datastore, layer partialLayer) error { - tx, err := datastore.Begin() - if err != nil { - return err - } - defer tx.Rollback() +// processLayers processes a set of post layer requests, stores layers and +// returns an ordered list of processed layers with detected features and +// namespaces. +func processLayers(datastore database.Datastore, imageFormat string, requests []LayerRequest) ([]database.Layer, error) { + var ( + reqMap = make(map[string]*processRequest) + err error + ) - if err := tx.PersistLayer(layer.hash, layer.namespaces, layer.features, layer.processedBy); err != nil { - return err + for _, r := range requests { + reqMap[r.Hash], err = getProcessRequest(datastore, r) + if err != nil { + return nil, err + } } - return tx.Commit() -} - -func persistFeatures(datastore database.Datastore, features []database.Feature) error { - tx, err := datastore.Begin() + results, err := processRequests(imageFormat, reqMap) if err != nil { - return err + return nil, err } - defer tx.Rollback() - if err := tx.PersistFeatures(features); err != nil { - return err - } - return tx.Commit() -} - -func persistNamespaces(datastore database.Datastore, namespaces []database.Namespace) error { - tx, err := datastore.Begin() - if err != nil { - return err + if err := persistProcessResult(datastore, results); err != nil { + return nil, err } - defer tx.Rollback() - if err := tx.PersistNamespaces(namespaces); err != nil { - return err + completeLayers := getProcessResultLayers(results) + layers := make([]database.Layer, 0, len(requests)) + for _, r := range requests { + layers = append(layers, completeLayers[r.Hash]) } - return tx.Commit() + return layers, nil } -// combineLayers merges `layer` and `partial` without duplicated content. -func combineLayers(layer database.Layer, partial partialLayer) database.Layer { - mapF := map[database.Feature]struct{}{} - mapNS := map[database.Namespace]struct{}{} - for _, f := range layer.Features { - mapF[f] = struct{}{} - } - for _, ns := range layer.Namespaces { - mapNS[ns] = struct{}{} - } - for _, f := range partial.features { - mapF[f] = struct{}{} - } - for _, ns := range partial.namespaces { - mapNS[ns] = struct{}{} - } - features := make([]database.Feature, 0, len(mapF)) - namespaces := make([]database.Namespace, 0, len(mapNS)) - for f := range mapF { - features = append(features, f) - } - for ns := range mapNS { - namespaces = append(namespaces, ns) +func getProcessResultLayers(results map[string]*processResult) map[string]database.Layer { + layers := map[string]database.Layer{} + for name, r := range results { + layers[name] = *dbutil.MergeLayers(r.existingLayer, r.newLayerContent) } - layer.ProcessedBy.Detectors = append(layer.ProcessedBy.Detectors, strutil.CompareStringLists(partial.processedBy.Detectors, layer.ProcessedBy.Detectors)...) - layer.ProcessedBy.Listers = append(layer.ProcessedBy.Listers, strutil.CompareStringLists(partial.processedBy.Listers, layer.ProcessedBy.Listers)...) - return database.Layer{ - LayerMetadata: database.LayerMetadata{ - Hash: layer.Hash, - ProcessedBy: layer.ProcessedBy, - }, - Features: features, - Namespaces: namespaces, - } + return layers } func isAncestryProcessed(datastore database.Datastore, name string) (bool, error) { - tx, err := datastore.Begin() - if err != nil { - return false, err - } - defer tx.Rollback() - ancestry, ok, err := tx.FindAncestry(name) - if err != nil { - return false, err - } - if !ok { - return false, nil + ancestry, ok, err := dbutil.FindAncestry(datastore, name) + if err != nil || !ok { + return ok, err } - notProcessed := getNotProcessedBy(ancestry.ProcessedBy) - return len(notProcessed.Detectors) == 0 && len(notProcessed.Listers) == 0, nil + return len(dbutil.DiffDetectors(EnabledDetectors, ancestry.By)) == 0, nil } // ProcessAncestry downloads and scans an ancestry if it's not scanned by all // enabled processors in this instance of Clair. func ProcessAncestry(datastore database.Datastore, imageFormat, name string, layerRequest []LayerRequest) error { var ( - err error - ok bool - layers []database.Layer - commonProcessors database.Processors + err error + ok bool + layers []database.Layer ) if name == "" { @@ -360,10 +228,12 @@ func ProcessAncestry(datastore database.Datastore, imageFormat, name string, lay return commonerr.NewBadRequestError("could not process a layer which does not have a format") } + log.WithField("ancestry", name).Debug("start processing ancestry...") if ok, err = isAncestryProcessed(datastore, name); err != nil { + log.WithError(err).Error("could not determine if ancestry is processed") return err } else if ok { - log.WithField("name", name).Debug("ancestry is already processed") + log.WithField("ancestry", name).Debug("ancestry is already processed") return nil } @@ -371,155 +241,100 @@ func ProcessAncestry(datastore database.Datastore, imageFormat, name string, lay return err } - if commonProcessors, err = getProcessors(layers); err != nil { - return err - } - - return processAncestry(datastore, name, layers, commonProcessors) -} - -// getNamespacedFeatures extracts the namespaced features introduced in each -// layer into one array. -func getNamespacedFeatures(layers []database.AncestryLayer) []database.NamespacedFeature { - features := []database.NamespacedFeature{} - for _, layer := range layers { - features = append(features, layer.DetectedFeatures...) - } - return features + return processAncestry(datastore, name, layers) } -func processAncestry(datastore database.Datastore, name string, layers []database.Layer, commonProcessors database.Processors) error { +func processAncestry(datastore database.Datastore, name string, layers []database.Layer) error { var ( - ancestry database.Ancestry + ancestry = database.Ancestry{Name: name} err error ) - ancestry.Name = name - ancestry.ProcessedBy = commonProcessors - ancestry.Layers, err = computeAncestryLayers(layers, commonProcessors) + ancestry.Layers, ancestry.By, err = computeAncestryLayers(layers) if err != nil { return err } - ancestryFeatures := getNamespacedFeatures(ancestry.Layers) + ancestryFeatures := dbutil.GetAncestryFeatures(ancestry) log.WithFields(log.Fields{ - "ancestry": name, - "number of features": len(ancestryFeatures), - "processed by": Processors, - "number of layers": len(ancestry.Layers), + "ancestry": name, + "processed by": EnabledDetectors, + "features count": len(ancestryFeatures), + "layer count": len(ancestry.Layers), }).Debug("compute ancestry features") - if err := persistNamespacedFeatures(datastore, ancestryFeatures); err != nil { + if err := dbutil.PersistNamespacedFeatures(datastore, ancestryFeatures); err != nil { + log.WithField("ancestry", name).WithError(err).Error("could not persist namespaced features for ancestry") return err } - tx, err := datastore.Begin() - if err != nil { + if err := dbutil.CacheRelatedVulnerability(datastore, ancestryFeatures); err != nil { + log.WithField("ancestry", name).WithError(err).Error("failed to cache feature related vulnerability") return err } - err = tx.UpsertAncestry(ancestry) - if err != nil { - tx.Rollback() + if err := dbutil.UpsertAncestry(datastore, ancestry); err != nil { + log.WithField("ancestry", name).WithError(err).Error("could not upsert ancestry") return err } - err = tx.Commit() - if err != nil { - return err - } return nil } -func persistNamespacedFeatures(datastore database.Datastore, features []database.NamespacedFeature) error { - tx, err := datastore.Begin() - if err != nil { - return err - } - - if err := tx.PersistNamespacedFeatures(features); err != nil { - tx.Rollback() - return err - } - - if err := tx.Commit(); err != nil { - return err - } - - tx, err = datastore.Begin() - if err != nil { - return err +func getCommonDetectors(layers []database.Layer) mapset.Set { + // find the common detector for all layers and filter the namespaces and + // features based on that. + commonDetectors := mapset.NewSet() + for _, d := range layers[0].By { + commonDetectors.Add(d) } - if err := tx.CacheAffectedNamespacedFeatures(features); err != nil { - tx.Rollback() - return err - } - - return tx.Commit() -} - -// getProcessors retrieves common subset of the processors of each layer. -func getProcessors(layers []database.Layer) (database.Processors, error) { - if len(layers) == 0 { - return database.Processors{}, nil - } - - detectors := layers[0].ProcessedBy.Detectors - listers := layers[0].ProcessedBy.Listers - - detectorsLen := len(detectors) - listersLen := len(listers) - - for _, l := range layers[1:] { - detectors := strutil.CompareStringListsInBoth(detectors, l.ProcessedBy.Detectors) - listers := strutil.CompareStringListsInBoth(listers, l.ProcessedBy.Listers) - - if len(detectors) != detectorsLen || len(listers) != listersLen { - // This error might be triggered because of multiple workers are - // processing the same instance with different processors. - // TODO(sidchen): Once the features can be associated with - // Detectors/Listers, we can support dynamically generating ancestry's - // detector/lister based on the layers. - return database.Processors{}, errors.New("processing layers with different Clair instances is currently unsupported") + for _, l := range layers { + detectors := mapset.NewSet() + for _, d := range l.By { + detectors.Add(d) } + + commonDetectors = commonDetectors.Intersect(detectors) } - return database.Processors{ - Detectors: detectors, - Listers: listers, - }, nil -} -type introducedFeature struct { - feature database.NamespacedFeature - layerIndex int + return commonDetectors } // computeAncestryLayers computes ancestry's layers along with what features are // introduced. -func computeAncestryLayers(layers []database.Layer, commonProcessors database.Processors) ([]database.AncestryLayer, error) { - // TODO(sidchen): Once the features are linked to specific processor, we - // will use commonProcessors to filter out the features for this ancestry. +func computeAncestryLayers(layers []database.Layer) ([]database.AncestryLayer, []database.Detector, error) { + if len(layers) == 0 { + return nil, nil, nil + } + commonDetectors := getCommonDetectors(layers) // version format -> namespace - namespaces := map[string]database.Namespace{} + namespaces := map[string]database.LayerNamespace{} // version format -> feature ID -> feature features := map[string]map[string]introducedFeature{} ancestryLayers := []database.AncestryLayer{} for index, layer := range layers { - // Initialize the ancestry Layer - initializedLayer := database.AncestryLayer{LayerMetadata: layer.LayerMetadata, DetectedFeatures: []database.NamespacedFeature{}} + initializedLayer := database.AncestryLayer{Hash: layer.Hash} ancestryLayers = append(ancestryLayers, initializedLayer) // Precondition: namespaces and features contain the result from union // of all parents. for _, ns := range layer.Namespaces { + if !commonDetectors.Contains(ns.By) { + continue + } + namespaces[ns.VersionFormat] = ns } // version format -> feature ID -> feature currentFeatures := map[string]map[string]introducedFeature{} for _, f := range layer.Features { + if !commonDetectors.Contains(f.By) { + continue + } + if ns, ok := namespaces[f.VersionFormat]; ok { var currentMap map[string]introducedFeature if currentMap, ok = currentFeatures[f.VersionFormat]; !ok { @@ -537,16 +352,20 @@ func computeAncestryLayers(layers []database.Layer, commonProcessors database.Pr if !inherited { currentMap[f.Name+":"+f.Version] = introducedFeature{ - feature: database.NamespacedFeature{ - Feature: f, - Namespace: ns, + feature: database.AncestryFeature{ + NamespacedFeature: database.NamespacedFeature{ + Feature: f.Feature, + Namespace: ns.Namespace, + }, + NamespaceBy: ns.By, + FeatureBy: f.By, }, layerIndex: index, } } } else { - return nil, errors.New("No corresponding version format") + return nil, nil, errors.New("No corresponding version format") } } @@ -564,57 +383,97 @@ func computeAncestryLayers(layers []database.Layer, commonProcessors database.Pr for _, featureMap := range features { for _, feature := range featureMap { - ancestryLayers[feature.layerIndex].DetectedFeatures = append( - ancestryLayers[feature.layerIndex].DetectedFeatures, + ancestryLayers[feature.layerIndex].Features = append( + ancestryLayers[feature.layerIndex].Features, feature.feature, ) } } - return ancestryLayers, nil + detectors := make([]database.Detector, 0, commonDetectors.Cardinality()) + for d := range commonDetectors.Iter() { + detectors = append(detectors, d.(database.Detector)) + } + + return ancestryLayers, detectors, nil } -// getNotProcessedBy returns a processors, which contains the detectors and -// listers not in `processedBy` but implemented in the current clair instance. -func getNotProcessedBy(processedBy database.Processors) database.Processors { - notProcessedLister := strutil.CompareStringLists(Processors.Listers, processedBy.Listers) - notProcessedDetector := strutil.CompareStringLists(Processors.Detectors, processedBy.Detectors) - return database.Processors{ - Listers: notProcessedLister, - Detectors: notProcessedDetector, +func extractRequiredFiles(imageFormat string, req *processRequest) (tarutil.FilesMap, error) { + requiredFiles := append(featurefmt.RequiredFilenames(req.detectors), featurens.RequiredFilenames(req.detectors)...) + if len(requiredFiles) == 0 { + log.WithFields(log.Fields{ + "layer": req.Hash, + "detectors": req.detectors, + }).Info("layer requires no file to extract") + return make(tarutil.FilesMap), nil } -} -// detectContent downloads a layer and detects all features and namespaces. -func detectContent(imageFormat, name, path string, headers map[string]string, toProcess database.Processors) (namespaces []database.Namespace, featureVersions []database.Feature, err error) { - log.WithFields(log.Fields{"Hash": name}).Debug("Process Layer") - totalRequiredFiles := append(featurefmt.RequiredFilenames(toProcess.Listers), featurens.RequiredFilenames(toProcess.Detectors)...) - files, err := imagefmt.Extract(imageFormat, path, headers, totalRequiredFiles) + files, err := imagefmt.Extract(imageFormat, req.Path, req.Headers, requiredFiles) if err != nil { log.WithError(err).WithFields(log.Fields{ - logLayerName: name, - "path": cleanURL(path), + "layer": req.Hash, + "path": strutil.CleanURL(req.Path), }).Error("failed to extract data from path") + return nil, err + } + + return files, err +} + +// detectContent downloads a layer and detects all features and namespaces. +func detectContent(imageFormat string, req *processRequest) (res *processResult) { + var ( + files tarutil.FilesMap + layer = database.Layer{Hash: req.Hash, By: req.detectors} + ) + + res = &processResult{req.existingLayer, &layer, nil} + log.WithFields(log.Fields{ + "layer": req.Hash, + "detectors": req.detectors, + }).Info("detecting layer content...") + + files, res.err = extractRequiredFiles(imageFormat, req) + if res.err != nil { return } - namespaces, err = featurens.Detect(files, toProcess.Detectors) - if err != nil { + if layer.Namespaces, res.err = featurens.Detect(files, req.detectors); res.err != nil { return } - if len(featureVersions) > 0 { - log.WithFields(log.Fields{logLayerName: name, "count": len(namespaces)}).Debug("detected layer namespaces") + if layer.Features, res.err = featurefmt.ListFeatures(files, req.detectors); res.err != nil { + return } - featureVersions, err = featurefmt.ListFeatures(files, toProcess.Listers) - if err != nil { + log.WithFields(log.Fields{ + "layer": req.Hash, + "detectors": req.detectors, + "namespace count": len(layer.Namespaces), + "feature count": len(layer.Features), + }).Info("processed layer") + + return +} + +// InitWorker initializes the worker. +func InitWorker(datastore database.Datastore) { + if len(EnabledDetectors) == 0 { + log.Warn("no enabled detector, and therefore, no ancestry will be processed.") return } - if len(featureVersions) > 0 { - log.WithFields(log.Fields{logLayerName: name, "count": len(featureVersions)}).Debug("detected layer features") + tx, err := datastore.Begin() + if err != nil { + log.WithError(err).Fatal("cannot connect to database to initialize worker") } - return + defer tx.Rollback() + if err := tx.PersistDetectors(EnabledDetectors); err != nil { + log.WithError(err).Fatal("cannot insert detectors to initialize worker") + } + + if err := tx.Commit(); err != nil { + log.WithError(err).Fatal("cannot commit detector changes to initialize worker") + } } diff --git a/worker_test.go b/worker_test.go index 2df212e0..e6e58ad6 100644 --- a/worker_test.go +++ b/worker_test.go @@ -22,12 +22,14 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/featurefmt" "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/pkg/strutil" + "github.com/coreos/clair/pkg/dbutil" + "github.com/coreos/clair/pkg/testutil" // Register the required detectors. _ "github.com/coreos/clair/ext/featurefmt/dpkg" @@ -58,55 +60,27 @@ type mockSession struct { func copyDatastore(md *mockDatastore) mockDatastore { layers := map[string]database.Layer{} for k, l := range md.layers { - features := append([]database.Feature(nil), l.Features...) - namespaces := append([]database.Namespace(nil), l.Namespaces...) - listers := append([]string(nil), l.ProcessedBy.Listers...) - detectors := append([]string(nil), l.ProcessedBy.Detectors...) layers[k] = database.Layer{ - LayerMetadata: database.LayerMetadata{ - Hash: l.Hash, - ProcessedBy: database.Processors{ - Listers: listers, - Detectors: detectors, - }, - }, - Features: features, - Namespaces: namespaces, + Hash: l.Hash, + By: append([]database.Detector{}, l.By...), + Features: append([]database.LayerFeature{}, l.Features...), + Namespaces: append([]database.LayerNamespace{}, l.Namespaces...), } } ancestry := map[string]database.Ancestry{} for k, a := range md.ancestry { ancestryLayers := []database.AncestryLayer{} - layers := []database.LayerMetadata{} - for _, layer := range a.Layers { - layers = append(layers, database.LayerMetadata{ - Hash: layer.Hash, - ProcessedBy: database.Processors{ - Detectors: append([]string(nil), layer.LayerMetadata.ProcessedBy.Detectors...), - Listers: append([]string(nil), layer.LayerMetadata.ProcessedBy.Listers...), - }, - }) - ancestryLayers = append(ancestryLayers, database.AncestryLayer{ - LayerMetadata: database.LayerMetadata{ - Hash: layer.Hash, - ProcessedBy: database.Processors{ - Detectors: append([]string(nil), layer.LayerMetadata.ProcessedBy.Detectors...), - Listers: append([]string(nil), layer.LayerMetadata.ProcessedBy.Listers...), - }, - }, - DetectedFeatures: append([]database.NamespacedFeature(nil), layer.DetectedFeatures...), + Hash: layer.Hash, + Features: append([]database.AncestryFeature{}, layer.Features...), }) } ancestry[k] = database.Ancestry{ - Name: a.Name, - ProcessedBy: database.Processors{ - Detectors: append([]string(nil), a.ProcessedBy.Detectors...), - Listers: append([]string(nil), a.ProcessedBy.Listers...), - }, + Name: a.Name, + By: append([]database.Detector{}, a.By...), Layers: ancestryLayers, } } @@ -125,6 +99,7 @@ func copyDatastore(md *mockDatastore) mockDatastore { for k, f := range md.namespacedFeatures { namespacedFeatures[k] = f } + return mockDatastore{ layers: layers, ancestry: ancestry, @@ -194,10 +169,7 @@ func newMockDatastore() *mockDatastore { return errSessionDone } for _, n := range ns { - _, ok := session.copy.namespaces[n.Name] - if !ok { - session.copy.namespaces[n.Name] = n - } + session.copy.namespaces[NamespaceKey(&n)] = n } return nil } @@ -207,63 +179,36 @@ func newMockDatastore() *mockDatastore { return errSessionDone } for _, f := range fs { - key := FeatureKey(&f) - _, ok := session.copy.features[key] - if !ok { - session.copy.features[key] = f - } + session.copy.features[FeatureKey(&f)] = f } + return nil } - session.FctPersistLayer = func(hash string, namespaces []database.Namespace, features []database.Feature, processedBy database.Processors) error { + session.FctPersistLayer = func(hash string, features []database.LayerFeature, namespaces []database.LayerNamespace, by []database.Detector) error { if session.terminated { return errSessionDone } - // update the layer - _, ok := session.copy.layers[hash] - if !ok { - session.copy.layers[hash] = database.Layer{} - } - - layer, ok := session.copy.layers[hash] - if !ok { - return errors.New("Failed to insert layer") - } - - layerFeatures := map[string]database.Feature{} - layerNamespaces := map[string]database.Namespace{} - for _, f := range layer.Features { - layerFeatures[FeatureKey(&f)] = f - } - for _, n := range layer.Namespaces { - layerNamespaces[n.Name] = n - } - - // ensure that all the namespaces, features are in the database for _, ns := range namespaces { - if _, ok := session.copy.namespaces[ns.Name]; !ok { - return errors.New("Namespaces should be in the database") - } - if _, ok := layerNamespaces[ns.Name]; !ok { - layer.Namespaces = append(layer.Namespaces, ns) - layerNamespaces[ns.Name] = ns + if _, ok := session.copy.namespaces[NamespaceKey(&ns.Namespace)]; !ok { + panic("") } } for _, f := range features { - if _, ok := session.copy.features[FeatureKey(&f)]; !ok { - return errors.New("Namespaces should be in the database") - } - if _, ok := layerFeatures[FeatureKey(&f)]; !ok { - layer.Features = append(layer.Features, f) - layerFeatures[FeatureKey(&f)] = f + if _, ok := session.copy.features[FeatureKey(&f.Feature)]; !ok { + panic("") } } - layer.ProcessedBy.Detectors = append(layer.ProcessedBy.Detectors, strutil.CompareStringLists(processedBy.Detectors, layer.ProcessedBy.Detectors)...) - layer.ProcessedBy.Listers = append(layer.ProcessedBy.Listers, strutil.CompareStringLists(processedBy.Listers, layer.ProcessedBy.Listers)...) + layer, _ := session.copy.layers[hash] + dbutil.MergeLayers(&layer, &database.Layer{ + Hash: hash, + By: by, + Namespaces: namespaces, + Features: features, + }) session.copy.layers[hash] = layer return nil @@ -274,11 +219,12 @@ func newMockDatastore() *mockDatastore { return errSessionDone } - features := getNamespacedFeatures(ancestry.Layers) - // ensure features are in the database - for _, f := range features { - if _, ok := session.copy.namespacedFeatures[NamespacedFeatureKey(&f)]; !ok { - return errors.New("namespaced feature not in db") + // ensure the namespaces features are in the code base + for _, l := range ancestry.Layers { + for _, f := range l.GetFeatures() { + if _, ok := session.copy.namespacedFeatures[NamespacedFeatureKey(&f)]; !ok { + panic("") + } } } @@ -288,6 +234,14 @@ func newMockDatastore() *mockDatastore { session.FctPersistNamespacedFeatures = func(namespacedFeatures []database.NamespacedFeature) error { for i, f := range namespacedFeatures { + if _, ok := session.copy.features[FeatureKey(&f.Feature)]; !ok { + panic("") + } + + if _, ok := session.copy.namespaces[NamespaceKey(&f.Namespace)]; !ok { + panic("") + } + session.copy.namespacedFeatures[NamespacedFeatureKey(&f)] = namespacedFeatures[i] } return nil @@ -304,10 +258,7 @@ func newMockDatastore() *mockDatastore { } func TestMain(m *testing.M) { - Processors = database.Processors{ - Listers: featurefmt.ListListers(), - Detectors: featurens.ListDetectors(), - } + EnabledDetectors = append(featurefmt.ListListers(), featurens.ListDetectors()...) m.Run() } @@ -315,11 +266,16 @@ func FeatureKey(f *database.Feature) string { return strings.Join([]string{f.Name, f.VersionFormat, f.Version}, "__") } +func NamespaceKey(ns *database.Namespace) string { + return strings.Join([]string{ns.Name, ns.VersionFormat}, "__") +} + func NamespacedFeatureKey(f *database.NamespacedFeature) string { return strings.Join([]string{f.Name, f.Namespace.Name}, "__") } func TestProcessAncestryWithDistUpgrade(t *testing.T) { + // TODO(sidac): Change to use table driven tests. // Create the list of Features that should not been upgraded from one layer to another. nonUpgradedFeatures := []database.Feature{ {Name: "libtext-wrapi18n-perl", Version: "0.06-7"}, @@ -358,7 +314,12 @@ func TestProcessAncestryWithDistUpgrade(t *testing.T) { assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock", layers)) // check the ancestry features - features := getNamespacedFeatures(datastore.ancestry["Mock"].Layers) + features := []database.AncestryFeature{} + for i, l := range datastore.ancestry["Mock"].Layers { + assert.Equal(t, layers[i].Hash, l.Hash) + features = append(features, l.Features...) + } + assert.Len(t, features, 74) for _, f := range features { if _, ok := nonUpgradedMap[f.Feature]; ok { @@ -367,12 +328,6 @@ func TestProcessAncestryWithDistUpgrade(t *testing.T) { assert.Equal(t, "debian:8", f.Namespace.Name) } } - - assert.Equal(t, []database.LayerMetadata{ - {Hash: "blank"}, - {Hash: "wheezy"}, - {Hash: "jessie"}, - }, datastore.ancestry["Mock"].Layers) } func TestProcessLayers(t *testing.T) { @@ -404,8 +359,7 @@ func TestProcessLayers(t *testing.T) { // Ensure each layer has expected namespaces and features detected if blank, ok := datastore.layers["blank"]; ok { - assert.Equal(t, blank.ProcessedBy.Detectors, Processors.Detectors) - assert.Equal(t, blank.ProcessedBy.Listers, Processors.Listers) + testutil.AssertDetectorsEqual(t, EnabledDetectors, blank.By) assert.Len(t, blank.Namespaces, 0) assert.Len(t, blank.Features, 0) } else { @@ -414,9 +368,11 @@ func TestProcessLayers(t *testing.T) { } if wheezy, ok := datastore.layers["wheezy"]; ok { - assert.Equal(t, wheezy.ProcessedBy.Detectors, Processors.Detectors) - assert.Equal(t, wheezy.ProcessedBy.Listers, Processors.Listers) - assert.Equal(t, wheezy.Namespaces, []database.Namespace{{Name: "debian:7", VersionFormat: dpkg.ParserName}}) + testutil.AssertDetectorsEqual(t, EnabledDetectors, wheezy.By) + assert.Equal(t, []database.LayerNamespace{ + {database.Namespace{"debian:7", dpkg.ParserName}, database.NewNamespaceDetector("os-release", "1.0")}, + }, wheezy.Namespaces) + assert.Len(t, wheezy.Features, 52) } else { assert.Fail(t, "wheezy is not stored") @@ -424,9 +380,10 @@ func TestProcessLayers(t *testing.T) { } if jessie, ok := datastore.layers["jessie"]; ok { - assert.Equal(t, jessie.ProcessedBy.Detectors, Processors.Detectors) - assert.Equal(t, jessie.ProcessedBy.Listers, Processors.Listers) - assert.Equal(t, jessie.Namespaces, []database.Namespace{{Name: "debian:8", VersionFormat: dpkg.ParserName}}) + testutil.AssertDetectorsEqual(t, EnabledDetectors, jessie.By) + assert.Equal(t, []database.LayerNamespace{ + {database.Namespace{"debian:8", dpkg.ParserName}, database.NewNamespaceDetector("os-release", "1.0")}, + }, jessie.Namespaces) assert.Len(t, jessie.Features, 74) } else { assert.Fail(t, "jessie is not stored") @@ -434,157 +391,124 @@ func TestProcessLayers(t *testing.T) { } } -// TestUpgradeClair checks if a clair is upgraded and certain ancestry's -// features should not change. We assume that Clair should only upgrade -func TestClairUpgrade(t *testing.T) { - _, f, _, _ := runtime.Caller(0) - testDataPath := filepath.Join(filepath.Dir(f)) + "/testdata/DistUpgrade/" - - datastore := newMockDatastore() - - // suppose there are two ancestries. - layers := []LayerRequest{ - {Hash: "blank", Path: testDataPath + "blank.tar.gz"}, - {Hash: "wheezy", Path: testDataPath + "wheezy.tar.gz"}, - {Hash: "jessie", Path: testDataPath + "jessie.tar.gz"}, +func getFeatures(a database.Ancestry) []database.AncestryFeature { + features := []database.AncestryFeature{} + for _, l := range a.Layers { + features = append(features, l.Features...) } - layers2 := []LayerRequest{ - {Hash: "blank", Path: testDataPath + "blank.tar.gz"}, - {Hash: "wheezy", Path: testDataPath + "wheezy.tar.gz"}, - } - - // Suppose user scan an ancestry with an old instance of Clair. - Processors = database.Processors{ - Detectors: []string{"os-release"}, - Listers: []string{"rpm"}, - } - - assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock", layers)) - assert.Len(t, getNamespacedFeatures(datastore.ancestry["Mock"].Layers), 0) - - assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock2", layers2)) - assert.Len(t, getNamespacedFeatures(datastore.ancestry["Mock2"].Layers), 0) - - // Clair is upgraded to use a new namespace detector. The expected - // behavior is that all layers will be rescanned with "apt-sources" and - // the ancestry's features are recalculated. - Processors = database.Processors{ - Detectors: []string{"os-release", "apt-sources"}, - Listers: []string{"rpm"}, - } - - // Even though Clair processors are upgraded, the ancestry's features should - // not be upgraded without posting the ancestry to Clair again. - assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock", layers)) - assert.Len(t, getNamespacedFeatures(datastore.ancestry["Mock"].Layers), 0) - - // Clair is upgraded to use a new feature lister. The expected behavior is - // that all layers will be rescanned with "dpkg" and the ancestry's features - // are invalidated and recalculated. - Processors = database.Processors{ - Detectors: []string{"os-release", "apt-sources"}, - Listers: []string{"rpm", "dpkg"}, - } - - assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock", layers)) - assert.Len(t, getNamespacedFeatures(datastore.ancestry["Mock"].Layers), 74) - assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock2", layers2)) - assert.Len(t, getNamespacedFeatures(datastore.ancestry["Mock2"].Layers), 52) - - // check the namespaces are correct - for _, f := range getNamespacedFeatures(datastore.ancestry["Mock"].Layers) { - if !assert.NotEqual(t, database.Namespace{}, f.Namespace) { - assert.Fail(t, "Every feature should have a namespace attached") - } - } - - for _, f := range getNamespacedFeatures(datastore.ancestry["Mock2"].Layers) { - if !assert.NotEqual(t, database.Namespace{}, f.Namespace) { - assert.Fail(t, "Every feature should have a namespace attached") - } - } + return features } -// TestMultipleNamespaces tests computing ancestry features func TestComputeAncestryFeatures(t *testing.T) { vf1 := "format 1" vf2 := "format 2" - ns1a := database.Namespace{ - Name: "namespace 1:a", - VersionFormat: vf1, - } - - ns1b := database.Namespace{ - Name: "namespace 1:b", - VersionFormat: vf1, - } - - ns2a := database.Namespace{ - Name: "namespace 2:a", - VersionFormat: vf2, - } - - ns2b := database.Namespace{ - Name: "namespace 2:b", - VersionFormat: vf2, - } - - f1 := database.Feature{ - Name: "feature 1", - Version: "0.1", - VersionFormat: vf1, - } - - f2 := database.Feature{ + nd1 := database.NewNamespaceDetector("apk", "1.0") + fd1 := database.NewFeatureDetector("fd1", "1.0") + // this detector only scans one layer with one extra feature, this one + // should be omitted. + fd2 := database.NewFeatureDetector("fd2", "1.0") + + ns1a := database.LayerNamespace{ + database.Namespace{ + Name: "namespace 1:a", + VersionFormat: vf1, + }, nd1, + } + + ns1b := database.LayerNamespace{ + database.Namespace{ + Name: "namespace 1:b", + VersionFormat: vf1, + }, nd1} + + ns2a := database.LayerNamespace{ + database.Namespace{ + Name: "namespace 2:a", + VersionFormat: vf2, + }, nd1} + + ns2b := database.LayerNamespace{ + database.Namespace{ + Name: "namespace 2:b", + VersionFormat: vf2, + }, nd1} + + f1 := database.LayerFeature{ + database.Feature{ + Name: "feature 1", + Version: "0.1", + VersionFormat: vf1, + }, fd1} + + f2 := database.LayerFeature{database.Feature{ Name: "feature 2", Version: "0.2", VersionFormat: vf1, - } - - f3 := database.Feature{ - Name: "feature 1", - Version: "0.3", - VersionFormat: vf2, - } - - f4 := database.Feature{ - Name: "feature 2", - Version: "0.3", - VersionFormat: vf2, + }, fd2} + + f3 := database.LayerFeature{ + database.Feature{ + Name: "feature 1", + Version: "0.3", + VersionFormat: vf2, + }, fd1} + + f4 := database.LayerFeature{ + database.Feature{ + Name: "feature 2", + Version: "0.3", + VersionFormat: vf2, + }, fd1} + + f5 := database.LayerFeature{ + database.Feature{ + Name: "feature 3", + Version: "0.3", + VersionFormat: vf2, + }, + fd2, } // Suppose Clair is watching two files for namespaces one containing ns1 // changes e.g. os-release and the other one containing ns2 changes e.g. // node. - blank := database.Layer{LayerMetadata: database.LayerMetadata{Hash: "blank"}} + blank := database.Layer{ + Hash: "blank", + By: []database.Detector{nd1, fd1, fd1}, + } initNS1a := database.Layer{ - LayerMetadata: database.LayerMetadata{Hash: "init ns1a"}, - Namespaces: []database.Namespace{ns1a}, - Features: []database.Feature{f1, f2}, + Hash: "initNS1a", + By: []database.Detector{nd1, fd1, fd1}, + Namespaces: []database.LayerNamespace{ns1a}, + Features: []database.LayerFeature{f1, f2}, } upgradeNS2b := database.Layer{ - LayerMetadata: database.LayerMetadata{Hash: "upgrade ns2b"}, - Namespaces: []database.Namespace{ns2b}, + Hash: "upgradeNS2b", + By: []database.Detector{nd1, fd1, fd1}, + Namespaces: []database.LayerNamespace{ns2b}, } upgradeNS1b := database.Layer{ - LayerMetadata: database.LayerMetadata{Hash: "upgrade ns1b"}, - Namespaces: []database.Namespace{ns1b}, - Features: []database.Feature{f1, f2}, + Hash: "upgradeNS1b", + By: []database.Detector{nd1, fd1, fd1, fd2}, + Namespaces: []database.LayerNamespace{ns1b}, + Features: []database.LayerFeature{f1, f2, f5}, } initNS2a := database.Layer{ - LayerMetadata: database.LayerMetadata{Hash: "init ns2a"}, - Namespaces: []database.Namespace{ns2a}, - Features: []database.Feature{f3, f4}, + Hash: "initNS2a", + By: []database.Detector{nd1, fd1, fd1}, + Namespaces: []database.LayerNamespace{ns2a}, + Features: []database.LayerFeature{f3, f4}, } removeF2 := database.Layer{ - LayerMetadata: database.LayerMetadata{Hash: "remove f2"}, - Features: []database.Feature{f1}, + Hash: "removeF2", + By: []database.Detector{nd1, fd1, fd1}, + Features: []database.LayerFeature{f1}, } // blank -> ns1:a, f1 f2 (init) @@ -597,44 +521,65 @@ func TestComputeAncestryFeatures(t *testing.T) { // -> blank (empty) layers := []database.Layer{ - blank, - initNS1a, - removeF2, - initNS2a, - upgradeNS2b, - blank, - upgradeNS1b, - removeF2, + blank, // empty + initNS1a, // namespace: NS1a, features: f1, f2 + removeF2, // namespace: , features: f1 + initNS2a, // namespace: NS2a, features: f3, f4 ( under NS2a ) + upgradeNS2b, // namespace: NS2b, ( f3, f4 are now under NS2b ) + blank, // empty + upgradeNS1b, // namespace: NS1b, ( f1, f2 are now under NS1b, and they are introduced in this layer. ) + removeF2, // namespace: , features: f1 blank, } - expected := map[database.NamespacedFeature]bool{ + expected := []database.AncestryLayer{ + { + "blank", + []database.AncestryFeature{}, + }, { - Feature: f1, - Namespace: ns1a, - }: false, + "initNS1a", + []database.AncestryFeature{{database.NamespacedFeature{f1.Feature, ns1a.Namespace}, f1.By, ns1a.By}}, + }, { - Feature: f3, - Namespace: ns2a, - }: false, + "removeF2", + []database.AncestryFeature{}, + }, { - Feature: f4, - Namespace: ns2a, - }: false, + "initNS2a", + []database.AncestryFeature{ + {database.NamespacedFeature{f3.Feature, ns2a.Namespace}, f3.By, ns2a.By}, + {database.NamespacedFeature{f4.Feature, ns2a.Namespace}, f4.By, ns2a.By}, + }, + }, + { + "upgradeNS2b", + []database.AncestryFeature{}, + }, + { + "blank", + []database.AncestryFeature{}, + }, + { + "upgradeNS1b", + []database.AncestryFeature{}, + }, + { + "removeF2", + []database.AncestryFeature{}, + }, + { + "blank", + []database.AncestryFeature{}, + }, } - ancestryLayers, err := computeAncestryLayers(layers, database.Processors{}) - assert.Nil(t, err) - features := getNamespacedFeatures(ancestryLayers) - for _, f := range features { - if assert.Contains(t, expected, f) { - if assert.False(t, expected[f]) { - expected[f] = true - } - } - } + expectedDetectors := []database.Detector{nd1, fd1} + ancestryLayers, detectors, err := computeAncestryLayers(layers) + require.Nil(t, err) - for f, visited := range expected { - assert.True(t, visited, "expected feature is missing : "+f.Namespace.Name+":"+f.Name) + testutil.AssertDetectorsEqual(t, expectedDetectors, detectors) + for i := range expected { + testutil.AssertAncestryLayerEqual(t, &expected[i], &ancestryLayers[i]) } } From 0c1b80b2ed54dcbe227f7233468a5bdc66d4a17e Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Mon, 8 Oct 2018 11:11:30 -0400 Subject: [PATCH 11/14] pgsql: Implement database queries for detector relationship * Refactor layer and ancestry * Add tests * Fix bugs introduced when the queries were moved --- database/pgsql/ancestry.go | 346 +++++++++------- database/pgsql/ancestry_test.go | 263 +++++------- database/pgsql/complex_test.go | 4 +- database/pgsql/detector.go | 198 +++++++++ database/pgsql/detector_test.go | 119 ++++++ database/pgsql/feature.go | 17 +- database/pgsql/feature_test.go | 23 +- database/pgsql/layer.go | 383 ++++++++++-------- database/pgsql/layer_test.go | 241 +++++++---- .../pgsql/migrations/00001_initial_schema.go | 8 +- database/pgsql/namespace.go | 5 - database/pgsql/namespace_test.go | 39 -- database/pgsql/notification.go | 16 +- database/pgsql/notification_test.go | 225 +++++----- database/pgsql/pgsql.go | 3 +- database/pgsql/pgsql_test.go | 8 +- database/pgsql/queries.go | 22 +- database/pgsql/testdata/data.sql | 161 ++++---- database/pgsql/testutil.go | 263 ++++++++++++ database/pgsql/vulnerability_test.go | 4 +- pkg/testutil/testutil.go | 285 +++++++++++++ 21 files changed, 1766 insertions(+), 867 deletions(-) create mode 100644 database/pgsql/detector.go create mode 100644 database/pgsql/detector_test.go create mode 100644 database/pgsql/testutil.go create mode 100644 pkg/testutil/testutil.go diff --git a/database/pgsql/ancestry.go b/database/pgsql/ancestry.go index 36d8fcde..fa0c0ad5 100644 --- a/database/pgsql/ancestry.go +++ b/database/pgsql/ancestry.go @@ -14,15 +14,17 @@ const ( insertAncestry = ` INSERT INTO ancestry (name) VALUES ($1) RETURNING id` - searchAncestryLayer = ` - SELECT layer.hash, layer.id, ancestry_layer.ancestry_index + findAncestryLayerHashes = ` + SELECT layer.hash, ancestry_layer.ancestry_index FROM layer, ancestry_layer WHERE ancestry_layer.ancestry_id = $1 AND ancestry_layer.layer_id = layer.id ORDER BY ancestry_layer.ancestry_index ASC` - searchAncestryFeatures = ` - SELECT namespace.name, namespace.version_format, feature.name, feature.version, feature.version_format, ancestry_layer.ancestry_index + findAncestryFeatures = ` + SELECT namespace.name, namespace.version_format, feature.name, + feature.version, feature.version_format, ancestry_layer.ancestry_index, + ancestry_feature.feature_detector_id, ancestry_feature.namespace_detector_id FROM namespace, feature, namespaced_feature, ancestry_layer, ancestry_feature WHERE ancestry_layer.ancestry_id = $1 AND ancestry_feature.ancestry_layer_id = ancestry_layer.id @@ -30,203 +32,220 @@ const ( AND namespaced_feature.feature_id = feature.id AND namespaced_feature.namespace_id = namespace.id` - searchAncestry = `SELECT id FROM ancestry WHERE name = $1` - removeAncestry = `DELETE FROM ancestry WHERE name = $1` - insertAncestryLayer = ` - INSERT INTO ancestry_layer (ancestry_id, ancestry_index, layer_id) VALUES - ($1, $2, (SELECT layer.id FROM layer WHERE hash = $3 LIMIT 1)) + findAncestryID = `SELECT id FROM ancestry WHERE name = $1` + removeAncestry = `DELETE FROM ancestry WHERE name = $1` + insertAncestryLayers = ` + INSERT INTO ancestry_layer (ancestry_id, ancestry_index, layer_id) VALUES ($1, $2, $3) RETURNING id` - insertAncestryLayerFeature = ` + insertAncestryFeatures = ` INSERT INTO ancestry_feature (ancestry_layer_id, namespaced_feature_id, feature_detector_id, namespace_detector_id) VALUES ($1, $2, $3, $4)` ) -type ancestryLayerWithID struct { - database.AncestryLayer +func (tx *pgSession) FindAncestry(name string) (database.Ancestry, bool, error) { + var ( + ancestry = database.Ancestry{Name: name} + err error + ) + + id, ok, err := tx.findAncestryID(name) + if !ok || err != nil { + return ancestry, ok, err + } + + if ancestry.By, err = tx.findAncestryDetectors(id); err != nil { + return ancestry, false, err + } + + if ancestry.Layers, err = tx.findAncestryLayers(id); err != nil { + return ancestry, false, err + } - layerID int64 + return ancestry, true, nil } func (tx *pgSession) UpsertAncestry(ancestry database.Ancestry) error { - if ancestry.Name == "" { - log.Error("Empty ancestry name is not allowed") - return commonerr.NewBadRequestError("could not insert an ancestry with empty name") + if !ancestry.Valid() { + return database.ErrInvalidParameters } - if len(ancestry.Layers) == 0 { - log.Error("Empty ancestry is not allowed") - return commonerr.NewBadRequestError("could not insert an ancestry with 0 layers") + if err := tx.removeAncestry(ancestry.Name); err != nil { + return err } - if err := tx.deleteAncestry(ancestry.Name); err != nil { + id, err := tx.insertAncestry(ancestry.Name) + if err != nil { return err } - var ancestryID int64 - if err := tx.QueryRow(insertAncestry, ancestry.Name).Scan(&ancestryID); err != nil { - if isErrUniqueViolation(err) { - return handleError("insertAncestry", errors.New("other Go-routine is processing this ancestry (skip)")) - } - return handleError("insertAncestry", err) + detectorIDs, err := tx.findDetectorIDs(ancestry.By) + if err != nil { + return err } - if err := tx.insertAncestryLayers(ancestryID, ancestry.Layers); err != nil { + // insert ancestry metadata + if err := tx.insertAncestryDetectors(id, detectorIDs); err != nil { return err } - return tx.persistProcessors(persistAncestryLister, - "persistAncestryLister", - persistAncestryDetector, - "persistAncestryDetector", - ancestryID, ancestry.ProcessedBy) -} - -func (tx *pgSession) findAncestryID(name string) (int64, bool, error) { - var id sql.NullInt64 - if err := tx.QueryRow(searchAncestry, name).Scan(&id); err != nil { - if err == sql.ErrNoRows { - return 0, false, nil - } - - return 0, false, handleError("searchAncestry", err) + layers := make([]string, 0, len(ancestry.Layers)) + for _, layer := range ancestry.Layers { + layers = append(layers, layer.Hash) } - return id.Int64, true, nil -} + layerIDs, ok, err := tx.findLayerIDs(layers) + if err != nil { + return err + } -func (tx *pgSession) findAncestryProcessors(id int64) (database.Processors, error) { - var ( - processors database.Processors - err error - ) + if !ok { + log.Error("layer cannot be found, this indicates that the internal logic of calling UpsertAncestry is wrong or the database is corrupted.") + return database.ErrMissingEntities + } - if processors.Detectors, err = tx.findProcessors(searchAncestryDetectors, id); err != nil { - return processors, handleError("searchAncestryDetectors", err) + ancestryLayerIDs, err := tx.insertAncestryLayers(id, layerIDs) + if err != nil { + return err } - if processors.Listers, err = tx.findProcessors(searchAncestryListers, id); err != nil { - return processors, handleError("searchAncestryListers", err) + for i, id := range ancestryLayerIDs { + if err := tx.insertAncestryFeatures(id, ancestry.Layers[i]); err != nil { + return err + } } - return processors, err + return nil } -func (tx *pgSession) FindAncestry(name string) (database.Ancestry, bool, error) { - var ( - ancestry = database.Ancestry{Name: name} - err error - ) +func (tx *pgSession) insertAncestry(name string) (int64, error) { + var id int64 + err := tx.QueryRow(insertAncestry, name).Scan(&id) + if err != nil { + if isErrUniqueViolation(err) { + return 0, handleError("insertAncestry", errors.New("other Go-routine is processing this ancestry (skip)")) + } - id, ok, err := tx.findAncestryID(name) - if !ok || err != nil { - return ancestry, ok, err + return 0, handleError("insertAncestry", err) } - if ancestry.ProcessedBy, err = tx.findAncestryProcessors(id); err != nil { - return ancestry, false, err - } + log.WithFields(log.Fields{"ancestry": name, "id": id}).Debug("database: inserted ancestry") + return id, nil +} - if ancestry.Layers, err = tx.findAncestryLayers(id); err != nil { - return ancestry, false, err +func (tx *pgSession) findAncestryID(name string) (int64, bool, error) { + var id sql.NullInt64 + if err := tx.QueryRow(findAncestryID, name).Scan(&id); err != nil { + if err == sql.ErrNoRows { + return 0, false, nil + } + + return 0, false, handleError("findAncestryID", err) } - return ancestry, true, nil + return id.Int64, true, nil } -func (tx *pgSession) deleteAncestry(name string) error { +func (tx *pgSession) removeAncestry(name string) error { result, err := tx.Exec(removeAncestry, name) if err != nil { return handleError("removeAncestry", err) } - _, err = result.RowsAffected() + affected, err := result.RowsAffected() if err != nil { return handleError("removeAncestry", err) } + if affected != 0 { + log.WithField("ancestry", name).Debug("removed ancestry") + } + return nil } -func (tx *pgSession) findProcessors(query string, id int64) ([]string, error) { - var ( - processors []string - processor string - ) +func (tx *pgSession) findAncestryLayers(id int64) ([]database.AncestryLayer, error) { + detectors, err := tx.findAllDetectors() + if err != nil { + return nil, err + } - rows, err := tx.Query(query, id) + layerMap, err := tx.findAncestryLayerHashes(id) if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } + return nil, err + } + log.WithField("map", layerMap).Debug("found layer hashes") + featureMap, err := tx.findAncestryFeatures(id, detectors) + if err != nil { return nil, err } - for rows.Next() { - if err := rows.Scan(&processor); err != nil { - return nil, err + layers := make([]database.AncestryLayer, len(layerMap)) + for index, layer := range layerMap { + // index MUST match the ancestry layer slice index. + if layers[index].Hash == "" && len(layers[index].Features) == 0 { + layers[index] = database.AncestryLayer{ + Hash: layer, + Features: featureMap[index], + } + } else { + log.WithFields(log.Fields{ + "ancestry ID": id, + "duplicated ancestry index": index, + }).WithError(database.ErrInconsistent).Error("ancestry layers with same ancestry_index is not allowed") + return nil, database.ErrInconsistent } - - processors = append(processors, processor) } - return processors, nil + return layers, nil } -func (tx *pgSession) findAncestryLayers(id int64) ([]database.AncestryLayer, error) { - var ( - err error - rows *sql.Rows - // layer index -> Ancestry Layer + Layer ID - layers = map[int64]ancestryLayerWithID{} - // layer index -> layer-wise features - features = map[int64][]database.NamespacedFeature{} - ancestryLayers []database.AncestryLayer - ) - - // retrieve ancestry layer metadata - if rows, err = tx.Query(searchAncestryLayer, id); err != nil { - return nil, handleError("searchAncestryLayer", err) +func (tx *pgSession) findAncestryLayerHashes(ancestryID int64) (map[int64]string, error) { + // retrieve layer indexes and hashes + rows, err := tx.Query(findAncestryLayerHashes, ancestryID) + if err != nil { + return nil, handleError("findAncestryLayerHashes", err) } + layerHashes := map[int64]string{} for rows.Next() { var ( - layer database.AncestryLayer - index sql.NullInt64 - id sql.NullInt64 + hash string + index int64 ) - if err = rows.Scan(&layer.Hash, &id, &index); err != nil { - return nil, handleError("searchAncestryLayer", err) + if err = rows.Scan(&hash, &index); err != nil { + return nil, handleError("findAncestryLayerHashes", err) } - if !index.Valid || !id.Valid { - panic("null ancestry ID or ancestry index violates database constraints") - } - - if _, ok := layers[index.Int64]; ok { + if _, ok := layerHashes[index]; ok { // one ancestry index should correspond to only one layer return nil, database.ErrInconsistent } - layers[index.Int64] = ancestryLayerWithID{layer, id.Int64} + layerHashes[index] = hash } - for _, layer := range layers { - if layer.ProcessedBy, err = tx.findLayerProcessors(layer.layerID); err != nil { - return nil, err - } - } + return layerHashes, nil +} +func (tx *pgSession) findAncestryFeatures(ancestryID int64, detectors detectorMap) (map[int64][]database.AncestryFeature, error) { + // ancestry_index -> ancestry features + featureMap := make(map[int64][]database.AncestryFeature) // retrieve ancestry layer's namespaced features - if rows, err = tx.Query(searchAncestryFeatures, id); err != nil { - return nil, handleError("searchAncestryFeatures", err) + rows, err := tx.Query(findAncestryFeatures, ancestryID) + if err != nil { + return nil, handleError("findAncestryFeatures", err) } + defer rows.Close() + for rows.Next() { var ( - feature database.NamespacedFeature + featureDetectorID int64 + namespaceDetectorID int64 + feature database.NamespacedFeature // index is used to determine which layer the feature belongs to. index sql.NullInt64 ) @@ -238,8 +257,10 @@ func (tx *pgSession) findAncestryLayers(id int64) ([]database.AncestryLayer, err &feature.Feature.Version, &feature.Feature.VersionFormat, &index, + &featureDetectorID, + &namespaceDetectorID, ); err != nil { - return nil, handleError("searchAncestryFeatures", err) + return nil, handleError("findAncestryFeatures", err) } if feature.Feature.VersionFormat != feature.Namespace.VersionFormat { @@ -248,59 +269,88 @@ func (tx *pgSession) findAncestryLayers(id int64) ([]database.AncestryLayer, err return nil, database.ErrInconsistent } - features[index.Int64] = append(features[index.Int64], feature) - } + fDetector, ok := detectors.byID[featureDetectorID] + if !ok { + return nil, database.ErrInconsistent + } + + nsDetector, ok := detectors.byID[namespaceDetectorID] + if !ok { + return nil, database.ErrInconsistent + } - for index, layer := range layers { - layer.DetectedFeatures = features[index] - ancestryLayers = append(ancestryLayers, layer.AncestryLayer) + featureMap[index.Int64] = append(featureMap[index.Int64], database.AncestryFeature{ + NamespacedFeature: feature, + FeatureBy: fDetector, + NamespaceBy: nsDetector, + }) } - return ancestryLayers, nil + return featureMap, nil } // insertAncestryLayers inserts the ancestry layers along with its content into // the database. The layers are 0 based indexed in the original order. -func (tx *pgSession) insertAncestryLayers(ancestryID int64, layers []database.AncestryLayer) error { - //TODO(Sida): use bulk insert. - stmt, err := tx.Prepare(insertAncestryLayer) +func (tx *pgSession) insertAncestryLayers(ancestryID int64, layers []int64) ([]int64, error) { + stmt, err := tx.Prepare(insertAncestryLayers) if err != nil { - return handleError("insertAncestryLayer", err) + return nil, handleError("insertAncestryLayers", err) } - ancestryLayerIDs := []sql.NullInt64{} - for index, layer := range layers { + ancestryLayerIDs := []int64{} + for index, layerID := range layers { var ancestryLayerID sql.NullInt64 - if err := stmt.QueryRow(ancestryID, index, layer.Hash).Scan(&ancestryLayerID); err != nil { - return handleError("insertAncestryLayer", commonerr.CombineErrors(err, stmt.Close())) + if err := stmt.QueryRow(ancestryID, index, layerID).Scan(&ancestryLayerID); err != nil { + return nil, handleError("insertAncestryLayers", commonerr.CombineErrors(err, stmt.Close())) + } + + if !ancestryLayerID.Valid { + return nil, database.ErrInconsistent } - ancestryLayerIDs = append(ancestryLayerIDs, ancestryLayerID) + ancestryLayerIDs = append(ancestryLayerIDs, ancestryLayerID.Int64) } if err := stmt.Close(); err != nil { - return handleError("Failed to close insertAncestryLayer statement", err) + return nil, handleError("insertAncestryLayers", err) } - stmt, err = tx.Prepare(insertAncestryLayerFeature) - defer stmt.Close() + return ancestryLayerIDs, nil +} - for i, layer := range layers { - var ( - nsFeatureIDs []sql.NullInt64 - layerID = ancestryLayerIDs[i] - ) +func (tx *pgSession) insertAncestryFeatures(ancestryLayerID int64, layer database.AncestryLayer) error { + detectors, err := tx.findAllDetectors() + if err != nil { + return err + } - if nsFeatureIDs, err = tx.findNamespacedFeatureIDs(layer.DetectedFeatures); err != nil { - return err + nsFeatureIDs, err := tx.findNamespacedFeatureIDs(layer.GetFeatures()) + if err != nil { + return err + } + + // find the detectors for each feature + stmt, err := tx.Prepare(insertAncestryFeatures) + if err != nil { + return handleError("insertAncestryFeatures", err) + } + + defer stmt.Close() + + for index, id := range nsFeatureIDs { + namespaceDetectorID, ok := detectors.byValue[layer.Features[index].NamespaceBy] + if !ok { + return database.ErrMissingEntities } - for _, id := range nsFeatureIDs { - if _, err := stmt.Exec(layerID, id); err != nil { - return handleError("insertAncestryLayerFeature", commonerr.CombineErrors(err, stmt.Close())) - } + featureDetectorID, ok := detectors.byValue[layer.Features[index].FeatureBy] + if !ok { + return database.ErrMissingEntities } + if _, err := stmt.Exec(ancestryLayerID, id, featureDetectorID, namespaceDetectorID); err != nil { + return handleError("insertAncestryFeatures", commonerr.CombineErrors(err, stmt.Close())) + } } return nil diff --git a/database/pgsql/ancestry_test.go b/database/pgsql/ancestry_test.go index 9d1f1c5c..6cceb718 100644 --- a/database/pgsql/ancestry_test.go +++ b/database/pgsql/ancestry_test.go @@ -15,198 +15,125 @@ package pgsql import ( - "sort" "testing" "github.com/stretchr/testify/assert" "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/testutil" ) -func TestUpsertAncestry(t *testing.T) { - store, tx := openSessionForTest(t, "UpsertAncestry", true) - defer closeTest(t, store, tx) - a1 := database.Ancestry{ - Name: "a1", - Layers: []database.AncestryLayer{ - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-N", +var upsertAncestryTests = []struct { + in *database.Ancestry + err string + title string +}{ + { + title: "ancestry with invalid layer", + in: &database.Ancestry{ + Name: "a1", + Layers: []database.AncestryLayer{ + { + Hash: "layer-non-existing", }, }, }, - } - - a2 := database.Ancestry{} - - a3 := database.Ancestry{ - Name: "a", - Layers: []database.AncestryLayer{ - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-0", - }, - }, + err: database.ErrMissingEntities.Error(), + }, + { + title: "ancestry with invalid name", + in: &database.Ancestry{}, + err: database.ErrInvalidParameters.Error(), + }, + { + title: "new valid ancestry", + in: &database.Ancestry{ + Name: "a", + Layers: []database.AncestryLayer{{Hash: "layer-0"}}, }, - } - - a4 := database.Ancestry{ - Name: "a", - Layers: []database.AncestryLayer{ - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-1", - }, + }, + { + title: "ancestry with invalid feature", + in: &database.Ancestry{ + Name: "a", + By: []database.Detector{realDetectors[1], realDetectors[2]}, + Layers: []database.AncestryLayer{{Hash: "layer-1", Features: []database.AncestryFeature{ + {fakeNamespacedFeatures[1], fakeDetector[1], fakeDetector[2]}, + }}}, + }, + err: database.ErrMissingEntities.Error(), + }, + { + title: "replace old ancestry", + in: &database.Ancestry{ + Name: "a", + By: []database.Detector{realDetectors[1], realDetectors[2]}, + Layers: []database.AncestryLayer{ + {"layer-1", []database.AncestryFeature{{realNamespacedFeatures[1], realDetectors[2], realDetectors[1]}}}, }, }, - } - - f1 := database.Feature{ - Name: "wechat", - Version: "0.5", - VersionFormat: "dpkg", - } - - // not in database - f2 := database.Feature{ - Name: "wechat", - Version: "0.6", - VersionFormat: "dpkg", - } - - n1 := database.Namespace{ - Name: "debian:7", - VersionFormat: "dpkg", - } - - p := database.Processors{ - Listers: []string{"dpkg", "non-existing"}, - Detectors: []string{"os-release", "non-existing"}, - } - - nsf1 := database.NamespacedFeature{ - Namespace: n1, - Feature: f1, - } - - // not in database - nsf2 := database.NamespacedFeature{ - Namespace: n1, - Feature: f2, - } - - a4.ProcessedBy = p - // invalid case - assert.NotNil(t, tx.UpsertAncestry(a1)) - assert.NotNil(t, tx.UpsertAncestry(a2)) - // valid case - assert.Nil(t, tx.UpsertAncestry(a3)) - a4.Layers[0].DetectedFeatures = []database.NamespacedFeature{nsf1, nsf2} - // replace invalid case - assert.NotNil(t, tx.UpsertAncestry(a4)) - a4.Layers[0].DetectedFeatures = []database.NamespacedFeature{nsf1} - // replace valid case - assert.Nil(t, tx.UpsertAncestry(a4)) - // validate - ancestry, ok, err := tx.FindAncestry("a") - assert.Nil(t, err) - assert.True(t, ok) - assertAncestryEqual(t, a4, ancestry) -} - -func assertProcessorsEqual(t *testing.T, expected database.Processors, actual database.Processors) bool { - sort.Strings(expected.Detectors) - sort.Strings(actual.Detectors) - sort.Strings(expected.Listers) - sort.Strings(actual.Listers) - return assert.Equal(t, expected.Detectors, actual.Detectors) && assert.Equal(t, expected.Listers, actual.Listers) + }, } -func assertAncestryEqual(t *testing.T, expected database.Ancestry, actual database.Ancestry) bool { - assert.Equal(t, expected.Name, actual.Name) - assertProcessorsEqual(t, expected.ProcessedBy, actual.ProcessedBy) - if assert.Equal(t, len(expected.Layers), len(actual.Layers)) { - for index, layer := range expected.Layers { - if !assertAncestryLayerEqual(t, layer, actual.Layers[index]) { - return false +func TestUpsertAncestry(t *testing.T) { + store, tx := openSessionForTest(t, "UpsertAncestry", true) + defer closeTest(t, store, tx) + for _, test := range upsertAncestryTests { + t.Run(test.title, func(t *testing.T) { + err := tx.UpsertAncestry(*test.in) + if test.err != "" { + assert.EqualError(t, err, test.err, "unexpected error") + return } - } - return true + assert.Nil(t, err) + actual, ok, err := tx.FindAncestry(test.in.Name) + assert.Nil(t, err) + assert.True(t, ok) + testutil.AssertAncestryEqual(t, test.in, &actual) + }) } - return false } -func assertAncestryLayerEqual(t *testing.T, expected database.AncestryLayer, actual database.AncestryLayer) bool { - return assertLayerEqual(t, expected.LayerMetadata, actual.LayerMetadata) && - assertNamespacedFeatureEqual(t, expected.DetectedFeatures, actual.DetectedFeatures) +var findAncestryTests = []struct { + title string + in string + + ancestry *database.Ancestry + err string + ok bool +}{ + { + title: "missing ancestry", + in: "ancestry-non", + err: "", + ancestry: nil, + ok: false, + }, + { + title: "valid ancestry", + in: "ancestry-2", + err: "", + ok: true, + ancestry: takeAncestryPointerFromMap(realAncestries, 2), + }, } func TestFindAncestry(t *testing.T) { store, tx := openSessionForTest(t, "FindAncestry", true) defer closeTest(t, store, tx) + for _, test := range findAncestryTests { + t.Run(test.title, func(t *testing.T) { + ancestry, ok, err := tx.FindAncestry(test.in) + if test.err != "" { + assert.EqualError(t, err, test.err, "unexpected error") + return + } - // invalid - _, ok, err := tx.FindAncestry("ancestry-non") - if assert.Nil(t, err) { - assert.False(t, ok) - } - - expected := database.Ancestry{ - Name: "ancestry-2", - ProcessedBy: database.Processors{ - Detectors: []string{"os-release"}, - Listers: []string{"dpkg"}, - }, - Layers: []database.AncestryLayer{ - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-0", - }, - DetectedFeatures: []database.NamespacedFeature{ - { - Namespace: database.Namespace{ - Name: "debian:7", - VersionFormat: "dpkg", - }, - Feature: database.Feature{ - Name: "wechat", - Version: "0.5", - VersionFormat: "dpkg", - }, - }, - { - Namespace: database.Namespace{ - Name: "debian:8", - VersionFormat: "dpkg", - }, - Feature: database.Feature{ - Name: "openssl", - Version: "1.0", - VersionFormat: "dpkg", - }, - }, - }, - }, - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-1", - }, - }, - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-2", - }, - }, - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-3b", - }, - }, - }, - } - // valid - ancestry, ok, err := tx.FindAncestry("ancestry-2") - if assert.Nil(t, err) && assert.True(t, ok) { - assertAncestryEqual(t, expected, ancestry) + assert.Nil(t, err) + assert.Equal(t, test.ok, ok) + if test.ok { + testutil.AssertAncestryEqual(t, test.ancestry, &ancestry) + } + }) } } diff --git a/database/pgsql/complex_test.go b/database/pgsql/complex_test.go index 07d6f55f..de8b0f20 100644 --- a/database/pgsql/complex_test.go +++ b/database/pgsql/complex_test.go @@ -220,7 +220,7 @@ func TestCaching(t *testing.T) { actualAffectedNames = append(actualAffectedNames, s.Name) } - assert.Len(t, strutil.CompareStringLists(expectedAffectedNames, actualAffectedNames), 0) - assert.Len(t, strutil.CompareStringLists(actualAffectedNames, expectedAffectedNames), 0) + assert.Len(t, strutil.Difference(expectedAffectedNames, actualAffectedNames), 0) + assert.Len(t, strutil.Difference(actualAffectedNames, expectedAffectedNames), 0) } } diff --git a/database/pgsql/detector.go b/database/pgsql/detector.go new file mode 100644 index 00000000..99b5c283 --- /dev/null +++ b/database/pgsql/detector.go @@ -0,0 +1,198 @@ +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pgsql + +import ( + "database/sql" + + "github.com/deckarep/golang-set" + log "github.com/sirupsen/logrus" + + "github.com/coreos/clair/database" +) + +const ( + soiDetector = ` + INSERT INTO detector (name, version, dtype) + SELECT CAST ($1 AS TEXT), CAST ($2 AS TEXT), CAST ($3 AS detector_type ) + WHERE NOT EXISTS (SELECT id FROM detector WHERE name = $1 AND version = $2 AND dtype = $3);` + + selectAncestryDetectors = ` + SELECT d.name, d.version, d.dtype + FROM ancestry_detector, detector AS d + WHERE ancestry_detector.detector_id = d.id AND ancestry_detector.ancestry_id = $1;` + + selectLayerDetectors = ` + SELECT d.name, d.version, d.dtype + FROM layer_detector, detector AS d + WHERE layer_detector.detector_id = d.id AND layer_detector.layer_id = $1;` + + insertAncestryDetectors = ` + INSERT INTO ancestry_detector (ancestry_id, detector_id) + SELECT $1, $2 + WHERE NOT EXISTS (SELECT id FROM ancestry_detector WHERE ancestry_id = $1 AND detector_id = $2)` + + persistLayerDetector = ` + INSERT INTO layer_detector (layer_id, detector_id) + SELECT $1, $2 + WHERE NOT EXISTS (SELECT id FROM layer_detector WHERE layer_id = $1 AND detector_id = $2)` + + findDetectorID = `SELECT id FROM detector WHERE name = $1 AND version = $2 AND dtype = $3` + findAllDetectors = `SELECT id, name, version, dtype FROM detector` +) + +type detectorMap struct { + byID map[int64]database.Detector + byValue map[database.Detector]int64 +} + +func (tx *pgSession) PersistDetectors(detectors []database.Detector) error { + for _, d := range detectors { + if !d.Valid() { + log.WithField("detector", d).Debug("Invalid Detector") + return database.ErrInvalidParameters + } + + r, err := tx.Exec(soiDetector, d.Name, d.Version, d.DType) + if err != nil { + return handleError("soiDetector", err) + } + + count, err := r.RowsAffected() + if err != nil { + return handleError("soiDetector", err) + } + + if count == 0 { + log.Debug("detector already exists: ", d) + } + } + + return nil +} + +func (tx *pgSession) persistLayerDetector(layerID int64, detectorID int64) error { + if _, err := tx.Exec(persistLayerDetector, layerID, detectorID); err != nil { + return handleError("persistLayerDetector", err) + } + + return nil +} + +func (tx *pgSession) persistLayerDetectors(layerID int64, detectorIDs []int64) error { + alreadySaved := mapset.NewSet() + for _, id := range detectorIDs { + if alreadySaved.Contains(id) { + continue + } + + alreadySaved.Add(id) + if err := tx.persistLayerDetector(layerID, id); err != nil { + return err + } + } + + return nil +} + +func (tx *pgSession) insertAncestryDetectors(ancestryID int64, detectorIDs []int64) error { + for _, detectorID := range detectorIDs { + if _, err := tx.Exec(insertAncestryDetectors, ancestryID, detectorID); err != nil { + return handleError("insertAncestryDetectors", err) + } + } + + return nil +} + +func (tx *pgSession) findAncestryDetectors(id int64) ([]database.Detector, error) { + detectors, err := tx.getDetectors(selectAncestryDetectors, id) + log.WithField("detectors", detectors).Debug("found ancestry detectors") + return detectors, err +} + +func (tx *pgSession) findLayerDetectors(id int64) ([]database.Detector, error) { + detectors, err := tx.getDetectors(selectLayerDetectors, id) + log.WithField("detectors", detectors).Debug("found layer detectors") + return detectors, err +} + +// findDetectorIDs retrieve ids of the detectors from the database, if any is not +// found, return the error. +func (tx *pgSession) findDetectorIDs(detectors []database.Detector) ([]int64, error) { + ids := []int64{} + for _, d := range detectors { + id := sql.NullInt64{} + err := tx.QueryRow(findDetectorID, d.Name, d.Version, d.DType).Scan(&id) + if err != nil { + return nil, handleError("findDetectorID", err) + } + + if !id.Valid { + return nil, database.ErrInconsistent + } + + ids = append(ids, id.Int64) + } + + return ids, nil +} + +func (tx *pgSession) getDetectors(query string, id int64) ([]database.Detector, error) { + rows, err := tx.Query(query, id) + if err != nil { + return nil, handleError("getDetectors", err) + } + + detectors := []database.Detector{} + for rows.Next() { + d := database.Detector{} + err := rows.Scan(&d.Name, &d.Version, &d.DType) + if err != nil { + return nil, handleError("getDetectors", err) + } + + if !d.Valid() { + return nil, database.ErrInvalidDetector + } + + detectors = append(detectors, d) + } + + return detectors, nil +} + +func (tx *pgSession) findAllDetectors() (detectorMap, error) { + rows, err := tx.Query(findAllDetectors) + if err != nil { + return detectorMap{}, handleError("searchAllDetectors", err) + } + + detectors := detectorMap{byID: make(map[int64]database.Detector), byValue: make(map[database.Detector]int64)} + for rows.Next() { + var ( + id int64 + d database.Detector + ) + if err := rows.Scan(&id, &d.Name, &d.Version, &d.DType); err != nil { + return detectorMap{}, handleError("searchAllDetectors", err) + } + + detectors.byID[id] = d + detectors.byValue[d] = id + } + + return detectors, nil +} diff --git a/database/pgsql/detector_test.go b/database/pgsql/detector_test.go new file mode 100644 index 00000000..582da60b --- /dev/null +++ b/database/pgsql/detector_test.go @@ -0,0 +1,119 @@ +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pgsql + +import ( + "testing" + + "github.com/deckarep/golang-set" + "github.com/stretchr/testify/require" + + "github.com/coreos/clair/database" +) + +func testGetAllDetectors(tx *pgSession) []database.Detector { + query := `SELECT name, version, dtype FROM detector` + rows, err := tx.Query(query) + if err != nil { + panic(err) + } + + detectors := []database.Detector{} + for rows.Next() { + d := database.Detector{} + if err := rows.Scan(&d.Name, &d.Version, &d.DType); err != nil { + panic(err) + } + + detectors = append(detectors, d) + } + + return detectors +} + +var persistDetectorTests = []struct { + title string + in []database.Detector + err string +}{ + { + title: "invalid detector", + in: []database.Detector{ + {}, + database.NewFeatureDetector("name", "2.0"), + }, + err: database.ErrInvalidParameters.Error(), + }, + { + title: "invalid detector 2", + in: []database.Detector{ + database.NewFeatureDetector("name", "2.0"), + {"name", "1.0", "random not valid dtype"}, + }, + err: database.ErrInvalidParameters.Error(), + }, + { + title: "detectors with some different fields", + in: []database.Detector{ + database.NewFeatureDetector("name", "2.0"), + database.NewFeatureDetector("name", "1.0"), + database.NewNamespaceDetector("name", "1.0"), + }, + }, + { + title: "duplicated detectors (parameter level)", + in: []database.Detector{ + database.NewFeatureDetector("name", "1.0"), + database.NewFeatureDetector("name", "1.0"), + }, + }, + { + title: "duplicated detectors (db level)", + in: []database.Detector{ + database.NewNamespaceDetector("os-release", "1.0"), + database.NewNamespaceDetector("os-release", "1.0"), + database.NewFeatureDetector("dpkg", "1.0"), + }, + }, +} + +func TestPersistDetector(t *testing.T) { + datastore, tx := openSessionForTest(t, "PersistDetector", true) + defer closeTest(t, datastore, tx) + + for _, test := range persistDetectorTests { + t.Run(test.title, func(t *testing.T) { + err := tx.PersistDetectors(test.in) + if test.err != "" { + require.EqualError(t, err, test.err) + return + } + + detectors := testGetAllDetectors(tx) + + // ensure no duplicated detectors + detectorSet := mapset.NewSet() + for _, d := range detectors { + require.False(t, detectorSet.Contains(d), "duplicated: %v", d) + detectorSet.Add(d) + } + + // ensure all persisted detectors are actually saved + for _, d := range test.in { + require.True(t, detectorSet.Contains(d), "detector: %v, detectors: %v", d, detectorSet) + } + }) + } +} diff --git a/database/pgsql/feature.go b/database/pgsql/feature.go index e1c0781c..345b73a3 100644 --- a/database/pgsql/feature.go +++ b/database/pgsql/feature.go @@ -16,7 +16,6 @@ package pgsql import ( "database/sql" - "errors" "sort" "github.com/lib/pq" @@ -28,7 +27,6 @@ import ( ) const ( - // feature.go soiNamespacedFeature = ` WITH new_feature_ns AS ( INSERT INTO namespaced_feature(feature_id, namespace_id) @@ -65,15 +63,6 @@ const ( AND v.deleted_at IS NULL` ) -var ( - errFeatureNotFound = errors.New("Feature not found") -) - -type vulnerabilityAffecting struct { - vulnerabilityID int64 - addedByID int64 -} - func (tx *pgSession) PersistFeatures(features []database.Feature) error { if len(features) == 0 { return nil @@ -126,7 +115,7 @@ func (tx *pgSession) searchAffectingVulnerabilities(features []database.Namespac fMap := map[int64]database.NamespacedFeature{} for i, f := range features { if !ids[i].Valid { - return nil, errFeatureNotFound + return nil, database.ErrMissingEntities } fMap[ids[i].Int64] = f } @@ -218,7 +207,7 @@ func (tx *pgSession) PersistNamespacedFeatures(features []database.NamespacedFea if ids, err := tx.findFeatureIDs(fToFind); err == nil { for i, id := range ids { if !id.Valid { - return errFeatureNotFound + return database.ErrMissingEntities } fIDs[fToFind[i]] = id } @@ -234,7 +223,7 @@ func (tx *pgSession) PersistNamespacedFeatures(features []database.NamespacedFea if ids, err := tx.findNamespaceIDs(nsToFind); err == nil { for i, id := range ids { if !id.Valid { - return errNamespaceNotFound + return database.ErrMissingEntities } nsIDs[nsToFind[i]] = id } diff --git a/database/pgsql/feature_test.go b/database/pgsql/feature_test.go index 934b8cc1..2823e1e8 100644 --- a/database/pgsql/feature_test.go +++ b/database/pgsql/feature_test.go @@ -52,7 +52,7 @@ func TestPersistNamespacedFeatures(t *testing.T) { // existing features f1 := database.Feature{ - Name: "wechat", + Name: "ourchat", Version: "0.5", VersionFormat: "dpkg", } @@ -213,27 +213,6 @@ func listFeatures(t *testing.T, tx *pgSession) []database.Feature { return fs } -func assertFeaturesEqual(t *testing.T, expected []database.Feature, actual []database.Feature) bool { - if assert.Len(t, actual, len(expected)) { - has := map[database.Feature]bool{} - for _, nf := range expected { - has[nf] = false - } - - for _, nf := range actual { - has[nf] = true - } - - for nf, visited := range has { - if !assert.True(t, visited, nf.Name+" is expected") { - return false - } - return true - } - } - return false -} - func assertNamespacedFeatureEqual(t *testing.T, expected []database.NamespacedFeature, actual []database.NamespacedFeature) bool { if assert.Len(t, actual, len(expected)) { has := map[database.NamespacedFeature]bool{} diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go index e474164e..ebea1849 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer.go @@ -18,6 +18,8 @@ import ( "database/sql" "sort" + "github.com/deckarep/golang-set" + "github.com/coreos/clair/database" "github.com/coreos/clair/pkg/commonerr" ) @@ -34,300 +36,331 @@ const ( UNION SELECT id FROM layer WHERE hash = $1` - searchLayerFeatures = ` - SELECT feature_id, detector_id - FROM layer_feature - WHERE layer_id = $1` + findLayerFeatures = ` + SELECT f.name, f.version, f.version_format, lf.detector_id + FROM layer_feature AS lf, feature AS f + WHERE lf.feature_id = f.id + AND lf.layer_id = $1` - searchLayerNamespaces = ` - SELECT namespace.Name, namespace.version_format - FROM namespace, layer_namespace - WHERE layer_namespace.layer_id = $1 - AND layer_namespace.namespace_id = namespace.id` + findLayerNamespaces = ` + SELECT ns.name, ns.version_format, ln.detector_id + FROM layer_namespace AS ln, namespace AS ns + WHERE ln.namespace_id = ns.id + AND ln.layer_id = $1` - searchLayer = `SELECT id FROM layer WHERE hash = $1` + findLayerID = `SELECT id FROM layer WHERE hash = $1` ) + +// dbLayerNamespace represents the layer_namespace table. +type dbLayerNamespace struct { + layerID int64 + namespaceID int64 + detectorID int64 +} + +// dbLayerFeature represents the layer_feature table +type dbLayerFeature struct { + layerID int64 + featureID int64 + detectorID int64 +} + +func (tx *pgSession) FindLayer(hash string) (database.Layer, bool, error) { + layer := database.Layer{Hash: hash} + if hash == "" { + return layer, false, commonerr.NewBadRequestError("non empty layer hash is expected.") + } + + layerID, ok, err := tx.findLayerID(hash) + if err != nil || !ok { + return layer, ok, err + } + + detectorMap, err := tx.findAllDetectors() if err != nil { return layer, false, err } - if !ok { - return layer, false, nil + if layer.By, err = tx.findLayerDetectors(layerID); err != nil { + return layer, false, err + } + + if layer.Features, err = tx.findLayerFeatures(layerID, detectorMap); err != nil { + return layer, false, err + } + + if layer.Namespaces, err = tx.findLayerNamespaces(layerID, detectorMap); err != nil { + return layer, false, err } - layer.Features, err = tx.findLayerFeatures(layerID) - layer.Namespaces, err = tx.findLayerNamespaces(layerID) return layer, true, nil } -func (tx *pgSession) persistLayer(hash string) (int64, error) { +func sanitizePersistLayerInput(hash string, features []database.LayerFeature, namespaces []database.LayerNamespace, detectedBy []database.Detector) error { if hash == "" { - return -1, commonerr.NewBadRequestError("Empty Layer Hash is not allowed") + return commonerr.NewBadRequestError("expected non-empty layer hash") } - id := sql.NullInt64{} - if err := tx.QueryRow(soiLayer, hash).Scan(&id); err != nil { - return -1, handleError("queryPersistLayer", err) + detectedBySet := mapset.NewSet() + for _, d := range detectedBy { + detectedBySet.Add(d) } - if !id.Valid { - panic("null layer.id violates database constraint") + for _, f := range features { + if !detectedBySet.Contains(f.By) { + return database.ErrInvalidParameters + } } - return id.Int64, nil -} - -// PersistLayer relates layer identified by hash with namespaces, -// features and processors provided. If the layer, namespaces, features are not -// in database, the function returns an error. -func (tx *pgSession) PersistLayer(hash string, namespaces []database.Namespace, features []database.Feature, processedBy database.Processors) error { - if hash == "" { - return commonerr.NewBadRequestError("Empty layer hash is not allowed") + for _, n := range namespaces { + if !detectedBySet.Contains(n.By) { + return database.ErrInvalidParameters + } } + return nil +} + +// PersistLayer saves the content of a layer to the database. +func (tx *pgSession) PersistLayer(hash string, features []database.LayerFeature, namespaces []database.LayerNamespace, detectedBy []database.Detector) error { var ( - err error - id int64 + err error + id int64 + detectorIDs []int64 ) - if id, err = tx.persistLayer(hash); err != nil { + if err = sanitizePersistLayerInput(hash, features, namespaces, detectedBy); err != nil { return err } - if err = tx.persistLayerNamespace(id, namespaces); err != nil { + if id, err = tx.soiLayer(hash); err != nil { return err } - if err = tx.persistLayerFeatures(id, features); err != nil { + if detectorIDs, err = tx.findDetectorIDs(detectedBy); err != nil { + if err == commonerr.ErrNotFound { + return database.ErrMissingEntities + } + return err } - if err = tx.persistLayerDetectors(id, processedBy.Detectors); err != nil { + if err = tx.persistLayerDetectors(id, detectorIDs); err != nil { return err } - if err = tx.persistLayerListers(id, processedBy.Listers); err != nil { + if err = tx.persistAllLayerFeatures(id, features); err != nil { + return err + } + + if err = tx.persistAllLayerNamespaces(id, namespaces); err != nil { return err } return nil } -func (tx *pgSession) persistLayerDetectors(id int64, detectors []string) error { - if len(detectors) == 0 { - return nil +func (tx *pgSession) persistAllLayerNamespaces(layerID int64, namespaces []database.LayerNamespace) error { + detectorMap, err := tx.findAllDetectors() + if err != nil { + return err } - // Sorting is needed before inserting into database to prevent deadlock. - sort.Strings(detectors) - keys := make([]interface{}, len(detectors)*2) - for i, d := range detectors { - keys[i*2] = id - keys[i*2+1] = d + // TODO(sidac): This kind of type conversion is very useless and wasteful, + // we need interfaces around the database models to reduce these kind of + // operations. + rawNamespaces := make([]database.Namespace, 0, len(namespaces)) + for _, ns := range namespaces { + rawNamespaces = append(rawNamespaces, ns.Namespace) } - _, err := tx.Exec(queryPersistLayerDetectors(len(detectors)), keys...) + + rawNamespaceIDs, err := tx.findNamespaceIDs(rawNamespaces) if err != nil { - return handleError("queryPersistLayerDetectors", err) + return err } - return nil -} -func (tx *pgSession) persistLayerListers(id int64, listers []string) error { - if len(listers) == 0 { - return nil - } + dbLayerNamespaces := make([]dbLayerNamespace, 0, len(namespaces)) + for i, ns := range namespaces { + detectorID := detectorMap.byValue[ns.By] + namespaceID := rawNamespaceIDs[i].Int64 + if !rawNamespaceIDs[i].Valid { + return database.ErrMissingEntities + } - sort.Strings(listers) - keys := make([]interface{}, len(listers)*2) - for i, d := range listers { - keys[i*2] = id - keys[i*2+1] = d + dbLayerNamespaces = append(dbLayerNamespaces, dbLayerNamespace{layerID, namespaceID, detectorID}) } - _, err := tx.Exec(queryPersistLayerListers(len(listers)), keys...) + return tx.persistLayerNamespaces(dbLayerNamespaces) +} + +func (tx *pgSession) persistAllLayerFeatures(layerID int64, features []database.LayerFeature) error { + detectorMap, err := tx.findAllDetectors() if err != nil { - return handleError("queryPersistLayerDetectors", err) + return err } - return nil -} -func (tx *pgSession) persistLayerFeatures(id int64, features []database.Feature) error { - if len(features) == 0 { - return nil + rawFeatures := make([]database.Feature, 0, len(features)) + for _, f := range features { + rawFeatures = append(rawFeatures, f.Feature) } - fIDs, err := tx.findFeatureIDs(features) + featureIDs, err := tx.findFeatureIDs(rawFeatures) if err != nil { return err } - ids := make([]int, len(fIDs)) - for i, fID := range fIDs { - if !fID.Valid { - return errNamespaceNotFound + dbFeatures := make([]dbLayerFeature, 0, len(features)) + for i, f := range features { + detectorID := detectorMap.byValue[f.By] + featureID := featureIDs[i].Int64 + if !featureIDs[i].Valid { + return database.ErrMissingEntities } - ids[i] = int(fID.Int64) - } - sort.IntSlice(ids).Sort() - keys := make([]interface{}, len(features)*2) - for i, fID := range ids { - keys[i*2] = id - keys[i*2+1] = fID + dbFeatures = append(dbFeatures, dbLayerFeature{layerID, featureID, detectorID}) } - _, err = tx.Exec(queryPersistLayerFeature(len(features)), keys...) - if err != nil { - return handleError("queryPersistLayerFeature", err) + if err := tx.persistLayerFeatures(dbFeatures); err != nil { + return err } + return nil } -func (tx *pgSession) persistLayerNamespace(id int64, namespaces []database.Namespace) error { - if len(namespaces) == 0 { +func (tx *pgSession) persistLayerFeatures(features []dbLayerFeature) error { + if len(features) == 0 { return nil } - nsIDs, err := tx.findNamespaceIDs(namespaces) - if err != nil { - return err - } - - // for every bulk persist operation, the input data should be sorted. - ids := make([]int, len(nsIDs)) - for i, nsID := range nsIDs { - if !nsID.Valid { - panic(errNamespaceNotFound) - } - ids[i] = int(nsID.Int64) - } - - sort.IntSlice(ids).Sort() + sort.Slice(features, func(i, j int) bool { + return features[i].featureID < features[j].featureID + }) - keys := make([]interface{}, len(namespaces)*2) - for i, nsID := range ids { - keys[i*2] = id - keys[i*2+1] = nsID + keys := make([]interface{}, len(features)*3) + for i, feature := range features { + keys[i*3] = feature.layerID + keys[i*3+1] = feature.featureID + keys[i*3+2] = feature.detectorID } - _, err = tx.Exec(queryPersistLayerNamespace(len(namespaces)), keys...) + _, err := tx.Exec(queryPersistLayerFeature(len(features)), keys...) if err != nil { - return handleError("queryPersistLayerNamespace", err) + return handleError("queryPersistLayerFeature", err) } return nil } -func (tx *pgSession) persistProcessors(listerQuery, listerQueryName, detectorQuery, detectorQueryName string, id int64, processors database.Processors) error { - stmt, err := tx.Prepare(listerQuery) - if err != nil { - return handleError(listerQueryName, err) +func (tx *pgSession) persistLayerNamespaces(namespaces []dbLayerNamespace) error { + if len(namespaces) == 0 { + return nil } - for _, l := range processors.Listers { - _, err := stmt.Exec(id, l) - if err != nil { - stmt.Close() - return handleError(listerQueryName, err) - } - } + // for every bulk persist operation, the input data should be sorted. + sort.Slice(namespaces, func(i, j int) bool { + return namespaces[i].namespaceID < namespaces[j].namespaceID + }) - if err := stmt.Close(); err != nil { - return handleError(listerQueryName, err) + elementSize := 3 + keys := make([]interface{}, len(namespaces)*elementSize) + for i, row := range namespaces { + keys[i*3] = row.layerID + keys[i*3+1] = row.namespaceID + keys[i*3+2] = row.detectorID } - stmt, err = tx.Prepare(detectorQuery) + _, err := tx.Exec(queryPersistLayerNamespace(len(namespaces)), keys...) if err != nil { - return handleError(detectorQueryName, err) - } - - for _, d := range processors.Detectors { - _, err := stmt.Exec(id, d) - if err != nil { - stmt.Close() - return handleError(detectorQueryName, err) - } - } - - if err := stmt.Close(); err != nil { - return handleError(detectorQueryName, err) + return handleError("queryPersistLayerNamespace", err) } return nil } -func (tx *pgSession) findLayerNamespaces(layerID int64) ([]database.Namespace, error) { - var namespaces []database.Namespace - - rows, err := tx.Query(searchLayerNamespaces, layerID) +func (tx *pgSession) findLayerNamespaces(layerID int64, detectors detectorMap) ([]database.LayerNamespace, error) { + rows, err := tx.Query(findLayerNamespaces, layerID) if err != nil { - return nil, handleError("searchLayerFeatures", err) + return nil, handleError("findLayerNamespaces", err) } + namespaces := []database.LayerNamespace{} for rows.Next() { - ns := database.Namespace{} - err := rows.Scan(&ns.Name, &ns.VersionFormat) - if err != nil { + var ( + namespace database.LayerNamespace + detectorID int64 + ) + + if err := rows.Scan(&namespace.Name, &namespace.VersionFormat, &detectorID); err != nil { return nil, err } - namespaces = append(namespaces, ns) + + namespace.By = detectors.byID[detectorID] + namespaces = append(namespaces, namespace) } + return namespaces, nil } -func (tx *pgSession) findLayerFeatures(layerID int64) ([]database.Feature, error) { - var features []database.Feature - - rows, err := tx.Query(searchLayerFeatures, layerID) +func (tx *pgSession) findLayerFeatures(layerID int64, detectors detectorMap) ([]database.LayerFeature, error) { + rows, err := tx.Query(findLayerFeatures, layerID) if err != nil { - return nil, handleError("searchLayerFeatures", err) + return nil, handleError("findLayerFeatures", err) } + defer rows.Close() + features := []database.LayerFeature{} for rows.Next() { - f := database.Feature{} - err := rows.Scan(&f.Name, &f.Version, &f.VersionFormat) - if err != nil { - return nil, err + var ( + detectorID int64 + feature database.LayerFeature + ) + if err := rows.Scan(&feature.Name, &feature.Version, &feature.VersionFormat, &detectorID); err != nil { + return nil, handleError("findLayerFeatures", err) } - features = append(features, f) + + feature.By = detectors.byID[detectorID] + features = append(features, feature) } + return features, nil } -func (tx *pgSession) findLayer(hash string) (database.LayerMetadata, int64, bool, error) { - var ( - layerID int64 - layer = database.LayerMetadata{Hash: hash, ProcessedBy: database.Processors{}} - ) - - if hash == "" { - return layer, layerID, false, commonerr.NewBadRequestError("Empty Layer Hash is not allowed") - } - - err := tx.QueryRow(searchLayer, hash).Scan(&layerID) +func (tx *pgSession) findLayerID(hash string) (int64, bool, error) { + var layerID int64 + err := tx.QueryRow(findLayerID, hash).Scan(&layerID) if err != nil { if err == sql.ErrNoRows { - return layer, layerID, false, nil + return layerID, false, nil } - return layer, layerID, false, err + + return layerID, false, handleError("findLayerID", err) } - layer.ProcessedBy, err = tx.findLayerProcessors(layerID) - return layer, layerID, true, err + return layerID, true, nil } -func (tx *pgSession) findLayerProcessors(id int64) (database.Processors, error) { - var ( - err error - processors database.Processors - ) +func (tx *pgSession) findLayerIDs(hashes []string) ([]int64, bool, error) { + layerIDs := make([]int64, 0, len(hashes)) + for _, hash := range hashes { + id, ok, err := tx.findLayerID(hash) + if !ok { + return nil, false, nil + } - if processors.Detectors, err = tx.findProcessors(searchLayerDetectors, id); err != nil { - return processors, handleError("searchLayerDetectors", err) + if err != nil { + return nil, false, err + } + + layerIDs = append(layerIDs, id) } - if processors.Listers, err = tx.findProcessors(searchLayerListers, id); err != nil { - return processors, handleError("searchLayerListers", err) + return layerIDs, true, nil +} + +func (tx *pgSession) soiLayer(hash string) (int64, error) { + var id int64 + if err := tx.QueryRow(soiLayer, hash).Scan(&id); err != nil { + return 0, handleError("soiLayer", err) } - return processors, nil + return id, nil } diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer_test.go index 6fe8bed3..fc22a83c 100644 --- a/database/pgsql/layer_test.go +++ b/database/pgsql/layer_test.go @@ -20,107 +20,172 @@ import ( "github.com/stretchr/testify/assert" "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/testutil" ) -func TestPersistLayer(t *testing.T) { - datastore, tx := openSessionForTest(t, "PersistLayer", false) - defer closeTest(t, datastore, tx) - - // invalid - assert.NotNil(t, tx.PersistLayer("", nil, nil, database.Processors{})) - // insert namespaces + features to - namespaces := []database.Namespace{ - { - Name: "sushi shop", - VersionFormat: "apk", +var persistLayerTests = []struct { + title string + name string + by []database.Detector + features []database.LayerFeature + namespaces []database.LayerNamespace + layer *database.Layer + err string +}{ + { + title: "invalid layer name", + name: "", + err: "expected non-empty layer hash", + }, + { + title: "layer with inconsistent feature and detectors", + name: "random-forest", + by: []database.Detector{realDetectors[2]}, + features: []database.LayerFeature{ + {realFeatures[1], realDetectors[1]}, }, - } - - features := []database.Feature{ - { - Name: "blue fin sashimi", - Version: "v1.0", - VersionFormat: "apk", + err: "database: parameters are not valid", + }, + { + title: "layer with non-existing feature", + name: "random-forest", + err: "database: associated immutable entities are missing in the database", + by: []database.Detector{realDetectors[2]}, + features: []database.LayerFeature{ + {fakeFeatures[1], realDetectors[2]}, }, - } - - processors := database.Processors{ - Listers: []string{"release"}, - Detectors: []string{"apk"}, - } - - assert.Nil(t, tx.PersistNamespaces(namespaces)) - assert.Nil(t, tx.PersistFeatures(features)) - - // Valid - assert.Nil(t, tx.PersistLayer("RANDOM_FOREST", namespaces, features, processors)) - - nonExistingFeature := []database.Feature{{Name: "lobster sushi", Version: "v0.1", VersionFormat: "apk"}} - // Invalid: - assert.NotNil(t, tx.PersistLayer("RANDOM_FOREST", namespaces, nonExistingFeature, processors)) - - assert.Nil(t, tx.PersistFeatures(nonExistingFeature)) - // Update the layer - assert.Nil(t, tx.PersistLayer("RANDOM_FOREST", namespaces, nonExistingFeature, processors)) - - // confirm update - layer, ok, err := tx.FindLayer("RANDOM_FOREST") - assert.Nil(t, err) - assert.True(t, ok) - - expectedLayer := database.Layer{ - LayerMetadata: database.LayerMetadata{ - Hash: "RANDOM_FOREST", - ProcessedBy: processors, + }, + { + title: "layer with non-existing namespace", + name: "random-forest2", + err: "database: associated immutable entities are missing in the database", + by: []database.Detector{realDetectors[1]}, + namespaces: []database.LayerNamespace{ + {fakeNamespaces[1], realDetectors[1]}, }, - Features: append(features, nonExistingFeature...), - Namespaces: namespaces, - } - - assertLayerWithContentEqual(t, expectedLayer, layer) -} - -func TestFindLayer(t *testing.T) { - datastore, tx := openSessionForTest(t, "FindLayer", true) - defer closeTest(t, datastore, tx) - - _, _, err := tx.FindLayer("") - assert.NotNil(t, err) - _, ok, err := tx.FindLayer("layer-non") - assert.Nil(t, err) - assert.False(t, ok) - - expectedL := database.Layer{ - LayerMetadata: database.LayerMetadata{ - Hash: "layer-4", - ProcessedBy: database.Processors{ - Detectors: []string{"os-release", "apt-sources"}, - Listers: []string{"dpkg", "rpm"}, + }, + { + title: "layer with non-existing detector", + name: "random-forest3", + err: "database: associated immutable entities are missing in the database", + by: []database.Detector{fakeDetector[1]}, + }, + { + title: "valid layer", + name: "hamsterhouse", + by: []database.Detector{realDetectors[1], realDetectors[2]}, + features: []database.LayerFeature{ + {realFeatures[1], realDetectors[2]}, + {realFeatures[2], realDetectors[2]}, + }, + namespaces: []database.LayerNamespace{ + {realNamespaces[1], realDetectors[1]}, + }, + layer: &database.Layer{ + Hash: "hamsterhouse", + By: []database.Detector{realDetectors[1], realDetectors[2]}, + Features: []database.LayerFeature{ + {realFeatures[1], realDetectors[2]}, + {realFeatures[2], realDetectors[2]}, + }, + Namespaces: []database.LayerNamespace{ + {realNamespaces[1], realDetectors[1]}, }, }, - Features: []database.Feature{ - {Name: "fake", Version: "2.0", VersionFormat: "rpm"}, - {Name: "openssl", Version: "2.0", VersionFormat: "dpkg"}, + }, + { + title: "update existing layer", + name: "layer-1", + by: []database.Detector{realDetectors[3], realDetectors[4]}, + features: []database.LayerFeature{ + {realFeatures[4], realDetectors[3]}, }, - Namespaces: []database.Namespace{ - {Name: "debian:7", VersionFormat: "dpkg"}, - {Name: "fake:1.0", VersionFormat: "rpm"}, + namespaces: []database.LayerNamespace{ + {realNamespaces[3], realDetectors[4]}, }, - } + layer: &database.Layer{ + Hash: "layer-1", + By: []database.Detector{realDetectors[1], realDetectors[2], realDetectors[3], realDetectors[4]}, + Features: []database.LayerFeature{ + {realFeatures[1], realDetectors[2]}, + {realFeatures[2], realDetectors[2]}, + {realFeatures[4], realDetectors[3]}, + }, + Namespaces: []database.LayerNamespace{ + {realNamespaces[1], realDetectors[1]}, + {realNamespaces[3], realDetectors[4]}, + }, + }, + }, +} - layer, ok2, err := tx.FindLayer("layer-4") - if assert.Nil(t, err) && assert.True(t, ok2) { - assertLayerWithContentEqual(t, expectedL, layer) +func TestPersistLayer(t *testing.T) { + datastore, tx := openSessionForTest(t, "PersistLayer", true) + defer closeTest(t, datastore, tx) + + for _, test := range persistLayerTests { + t.Run(test.title, func(t *testing.T) { + err := tx.PersistLayer(test.name, test.features, test.namespaces, test.by) + if test.err != "" { + assert.EqualError(t, err, test.err, "unexpected error") + return + } + + assert.Nil(t, err) + if test.layer != nil { + layer, ok, err := tx.FindLayer(test.name) + assert.Nil(t, err) + assert.True(t, ok) + testutil.AssertLayerEqual(t, test.layer, &layer) + } + }) } } -func assertLayerWithContentEqual(t *testing.T, expected database.Layer, actual database.Layer) bool { - return assertLayerEqual(t, expected.LayerMetadata, actual.LayerMetadata) && - assertFeaturesEqual(t, expected.Features, actual.Features) && - assertNamespacesEqual(t, expected.Namespaces, actual.Namespaces) +var findLayerTests = []struct { + title string + in string + + out *database.Layer + err string + ok bool +}{ + { + title: "invalid layer name", + in: "", + err: "non empty layer hash is expected.", + }, + { + title: "non-existing layer", + in: "layer-non-existing", + ok: false, + out: nil, + }, + { + title: "existing layer", + in: "layer-4", + ok: true, + out: takeLayerPointerFromMap(realLayers, 6), + }, } -func assertLayerEqual(t *testing.T, expected database.LayerMetadata, actual database.LayerMetadata) bool { - return assertProcessorsEqual(t, expected.ProcessedBy, actual.ProcessedBy) && - assert.Equal(t, expected.Hash, actual.Hash) +func TestFindLayer(t *testing.T) { + datastore, tx := openSessionForTest(t, "FindLayer", true) + defer closeTest(t, datastore, tx) + + for _, test := range findLayerTests { + t.Run(test.title, func(t *testing.T) { + layer, ok, err := tx.FindLayer(test.in) + if test.err != "" { + assert.EqualError(t, err, test.err, "unexpected error") + return + } + + assert.Nil(t, err) + assert.Equal(t, test.ok, ok) + if test.ok { + testutil.AssertLayerEqual(t, test.out, &layer) + } + }) + } } diff --git a/database/pgsql/migrations/00001_initial_schema.go b/database/pgsql/migrations/00001_initial_schema.go index 69bd4081..c073e286 100644 --- a/database/pgsql/migrations/00001_initial_schema.go +++ b/database/pgsql/migrations/00001_initial_schema.go @@ -38,8 +38,8 @@ var ( `CREATE TABLE IF NOT EXISTS namespaced_feature ( id SERIAL PRIMARY KEY, - namespace_id INT REFERENCES namespace, - feature_id INT REFERENCES feature, + namespace_id INT REFERENCES namespace ON DELETE CASCADE, + feature_id INT REFERENCES feature ON DELETE CASCADE, UNIQUE (namespace_id, feature_id));`, }, Down: []string{ @@ -116,7 +116,7 @@ var ( id SERIAL PRIMARY KEY, ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, ancestry_index INT NOT NULL, - layer_id INT REFERENCES layer ON DELETE RESTRICT, + layer_id INT NOT NULL REFERENCES layer ON DELETE RESTRICT, UNIQUE (ancestry_id, ancestry_index));`, `CREATE INDEX ON ancestry_layer(ancestry_id);`, @@ -130,7 +130,7 @@ var ( `CREATE TABLE IF NOT EXISTS ancestry_detector( id SERIAL PRIMARY KEY, - ancestry_id INT REFERENCES layer ON DELETE CASCADE, + ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, detector_id INT REFERENCES detector ON DELETE CASCADE, UNIQUE(ancestry_id, detector_id));`, `CREATE INDEX ON ancestry_detector(ancestry_id);`, diff --git a/database/pgsql/namespace.go b/database/pgsql/namespace.go index bd0dae34..87d25e33 100644 --- a/database/pgsql/namespace.go +++ b/database/pgsql/namespace.go @@ -16,7 +16,6 @@ package pgsql import ( "database/sql" - "errors" "sort" "github.com/coreos/clair/database" @@ -27,10 +26,6 @@ const ( searchNamespaceID = `SELECT id FROM Namespace WHERE name = $1 AND version_format = $2` ) -var ( - errNamespaceNotFound = errors.New("Requested Namespace is not in database") -) - // PersistNamespaces soi namespaces into database. func (tx *pgSession) PersistNamespaces(namespaces []database.Namespace) error { if len(namespaces) == 0 { diff --git a/database/pgsql/namespace_test.go b/database/pgsql/namespace_test.go index 27ceefef..8f2af288 100644 --- a/database/pgsql/namespace_test.go +++ b/database/pgsql/namespace_test.go @@ -42,42 +42,3 @@ func TestPersistNamespaces(t *testing.T) { assert.Len(t, nsList, 1) assert.Equal(t, ns2, nsList[0]) } - -func assertNamespacesEqual(t *testing.T, expected []database.Namespace, actual []database.Namespace) bool { - if assert.Len(t, actual, len(expected)) { - has := map[database.Namespace]bool{} - for _, i := range expected { - has[i] = false - } - for _, i := range actual { - has[i] = true - } - for key, v := range has { - if !assert.True(t, v, key.Name+"is expected") { - return false - } - } - return true - } - return false -} - -func listNamespaces(t *testing.T, tx *pgSession) []database.Namespace { - rows, err := tx.Query("SELECT name, version_format FROM namespace") - if err != nil { - t.FailNow() - } - defer rows.Close() - - namespaces := []database.Namespace{} - for rows.Next() { - var ns database.Namespace - err := rows.Scan(&ns.Name, &ns.VersionFormat) - if err != nil { - t.FailNow() - } - namespaces = append(namespaces, ns) - } - - return namespaces -} diff --git a/database/pgsql/notification.go b/database/pgsql/notification.go index 44eff64b..7d2b750d 100644 --- a/database/pgsql/notification.go +++ b/database/pgsql/notification.go @@ -27,7 +27,6 @@ import ( ) const ( - // notification.go insertNotification = ` INSERT INTO Vulnerability_Notification(name, created_at, old_vulnerability_id, new_vulnerability_id) VALUES ($1, $2, $3, $4)` @@ -60,9 +59,10 @@ const ( SELECT DISTINCT ON (a.id) a.id, a.name FROM vulnerability_affected_namespaced_feature AS vanf, - ancestry_layer AS al, ancestry_feature AS af + ancestry_layer AS al, ancestry_feature AS af, ancestry AS a WHERE vanf.vulnerability_id = $1 - AND al.ancestry_id >= $2 + AND a.id >= $2 + AND al.ancestry_id = a.id AND al.id = af.ancestry_layer_id AND af.namespaced_feature_id = vanf.namespaced_feature_id ORDER BY a.id ASC @@ -211,14 +211,12 @@ func (tx *pgSession) findPagedVulnerableAncestries(vulnID int64, limit int, curr vulnPage := database.PagedVulnerableAncestries{Limit: limit} currentPage := Page{0} if currentToken != pagination.FirstPageToken { - var err error - err = tx.key.UnmarshalToken(currentToken, ¤tPage) - if err != nil { + if err := tx.key.UnmarshalToken(currentToken, ¤tPage); err != nil { return vulnPage, err } } - err := tx.QueryRow(searchVulnerabilityByID, vulnID).Scan( + if err := tx.QueryRow(searchVulnerabilityByID, vulnID).Scan( &vulnPage.Name, &vulnPage.Description, &vulnPage.Link, @@ -226,8 +224,7 @@ func (tx *pgSession) findPagedVulnerableAncestries(vulnID int64, limit int, curr &vulnPage.Metadata, &vulnPage.Namespace.Name, &vulnPage.Namespace.VersionFormat, - ) - if err != nil { + ); err != nil { return vulnPage, handleError("searchVulnerabilityByID", err) } @@ -290,7 +287,6 @@ func (tx *pgSession) FindVulnerabilityNotification(name string, limit int, oldPa } noti.Name = name - err := tx.QueryRow(searchNotification, name).Scan(&created, ¬ified, &deleted, &oldVulnID, &newVulnID) diff --git a/database/pgsql/notification_test.go b/database/pgsql/notification_test.go index 9d36f4cb..0a23abca 100644 --- a/database/pgsql/notification_test.go +++ b/database/pgsql/notification_test.go @@ -19,121 +19,144 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/pagination" ) -func TestPagination(t *testing.T) { - datastore, tx := openSessionForTest(t, "Pagination", true) - defer closeTest(t, datastore, tx) - - ns := database.Namespace{ - Name: "debian:7", - VersionFormat: "dpkg", - } - - vNew := database.Vulnerability{ - Namespace: ns, - Name: "CVE-OPENSSL-1-DEB7", - Description: "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", - Link: "http://google.com/#q=CVE-OPENSSL-1-DEB7", - Severity: database.HighSeverity, - } - - vOld := database.Vulnerability{ - Namespace: ns, - Name: "CVE-NOPE", - Description: "A vulnerability affecting nothing", - Severity: database.UnknownSeverity, - } - - noti, ok, err := tx.FindVulnerabilityNotification("test", 1, "", "") - oldPage := database.PagedVulnerableAncestries{ - Vulnerability: vOld, - Limit: 1, - Affected: make(map[int]string), - End: true, - } - - newPage1 := database.PagedVulnerableAncestries{ - Vulnerability: vNew, - Limit: 1, - Affected: map[int]string{3: "ancestry-3"}, - End: false, - } +type findVulnerabilityNotificationIn struct { + notificationName string + pageSize int + oldAffectedAncestryPage pagination.Token + newAffectedAncestryPage pagination.Token +} - newPage2 := database.PagedVulnerableAncestries{ - Vulnerability: vNew, - Limit: 1, - Affected: map[int]string{4: "ancestry-4"}, - Next: "", - End: true, - } +type findVulnerabilityNotificationOut struct { + notification *database.VulnerabilityNotificationWithVulnerable + ok bool + err string +} - if assert.Nil(t, err) && assert.True(t, ok) { - assert.Equal(t, "test", noti.Name) - if assert.NotNil(t, noti.Old) && assert.NotNil(t, noti.New) { - var oldPage Page - err := tx.key.UnmarshalToken(noti.Old.Current, &oldPage) - if !assert.Nil(t, err) { - assert.FailNow(t, "") - } +var findVulnerabilityNotificationTests = []struct { + title string + in findVulnerabilityNotificationIn + out findVulnerabilityNotificationOut +}{ + { + title: "find notification with invalid page", + in: findVulnerabilityNotificationIn{ + notificationName: "test", + pageSize: 1, + oldAffectedAncestryPage: pagination.FirstPageToken, + newAffectedAncestryPage: pagination.Token("random non sense"), + }, + out: findVulnerabilityNotificationOut{ + err: pagination.ErrInvalidToken.Error(), + }, + }, + { + title: "find non-existing notification", + in: findVulnerabilityNotificationIn{ + notificationName: "non-existing", + pageSize: 1, + oldAffectedAncestryPage: pagination.FirstPageToken, + newAffectedAncestryPage: pagination.FirstPageToken, + }, + out: findVulnerabilityNotificationOut{ + ok: false, + }, + }, + { + title: "find existing notification first page", + in: findVulnerabilityNotificationIn{ + notificationName: "test", + pageSize: 1, + oldAffectedAncestryPage: pagination.FirstPageToken, + newAffectedAncestryPage: pagination.FirstPageToken, + }, + out: findVulnerabilityNotificationOut{ + &database.VulnerabilityNotificationWithVulnerable{ + NotificationHook: realNotification[1].NotificationHook, + Old: &database.PagedVulnerableAncestries{ + Vulnerability: realVulnerability[2], + Limit: 1, + Affected: make(map[int]string), + Current: mustMarshalToken(testPaginationKey, Page{0}), + Next: mustMarshalToken(testPaginationKey, Page{0}), + End: true, + }, + New: &database.PagedVulnerableAncestries{ + Vulnerability: realVulnerability[1], + Limit: 1, + Affected: map[int]string{3: "ancestry-3"}, + Current: mustMarshalToken(testPaginationKey, Page{0}), + Next: mustMarshalToken(testPaginationKey, Page{4}), + End: false, + }, + }, - assert.Equal(t, int64(0), oldPage.StartID) - var newPage Page - err = tx.key.UnmarshalToken(noti.New.Current, &newPage) - if !assert.Nil(t, err) { - assert.FailNow(t, "") - } - var newPageNext Page - err = tx.key.UnmarshalToken(noti.New.Next, &newPageNext) - if !assert.Nil(t, err) { - assert.FailNow(t, "") - } - assert.Equal(t, int64(0), newPage.StartID) - assert.Equal(t, int64(4), newPageNext.StartID) - - noti.Old.Current = "" - noti.New.Current = "" - noti.New.Next = "" - assert.Equal(t, oldPage, *noti.Old) - assert.Equal(t, newPage1, *noti.New) - } - } + true, + "", + }, + }, + + { + title: "find existing notification of second page of new affected ancestry", + in: findVulnerabilityNotificationIn{ + notificationName: "test", + pageSize: 1, + oldAffectedAncestryPage: pagination.FirstPageToken, + newAffectedAncestryPage: mustMarshalToken(testPaginationKey, Page{4}), + }, + out: findVulnerabilityNotificationOut{ + &database.VulnerabilityNotificationWithVulnerable{ + NotificationHook: realNotification[1].NotificationHook, + Old: &database.PagedVulnerableAncestries{ + Vulnerability: realVulnerability[2], + Limit: 1, + Affected: make(map[int]string), + Current: mustMarshalToken(testPaginationKey, Page{0}), + Next: mustMarshalToken(testPaginationKey, Page{0}), + End: true, + }, + New: &database.PagedVulnerableAncestries{ + Vulnerability: realVulnerability[1], + Limit: 1, + Affected: map[int]string{4: "ancestry-4"}, + Current: mustMarshalToken(testPaginationKey, Page{4}), + Next: mustMarshalToken(testPaginationKey, Page{0}), + End: true, + }, + }, - pageNum1, err := tx.key.MarshalToken(Page{0}) - if !assert.Nil(t, err) { - assert.FailNow(t, "") - } + true, + "", + }, + }, +} - pageNum2, err := tx.key.MarshalToken(Page{4}) - if !assert.Nil(t, err) { - assert.FailNow(t, "") - } +func TestFindVulnerabilityNotification(t *testing.T) { + datastore, tx := openSessionForTest(t, "pagination", true) + defer closeTest(t, datastore, tx) - noti, ok, err = tx.FindVulnerabilityNotification("test", 1, pageNum1, pageNum2) - if assert.Nil(t, err) && assert.True(t, ok) { - assert.Equal(t, "test", noti.Name) - if assert.NotNil(t, noti.Old) && assert.NotNil(t, noti.New) { - var oldCurrentPage Page - err = tx.key.UnmarshalToken(noti.Old.Current, &oldCurrentPage) - if !assert.Nil(t, err) { - assert.FailNow(t, "") + for _, test := range findVulnerabilityNotificationTests { + t.Run(test.title, func(t *testing.T) { + notification, ok, err := tx.FindVulnerabilityNotification(test.in.notificationName, test.in.pageSize, test.in.oldAffectedAncestryPage, test.in.newAffectedAncestryPage) + if test.out.err != "" { + require.EqualError(t, err, test.out.err) + return } - var newCurrentPage Page - err = tx.key.UnmarshalToken(noti.New.Current, &newCurrentPage) - if !assert.Nil(t, err) { - assert.FailNow(t, "") + require.Nil(t, err) + if !test.out.ok { + require.Equal(t, test.out.ok, ok) + return } - assert.Equal(t, int64(0), oldCurrentPage.StartID) - assert.Equal(t, int64(4), newCurrentPage.StartID) - noti.Old.Current = "" - noti.New.Current = "" - assert.Equal(t, oldPage, *noti.Old) - assert.Equal(t, newPage2, *noti.New) - } + require.True(t, ok) + assertVulnerabilityNotificationWithVulnerableEqual(t, testPaginationKey, test.out.notification, ¬ification) + }) } } diff --git a/database/pgsql/pgsql.go b/database/pgsql/pgsql.go index 9af010b6..4b23e014 100644 --- a/database/pgsql/pgsql.go +++ b/database/pgsql/pgsql.go @@ -270,6 +270,7 @@ func migrateDatabase(db *sql.DB) error { // createDatabase creates a new database. // The source parameter should not contain a dbname. func createDatabase(source, dbName string) error { + log.WithFields(log.Fields{"source": source, "dbName": dbName}).Debug("creating database...") // Open database. db, err := sql.Open("postgres", source) if err != nil { @@ -325,7 +326,7 @@ func handleError(desc string, err error) error { return commonerr.ErrNotFound } - log.WithError(err).WithField("Description", desc).Error("Handled Database Error") + log.WithError(err).WithField("Description", desc).Error("database: handled database error") promErrorsTotal.WithLabelValues(desc).Inc() if _, o := err.(*pq.Error); o || err == sql.ErrTxDone || strings.HasPrefix(err.Error(), "sql:") { diff --git a/database/pgsql/pgsql_test.go b/database/pgsql/pgsql_test.go index e4a8c8b4..863445a5 100644 --- a/database/pgsql/pgsql_test.go +++ b/database/pgsql/pgsql_test.go @@ -37,6 +37,8 @@ var ( withFixtureName, withoutFixtureName string ) +var testPaginationKey = pagination.Must(pagination.NewKey()) + func genTemplateDatabase(name string, loadFixture bool) (sourceURL string, dbName string) { config := generateTestConfig(name, loadFixture, false) source := config.Options["source"].(string) @@ -215,13 +217,15 @@ func generateTestConfig(testName string, loadFixture bool, manageLife bool) data source = fmt.Sprintf(sourceEnv, dbName) } + log.Infof("pagination key for current test: %s", testPaginationKey.String()) + return database.RegistrableComponentConfig{ Options: map[string]interface{}{ "source": source, "cachesize": 0, "managedatabaselifecycle": manageLife, "fixturepath": fixturePath, - "paginationkey": pagination.Must(pagination.NewKey()).String(), + "paginationkey": testPaginationKey.String(), }, } } @@ -247,6 +251,8 @@ func openSessionForTest(t *testing.T, name string, loadFixture bool) (*pgSQL, *p t.Error(err) t.FailNow() } + + log.Infof("transaction pagination key: '%s'", tx.(*pgSession).key.String()) return store, tx.(*pgSession) } diff --git a/database/pgsql/queries.go b/database/pgsql/queries.go index ad7cfc44..2d4b7e99 100644 --- a/database/pgsql/queries.go +++ b/database/pgsql/queries.go @@ -121,7 +121,8 @@ func queryPersistLayerFeature(count int) string { "layer_feature", "layer_feature_layer_id_feature_id_key", "layer_id", - "feature_id") + "feature_id", + "detector_id") } func queryPersistNamespace(count int) string { @@ -132,28 +133,13 @@ func queryPersistNamespace(count int) string { "version_format") } -func queryPersistLayerListers(count int) string { - return queryPersist(count, - "layer_lister", - "layer_lister_layer_id_lister_key", - "layer_id", - "lister") -} - -func queryPersistLayerDetectors(count int) string { - return queryPersist(count, - "layer_detector", - "layer_detector_layer_id_detector_key", - "layer_id", - "detector") -} - func queryPersistLayerNamespace(count int) string { return queryPersist(count, "layer_namespace", "layer_namespace_layer_id_namespace_id_key", "layer_id", - "namespace_id") + "namespace_id", + "detector_id") } // size of key and array should be both greater than 0 diff --git a/database/pgsql/testdata/data.sql b/database/pgsql/testdata/data.sql index 9d8e0323..e7484209 100644 --- a/database/pgsql/testdata/data.sql +++ b/database/pgsql/testdata/data.sql @@ -1,57 +1,69 @@ +-- initialize entities INSERT INTO namespace (id, name, version_format) VALUES -(1, 'debian:7', 'dpkg'), -(2, 'debian:8', 'dpkg'), -(3, 'fake:1.0', 'rpm'); + (1, 'debian:7', 'dpkg'), + (2, 'debian:8', 'dpkg'), + (3, 'fake:1.0', 'rpm'); INSERT INTO feature (id, name, version, version_format) VALUES -(1, 'wechat', '0.5', 'dpkg'), -(2, 'openssl', '1.0', 'dpkg'), -(3, 'openssl', '2.0', 'dpkg'), -(4, 'fake', '2.0', 'rpm'); + (1, 'ourchat', '0.5', 'dpkg'), + (2, 'openssl', '1.0', 'dpkg'), + (3, 'openssl', '2.0', 'dpkg'), + (4, 'fake', '2.0', 'rpm'); +INSERT INTO namespaced_feature(id, feature_id, namespace_id) VALUES + (1, 1, 1), -- ourchat 0.5, debian:7 + (2, 2, 1), -- openssl 1.0, debian:7 + (3, 2, 2), -- openssl 1.0, debian:8 + (4, 3, 1); -- openssl 2.0, debian:7 + +INSERT INTO detector(id, name, version, dtype) VALUES + (1, 'os-release', '1.0', 'namespace'), + (2, 'dpkg', '1.0', 'feature'), + (3, 'rpm', '1.0', 'feature'), + (4, 'apt-sources', '1.0', 'namespace'); + +-- initialize layers INSERT INTO layer (id, hash) VALUES (1, 'layer-0'), -- blank - (2, 'layer-1'), -- debian:7; wechat 0.5, openssl 1.0 - (3, 'layer-2'), -- debian:7; wechat 0.5, openssl 2.0 + (2, 'layer-1'), -- debian:7; ourchat 0.5, openssl 1.0 + (3, 'layer-2'), -- debian:7; ourchat 0.5, openssl 2.0 (4, 'layer-3a'),-- debian:7; - (5, 'layer-3b'),-- debian:8; wechat 0.5, openssl 1.0 + (5, 'layer-3b'),-- debian:8; ourchat 0.5, openssl 1.0 (6, 'layer-4'); -- debian:7, fake:1.0; openssl 2.0 (debian), fake 2.0 (fake) -INSERT INTO layer_namespace(id, layer_id, namespace_id) VALUES - (1, 2, 1), - (2, 3, 1), - (3, 4, 1), - (4, 5, 2), - (5, 6, 1), - (6, 6, 3); - -INSERT INTO layer_feature(id, layer_id, feature_id) VALUES - (1, 2, 1), - (2, 2, 2), - (3, 3, 1), - (4, 3, 3), - (5, 5, 1), - (6, 5, 2), - (7, 6, 4), - (8, 6, 3); - -INSERT INTO layer_lister(id, layer_id, lister) VALUES - (1, 1, 'dpkg'), - (2, 2, 'dpkg'), - (3, 3, 'dpkg'), - (4, 4, 'dpkg'), - (5, 5, 'dpkg'), - (6, 6, 'dpkg'), - (7, 6, 'rpm'); - -INSERT INTO layer_detector(id, layer_id, detector) VALUES - (1, 1, 'os-release'), - (2, 2, 'os-release'), - (3, 3, 'os-release'), - (4, 4, 'os-release'), - (5, 5, 'os-release'), - (6, 6, 'os-release'), - (7, 6, 'apt-sources'); +INSERT INTO layer_namespace(id, layer_id, namespace_id, detector_id) VALUES + (1, 2, 1, 1), -- layer-1: debian:7 + (2, 3, 1, 1), -- layer-2: debian:7 + (3, 4, 1, 1), -- layer-3a: debian:7 + (4, 5, 2, 1), -- layer-3b: debian:8 + (5, 6, 1, 1), -- layer-4: debian:7 + (6, 6, 3, 4); -- layer-4: fake:1.0 + +INSERT INTO layer_feature(id, layer_id, feature_id, detector_id) VALUES + (1, 2, 1, 2), -- layer-1: ourchat 0.5 + (2, 2, 2, 2), -- layer-1: openssl 1.0 + (3, 3, 1, 2), -- layer-2: ourchat 0.5 + (4, 3, 3, 2), -- layer-2: openssl 2.0 + (5, 5, 1, 2), -- layer-3b: ourchat 0.5 + (6, 5, 2, 2), -- layer-3b: openssl 1.0 + (7, 6, 4, 3), -- layer-4: fake 2.0 + (8, 6, 3, 2); -- layer-4: openssl 2.0 + +INSERT INTO layer_detector(layer_id, detector_id) VALUES + (1, 1), + (2, 1), + (3, 1), + (4, 1), + (5, 1), + (6, 1), + (6, 4), + (1, 2), + (2, 2), + (3, 2), + (4, 2), + (5, 2), + (6, 2), + (6, 3); INSERT INTO ancestry (id, name) VALUES (1, 'ancestry-1'), -- layer-0, layer-1, layer-2, layer-3a @@ -59,32 +71,39 @@ INSERT INTO ancestry (id, name) VALUES (3, 'ancestry-3'), -- layer-0 (4, 'ancestry-4'); -- layer-0 -INSERT INTO ancestry_lister (id, ancestry_id, lister) VALUES - (1, 1, 'dpkg'), - (2, 2, 'dpkg'); - -INSERT INTO ancestry_detector (id, ancestry_id, detector) VALUES - (1, 1, 'os-release'), - (2, 2, 'os-release'); +INSERT INTO ancestry_detector (ancestry_id, detector_id) VALUES + (1, 2), + (2, 2), + (1, 1), + (2, 1); INSERT INTO ancestry_layer (id, ancestry_id, layer_id, ancestry_index) VALUES + -- ancestry-1: layer-0, layer-1, layer-2, layer-3a (1, 1, 1, 0),(2, 1, 2, 1),(3, 1, 3, 2),(4, 1, 4, 3), + -- ancestry-2: layer-0, layer-1, layer-2, layer-3b (5, 2, 1, 0),(6, 2, 2, 1),(7, 2, 3, 2),(8, 2, 5, 3), - (9, 3, 1, 0), - (10, 4, 1, 0); - -INSERT INTO namespaced_feature(id, feature_id, namespace_id) VALUES - (1, 1, 1), -- wechat 0.5, debian:7 - (2, 2, 1), -- openssl 1.0, debian:7 - (3, 2, 2), -- openssl 1.0, debian:8 - (4, 3, 1); -- openssl 2.0, debian:7 + -- ancestry-3: layer-1 + (9, 3, 2, 0), + -- ancestry-4: layer-1 + (10, 4, 2, 0); -- assume that ancestry-3 and ancestry-4 are vulnerable. -INSERT INTO ancestry_feature (id, ancestry_layer_id, namespaced_feature_id) VALUES - (1, 1, 1), (2, 1, 4), -- ancestry-1, layer 0 introduces 1, 4 - (3, 5, 1), (4, 5, 3), -- ancestry-2, layer 0 introduces 1, 3 - (5, 9, 2), -- ancestry-3, layer 0 introduces 2 - (6, 10, 2); -- ancestry-4, layer 0 introduces 2 +INSERT INTO ancestry_feature (id, ancestry_layer_id, namespaced_feature_id, feature_detector_id, namespace_detector_id) VALUES + -- ancestry-1: + -- layer-2: ourchat 0.5 <- detected by dpkg 1.0 (2); debian: 7 <- detected by os-release 1.0 (1) + -- layer-2: openssl 2.0, debian:7 + (1, 3, 1, 2, 1), (2, 3, 4, 2, 1), + -- ancestry 2: + -- 1(ourchat 0.5; debian:7 layer-2) + -- 3(openssl 1.0; debian:8 layer-3b) + (3, 7, 1, 2, 1), (4, 8, 3, 2, 1), + -- ancestry-3: + -- 2(openssl 1.0, debian:7 layer-1) + -- 1(ourchat 0.5, debian:7 layer-1) + (5, 9, 2, 2, 1), (6, 9, 1, 2, 1), -- vulnerable + -- ancestry-4: + -- same as ancestry-3 + (7, 10, 2, 2, 1), (8, 10, 1, 2, 1); -- vulnerable INSERT INTO vulnerability (id, namespace_id, name, description, link, severity) VALUES (1, 1, 'CVE-OPENSSL-1-DEB7', 'A vulnerability affecting OpenSSL < 2.0 on Debian 7.0', 'http://google.com/#q=CVE-OPENSSL-1-DEB7', 'High'), @@ -103,19 +122,23 @@ INSERT INTO vulnerability_affected_namespaced_feature(id, vulnerability_id, name INSERT INTO vulnerability_notification(id, name, created_at, notified_at, deleted_at, old_vulnerability_id, new_vulnerability_id) VALUES (1, 'test', NULL, NULL, NULL, 2, 1); -- 'CVE-NOPE' -> 'CVE-OPENSSL-1-DEB7' +SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('namespace', 'id'), (SELECT MAX(id) FROM namespace)+1); +SELECT pg_catalog.setval(pg_get_serial_sequence('namespaced_feature', 'id'), (SELECT MAX(id) FROM namespaced_feature)+1); +SELECT pg_catalog.setval(pg_get_serial_sequence('detector', 'id'), (SELECT MAX(id) FROM detector)+1); + SELECT pg_catalog.setval(pg_get_serial_sequence('ancestry', 'id'), (SELECT MAX(id) FROM ancestry)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('ancestry_layer', 'id'), (SELECT MAX(id) FROM ancestry_layer)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('ancestry_feature', 'id'), (SELECT MAX(id) FROM ancestry_feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('ancestry_detector', 'id'), (SELECT MAX(id) FROM ancestry_detector)+1); -SELECT pg_catalog.setval(pg_get_serial_sequence('ancestry_lister', 'id'), (SELECT MAX(id) FROM ancestry_lister)+1); -SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1); -SELECT pg_catalog.setval(pg_get_serial_sequence('namespaced_feature', 'id'), (SELECT MAX(id) FROM namespaced_feature)+1); + SELECT pg_catalog.setval(pg_get_serial_sequence('layer', 'id'), (SELECT MAX(id) FROM layer)+1); +SELECT pg_catalog.setval(pg_get_serial_sequence('layer_feature', 'id'), (SELECT MAX(id) FROM layer_feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('layer_namespace', 'id'), (SELECT MAX(id) FROM layer_namespace)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('layer_detector', 'id'), (SELECT MAX(id) FROM layer_detector)+1); -SELECT pg_catalog.setval(pg_get_serial_sequence('layer_lister', 'id'), (SELECT MAX(id) FROM layer_lister)+1); + SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability', 'id'), (SELECT MAX(id) FROM vulnerability)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_affected_feature', 'id'), (SELECT MAX(id) FROM vulnerability_affected_feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_affected_namespaced_feature', 'id'), (SELECT MAX(id) FROM vulnerability_affected_namespaced_feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_notification', 'id'), (SELECT MAX(id) FROM vulnerability_notification)+1); +SELECT pg_catalog.setval(pg_get_serial_sequence('detector', 'id'), (SELECT MAX(id) FROM detector)+1); diff --git a/database/pgsql/testutil.go b/database/pgsql/testutil.go new file mode 100644 index 00000000..05ed4fbf --- /dev/null +++ b/database/pgsql/testutil.go @@ -0,0 +1,263 @@ +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pgsql + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/pagination" + "github.com/coreos/clair/pkg/testutil" +) + +// int keys must be the consistent with the database ID. +var ( + realFeatures = map[int]database.Feature{ + 1: {"ourchat", "0.5", "dpkg"}, + 2: {"openssl", "1.0", "dpkg"}, + 3: {"openssl", "2.0", "dpkg"}, + 4: {"fake", "2.0", "rpm"}, + } + + realNamespaces = map[int]database.Namespace{ + 1: {"debian:7", "dpkg"}, + 2: {"debian:8", "dpkg"}, + 3: {"fake:1.0", "rpm"}, + } + + realNamespacedFeatures = map[int]database.NamespacedFeature{ + 1: {realFeatures[1], realNamespaces[1]}, + 2: {realFeatures[2], realNamespaces[1]}, + 3: {realFeatures[2], realNamespaces[2]}, + 4: {realFeatures[3], realNamespaces[1]}, + } + + realDetectors = map[int]database.Detector{ + 1: database.NewNamespaceDetector("os-release", "1.0"), + 2: database.NewFeatureDetector("dpkg", "1.0"), + 3: database.NewFeatureDetector("rpm", "1.0"), + 4: database.NewNamespaceDetector("apt-sources", "1.0"), + } + + realLayers = map[int]database.Layer{ + 2: { + Hash: "layer-1", + By: []database.Detector{realDetectors[1], realDetectors[2]}, + Features: []database.LayerFeature{ + {realFeatures[1], realDetectors[2]}, + {realFeatures[2], realDetectors[2]}, + }, + Namespaces: []database.LayerNamespace{ + {realNamespaces[1], realDetectors[1]}, + }, + }, + 6: { + Hash: "layer-4", + By: []database.Detector{realDetectors[1], realDetectors[2], realDetectors[3], realDetectors[4]}, + Features: []database.LayerFeature{ + {realFeatures[4], realDetectors[3]}, + {realFeatures[3], realDetectors[2]}, + }, + Namespaces: []database.LayerNamespace{ + {realNamespaces[1], realDetectors[1]}, + {realNamespaces[3], realDetectors[4]}, + }, + }, + } + + realAncestries = map[int]database.Ancestry{ + 2: { + Name: "ancestry-2", + By: []database.Detector{realDetectors[2], realDetectors[1]}, + Layers: []database.AncestryLayer{ + { + "layer-0", + []database.AncestryFeature{}, + }, + { + "layer-1", + []database.AncestryFeature{}, + }, + { + "layer-2", + []database.AncestryFeature{ + { + realNamespacedFeatures[1], + realDetectors[2], + realDetectors[1], + }, + }, + }, + { + "layer-3b", + []database.AncestryFeature{ + { + realNamespacedFeatures[3], + realDetectors[2], + realDetectors[1], + }, + }, + }, + }, + }, + } + + realVulnerability = map[int]database.Vulnerability{ + 1: { + Name: "CVE-OPENSSL-1-DEB7", + Namespace: realNamespaces[1], + Description: "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", + Link: "http://google.com/#q=CVE-OPENSSL-1-DEB7", + Severity: database.HighSeverity, + }, + 2: { + Name: "CVE-NOPE", + Namespace: realNamespaces[1], + Description: "A vulnerability affecting nothing", + Severity: database.UnknownSeverity, + }, + } + + realNotification = map[int]database.VulnerabilityNotification{ + 1: { + NotificationHook: database.NotificationHook{ + Name: "test", + }, + Old: takeVulnerabilityPointerFromMap(realVulnerability, 2), + New: takeVulnerabilityPointerFromMap(realVulnerability, 1), + }, + } + + fakeFeatures = map[int]database.Feature{ + 1: { + Name: "ourchat", + Version: "0.6", + VersionFormat: "dpkg", + }, + } + + fakeNamespaces = map[int]database.Namespace{ + 1: {"green hat", "rpm"}, + } + fakeNamespacedFeatures = map[int]database.NamespacedFeature{ + 1: { + Feature: fakeFeatures[0], + Namespace: realNamespaces[0], + }, + } + + fakeDetector = map[int]database.Detector{ + 1: { + Name: "fake", + Version: "1.0", + DType: database.FeatureDetectorType, + }, + 2: { + Name: "fake2", + Version: "2.0", + DType: database.NamespaceDetectorType, + }, + } +) + +func takeVulnerabilityPointerFromMap(m map[int]database.Vulnerability, id int) *database.Vulnerability { + x := m[id] + return &x +} + +func takeAncestryPointerFromMap(m map[int]database.Ancestry, id int) *database.Ancestry { + x := m[id] + return &x +} + +func takeLayerPointerFromMap(m map[int]database.Layer, id int) *database.Layer { + x := m[id] + return &x +} + +func listNamespaces(t *testing.T, tx *pgSession) []database.Namespace { + rows, err := tx.Query("SELECT name, version_format FROM namespace") + if err != nil { + t.FailNow() + } + defer rows.Close() + + namespaces := []database.Namespace{} + for rows.Next() { + var ns database.Namespace + err := rows.Scan(&ns.Name, &ns.VersionFormat) + if err != nil { + t.FailNow() + } + namespaces = append(namespaces, ns) + } + + return namespaces +} + +func assertVulnerabilityNotificationWithVulnerableEqual(t *testing.T, key pagination.Key, expected, actual *database.VulnerabilityNotificationWithVulnerable) bool { + if expected == actual { + return true + } + + if expected == nil || actual == nil { + return assert.Equal(t, expected, actual) + } + + return assert.Equal(t, expected.NotificationHook, actual.NotificationHook) && + AssertPagedVulnerableAncestriesEqual(t, key, expected.Old, actual.Old) && + AssertPagedVulnerableAncestriesEqual(t, key, expected.New, actual.New) +} + +func AssertPagedVulnerableAncestriesEqual(t *testing.T, key pagination.Key, expected, actual *database.PagedVulnerableAncestries) bool { + if expected == actual { + return true + } + + if expected == nil || actual == nil { + return assert.Equal(t, expected, actual) + } + + return testutil.AssertVulnerabilityEqual(t, &expected.Vulnerability, &actual.Vulnerability) && + assert.Equal(t, expected.Limit, actual.Limit) && + assert.Equal(t, mustUnmarshalToken(key, expected.Current), mustUnmarshalToken(key, actual.Current)) && + assert.Equal(t, mustUnmarshalToken(key, expected.Next), mustUnmarshalToken(key, actual.Next)) && + assert.Equal(t, expected.End, actual.End) && + testutil.AssertIntStringMapEqual(t, expected.Affected, actual.Affected) +} + +func mustUnmarshalToken(key pagination.Key, token pagination.Token) Page { + if token == pagination.FirstPageToken { + return Page{} + } + + p := Page{} + if err := key.UnmarshalToken(token, &p); err != nil { + panic(err) + } + + return p +} + +func mustMarshalToken(key pagination.Key, v interface{}) pagination.Token { + token, err := key.MarshalToken(v) + if err != nil { + panic(err) + } + + return token +} diff --git a/database/pgsql/vulnerability_test.go b/database/pgsql/vulnerability_test.go index 9fe2c23b..bfa465b2 100644 --- a/database/pgsql/vulnerability_test.go +++ b/database/pgsql/vulnerability_test.go @@ -306,14 +306,14 @@ func TestFindVulnerabilityIDs(t *testing.T) { ids, err := tx.findLatestDeletedVulnerabilityIDs([]database.VulnerabilityID{{Name: "CVE-DELETED", Namespace: "debian:7"}}) if assert.Nil(t, err) { - if !(assert.Len(t, ids, 1) && assert.True(t, ids[0].Valid) && assert.Equal(t, 3, ids[0].Int64)) { + if !(assert.Len(t, ids, 1) && assert.True(t, ids[0].Valid) && assert.Equal(t, 3, int(ids[0].Int64))) { assert.Fail(t, "") } } ids, err = tx.findNotDeletedVulnerabilityIDs([]database.VulnerabilityID{{Name: "CVE-NOPE", Namespace: "debian:7"}}) if assert.Nil(t, err) { - if !(assert.Len(t, ids, 1) && assert.True(t, ids[0].Valid) && assert.Equal(t, 2, ids[0].Int64)) { + if !(assert.Len(t, ids, 1) && assert.True(t, ids[0].Valid) && assert.Equal(t, 2, int(ids[0].Int64))) { assert.Fail(t, "") } } diff --git a/pkg/testutil/testutil.go b/pkg/testutil/testutil.go new file mode 100644 index 00000000..91c2abd4 --- /dev/null +++ b/pkg/testutil/testutil.go @@ -0,0 +1,285 @@ +package testutil + +import ( + "encoding/json" + "sort" + "testing" + + "github.com/deckarep/golang-set" + "github.com/stretchr/testify/assert" + + "github.com/coreos/clair/database" +) + +// AssertDetectorsEqual asserts actual detectors are content wise equal to +// expected detectors regardless of the ordering. +func AssertDetectorsEqual(t *testing.T, expected, actual []database.Detector) bool { + if len(expected) != len(actual) { + return assert.Fail(t, "detectors are not equal", "expected: '%v', actual: '%v'", expected, actual) + } + + sort.Slice(expected, func(i, j int) bool { + return expected[i].String() < expected[j].String() + }) + + sort.Slice(actual, func(i, j int) bool { + return actual[i].String() < actual[j].String() + }) + + for i := range expected { + if expected[i] != actual[i] { + return assert.Fail(t, "detectors are not equal", "expected: '%v', actual: '%v'", expected, actual) + } + } + + return true +} + +// AssertAncestryEqual asserts actual ancestry equals to expected ancestry +// content wise. +func AssertAncestryEqual(t *testing.T, expected, actual *database.Ancestry) bool { + if expected == actual { + return true + } + + if actual == nil || expected == nil { + return assert.Equal(t, expected, actual) + } + + if !assert.Equal(t, expected.Name, actual.Name) || !AssertDetectorsEqual(t, expected.By, actual.By) { + return false + } + + if assert.Equal(t, len(expected.Layers), len(actual.Layers)) { + for index := range expected.Layers { + if !AssertAncestryLayerEqual(t, &expected.Layers[index], &actual.Layers[index]) { + return false + } + } + return true + } + return false +} + +// AssertAncestryLayerEqual asserts actual ancestry layer equals to expected +// ancestry layer content wise. +func AssertAncestryLayerEqual(t *testing.T, expected, actual *database.AncestryLayer) bool { + if !assert.Equal(t, expected.Hash, actual.Hash) { + return false + } + + if !assert.Equal(t, len(expected.Features), len(actual.Features), + "layer: %s\nExpected: %v\n Actual: %v", + expected.Hash, expected.Features, actual.Features, + ) { + return false + } + + // feature -> is in actual layer + hitCounter := map[database.AncestryFeature]bool{} + for _, f := range expected.Features { + hitCounter[f] = false + } + + // if there's no extra features and no duplicated features, since expected + // and actual have the same length, their result must equal. + for _, f := range actual.Features { + v, ok := hitCounter[f] + assert.True(t, ok, "unexpected feature %s", f) + assert.False(t, v, "duplicated feature %s", f) + hitCounter[f] = true + } + + for f, visited := range hitCounter { + assert.True(t, visited, "missing feature %s", f) + } + + return true +} + +// AssertElementsEqual asserts that content in actual equals to content in +// expected array regardless of ordering. +// +// Note: This function uses interface wise comparison. +func AssertElementsEqual(t *testing.T, expected, actual []interface{}) bool { + counter := map[interface{}]bool{} + for _, f := range expected { + counter[f] = false + } + + for _, f := range actual { + v, ok := counter[f] + if !assert.True(t, ok, "unexpected element %v\nExpected: %v\n Actual: %v\n", f, expected, actual) { + return false + } + + if !assert.False(t, v, "duplicated element %v\nExpected: %v\n Actual: %v\n", f, expected, actual) { + return false + } + + counter[f] = true + } + + for f, visited := range counter { + if !assert.True(t, visited, "missing feature %v\nExpected: %v\n Actual: %v\n", f, expected, actual) { + return false + } + } + + return true +} + +// AssertFeaturesEqual asserts content in actual equals content in expected +// regardless of ordering. +func AssertFeaturesEqual(t *testing.T, expected, actual []database.Feature) bool { + if assert.Len(t, actual, len(expected)) { + has := map[database.Feature]bool{} + for _, nf := range expected { + has[nf] = false + } + + for _, nf := range actual { + has[nf] = true + } + + for nf, visited := range has { + if !assert.True(t, visited, nf.Name+" is expected") { + return false + } + return true + } + } + return false +} + +// AssertLayerFeaturesEqual asserts content in actual equals to content in +// expected regardless of ordering. +func AssertLayerFeaturesEqual(t *testing.T, expected, actual []database.LayerFeature) bool { + if !assert.Len(t, actual, len(expected)) { + return false + } + + expectedInterfaces := []interface{}{} + for _, e := range expected { + expectedInterfaces = append(expectedInterfaces, e) + } + + actualInterfaces := []interface{}{} + for _, a := range actual { + actualInterfaces = append(actualInterfaces, a) + } + + return AssertElementsEqual(t, expectedInterfaces, actualInterfaces) +} + +// AssertNamespacesEqual asserts content in actual equals to content in +// expected regardless of ordering. +func AssertNamespacesEqual(t *testing.T, expected, actual []database.Namespace) bool { + expectedInterfaces := []interface{}{} + for _, e := range expected { + expectedInterfaces = append(expectedInterfaces, e) + } + + actualInterfaces := []interface{}{} + for _, a := range actual { + actualInterfaces = append(actualInterfaces, a) + } + + return AssertElementsEqual(t, expectedInterfaces, actualInterfaces) +} + +// AssertLayerNamespacesEqual asserts content in actual equals to content in +// expected regardless of ordering. +func AssertLayerNamespacesEqual(t *testing.T, expected, actual []database.LayerNamespace) bool { + expectedInterfaces := []interface{}{} + for _, e := range expected { + expectedInterfaces = append(expectedInterfaces, e) + } + + actualInterfaces := []interface{}{} + for _, a := range actual { + actualInterfaces = append(actualInterfaces, a) + } + + return AssertElementsEqual(t, expectedInterfaces, actualInterfaces) +} + +// AssertLayerEqual asserts actual layer equals to expected layer content wise. +func AssertLayerEqual(t *testing.T, expected, actual *database.Layer) bool { + if expected == actual { + return true + } + + if expected == nil || actual == nil { + return assert.Equal(t, expected, actual) + } + + return assert.Equal(t, expected.Hash, actual.Hash) && + AssertDetectorsEqual(t, expected.By, actual.By) && + AssertLayerFeaturesEqual(t, expected.Features, actual.Features) && + AssertLayerNamespacesEqual(t, expected.Namespaces, actual.Namespaces) +} + +// AssertIntStringMapEqual asserts two maps with integer as key and string as +// value are equal. +func AssertIntStringMapEqual(t *testing.T, expected, actual map[int]string) bool { + checked := mapset.NewSet() + for k, v := range expected { + assert.Equal(t, v, actual[k]) + checked.Add(k) + } + + for k := range actual { + if !assert.True(t, checked.Contains(k)) { + return false + } + } + + return true +} + +// AssertVulnerabilityEqual asserts two vulnerabilities are equal. +func AssertVulnerabilityEqual(t *testing.T, expected, actual *database.Vulnerability) bool { + return assert.Equal(t, expected.Name, actual.Name) && + assert.Equal(t, expected.Link, actual.Link) && + assert.Equal(t, expected.Description, actual.Description) && + assert.Equal(t, expected.Namespace, actual.Namespace) && + assert.Equal(t, expected.Severity, actual.Severity) && + AssertMetadataMapEqual(t, expected.Metadata, actual.Metadata) +} + +func castMetadataMapToInterface(metadata database.MetadataMap) map[string]interface{} { + content, err := json.Marshal(metadata) + if err != nil { + panic(err) + } + + data := make(map[string]interface{}) + if err := json.Unmarshal(content, &data); err != nil { + panic(err) + } + + return data +} + +// AssertMetadataMapEqual asserts two metadata maps are equal. +func AssertMetadataMapEqual(t *testing.T, expected, actual database.MetadataMap) bool { + expectedMap := castMetadataMapToInterface(expected) + actualMap := castMetadataMapToInterface(actual) + checked := mapset.NewSet() + for k, v := range expectedMap { + if !assert.Equal(t, v, (actualMap)[k]) { + return false + } + + checked.Add(k) + } + + for k := range actual { + if !assert.True(t, checked.Contains(k)) { + return false + } + } + + return true +} From e657d26313b1b91fe4dab17298597119dc919cd2 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 2 Oct 2018 10:50:53 -0400 Subject: [PATCH 12/14] database: move dbutil and testutil to database from pkg Move dbutil and testutil to database from pkg Rename all "result" --- {pkg/dbutil => database}/dbutil.go | 70 +++++++++++++------------- database/pgsql/ancestry_test.go | 5 +- database/pgsql/layer_test.go | 5 +- database/pgsql/testutil.go | 5 +- {pkg/testutil => database}/testutil.go | 44 ++++++++++------ ext/featurens/driver_test.go | 3 +- updater.go | 4 +- worker.go | 29 ++++++----- worker_test.go | 14 +++--- 9 files changed, 90 insertions(+), 89 deletions(-) rename {pkg/dbutil => database}/dbutil.go (69%) rename {pkg/testutil => database}/testutil.go (82%) diff --git a/pkg/dbutil/dbutil.go b/database/dbutil.go similarity index 69% rename from pkg/dbutil/dbutil.go rename to database/dbutil.go index 57334298..4c205eb1 100644 --- a/pkg/dbutil/dbutil.go +++ b/database/dbutil.go @@ -12,47 +12,45 @@ // See the License for the specific language governing permissions and // limitations under the License. -package dbutil +package database import ( "github.com/deckarep/golang-set" - - "github.com/coreos/clair/database" ) // DeduplicateNamespaces deduplicates a list of namespaces. -func DeduplicateNamespaces(namespaces ...database.Namespace) []database.Namespace { +func DeduplicateNamespaces(namespaces ...Namespace) []Namespace { nsSet := mapset.NewSet() for _, ns := range namespaces { nsSet.Add(ns) } - result := make([]database.Namespace, 0, nsSet.Cardinality()) + uniqueNamespaces := make([]Namespace, 0, nsSet.Cardinality()) for ns := range nsSet.Iter() { - result = append(result, ns.(database.Namespace)) + uniqueNamespaces = append(uniqueNamespaces, ns.(Namespace)) } - return result + return uniqueNamespaces } // DeduplicateFeatures deduplicates a list of list of features. -func DeduplicateFeatures(features ...database.Feature) []database.Feature { +func DeduplicateFeatures(features ...Feature) []Feature { fSet := mapset.NewSet() for _, f := range features { fSet.Add(f) } - result := make([]database.Feature, 0, fSet.Cardinality()) + uniqueFeatures := make([]Feature, 0, fSet.Cardinality()) for f := range fSet.Iter() { - result = append(result, f.(database.Feature)) + uniqueFeatures = append(uniqueFeatures, f.(Feature)) } - return result + return uniqueFeatures } // PersistPartialLayer wraps session PersistLayer function with begin and // commit. -func PersistPartialLayer(datastore database.Datastore, layer *database.Layer) error { +func PersistPartialLayer(datastore Datastore, layer *Layer) error { tx, err := datastore.Begin() if err != nil { return err @@ -67,7 +65,7 @@ func PersistPartialLayer(datastore database.Datastore, layer *database.Layer) er } // PersistFeatures wraps session PersistFeatures function with begin and commit. -func PersistFeatures(datastore database.Datastore, features []database.Feature) error { +func PersistFeatures(datastore Datastore, features []Feature) error { tx, err := datastore.Begin() if err != nil { return err @@ -82,7 +80,7 @@ func PersistFeatures(datastore database.Datastore, features []database.Feature) // PersistNamespaces wraps session PersistNamespaces function with begin and // commit. -func PersistNamespaces(datastore database.Datastore, namespaces []database.Namespace) error { +func PersistNamespaces(datastore Datastore, namespaces []Namespace) error { tx, err := datastore.Begin() if err != nil { return err @@ -97,20 +95,20 @@ func PersistNamespaces(datastore database.Datastore, namespaces []database.Names } // FindAncestry wraps session FindAncestry function with begin and rollback. -func FindAncestry(datastore database.Datastore, name string) (database.Ancestry, bool, error) { +func FindAncestry(datastore Datastore, name string) (Ancestry, bool, error) { tx, err := datastore.Begin() defer tx.Rollback() if err != nil { - return database.Ancestry{}, false, err + return Ancestry{}, false, err } return tx.FindAncestry(name) } // FindLayer wraps session FindLayer function with begin and rollback. -func FindLayer(datastore database.Datastore, hash string) (layer database.Layer, ok bool, err error) { - var tx database.Session +func FindLayer(datastore Datastore, hash string) (layer Layer, ok bool, err error) { + var tx Session if tx, err = datastore.Begin(); err != nil { return } @@ -122,24 +120,24 @@ func FindLayer(datastore database.Datastore, hash string) (layer database.Layer, // DeduplicateNamespacedFeatures returns a copy of all unique features in the // input. -func DeduplicateNamespacedFeatures(features []database.NamespacedFeature) []database.NamespacedFeature { +func DeduplicateNamespacedFeatures(features []NamespacedFeature) []NamespacedFeature { nsSet := mapset.NewSet() for _, ns := range features { nsSet.Add(ns) } - result := make([]database.NamespacedFeature, 0, nsSet.Cardinality()) + uniqueFeatures := make([]NamespacedFeature, 0, nsSet.Cardinality()) for ns := range nsSet.Iter() { - result = append(result, ns.(database.NamespacedFeature)) + uniqueFeatures = append(uniqueFeatures, ns.(NamespacedFeature)) } - return result + return uniqueFeatures } // GetAncestryFeatures returns a list of unique namespaced features in the // ancestry. -func GetAncestryFeatures(ancestry database.Ancestry) []database.NamespacedFeature { - features := []database.NamespacedFeature{} +func GetAncestryFeatures(ancestry Ancestry) []NamespacedFeature { + features := []NamespacedFeature{} for _, layer := range ancestry.Layers { features = append(features, layer.GetFeatures()...) } @@ -148,7 +146,7 @@ func GetAncestryFeatures(ancestry database.Ancestry) []database.NamespacedFeatur } // UpsertAncestry wraps session UpsertAncestry function with begin and commit. -func UpsertAncestry(datastore database.Datastore, ancestry database.Ancestry) error { +func UpsertAncestry(datastore Datastore, ancestry Ancestry) error { tx, err := datastore.Begin() if err != nil { return err @@ -168,7 +166,7 @@ func UpsertAncestry(datastore database.Datastore, ancestry database.Ancestry) er // PersistNamespacedFeatures wraps session PersistNamespacedFeatures function // with begin and commit. -func PersistNamespacedFeatures(datastore database.Datastore, features []database.NamespacedFeature) error { +func PersistNamespacedFeatures(datastore Datastore, features []NamespacedFeature) error { tx, err := datastore.Begin() if err != nil { return err @@ -188,7 +186,7 @@ func PersistNamespacedFeatures(datastore database.Datastore, features []database // CacheRelatedVulnerability wraps session CacheAffectedNamespacedFeatures // function with begin and commit. -func CacheRelatedVulnerability(datastore database.Datastore, features []database.NamespacedFeature) error { +func CacheRelatedVulnerability(datastore Datastore, features []NamespacedFeature) error { tx, err := datastore.Begin() if err != nil { return err @@ -203,7 +201,7 @@ func CacheRelatedVulnerability(datastore database.Datastore, features []database } // IntersectDetectors returns the detectors in both d1 and d2. -func IntersectDetectors(d1 []database.Detector, d2 []database.Detector) []database.Detector { +func IntersectDetectors(d1 []Detector, d2 []Detector) []Detector { d1Set := mapset.NewSet() for _, d := range d1 { d1Set.Add(d) @@ -215,16 +213,16 @@ func IntersectDetectors(d1 []database.Detector, d2 []database.Detector) []databa } inter := d1Set.Intersect(d2Set) - result := make([]database.Detector, 0, inter.Cardinality()) + detectors := make([]Detector, 0, inter.Cardinality()) for d := range inter.Iter() { - result = append(result, d.(database.Detector)) + detectors = append(detectors, d.(Detector)) } - return result + return detectors } // DiffDetectors returns the detectors belongs to d1 but not d2 -func DiffDetectors(d1 []database.Detector, d2 []database.Detector) []database.Detector { +func DiffDetectors(d1 []Detector, d2 []Detector) []Detector { d1Set := mapset.NewSet() for _, d := range d1 { d1Set.Add(d) @@ -236,17 +234,17 @@ func DiffDetectors(d1 []database.Detector, d2 []database.Detector) []database.De } diff := d1Set.Difference(d2Set) - result := make([]database.Detector, 0, diff.Cardinality()) + detectors := make([]Detector, 0, diff.Cardinality()) for d := range diff.Iter() { - result = append(result, d.(database.Detector)) + detectors = append(detectors, d.(Detector)) } - return result + return detectors } // MergeLayers merges all content in new layer to l, where the content is // updated. -func MergeLayers(l *database.Layer, new *database.Layer) *database.Layer { +func MergeLayers(l *Layer, new *Layer) *Layer { featureSet := mapset.NewSet() namespaceSet := mapset.NewSet() bySet := mapset.NewSet() diff --git a/database/pgsql/ancestry_test.go b/database/pgsql/ancestry_test.go index 6cceb718..7f0a37f8 100644 --- a/database/pgsql/ancestry_test.go +++ b/database/pgsql/ancestry_test.go @@ -20,7 +20,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/coreos/clair/database" - "github.com/coreos/clair/pkg/testutil" ) var upsertAncestryTests = []struct { @@ -89,7 +88,7 @@ func TestUpsertAncestry(t *testing.T) { actual, ok, err := tx.FindAncestry(test.in.Name) assert.Nil(t, err) assert.True(t, ok) - testutil.AssertAncestryEqual(t, test.in, &actual) + database.AssertAncestryEqual(t, test.in, &actual) }) } } @@ -132,7 +131,7 @@ func TestFindAncestry(t *testing.T) { assert.Nil(t, err) assert.Equal(t, test.ok, ok) if test.ok { - testutil.AssertAncestryEqual(t, test.ancestry, &ancestry) + database.AssertAncestryEqual(t, test.ancestry, &ancestry) } }) } diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer_test.go index fc22a83c..478b2171 100644 --- a/database/pgsql/layer_test.go +++ b/database/pgsql/layer_test.go @@ -20,7 +20,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/coreos/clair/database" - "github.com/coreos/clair/pkg/testutil" ) var persistLayerTests = []struct { @@ -136,7 +135,7 @@ func TestPersistLayer(t *testing.T) { layer, ok, err := tx.FindLayer(test.name) assert.Nil(t, err) assert.True(t, ok) - testutil.AssertLayerEqual(t, test.layer, &layer) + database.AssertLayerEqual(t, test.layer, &layer) } }) } @@ -184,7 +183,7 @@ func TestFindLayer(t *testing.T) { assert.Nil(t, err) assert.Equal(t, test.ok, ok) if test.ok { - testutil.AssertLayerEqual(t, test.out, &layer) + database.AssertLayerEqual(t, test.out, &layer) } }) } diff --git a/database/pgsql/testutil.go b/database/pgsql/testutil.go index 05ed4fbf..1561e59d 100644 --- a/database/pgsql/testutil.go +++ b/database/pgsql/testutil.go @@ -21,7 +21,6 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/pkg/pagination" - "github.com/coreos/clair/pkg/testutil" ) // int keys must be the consistent with the database ID. @@ -232,12 +231,12 @@ func AssertPagedVulnerableAncestriesEqual(t *testing.T, key pagination.Key, expe return assert.Equal(t, expected, actual) } - return testutil.AssertVulnerabilityEqual(t, &expected.Vulnerability, &actual.Vulnerability) && + return database.AssertVulnerabilityEqual(t, &expected.Vulnerability, &actual.Vulnerability) && assert.Equal(t, expected.Limit, actual.Limit) && assert.Equal(t, mustUnmarshalToken(key, expected.Current), mustUnmarshalToken(key, actual.Current)) && assert.Equal(t, mustUnmarshalToken(key, expected.Next), mustUnmarshalToken(key, actual.Next)) && assert.Equal(t, expected.End, actual.End) && - testutil.AssertIntStringMapEqual(t, expected.Affected, actual.Affected) + database.AssertIntStringMapEqual(t, expected.Affected, actual.Affected) } func mustUnmarshalToken(key pagination.Key, token pagination.Token) Page { diff --git a/pkg/testutil/testutil.go b/database/testutil.go similarity index 82% rename from pkg/testutil/testutil.go rename to database/testutil.go index 91c2abd4..232e9202 100644 --- a/pkg/testutil/testutil.go +++ b/database/testutil.go @@ -1,4 +1,18 @@ -package testutil +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database import ( "encoding/json" @@ -7,13 +21,11 @@ import ( "github.com/deckarep/golang-set" "github.com/stretchr/testify/assert" - - "github.com/coreos/clair/database" ) // AssertDetectorsEqual asserts actual detectors are content wise equal to // expected detectors regardless of the ordering. -func AssertDetectorsEqual(t *testing.T, expected, actual []database.Detector) bool { +func AssertDetectorsEqual(t *testing.T, expected, actual []Detector) bool { if len(expected) != len(actual) { return assert.Fail(t, "detectors are not equal", "expected: '%v', actual: '%v'", expected, actual) } @@ -37,7 +49,7 @@ func AssertDetectorsEqual(t *testing.T, expected, actual []database.Detector) bo // AssertAncestryEqual asserts actual ancestry equals to expected ancestry // content wise. -func AssertAncestryEqual(t *testing.T, expected, actual *database.Ancestry) bool { +func AssertAncestryEqual(t *testing.T, expected, actual *Ancestry) bool { if expected == actual { return true } @@ -63,7 +75,7 @@ func AssertAncestryEqual(t *testing.T, expected, actual *database.Ancestry) bool // AssertAncestryLayerEqual asserts actual ancestry layer equals to expected // ancestry layer content wise. -func AssertAncestryLayerEqual(t *testing.T, expected, actual *database.AncestryLayer) bool { +func AssertAncestryLayerEqual(t *testing.T, expected, actual *AncestryLayer) bool { if !assert.Equal(t, expected.Hash, actual.Hash) { return false } @@ -76,7 +88,7 @@ func AssertAncestryLayerEqual(t *testing.T, expected, actual *database.AncestryL } // feature -> is in actual layer - hitCounter := map[database.AncestryFeature]bool{} + hitCounter := map[AncestryFeature]bool{} for _, f := range expected.Features { hitCounter[f] = false } @@ -131,9 +143,9 @@ func AssertElementsEqual(t *testing.T, expected, actual []interface{}) bool { // AssertFeaturesEqual asserts content in actual equals content in expected // regardless of ordering. -func AssertFeaturesEqual(t *testing.T, expected, actual []database.Feature) bool { +func AssertFeaturesEqual(t *testing.T, expected, actual []Feature) bool { if assert.Len(t, actual, len(expected)) { - has := map[database.Feature]bool{} + has := map[Feature]bool{} for _, nf := range expected { has[nf] = false } @@ -154,7 +166,7 @@ func AssertFeaturesEqual(t *testing.T, expected, actual []database.Feature) bool // AssertLayerFeaturesEqual asserts content in actual equals to content in // expected regardless of ordering. -func AssertLayerFeaturesEqual(t *testing.T, expected, actual []database.LayerFeature) bool { +func AssertLayerFeaturesEqual(t *testing.T, expected, actual []LayerFeature) bool { if !assert.Len(t, actual, len(expected)) { return false } @@ -174,7 +186,7 @@ func AssertLayerFeaturesEqual(t *testing.T, expected, actual []database.LayerFea // AssertNamespacesEqual asserts content in actual equals to content in // expected regardless of ordering. -func AssertNamespacesEqual(t *testing.T, expected, actual []database.Namespace) bool { +func AssertNamespacesEqual(t *testing.T, expected, actual []Namespace) bool { expectedInterfaces := []interface{}{} for _, e := range expected { expectedInterfaces = append(expectedInterfaces, e) @@ -190,7 +202,7 @@ func AssertNamespacesEqual(t *testing.T, expected, actual []database.Namespace) // AssertLayerNamespacesEqual asserts content in actual equals to content in // expected regardless of ordering. -func AssertLayerNamespacesEqual(t *testing.T, expected, actual []database.LayerNamespace) bool { +func AssertLayerNamespacesEqual(t *testing.T, expected, actual []LayerNamespace) bool { expectedInterfaces := []interface{}{} for _, e := range expected { expectedInterfaces = append(expectedInterfaces, e) @@ -205,7 +217,7 @@ func AssertLayerNamespacesEqual(t *testing.T, expected, actual []database.LayerN } // AssertLayerEqual asserts actual layer equals to expected layer content wise. -func AssertLayerEqual(t *testing.T, expected, actual *database.Layer) bool { +func AssertLayerEqual(t *testing.T, expected, actual *Layer) bool { if expected == actual { return true } @@ -239,7 +251,7 @@ func AssertIntStringMapEqual(t *testing.T, expected, actual map[int]string) bool } // AssertVulnerabilityEqual asserts two vulnerabilities are equal. -func AssertVulnerabilityEqual(t *testing.T, expected, actual *database.Vulnerability) bool { +func AssertVulnerabilityEqual(t *testing.T, expected, actual *Vulnerability) bool { return assert.Equal(t, expected.Name, actual.Name) && assert.Equal(t, expected.Link, actual.Link) && assert.Equal(t, expected.Description, actual.Description) && @@ -248,7 +260,7 @@ func AssertVulnerabilityEqual(t *testing.T, expected, actual *database.Vulnerabi AssertMetadataMapEqual(t, expected.Metadata, actual.Metadata) } -func castMetadataMapToInterface(metadata database.MetadataMap) map[string]interface{} { +func castMetadataMapToInterface(metadata MetadataMap) map[string]interface{} { content, err := json.Marshal(metadata) if err != nil { panic(err) @@ -263,7 +275,7 @@ func castMetadataMapToInterface(metadata database.MetadataMap) map[string]interf } // AssertMetadataMapEqual asserts two metadata maps are equal. -func AssertMetadataMapEqual(t *testing.T, expected, actual database.MetadataMap) bool { +func AssertMetadataMapEqual(t *testing.T, expected, actual MetadataMap) bool { expectedMap := castMetadataMapToInterface(expected) actualMap := castMetadataMapToInterface(actual) checked := mapset.NewSet() diff --git a/ext/featurens/driver_test.go b/ext/featurens/driver_test.go index 1218a852..1f463d45 100644 --- a/ext/featurens/driver_test.go +++ b/ext/featurens/driver_test.go @@ -8,7 +8,6 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/pkg/tarutil" - "github.com/coreos/clair/pkg/testutil" _ "github.com/coreos/clair/ext/featurens/alpinerelease" _ "github.com/coreos/clair/ext/featurens/aptsources" @@ -50,6 +49,6 @@ func TestNamespaceDetector(t *testing.T) { return } - testutil.AssertLayerNamespacesEqual(t, test.out, out) + database.AssertLayerNamespacesEqual(t, test.out, out) } } diff --git a/updater.go b/updater.go index d8184178..e8eba706 100644 --- a/updater.go +++ b/updater.go @@ -21,8 +21,6 @@ import ( "sync" "time" - "github.com/coreos/clair/pkg/dbutil" - "github.com/pborman/uuid" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -210,7 +208,7 @@ func update(datastore database.Datastore, firstUpdate bool) { namespaces = append(namespaces, ns) } - if err := dbutil.PersistNamespaces(datastore, namespaces); err != nil { + if err := database.PersistNamespaces(datastore, namespaces); err != nil { log.WithError(err).Error("Unable to insert namespaces") return } diff --git a/worker.go b/worker.go index 16f6a3e6..fc48c0c3 100644 --- a/worker.go +++ b/worker.go @@ -26,7 +26,6 @@ import ( "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/imagefmt" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/pkg/dbutil" "github.com/coreos/clair/pkg/strutil" "github.com/coreos/clair/pkg/tarutil" ) @@ -102,7 +101,7 @@ func processRequests(imageFormat string, toDetect map[string]*processRequest) (m } func getProcessRequest(datastore database.Datastore, req LayerRequest) (preq *processRequest, err error) { - layer, ok, err := dbutil.FindLayer(datastore, req.Hash) + layer, ok, err := database.FindLayer(datastore, req.Hash) if err != nil { return } @@ -125,7 +124,7 @@ func getProcessRequest(datastore database.Datastore, req LayerRequest) (preq *pr preq = &processRequest{ LayerRequest: req, existingLayer: &layer, - detectors: dbutil.DiffDetectors(EnabledDetectors, layer.By), + detectors: database.DiffDetectors(EnabledDetectors, layer.By), } } @@ -140,18 +139,18 @@ func persistProcessResult(datastore database.Datastore, results map[string]*proc namespaces = append(namespaces, r.newLayerContent.GetNamespaces()...) } - features = dbutil.DeduplicateFeatures(features...) - namespaces = dbutil.DeduplicateNamespaces(namespaces...) - if err := dbutil.PersistNamespaces(datastore, namespaces); err != nil { + features = database.DeduplicateFeatures(features...) + namespaces = database.DeduplicateNamespaces(namespaces...) + if err := database.PersistNamespaces(datastore, namespaces); err != nil { return err } - if err := dbutil.PersistFeatures(datastore, features); err != nil { + if err := database.PersistFeatures(datastore, features); err != nil { return err } for _, layer := range results { - if err := dbutil.PersistPartialLayer(datastore, layer.newLayerContent); err != nil { + if err := database.PersistPartialLayer(datastore, layer.newLayerContent); err != nil { return err } } @@ -196,19 +195,19 @@ func processLayers(datastore database.Datastore, imageFormat string, requests [] func getProcessResultLayers(results map[string]*processResult) map[string]database.Layer { layers := map[string]database.Layer{} for name, r := range results { - layers[name] = *dbutil.MergeLayers(r.existingLayer, r.newLayerContent) + layers[name] = *database.MergeLayers(r.existingLayer, r.newLayerContent) } return layers } func isAncestryProcessed(datastore database.Datastore, name string) (bool, error) { - ancestry, ok, err := dbutil.FindAncestry(datastore, name) + ancestry, ok, err := database.FindAncestry(datastore, name) if err != nil || !ok { return ok, err } - return len(dbutil.DiffDetectors(EnabledDetectors, ancestry.By)) == 0, nil + return len(database.DiffDetectors(EnabledDetectors, ancestry.By)) == 0, nil } // ProcessAncestry downloads and scans an ancestry if it's not scanned by all @@ -255,7 +254,7 @@ func processAncestry(datastore database.Datastore, name string, layers []databas return err } - ancestryFeatures := dbutil.GetAncestryFeatures(ancestry) + ancestryFeatures := database.GetAncestryFeatures(ancestry) log.WithFields(log.Fields{ "ancestry": name, "processed by": EnabledDetectors, @@ -263,17 +262,17 @@ func processAncestry(datastore database.Datastore, name string, layers []databas "layer count": len(ancestry.Layers), }).Debug("compute ancestry features") - if err := dbutil.PersistNamespacedFeatures(datastore, ancestryFeatures); err != nil { + if err := database.PersistNamespacedFeatures(datastore, ancestryFeatures); err != nil { log.WithField("ancestry", name).WithError(err).Error("could not persist namespaced features for ancestry") return err } - if err := dbutil.CacheRelatedVulnerability(datastore, ancestryFeatures); err != nil { + if err := database.CacheRelatedVulnerability(datastore, ancestryFeatures); err != nil { log.WithField("ancestry", name).WithError(err).Error("failed to cache feature related vulnerability") return err } - if err := dbutil.UpsertAncestry(datastore, ancestry); err != nil { + if err := database.UpsertAncestry(datastore, ancestry); err != nil { log.WithField("ancestry", name).WithError(err).Error("could not upsert ancestry") return err } diff --git a/worker_test.go b/worker_test.go index e6e58ad6..aef37985 100644 --- a/worker_test.go +++ b/worker_test.go @@ -28,8 +28,6 @@ import ( "github.com/coreos/clair/ext/featurefmt" "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/pkg/dbutil" - "github.com/coreos/clair/pkg/testutil" // Register the required detectors. _ "github.com/coreos/clair/ext/featurefmt/dpkg" @@ -203,7 +201,7 @@ func newMockDatastore() *mockDatastore { } layer, _ := session.copy.layers[hash] - dbutil.MergeLayers(&layer, &database.Layer{ + database.MergeLayers(&layer, &database.Layer{ Hash: hash, By: by, Namespaces: namespaces, @@ -359,7 +357,7 @@ func TestProcessLayers(t *testing.T) { // Ensure each layer has expected namespaces and features detected if blank, ok := datastore.layers["blank"]; ok { - testutil.AssertDetectorsEqual(t, EnabledDetectors, blank.By) + database.AssertDetectorsEqual(t, EnabledDetectors, blank.By) assert.Len(t, blank.Namespaces, 0) assert.Len(t, blank.Features, 0) } else { @@ -368,7 +366,7 @@ func TestProcessLayers(t *testing.T) { } if wheezy, ok := datastore.layers["wheezy"]; ok { - testutil.AssertDetectorsEqual(t, EnabledDetectors, wheezy.By) + database.AssertDetectorsEqual(t, EnabledDetectors, wheezy.By) assert.Equal(t, []database.LayerNamespace{ {database.Namespace{"debian:7", dpkg.ParserName}, database.NewNamespaceDetector("os-release", "1.0")}, }, wheezy.Namespaces) @@ -380,7 +378,7 @@ func TestProcessLayers(t *testing.T) { } if jessie, ok := datastore.layers["jessie"]; ok { - testutil.AssertDetectorsEqual(t, EnabledDetectors, jessie.By) + database.AssertDetectorsEqual(t, EnabledDetectors, jessie.By) assert.Equal(t, []database.LayerNamespace{ {database.Namespace{"debian:8", dpkg.ParserName}, database.NewNamespaceDetector("os-release", "1.0")}, }, jessie.Namespaces) @@ -578,8 +576,8 @@ func TestComputeAncestryFeatures(t *testing.T) { ancestryLayers, detectors, err := computeAncestryLayers(layers) require.Nil(t, err) - testutil.AssertDetectorsEqual(t, expectedDetectors, detectors) + database.AssertDetectorsEqual(t, expectedDetectors, detectors) for i := range expected { - testutil.AssertAncestryLayerEqual(t, &expected[i], &ancestryLayers[i]) + database.AssertAncestryLayerEqual(t, &expected[i], &ancestryLayers[i]) } } From a3e9b5b55d13921b61e2f92a1ade9392b6e7d7a0 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Mon, 8 Oct 2018 13:12:18 -0400 Subject: [PATCH 13/14] database: rename utility functions with commit/rollback All database utility functions are renamed to explicitly say if it will commit changes or rollback changes on success. --- database/dbutil.go | 35 ++++++++++++++++++----------------- updater.go | 2 +- worker.go | 16 ++++++++-------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/database/dbutil.go b/database/dbutil.go index 4c205eb1..f1c90a67 100644 --- a/database/dbutil.go +++ b/database/dbutil.go @@ -48,9 +48,9 @@ func DeduplicateFeatures(features ...Feature) []Feature { return uniqueFeatures } -// PersistPartialLayer wraps session PersistLayer function with begin and +// PersistPartialLayerAndCommit wraps session PersistLayer function with begin and // commit. -func PersistPartialLayer(datastore Datastore, layer *Layer) error { +func PersistPartialLayerAndCommit(datastore Datastore, layer *Layer) error { tx, err := datastore.Begin() if err != nil { return err @@ -64,8 +64,8 @@ func PersistPartialLayer(datastore Datastore, layer *Layer) error { return tx.Commit() } -// PersistFeatures wraps session PersistFeatures function with begin and commit. -func PersistFeatures(datastore Datastore, features []Feature) error { +// PersistFeaturesAndCommit wraps session PersistFeaturesAndCommit function with begin and commit. +func PersistFeaturesAndCommit(datastore Datastore, features []Feature) error { tx, err := datastore.Begin() if err != nil { return err @@ -78,9 +78,9 @@ func PersistFeatures(datastore Datastore, features []Feature) error { return tx.Commit() } -// PersistNamespaces wraps session PersistNamespaces function with begin and -// commit. -func PersistNamespaces(datastore Datastore, namespaces []Namespace) error { +// PersistNamespacesAndCommit wraps session PersistNamespaces function with +// begin and commit. +func PersistNamespacesAndCommit(datastore Datastore, namespaces []Namespace) error { tx, err := datastore.Begin() if err != nil { return err @@ -94,8 +94,9 @@ func PersistNamespaces(datastore Datastore, namespaces []Namespace) error { return tx.Commit() } -// FindAncestry wraps session FindAncestry function with begin and rollback. -func FindAncestry(datastore Datastore, name string) (Ancestry, bool, error) { +// FindAncestryAndRollback wraps session FindAncestry function with begin and +// rollback. +func FindAncestryAndRollback(datastore Datastore, name string) (Ancestry, bool, error) { tx, err := datastore.Begin() defer tx.Rollback() @@ -106,8 +107,8 @@ func FindAncestry(datastore Datastore, name string) (Ancestry, bool, error) { return tx.FindAncestry(name) } -// FindLayer wraps session FindLayer function with begin and rollback. -func FindLayer(datastore Datastore, hash string) (layer Layer, ok bool, err error) { +// FindLayerAndRollback wraps session FindLayer function with begin and rollback. +func FindLayerAndRollback(datastore Datastore, hash string) (layer Layer, ok bool, err error) { var tx Session if tx, err = datastore.Begin(); err != nil { return @@ -145,8 +146,8 @@ func GetAncestryFeatures(ancestry Ancestry) []NamespacedFeature { return DeduplicateNamespacedFeatures(features) } -// UpsertAncestry wraps session UpsertAncestry function with begin and commit. -func UpsertAncestry(datastore Datastore, ancestry Ancestry) error { +// UpsertAncestryAndCommit wraps session UpsertAncestry function with begin and commit. +func UpsertAncestryAndCommit(datastore Datastore, ancestry Ancestry) error { tx, err := datastore.Begin() if err != nil { return err @@ -164,9 +165,9 @@ func UpsertAncestry(datastore Datastore, ancestry Ancestry) error { return nil } -// PersistNamespacedFeatures wraps session PersistNamespacedFeatures function +// PersistNamespacedFeaturesAndCommit wraps session PersistNamespacedFeatures function // with begin and commit. -func PersistNamespacedFeatures(datastore Datastore, features []NamespacedFeature) error { +func PersistNamespacedFeaturesAndCommit(datastore Datastore, features []NamespacedFeature) error { tx, err := datastore.Begin() if err != nil { return err @@ -184,9 +185,9 @@ func PersistNamespacedFeatures(datastore Datastore, features []NamespacedFeature return nil } -// CacheRelatedVulnerability wraps session CacheAffectedNamespacedFeatures +// CacheRelatedVulnerabilityAndCommit wraps session CacheAffectedNamespacedFeatures // function with begin and commit. -func CacheRelatedVulnerability(datastore Datastore, features []NamespacedFeature) error { +func CacheRelatedVulnerabilityAndCommit(datastore Datastore, features []NamespacedFeature) error { tx, err := datastore.Begin() if err != nil { return err diff --git a/updater.go b/updater.go index e8eba706..567db2c2 100644 --- a/updater.go +++ b/updater.go @@ -208,7 +208,7 @@ func update(datastore database.Datastore, firstUpdate bool) { namespaces = append(namespaces, ns) } - if err := database.PersistNamespaces(datastore, namespaces); err != nil { + if err := database.PersistNamespacesAndCommit(datastore, namespaces); err != nil { log.WithError(err).Error("Unable to insert namespaces") return } diff --git a/worker.go b/worker.go index fc48c0c3..07a547b3 100644 --- a/worker.go +++ b/worker.go @@ -101,7 +101,7 @@ func processRequests(imageFormat string, toDetect map[string]*processRequest) (m } func getProcessRequest(datastore database.Datastore, req LayerRequest) (preq *processRequest, err error) { - layer, ok, err := database.FindLayer(datastore, req.Hash) + layer, ok, err := database.FindLayerAndRollback(datastore, req.Hash) if err != nil { return } @@ -141,16 +141,16 @@ func persistProcessResult(datastore database.Datastore, results map[string]*proc features = database.DeduplicateFeatures(features...) namespaces = database.DeduplicateNamespaces(namespaces...) - if err := database.PersistNamespaces(datastore, namespaces); err != nil { + if err := database.PersistNamespacesAndCommit(datastore, namespaces); err != nil { return err } - if err := database.PersistFeatures(datastore, features); err != nil { + if err := database.PersistFeaturesAndCommit(datastore, features); err != nil { return err } for _, layer := range results { - if err := database.PersistPartialLayer(datastore, layer.newLayerContent); err != nil { + if err := database.PersistPartialLayerAndCommit(datastore, layer.newLayerContent); err != nil { return err } } @@ -202,7 +202,7 @@ func getProcessResultLayers(results map[string]*processResult) map[string]databa } func isAncestryProcessed(datastore database.Datastore, name string) (bool, error) { - ancestry, ok, err := database.FindAncestry(datastore, name) + ancestry, ok, err := database.FindAncestryAndRollback(datastore, name) if err != nil || !ok { return ok, err } @@ -262,17 +262,17 @@ func processAncestry(datastore database.Datastore, name string, layers []databas "layer count": len(ancestry.Layers), }).Debug("compute ancestry features") - if err := database.PersistNamespacedFeatures(datastore, ancestryFeatures); err != nil { + if err := database.PersistNamespacedFeaturesAndCommit(datastore, ancestryFeatures); err != nil { log.WithField("ancestry", name).WithError(err).Error("could not persist namespaced features for ancestry") return err } - if err := database.CacheRelatedVulnerability(datastore, ancestryFeatures); err != nil { + if err := database.CacheRelatedVulnerabilityAndCommit(datastore, ancestryFeatures); err != nil { log.WithField("ancestry", name).WithError(err).Error("failed to cache feature related vulnerability") return err } - if err := database.UpsertAncestry(datastore, ancestry); err != nil { + if err := database.UpsertAncestryAndCommit(datastore, ancestry); err != nil { log.WithField("ancestry", name).WithError(err).Error("could not upsert ancestry") return err } From 69c0c84348c74749cd1d12ee4e4959991621a59d Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Mon, 8 Oct 2018 14:12:00 -0400 Subject: [PATCH 14/14] api: Rename detector type to DType Rename detector type to DType because all reserved key words should be avoided used as type name or variable name. --- api/v3/clairpb/clair.pb.go | 210 +++++++++++++++--------------- api/v3/clairpb/clair.proto | 10 +- api/v3/clairpb/clair.swagger.json | 22 ++-- api/v3/clairpb/convert.go | 8 +- 4 files changed, 125 insertions(+), 125 deletions(-) diff --git a/api/v3/clairpb/clair.pb.go b/api/v3/clairpb/clair.pb.go index b6b2184e..1306dbee 100644 --- a/api/v3/clairpb/clair.pb.go +++ b/api/v3/clairpb/clair.pb.go @@ -50,29 +50,29 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package -type Detector_Type int32 +type Detector_DType int32 const ( - Detector_DETECTOR_TYPE_INVALID Detector_Type = 0 - Detector_DETECTOR_TYPE_NAMESPACE Detector_Type = 1 - Detector_DETECTOR_TYPE_FEATURE Detector_Type = 2 + Detector_DETECTOR_D_TYPE_INVALID Detector_DType = 0 + Detector_DETECTOR_D_TYPE_NAMESPACE Detector_DType = 1 + Detector_DETECTOR_D_TYPE_FEATURE Detector_DType = 2 ) -var Detector_Type_name = map[int32]string{ - 0: "DETECTOR_TYPE_INVALID", - 1: "DETECTOR_TYPE_NAMESPACE", - 2: "DETECTOR_TYPE_FEATURE", +var Detector_DType_name = map[int32]string{ + 0: "DETECTOR_D_TYPE_INVALID", + 1: "DETECTOR_D_TYPE_NAMESPACE", + 2: "DETECTOR_D_TYPE_FEATURE", } -var Detector_Type_value = map[string]int32{ - "DETECTOR_TYPE_INVALID": 0, - "DETECTOR_TYPE_NAMESPACE": 1, - "DETECTOR_TYPE_FEATURE": 2, +var Detector_DType_value = map[string]int32{ + "DETECTOR_D_TYPE_INVALID": 0, + "DETECTOR_D_TYPE_NAMESPACE": 1, + "DETECTOR_D_TYPE_FEATURE": 2, } -func (x Detector_Type) String() string { - return proto.EnumName(Detector_Type_name, int32(x)) +func (x Detector_DType) String() string { + return proto.EnumName(Detector_DType_name, int32(x)) } -func (Detector_Type) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1, 0} } +func (Detector_DType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1, 0} } type Vulnerability struct { // The name of the vulnerability. @@ -162,7 +162,7 @@ type Detector struct { // The version of the detector. Version string `protobuf:"bytes,2,opt,name=version" json:"version,omitempty"` // The type of the detector. - Type Detector_Type `protobuf:"varint,3,opt,name=type,enum=coreos.clair.Detector_Type" json:"type,omitempty"` + Dtype Detector_DType `protobuf:"varint,3,opt,name=dtype,enum=coreos.clair.Detector_DType" json:"dtype,omitempty"` } func (m *Detector) Reset() { *m = Detector{} } @@ -184,11 +184,11 @@ func (m *Detector) GetVersion() string { return "" } -func (m *Detector) GetType() Detector_Type { +func (m *Detector) GetDtype() Detector_DType { if m != nil { - return m.Type + return m.Dtype } - return Detector_DETECTOR_TYPE_INVALID + return Detector_DETECTOR_D_TYPE_INVALID } type Namespace struct { @@ -809,7 +809,7 @@ func init() { proto.RegisterType((*MarkNotificationAsReadResponse)(nil), "coreos.clair.MarkNotificationAsReadResponse") proto.RegisterType((*GetStatusRequest)(nil), "coreos.clair.GetStatusRequest") proto.RegisterType((*GetStatusResponse)(nil), "coreos.clair.GetStatusResponse") - proto.RegisterEnum("coreos.clair.Detector_Type", Detector_Type_name, Detector_Type_value) + proto.RegisterEnum("coreos.clair.Detector_DType", Detector_DType_name, Detector_DType_value) } // Reference imports to suppress errors if they are not otherwise used. @@ -1091,89 +1091,89 @@ var _StatusService_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("api/v3/clairpb/clair.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1334 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4d, 0x6f, 0x1b, 0xc5, - 0x1b, 0xff, 0xaf, 0x13, 0xc7, 0xf6, 0x63, 0x3b, 0x71, 0x27, 0x69, 0xe2, 0x6c, 0xfe, 0x6d, 0x93, - 0x85, 0x8a, 0x52, 0x90, 0x2d, 0xdc, 0x22, 0xb5, 0xe5, 0x80, 0xdc, 0xc4, 0x09, 0x91, 0xda, 0x10, - 0x6d, 0xd2, 0x48, 0x05, 0x21, 0x6b, 0xe2, 0x7d, 0x9c, 0xac, 0xb2, 0xde, 0x5d, 0x76, 0xc7, 0x49, - 0xad, 0xaa, 0x1c, 0xb8, 0x71, 0x43, 0x70, 0xe0, 0xc4, 0x07, 0xe0, 0x82, 0xf8, 0x0e, 0xf0, 0x09, - 0xe0, 0x0a, 0x37, 0x0e, 0x7c, 0x01, 0xee, 0x68, 0x66, 0x67, 0x36, 0xbb, 0xc9, 0x26, 0x71, 0x7b, - 0xf2, 0xcc, 0xf3, 0xfe, 0xf2, 0x9b, 0xe7, 0x59, 0x83, 0x4e, 0x7d, 0xbb, 0x79, 0x7c, 0xaf, 0xd9, - 0x73, 0xa8, 0x1d, 0xf8, 0xfb, 0xd1, 0x6f, 0xc3, 0x0f, 0x3c, 0xe6, 0x91, 0x4a, 0xcf, 0x0b, 0xd0, - 0x0b, 0x1b, 0x82, 0xa6, 0xdf, 0x3a, 0xf0, 0xbc, 0x03, 0x07, 0x9b, 0x82, 0xb7, 0x3f, 0xec, 0x37, - 0x99, 0x3d, 0xc0, 0x90, 0xd1, 0x81, 0x1f, 0x89, 0xeb, 0xff, 0x97, 0x02, 0xdc, 0x22, 0x75, 0x5d, - 0x8f, 0x51, 0x66, 0x7b, 0x6e, 0x18, 0x71, 0x8d, 0x1f, 0x72, 0x50, 0xdd, 0x1b, 0x3a, 0x2e, 0x06, - 0x74, 0xdf, 0x76, 0x6c, 0x36, 0x22, 0x04, 0x26, 0x5d, 0x3a, 0xc0, 0xba, 0xb6, 0xac, 0xdd, 0x29, - 0x99, 0xe2, 0x4c, 0x6e, 0xc3, 0x34, 0xff, 0x0d, 0x7d, 0xda, 0xc3, 0xae, 0xe0, 0xe6, 0x04, 0xb7, - 0x1a, 0x53, 0xb7, 0xb8, 0xd8, 0x32, 0x94, 0x2d, 0x0c, 0x7b, 0x81, 0xed, 0x73, 0x17, 0xf5, 0x09, - 0x21, 0x93, 0x24, 0x71, 0xe3, 0x8e, 0xed, 0x1e, 0xd5, 0x27, 0x23, 0xe3, 0xfc, 0x4c, 0x74, 0x28, - 0x86, 0x78, 0x8c, 0x81, 0xcd, 0x46, 0xf5, 0xbc, 0xa0, 0xc7, 0x77, 0xce, 0x1b, 0x20, 0xa3, 0x16, - 0x65, 0xb4, 0x3e, 0x15, 0xf1, 0xd4, 0x9d, 0x2c, 0x42, 0xb1, 0x6f, 0xbf, 0x40, 0xab, 0xbb, 0x3f, - 0xaa, 0x17, 0x04, 0xaf, 0x20, 0xee, 0x8f, 0x47, 0xe4, 0x31, 0x5c, 0xa3, 0xfd, 0x3e, 0xf6, 0x18, - 0x5a, 0xdd, 0x63, 0x0c, 0x42, 0x9e, 0x70, 0xbd, 0xb8, 0x3c, 0x71, 0xa7, 0xdc, 0xba, 0xde, 0x48, - 0x96, 0xaf, 0xb1, 0x8e, 0x94, 0x0d, 0x03, 0x34, 0x6b, 0x4a, 0x7e, 0x4f, 0x8a, 0x1b, 0xbf, 0x69, - 0x50, 0x5c, 0x43, 0x86, 0x3d, 0xe6, 0x05, 0x99, 0x45, 0xa9, 0x43, 0x41, 0xda, 0x96, 0xd5, 0x50, - 0x57, 0xd2, 0x84, 0x49, 0x36, 0xf2, 0x51, 0x14, 0x60, 0xba, 0xb5, 0x94, 0xf6, 0xa8, 0x6c, 0x36, - 0x76, 0x47, 0x3e, 0x9a, 0x42, 0xd0, 0x78, 0x0e, 0x93, 0xfc, 0x46, 0x16, 0xe1, 0xfa, 0x5a, 0x67, - 0xb7, 0xb3, 0xba, 0xfb, 0xa9, 0xd9, 0xdd, 0x7d, 0xbe, 0xdd, 0xe9, 0x6e, 0x6e, 0xed, 0xb5, 0x9f, - 0x6c, 0xae, 0xd5, 0xfe, 0x47, 0x96, 0x60, 0x21, 0xcd, 0xda, 0x6a, 0x3f, 0xed, 0xec, 0x6c, 0xb7, - 0x57, 0x3b, 0x35, 0xed, 0xbc, 0xde, 0x7a, 0xa7, 0xbd, 0xfb, 0xcc, 0xec, 0xd4, 0x72, 0xc6, 0x0e, - 0x94, 0xb6, 0x54, 0x93, 0x32, 0xd3, 0x68, 0x41, 0xd1, 0x92, 0x21, 0x89, 0x3c, 0xca, 0xad, 0xf9, - 0xec, 0x80, 0xcd, 0x58, 0xce, 0xf8, 0x36, 0x07, 0x05, 0x59, 0xb9, 0x4c, 0x9b, 0x1f, 0x42, 0x29, - 0x46, 0x86, 0x34, 0xba, 0x90, 0x36, 0x1a, 0xc7, 0x64, 0x9e, 0x4a, 0x26, 0x2b, 0x3a, 0x91, 0xae, - 0xe8, 0x6d, 0x98, 0x96, 0xc7, 0x6e, 0xdf, 0x0b, 0x06, 0x94, 0x49, 0x04, 0x55, 0x25, 0x75, 0x5d, - 0x10, 0x53, 0xb9, 0xe4, 0xc7, 0xcb, 0x85, 0x74, 0x60, 0xe6, 0x38, 0xf1, 0x00, 0x6c, 0x0c, 0xeb, - 0x53, 0x02, 0x29, 0x67, 0xfa, 0x96, 0x7a, 0x25, 0xe6, 0x59, 0x1d, 0x63, 0x09, 0xf2, 0x4f, 0xe8, - 0x08, 0x05, 0x54, 0x0e, 0x69, 0x78, 0xa8, 0xea, 0xc1, 0xcf, 0xc6, 0x37, 0x1a, 0x94, 0x57, 0xb9, - 0x95, 0x1d, 0x46, 0xd9, 0x30, 0x24, 0xf7, 0xa1, 0xa4, 0xfc, 0x87, 0x75, 0x4d, 0x78, 0xbb, 0x28, - 0xd0, 0x53, 0x41, 0xb2, 0x06, 0x35, 0x87, 0x86, 0xac, 0x3b, 0xf4, 0x2d, 0xca, 0xb0, 0xcb, 0x1f, - 0xba, 0x2c, 0xae, 0xde, 0x88, 0x1e, 0x79, 0x43, 0x4d, 0x81, 0xc6, 0xae, 0x9a, 0x02, 0xe6, 0x34, - 0xd7, 0x79, 0x26, 0x54, 0x38, 0xd1, 0x78, 0x08, 0x64, 0x03, 0x59, 0xdb, 0xed, 0x61, 0xc8, 0x82, - 0x91, 0x89, 0x5f, 0x0e, 0x31, 0x64, 0xe4, 0x2d, 0xa8, 0x52, 0x49, 0xea, 0x26, 0xda, 0x59, 0x51, - 0x44, 0xde, 0x2f, 0xe3, 0x97, 0x09, 0x98, 0x4d, 0xe9, 0x86, 0xbe, 0xe7, 0x86, 0x48, 0xd6, 0xa1, - 0xa8, 0xe4, 0x84, 0x5e, 0xb9, 0x75, 0x37, 0x9d, 0x4d, 0x86, 0x52, 0x23, 0x26, 0xc4, 0xba, 0xe4, - 0x03, 0x98, 0x0a, 0x45, 0x81, 0x64, 0x5a, 0x8b, 0x69, 0x2b, 0x89, 0x0a, 0x9a, 0x52, 0x50, 0xff, - 0x0a, 0xaa, 0xca, 0x50, 0x54, 0xfe, 0x77, 0x21, 0xef, 0xf0, 0x83, 0x0c, 0x64, 0x36, 0x6d, 0x42, - 0xc8, 0x98, 0x91, 0x04, 0x9f, 0x12, 0x51, 0x71, 0xd1, 0xea, 0xf6, 0x23, 0x34, 0x73, 0xcf, 0x97, - 0x4d, 0x09, 0x25, 0x2f, 0x09, 0xa1, 0xfe, 0xa3, 0x06, 0x45, 0x15, 0x40, 0xe6, 0x53, 0x48, 0xb5, - 0x3a, 0x37, 0x6e, 0xab, 0x37, 0x60, 0x4a, 0xc4, 0x18, 0xd6, 0x27, 0x84, 0x4a, 0x73, 0xfc, 0x7a, - 0x46, 0x29, 0x4a, 0x75, 0xe3, 0xaf, 0x1c, 0xcc, 0x6e, 0x7b, 0xe1, 0x1b, 0xf5, 0x9b, 0xcc, 0xc3, - 0x94, 0x7c, 0x6d, 0xd1, 0x80, 0x93, 0x37, 0xb2, 0x7a, 0x26, 0xba, 0xf7, 0xd2, 0xd1, 0x65, 0xf8, - 0x13, 0xb4, 0x54, 0x64, 0xfa, 0xaf, 0x1a, 0x94, 0x62, 0x6a, 0xd6, 0xab, 0xe1, 0x34, 0x9f, 0xb2, - 0x43, 0xe9, 0x5c, 0x9c, 0x89, 0x09, 0x85, 0x43, 0xa4, 0xd6, 0xa9, 0xef, 0x07, 0xaf, 0xe1, 0xbb, - 0xf1, 0x49, 0xa4, 0xda, 0x71, 0x39, 0x57, 0x19, 0xd2, 0x1f, 0x41, 0x25, 0xc9, 0x20, 0x35, 0x98, - 0x38, 0xc2, 0x91, 0x0c, 0x85, 0x1f, 0xc9, 0x1c, 0xe4, 0x8f, 0xa9, 0x33, 0x54, 0x6b, 0x2f, 0xba, - 0x3c, 0xca, 0x3d, 0xd0, 0x8c, 0x4d, 0x98, 0x4b, 0xbb, 0x94, 0x4f, 0xe2, 0x14, 0xca, 0xda, 0x98, - 0x50, 0x36, 0x7e, 0xd6, 0x60, 0x7e, 0x03, 0xd9, 0x96, 0xc7, 0xec, 0xbe, 0xdd, 0x13, 0x5b, 0x5a, - 0x75, 0xeb, 0x3e, 0xcc, 0x7b, 0x8e, 0xd5, 0x4d, 0xce, 0x9c, 0x51, 0xd7, 0xa7, 0x07, 0xaa, 0x6d, - 0x73, 0x9e, 0x63, 0xa5, 0xe6, 0xd3, 0x36, 0x3d, 0xe0, 0xd0, 0x9b, 0x77, 0xf1, 0x24, 0x4b, 0x2b, - 0x4a, 0x63, 0xce, 0xc5, 0x93, 0xf3, 0x5a, 0x73, 0x90, 0x77, 0xec, 0x81, 0xcd, 0xc4, 0x08, 0xce, - 0x9b, 0xd1, 0x25, 0x86, 0xf6, 0xe4, 0x29, 0xb4, 0x8d, 0x3f, 0x73, 0xb0, 0x70, 0x2e, 0x60, 0x99, - 0xff, 0x1e, 0x54, 0xdc, 0x04, 0x5d, 0x56, 0xa1, 0x75, 0x0e, 0xc6, 0x59, 0xca, 0x8d, 0x14, 0x31, - 0x65, 0x47, 0xff, 0x47, 0x83, 0x4a, 0x92, 0x7d, 0xd1, 0x66, 0xee, 0x05, 0x48, 0x19, 0x5a, 0x6a, - 0x33, 0xcb, 0x2b, 0xff, 0x9e, 0x88, 0xcc, 0xa1, 0x25, 0x57, 0x4c, 0x7c, 0xe7, 0x5a, 0x16, 0x3a, - 0xc8, 0xb5, 0xa2, 0x2c, 0xd5, 0x95, 0x3c, 0x84, 0x09, 0xcf, 0xb1, 0xe4, 0x46, 0x79, 0xe7, 0x0c, - 0xe0, 0xe8, 0x01, 0xc6, 0xb5, 0x77, 0x50, 0x02, 0xc1, 0xc6, 0xd0, 0xe4, 0x3a, 0x5c, 0xd5, 0xc5, - 0x13, 0xf1, 0xed, 0xf2, 0x3a, 0xaa, 0x2e, 0x9e, 0x18, 0xbf, 0xe7, 0x60, 0xf1, 0x42, 0x11, 0xb2, - 0x02, 0x95, 0xde, 0x30, 0x08, 0xd0, 0x65, 0x49, 0x20, 0x94, 0x25, 0x4d, 0x74, 0x72, 0x09, 0x4a, - 0x2e, 0xbe, 0x60, 0xc9, 0x96, 0x17, 0x39, 0xe1, 0x92, 0x36, 0xb7, 0xa1, 0x9a, 0x82, 0x8b, 0xa8, - 0xc4, 0x15, 0xab, 0x30, 0xad, 0x41, 0x3e, 0x07, 0xa0, 0x71, 0x98, 0xf5, 0xbc, 0x78, 0xa4, 0x1f, - 0x8d, 0x99, 0x78, 0x63, 0xd3, 0xb5, 0xf0, 0x05, 0x5a, 0xed, 0xc4, 0x14, 0x32, 0x13, 0xe6, 0xf4, - 0x8f, 0x61, 0x36, 0x43, 0x84, 0x27, 0x63, 0x73, 0xb2, 0xa8, 0x42, 0xde, 0x8c, 0x2e, 0x31, 0x34, - 0x72, 0x09, 0xcc, 0xde, 0x83, 0x1b, 0x4f, 0x69, 0x70, 0x94, 0x84, 0x50, 0x3b, 0x34, 0x91, 0x5a, - 0xea, 0xa9, 0x65, 0xe0, 0xc9, 0x58, 0x86, 0x9b, 0x17, 0x29, 0x45, 0x88, 0x35, 0x08, 0xd4, 0x36, - 0x90, 0xc9, 0x07, 0x1d, 0x59, 0x32, 0xd6, 0xe1, 0x5a, 0x82, 0xf6, 0xc6, 0x73, 0xa1, 0xf5, 0xaf, - 0x06, 0x33, 0x2a, 0xdb, 0x1d, 0x0c, 0x8e, 0xed, 0x1e, 0x92, 0x21, 0x94, 0x13, 0x3b, 0x80, 0x2c, - 0x5f, 0xb2, 0x1e, 0x44, 0x30, 0xfa, 0xca, 0x95, 0x0b, 0xc4, 0x58, 0xf9, 0xfa, 0x8f, 0xbf, 0xbf, - 0xcf, 0x2d, 0x91, 0xc5, 0xa6, 0x5a, 0x02, 0xcd, 0x97, 0xa9, 0x1d, 0xf1, 0x8a, 0x1c, 0x41, 0x25, - 0x39, 0xed, 0xc8, 0xca, 0x95, 0xc3, 0x57, 0x37, 0x2e, 0x13, 0x91, 0x9e, 0xe7, 0x84, 0xe7, 0x69, - 0xa3, 0x14, 0x7b, 0x7e, 0xa4, 0xdd, 0x6d, 0xfd, 0x94, 0x83, 0xd9, 0x64, 0xc9, 0x55, 0xee, 0xaf, - 0x60, 0xe6, 0xcc, 0xe0, 0x20, 0x6f, 0x5f, 0x31, 0x57, 0xa2, 0x50, 0x6e, 0x8f, 0x35, 0x7d, 0x8c, - 0x1b, 0x22, 0x9a, 0x05, 0x72, 0xbd, 0x99, 0x9c, 0x3c, 0x61, 0xf3, 0x65, 0x54, 0x83, 0xef, 0x34, - 0x98, 0xcf, 0x46, 0x03, 0x39, 0xb3, 0x07, 0x2f, 0x05, 0x9a, 0xfe, 0xfe, 0x78, 0xc2, 0xe9, 0xa0, - 0xee, 0x66, 0x07, 0xd5, 0x72, 0xa1, 0x1a, 0xa1, 0x46, 0x15, 0xe9, 0x0b, 0x28, 0xc5, 0xe0, 0x23, - 0x37, 0xcf, 0x25, 0x9e, 0x42, 0xaa, 0x7e, 0xeb, 0x42, 0xbe, 0xf4, 0x3e, 0x23, 0xbc, 0x97, 0x48, - 0xa1, 0x19, 0x61, 0xf2, 0xf1, 0x4d, 0x98, 0xed, 0x79, 0x83, 0xb4, 0x9a, 0xbf, 0xff, 0x59, 0x41, - 0xfe, 0x5f, 0xdd, 0x9f, 0x12, 0x1f, 0xa2, 0xf7, 0xfe, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x9f, 0xd1, - 0xa1, 0x1f, 0xc8, 0x0e, 0x00, 0x00, + // 1336 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4b, 0x6f, 0x1b, 0x55, + 0x14, 0x66, 0x9c, 0x3a, 0xb6, 0x8f, 0xed, 0xc4, 0xbd, 0x49, 0x13, 0x67, 0xd2, 0x47, 0x32, 0x50, + 0x51, 0x0a, 0xb2, 0x85, 0x5b, 0xa4, 0xb6, 0x2c, 0x90, 0x9b, 0x38, 0x21, 0x52, 0x1b, 0xa2, 0x49, + 0x1a, 0x09, 0x10, 0x32, 0x37, 0x9e, 0xe3, 0x64, 0x94, 0xf1, 0xcc, 0x30, 0x73, 0x9d, 0xd4, 0xaa, + 0xca, 0x82, 0x1d, 0x3b, 0x04, 0x0b, 0x56, 0xfc, 0x00, 0x36, 0x88, 0xff, 0xc0, 0x9e, 0x05, 0x6c, + 0x61, 0xc7, 0x82, 0x3f, 0xc0, 0x1e, 0xdd, 0xc7, 0x4c, 0x66, 0x92, 0x49, 0xe2, 0x76, 0xe5, 0x7b, + 0xde, 0x8f, 0xfb, 0xdd, 0x73, 0xc6, 0xa0, 0x53, 0xdf, 0x6e, 0x1e, 0xdd, 0x6b, 0xf6, 0x1c, 0x6a, + 0x07, 0xfe, 0x9e, 0xfc, 0x6d, 0xf8, 0x81, 0xc7, 0x3c, 0x52, 0xe9, 0x79, 0x01, 0x7a, 0x61, 0x43, + 0xf0, 0xf4, 0x5b, 0xfb, 0x9e, 0xb7, 0xef, 0x60, 0x53, 0xc8, 0xf6, 0x86, 0xfd, 0x26, 0xb3, 0x07, + 0x18, 0x32, 0x3a, 0xf0, 0xa5, 0xba, 0x7e, 0x5d, 0x29, 0x70, 0x8f, 0xd4, 0x75, 0x3d, 0x46, 0x99, + 0xed, 0xb9, 0xa1, 0x94, 0x1a, 0x3f, 0xe6, 0xa0, 0xba, 0x3b, 0x74, 0x5c, 0x0c, 0xe8, 0x9e, 0xed, + 0xd8, 0x6c, 0x44, 0x08, 0x5c, 0x71, 0xe9, 0x00, 0xeb, 0xda, 0x92, 0x76, 0xa7, 0x64, 0x8a, 0x33, + 0xb9, 0x0d, 0x53, 0xfc, 0x37, 0xf4, 0x69, 0x0f, 0xbb, 0x42, 0x9a, 0x13, 0xd2, 0x6a, 0xcc, 0xdd, + 0xe4, 0x6a, 0x4b, 0x50, 0xb6, 0x30, 0xec, 0x05, 0xb6, 0xcf, 0x43, 0xd4, 0x27, 0x84, 0x4e, 0x92, + 0xc5, 0x9d, 0x3b, 0xb6, 0x7b, 0x58, 0xbf, 0x22, 0x9d, 0xf3, 0x33, 0xd1, 0xa1, 0x18, 0xe2, 0x11, + 0x06, 0x36, 0x1b, 0xd5, 0xf3, 0x82, 0x1f, 0xd3, 0x5c, 0x36, 0x40, 0x46, 0x2d, 0xca, 0x68, 0x7d, + 0x52, 0xca, 0x22, 0x9a, 0x2c, 0x40, 0xb1, 0x6f, 0x3f, 0x47, 0xab, 0xbb, 0x37, 0xaa, 0x17, 0x84, + 0xac, 0x20, 0xe8, 0xc7, 0x23, 0xf2, 0x18, 0xae, 0xd2, 0x7e, 0x1f, 0x7b, 0x0c, 0xad, 0xee, 0x11, + 0x06, 0x21, 0x2f, 0xb8, 0x5e, 0x5c, 0x9a, 0xb8, 0x53, 0x6e, 0x5d, 0x6b, 0x24, 0xdb, 0xd7, 0x58, + 0x43, 0xca, 0x86, 0x01, 0x9a, 0xb5, 0x48, 0x7f, 0x57, 0xa9, 0x1b, 0xbf, 0x6b, 0x50, 0x5c, 0x45, + 0x86, 0x3d, 0xe6, 0x05, 0x99, 0x4d, 0xa9, 0x43, 0x41, 0xf9, 0x56, 0xdd, 0x88, 0x48, 0xd2, 0x82, + 0xbc, 0xc5, 0x46, 0x3e, 0x8a, 0x0e, 0x4c, 0xb5, 0xae, 0xa7, 0x43, 0x46, 0x4e, 0x1b, 0xab, 0x3b, + 0x23, 0x1f, 0x4d, 0xa9, 0x6a, 0x7c, 0x09, 0x79, 0x41, 0x93, 0x45, 0x98, 0x5f, 0xed, 0xec, 0x74, + 0x56, 0x76, 0x3e, 0x31, 0xbb, 0xab, 0xdd, 0x9d, 0x4f, 0xb7, 0x3a, 0xdd, 0x8d, 0xcd, 0xdd, 0xf6, + 0x93, 0x8d, 0xd5, 0xda, 0x1b, 0xe4, 0x06, 0x2c, 0x9c, 0x16, 0x6e, 0xb6, 0x9f, 0x76, 0xb6, 0xb7, + 0xda, 0x2b, 0x9d, 0x9a, 0x96, 0x65, 0xbb, 0xd6, 0x69, 0xef, 0x3c, 0x33, 0x3b, 0xb5, 0x9c, 0xb1, + 0x0d, 0xa5, 0xcd, 0xe8, 0xba, 0x32, 0x0b, 0x6a, 0x41, 0xd1, 0x52, 0xb9, 0x89, 0x8a, 0xca, 0xad, + 0xb9, 0xec, 0xcc, 0xcd, 0x58, 0xcf, 0xf8, 0x2e, 0x07, 0x05, 0xd5, 0xc3, 0x4c, 0x9f, 0x1f, 0x40, + 0x29, 0xc6, 0x88, 0x72, 0x3a, 0x9f, 0x76, 0x1a, 0xe7, 0x64, 0x9e, 0x68, 0x26, 0x7b, 0x3b, 0x91, + 0xee, 0xed, 0x6d, 0x98, 0x52, 0xc7, 0x6e, 0xdf, 0x0b, 0x06, 0x94, 0x29, 0x2c, 0x55, 0x15, 0x77, + 0x4d, 0x30, 0x53, 0xb5, 0xe4, 0xc7, 0xab, 0x85, 0x74, 0x60, 0xfa, 0x28, 0xf1, 0x14, 0x6c, 0x0c, + 0xeb, 0x93, 0x02, 0x33, 0x8b, 0x69, 0xd3, 0xd4, 0x7b, 0x31, 0x4f, 0xdb, 0x18, 0x8b, 0x90, 0x7f, + 0x42, 0x47, 0x28, 0x40, 0x73, 0x40, 0xc3, 0x83, 0xa8, 0x1f, 0xfc, 0x6c, 0x7c, 0xab, 0x41, 0x79, + 0x85, 0x7b, 0xd9, 0x66, 0x94, 0x0d, 0x43, 0x72, 0x1f, 0x4a, 0x51, 0xfc, 0xb0, 0xae, 0x89, 0x68, + 0xe7, 0x25, 0x7a, 0xa2, 0x48, 0x56, 0xa1, 0xe6, 0xd0, 0x90, 0x75, 0x87, 0xbe, 0x45, 0x19, 0x76, + 0xf9, 0x93, 0x57, 0xcd, 0xd5, 0x1b, 0xf2, 0xb9, 0x37, 0xa2, 0x79, 0xd0, 0xd8, 0x89, 0xe6, 0x81, + 0x39, 0xc5, 0x6d, 0x9e, 0x09, 0x13, 0xce, 0x34, 0x1e, 0x02, 0x59, 0x47, 0xd6, 0x76, 0x7b, 0x18, + 0xb2, 0x60, 0x64, 0xe2, 0x57, 0x43, 0x0c, 0x19, 0x79, 0x13, 0xaa, 0x54, 0xb1, 0xba, 0x89, 0xeb, + 0xac, 0x44, 0x4c, 0x7e, 0x5f, 0xc6, 0xaf, 0x13, 0x30, 0x93, 0xb2, 0x0d, 0x7d, 0xcf, 0x0d, 0x91, + 0xac, 0x41, 0x31, 0xd2, 0x13, 0x76, 0xe5, 0xd6, 0xdd, 0x74, 0x35, 0x19, 0x46, 0x8d, 0x98, 0x11, + 0xdb, 0x92, 0xf7, 0x61, 0x32, 0x14, 0x0d, 0x52, 0x65, 0x2d, 0xa4, 0xbd, 0x24, 0x3a, 0x68, 0x2a, + 0x45, 0xfd, 0x6b, 0xa8, 0x46, 0x8e, 0x64, 0xfb, 0xdf, 0x81, 0xbc, 0xc3, 0x0f, 0x2a, 0x91, 0x99, + 0xb4, 0x0b, 0xa1, 0x63, 0x4a, 0x0d, 0x3e, 0x2f, 0x64, 0x73, 0xd1, 0xea, 0xf6, 0x25, 0x9a, 0x79, + 0xe4, 0x8b, 0xe6, 0x45, 0xa4, 0xaf, 0x18, 0xa1, 0xfe, 0x93, 0x06, 0xc5, 0x28, 0x81, 0xcc, 0xa7, + 0x90, 0xba, 0xea, 0xdc, 0xb8, 0x57, 0xbd, 0x0e, 0x93, 0x22, 0xc7, 0xb0, 0x3e, 0x21, 0x4c, 0x9a, + 0xe3, 0xf7, 0x53, 0x96, 0xa8, 0xcc, 0x8d, 0xbf, 0x73, 0x30, 0xb3, 0xe5, 0x85, 0xaf, 0x75, 0xdf, + 0x64, 0x0e, 0x26, 0xd5, 0x6b, 0x93, 0xa3, 0x4e, 0x51, 0x64, 0xe5, 0x54, 0x76, 0xef, 0xa6, 0xb3, + 0xcb, 0x88, 0x27, 0x78, 0xa9, 0xcc, 0xf4, 0xdf, 0x34, 0x28, 0xc5, 0xdc, 0xac, 0x57, 0xc3, 0x79, + 0x3e, 0x65, 0x07, 0x2a, 0xb8, 0x38, 0x13, 0x13, 0x0a, 0x07, 0x48, 0xad, 0x93, 0xd8, 0x0f, 0x5e, + 0x21, 0x76, 0xe3, 0x63, 0x69, 0xda, 0x71, 0xb9, 0x34, 0x72, 0xa4, 0x3f, 0x82, 0x4a, 0x52, 0x40, + 0x6a, 0x30, 0x71, 0x88, 0x23, 0x95, 0x0a, 0x3f, 0x92, 0x59, 0xc8, 0x1f, 0x51, 0x67, 0x18, 0x2d, + 0x40, 0x49, 0x3c, 0xca, 0x3d, 0xd0, 0x8c, 0x0d, 0x98, 0x4d, 0x87, 0x54, 0x4f, 0xe2, 0x04, 0xca, + 0xda, 0x98, 0x50, 0x36, 0x7e, 0xd1, 0x60, 0x6e, 0x1d, 0xd9, 0xa6, 0xc7, 0xec, 0xbe, 0xdd, 0x13, + 0xfb, 0x3a, 0xba, 0xad, 0xfb, 0x30, 0xe7, 0x39, 0x56, 0x37, 0x39, 0x73, 0x46, 0x5d, 0x9f, 0xee, + 0x47, 0xd7, 0x36, 0xeb, 0x39, 0x56, 0x6a, 0x3e, 0x6d, 0xd1, 0x7d, 0x0e, 0xbd, 0x39, 0x17, 0x8f, + 0xb3, 0xac, 0x64, 0x19, 0xb3, 0x2e, 0x1e, 0x9f, 0xb5, 0x9a, 0x85, 0xbc, 0x63, 0x0f, 0x6c, 0x26, + 0x46, 0x70, 0xde, 0x94, 0x44, 0x0c, 0xed, 0x2b, 0x27, 0xd0, 0x36, 0xfe, 0xca, 0xc1, 0xfc, 0x99, + 0x84, 0x55, 0xfd, 0xbb, 0x50, 0x71, 0x13, 0x7c, 0xd5, 0x85, 0xd6, 0x19, 0x18, 0x67, 0x19, 0x37, + 0x52, 0xcc, 0x94, 0x1f, 0xfd, 0x5f, 0x0d, 0x2a, 0x49, 0xf1, 0x79, 0x3b, 0xba, 0x17, 0x20, 0x65, + 0x68, 0x45, 0x3b, 0x5a, 0x91, 0xfc, 0xcb, 0x42, 0xba, 0x43, 0x4b, 0xad, 0x98, 0x98, 0xe6, 0x56, + 0x16, 0x3a, 0xc8, 0xad, 0x64, 0x95, 0x11, 0x49, 0x1e, 0xc2, 0x84, 0xe7, 0x58, 0x6a, 0xa3, 0xbc, + 0x7d, 0x0a, 0x70, 0x74, 0x1f, 0xe3, 0xde, 0x3b, 0xa8, 0x80, 0x60, 0x63, 0x68, 0x72, 0x1b, 0x6e, + 0xea, 0xe2, 0xb1, 0xf8, 0x8a, 0x79, 0x15, 0x53, 0x17, 0x8f, 0x8d, 0x3f, 0x72, 0xb0, 0x70, 0xae, + 0x0a, 0x59, 0x86, 0x4a, 0x6f, 0x18, 0x04, 0xe8, 0xb2, 0x24, 0x10, 0xca, 0x8a, 0x27, 0x6e, 0x72, + 0x11, 0x4a, 0x2e, 0x3e, 0x67, 0xc9, 0x2b, 0x2f, 0x72, 0xc6, 0x05, 0xd7, 0xdc, 0x86, 0x6a, 0x0a, + 0x2e, 0xa2, 0x13, 0x97, 0xac, 0xc2, 0xb4, 0x05, 0xf9, 0x1c, 0x80, 0xc6, 0x69, 0xd6, 0xf3, 0xe2, + 0x91, 0x7e, 0x38, 0x66, 0xe1, 0x8d, 0x0d, 0xd7, 0xc2, 0xe7, 0x68, 0xb5, 0x13, 0x53, 0xc8, 0x4c, + 0xb8, 0xd3, 0x3f, 0x82, 0x99, 0x0c, 0x15, 0x5e, 0x8c, 0xcd, 0xd9, 0xa2, 0x0b, 0x79, 0x53, 0x12, + 0x31, 0x34, 0x72, 0x09, 0xcc, 0xde, 0x83, 0x1b, 0x4f, 0x69, 0x70, 0x98, 0x84, 0x50, 0x3b, 0x34, + 0x91, 0x5a, 0xd1, 0x53, 0xcb, 0xc0, 0x93, 0xb1, 0x04, 0x37, 0xcf, 0x33, 0x92, 0x88, 0x35, 0x08, + 0xd4, 0xd6, 0x91, 0xa9, 0x07, 0x2d, 0x3d, 0x19, 0x6b, 0x70, 0x35, 0xc1, 0x7b, 0xed, 0xb9, 0xd0, + 0xfa, 0x4f, 0x83, 0xe9, 0xa8, 0xda, 0x6d, 0x0c, 0x8e, 0xec, 0x1e, 0x92, 0x21, 0x94, 0x13, 0x3b, + 0x80, 0x2c, 0x5d, 0xb0, 0x1e, 0x44, 0x32, 0xfa, 0xf2, 0xa5, 0x0b, 0xc4, 0x58, 0xfe, 0xe6, 0xcf, + 0x7f, 0x7e, 0xc8, 0x2d, 0x92, 0x85, 0x66, 0xb4, 0x04, 0x9a, 0x2f, 0x52, 0x3b, 0xe2, 0x25, 0x39, + 0x84, 0x4a, 0x72, 0xda, 0x91, 0xe5, 0x4b, 0x87, 0xaf, 0x6e, 0x5c, 0xa4, 0xa2, 0x22, 0xcf, 0x8a, + 0xc8, 0x53, 0x46, 0x29, 0x8e, 0xfc, 0x48, 0xbb, 0xdb, 0xfa, 0x39, 0x07, 0x33, 0xc9, 0x96, 0x47, + 0xb5, 0xbf, 0x84, 0xe9, 0x53, 0x83, 0x83, 0xbc, 0x75, 0xc9, 0x5c, 0x91, 0xa9, 0xdc, 0x1e, 0x6b, + 0xfa, 0x18, 0x37, 0x44, 0x36, 0xf3, 0xe4, 0x5a, 0x33, 0x39, 0x79, 0xc2, 0xe6, 0x0b, 0xd9, 0x83, + 0xef, 0x35, 0x98, 0xcb, 0x46, 0x03, 0x39, 0xb5, 0x07, 0x2f, 0x04, 0x9a, 0xfe, 0xde, 0x78, 0xca, + 0xe9, 0xa4, 0xee, 0x66, 0x27, 0xd5, 0x72, 0xa1, 0x2a, 0x51, 0x13, 0x35, 0xe9, 0x0b, 0x28, 0xc5, + 0xe0, 0x23, 0x37, 0xcf, 0x14, 0x9e, 0x42, 0xaa, 0x7e, 0xeb, 0x5c, 0xb9, 0x8a, 0x3e, 0x2d, 0xa2, + 0x97, 0x48, 0xa1, 0x29, 0x31, 0xf9, 0xf8, 0x26, 0xcc, 0xf4, 0xbc, 0x41, 0xda, 0xcc, 0xdf, 0xfb, + 0xac, 0xa0, 0xfe, 0xb9, 0xee, 0x4d, 0x8a, 0x0f, 0xd1, 0x7b, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, + 0xcb, 0x5c, 0xce, 0x34, 0xd2, 0x0e, 0x00, 0x00, } diff --git a/api/v3/clairpb/clair.proto b/api/v3/clairpb/clair.proto index f8545cff..015ab3ac 100644 --- a/api/v3/clairpb/clair.proto +++ b/api/v3/clairpb/clair.proto @@ -45,17 +45,17 @@ message Vulnerability { } message Detector { - enum Type { - DETECTOR_TYPE_INVALID = 0; - DETECTOR_TYPE_NAMESPACE = 1; - DETECTOR_TYPE_FEATURE = 2; + enum DType { + DETECTOR_D_TYPE_INVALID = 0; + DETECTOR_D_TYPE_NAMESPACE = 1; + DETECTOR_D_TYPE_FEATURE = 2; } // The name of the detector. string name = 1; // The version of the detector. string version = 2; // The type of the detector. - Type type = 3; + DType dtype = 3; } message Namespace { diff --git a/api/v3/clairpb/clair.swagger.json b/api/v3/clairpb/clair.swagger.json index a967092f..19396b43 100644 --- a/api/v3/clairpb/clair.swagger.json +++ b/api/v3/clairpb/clair.swagger.json @@ -156,6 +156,15 @@ } }, "definitions": { + "DetectorDType": { + "type": "string", + "enum": [ + "DETECTOR_D_TYPE_INVALID", + "DETECTOR_D_TYPE_NAMESPACE", + "DETECTOR_D_TYPE_FEATURE" + ], + "default": "DETECTOR_D_TYPE_INVALID" + }, "GetAncestryResponseAncestry": { "type": "object", "properties": { @@ -286,21 +295,12 @@ "type": "string", "description": "The version of the detector." }, - "type": { - "$ref": "#/definitions/clairDetectorType", + "dtype": { + "$ref": "#/definitions/DetectorDType", "description": "The type of the detector." } } }, - "clairDetectorType": { - "type": "string", - "enum": [ - "DETECTOR_TYPE_INVALID", - "DETECTOR_TYPE_NAMESPACE", - "DETECTOR_TYPE_FEATURE" - ], - "default": "DETECTOR_TYPE_INVALID" - }, "clairFeature": { "type": "object", "properties": { diff --git a/api/v3/clairpb/convert.go b/api/v3/clairpb/convert.go index da4e9e33..fe54b2f9 100644 --- a/api/v3/clairpb/convert.go +++ b/api/v3/clairpb/convert.go @@ -24,9 +24,9 @@ import ( // DatabaseDetectorTypeMapping maps the database detector type to the integer // enum proto. -var DatabaseDetectorTypeMapping = map[database.DetectorType]Detector_Type{ - database.NamespaceDetectorType: Detector_Type(1), - database.FeatureDetectorType: Detector_Type(2), +var DatabaseDetectorTypeMapping = map[database.DetectorType]Detector_DType{ + database.NamespaceDetectorType: Detector_DType(1), + database.FeatureDetectorType: Detector_DType(2), } // PagedVulnerableAncestriesFromDatabaseModel converts database @@ -152,7 +152,7 @@ func DetectorFromDatabaseModel(detector database.Detector) *Detector { return &Detector{ Name: detector.Name, Version: detector.Version, - Type: DatabaseDetectorTypeMapping[detector.DType], + Dtype: DatabaseDetectorTypeMapping[detector.DType], } }