diff --git a/api/v3/clairpb/clair.pb.go b/api/v3/clairpb/clair.pb.go index ac441ec0..1306dbee 100644 --- a/api/v3/clairpb/clair.pb.go +++ b/api/v3/clairpb/clair.pb.go @@ -9,6 +9,8 @@ It is generated from these files: It has these top-level messages: Vulnerability + Detector + Namespace Feature Layer ClairStatus @@ -48,6 +50,30 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +type Detector_DType int32 + +const ( + Detector_DETECTOR_D_TYPE_INVALID Detector_DType = 0 + Detector_DETECTOR_D_TYPE_NAMESPACE Detector_DType = 1 + Detector_DETECTOR_D_TYPE_FEATURE Detector_DType = 2 +) + +var Detector_DType_name = map[int32]string{ + 0: "DETECTOR_D_TYPE_INVALID", + 1: "DETECTOR_D_TYPE_NAMESPACE", + 2: "DETECTOR_D_TYPE_FEATURE", +} +var Detector_DType_value = map[string]int32{ + "DETECTOR_D_TYPE_INVALID": 0, + "DETECTOR_D_TYPE_NAMESPACE": 1, + "DETECTOR_D_TYPE_FEATURE": 2, +} + +func (x Detector_DType) String() string { + return proto.EnumName(Detector_DType_name, int32(x)) +} +func (Detector_DType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1, 0} } + type Vulnerability struct { // The name of the vulnerability. Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` @@ -130,23 +156,88 @@ func (m *Vulnerability) GetAffectedVersions() []*Feature { return nil } +type Detector struct { + // The name of the detector. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The version of the detector. + Version string `protobuf:"bytes,2,opt,name=version" json:"version,omitempty"` + // The type of the detector. + Dtype Detector_DType `protobuf:"varint,3,opt,name=dtype,enum=coreos.clair.Detector_DType" json:"dtype,omitempty"` +} + +func (m *Detector) Reset() { *m = Detector{} } +func (m *Detector) String() string { return proto.CompactTextString(m) } +func (*Detector) ProtoMessage() {} +func (*Detector) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *Detector) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Detector) GetVersion() string { + if m != nil { + return m.Version + } + return "" +} + +func (m *Detector) GetDtype() Detector_DType { + if m != nil { + return m.Dtype + } + return Detector_DETECTOR_D_TYPE_INVALID +} + +type Namespace struct { + // The name of the namespace. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The detector used to detect the namespace. This only exists when present in + // an Ancestry Feature. + Detector *Detector `protobuf:"bytes,2,opt,name=detector" json:"detector,omitempty"` +} + +func (m *Namespace) Reset() { *m = Namespace{} } +func (m *Namespace) String() string { return proto.CompactTextString(m) } +func (*Namespace) ProtoMessage() {} +func (*Namespace) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } + +func (m *Namespace) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Namespace) GetDetector() *Detector { + if m != nil { + return m.Detector + } + return nil +} + type Feature struct { // The name of the feature. Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - // The name of the namespace in which the feature is detected. - NamespaceName string `protobuf:"bytes,2,opt,name=namespace_name,json=namespaceName" json:"namespace_name,omitempty"` + // The namespace in which the feature is detected. + Namespace *Namespace `protobuf:"bytes,2,opt,name=namespace" json:"namespace,omitempty"` // The specific version of this feature. Version string `protobuf:"bytes,3,opt,name=version" json:"version,omitempty"` // The format used to parse version numbers for the feature. VersionFormat string `protobuf:"bytes,4,opt,name=version_format,json=versionFormat" json:"version_format,omitempty"` + // The detector used to detect this feature. This only exists when present in + // an Ancestry. + Detector *Detector `protobuf:"bytes,5,opt,name=detector" json:"detector,omitempty"` // The list of vulnerabilities that affect the feature. - Vulnerabilities []*Vulnerability `protobuf:"bytes,5,rep,name=vulnerabilities" json:"vulnerabilities,omitempty"` + Vulnerabilities []*Vulnerability `protobuf:"bytes,6,rep,name=vulnerabilities" json:"vulnerabilities,omitempty"` } func (m *Feature) Reset() { *m = Feature{} } func (m *Feature) String() string { return proto.CompactTextString(m) } func (*Feature) ProtoMessage() {} -func (*Feature) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } +func (*Feature) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } func (m *Feature) GetName() string { if m != nil { @@ -155,11 +246,11 @@ func (m *Feature) GetName() string { return "" } -func (m *Feature) GetNamespaceName() string { +func (m *Feature) GetNamespace() *Namespace { if m != nil { - return m.NamespaceName + return m.Namespace } - return "" + return nil } func (m *Feature) GetVersion() string { @@ -176,6 +267,13 @@ func (m *Feature) GetVersionFormat() string { return "" } +func (m *Feature) GetDetector() *Detector { + if m != nil { + return m.Detector + } + return nil +} + func (m *Feature) GetVulnerabilities() []*Vulnerability { if m != nil { return m.Vulnerabilities @@ -191,7 +289,7 @@ type Layer struct { func (m *Layer) Reset() { *m = Layer{} } func (m *Layer) String() string { return proto.CompactTextString(m) } func (*Layer) ProtoMessage() {} -func (*Layer) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } +func (*Layer) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } func (m *Layer) GetHash() string { if m != nil { @@ -201,27 +299,18 @@ func (m *Layer) GetHash() string { } type ClairStatus struct { - // The configured list of feature listers used to scan an ancestry. - Listers []string `protobuf:"bytes,1,rep,name=listers" json:"listers,omitempty"` - // The configured list of namespace detectors used to scan an ancestry. - Detectors []string `protobuf:"bytes,2,rep,name=detectors" json:"detectors,omitempty"` + // The implemented detectors in this Clair instance + Detectors []*Detector `protobuf:"bytes,1,rep,name=detectors" json:"detectors,omitempty"` // The time at which the updater last ran. - LastUpdateTime *google_protobuf.Timestamp `protobuf:"bytes,3,opt,name=last_update_time,json=lastUpdateTime" json:"last_update_time,omitempty"` + LastUpdateTime *google_protobuf.Timestamp `protobuf:"bytes,2,opt,name=last_update_time,json=lastUpdateTime" json:"last_update_time,omitempty"` } func (m *ClairStatus) Reset() { *m = ClairStatus{} } func (m *ClairStatus) String() string { return proto.CompactTextString(m) } func (*ClairStatus) ProtoMessage() {} -func (*ClairStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } +func (*ClairStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } -func (m *ClairStatus) GetListers() []string { - if m != nil { - return m.Listers - } - return nil -} - -func (m *ClairStatus) GetDetectors() []string { +func (m *ClairStatus) GetDetectors() []*Detector { if m != nil { return m.Detectors } @@ -243,7 +332,7 @@ type GetAncestryRequest struct { func (m *GetAncestryRequest) Reset() { *m = GetAncestryRequest{} } func (m *GetAncestryRequest) String() string { return proto.CompactTextString(m) } func (*GetAncestryRequest) ProtoMessage() {} -func (*GetAncestryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } +func (*GetAncestryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } func (m *GetAncestryRequest) GetAncestryName() string { if m != nil { @@ -262,7 +351,7 @@ type GetAncestryResponse struct { func (m *GetAncestryResponse) Reset() { *m = GetAncestryResponse{} } func (m *GetAncestryResponse) String() string { return proto.CompactTextString(m) } func (*GetAncestryResponse) ProtoMessage() {} -func (*GetAncestryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } +func (*GetAncestryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } func (m *GetAncestryResponse) GetAncestry() *GetAncestryResponse_Ancestry { if m != nil { @@ -289,7 +378,7 @@ func (m *GetAncestryResponse_AncestryLayer) Reset() { *m = GetAncestryRe func (m *GetAncestryResponse_AncestryLayer) String() string { return proto.CompactTextString(m) } func (*GetAncestryResponse_AncestryLayer) ProtoMessage() {} func (*GetAncestryResponse_AncestryLayer) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{5, 0} + return fileDescriptor0, []int{7, 0} } func (m *GetAncestryResponse_AncestryLayer) GetLayer() *Layer { @@ -309,18 +398,17 @@ func (m *GetAncestryResponse_AncestryLayer) GetDetectedFeatures() []*Feature { type GetAncestryResponse_Ancestry struct { // The name of the desired ancestry. Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - // The configured list of feature listers used to scan this ancestry. - ScannedListers []string `protobuf:"bytes,4,rep,name=scanned_listers,json=scannedListers" json:"scanned_listers,omitempty"` - // The configured list of namespace detectors used to scan an ancestry. - ScannedDetectors []string `protobuf:"bytes,5,rep,name=scanned_detectors,json=scannedDetectors" json:"scanned_detectors,omitempty"` + // The detectors used to scan this Ancestry. It may not be the current set + // of detectors in clair status. + Detectors []*Detector `protobuf:"bytes,2,rep,name=detectors" json:"detectors,omitempty"` // The list of layers along with detected features in each. - Layers []*GetAncestryResponse_AncestryLayer `protobuf:"bytes,6,rep,name=layers" json:"layers,omitempty"` + Layers []*GetAncestryResponse_AncestryLayer `protobuf:"bytes,3,rep,name=layers" json:"layers,omitempty"` } func (m *GetAncestryResponse_Ancestry) Reset() { *m = GetAncestryResponse_Ancestry{} } func (m *GetAncestryResponse_Ancestry) String() string { return proto.CompactTextString(m) } func (*GetAncestryResponse_Ancestry) ProtoMessage() {} -func (*GetAncestryResponse_Ancestry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5, 1} } +func (*GetAncestryResponse_Ancestry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7, 1} } func (m *GetAncestryResponse_Ancestry) GetName() string { if m != nil { @@ -329,16 +417,9 @@ func (m *GetAncestryResponse_Ancestry) GetName() string { return "" } -func (m *GetAncestryResponse_Ancestry) GetScannedListers() []string { +func (m *GetAncestryResponse_Ancestry) GetDetectors() []*Detector { if m != nil { - return m.ScannedListers - } - return nil -} - -func (m *GetAncestryResponse_Ancestry) GetScannedDetectors() []string { - if m != nil { - return m.ScannedDetectors + return m.Detectors } return nil } @@ -364,7 +445,7 @@ type PostAncestryRequest struct { func (m *PostAncestryRequest) Reset() { *m = PostAncestryRequest{} } func (m *PostAncestryRequest) String() string { return proto.CompactTextString(m) } func (*PostAncestryRequest) ProtoMessage() {} -func (*PostAncestryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } +func (*PostAncestryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } func (m *PostAncestryRequest) GetAncestryName() string { if m != nil { @@ -400,7 +481,7 @@ func (m *PostAncestryRequest_PostLayer) Reset() { *m = PostAncestryReque func (m *PostAncestryRequest_PostLayer) String() string { return proto.CompactTextString(m) } func (*PostAncestryRequest_PostLayer) ProtoMessage() {} func (*PostAncestryRequest_PostLayer) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{6, 0} + return fileDescriptor0, []int{8, 0} } func (m *PostAncestryRequest_PostLayer) GetHash() string { @@ -432,7 +513,7 @@ type PostAncestryResponse struct { func (m *PostAncestryResponse) Reset() { *m = PostAncestryResponse{} } func (m *PostAncestryResponse) String() string { return proto.CompactTextString(m) } func (*PostAncestryResponse) ProtoMessage() {} -func (*PostAncestryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } +func (*PostAncestryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } func (m *PostAncestryResponse) GetStatus() *ClairStatus { if m != nil { @@ -457,7 +538,7 @@ type GetNotificationRequest struct { func (m *GetNotificationRequest) Reset() { *m = GetNotificationRequest{} } func (m *GetNotificationRequest) String() string { return proto.CompactTextString(m) } func (*GetNotificationRequest) ProtoMessage() {} -func (*GetNotificationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } +func (*GetNotificationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } func (m *GetNotificationRequest) GetOldVulnerabilityPage() string { if m != nil { @@ -495,7 +576,7 @@ type GetNotificationResponse struct { func (m *GetNotificationResponse) Reset() { *m = GetNotificationResponse{} } func (m *GetNotificationResponse) String() string { return proto.CompactTextString(m) } func (*GetNotificationResponse) ProtoMessage() {} -func (*GetNotificationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } +func (*GetNotificationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } func (m *GetNotificationResponse) GetNotification() *GetNotificationResponse_Notification { if m != nil { @@ -523,7 +604,7 @@ func (m *GetNotificationResponse_Notification) Reset() { *m = GetNotific func (m *GetNotificationResponse_Notification) String() string { return proto.CompactTextString(m) } func (*GetNotificationResponse_Notification) ProtoMessage() {} func (*GetNotificationResponse_Notification) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{9, 0} + return fileDescriptor0, []int{11, 0} } func (m *GetNotificationResponse_Notification) GetName() string { @@ -585,7 +666,7 @@ type PagedVulnerableAncestries struct { func (m *PagedVulnerableAncestries) Reset() { *m = PagedVulnerableAncestries{} } func (m *PagedVulnerableAncestries) String() string { return proto.CompactTextString(m) } func (*PagedVulnerableAncestries) ProtoMessage() {} -func (*PagedVulnerableAncestries) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } +func (*PagedVulnerableAncestries) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } func (m *PagedVulnerableAncestries) GetCurrentPage() string { if m != nil { @@ -638,7 +719,7 @@ func (m *PagedVulnerableAncestries_IndexedAncestryName) String() string { } func (*PagedVulnerableAncestries_IndexedAncestryName) ProtoMessage() {} func (*PagedVulnerableAncestries_IndexedAncestryName) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{10, 0} + return fileDescriptor0, []int{12, 0} } func (m *PagedVulnerableAncestries_IndexedAncestryName) GetIndex() int32 { @@ -663,7 +744,7 @@ type MarkNotificationAsReadRequest struct { func (m *MarkNotificationAsReadRequest) Reset() { *m = MarkNotificationAsReadRequest{} } func (m *MarkNotificationAsReadRequest) String() string { return proto.CompactTextString(m) } func (*MarkNotificationAsReadRequest) ProtoMessage() {} -func (*MarkNotificationAsReadRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } +func (*MarkNotificationAsReadRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } func (m *MarkNotificationAsReadRequest) GetName() string { if m != nil { @@ -678,7 +759,7 @@ type MarkNotificationAsReadResponse struct { func (m *MarkNotificationAsReadResponse) Reset() { *m = MarkNotificationAsReadResponse{} } func (m *MarkNotificationAsReadResponse) String() string { return proto.CompactTextString(m) } func (*MarkNotificationAsReadResponse) ProtoMessage() {} -func (*MarkNotificationAsReadResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } +func (*MarkNotificationAsReadResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} } type GetStatusRequest struct { } @@ -686,7 +767,7 @@ type GetStatusRequest struct { func (m *GetStatusRequest) Reset() { *m = GetStatusRequest{} } func (m *GetStatusRequest) String() string { return proto.CompactTextString(m) } func (*GetStatusRequest) ProtoMessage() {} -func (*GetStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } +func (*GetStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} } type GetStatusResponse struct { // The status of the current Clair instance. @@ -696,7 +777,7 @@ type GetStatusResponse struct { func (m *GetStatusResponse) Reset() { *m = GetStatusResponse{} } func (m *GetStatusResponse) String() string { return proto.CompactTextString(m) } func (*GetStatusResponse) ProtoMessage() {} -func (*GetStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} } +func (*GetStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} } func (m *GetStatusResponse) GetStatus() *ClairStatus { if m != nil { @@ -707,6 +788,8 @@ func (m *GetStatusResponse) GetStatus() *ClairStatus { func init() { proto.RegisterType((*Vulnerability)(nil), "coreos.clair.Vulnerability") + proto.RegisterType((*Detector)(nil), "coreos.clair.Detector") + proto.RegisterType((*Namespace)(nil), "coreos.clair.Namespace") proto.RegisterType((*Feature)(nil), "coreos.clair.Feature") proto.RegisterType((*Layer)(nil), "coreos.clair.Layer") proto.RegisterType((*ClairStatus)(nil), "coreos.clair.ClairStatus") @@ -726,6 +809,7 @@ func init() { proto.RegisterType((*MarkNotificationAsReadResponse)(nil), "coreos.clair.MarkNotificationAsReadResponse") proto.RegisterType((*GetStatusRequest)(nil), "coreos.clair.GetStatusRequest") proto.RegisterType((*GetStatusResponse)(nil), "coreos.clair.GetStatusResponse") + proto.RegisterEnum("coreos.clair.Detector_DType", Detector_DType_name, Detector_DType_value) } // Reference imports to suppress errors if they are not otherwise used. @@ -1007,83 +1091,89 @@ var _StatusService_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("api/v3/clairpb/clair.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1237 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x57, 0x4b, 0x6f, 0x1b, 0xd5, - 0x17, 0xd7, 0xd8, 0x71, 0x1c, 0x1f, 0xdb, 0x49, 0x7a, 0x93, 0xa6, 0x93, 0x49, 0x1f, 0xc9, 0xfc, - 0xff, 0x55, 0x4b, 0x8b, 0x6c, 0xe1, 0xb2, 0x68, 0xcb, 0x02, 0xa5, 0x8f, 0x84, 0x4a, 0xa5, 0xaa, - 0xa6, 0xd0, 0x05, 0x08, 0x59, 0xd7, 0x33, 0xc7, 0xc9, 0x28, 0xe3, 0x19, 0x33, 0xf7, 0xda, 0x89, - 0x55, 0x95, 0x05, 0x5b, 0x76, 0xb0, 0xe0, 0x33, 0xb0, 0xe1, 0x1b, 0xb0, 0x62, 0xcb, 0x02, 0xc1, - 0x16, 0x76, 0x2c, 0xf8, 0x02, 0xec, 0xd1, 0x7d, 0x4d, 0x66, 0x12, 0xe7, 0xd1, 0xb2, 0xf2, 0x9c, - 0xf7, 0xeb, 0x77, 0xcf, 0x49, 0xc0, 0xa1, 0xc3, 0xb0, 0x3d, 0xbe, 0xd3, 0xf6, 0x23, 0x1a, 0xa6, - 0xc3, 0x9e, 0xfa, 0x6d, 0x0d, 0xd3, 0x84, 0x27, 0xa4, 0xe1, 0x27, 0x29, 0x26, 0xac, 0x25, 0x79, - 0xce, 0xb5, 0x9d, 0x24, 0xd9, 0x89, 0xb0, 0x2d, 0x65, 0xbd, 0x51, 0xbf, 0xcd, 0xc3, 0x01, 0x32, - 0x4e, 0x07, 0x43, 0xa5, 0xee, 0x5c, 0xd6, 0x0a, 0xc2, 0x23, 0x8d, 0xe3, 0x84, 0x53, 0x1e, 0x26, - 0x31, 0x53, 0x52, 0xf7, 0xfb, 0x12, 0x34, 0x5f, 0x8e, 0xa2, 0x18, 0x53, 0xda, 0x0b, 0xa3, 0x90, - 0x4f, 0x08, 0x81, 0x99, 0x98, 0x0e, 0xd0, 0xb6, 0xd6, 0xad, 0x9b, 0x35, 0x4f, 0x7e, 0x93, 0xeb, - 0x30, 0x2f, 0x7e, 0xd9, 0x90, 0xfa, 0xd8, 0x95, 0xd2, 0x92, 0x94, 0x36, 0x33, 0xee, 0x33, 0xa1, - 0xb6, 0x0e, 0xf5, 0x00, 0x99, 0x9f, 0x86, 0x43, 0x11, 0xc2, 0x2e, 0x4b, 0x9d, 0x3c, 0x4b, 0x38, - 0x8f, 0xc2, 0x78, 0xcf, 0x9e, 0x51, 0xce, 0xc5, 0x37, 0x71, 0x60, 0x8e, 0xe1, 0x18, 0xd3, 0x90, - 0x4f, 0xec, 0x8a, 0xe4, 0x67, 0xb4, 0x90, 0x0d, 0x90, 0xd3, 0x80, 0x72, 0x6a, 0xcf, 0x2a, 0x99, - 0xa1, 0xc9, 0x2a, 0xcc, 0xf5, 0xc3, 0x03, 0x0c, 0xba, 0xbd, 0x89, 0x5d, 0x95, 0xb2, 0xaa, 0xa4, - 0x1f, 0x4c, 0xc8, 0x03, 0xb8, 0x40, 0xfb, 0x7d, 0xf4, 0x39, 0x06, 0xdd, 0x31, 0xa6, 0x4c, 0x14, - 0x6c, 0xcf, 0xad, 0x97, 0x6f, 0xd6, 0x3b, 0x17, 0x5b, 0xf9, 0xf6, 0xb5, 0xb6, 0x90, 0xf2, 0x51, - 0x8a, 0xde, 0xa2, 0xd1, 0x7f, 0xa9, 0xd5, 0xdd, 0x5f, 0x2c, 0xa8, 0x6a, 0xe9, 0x7f, 0xe9, 0x89, - 0x0d, 0x55, 0x9d, 0x81, 0xee, 0x87, 0x21, 0x85, 0x03, 0xfd, 0xd9, 0xed, 0x27, 0xe9, 0x80, 0x72, - 0xdd, 0x95, 0xa6, 0xe6, 0x6e, 0x49, 0x26, 0x79, 0x0c, 0x0b, 0xe3, 0xdc, 0x80, 0x42, 0x64, 0x76, - 0x45, 0x56, 0xb2, 0x56, 0xac, 0xa4, 0x30, 0x45, 0xef, 0xa8, 0x8d, 0xbb, 0x06, 0x95, 0xa7, 0x74, - 0x82, 0xa9, 0xa8, 0x65, 0x97, 0xb2, 0x5d, 0x53, 0x8b, 0xf8, 0x76, 0xbf, 0xb1, 0xa0, 0xfe, 0x50, - 0x78, 0x79, 0xc1, 0x29, 0x1f, 0x31, 0x91, 0x74, 0x14, 0x32, 0x8e, 0x29, 0xb3, 0xad, 0xf5, 0xb2, - 0x48, 0x5a, 0x93, 0xe4, 0x32, 0xd4, 0x02, 0xe4, 0xe8, 0xf3, 0x24, 0x65, 0x76, 0x49, 0xca, 0x0e, - 0x19, 0xe4, 0x11, 0x2c, 0x46, 0x94, 0xf1, 0xee, 0x68, 0x18, 0x50, 0x8e, 0x5d, 0x01, 0x45, 0x59, - 0x75, 0xbd, 0xe3, 0xb4, 0x14, 0x0c, 0x5b, 0x06, 0xa7, 0xad, 0x4f, 0x0c, 0x4e, 0xbd, 0x79, 0x61, - 0xf3, 0xa9, 0x34, 0x11, 0x4c, 0xf7, 0x1e, 0x90, 0x6d, 0xe4, 0x9b, 0xb1, 0x8f, 0x8c, 0xa7, 0x13, - 0x0f, 0xbf, 0x1c, 0x21, 0xe3, 0xe4, 0x7f, 0xd0, 0xa4, 0x9a, 0xd5, 0xcd, 0x0d, 0xa3, 0x61, 0x98, - 0xa2, 0xdb, 0xee, 0xaf, 0x65, 0x58, 0x2a, 0xd8, 0xb2, 0x61, 0x12, 0x33, 0x24, 0x5b, 0x30, 0x67, - 0xf4, 0xa4, 0x5d, 0xbd, 0x73, 0xab, 0xd8, 0xbd, 0x29, 0x46, 0xad, 0x8c, 0x91, 0xd9, 0x92, 0xf7, - 0x60, 0x96, 0xc9, 0x16, 0xc9, 0x61, 0xd7, 0x3b, 0xab, 0x45, 0x2f, 0xb9, 0x1e, 0x7a, 0x5a, 0xd1, - 0xf9, 0x0a, 0x9a, 0xc6, 0x91, 0x1a, 0xc0, 0x3b, 0x50, 0x89, 0xc4, 0x87, 0x4e, 0x64, 0xa9, 0xe8, - 0x42, 0xea, 0x78, 0x4a, 0x43, 0xe0, 0x58, 0x35, 0x17, 0x83, 0x6e, 0x5f, 0x61, 0x51, 0x75, 0xfd, - 0x64, 0x1c, 0x1b, 0x7d, 0xcd, 0x60, 0xce, 0x4f, 0x16, 0xcc, 0x99, 0x04, 0xa6, 0x02, 0xf9, 0x06, - 0x2c, 0x30, 0x9f, 0xc6, 0x31, 0x06, 0x5d, 0x33, 0xf4, 0x19, 0x39, 0xd8, 0x79, 0xcd, 0x7e, 0xaa, - 0x67, 0x7f, 0x1b, 0x2e, 0x18, 0xc5, 0x43, 0x0c, 0x54, 0xa4, 0xea, 0xa2, 0x16, 0x3c, 0xca, 0xa0, - 0xb0, 0x0d, 0xb3, 0xb2, 0x06, 0x66, 0xcf, 0xca, 0x7c, 0xdb, 0xe7, 0xef, 0xb7, 0x6a, 0x81, 0x36, - 0x77, 0xff, 0x2c, 0xc1, 0xd2, 0xf3, 0x84, 0xbd, 0x15, 0x1e, 0xc8, 0x0a, 0xcc, 0xea, 0xb7, 0xa5, - 0x1e, 0xa7, 0xa6, 0xc8, 0xc3, 0x2c, 0xbb, 0xb2, 0xcc, 0xee, 0x76, 0x31, 0xbb, 0x29, 0xf1, 0x24, - 0xaf, 0x90, 0x99, 0xf3, 0xb3, 0x05, 0xb5, 0x8c, 0x3b, 0xed, 0x5d, 0x09, 0xde, 0x90, 0xf2, 0x5d, - 0x1d, 0x5c, 0x7e, 0x13, 0x0f, 0xaa, 0xbb, 0x48, 0x83, 0xc3, 0xd8, 0x77, 0xdf, 0x20, 0x76, 0xeb, - 0x23, 0x65, 0xfa, 0x38, 0x16, 0x52, 0xe3, 0xc8, 0xb9, 0x0f, 0x8d, 0xbc, 0x80, 0x2c, 0x42, 0x79, - 0x0f, 0x27, 0x3a, 0x15, 0xf1, 0x49, 0x96, 0xa1, 0x32, 0xa6, 0xd1, 0xc8, 0x2c, 0x29, 0x45, 0xdc, - 0x2f, 0xdd, 0xb5, 0xdc, 0x27, 0xb0, 0x5c, 0x0c, 0xa9, 0x9f, 0xcc, 0x21, 0xd4, 0xad, 0x73, 0x42, - 0xdd, 0xfd, 0xd1, 0x82, 0x95, 0x6d, 0xe4, 0xcf, 0x12, 0x1e, 0xf6, 0x43, 0x5f, 0xde, 0x19, 0x33, - 0xad, 0xf7, 0x61, 0x25, 0x89, 0x82, 0x6e, 0x7e, 0x2b, 0x4d, 0xba, 0x43, 0xba, 0x63, 0xc6, 0xb6, - 0x9c, 0x44, 0x41, 0x61, 0x83, 0x3d, 0xa7, 0x3b, 0x28, 0xac, 0x62, 0xdc, 0x9f, 0x66, 0xa5, 0xca, - 0x58, 0x8e, 0x71, 0xff, 0xb8, 0xd5, 0x32, 0x54, 0xa2, 0x70, 0x10, 0x72, 0xb9, 0x7a, 0x2a, 0x9e, - 0x22, 0x32, 0xe8, 0xcf, 0x1c, 0x42, 0xdf, 0xfd, 0xa3, 0x04, 0x97, 0x8e, 0x25, 0xac, 0xeb, 0x7f, - 0x09, 0x8d, 0x38, 0xc7, 0xd7, 0x5d, 0xe8, 0x1c, 0x83, 0xf1, 0x34, 0xe3, 0x56, 0x81, 0x59, 0xf0, - 0xe3, 0xfc, 0x6d, 0x41, 0x23, 0x2f, 0x9e, 0xfa, 0x26, 0x6d, 0xa8, 0xfa, 0x29, 0x52, 0x8e, 0x81, - 0xae, 0xd4, 0x90, 0xe2, 0x22, 0x2a, 0x77, 0x18, 0xe8, 0x83, 0x92, 0xd1, 0xc2, 0x2a, 0xc0, 0x08, - 0x85, 0x95, 0xaa, 0xd2, 0x90, 0xe4, 0x1e, 0x94, 0x93, 0x28, 0x90, 0xe7, 0xb5, 0xde, 0xb9, 0x71, - 0x04, 0x70, 0x74, 0x07, 0xb3, 0xde, 0x47, 0xa8, 0x81, 0x10, 0x22, 0xf3, 0x84, 0x8d, 0x30, 0x8d, - 0x71, 0x5f, 0x5e, 0xdf, 0x37, 0x31, 0x8d, 0x71, 0xdf, 0xfd, 0xad, 0x04, 0xab, 0x27, 0xaa, 0x90, - 0x0d, 0x68, 0xf8, 0xa3, 0x34, 0xc5, 0x98, 0xe7, 0x81, 0x50, 0xd7, 0x3c, 0x39, 0xc9, 0x35, 0xa8, - 0xc5, 0x78, 0xc0, 0xf3, 0x23, 0x9f, 0x13, 0x8c, 0x53, 0xc6, 0xbc, 0x09, 0xcd, 0x02, 0x5c, 0x64, - 0x27, 0xce, 0x38, 0x96, 0x45, 0x0b, 0xf2, 0x39, 0x00, 0xcd, 0xd2, 0xd4, 0xc7, 0xf6, 0x83, 0x73, - 0x16, 0xde, 0x7a, 0x12, 0x07, 0x78, 0x80, 0xc1, 0x66, 0x6e, 0x0b, 0x79, 0x39, 0x77, 0xce, 0x87, - 0xb0, 0x34, 0x45, 0x45, 0x14, 0x13, 0x0a, 0xb6, 0xec, 0x42, 0xc5, 0x53, 0x44, 0x06, 0x8d, 0x52, - 0x0e, 0xb3, 0x77, 0xe0, 0xca, 0xc7, 0x34, 0xdd, 0xcb, 0x43, 0x68, 0x93, 0x79, 0x48, 0x03, 0xf3, - 0xd4, 0xa6, 0xe0, 0xc9, 0x5d, 0x87, 0xab, 0x27, 0x19, 0x29, 0xc4, 0xba, 0x04, 0x16, 0xb7, 0x91, - 0xeb, 0x07, 0xad, 0x3c, 0xb9, 0x5b, 0x70, 0x21, 0xc7, 0x7b, 0xeb, 0xbd, 0xd0, 0xf9, 0xc7, 0x82, - 0x05, 0x53, 0xed, 0x0b, 0x4c, 0xc7, 0xa1, 0x8f, 0x64, 0x04, 0xf5, 0xdc, 0x0d, 0x20, 0xeb, 0xa7, - 0x9c, 0x07, 0x99, 0x8c, 0xb3, 0x71, 0xe6, 0x01, 0x71, 0x37, 0xbe, 0xfe, 0xfd, 0xaf, 0xef, 0x4a, - 0x6b, 0x64, 0xb5, 0x6d, 0x8e, 0x40, 0xfb, 0x55, 0xe1, 0x46, 0xbc, 0x26, 0x7b, 0xd0, 0xc8, 0x6f, - 0x3b, 0xb2, 0x71, 0xe6, 0xf2, 0x75, 0xdc, 0xd3, 0x54, 0x74, 0xe4, 0x65, 0x19, 0x79, 0xde, 0xad, - 0x65, 0x91, 0xef, 0x5b, 0xb7, 0x3a, 0x3f, 0x94, 0x60, 0x29, 0xdf, 0x72, 0x53, 0xfb, 0x6b, 0x58, - 0x38, 0xb2, 0x38, 0xc8, 0xff, 0xcf, 0xd8, 0x2b, 0x2a, 0x95, 0xeb, 0xe7, 0xda, 0x3e, 0xee, 0x15, - 0x99, 0xcd, 0x25, 0x72, 0xb1, 0x9d, 0xdf, 0x3c, 0xac, 0xfd, 0x4a, 0xf5, 0xe0, 0x5b, 0x0b, 0x56, - 0xa6, 0xa3, 0x81, 0x1c, 0xb9, 0x83, 0xa7, 0x02, 0xcd, 0x79, 0xf7, 0x7c, 0xca, 0xc5, 0xa4, 0x6e, - 0x4d, 0x4f, 0xaa, 0x13, 0x43, 0x53, 0xa1, 0xc6, 0x34, 0xe9, 0x0b, 0xa8, 0x65, 0xe0, 0x23, 0x57, - 0x8f, 0x15, 0x5e, 0x40, 0xaa, 0x73, 0xed, 0x44, 0xb9, 0x8e, 0xbe, 0x20, 0xa3, 0xd7, 0x48, 0xb5, - 0xad, 0x30, 0xf9, 0xe0, 0x2a, 0x2c, 0xf9, 0xc9, 0xa0, 0x68, 0x36, 0xec, 0x7d, 0x56, 0xd5, 0xff, - 0x71, 0xf5, 0x66, 0xe5, 0x1f, 0xaa, 0x77, 0xfe, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x7b, 0xd2, 0x74, - 0xfa, 0x8a, 0x0d, 0x00, 0x00, + // 1336 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4b, 0x6f, 0x1b, 0x55, + 0x14, 0x66, 0x9c, 0x3a, 0xb6, 0x8f, 0xed, 0xc4, 0xbd, 0x49, 0x13, 0x67, 0xd2, 0x47, 0x32, 0x50, + 0x51, 0x0a, 0xb2, 0x85, 0x5b, 0xa4, 0xb6, 0x2c, 0x90, 0x9b, 0x38, 0x21, 0x52, 0x1b, 0xa2, 0x49, + 0x1a, 0x09, 0x10, 0x32, 0x37, 0x9e, 0xe3, 0x64, 0x94, 0xf1, 0xcc, 0x30, 0x73, 0x9d, 0xd4, 0xaa, + 0xca, 0x82, 0x1d, 0x3b, 0x04, 0x0b, 0x56, 0xfc, 0x00, 0x36, 0x88, 0xff, 0xc0, 0x9e, 0x05, 0x6c, + 0x61, 0xc7, 0x82, 0x3f, 0xc0, 0x1e, 0xdd, 0xc7, 0x4c, 0x66, 0x92, 0x49, 0xe2, 0x76, 0xe5, 0x7b, + 0xde, 0x8f, 0xfb, 0xdd, 0x73, 0xc6, 0xa0, 0x53, 0xdf, 0x6e, 0x1e, 0xdd, 0x6b, 0xf6, 0x1c, 0x6a, + 0x07, 0xfe, 0x9e, 0xfc, 0x6d, 0xf8, 0x81, 0xc7, 0x3c, 0x52, 0xe9, 0x79, 0x01, 0x7a, 0x61, 0x43, + 0xf0, 0xf4, 0x5b, 0xfb, 0x9e, 0xb7, 0xef, 0x60, 0x53, 0xc8, 0xf6, 0x86, 0xfd, 0x26, 0xb3, 0x07, + 0x18, 0x32, 0x3a, 0xf0, 0xa5, 0xba, 0x7e, 0x5d, 0x29, 0x70, 0x8f, 0xd4, 0x75, 0x3d, 0x46, 0x99, + 0xed, 0xb9, 0xa1, 0x94, 0x1a, 0x3f, 0xe6, 0xa0, 0xba, 0x3b, 0x74, 0x5c, 0x0c, 0xe8, 0x9e, 0xed, + 0xd8, 0x6c, 0x44, 0x08, 0x5c, 0x71, 0xe9, 0x00, 0xeb, 0xda, 0x92, 0x76, 0xa7, 0x64, 0x8a, 0x33, + 0xb9, 0x0d, 0x53, 0xfc, 0x37, 0xf4, 0x69, 0x0f, 0xbb, 0x42, 0x9a, 0x13, 0xd2, 0x6a, 0xcc, 0xdd, + 0xe4, 0x6a, 0x4b, 0x50, 0xb6, 0x30, 0xec, 0x05, 0xb6, 0xcf, 0x43, 0xd4, 0x27, 0x84, 0x4e, 0x92, + 0xc5, 0x9d, 0x3b, 0xb6, 0x7b, 0x58, 0xbf, 0x22, 0x9d, 0xf3, 0x33, 0xd1, 0xa1, 0x18, 0xe2, 0x11, + 0x06, 0x36, 0x1b, 0xd5, 0xf3, 0x82, 0x1f, 0xd3, 0x5c, 0x36, 0x40, 0x46, 0x2d, 0xca, 0x68, 0x7d, + 0x52, 0xca, 0x22, 0x9a, 0x2c, 0x40, 0xb1, 0x6f, 0x3f, 0x47, 0xab, 0xbb, 0x37, 0xaa, 0x17, 0x84, + 0xac, 0x20, 0xe8, 0xc7, 0x23, 0xf2, 0x18, 0xae, 0xd2, 0x7e, 0x1f, 0x7b, 0x0c, 0xad, 0xee, 0x11, + 0x06, 0x21, 0x2f, 0xb8, 0x5e, 0x5c, 0x9a, 0xb8, 0x53, 0x6e, 0x5d, 0x6b, 0x24, 0xdb, 0xd7, 0x58, + 0x43, 0xca, 0x86, 0x01, 0x9a, 0xb5, 0x48, 0x7f, 0x57, 0xa9, 0x1b, 0xbf, 0x6b, 0x50, 0x5c, 0x45, + 0x86, 0x3d, 0xe6, 0x05, 0x99, 0x4d, 0xa9, 0x43, 0x41, 0xf9, 0x56, 0xdd, 0x88, 0x48, 0xd2, 0x82, + 0xbc, 0xc5, 0x46, 0x3e, 0x8a, 0x0e, 0x4c, 0xb5, 0xae, 0xa7, 0x43, 0x46, 0x4e, 0x1b, 0xab, 0x3b, + 0x23, 0x1f, 0x4d, 0xa9, 0x6a, 0x7c, 0x09, 0x79, 0x41, 0x93, 0x45, 0x98, 0x5f, 0xed, 0xec, 0x74, + 0x56, 0x76, 0x3e, 0x31, 0xbb, 0xab, 0xdd, 0x9d, 0x4f, 0xb7, 0x3a, 0xdd, 0x8d, 0xcd, 0xdd, 0xf6, + 0x93, 0x8d, 0xd5, 0xda, 0x1b, 0xe4, 0x06, 0x2c, 0x9c, 0x16, 0x6e, 0xb6, 0x9f, 0x76, 0xb6, 0xb7, + 0xda, 0x2b, 0x9d, 0x9a, 0x96, 0x65, 0xbb, 0xd6, 0x69, 0xef, 0x3c, 0x33, 0x3b, 0xb5, 0x9c, 0xb1, + 0x0d, 0xa5, 0xcd, 0xe8, 0xba, 0x32, 0x0b, 0x6a, 0x41, 0xd1, 0x52, 0xb9, 0x89, 0x8a, 0xca, 0xad, + 0xb9, 0xec, 0xcc, 0xcd, 0x58, 0xcf, 0xf8, 0x2e, 0x07, 0x05, 0xd5, 0xc3, 0x4c, 0x9f, 0x1f, 0x40, + 0x29, 0xc6, 0x88, 0x72, 0x3a, 0x9f, 0x76, 0x1a, 0xe7, 0x64, 0x9e, 0x68, 0x26, 0x7b, 0x3b, 0x91, + 0xee, 0xed, 0x6d, 0x98, 0x52, 0xc7, 0x6e, 0xdf, 0x0b, 0x06, 0x94, 0x29, 0x2c, 0x55, 0x15, 0x77, + 0x4d, 0x30, 0x53, 0xb5, 0xe4, 0xc7, 0xab, 0x85, 0x74, 0x60, 0xfa, 0x28, 0xf1, 0x14, 0x6c, 0x0c, + 0xeb, 0x93, 0x02, 0x33, 0x8b, 0x69, 0xd3, 0xd4, 0x7b, 0x31, 0x4f, 0xdb, 0x18, 0x8b, 0x90, 0x7f, + 0x42, 0x47, 0x28, 0x40, 0x73, 0x40, 0xc3, 0x83, 0xa8, 0x1f, 0xfc, 0x6c, 0x7c, 0xab, 0x41, 0x79, + 0x85, 0x7b, 0xd9, 0x66, 0x94, 0x0d, 0x43, 0x72, 0x1f, 0x4a, 0x51, 0xfc, 0xb0, 0xae, 0x89, 0x68, + 0xe7, 0x25, 0x7a, 0xa2, 0x48, 0x56, 0xa1, 0xe6, 0xd0, 0x90, 0x75, 0x87, 0xbe, 0x45, 0x19, 0x76, + 0xf9, 0x93, 0x57, 0xcd, 0xd5, 0x1b, 0xf2, 0xb9, 0x37, 0xa2, 0x79, 0xd0, 0xd8, 0x89, 0xe6, 0x81, + 0x39, 0xc5, 0x6d, 0x9e, 0x09, 0x13, 0xce, 0x34, 0x1e, 0x02, 0x59, 0x47, 0xd6, 0x76, 0x7b, 0x18, + 0xb2, 0x60, 0x64, 0xe2, 0x57, 0x43, 0x0c, 0x19, 0x79, 0x13, 0xaa, 0x54, 0xb1, 0xba, 0x89, 0xeb, + 0xac, 0x44, 0x4c, 0x7e, 0x5f, 0xc6, 0xaf, 0x13, 0x30, 0x93, 0xb2, 0x0d, 0x7d, 0xcf, 0x0d, 0x91, + 0xac, 0x41, 0x31, 0xd2, 0x13, 0x76, 0xe5, 0xd6, 0xdd, 0x74, 0x35, 0x19, 0x46, 0x8d, 0x98, 0x11, + 0xdb, 0x92, 0xf7, 0x61, 0x32, 0x14, 0x0d, 0x52, 0x65, 0x2d, 0xa4, 0xbd, 0x24, 0x3a, 0x68, 0x2a, + 0x45, 0xfd, 0x6b, 0xa8, 0x46, 0x8e, 0x64, 0xfb, 0xdf, 0x81, 0xbc, 0xc3, 0x0f, 0x2a, 0x91, 0x99, + 0xb4, 0x0b, 0xa1, 0x63, 0x4a, 0x0d, 0x3e, 0x2f, 0x64, 0x73, 0xd1, 0xea, 0xf6, 0x25, 0x9a, 0x79, + 0xe4, 0x8b, 0xe6, 0x45, 0xa4, 0xaf, 0x18, 0xa1, 0xfe, 0x93, 0x06, 0xc5, 0x28, 0x81, 0xcc, 0xa7, + 0x90, 0xba, 0xea, 0xdc, 0xb8, 0x57, 0xbd, 0x0e, 0x93, 0x22, 0xc7, 0xb0, 0x3e, 0x21, 0x4c, 0x9a, + 0xe3, 0xf7, 0x53, 0x96, 0xa8, 0xcc, 0x8d, 0xbf, 0x73, 0x30, 0xb3, 0xe5, 0x85, 0xaf, 0x75, 0xdf, + 0x64, 0x0e, 0x26, 0xd5, 0x6b, 0x93, 0xa3, 0x4e, 0x51, 0x64, 0xe5, 0x54, 0x76, 0xef, 0xa6, 0xb3, + 0xcb, 0x88, 0x27, 0x78, 0xa9, 0xcc, 0xf4, 0xdf, 0x34, 0x28, 0xc5, 0xdc, 0xac, 0x57, 0xc3, 0x79, + 0x3e, 0x65, 0x07, 0x2a, 0xb8, 0x38, 0x13, 0x13, 0x0a, 0x07, 0x48, 0xad, 0x93, 0xd8, 0x0f, 0x5e, + 0x21, 0x76, 0xe3, 0x63, 0x69, 0xda, 0x71, 0xb9, 0x34, 0x72, 0xa4, 0x3f, 0x82, 0x4a, 0x52, 0x40, + 0x6a, 0x30, 0x71, 0x88, 0x23, 0x95, 0x0a, 0x3f, 0x92, 0x59, 0xc8, 0x1f, 0x51, 0x67, 0x18, 0x2d, + 0x40, 0x49, 0x3c, 0xca, 0x3d, 0xd0, 0x8c, 0x0d, 0x98, 0x4d, 0x87, 0x54, 0x4f, 0xe2, 0x04, 0xca, + 0xda, 0x98, 0x50, 0x36, 0x7e, 0xd1, 0x60, 0x6e, 0x1d, 0xd9, 0xa6, 0xc7, 0xec, 0xbe, 0xdd, 0x13, + 0xfb, 0x3a, 0xba, 0xad, 0xfb, 0x30, 0xe7, 0x39, 0x56, 0x37, 0x39, 0x73, 0x46, 0x5d, 0x9f, 0xee, + 0x47, 0xd7, 0x36, 0xeb, 0x39, 0x56, 0x6a, 0x3e, 0x6d, 0xd1, 0x7d, 0x0e, 0xbd, 0x39, 0x17, 0x8f, + 0xb3, 0xac, 0x64, 0x19, 0xb3, 0x2e, 0x1e, 0x9f, 0xb5, 0x9a, 0x85, 0xbc, 0x63, 0x0f, 0x6c, 0x26, + 0x46, 0x70, 0xde, 0x94, 0x44, 0x0c, 0xed, 0x2b, 0x27, 0xd0, 0x36, 0xfe, 0xca, 0xc1, 0xfc, 0x99, + 0x84, 0x55, 0xfd, 0xbb, 0x50, 0x71, 0x13, 0x7c, 0xd5, 0x85, 0xd6, 0x19, 0x18, 0x67, 0x19, 0x37, + 0x52, 0xcc, 0x94, 0x1f, 0xfd, 0x5f, 0x0d, 0x2a, 0x49, 0xf1, 0x79, 0x3b, 0xba, 0x17, 0x20, 0x65, + 0x68, 0x45, 0x3b, 0x5a, 0x91, 0xfc, 0xcb, 0x42, 0xba, 0x43, 0x4b, 0xad, 0x98, 0x98, 0xe6, 0x56, + 0x16, 0x3a, 0xc8, 0xad, 0x64, 0x95, 0x11, 0x49, 0x1e, 0xc2, 0x84, 0xe7, 0x58, 0x6a, 0xa3, 0xbc, + 0x7d, 0x0a, 0x70, 0x74, 0x1f, 0xe3, 0xde, 0x3b, 0xa8, 0x80, 0x60, 0x63, 0x68, 0x72, 0x1b, 0x6e, + 0xea, 0xe2, 0xb1, 0xf8, 0x8a, 0x79, 0x15, 0x53, 0x17, 0x8f, 0x8d, 0x3f, 0x72, 0xb0, 0x70, 0xae, + 0x0a, 0x59, 0x86, 0x4a, 0x6f, 0x18, 0x04, 0xe8, 0xb2, 0x24, 0x10, 0xca, 0x8a, 0x27, 0x6e, 0x72, + 0x11, 0x4a, 0x2e, 0x3e, 0x67, 0xc9, 0x2b, 0x2f, 0x72, 0xc6, 0x05, 0xd7, 0xdc, 0x86, 0x6a, 0x0a, + 0x2e, 0xa2, 0x13, 0x97, 0xac, 0xc2, 0xb4, 0x05, 0xf9, 0x1c, 0x80, 0xc6, 0x69, 0xd6, 0xf3, 0xe2, + 0x91, 0x7e, 0x38, 0x66, 0xe1, 0x8d, 0x0d, 0xd7, 0xc2, 0xe7, 0x68, 0xb5, 0x13, 0x53, 0xc8, 0x4c, + 0xb8, 0xd3, 0x3f, 0x82, 0x99, 0x0c, 0x15, 0x5e, 0x8c, 0xcd, 0xd9, 0xa2, 0x0b, 0x79, 0x53, 0x12, + 0x31, 0x34, 0x72, 0x09, 0xcc, 0xde, 0x83, 0x1b, 0x4f, 0x69, 0x70, 0x98, 0x84, 0x50, 0x3b, 0x34, + 0x91, 0x5a, 0xd1, 0x53, 0xcb, 0xc0, 0x93, 0xb1, 0x04, 0x37, 0xcf, 0x33, 0x92, 0x88, 0x35, 0x08, + 0xd4, 0xd6, 0x91, 0xa9, 0x07, 0x2d, 0x3d, 0x19, 0x6b, 0x70, 0x35, 0xc1, 0x7b, 0xed, 0xb9, 0xd0, + 0xfa, 0x4f, 0x83, 0xe9, 0xa8, 0xda, 0x6d, 0x0c, 0x8e, 0xec, 0x1e, 0x92, 0x21, 0x94, 0x13, 0x3b, + 0x80, 0x2c, 0x5d, 0xb0, 0x1e, 0x44, 0x32, 0xfa, 0xf2, 0xa5, 0x0b, 0xc4, 0x58, 0xfe, 0xe6, 0xcf, + 0x7f, 0x7e, 0xc8, 0x2d, 0x92, 0x85, 0x66, 0xb4, 0x04, 0x9a, 0x2f, 0x52, 0x3b, 0xe2, 0x25, 0x39, + 0x84, 0x4a, 0x72, 0xda, 0x91, 0xe5, 0x4b, 0x87, 0xaf, 0x6e, 0x5c, 0xa4, 0xa2, 0x22, 0xcf, 0x8a, + 0xc8, 0x53, 0x46, 0x29, 0x8e, 0xfc, 0x48, 0xbb, 0xdb, 0xfa, 0x39, 0x07, 0x33, 0xc9, 0x96, 0x47, + 0xb5, 0xbf, 0x84, 0xe9, 0x53, 0x83, 0x83, 0xbc, 0x75, 0xc9, 0x5c, 0x91, 0xa9, 0xdc, 0x1e, 0x6b, + 0xfa, 0x18, 0x37, 0x44, 0x36, 0xf3, 0xe4, 0x5a, 0x33, 0x39, 0x79, 0xc2, 0xe6, 0x0b, 0xd9, 0x83, + 0xef, 0x35, 0x98, 0xcb, 0x46, 0x03, 0x39, 0xb5, 0x07, 0x2f, 0x04, 0x9a, 0xfe, 0xde, 0x78, 0xca, + 0xe9, 0xa4, 0xee, 0x66, 0x27, 0xd5, 0x72, 0xa1, 0x2a, 0x51, 0x13, 0x35, 0xe9, 0x0b, 0x28, 0xc5, + 0xe0, 0x23, 0x37, 0xcf, 0x14, 0x9e, 0x42, 0xaa, 0x7e, 0xeb, 0x5c, 0xb9, 0x8a, 0x3e, 0x2d, 0xa2, + 0x97, 0x48, 0xa1, 0x29, 0x31, 0xf9, 0xf8, 0x26, 0xcc, 0xf4, 0xbc, 0x41, 0xda, 0xcc, 0xdf, 0xfb, + 0xac, 0xa0, 0xfe, 0xb9, 0xee, 0x4d, 0x8a, 0x0f, 0xd1, 0x7b, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, + 0xcb, 0x5c, 0xce, 0x34, 0xd2, 0x0e, 0x00, 0x00, } diff --git a/api/v3/clairpb/clair.proto b/api/v3/clairpb/clair.proto index c0e3ff8a..015ab3ac 100644 --- a/api/v3/clairpb/clair.proto +++ b/api/v3/clairpb/clair.proto @@ -44,17 +44,42 @@ message Vulnerability { repeated Feature affected_versions = 8; } +message Detector { + enum DType { + DETECTOR_D_TYPE_INVALID = 0; + DETECTOR_D_TYPE_NAMESPACE = 1; + DETECTOR_D_TYPE_FEATURE = 2; + } + // The name of the detector. + string name = 1; + // The version of the detector. + string version = 2; + // The type of the detector. + DType dtype = 3; +} + +message Namespace { + // The name of the namespace. + string name = 1; + // The detector used to detect the namespace. This only exists when present in + // an Ancestry Feature. + Detector detector = 2; +} + message Feature { // The name of the feature. string name = 1; - // The name of the namespace in which the feature is detected. - string namespace_name = 2; + // The namespace in which the feature is detected. + Namespace namespace = 2; // The specific version of this feature. string version = 3; // The format used to parse version numbers for the feature. string version_format = 4; + // The detector used to detect this feature. This only exists when present in + // an Ancestry. + Detector detector = 5; // The list of vulnerabilities that affect the feature. - repeated Vulnerability vulnerabilities = 5; + repeated Vulnerability vulnerabilities = 6; } message Layer { @@ -77,12 +102,10 @@ service AncestryService { } message ClairStatus { - // The configured list of feature listers used to scan an ancestry. - repeated string listers = 1; - // The configured list of namespace detectors used to scan an ancestry. - repeated string detectors = 2; + // The implemented detectors in this Clair instance + repeated Detector detectors = 1; // The time at which the updater last ran. - google.protobuf.Timestamp last_update_time = 3; + google.protobuf.Timestamp last_update_time = 2; } message GetAncestryRequest { @@ -100,12 +123,11 @@ message GetAncestryResponse { message Ancestry { // The name of the desired ancestry. string name = 1; - // The configured list of feature listers used to scan this ancestry. - repeated string scanned_listers = 4; - // The configured list of namespace detectors used to scan an ancestry. - repeated string scanned_detectors = 5; + // The detectors used to scan this Ancestry. It may not be the current set + // of detectors in clair status. + repeated Detector detectors = 2; // The list of layers along with detected features in each. - repeated AncestryLayer layers = 6; + repeated AncestryLayer layers = 3; } // The ancestry requested. Ancestry ancestry = 1; diff --git a/api/v3/clairpb/clair.swagger.json b/api/v3/clairpb/clair.swagger.json index 32e00e53..19396b43 100644 --- a/api/v3/clairpb/clair.swagger.json +++ b/api/v3/clairpb/clair.swagger.json @@ -156,6 +156,15 @@ } }, "definitions": { + "DetectorDType": { + "type": "string", + "enum": [ + "DETECTOR_D_TYPE_INVALID", + "DETECTOR_D_TYPE_NAMESPACE", + "DETECTOR_D_TYPE_FEATURE" + ], + "default": "DETECTOR_D_TYPE_INVALID" + }, "GetAncestryResponseAncestry": { "type": "object", "properties": { @@ -163,19 +172,12 @@ "type": "string", "description": "The name of the desired ancestry." }, - "scanned_listers": { - "type": "array", - "items": { - "type": "string" - }, - "description": "The configured list of feature listers used to scan this ancestry." - }, - "scanned_detectors": { + "detectors": { "type": "array", "items": { - "type": "string" + "$ref": "#/definitions/clairDetector" }, - "description": "The configured list of namespace detectors used to scan an ancestry." + "description": "The detectors used to scan this Ancestry. It may not be the current set\nof detectors in clair status." }, "layers": { "type": "array", @@ -268,19 +270,12 @@ "clairClairStatus": { "type": "object", "properties": { - "listers": { - "type": "array", - "items": { - "type": "string" - }, - "description": "The configured list of feature listers used to scan an ancestry." - }, "detectors": { "type": "array", "items": { - "type": "string" + "$ref": "#/definitions/clairDetector" }, - "description": "The configured list of namespace detectors used to scan an ancestry." + "title": "The implemented detectors in this Clair instance" }, "last_update_time": { "type": "string", @@ -289,6 +284,23 @@ } } }, + "clairDetector": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the detector." + }, + "version": { + "type": "string", + "description": "The version of the detector." + }, + "dtype": { + "$ref": "#/definitions/DetectorDType", + "description": "The type of the detector." + } + } + }, "clairFeature": { "type": "object", "properties": { @@ -296,9 +308,9 @@ "type": "string", "description": "The name of the feature." }, - "namespace_name": { - "type": "string", - "description": "The name of the namespace in which the feature is detected." + "namespace": { + "$ref": "#/definitions/clairNamespace", + "description": "The namespace in which the feature is detected." }, "version": { "type": "string", @@ -308,6 +320,10 @@ "type": "string", "description": "The format used to parse version numbers for the feature." }, + "detector": { + "$ref": "#/definitions/clairDetector", + "description": "The detector used to detect this feature. This only exists when present in\nan Ancestry." + }, "vulnerabilities": { "type": "array", "items": { @@ -360,6 +376,19 @@ "clairMarkNotificationAsReadResponse": { "type": "object" }, + "clairNamespace": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the namespace." + }, + "detector": { + "$ref": "#/definitions/clairDetector", + "description": "The detector used to detect the namespace. This only exists when present in\nan Ancestry Feature." + } + } + }, "clairPagedVulnerableAncestries": { "type": "object", "properties": { diff --git a/api/v3/clairpb/convert.go b/api/v3/clairpb/convert.go index 8414ed92..fe54b2f9 100644 --- a/api/v3/clairpb/convert.go +++ b/api/v3/clairpb/convert.go @@ -22,6 +22,13 @@ import ( "github.com/coreos/clair/ext/versionfmt" ) +// DatabaseDetectorTypeMapping maps the database detector type to the integer +// enum proto. +var DatabaseDetectorTypeMapping = map[database.DetectorType]Detector_DType{ + database.NamespaceDetectorType: Detector_DType(1), + database.FeatureDetectorType: Detector_DType(2), +} + // PagedVulnerableAncestriesFromDatabaseModel converts database // PagedVulnerableAncestries to api PagedVulnerableAncestries and assigns // indexes to ancestries. @@ -122,23 +129,38 @@ func VulnerabilityWithFixedInFromDatabaseModel(dbVuln database.VulnerabilityWith return vuln, nil } -// LayerFromDatabaseModel converts database layer to api layer. -func LayerFromDatabaseModel(dbLayer database.LayerMetadata) *Layer { - layer := Layer{Hash: dbLayer.Hash} - return &layer -} - // NamespacedFeatureFromDatabaseModel converts database namespacedFeature to api Feature. -func NamespacedFeatureFromDatabaseModel(feature database.NamespacedFeature) *Feature { +func NamespacedFeatureFromDatabaseModel(feature database.AncestryFeature) *Feature { version := feature.Feature.Version if version == versionfmt.MaxVersion { version = "None" } return &Feature{ - Name: feature.Feature.Name, - NamespaceName: feature.Namespace.Name, + Name: feature.Feature.Name, + Namespace: &Namespace{ + Name: feature.Namespace.Name, + Detector: DetectorFromDatabaseModel(feature.NamespaceBy), + }, VersionFormat: feature.Namespace.VersionFormat, Version: version, + Detector: DetectorFromDatabaseModel(feature.FeatureBy), } } + +func DetectorFromDatabaseModel(detector database.Detector) *Detector { + return &Detector{ + Name: detector.Name, + Version: detector.Version, + Dtype: DatabaseDetectorTypeMapping[detector.DType], + } +} + +func DetectorsFromDatabaseModel(dbDetectors []database.Detector) []*Detector { + detectors := make([]*Detector, 0, len(dbDetectors)) + for _, d := range dbDetectors { + detectors = append(detectors, DetectorFromDatabaseModel(d)) + } + + return detectors +} diff --git a/api/v3/rpc.go b/api/v3/rpc.go index 830bde9b..51502c2c 100644 --- a/api/v3/rpc.go +++ b/api/v3/rpc.go @@ -129,9 +129,8 @@ func (s *AncestryServer) GetAncestry(ctx context.Context, req *pb.GetAncestryReq } pbAncestry := &pb.GetAncestryResponse_Ancestry{ - Name: ancestry.Name, - ScannedDetectors: ancestry.ProcessedBy.Detectors, - ScannedListers: ancestry.ProcessedBy.Listers, + Name: ancestry.Name, + Detectors: pb.DetectorsFromDatabaseModel(ancestry.By), } for _, layer := range ancestry.Layers { diff --git a/api/v3/util.go b/api/v3/util.go index 6b6baa0a..392a147e 100644 --- a/api/v3/util.go +++ b/api/v3/util.go @@ -13,8 +13,7 @@ import ( // protobuf struct. func GetClairStatus(store database.Datastore) (*pb.ClairStatus, error) { status := &pb.ClairStatus{ - Listers: clair.Processors.Listers, - Detectors: clair.Processors.Detectors, + Detectors: pb.DetectorsFromDatabaseModel(clair.EnabledDetectors), } t, firstUpdate, err := clair.GetLastUpdateTime(store) @@ -34,41 +33,49 @@ func GetClairStatus(store database.Datastore) (*pb.ClairStatus, error) { // GetPbAncestryLayer retrieves an ancestry layer with vulnerabilities and // features in an ancestry based on the provided database layer. -func GetPbAncestryLayer(session database.Session, layer database.AncestryLayer) (*pb.GetAncestryResponse_AncestryLayer, error) { +func GetPbAncestryLayer(tx database.Session, layer database.AncestryLayer) (*pb.GetAncestryResponse_AncestryLayer, error) { pbLayer := &pb.GetAncestryResponse_AncestryLayer{ Layer: &pb.Layer{ Hash: layer.Hash, }, } - var ( - features []database.NullableAffectedNamespacedFeature - err error - ) - - if features, err = session.FindAffectedNamespacedFeatures(layer.DetectedFeatures); err != nil { + features := layer.GetFeatures() + affectedFeatures, err := tx.FindAffectedNamespacedFeatures(features) + if err != nil { return nil, status.Error(codes.Internal, err.Error()) } - for _, feature := range features { + // NOTE(sidac): It's quite inefficient, but the easiest way to implement + // this feature for now, we should refactor the implementation if there's + // any performance issue. It's expected that the number of features is less + // than 1000. + for _, feature := range affectedFeatures { if !feature.Valid { return nil, status.Error(codes.Internal, "ancestry feature is not found") } - var ( - pbFeature = pb.NamespacedFeatureFromDatabaseModel(feature.NamespacedFeature) - pbVuln *pb.Vulnerability - err error - ) - for _, vuln := range feature.AffectedBy { - if pbVuln, err = pb.VulnerabilityWithFixedInFromDatabaseModel(vuln); err != nil { - return nil, status.Error(codes.Internal, err.Error()) + for _, detectedFeature := range layer.Features { + if detectedFeature.NamespacedFeature != feature.NamespacedFeature { + continue } - pbFeature.Vulnerabilities = append(pbFeature.Vulnerabilities, pbVuln) - } + var ( + pbFeature = pb.NamespacedFeatureFromDatabaseModel(detectedFeature) + pbVuln *pb.Vulnerability + err error + ) + + for _, vuln := range feature.AffectedBy { + if pbVuln, err = pb.VulnerabilityWithFixedInFromDatabaseModel(vuln); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } - pbLayer.DetectedFeatures = append(pbLayer.DetectedFeatures, pbFeature) + pbFeature.Vulnerabilities = append(pbFeature.Vulnerabilities, pbVuln) + } + + pbLayer.DetectedFeatures = append(pbLayer.DetectedFeatures, pbFeature) + } } return pbLayer, nil diff --git a/cmd/clair/config.go b/cmd/clair/config.go index 09a01364..d5d2c402 100644 --- a/cmd/clair/config.go +++ b/cmd/clair/config.go @@ -26,8 +26,6 @@ import ( "github.com/coreos/clair" "github.com/coreos/clair/api" "github.com/coreos/clair/database" - "github.com/coreos/clair/ext/featurefmt" - "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/notification" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/pkg/pagination" @@ -47,7 +45,6 @@ type File struct { type Config struct { Database database.RegistrableComponentConfig Updater *clair.UpdaterConfig - Worker *clair.WorkerConfig Notifier *notification.Config API *api.Config } @@ -62,10 +59,6 @@ func DefaultConfig() Config { EnabledUpdaters: vulnsrc.ListUpdaters(), Interval: 1 * time.Hour, }, - Worker: &clair.WorkerConfig{ - EnabledDetectors: featurens.ListDetectors(), - EnabledListers: featurefmt.ListListers(), - }, API: &api.Config{ HealthAddr: "0.0.0.0:6061", Addr: "0.0.0.0:6060", diff --git a/cmd/clair/main.go b/cmd/clair/main.go index da046351..e28a30d5 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -102,40 +102,13 @@ func stopCPUProfiling(f *os.File) { } func configClairVersion(config *Config) { - listers := featurefmt.ListListers() - detectors := featurens.ListDetectors() - updaters := vulnsrc.ListUpdaters() + clair.EnabledDetectors = append(featurefmt.ListListers(), featurens.ListDetectors()...) + clair.EnabledUpdaters = strutil.Intersect(config.Updater.EnabledUpdaters, vulnsrc.ListUpdaters()) log.WithFields(log.Fields{ - "Listers": strings.Join(listers, ","), - "Detectors": strings.Join(detectors, ","), - "Updaters": strings.Join(updaters, ","), - }).Info("Clair registered components") - - unregDetectors := strutil.CompareStringLists(config.Worker.EnabledDetectors, detectors) - unregListers := strutil.CompareStringLists(config.Worker.EnabledListers, listers) - unregUpdaters := strutil.CompareStringLists(config.Updater.EnabledUpdaters, updaters) - if len(unregDetectors) != 0 || len(unregListers) != 0 || len(unregUpdaters) != 0 { - log.WithFields(log.Fields{ - "Unknown Detectors": strings.Join(unregDetectors, ","), - "Unknown Listers": strings.Join(unregListers, ","), - "Unknown Updaters": strings.Join(unregUpdaters, ","), - "Available Listers": strings.Join(featurefmt.ListListers(), ","), - "Available Detectors": strings.Join(featurens.ListDetectors(), ","), - "Available Updaters": strings.Join(vulnsrc.ListUpdaters(), ","), - }).Fatal("Unknown or unregistered components are configured") - } - - // verify the user specified detectors/listers/updaters are implemented. If - // some are not registered, it logs warning and won't use the unregistered - // extensions. - - clair.Processors = database.Processors{ - Detectors: strutil.CompareStringListsInBoth(config.Worker.EnabledDetectors, detectors), - Listers: strutil.CompareStringListsInBoth(config.Worker.EnabledListers, listers), - } - - clair.EnabledUpdaters = strutil.CompareStringListsInBoth(config.Updater.EnabledUpdaters, updaters) + "Detectors": database.SerializeDetectors(clair.EnabledDetectors), + "Updaters": clair.EnabledUpdaters, + }).Info("enabled Clair extensions") } // Boot starts Clair instance with the provided config. @@ -160,6 +133,7 @@ func Boot(config *Config) { defer db.Close() + clair.InitWorker(db) // Start notifier st.Begin() go clair.RunNotifier(config.Notifier, db, st) @@ -180,6 +154,18 @@ func Boot(config *Config) { st.Stop() } +// Initialize logging system +func configureLogger(flagLogLevel *string) { + logLevel, err := log.ParseLevel(strings.ToUpper(*flagLogLevel)) + if err != nil { + log.WithError(err).Error("failed to set logger parser level") + } + + log.SetLevel(logLevel) + log.SetOutput(os.Stdout) + log.SetFormatter(&formatter.JSONExtendedFormatter{ShowLn: true}) +} + func main() { // Parse command-line arguments flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) @@ -189,6 +175,7 @@ func main() { flagInsecureTLS := flag.Bool("insecure-tls", false, "Disable TLS server's certificate chain and hostname verification when pulling layers.") flag.Parse() + configureLogger(flagLogLevel) // Check for dependencies. for _, bin := range BinaryDependencies { _, err := exec.LookPath(bin) @@ -197,12 +184,6 @@ func main() { } } - // Initialize logging system - logLevel, err := log.ParseLevel(strings.ToUpper(*flagLogLevel)) - log.SetLevel(logLevel) - log.SetOutput(os.Stdout) - log.SetFormatter(&formatter.JSONExtendedFormatter{ShowLn: true}) - config, err := LoadConfig(*flagConfigPath) if err != nil { log.WithError(err).Fatal("failed to load configuration") diff --git a/config.yaml.sample b/config.yaml.sample index cb6fd5d3..1128aa10 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -51,19 +51,6 @@ clair: keyfile: certfile: - worker: - namespace_detectors: - - os-release - - lsb-release - - apt-sources - - alpine-release - - redhat-release - - feature_listers: - - apk - - dpkg - - rpm - updater: # Frequency the database will be updated with vulnerabilities from the default data sources # The value 0 disables the updater entirely. diff --git a/database/database.go b/database/database.go index c77effac..f760061f 100644 --- a/database/database.go +++ b/database/database.go @@ -33,6 +33,14 @@ var ( // fails (i.e. when an entity which is supposed to be unique is detected // twice) ErrInconsistent = errors.New("database: inconsistent database") + + // ErrInvalidParameters is an error that occurs when the parameters are not valid. + ErrInvalidParameters = errors.New("database: parameters are not valid") + + // ErrMissingEntities is an error that occurs when an associated immutable + // entity doesn't exist in the database. This error can indicate a wrong + // implementation or corrupted database. + ErrMissingEntities = errors.New("database: associated immutable entities are missing in the database") ) // RegistrableComponentConfig is a configuration block that can be used to @@ -99,6 +107,9 @@ type Session interface { // namespaced features. If the ancestry is not found, return false. FindAncestry(name string) (ancestry Ancestry, found bool, err error) + // PersistDetector inserts a slice of detectors if not in the database. + PersistDetectors(detectors []Detector) error + // PersistFeatures inserts a set of features if not in the database. PersistFeatures(features []Feature) error @@ -120,12 +131,10 @@ type Session interface { // PersistNamespaces inserts a set of namespaces if not in the database. PersistNamespaces([]Namespace) error - // PersistLayer persists a layer's content in the database. The given - // namespaces and features can be partial content of this layer. + // PersistLayer appends a layer's content in the database. // - // The layer, namespaces and features are expected to be already existing - // in the database. - PersistLayer(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error + // If any feature, namespace, or detector is not in the database, it returns not found error. + PersistLayer(hash string, features []LayerFeature, namespaces []LayerNamespace, detectedBy []Detector) error // FindLayer returns a layer with all detected features and // namespaces. @@ -157,8 +166,8 @@ type Session interface { // affected ancestries affected by old or new vulnerability. // // Because the number of affected ancestries maybe large, they are paginated - // and their pages are specified by the paination token, which, if empty, are - // always considered first page. + // and their pages are specified by the pagination token, which should be + // considered first page when it's empty. FindVulnerabilityNotification(name string, limit int, oldVulnerabilityPage pagination.Token, newVulnerabilityPage pagination.Token) (noti VulnerabilityNotificationWithVulnerable, found bool, err error) // MarkNotificationAsRead marks a Notification as notified now, assuming diff --git a/database/dbutil.go b/database/dbutil.go new file mode 100644 index 00000000..f1c90a67 --- /dev/null +++ b/database/dbutil.go @@ -0,0 +1,287 @@ +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "github.com/deckarep/golang-set" +) + +// DeduplicateNamespaces deduplicates a list of namespaces. +func DeduplicateNamespaces(namespaces ...Namespace) []Namespace { + nsSet := mapset.NewSet() + for _, ns := range namespaces { + nsSet.Add(ns) + } + + uniqueNamespaces := make([]Namespace, 0, nsSet.Cardinality()) + for ns := range nsSet.Iter() { + uniqueNamespaces = append(uniqueNamespaces, ns.(Namespace)) + } + + return uniqueNamespaces +} + +// DeduplicateFeatures deduplicates a list of list of features. +func DeduplicateFeatures(features ...Feature) []Feature { + fSet := mapset.NewSet() + for _, f := range features { + fSet.Add(f) + } + + uniqueFeatures := make([]Feature, 0, fSet.Cardinality()) + for f := range fSet.Iter() { + uniqueFeatures = append(uniqueFeatures, f.(Feature)) + } + + return uniqueFeatures +} + +// PersistPartialLayerAndCommit wraps session PersistLayer function with begin and +// commit. +func PersistPartialLayerAndCommit(datastore Datastore, layer *Layer) error { + tx, err := datastore.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := tx.PersistLayer(layer.Hash, layer.Features, layer.Namespaces, layer.By); err != nil { + return err + } + + return tx.Commit() +} + +// PersistFeaturesAndCommit wraps session PersistFeaturesAndCommit function with begin and commit. +func PersistFeaturesAndCommit(datastore Datastore, features []Feature) error { + tx, err := datastore.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := tx.PersistFeatures(features); err != nil { + return err + } + return tx.Commit() +} + +// PersistNamespacesAndCommit wraps session PersistNamespaces function with +// begin and commit. +func PersistNamespacesAndCommit(datastore Datastore, namespaces []Namespace) error { + tx, err := datastore.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if err := tx.PersistNamespaces(namespaces); err != nil { + return err + } + + return tx.Commit() +} + +// FindAncestryAndRollback wraps session FindAncestry function with begin and +// rollback. +func FindAncestryAndRollback(datastore Datastore, name string) (Ancestry, bool, error) { + tx, err := datastore.Begin() + defer tx.Rollback() + + if err != nil { + return Ancestry{}, false, err + } + + return tx.FindAncestry(name) +} + +// FindLayerAndRollback wraps session FindLayer function with begin and rollback. +func FindLayerAndRollback(datastore Datastore, hash string) (layer Layer, ok bool, err error) { + var tx Session + if tx, err = datastore.Begin(); err != nil { + return + } + + defer tx.Rollback() + layer, ok, err = tx.FindLayer(hash) + return +} + +// DeduplicateNamespacedFeatures returns a copy of all unique features in the +// input. +func DeduplicateNamespacedFeatures(features []NamespacedFeature) []NamespacedFeature { + nsSet := mapset.NewSet() + for _, ns := range features { + nsSet.Add(ns) + } + + uniqueFeatures := make([]NamespacedFeature, 0, nsSet.Cardinality()) + for ns := range nsSet.Iter() { + uniqueFeatures = append(uniqueFeatures, ns.(NamespacedFeature)) + } + + return uniqueFeatures +} + +// GetAncestryFeatures returns a list of unique namespaced features in the +// ancestry. +func GetAncestryFeatures(ancestry Ancestry) []NamespacedFeature { + features := []NamespacedFeature{} + for _, layer := range ancestry.Layers { + features = append(features, layer.GetFeatures()...) + } + + return DeduplicateNamespacedFeatures(features) +} + +// UpsertAncestryAndCommit wraps session UpsertAncestry function with begin and commit. +func UpsertAncestryAndCommit(datastore Datastore, ancestry Ancestry) error { + tx, err := datastore.Begin() + if err != nil { + return err + } + + if err = tx.UpsertAncestry(ancestry); err != nil { + tx.Rollback() + return err + } + + if err = tx.Commit(); err != nil { + return err + } + + return nil +} + +// PersistNamespacedFeaturesAndCommit wraps session PersistNamespacedFeatures function +// with begin and commit. +func PersistNamespacedFeaturesAndCommit(datastore Datastore, features []NamespacedFeature) error { + tx, err := datastore.Begin() + if err != nil { + return err + } + + if err := tx.PersistNamespacedFeatures(features); err != nil { + tx.Rollback() + return err + } + + if err := tx.Commit(); err != nil { + return err + } + + return nil +} + +// CacheRelatedVulnerabilityAndCommit wraps session CacheAffectedNamespacedFeatures +// function with begin and commit. +func CacheRelatedVulnerabilityAndCommit(datastore Datastore, features []NamespacedFeature) error { + tx, err := datastore.Begin() + if err != nil { + return err + } + + if err := tx.CacheAffectedNamespacedFeatures(features); err != nil { + tx.Rollback() + return err + } + + return tx.Commit() +} + +// IntersectDetectors returns the detectors in both d1 and d2. +func IntersectDetectors(d1 []Detector, d2 []Detector) []Detector { + d1Set := mapset.NewSet() + for _, d := range d1 { + d1Set.Add(d) + } + + d2Set := mapset.NewSet() + for _, d := range d2 { + d2Set.Add(d) + } + + inter := d1Set.Intersect(d2Set) + detectors := make([]Detector, 0, inter.Cardinality()) + for d := range inter.Iter() { + detectors = append(detectors, d.(Detector)) + } + + return detectors +} + +// DiffDetectors returns the detectors belongs to d1 but not d2 +func DiffDetectors(d1 []Detector, d2 []Detector) []Detector { + d1Set := mapset.NewSet() + for _, d := range d1 { + d1Set.Add(d) + } + + d2Set := mapset.NewSet() + for _, d := range d2 { + d2Set.Add(d) + } + + diff := d1Set.Difference(d2Set) + detectors := make([]Detector, 0, diff.Cardinality()) + for d := range diff.Iter() { + detectors = append(detectors, d.(Detector)) + } + + return detectors +} + +// MergeLayers merges all content in new layer to l, where the content is +// updated. +func MergeLayers(l *Layer, new *Layer) *Layer { + featureSet := mapset.NewSet() + namespaceSet := mapset.NewSet() + bySet := mapset.NewSet() + + for _, f := range l.Features { + featureSet.Add(f) + } + + for _, ns := range l.Namespaces { + namespaceSet.Add(ns) + } + + for _, d := range l.By { + bySet.Add(d) + } + + for _, feature := range new.Features { + if !featureSet.Contains(feature) { + l.Features = append(l.Features, feature) + featureSet.Add(feature) + } + } + + for _, namespace := range new.Namespaces { + if !namespaceSet.Contains(namespace) { + l.Namespaces = append(l.Namespaces, namespace) + namespaceSet.Add(namespace) + } + } + + for _, detector := range new.By { + if !bySet.Contains(detector) { + l.By = append(l.By, detector) + bySet.Add(detector) + } + } + + return l +} diff --git a/database/detector.go b/database/detector.go new file mode 100644 index 00000000..e1295340 --- /dev/null +++ b/database/detector.go @@ -0,0 +1,144 @@ +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "database/sql/driver" + "errors" + "fmt" + "strings" +) + +const ( + // NamespaceDetectorType is a type of detector that extracts the namespaces. + NamespaceDetectorType DetectorType = "namespace" + // FeatureDetectorType is a type of detector that extracts the features. + FeatureDetectorType DetectorType = "feature" +) + +// DetectorTypes contains all detector types. +var ( + DetectorTypes = []DetectorType{ + NamespaceDetectorType, + FeatureDetectorType, + } + // ErrFailedToParseDetectorType is the error returned when a detector type could + // not be parsed from a string. + ErrFailedToParseDetectorType = errors.New("failed to parse DetectorType from input") + // ErrInvalidDetector is the error returned when a detector from database has + // invalid name or version or type. + ErrInvalidDetector = errors.New("the detector has invalid metadata") +) + +// DetectorType is the type of a detector. +type DetectorType string + +// Value implements the database/sql/driver.Valuer interface. +func (s DetectorType) Value() (driver.Value, error) { + return string(s), nil +} + +// Scan implements the database/sql.Scanner interface. +func (s *DetectorType) Scan(value interface{}) error { + val, ok := value.([]byte) + if !ok { + return errors.New("could not scan a Severity from a non-string input") + } + + var err error + *s, err = NewDetectorType(string(val)) + if err != nil { + return err + } + + return nil +} + +// NewDetectorType attempts to parse a string into a standard DetectorType +// value. +func NewDetectorType(s string) (DetectorType, error) { + for _, ss := range DetectorTypes { + if strings.EqualFold(s, string(ss)) { + return ss, nil + } + } + + return "", ErrFailedToParseDetectorType +} + +// Valid checks if a detector type is defined. +func (s DetectorType) Valid() bool { + for _, t := range DetectorTypes { + if s == t { + return true + } + } + + return false +} + +// Detector is an versioned Clair extension. +type Detector struct { + // Name of an extension should be non-empty and uniquely identifies the + // extension. + Name string + // Version of an extension should be non-empty. + Version string + // DType is the type of the extension and should be one of the types in + // DetectorTypes. + DType DetectorType +} + +// Valid checks if all fields in the detector satisfies the spec. +func (d Detector) Valid() bool { + if d.Name == "" || d.Version == "" || !d.DType.Valid() { + return false + } + + return true +} + +// String returns a unique string representation of the detector. +func (d Detector) String() string { + return fmt.Sprintf("%s:%s", d.Name, d.Version) +} + +// NewNamespaceDetector returns a new namespace detector. +func NewNamespaceDetector(name, version string) Detector { + return Detector{ + Name: name, + Version: version, + DType: NamespaceDetectorType, + } +} + +// NewFeatureDetector returns a new feature detector. +func NewFeatureDetector(name, version string) Detector { + return Detector{ + Name: name, + Version: version, + DType: FeatureDetectorType, + } +} + +// SerializeDetectors returns the string representation of given detectors. +func SerializeDetectors(detectors []Detector) []string { + strDetectors := []string{} + for _, d := range detectors { + strDetectors = append(strDetectors, d.String()) + } + + return strDetectors +} diff --git a/database/mock.go b/database/mock.go index 9995bc49..b95d1b4d 100644 --- a/database/mock.go +++ b/database/mock.go @@ -30,9 +30,10 @@ type MockSession struct { FctFindAffectedNamespacedFeatures func(features []NamespacedFeature) ([]NullableAffectedNamespacedFeature, error) FctPersistNamespaces func([]Namespace) error FctPersistFeatures func([]Feature) error + FctPersistDetectors func(detectors []Detector) error FctPersistNamespacedFeatures func([]NamespacedFeature) error FctCacheAffectedNamespacedFeatures func([]NamespacedFeature) error - FctPersistLayer func(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error + FctPersistLayer func(hash string, features []LayerFeature, namespaces []LayerNamespace, by []Detector) error FctFindLayer func(name string) (Layer, bool, error) FctInsertVulnerabilities func([]VulnerabilityWithAffected) error FctFindVulnerabilities func([]VulnerabilityID) ([]NullableVulnerability, error) @@ -85,6 +86,13 @@ func (ms *MockSession) FindAffectedNamespacedFeatures(features []NamespacedFeatu panic("required mock function not implemented") } +func (ms *MockSession) PersistDetectors(detectors []Detector) error { + if ms.FctPersistDetectors != nil { + return ms.FctPersistDetectors(detectors) + } + panic("required mock function not implemented") +} + func (ms *MockSession) PersistNamespaces(namespaces []Namespace) error { if ms.FctPersistNamespaces != nil { return ms.FctPersistNamespaces(namespaces) @@ -113,9 +121,9 @@ func (ms *MockSession) CacheAffectedNamespacedFeatures(namespacedFeatures []Name panic("required mock function not implemented") } -func (ms *MockSession) PersistLayer(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error { +func (ms *MockSession) PersistLayer(hash string, features []LayerFeature, namespaces []LayerNamespace, detectors []Detector) error { if ms.FctPersistLayer != nil { - return ms.FctPersistLayer(hash, namespaces, features, processedBy) + return ms.FctPersistLayer(hash, features, namespaces, detectors) } panic("required mock function not implemented") } diff --git a/database/models.go b/database/models.go index b0157b90..4c1d0905 100644 --- a/database/models.go +++ b/database/models.go @@ -22,47 +22,129 @@ import ( "github.com/coreos/clair/pkg/pagination" ) -// Processors are extentions to scan a layer's content. -type Processors struct { - Listers []string - Detectors []string -} - // Ancestry is a manifest that keeps all layers in an image in order. type Ancestry struct { + // Name is a globally unique value for a set of layers. This is often the + // sha256 digest of an OCI/Docker manifest. Name string - // ProcessedBy contains the processors that are used when computing the + // By contains the processors that are used when computing the // content of this ancestry. - ProcessedBy Processors + By []Detector // Layers should be ordered and i_th layer is the parent of i+1_th layer in // the slice. Layers []AncestryLayer } +// Valid checks if the ancestry is compliant to spec. +func (a *Ancestry) Valid() bool { + if a == nil { + return false + } + + if a.Name == "" { + return false + } + + for _, d := range a.By { + if !d.Valid() { + return false + } + } + + for _, l := range a.Layers { + if !l.Valid() { + return false + } + } + + return true +} + // AncestryLayer is a layer with all detected namespaced features. type AncestryLayer struct { - LayerMetadata - - // DetectedFeatures are the features introduced by this layer when it was + // Hash is the sha-256 tarsum on the layer's blob content. + Hash string + // Features are the features introduced by this layer when it was // processed. - DetectedFeatures []NamespacedFeature + Features []AncestryFeature } -// LayerMetadata contains the metadata of a layer. -type LayerMetadata struct { - // Hash is content hash of the layer. - Hash string - // ProcessedBy contains the processors that processed this layer. - ProcessedBy Processors +// Valid checks if the Ancestry Layer is compliant to the spec. +func (l *AncestryLayer) Valid() bool { + if l == nil { + return false + } + + if l.Hash == "" { + return false + } + + return true +} + +// GetFeatures returns the Ancestry's features. +func (l *AncestryLayer) GetFeatures() []NamespacedFeature { + nsf := make([]NamespacedFeature, 0, len(l.Features)) + for _, f := range l.Features { + nsf = append(nsf, f.NamespacedFeature) + } + + return nsf +} + +// AncestryFeature is a namespaced feature with the detectors used to +// find this feature. +type AncestryFeature struct { + NamespacedFeature + + // FeatureBy is the detector that detected the feature. + FeatureBy Detector + // NamespaceBy is the detector that detected the namespace. + NamespaceBy Detector } -// Layer is a layer with its detected namespaces and features by -// ProcessedBy. +// Layer is a layer with all the detected features and namespaces. type Layer struct { - LayerMetadata + // Hash is the sha-256 tarsum on the layer's blob content. + Hash string + // By contains a list of detectors scanned this Layer. + By []Detector + Namespaces []LayerNamespace + Features []LayerFeature +} + +func (l *Layer) GetFeatures() []Feature { + features := make([]Feature, 0, len(l.Features)) + for _, f := range l.Features { + features = append(features, f.Feature) + } + + return features +} + +func (l *Layer) GetNamespaces() []Namespace { + namespaces := make([]Namespace, 0, len(l.Namespaces)) + for _, ns := range l.Namespaces { + namespaces = append(namespaces, ns.Namespace) + } + + return namespaces +} + +// LayerNamespace is a namespace with detection information. +type LayerNamespace struct { + Namespace + + // By is the detector found the namespace. + By Detector +} + +// LayerFeature is a feature with detection information. +type LayerFeature struct { + Feature - Namespaces []Namespace - Features []Feature + // By is the detector found the feature. + By Detector } // Namespace is the contextual information around features. diff --git a/database/pgsql/ancestry.go b/database/pgsql/ancestry.go index 495d299f..fa0c0ad5 100644 --- a/database/pgsql/ancestry.go +++ b/database/pgsql/ancestry.go @@ -10,191 +10,242 @@ import ( "github.com/coreos/clair/pkg/commonerr" ) -type ancestryLayerWithID struct { - database.AncestryLayer +const ( + insertAncestry = ` + INSERT INTO ancestry (name) VALUES ($1) RETURNING id` + + findAncestryLayerHashes = ` + SELECT layer.hash, ancestry_layer.ancestry_index + FROM layer, ancestry_layer + WHERE ancestry_layer.ancestry_id = $1 + AND ancestry_layer.layer_id = layer.id + ORDER BY ancestry_layer.ancestry_index ASC` + + findAncestryFeatures = ` + SELECT namespace.name, namespace.version_format, feature.name, + feature.version, feature.version_format, ancestry_layer.ancestry_index, + ancestry_feature.feature_detector_id, ancestry_feature.namespace_detector_id + FROM namespace, feature, namespaced_feature, ancestry_layer, ancestry_feature + WHERE ancestry_layer.ancestry_id = $1 + AND ancestry_feature.ancestry_layer_id = ancestry_layer.id + AND ancestry_feature.namespaced_feature_id = namespaced_feature.id + AND namespaced_feature.feature_id = feature.id + AND namespaced_feature.namespace_id = namespace.id` + + findAncestryID = `SELECT id FROM ancestry WHERE name = $1` + removeAncestry = `DELETE FROM ancestry WHERE name = $1` + insertAncestryLayers = ` + INSERT INTO ancestry_layer (ancestry_id, ancestry_index, layer_id) VALUES ($1, $2, $3) + RETURNING id` + insertAncestryFeatures = ` + INSERT INTO ancestry_feature + (ancestry_layer_id, namespaced_feature_id, feature_detector_id, namespace_detector_id) VALUES + ($1, $2, $3, $4)` +) - layerID int64 +func (tx *pgSession) FindAncestry(name string) (database.Ancestry, bool, error) { + var ( + ancestry = database.Ancestry{Name: name} + err error + ) + + id, ok, err := tx.findAncestryID(name) + if !ok || err != nil { + return ancestry, ok, err + } + + if ancestry.By, err = tx.findAncestryDetectors(id); err != nil { + return ancestry, false, err + } + + if ancestry.Layers, err = tx.findAncestryLayers(id); err != nil { + return ancestry, false, err + } + + return ancestry, true, nil } func (tx *pgSession) UpsertAncestry(ancestry database.Ancestry) error { - if ancestry.Name == "" { - log.Error("Empty ancestry name is not allowed") - return commonerr.NewBadRequestError("could not insert an ancestry with empty name") + if !ancestry.Valid() { + return database.ErrInvalidParameters } - if len(ancestry.Layers) == 0 { - log.Error("Empty ancestry is not allowed") - return commonerr.NewBadRequestError("could not insert an ancestry with 0 layers") + if err := tx.removeAncestry(ancestry.Name); err != nil { + return err } - if err := tx.deleteAncestry(ancestry.Name); err != nil { + id, err := tx.insertAncestry(ancestry.Name) + if err != nil { return err } - var ancestryID int64 - if err := tx.QueryRow(insertAncestry, ancestry.Name).Scan(&ancestryID); err != nil { - if isErrUniqueViolation(err) { - return handleError("insertAncestry", errors.New("other Go-routine is processing this ancestry (skip)")) - } - return handleError("insertAncestry", err) + detectorIDs, err := tx.findDetectorIDs(ancestry.By) + if err != nil { + return err } - if err := tx.insertAncestryLayers(ancestryID, ancestry.Layers); err != nil { + // insert ancestry metadata + if err := tx.insertAncestryDetectors(id, detectorIDs); err != nil { return err } - return tx.persistProcessors(persistAncestryLister, - "persistAncestryLister", - persistAncestryDetector, - "persistAncestryDetector", - ancestryID, ancestry.ProcessedBy) -} - -func (tx *pgSession) findAncestryID(name string) (int64, bool, error) { - var id sql.NullInt64 - if err := tx.QueryRow(searchAncestry, name).Scan(&id); err != nil { - if err == sql.ErrNoRows { - return 0, false, nil - } - - return 0, false, handleError("searchAncestry", err) + layers := make([]string, 0, len(ancestry.Layers)) + for _, layer := range ancestry.Layers { + layers = append(layers, layer.Hash) } - return id.Int64, true, nil -} + layerIDs, ok, err := tx.findLayerIDs(layers) + if err != nil { + return err + } -func (tx *pgSession) findAncestryProcessors(id int64) (database.Processors, error) { - var ( - processors database.Processors - err error - ) + if !ok { + log.Error("layer cannot be found, this indicates that the internal logic of calling UpsertAncestry is wrong or the database is corrupted.") + return database.ErrMissingEntities + } - if processors.Detectors, err = tx.findProcessors(searchAncestryDetectors, id); err != nil { - return processors, handleError("searchAncestryDetectors", err) + ancestryLayerIDs, err := tx.insertAncestryLayers(id, layerIDs) + if err != nil { + return err } - if processors.Listers, err = tx.findProcessors(searchAncestryListers, id); err != nil { - return processors, handleError("searchAncestryListers", err) + for i, id := range ancestryLayerIDs { + if err := tx.insertAncestryFeatures(id, ancestry.Layers[i]); err != nil { + return err + } } - return processors, err + return nil } -func (tx *pgSession) FindAncestry(name string) (database.Ancestry, bool, error) { - var ( - ancestry = database.Ancestry{Name: name} - err error - ) +func (tx *pgSession) insertAncestry(name string) (int64, error) { + var id int64 + err := tx.QueryRow(insertAncestry, name).Scan(&id) + if err != nil { + if isErrUniqueViolation(err) { + return 0, handleError("insertAncestry", errors.New("other Go-routine is processing this ancestry (skip)")) + } - id, ok, err := tx.findAncestryID(name) - if !ok || err != nil { - return ancestry, ok, err + return 0, handleError("insertAncestry", err) } - if ancestry.ProcessedBy, err = tx.findAncestryProcessors(id); err != nil { - return ancestry, false, err - } + log.WithFields(log.Fields{"ancestry": name, "id": id}).Debug("database: inserted ancestry") + return id, nil +} - if ancestry.Layers, err = tx.findAncestryLayers(id); err != nil { - return ancestry, false, err +func (tx *pgSession) findAncestryID(name string) (int64, bool, error) { + var id sql.NullInt64 + if err := tx.QueryRow(findAncestryID, name).Scan(&id); err != nil { + if err == sql.ErrNoRows { + return 0, false, nil + } + + return 0, false, handleError("findAncestryID", err) } - return ancestry, true, nil + return id.Int64, true, nil } -func (tx *pgSession) deleteAncestry(name string) error { +func (tx *pgSession) removeAncestry(name string) error { result, err := tx.Exec(removeAncestry, name) if err != nil { return handleError("removeAncestry", err) } - _, err = result.RowsAffected() + affected, err := result.RowsAffected() if err != nil { return handleError("removeAncestry", err) } + if affected != 0 { + log.WithField("ancestry", name).Debug("removed ancestry") + } + return nil } -func (tx *pgSession) findProcessors(query string, id int64) ([]string, error) { - var ( - processors []string - processor string - ) +func (tx *pgSession) findAncestryLayers(id int64) ([]database.AncestryLayer, error) { + detectors, err := tx.findAllDetectors() + if err != nil { + return nil, err + } - rows, err := tx.Query(query, id) + layerMap, err := tx.findAncestryLayerHashes(id) if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } + return nil, err + } + log.WithField("map", layerMap).Debug("found layer hashes") + featureMap, err := tx.findAncestryFeatures(id, detectors) + if err != nil { return nil, err } - for rows.Next() { - if err := rows.Scan(&processor); err != nil { - return nil, err + layers := make([]database.AncestryLayer, len(layerMap)) + for index, layer := range layerMap { + // index MUST match the ancestry layer slice index. + if layers[index].Hash == "" && len(layers[index].Features) == 0 { + layers[index] = database.AncestryLayer{ + Hash: layer, + Features: featureMap[index], + } + } else { + log.WithFields(log.Fields{ + "ancestry ID": id, + "duplicated ancestry index": index, + }).WithError(database.ErrInconsistent).Error("ancestry layers with same ancestry_index is not allowed") + return nil, database.ErrInconsistent } - - processors = append(processors, processor) } - return processors, nil + return layers, nil } -func (tx *pgSession) findAncestryLayers(id int64) ([]database.AncestryLayer, error) { - var ( - err error - rows *sql.Rows - // layer index -> Ancestry Layer + Layer ID - layers = map[int64]ancestryLayerWithID{} - // layer index -> layer-wise features - features = map[int64][]database.NamespacedFeature{} - ancestryLayers []database.AncestryLayer - ) - - // retrieve ancestry layer metadata - if rows, err = tx.Query(searchAncestryLayer, id); err != nil { - return nil, handleError("searchAncestryLayer", err) +func (tx *pgSession) findAncestryLayerHashes(ancestryID int64) (map[int64]string, error) { + // retrieve layer indexes and hashes + rows, err := tx.Query(findAncestryLayerHashes, ancestryID) + if err != nil { + return nil, handleError("findAncestryLayerHashes", err) } + layerHashes := map[int64]string{} for rows.Next() { var ( - layer database.AncestryLayer - index sql.NullInt64 - id sql.NullInt64 + hash string + index int64 ) - if err = rows.Scan(&layer.Hash, &id, &index); err != nil { - return nil, handleError("searchAncestryLayer", err) - } - - if !index.Valid || !id.Valid { - panic("null ancestry ID or ancestry index violates database constraints") + if err = rows.Scan(&hash, &index); err != nil { + return nil, handleError("findAncestryLayerHashes", err) } - if _, ok := layers[index.Int64]; ok { + if _, ok := layerHashes[index]; ok { // one ancestry index should correspond to only one layer return nil, database.ErrInconsistent } - layers[index.Int64] = ancestryLayerWithID{layer, id.Int64} + layerHashes[index] = hash } - for _, layer := range layers { - if layer.ProcessedBy, err = tx.findLayerProcessors(layer.layerID); err != nil { - return nil, err - } - } + return layerHashes, nil +} +func (tx *pgSession) findAncestryFeatures(ancestryID int64, detectors detectorMap) (map[int64][]database.AncestryFeature, error) { + // ancestry_index -> ancestry features + featureMap := make(map[int64][]database.AncestryFeature) // retrieve ancestry layer's namespaced features - if rows, err = tx.Query(searchAncestryFeatures, id); err != nil { - return nil, handleError("searchAncestryFeatures", err) + rows, err := tx.Query(findAncestryFeatures, ancestryID) + if err != nil { + return nil, handleError("findAncestryFeatures", err) } + defer rows.Close() + for rows.Next() { var ( - feature database.NamespacedFeature + featureDetectorID int64 + namespaceDetectorID int64 + feature database.NamespacedFeature // index is used to determine which layer the feature belongs to. index sql.NullInt64 ) @@ -206,8 +257,10 @@ func (tx *pgSession) findAncestryLayers(id int64) ([]database.AncestryLayer, err &feature.Feature.Version, &feature.Feature.VersionFormat, &index, + &featureDetectorID, + &namespaceDetectorID, ); err != nil { - return nil, handleError("searchAncestryFeatures", err) + return nil, handleError("findAncestryFeatures", err) } if feature.Feature.VersionFormat != feature.Namespace.VersionFormat { @@ -216,59 +269,88 @@ func (tx *pgSession) findAncestryLayers(id int64) ([]database.AncestryLayer, err return nil, database.ErrInconsistent } - features[index.Int64] = append(features[index.Int64], feature) - } + fDetector, ok := detectors.byID[featureDetectorID] + if !ok { + return nil, database.ErrInconsistent + } + + nsDetector, ok := detectors.byID[namespaceDetectorID] + if !ok { + return nil, database.ErrInconsistent + } - for index, layer := range layers { - layer.DetectedFeatures = features[index] - ancestryLayers = append(ancestryLayers, layer.AncestryLayer) + featureMap[index.Int64] = append(featureMap[index.Int64], database.AncestryFeature{ + NamespacedFeature: feature, + FeatureBy: fDetector, + NamespaceBy: nsDetector, + }) } - return ancestryLayers, nil + return featureMap, nil } // insertAncestryLayers inserts the ancestry layers along with its content into // the database. The layers are 0 based indexed in the original order. -func (tx *pgSession) insertAncestryLayers(ancestryID int64, layers []database.AncestryLayer) error { - //TODO(Sida): use bulk insert. - stmt, err := tx.Prepare(insertAncestryLayer) +func (tx *pgSession) insertAncestryLayers(ancestryID int64, layers []int64) ([]int64, error) { + stmt, err := tx.Prepare(insertAncestryLayers) if err != nil { - return handleError("insertAncestryLayer", err) + return nil, handleError("insertAncestryLayers", err) } - ancestryLayerIDs := []sql.NullInt64{} - for index, layer := range layers { + ancestryLayerIDs := []int64{} + for index, layerID := range layers { var ancestryLayerID sql.NullInt64 - if err := stmt.QueryRow(ancestryID, index, layer.Hash).Scan(&ancestryLayerID); err != nil { - return handleError("insertAncestryLayer", commonerr.CombineErrors(err, stmt.Close())) + if err := stmt.QueryRow(ancestryID, index, layerID).Scan(&ancestryLayerID); err != nil { + return nil, handleError("insertAncestryLayers", commonerr.CombineErrors(err, stmt.Close())) + } + + if !ancestryLayerID.Valid { + return nil, database.ErrInconsistent } - ancestryLayerIDs = append(ancestryLayerIDs, ancestryLayerID) + ancestryLayerIDs = append(ancestryLayerIDs, ancestryLayerID.Int64) } if err := stmt.Close(); err != nil { - return handleError("Failed to close insertAncestryLayer statement", err) + return nil, handleError("insertAncestryLayers", err) } - stmt, err = tx.Prepare(insertAncestryLayerFeature) - defer stmt.Close() + return ancestryLayerIDs, nil +} - for i, layer := range layers { - var ( - nsFeatureIDs []sql.NullInt64 - layerID = ancestryLayerIDs[i] - ) +func (tx *pgSession) insertAncestryFeatures(ancestryLayerID int64, layer database.AncestryLayer) error { + detectors, err := tx.findAllDetectors() + if err != nil { + return err + } - if nsFeatureIDs, err = tx.findNamespacedFeatureIDs(layer.DetectedFeatures); err != nil { - return err + nsFeatureIDs, err := tx.findNamespacedFeatureIDs(layer.GetFeatures()) + if err != nil { + return err + } + + // find the detectors for each feature + stmt, err := tx.Prepare(insertAncestryFeatures) + if err != nil { + return handleError("insertAncestryFeatures", err) + } + + defer stmt.Close() + + for index, id := range nsFeatureIDs { + namespaceDetectorID, ok := detectors.byValue[layer.Features[index].NamespaceBy] + if !ok { + return database.ErrMissingEntities } - for _, id := range nsFeatureIDs { - if _, err := stmt.Exec(layerID, id); err != nil { - return handleError("insertAncestryLayerFeature", commonerr.CombineErrors(err, stmt.Close())) - } + featureDetectorID, ok := detectors.byValue[layer.Features[index].FeatureBy] + if !ok { + return database.ErrMissingEntities } + if _, err := stmt.Exec(ancestryLayerID, id, featureDetectorID, namespaceDetectorID); err != nil { + return handleError("insertAncestryFeatures", commonerr.CombineErrors(err, stmt.Close())) + } } return nil diff --git a/database/pgsql/ancestry_test.go b/database/pgsql/ancestry_test.go index 9d1f1c5c..7f0a37f8 100644 --- a/database/pgsql/ancestry_test.go +++ b/database/pgsql/ancestry_test.go @@ -15,7 +15,6 @@ package pgsql import ( - "sort" "testing" "github.com/stretchr/testify/assert" @@ -23,190 +22,117 @@ import ( "github.com/coreos/clair/database" ) -func TestUpsertAncestry(t *testing.T) { - store, tx := openSessionForTest(t, "UpsertAncestry", true) - defer closeTest(t, store, tx) - a1 := database.Ancestry{ - Name: "a1", - Layers: []database.AncestryLayer{ - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-N", +var upsertAncestryTests = []struct { + in *database.Ancestry + err string + title string +}{ + { + title: "ancestry with invalid layer", + in: &database.Ancestry{ + Name: "a1", + Layers: []database.AncestryLayer{ + { + Hash: "layer-non-existing", }, }, }, - } - - a2 := database.Ancestry{} - - a3 := database.Ancestry{ - Name: "a", - Layers: []database.AncestryLayer{ - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-0", - }, - }, + err: database.ErrMissingEntities.Error(), + }, + { + title: "ancestry with invalid name", + in: &database.Ancestry{}, + err: database.ErrInvalidParameters.Error(), + }, + { + title: "new valid ancestry", + in: &database.Ancestry{ + Name: "a", + Layers: []database.AncestryLayer{{Hash: "layer-0"}}, }, - } - - a4 := database.Ancestry{ - Name: "a", - Layers: []database.AncestryLayer{ - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-1", - }, + }, + { + title: "ancestry with invalid feature", + in: &database.Ancestry{ + Name: "a", + By: []database.Detector{realDetectors[1], realDetectors[2]}, + Layers: []database.AncestryLayer{{Hash: "layer-1", Features: []database.AncestryFeature{ + {fakeNamespacedFeatures[1], fakeDetector[1], fakeDetector[2]}, + }}}, + }, + err: database.ErrMissingEntities.Error(), + }, + { + title: "replace old ancestry", + in: &database.Ancestry{ + Name: "a", + By: []database.Detector{realDetectors[1], realDetectors[2]}, + Layers: []database.AncestryLayer{ + {"layer-1", []database.AncestryFeature{{realNamespacedFeatures[1], realDetectors[2], realDetectors[1]}}}, }, }, - } - - f1 := database.Feature{ - Name: "wechat", - Version: "0.5", - VersionFormat: "dpkg", - } - - // not in database - f2 := database.Feature{ - Name: "wechat", - Version: "0.6", - VersionFormat: "dpkg", - } - - n1 := database.Namespace{ - Name: "debian:7", - VersionFormat: "dpkg", - } - - p := database.Processors{ - Listers: []string{"dpkg", "non-existing"}, - Detectors: []string{"os-release", "non-existing"}, - } - - nsf1 := database.NamespacedFeature{ - Namespace: n1, - Feature: f1, - } - - // not in database - nsf2 := database.NamespacedFeature{ - Namespace: n1, - Feature: f2, - } - - a4.ProcessedBy = p - // invalid case - assert.NotNil(t, tx.UpsertAncestry(a1)) - assert.NotNil(t, tx.UpsertAncestry(a2)) - // valid case - assert.Nil(t, tx.UpsertAncestry(a3)) - a4.Layers[0].DetectedFeatures = []database.NamespacedFeature{nsf1, nsf2} - // replace invalid case - assert.NotNil(t, tx.UpsertAncestry(a4)) - a4.Layers[0].DetectedFeatures = []database.NamespacedFeature{nsf1} - // replace valid case - assert.Nil(t, tx.UpsertAncestry(a4)) - // validate - ancestry, ok, err := tx.FindAncestry("a") - assert.Nil(t, err) - assert.True(t, ok) - assertAncestryEqual(t, a4, ancestry) -} - -func assertProcessorsEqual(t *testing.T, expected database.Processors, actual database.Processors) bool { - sort.Strings(expected.Detectors) - sort.Strings(actual.Detectors) - sort.Strings(expected.Listers) - sort.Strings(actual.Listers) - return assert.Equal(t, expected.Detectors, actual.Detectors) && assert.Equal(t, expected.Listers, actual.Listers) + }, } -func assertAncestryEqual(t *testing.T, expected database.Ancestry, actual database.Ancestry) bool { - assert.Equal(t, expected.Name, actual.Name) - assertProcessorsEqual(t, expected.ProcessedBy, actual.ProcessedBy) - if assert.Equal(t, len(expected.Layers), len(actual.Layers)) { - for index, layer := range expected.Layers { - if !assertAncestryLayerEqual(t, layer, actual.Layers[index]) { - return false +func TestUpsertAncestry(t *testing.T) { + store, tx := openSessionForTest(t, "UpsertAncestry", true) + defer closeTest(t, store, tx) + for _, test := range upsertAncestryTests { + t.Run(test.title, func(t *testing.T) { + err := tx.UpsertAncestry(*test.in) + if test.err != "" { + assert.EqualError(t, err, test.err, "unexpected error") + return } - } - return true + assert.Nil(t, err) + actual, ok, err := tx.FindAncestry(test.in.Name) + assert.Nil(t, err) + assert.True(t, ok) + database.AssertAncestryEqual(t, test.in, &actual) + }) } - return false } -func assertAncestryLayerEqual(t *testing.T, expected database.AncestryLayer, actual database.AncestryLayer) bool { - return assertLayerEqual(t, expected.LayerMetadata, actual.LayerMetadata) && - assertNamespacedFeatureEqual(t, expected.DetectedFeatures, actual.DetectedFeatures) +var findAncestryTests = []struct { + title string + in string + + ancestry *database.Ancestry + err string + ok bool +}{ + { + title: "missing ancestry", + in: "ancestry-non", + err: "", + ancestry: nil, + ok: false, + }, + { + title: "valid ancestry", + in: "ancestry-2", + err: "", + ok: true, + ancestry: takeAncestryPointerFromMap(realAncestries, 2), + }, } func TestFindAncestry(t *testing.T) { store, tx := openSessionForTest(t, "FindAncestry", true) defer closeTest(t, store, tx) + for _, test := range findAncestryTests { + t.Run(test.title, func(t *testing.T) { + ancestry, ok, err := tx.FindAncestry(test.in) + if test.err != "" { + assert.EqualError(t, err, test.err, "unexpected error") + return + } - // invalid - _, ok, err := tx.FindAncestry("ancestry-non") - if assert.Nil(t, err) { - assert.False(t, ok) - } - - expected := database.Ancestry{ - Name: "ancestry-2", - ProcessedBy: database.Processors{ - Detectors: []string{"os-release"}, - Listers: []string{"dpkg"}, - }, - Layers: []database.AncestryLayer{ - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-0", - }, - DetectedFeatures: []database.NamespacedFeature{ - { - Namespace: database.Namespace{ - Name: "debian:7", - VersionFormat: "dpkg", - }, - Feature: database.Feature{ - Name: "wechat", - Version: "0.5", - VersionFormat: "dpkg", - }, - }, - { - Namespace: database.Namespace{ - Name: "debian:8", - VersionFormat: "dpkg", - }, - Feature: database.Feature{ - Name: "openssl", - Version: "1.0", - VersionFormat: "dpkg", - }, - }, - }, - }, - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-1", - }, - }, - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-2", - }, - }, - { - LayerMetadata: database.LayerMetadata{ - Hash: "layer-3b", - }, - }, - }, - } - // valid - ancestry, ok, err := tx.FindAncestry("ancestry-2") - if assert.Nil(t, err) && assert.True(t, ok) { - assertAncestryEqual(t, expected, ancestry) + assert.Nil(t, err) + assert.Equal(t, test.ok, ok) + if test.ok { + database.AssertAncestryEqual(t, test.ancestry, &ancestry) + } + }) } } diff --git a/database/pgsql/complex_test.go b/database/pgsql/complex_test.go index 07d6f55f..de8b0f20 100644 --- a/database/pgsql/complex_test.go +++ b/database/pgsql/complex_test.go @@ -220,7 +220,7 @@ func TestCaching(t *testing.T) { actualAffectedNames = append(actualAffectedNames, s.Name) } - assert.Len(t, strutil.CompareStringLists(expectedAffectedNames, actualAffectedNames), 0) - assert.Len(t, strutil.CompareStringLists(actualAffectedNames, expectedAffectedNames), 0) + assert.Len(t, strutil.Difference(expectedAffectedNames, actualAffectedNames), 0) + assert.Len(t, strutil.Difference(actualAffectedNames, expectedAffectedNames), 0) } } diff --git a/database/pgsql/detector.go b/database/pgsql/detector.go new file mode 100644 index 00000000..99b5c283 --- /dev/null +++ b/database/pgsql/detector.go @@ -0,0 +1,198 @@ +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pgsql + +import ( + "database/sql" + + "github.com/deckarep/golang-set" + log "github.com/sirupsen/logrus" + + "github.com/coreos/clair/database" +) + +const ( + soiDetector = ` + INSERT INTO detector (name, version, dtype) + SELECT CAST ($1 AS TEXT), CAST ($2 AS TEXT), CAST ($3 AS detector_type ) + WHERE NOT EXISTS (SELECT id FROM detector WHERE name = $1 AND version = $2 AND dtype = $3);` + + selectAncestryDetectors = ` + SELECT d.name, d.version, d.dtype + FROM ancestry_detector, detector AS d + WHERE ancestry_detector.detector_id = d.id AND ancestry_detector.ancestry_id = $1;` + + selectLayerDetectors = ` + SELECT d.name, d.version, d.dtype + FROM layer_detector, detector AS d + WHERE layer_detector.detector_id = d.id AND layer_detector.layer_id = $1;` + + insertAncestryDetectors = ` + INSERT INTO ancestry_detector (ancestry_id, detector_id) + SELECT $1, $2 + WHERE NOT EXISTS (SELECT id FROM ancestry_detector WHERE ancestry_id = $1 AND detector_id = $2)` + + persistLayerDetector = ` + INSERT INTO layer_detector (layer_id, detector_id) + SELECT $1, $2 + WHERE NOT EXISTS (SELECT id FROM layer_detector WHERE layer_id = $1 AND detector_id = $2)` + + findDetectorID = `SELECT id FROM detector WHERE name = $1 AND version = $2 AND dtype = $3` + findAllDetectors = `SELECT id, name, version, dtype FROM detector` +) + +type detectorMap struct { + byID map[int64]database.Detector + byValue map[database.Detector]int64 +} + +func (tx *pgSession) PersistDetectors(detectors []database.Detector) error { + for _, d := range detectors { + if !d.Valid() { + log.WithField("detector", d).Debug("Invalid Detector") + return database.ErrInvalidParameters + } + + r, err := tx.Exec(soiDetector, d.Name, d.Version, d.DType) + if err != nil { + return handleError("soiDetector", err) + } + + count, err := r.RowsAffected() + if err != nil { + return handleError("soiDetector", err) + } + + if count == 0 { + log.Debug("detector already exists: ", d) + } + } + + return nil +} + +func (tx *pgSession) persistLayerDetector(layerID int64, detectorID int64) error { + if _, err := tx.Exec(persistLayerDetector, layerID, detectorID); err != nil { + return handleError("persistLayerDetector", err) + } + + return nil +} + +func (tx *pgSession) persistLayerDetectors(layerID int64, detectorIDs []int64) error { + alreadySaved := mapset.NewSet() + for _, id := range detectorIDs { + if alreadySaved.Contains(id) { + continue + } + + alreadySaved.Add(id) + if err := tx.persistLayerDetector(layerID, id); err != nil { + return err + } + } + + return nil +} + +func (tx *pgSession) insertAncestryDetectors(ancestryID int64, detectorIDs []int64) error { + for _, detectorID := range detectorIDs { + if _, err := tx.Exec(insertAncestryDetectors, ancestryID, detectorID); err != nil { + return handleError("insertAncestryDetectors", err) + } + } + + return nil +} + +func (tx *pgSession) findAncestryDetectors(id int64) ([]database.Detector, error) { + detectors, err := tx.getDetectors(selectAncestryDetectors, id) + log.WithField("detectors", detectors).Debug("found ancestry detectors") + return detectors, err +} + +func (tx *pgSession) findLayerDetectors(id int64) ([]database.Detector, error) { + detectors, err := tx.getDetectors(selectLayerDetectors, id) + log.WithField("detectors", detectors).Debug("found layer detectors") + return detectors, err +} + +// findDetectorIDs retrieve ids of the detectors from the database, if any is not +// found, return the error. +func (tx *pgSession) findDetectorIDs(detectors []database.Detector) ([]int64, error) { + ids := []int64{} + for _, d := range detectors { + id := sql.NullInt64{} + err := tx.QueryRow(findDetectorID, d.Name, d.Version, d.DType).Scan(&id) + if err != nil { + return nil, handleError("findDetectorID", err) + } + + if !id.Valid { + return nil, database.ErrInconsistent + } + + ids = append(ids, id.Int64) + } + + return ids, nil +} + +func (tx *pgSession) getDetectors(query string, id int64) ([]database.Detector, error) { + rows, err := tx.Query(query, id) + if err != nil { + return nil, handleError("getDetectors", err) + } + + detectors := []database.Detector{} + for rows.Next() { + d := database.Detector{} + err := rows.Scan(&d.Name, &d.Version, &d.DType) + if err != nil { + return nil, handleError("getDetectors", err) + } + + if !d.Valid() { + return nil, database.ErrInvalidDetector + } + + detectors = append(detectors, d) + } + + return detectors, nil +} + +func (tx *pgSession) findAllDetectors() (detectorMap, error) { + rows, err := tx.Query(findAllDetectors) + if err != nil { + return detectorMap{}, handleError("searchAllDetectors", err) + } + + detectors := detectorMap{byID: make(map[int64]database.Detector), byValue: make(map[database.Detector]int64)} + for rows.Next() { + var ( + id int64 + d database.Detector + ) + if err := rows.Scan(&id, &d.Name, &d.Version, &d.DType); err != nil { + return detectorMap{}, handleError("searchAllDetectors", err) + } + + detectors.byID[id] = d + detectors.byValue[d] = id + } + + return detectors, nil +} diff --git a/database/pgsql/detector_test.go b/database/pgsql/detector_test.go new file mode 100644 index 00000000..582da60b --- /dev/null +++ b/database/pgsql/detector_test.go @@ -0,0 +1,119 @@ +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pgsql + +import ( + "testing" + + "github.com/deckarep/golang-set" + "github.com/stretchr/testify/require" + + "github.com/coreos/clair/database" +) + +func testGetAllDetectors(tx *pgSession) []database.Detector { + query := `SELECT name, version, dtype FROM detector` + rows, err := tx.Query(query) + if err != nil { + panic(err) + } + + detectors := []database.Detector{} + for rows.Next() { + d := database.Detector{} + if err := rows.Scan(&d.Name, &d.Version, &d.DType); err != nil { + panic(err) + } + + detectors = append(detectors, d) + } + + return detectors +} + +var persistDetectorTests = []struct { + title string + in []database.Detector + err string +}{ + { + title: "invalid detector", + in: []database.Detector{ + {}, + database.NewFeatureDetector("name", "2.0"), + }, + err: database.ErrInvalidParameters.Error(), + }, + { + title: "invalid detector 2", + in: []database.Detector{ + database.NewFeatureDetector("name", "2.0"), + {"name", "1.0", "random not valid dtype"}, + }, + err: database.ErrInvalidParameters.Error(), + }, + { + title: "detectors with some different fields", + in: []database.Detector{ + database.NewFeatureDetector("name", "2.0"), + database.NewFeatureDetector("name", "1.0"), + database.NewNamespaceDetector("name", "1.0"), + }, + }, + { + title: "duplicated detectors (parameter level)", + in: []database.Detector{ + database.NewFeatureDetector("name", "1.0"), + database.NewFeatureDetector("name", "1.0"), + }, + }, + { + title: "duplicated detectors (db level)", + in: []database.Detector{ + database.NewNamespaceDetector("os-release", "1.0"), + database.NewNamespaceDetector("os-release", "1.0"), + database.NewFeatureDetector("dpkg", "1.0"), + }, + }, +} + +func TestPersistDetector(t *testing.T) { + datastore, tx := openSessionForTest(t, "PersistDetector", true) + defer closeTest(t, datastore, tx) + + for _, test := range persistDetectorTests { + t.Run(test.title, func(t *testing.T) { + err := tx.PersistDetectors(test.in) + if test.err != "" { + require.EqualError(t, err, test.err) + return + } + + detectors := testGetAllDetectors(tx) + + // ensure no duplicated detectors + detectorSet := mapset.NewSet() + for _, d := range detectors { + require.False(t, detectorSet.Contains(d), "duplicated: %v", d) + detectorSet.Add(d) + } + + // ensure all persisted detectors are actually saved + for _, d := range test.in { + require.True(t, detectorSet.Contains(d), "detector: %v, detectors: %v", d, detectorSet) + } + }) + } +} diff --git a/database/pgsql/feature.go b/database/pgsql/feature.go index 81ef857d..345b73a3 100644 --- a/database/pgsql/feature.go +++ b/database/pgsql/feature.go @@ -16,7 +16,6 @@ package pgsql import ( "database/sql" - "errors" "sort" "github.com/lib/pq" @@ -27,15 +26,43 @@ import ( "github.com/coreos/clair/pkg/commonerr" ) -var ( - errFeatureNotFound = errors.New("Feature not found") +const ( + soiNamespacedFeature = ` + WITH new_feature_ns AS ( + INSERT INTO namespaced_feature(feature_id, namespace_id) + SELECT CAST ($1 AS INTEGER), CAST ($2 AS INTEGER) + WHERE NOT EXISTS ( SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2) + RETURNING id + ) + SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2 + UNION + SELECT id FROM new_feature_ns` + + searchPotentialAffectingVulneraibilities = ` + SELECT nf.id, v.id, vaf.affected_version, vaf.id + FROM vulnerability_affected_feature AS vaf, vulnerability AS v, + namespaced_feature AS nf, feature AS f + WHERE nf.id = ANY($1) + AND nf.feature_id = f.id + AND nf.namespace_id = v.namespace_id + AND vaf.feature_name = f.name + AND vaf.vulnerability_id = v.id + AND v.deleted_at IS NULL` + + searchNamespacedFeaturesVulnerabilities = ` + SELECT vanf.namespaced_feature_id, v.name, v.description, v.link, + v.severity, v.metadata, vaf.fixedin, n.name, n.version_format + FROM vulnerability_affected_namespaced_feature AS vanf, + Vulnerability AS v, + vulnerability_affected_feature AS vaf, + namespace AS n + WHERE vanf.namespaced_feature_id = ANY($1) + AND vaf.id = vanf.added_by + AND v.id = vanf.vulnerability_id + AND n.id = v.namespace_id + AND v.deleted_at IS NULL` ) -type vulnerabilityAffecting struct { - vulnerabilityID int64 - addedByID int64 -} - func (tx *pgSession) PersistFeatures(features []database.Feature) error { if len(features) == 0 { return nil @@ -88,7 +115,7 @@ func (tx *pgSession) searchAffectingVulnerabilities(features []database.Namespac fMap := map[int64]database.NamespacedFeature{} for i, f := range features { if !ids[i].Valid { - return nil, errFeatureNotFound + return nil, database.ErrMissingEntities } fMap[ids[i].Int64] = f } @@ -180,7 +207,7 @@ func (tx *pgSession) PersistNamespacedFeatures(features []database.NamespacedFea if ids, err := tx.findFeatureIDs(fToFind); err == nil { for i, id := range ids { if !id.Valid { - return errFeatureNotFound + return database.ErrMissingEntities } fIDs[fToFind[i]] = id } @@ -196,7 +223,7 @@ func (tx *pgSession) PersistNamespacedFeatures(features []database.NamespacedFea if ids, err := tx.findNamespaceIDs(nsToFind); err == nil { for i, id := range ids { if !id.Valid { - return errNamespaceNotFound + return database.ErrMissingEntities } nsIDs[nsToFind[i]] = id } diff --git a/database/pgsql/feature_test.go b/database/pgsql/feature_test.go index 934b8cc1..2823e1e8 100644 --- a/database/pgsql/feature_test.go +++ b/database/pgsql/feature_test.go @@ -52,7 +52,7 @@ func TestPersistNamespacedFeatures(t *testing.T) { // existing features f1 := database.Feature{ - Name: "wechat", + Name: "ourchat", Version: "0.5", VersionFormat: "dpkg", } @@ -213,27 +213,6 @@ func listFeatures(t *testing.T, tx *pgSession) []database.Feature { return fs } -func assertFeaturesEqual(t *testing.T, expected []database.Feature, actual []database.Feature) bool { - if assert.Len(t, actual, len(expected)) { - has := map[database.Feature]bool{} - for _, nf := range expected { - has[nf] = false - } - - for _, nf := range actual { - has[nf] = true - } - - for nf, visited := range has { - if !assert.True(t, visited, nf.Name+" is expected") { - return false - } - return true - } - } - return false -} - func assertNamespacedFeatureEqual(t *testing.T, expected []database.NamespacedFeature, actual []database.NamespacedFeature) bool { if assert.Len(t, actual, len(expected)) { has := map[database.NamespacedFeature]bool{} diff --git a/database/pgsql/keyvalue.go b/database/pgsql/keyvalue.go index 1f85fab5..9c985279 100644 --- a/database/pgsql/keyvalue.go +++ b/database/pgsql/keyvalue.go @@ -23,6 +23,15 @@ import ( "github.com/coreos/clair/pkg/commonerr" ) +const ( + searchKeyValue = `SELECT value FROM KeyValue WHERE key = $1` + upsertKeyValue = ` + INSERT INTO KeyValue(key, value) + VALUES ($1, $2) + ON CONFLICT ON CONSTRAINT keyvalue_key_key + DO UPDATE SET key=$1, value=$2` +) + func (tx *pgSession) UpdateKeyValue(key, value string) (err error) { if key == "" || value == "" { log.Warning("could not insert a flag which has an empty name or value") diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go index 8a35a144..ebea1849 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer.go @@ -18,300 +18,349 @@ import ( "database/sql" "sort" + "github.com/deckarep/golang-set" + "github.com/coreos/clair/database" "github.com/coreos/clair/pkg/commonerr" ) +const ( + soiLayer = ` + WITH new_layer AS ( + INSERT INTO layer (hash) + SELECT CAST ($1 AS VARCHAR) + WHERE NOT EXISTS (SELECT id FROM layer WHERE hash = $1) + RETURNING id + ) + SELECT id FROM new_Layer + UNION + SELECT id FROM layer WHERE hash = $1` + + findLayerFeatures = ` + SELECT f.name, f.version, f.version_format, lf.detector_id + FROM layer_feature AS lf, feature AS f + WHERE lf.feature_id = f.id + AND lf.layer_id = $1` + + findLayerNamespaces = ` + SELECT ns.name, ns.version_format, ln.detector_id + FROM layer_namespace AS ln, namespace AS ns + WHERE ln.namespace_id = ns.id + AND ln.layer_id = $1` + + findLayerID = `SELECT id FROM layer WHERE hash = $1` +) + +// dbLayerNamespace represents the layer_namespace table. +type dbLayerNamespace struct { + layerID int64 + namespaceID int64 + detectorID int64 +} + +// dbLayerFeature represents the layer_feature table +type dbLayerFeature struct { + layerID int64 + featureID int64 + detectorID int64 +} + func (tx *pgSession) FindLayer(hash string) (database.Layer, bool, error) { - var ( - layer database.Layer - layerID int64 - ok bool - err error - ) + layer := database.Layer{Hash: hash} + if hash == "" { + return layer, false, commonerr.NewBadRequestError("non empty layer hash is expected.") + } - layer.LayerMetadata, layerID, ok, err = tx.findLayer(hash) + layerID, ok, err := tx.findLayerID(hash) + if err != nil || !ok { + return layer, ok, err + } + + detectorMap, err := tx.findAllDetectors() if err != nil { return layer, false, err } - if !ok { - return layer, false, nil + if layer.By, err = tx.findLayerDetectors(layerID); err != nil { + return layer, false, err + } + + if layer.Features, err = tx.findLayerFeatures(layerID, detectorMap); err != nil { + return layer, false, err + } + + if layer.Namespaces, err = tx.findLayerNamespaces(layerID, detectorMap); err != nil { + return layer, false, err } - layer.Features, err = tx.findLayerFeatures(layerID) - layer.Namespaces, err = tx.findLayerNamespaces(layerID) return layer, true, nil } -func (tx *pgSession) persistLayer(hash string) (int64, error) { +func sanitizePersistLayerInput(hash string, features []database.LayerFeature, namespaces []database.LayerNamespace, detectedBy []database.Detector) error { if hash == "" { - return -1, commonerr.NewBadRequestError("Empty Layer Hash is not allowed") + return commonerr.NewBadRequestError("expected non-empty layer hash") } - id := sql.NullInt64{} - if err := tx.QueryRow(soiLayer, hash).Scan(&id); err != nil { - return -1, handleError("queryPersistLayer", err) + detectedBySet := mapset.NewSet() + for _, d := range detectedBy { + detectedBySet.Add(d) } - if !id.Valid { - panic("null layer.id violates database constraint") + for _, f := range features { + if !detectedBySet.Contains(f.By) { + return database.ErrInvalidParameters + } } - return id.Int64, nil -} - -// PersistLayer relates layer identified by hash with namespaces, -// features and processors provided. If the layer, namespaces, features are not -// in database, the function returns an error. -func (tx *pgSession) PersistLayer(hash string, namespaces []database.Namespace, features []database.Feature, processedBy database.Processors) error { - if hash == "" { - return commonerr.NewBadRequestError("Empty layer hash is not allowed") + for _, n := range namespaces { + if !detectedBySet.Contains(n.By) { + return database.ErrInvalidParameters + } } + return nil +} + +// PersistLayer saves the content of a layer to the database. +func (tx *pgSession) PersistLayer(hash string, features []database.LayerFeature, namespaces []database.LayerNamespace, detectedBy []database.Detector) error { var ( - err error - id int64 + err error + id int64 + detectorIDs []int64 ) - if id, err = tx.persistLayer(hash); err != nil { + if err = sanitizePersistLayerInput(hash, features, namespaces, detectedBy); err != nil { return err } - if err = tx.persistLayerNamespace(id, namespaces); err != nil { + if id, err = tx.soiLayer(hash); err != nil { + return err + } + + if detectorIDs, err = tx.findDetectorIDs(detectedBy); err != nil { + if err == commonerr.ErrNotFound { + return database.ErrMissingEntities + } + return err } - if err = tx.persistLayerFeatures(id, features); err != nil { + if err = tx.persistLayerDetectors(id, detectorIDs); err != nil { return err } - if err = tx.persistLayerDetectors(id, processedBy.Detectors); err != nil { + if err = tx.persistAllLayerFeatures(id, features); err != nil { return err } - if err = tx.persistLayerListers(id, processedBy.Listers); err != nil { + if err = tx.persistAllLayerNamespaces(id, namespaces); err != nil { return err } return nil } -func (tx *pgSession) persistLayerDetectors(id int64, detectors []string) error { - if len(detectors) == 0 { - return nil +func (tx *pgSession) persistAllLayerNamespaces(layerID int64, namespaces []database.LayerNamespace) error { + detectorMap, err := tx.findAllDetectors() + if err != nil { + return err } - // Sorting is needed before inserting into database to prevent deadlock. - sort.Strings(detectors) - keys := make([]interface{}, len(detectors)*2) - for i, d := range detectors { - keys[i*2] = id - keys[i*2+1] = d + // TODO(sidac): This kind of type conversion is very useless and wasteful, + // we need interfaces around the database models to reduce these kind of + // operations. + rawNamespaces := make([]database.Namespace, 0, len(namespaces)) + for _, ns := range namespaces { + rawNamespaces = append(rawNamespaces, ns.Namespace) } - _, err := tx.Exec(queryPersistLayerDetectors(len(detectors)), keys...) + + rawNamespaceIDs, err := tx.findNamespaceIDs(rawNamespaces) if err != nil { - return handleError("queryPersistLayerDetectors", err) + return err } - return nil -} -func (tx *pgSession) persistLayerListers(id int64, listers []string) error { - if len(listers) == 0 { - return nil - } + dbLayerNamespaces := make([]dbLayerNamespace, 0, len(namespaces)) + for i, ns := range namespaces { + detectorID := detectorMap.byValue[ns.By] + namespaceID := rawNamespaceIDs[i].Int64 + if !rawNamespaceIDs[i].Valid { + return database.ErrMissingEntities + } - sort.Strings(listers) - keys := make([]interface{}, len(listers)*2) - for i, d := range listers { - keys[i*2] = id - keys[i*2+1] = d + dbLayerNamespaces = append(dbLayerNamespaces, dbLayerNamespace{layerID, namespaceID, detectorID}) } - _, err := tx.Exec(queryPersistLayerListers(len(listers)), keys...) + return tx.persistLayerNamespaces(dbLayerNamespaces) +} + +func (tx *pgSession) persistAllLayerFeatures(layerID int64, features []database.LayerFeature) error { + detectorMap, err := tx.findAllDetectors() if err != nil { - return handleError("queryPersistLayerDetectors", err) + return err } - return nil -} -func (tx *pgSession) persistLayerFeatures(id int64, features []database.Feature) error { - if len(features) == 0 { - return nil + rawFeatures := make([]database.Feature, 0, len(features)) + for _, f := range features { + rawFeatures = append(rawFeatures, f.Feature) } - fIDs, err := tx.findFeatureIDs(features) + featureIDs, err := tx.findFeatureIDs(rawFeatures) if err != nil { return err } - ids := make([]int, len(fIDs)) - for i, fID := range fIDs { - if !fID.Valid { - return errNamespaceNotFound + dbFeatures := make([]dbLayerFeature, 0, len(features)) + for i, f := range features { + detectorID := detectorMap.byValue[f.By] + featureID := featureIDs[i].Int64 + if !featureIDs[i].Valid { + return database.ErrMissingEntities } - ids[i] = int(fID.Int64) - } - sort.IntSlice(ids).Sort() - keys := make([]interface{}, len(features)*2) - for i, fID := range ids { - keys[i*2] = id - keys[i*2+1] = fID + dbFeatures = append(dbFeatures, dbLayerFeature{layerID, featureID, detectorID}) } - _, err = tx.Exec(queryPersistLayerFeature(len(features)), keys...) - if err != nil { - return handleError("queryPersistLayerFeature", err) + if err := tx.persistLayerFeatures(dbFeatures); err != nil { + return err } + return nil } -func (tx *pgSession) persistLayerNamespace(id int64, namespaces []database.Namespace) error { - if len(namespaces) == 0 { +func (tx *pgSession) persistLayerFeatures(features []dbLayerFeature) error { + if len(features) == 0 { return nil } - nsIDs, err := tx.findNamespaceIDs(namespaces) - if err != nil { - return err - } - - // for every bulk persist operation, the input data should be sorted. - ids := make([]int, len(nsIDs)) - for i, nsID := range nsIDs { - if !nsID.Valid { - panic(errNamespaceNotFound) - } - ids[i] = int(nsID.Int64) - } + sort.Slice(features, func(i, j int) bool { + return features[i].featureID < features[j].featureID + }) - sort.IntSlice(ids).Sort() - - keys := make([]interface{}, len(namespaces)*2) - for i, nsID := range ids { - keys[i*2] = id - keys[i*2+1] = nsID + keys := make([]interface{}, len(features)*3) + for i, feature := range features { + keys[i*3] = feature.layerID + keys[i*3+1] = feature.featureID + keys[i*3+2] = feature.detectorID } - _, err = tx.Exec(queryPersistLayerNamespace(len(namespaces)), keys...) + _, err := tx.Exec(queryPersistLayerFeature(len(features)), keys...) if err != nil { - return handleError("queryPersistLayerNamespace", err) + return handleError("queryPersistLayerFeature", err) } return nil } -func (tx *pgSession) persistProcessors(listerQuery, listerQueryName, detectorQuery, detectorQueryName string, id int64, processors database.Processors) error { - stmt, err := tx.Prepare(listerQuery) - if err != nil { - return handleError(listerQueryName, err) +func (tx *pgSession) persistLayerNamespaces(namespaces []dbLayerNamespace) error { + if len(namespaces) == 0 { + return nil } - for _, l := range processors.Listers { - _, err := stmt.Exec(id, l) - if err != nil { - stmt.Close() - return handleError(listerQueryName, err) - } - } + // for every bulk persist operation, the input data should be sorted. + sort.Slice(namespaces, func(i, j int) bool { + return namespaces[i].namespaceID < namespaces[j].namespaceID + }) - if err := stmt.Close(); err != nil { - return handleError(listerQueryName, err) + elementSize := 3 + keys := make([]interface{}, len(namespaces)*elementSize) + for i, row := range namespaces { + keys[i*3] = row.layerID + keys[i*3+1] = row.namespaceID + keys[i*3+2] = row.detectorID } - stmt, err = tx.Prepare(detectorQuery) + _, err := tx.Exec(queryPersistLayerNamespace(len(namespaces)), keys...) if err != nil { - return handleError(detectorQueryName, err) - } - - for _, d := range processors.Detectors { - _, err := stmt.Exec(id, d) - if err != nil { - stmt.Close() - return handleError(detectorQueryName, err) - } - } - - if err := stmt.Close(); err != nil { - return handleError(detectorQueryName, err) + return handleError("queryPersistLayerNamespace", err) } return nil } -func (tx *pgSession) findLayerNamespaces(layerID int64) ([]database.Namespace, error) { - var namespaces []database.Namespace - - rows, err := tx.Query(searchLayerNamespaces, layerID) +func (tx *pgSession) findLayerNamespaces(layerID int64, detectors detectorMap) ([]database.LayerNamespace, error) { + rows, err := tx.Query(findLayerNamespaces, layerID) if err != nil { - return nil, handleError("searchLayerFeatures", err) + return nil, handleError("findLayerNamespaces", err) } + namespaces := []database.LayerNamespace{} for rows.Next() { - ns := database.Namespace{} - err := rows.Scan(&ns.Name, &ns.VersionFormat) - if err != nil { + var ( + namespace database.LayerNamespace + detectorID int64 + ) + + if err := rows.Scan(&namespace.Name, &namespace.VersionFormat, &detectorID); err != nil { return nil, err } - namespaces = append(namespaces, ns) + + namespace.By = detectors.byID[detectorID] + namespaces = append(namespaces, namespace) } + return namespaces, nil } -func (tx *pgSession) findLayerFeatures(layerID int64) ([]database.Feature, error) { - var features []database.Feature - - rows, err := tx.Query(searchLayerFeatures, layerID) +func (tx *pgSession) findLayerFeatures(layerID int64, detectors detectorMap) ([]database.LayerFeature, error) { + rows, err := tx.Query(findLayerFeatures, layerID) if err != nil { - return nil, handleError("searchLayerFeatures", err) + return nil, handleError("findLayerFeatures", err) } + defer rows.Close() + features := []database.LayerFeature{} for rows.Next() { - f := database.Feature{} - err := rows.Scan(&f.Name, &f.Version, &f.VersionFormat) - if err != nil { - return nil, err + var ( + detectorID int64 + feature database.LayerFeature + ) + if err := rows.Scan(&feature.Name, &feature.Version, &feature.VersionFormat, &detectorID); err != nil { + return nil, handleError("findLayerFeatures", err) } - features = append(features, f) + + feature.By = detectors.byID[detectorID] + features = append(features, feature) } + return features, nil } -func (tx *pgSession) findLayer(hash string) (database.LayerMetadata, int64, bool, error) { - var ( - layerID int64 - layer = database.LayerMetadata{Hash: hash, ProcessedBy: database.Processors{}} - ) - - if hash == "" { - return layer, layerID, false, commonerr.NewBadRequestError("Empty Layer Hash is not allowed") - } - - err := tx.QueryRow(searchLayer, hash).Scan(&layerID) +func (tx *pgSession) findLayerID(hash string) (int64, bool, error) { + var layerID int64 + err := tx.QueryRow(findLayerID, hash).Scan(&layerID) if err != nil { if err == sql.ErrNoRows { - return layer, layerID, false, nil + return layerID, false, nil } - return layer, layerID, false, err + + return layerID, false, handleError("findLayerID", err) } - layer.ProcessedBy, err = tx.findLayerProcessors(layerID) - return layer, layerID, true, err + return layerID, true, nil } -func (tx *pgSession) findLayerProcessors(id int64) (database.Processors, error) { - var ( - err error - processors database.Processors - ) +func (tx *pgSession) findLayerIDs(hashes []string) ([]int64, bool, error) { + layerIDs := make([]int64, 0, len(hashes)) + for _, hash := range hashes { + id, ok, err := tx.findLayerID(hash) + if !ok { + return nil, false, nil + } - if processors.Detectors, err = tx.findProcessors(searchLayerDetectors, id); err != nil { - return processors, handleError("searchLayerDetectors", err) + if err != nil { + return nil, false, err + } + + layerIDs = append(layerIDs, id) } - if processors.Listers, err = tx.findProcessors(searchLayerListers, id); err != nil { - return processors, handleError("searchLayerListers", err) + return layerIDs, true, nil +} + +func (tx *pgSession) soiLayer(hash string) (int64, error) { + var id int64 + if err := tx.QueryRow(soiLayer, hash).Scan(&id); err != nil { + return 0, handleError("soiLayer", err) } - return processors, nil + return id, nil } diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer_test.go index 6fe8bed3..478b2171 100644 --- a/database/pgsql/layer_test.go +++ b/database/pgsql/layer_test.go @@ -22,105 +22,169 @@ import ( "github.com/coreos/clair/database" ) -func TestPersistLayer(t *testing.T) { - datastore, tx := openSessionForTest(t, "PersistLayer", false) - defer closeTest(t, datastore, tx) - - // invalid - assert.NotNil(t, tx.PersistLayer("", nil, nil, database.Processors{})) - // insert namespaces + features to - namespaces := []database.Namespace{ - { - Name: "sushi shop", - VersionFormat: "apk", +var persistLayerTests = []struct { + title string + name string + by []database.Detector + features []database.LayerFeature + namespaces []database.LayerNamespace + layer *database.Layer + err string +}{ + { + title: "invalid layer name", + name: "", + err: "expected non-empty layer hash", + }, + { + title: "layer with inconsistent feature and detectors", + name: "random-forest", + by: []database.Detector{realDetectors[2]}, + features: []database.LayerFeature{ + {realFeatures[1], realDetectors[1]}, }, - } - - features := []database.Feature{ - { - Name: "blue fin sashimi", - Version: "v1.0", - VersionFormat: "apk", + err: "database: parameters are not valid", + }, + { + title: "layer with non-existing feature", + name: "random-forest", + err: "database: associated immutable entities are missing in the database", + by: []database.Detector{realDetectors[2]}, + features: []database.LayerFeature{ + {fakeFeatures[1], realDetectors[2]}, }, - } - - processors := database.Processors{ - Listers: []string{"release"}, - Detectors: []string{"apk"}, - } - - assert.Nil(t, tx.PersistNamespaces(namespaces)) - assert.Nil(t, tx.PersistFeatures(features)) - - // Valid - assert.Nil(t, tx.PersistLayer("RANDOM_FOREST", namespaces, features, processors)) - - nonExistingFeature := []database.Feature{{Name: "lobster sushi", Version: "v0.1", VersionFormat: "apk"}} - // Invalid: - assert.NotNil(t, tx.PersistLayer("RANDOM_FOREST", namespaces, nonExistingFeature, processors)) - - assert.Nil(t, tx.PersistFeatures(nonExistingFeature)) - // Update the layer - assert.Nil(t, tx.PersistLayer("RANDOM_FOREST", namespaces, nonExistingFeature, processors)) - - // confirm update - layer, ok, err := tx.FindLayer("RANDOM_FOREST") - assert.Nil(t, err) - assert.True(t, ok) - - expectedLayer := database.Layer{ - LayerMetadata: database.LayerMetadata{ - Hash: "RANDOM_FOREST", - ProcessedBy: processors, + }, + { + title: "layer with non-existing namespace", + name: "random-forest2", + err: "database: associated immutable entities are missing in the database", + by: []database.Detector{realDetectors[1]}, + namespaces: []database.LayerNamespace{ + {fakeNamespaces[1], realDetectors[1]}, }, - Features: append(features, nonExistingFeature...), - Namespaces: namespaces, - } - - assertLayerWithContentEqual(t, expectedLayer, layer) -} - -func TestFindLayer(t *testing.T) { - datastore, tx := openSessionForTest(t, "FindLayer", true) - defer closeTest(t, datastore, tx) - - _, _, err := tx.FindLayer("") - assert.NotNil(t, err) - _, ok, err := tx.FindLayer("layer-non") - assert.Nil(t, err) - assert.False(t, ok) - - expectedL := database.Layer{ - LayerMetadata: database.LayerMetadata{ - Hash: "layer-4", - ProcessedBy: database.Processors{ - Detectors: []string{"os-release", "apt-sources"}, - Listers: []string{"dpkg", "rpm"}, + }, + { + title: "layer with non-existing detector", + name: "random-forest3", + err: "database: associated immutable entities are missing in the database", + by: []database.Detector{fakeDetector[1]}, + }, + { + title: "valid layer", + name: "hamsterhouse", + by: []database.Detector{realDetectors[1], realDetectors[2]}, + features: []database.LayerFeature{ + {realFeatures[1], realDetectors[2]}, + {realFeatures[2], realDetectors[2]}, + }, + namespaces: []database.LayerNamespace{ + {realNamespaces[1], realDetectors[1]}, + }, + layer: &database.Layer{ + Hash: "hamsterhouse", + By: []database.Detector{realDetectors[1], realDetectors[2]}, + Features: []database.LayerFeature{ + {realFeatures[1], realDetectors[2]}, + {realFeatures[2], realDetectors[2]}, + }, + Namespaces: []database.LayerNamespace{ + {realNamespaces[1], realDetectors[1]}, }, }, - Features: []database.Feature{ - {Name: "fake", Version: "2.0", VersionFormat: "rpm"}, - {Name: "openssl", Version: "2.0", VersionFormat: "dpkg"}, + }, + { + title: "update existing layer", + name: "layer-1", + by: []database.Detector{realDetectors[3], realDetectors[4]}, + features: []database.LayerFeature{ + {realFeatures[4], realDetectors[3]}, }, - Namespaces: []database.Namespace{ - {Name: "debian:7", VersionFormat: "dpkg"}, - {Name: "fake:1.0", VersionFormat: "rpm"}, + namespaces: []database.LayerNamespace{ + {realNamespaces[3], realDetectors[4]}, }, - } + layer: &database.Layer{ + Hash: "layer-1", + By: []database.Detector{realDetectors[1], realDetectors[2], realDetectors[3], realDetectors[4]}, + Features: []database.LayerFeature{ + {realFeatures[1], realDetectors[2]}, + {realFeatures[2], realDetectors[2]}, + {realFeatures[4], realDetectors[3]}, + }, + Namespaces: []database.LayerNamespace{ + {realNamespaces[1], realDetectors[1]}, + {realNamespaces[3], realDetectors[4]}, + }, + }, + }, +} - layer, ok2, err := tx.FindLayer("layer-4") - if assert.Nil(t, err) && assert.True(t, ok2) { - assertLayerWithContentEqual(t, expectedL, layer) +func TestPersistLayer(t *testing.T) { + datastore, tx := openSessionForTest(t, "PersistLayer", true) + defer closeTest(t, datastore, tx) + + for _, test := range persistLayerTests { + t.Run(test.title, func(t *testing.T) { + err := tx.PersistLayer(test.name, test.features, test.namespaces, test.by) + if test.err != "" { + assert.EqualError(t, err, test.err, "unexpected error") + return + } + + assert.Nil(t, err) + if test.layer != nil { + layer, ok, err := tx.FindLayer(test.name) + assert.Nil(t, err) + assert.True(t, ok) + database.AssertLayerEqual(t, test.layer, &layer) + } + }) } } -func assertLayerWithContentEqual(t *testing.T, expected database.Layer, actual database.Layer) bool { - return assertLayerEqual(t, expected.LayerMetadata, actual.LayerMetadata) && - assertFeaturesEqual(t, expected.Features, actual.Features) && - assertNamespacesEqual(t, expected.Namespaces, actual.Namespaces) +var findLayerTests = []struct { + title string + in string + + out *database.Layer + err string + ok bool +}{ + { + title: "invalid layer name", + in: "", + err: "non empty layer hash is expected.", + }, + { + title: "non-existing layer", + in: "layer-non-existing", + ok: false, + out: nil, + }, + { + title: "existing layer", + in: "layer-4", + ok: true, + out: takeLayerPointerFromMap(realLayers, 6), + }, } -func assertLayerEqual(t *testing.T, expected database.LayerMetadata, actual database.LayerMetadata) bool { - return assertProcessorsEqual(t, expected.ProcessedBy, actual.ProcessedBy) && - assert.Equal(t, expected.Hash, actual.Hash) +func TestFindLayer(t *testing.T) { + datastore, tx := openSessionForTest(t, "FindLayer", true) + defer closeTest(t, datastore, tx) + + for _, test := range findLayerTests { + t.Run(test.title, func(t *testing.T) { + layer, ok, err := tx.FindLayer(test.in) + if test.err != "" { + assert.EqualError(t, err, test.err, "unexpected error") + return + } + + assert.Nil(t, err) + assert.Equal(t, test.ok, ok) + if test.ok { + database.AssertLayerEqual(t, test.out, &layer) + } + }) + } } diff --git a/database/pgsql/lock.go b/database/pgsql/lock.go index c8918ebc..b7e859e6 100644 --- a/database/pgsql/lock.go +++ b/database/pgsql/lock.go @@ -23,6 +23,14 @@ import ( "github.com/coreos/clair/pkg/commonerr" ) +const ( + soiLock = `INSERT INTO lock(name, owner, until) VALUES ($1, $2, $3)` + searchLock = `SELECT owner, until FROM Lock WHERE name = $1` + updateLock = `UPDATE Lock SET until = $3 WHERE name = $1 AND owner = $2` + removeLock = `DELETE FROM Lock WHERE name = $1 AND owner = $2` + removeLockExpired = `DELETE FROM LOCK WHERE until < CURRENT_TIMESTAMP` +) + var ( errLockNotFound = errors.New("lock is not in database") ) diff --git a/database/pgsql/migrations/00001_initial_schema.go b/database/pgsql/migrations/00001_initial_schema.go index e51037ee..c073e286 100644 --- a/database/pgsql/migrations/00001_initial_schema.go +++ b/database/pgsql/migrations/00001_initial_schema.go @@ -14,179 +14,223 @@ package migrations -import "github.com/remind101/migrate" - -func init() { - RegisterMigration(migrate.Migration{ - ID: 1, - Up: migrate.Queries([]string{ +var ( + // entities are the basic building blocks to relate the vulnerabilities with + // the ancestry. + entities = MigrationQuery{ + Up: []string{ // namespaces `CREATE TABLE IF NOT EXISTS namespace ( - id SERIAL PRIMARY KEY, - name TEXT NULL, - version_format TEXT, - UNIQUE (name, version_format));`, + id SERIAL PRIMARY KEY, + name TEXT NULL, + version_format TEXT, + UNIQUE (name, version_format));`, `CREATE INDEX ON namespace(name);`, // features `CREATE TABLE IF NOT EXISTS feature ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - version TEXT NOT NULL, - version_format TEXT NOT NULL, - UNIQUE (name, version, version_format));`, + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + version TEXT NOT NULL, + version_format TEXT NOT NULL, + UNIQUE (name, version, version_format));`, `CREATE INDEX ON feature(name);`, `CREATE TABLE IF NOT EXISTS namespaced_feature ( - id SERIAL PRIMARY KEY, - namespace_id INT REFERENCES namespace, - feature_id INT REFERENCES feature, - UNIQUE (namespace_id, feature_id));`, - + id SERIAL PRIMARY KEY, + namespace_id INT REFERENCES namespace ON DELETE CASCADE, + feature_id INT REFERENCES feature ON DELETE CASCADE, + UNIQUE (namespace_id, feature_id));`, + }, + Down: []string{ + `DROP TABLE IF EXISTS namespace, feature, namespaced_feature CASCADE;`, + }, + } + + // detector is analysis extensions used by the worker. + detector = MigrationQuery{ + Up: []string{ + // Detector Type + `CREATE TYPE detector_type AS ENUM ('namespace', 'feature');`, + + // Detector + `CREATE TABLE IF NOT EXISTS detector ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + version TEXT NOT NULL, + dtype detector_type NOT NULL, + UNIQUE (name, version, dtype));`, + }, + Down: []string{ + `DROP TABLE IF EXISTS detector CASCADE;`, + `DROP TYPE IF EXISTS detector_type;`, + }, + } + + // layer contains all metadata and scanned features and namespaces. + layer = MigrationQuery{ + Up: []string{ // layers `CREATE TABLE IF NOT EXISTS layer( - id SERIAL PRIMARY KEY, - hash TEXT NOT NULL UNIQUE);`, + id SERIAL PRIMARY KEY, + hash TEXT NOT NULL UNIQUE);`, + + `CREATE TABLE IF NOT EXISTS layer_detector( + id SERIAL PRIMARY KEY, + layer_id INT REFERENCES layer ON DELETE CASCADE, + detector_id INT REFERENCES detector ON DELETE CASCADE, + UNIQUE(layer_id, detector_id));`, + `CREATE INDEX ON layer_detector(layer_id);`, `CREATE TABLE IF NOT EXISTS layer_feature ( - id SERIAL PRIMARY KEY, - layer_id INT REFERENCES layer ON DELETE CASCADE, - feature_id INT REFERENCES feature ON DELETE CASCADE, - UNIQUE (layer_id, feature_id));`, + id SERIAL PRIMARY KEY, + layer_id INT REFERENCES layer ON DELETE CASCADE, + feature_id INT REFERENCES feature ON DELETE CASCADE, + detector_id INT REFERENCES detector ON DELETE CASCADE, + UNIQUE (layer_id, feature_id));`, `CREATE INDEX ON layer_feature(layer_id);`, - `CREATE TABLE IF NOT EXISTS layer_lister ( - id SERIAL PRIMARY KEY, - layer_id INT REFERENCES layer ON DELETE CASCADE, - lister TEXT NOT NULL, - UNIQUE (layer_id, lister));`, - `CREATE INDEX ON layer_lister(layer_id);`, - - `CREATE TABLE IF NOT EXISTS layer_detector ( - id SERIAL PRIMARY KEY, - layer_id INT REFERENCES layer ON DELETE CASCADE, - detector TEXT, - UNIQUE (layer_id, detector));`, - `CREATE INDEX ON layer_detector(layer_id);`, - `CREATE TABLE IF NOT EXISTS layer_namespace ( - id SERIAL PRIMARY KEY, - layer_id INT REFERENCES layer ON DELETE CASCADE, - namespace_id INT REFERENCES namespace ON DELETE CASCADE, - UNIQUE (layer_id, namespace_id));`, + id SERIAL PRIMARY KEY, + layer_id INT REFERENCES layer ON DELETE CASCADE, + namespace_id INT REFERENCES namespace ON DELETE CASCADE, + detector_id INT REFERENCES detector ON DELETE CASCADE, + UNIQUE (layer_id, namespace_id));`, `CREATE INDEX ON layer_namespace(layer_id);`, - + }, + Down: []string{ + `DROP TABLE IF EXISTS layer, layer_detector, layer_feature, layer_namespace CASCADE;`, + }, + } + + // ancestry contains all meta information around scanned manifest and its + // layers. + ancestry = MigrationQuery{ + Up: []string{ // ancestry `CREATE TABLE IF NOT EXISTS ancestry ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL UNIQUE);`, + id SERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE);`, `CREATE TABLE IF NOT EXISTS ancestry_layer ( - id SERIAL PRIMARY KEY, - ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, - ancestry_index INT NOT NULL, - layer_id INT REFERENCES layer ON DELETE RESTRICT, - UNIQUE (ancestry_id, ancestry_index));`, + id SERIAL PRIMARY KEY, + ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, + ancestry_index INT NOT NULL, + layer_id INT NOT NULL REFERENCES layer ON DELETE RESTRICT, + UNIQUE (ancestry_id, ancestry_index));`, `CREATE INDEX ON ancestry_layer(ancestry_id);`, `CREATE TABLE IF NOT EXISTS ancestry_feature( - id SERIAL PRIMARY KEY, - ancestry_layer_id INT REFERENCES ancestry_layer ON DELETE CASCADE, - namespaced_feature_id INT REFERENCES namespaced_feature ON DELETE CASCADE, - UNIQUE (ancestry_layer_id, namespaced_feature_id));`, - - `CREATE TABLE IF NOT EXISTS ancestry_lister ( - id SERIAL PRIMARY KEY, - ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, - lister TEXT, - UNIQUE (ancestry_id, lister));`, - `CREATE INDEX ON ancestry_lister(ancestry_id);`, - - `CREATE TABLE IF NOT EXISTS ancestry_detector ( - id SERIAL PRIMARY KEY, - ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, - detector TEXT, - UNIQUE (ancestry_id, detector));`, + id SERIAL PRIMARY KEY, + ancestry_layer_id INT REFERENCES ancestry_layer ON DELETE CASCADE, + namespaced_feature_id INT REFERENCES namespaced_feature ON DELETE CASCADE, + feature_detector_id INT REFERENCES detector ON DELETE CASCADE, + namespace_detector_id INT REFERENCES detector ON DELETE CASCADE, + UNIQUE (ancestry_layer_id, namespaced_feature_id));`, + + `CREATE TABLE IF NOT EXISTS ancestry_detector( + id SERIAL PRIMARY KEY, + ancestry_id INT REFERENCES ancestry ON DELETE CASCADE, + detector_id INT REFERENCES detector ON DELETE CASCADE, + UNIQUE(ancestry_id, detector_id));`, `CREATE INDEX ON ancestry_detector(ancestry_id);`, - + }, + Down: []string{ + `DROP TABLE IF EXISTS ancestry, ancestry_layer, ancestry_feature, ancestry_detector CASCADE;`, + }, + } + + // vulnerability contains the metadata and vulnerability affecting relation. + vulnerability = MigrationQuery{ + Up: []string{ `CREATE TYPE severity AS ENUM ('Unknown', 'Negligible', 'Low', 'Medium', 'High', 'Critical', 'Defcon1');`, // vulnerability `CREATE TABLE IF NOT EXISTS vulnerability ( - id SERIAL PRIMARY KEY, - namespace_id INT NOT NULL REFERENCES Namespace, - name TEXT NOT NULL, - description TEXT NULL, - link TEXT NULL, - severity severity NOT NULL, - metadata TEXT NULL, - created_at TIMESTAMP WITH TIME ZONE, - deleted_at TIMESTAMP WITH TIME ZONE NULL);`, + id SERIAL PRIMARY KEY, + namespace_id INT REFERENCES Namespace, + name TEXT NOT NULL, + description TEXT NULL, + link TEXT NULL, + severity severity NOT NULL, + metadata TEXT NULL, + created_at TIMESTAMP WITH TIME ZONE, + deleted_at TIMESTAMP WITH TIME ZONE NULL);`, `CREATE INDEX ON vulnerability(namespace_id, name);`, `CREATE INDEX ON vulnerability(namespace_id);`, `CREATE TABLE IF NOT EXISTS vulnerability_affected_feature ( - id SERIAL PRIMARY KEY, - vulnerability_id INT NOT NULL REFERENCES vulnerability ON DELETE CASCADE, - feature_name TEXT NOT NULL, - affected_version TEXT, - fixedin TEXT);`, + id SERIAL PRIMARY KEY, + vulnerability_id INT REFERENCES vulnerability ON DELETE CASCADE, + feature_name TEXT NOT NULL, + affected_version TEXT, + fixedin TEXT);`, `CREATE INDEX ON vulnerability_affected_feature(vulnerability_id, feature_name);`, `CREATE TABLE IF NOT EXISTS vulnerability_affected_namespaced_feature( - id SERIAL PRIMARY KEY, - vulnerability_id INT NOT NULL REFERENCES vulnerability ON DELETE CASCADE, - namespaced_feature_id INT NOT NULL REFERENCES namespaced_feature ON DELETE CASCADE, - added_by INT NOT NULL REFERENCES vulnerability_affected_feature ON DELETE CASCADE, - UNIQUE (vulnerability_id, namespaced_feature_id));`, + id SERIAL PRIMARY KEY, + vulnerability_id INT REFERENCES vulnerability ON DELETE CASCADE, + namespaced_feature_id INT REFERENCES namespaced_feature ON DELETE CASCADE, + added_by INT REFERENCES vulnerability_affected_feature ON DELETE CASCADE, + UNIQUE (vulnerability_id, namespaced_feature_id));`, `CREATE INDEX ON vulnerability_affected_namespaced_feature(namespaced_feature_id);`, - + }, + Down: []string{ + `DROP TYPE IF EXISTS severity;`, + `DROP TABLE IF EXISTS vulnerability, vulnerability_affected_feature, vulnerability_affected_namespaced_feature CASCADE;`, + }, + } + + // updaterLock is the lock to be used by updater to prevent multiple + // updaters running on the same vulnerability source. + updaterLock = MigrationQuery{ + Up: []string{ `CREATE TABLE IF NOT EXISTS KeyValue ( - id SERIAL PRIMARY KEY, - key TEXT NOT NULL UNIQUE, - value TEXT);`, + id SERIAL PRIMARY KEY, + key TEXT NOT NULL UNIQUE, + value TEXT);`, `CREATE TABLE IF NOT EXISTS Lock ( - id SERIAL PRIMARY KEY, - name VARCHAR(64) NOT NULL UNIQUE, - owner VARCHAR(64) NOT NULL, - until TIMESTAMP WITH TIME ZONE);`, + id SERIAL PRIMARY KEY, + name VARCHAR(64) NOT NULL UNIQUE, + owner VARCHAR(64) NOT NULL, + until TIMESTAMP WITH TIME ZONE);`, `CREATE INDEX ON Lock (owner);`, - - // Notification + }, + Down: []string{ + `DROP TABLE IF EXISTS KeyValue, Lock CASCADE;`, + }, + } + + // notification is the vulnerability notification spawned by the + // vulnerability changes. + notification = MigrationQuery{ + Up: []string{ `CREATE TABLE IF NOT EXISTS Vulnerability_Notification ( - id SERIAL PRIMARY KEY, - name VARCHAR(64) NOT NULL UNIQUE, - created_at TIMESTAMP WITH TIME ZONE, - notified_at TIMESTAMP WITH TIME ZONE NULL, - deleted_at TIMESTAMP WITH TIME ZONE NULL, - old_vulnerability_id INT NULL REFERENCES Vulnerability ON DELETE CASCADE, - new_vulnerability_id INT NULL REFERENCES Vulnerability ON DELETE CASCADE);`, + id SERIAL PRIMARY KEY, + name VARCHAR(64) NOT NULL UNIQUE, + created_at TIMESTAMP WITH TIME ZONE, + notified_at TIMESTAMP WITH TIME ZONE NULL, + deleted_at TIMESTAMP WITH TIME ZONE NULL, + old_vulnerability_id INT NULL REFERENCES Vulnerability ON DELETE CASCADE, + new_vulnerability_id INT NULL REFERENCES Vulnerability ON DELETE CASCADE);`, `CREATE INDEX ON Vulnerability_Notification (notified_at);`, - }), - Down: migrate.Queries([]string{ - `DROP TABLE IF EXISTS - ancestry, - ancestry_layer, - ancestry_detector, - ancestry_lister, - ancestry_feature, - feature, - namespaced_feature, - keyvalue, - layer, - layer_detector, - layer_feature, - layer_lister, - layer_namespace, - lock, - namespace, - vulnerability, - vulnerability_affected_feature, - vulnerability_affected_namespaced_feature, - vulnerability_notification - CASCADE;`, - `DROP TYPE IF EXISTS severity;`, - }), - }) + }, + Down: []string{ + `DROP TABLE IF EXISTS Vulnerability_Notification CASCADE;`, + }, + } +) + +func init() { + RegisterMigration(NewSimpleMigration(1, + []MigrationQuery{ + entities, + detector, + layer, + ancestry, + vulnerability, + updaterLock, + notification, + })) } diff --git a/database/pgsql/migrations/util.go b/database/pgsql/migrations/util.go new file mode 100644 index 00000000..8b8898e0 --- /dev/null +++ b/database/pgsql/migrations/util.go @@ -0,0 +1,30 @@ +package migrations + +import "github.com/remind101/migrate" + +// MigrationQuery contains the Up migration and Down migration in Plain strings. +type MigrationQuery struct { + Up []string + Down []string +} + +// ConcatMigrationQueries concats migration queries in the give order. +func ConcatMigrationQueries(qs []MigrationQuery) MigrationQuery { + r := MigrationQuery{} + for _, q := range qs { + r.Up = append(r.Up, q.Up...) + r.Down = append(r.Down, q.Down...) + } + return r +} + +// NewSimpleMigration returns a simple migration plan with all provided +// migration queries concatted in order. +func NewSimpleMigration(id int, qs []MigrationQuery) migrate.Migration { + q := ConcatMigrationQueries(qs) + return migrate.Migration{ + ID: id, + Up: migrate.Queries(q.Up), + Down: migrate.Queries(q.Down), + } +} diff --git a/database/pgsql/namespace.go b/database/pgsql/namespace.go index 1a78f837..87d25e33 100644 --- a/database/pgsql/namespace.go +++ b/database/pgsql/namespace.go @@ -16,15 +16,14 @@ package pgsql import ( "database/sql" - "errors" "sort" "github.com/coreos/clair/database" "github.com/coreos/clair/pkg/commonerr" ) -var ( - errNamespaceNotFound = errors.New("Requested Namespace is not in database") +const ( + searchNamespaceID = `SELECT id FROM Namespace WHERE name = $1 AND version_format = $2` ) // PersistNamespaces soi namespaces into database. diff --git a/database/pgsql/namespace_test.go b/database/pgsql/namespace_test.go index 27ceefef..8f2af288 100644 --- a/database/pgsql/namespace_test.go +++ b/database/pgsql/namespace_test.go @@ -42,42 +42,3 @@ func TestPersistNamespaces(t *testing.T) { assert.Len(t, nsList, 1) assert.Equal(t, ns2, nsList[0]) } - -func assertNamespacesEqual(t *testing.T, expected []database.Namespace, actual []database.Namespace) bool { - if assert.Len(t, actual, len(expected)) { - has := map[database.Namespace]bool{} - for _, i := range expected { - has[i] = false - } - for _, i := range actual { - has[i] = true - } - for key, v := range has { - if !assert.True(t, v, key.Name+"is expected") { - return false - } - } - return true - } - return false -} - -func listNamespaces(t *testing.T, tx *pgSession) []database.Namespace { - rows, err := tx.Query("SELECT name, version_format FROM namespace") - if err != nil { - t.FailNow() - } - defer rows.Close() - - namespaces := []database.Namespace{} - for rows.Next() { - var ns database.Namespace - err := rows.Scan(&ns.Name, &ns.VersionFormat) - if err != nil { - t.FailNow() - } - namespaces = append(namespaces, ns) - } - - return namespaces -} diff --git a/database/pgsql/notification.go b/database/pgsql/notification.go index 4ddf042f..7d2b750d 100644 --- a/database/pgsql/notification.go +++ b/database/pgsql/notification.go @@ -26,6 +26,49 @@ import ( "github.com/coreos/clair/pkg/pagination" ) +const ( + insertNotification = ` + INSERT INTO Vulnerability_Notification(name, created_at, old_vulnerability_id, new_vulnerability_id) + VALUES ($1, $2, $3, $4)` + + updatedNotificationAsRead = ` + UPDATE Vulnerability_Notification + SET notified_at = CURRENT_TIMESTAMP + WHERE name = $1` + + removeNotification = ` + UPDATE Vulnerability_Notification + SET deleted_at = CURRENT_TIMESTAMP + WHERE name = $1 AND deleted_at IS NULL` + + searchNotificationAvailable = ` + SELECT name, created_at, notified_at, deleted_at + FROM Vulnerability_Notification + WHERE (notified_at IS NULL OR notified_at < $1) + AND deleted_at IS NULL + AND name NOT IN (SELECT name FROM Lock) + ORDER BY Random() + LIMIT 1` + + searchNotification = ` + SELECT created_at, notified_at, deleted_at, old_vulnerability_id, new_vulnerability_id + FROM Vulnerability_Notification + WHERE name = $1` + + searchNotificationVulnerableAncestry = ` + SELECT DISTINCT ON (a.id) + a.id, a.name + FROM vulnerability_affected_namespaced_feature AS vanf, + ancestry_layer AS al, ancestry_feature AS af, ancestry AS a + WHERE vanf.vulnerability_id = $1 + AND a.id >= $2 + AND al.ancestry_id = a.id + AND al.id = af.ancestry_layer_id + AND af.namespaced_feature_id = vanf.namespaced_feature_id + ORDER BY a.id ASC + LIMIT $3;` +) + var ( errNotificationNotFound = errors.New("requested notification is not found") ) @@ -168,14 +211,12 @@ func (tx *pgSession) findPagedVulnerableAncestries(vulnID int64, limit int, curr vulnPage := database.PagedVulnerableAncestries{Limit: limit} currentPage := Page{0} if currentToken != pagination.FirstPageToken { - var err error - err = tx.key.UnmarshalToken(currentToken, ¤tPage) - if err != nil { + if err := tx.key.UnmarshalToken(currentToken, ¤tPage); err != nil { return vulnPage, err } } - err := tx.QueryRow(searchVulnerabilityByID, vulnID).Scan( + if err := tx.QueryRow(searchVulnerabilityByID, vulnID).Scan( &vulnPage.Name, &vulnPage.Description, &vulnPage.Link, @@ -183,8 +224,7 @@ func (tx *pgSession) findPagedVulnerableAncestries(vulnID int64, limit int, curr &vulnPage.Metadata, &vulnPage.Namespace.Name, &vulnPage.Namespace.VersionFormat, - ) - if err != nil { + ); err != nil { return vulnPage, handleError("searchVulnerabilityByID", err) } @@ -247,7 +287,6 @@ func (tx *pgSession) FindVulnerabilityNotification(name string, limit int, oldPa } noti.Name = name - err := tx.QueryRow(searchNotification, name).Scan(&created, ¬ified, &deleted, &oldVulnID, &newVulnID) diff --git a/database/pgsql/notification_test.go b/database/pgsql/notification_test.go index 9d36f4cb..0a23abca 100644 --- a/database/pgsql/notification_test.go +++ b/database/pgsql/notification_test.go @@ -19,121 +19,144 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/pagination" ) -func TestPagination(t *testing.T) { - datastore, tx := openSessionForTest(t, "Pagination", true) - defer closeTest(t, datastore, tx) - - ns := database.Namespace{ - Name: "debian:7", - VersionFormat: "dpkg", - } - - vNew := database.Vulnerability{ - Namespace: ns, - Name: "CVE-OPENSSL-1-DEB7", - Description: "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", - Link: "http://google.com/#q=CVE-OPENSSL-1-DEB7", - Severity: database.HighSeverity, - } - - vOld := database.Vulnerability{ - Namespace: ns, - Name: "CVE-NOPE", - Description: "A vulnerability affecting nothing", - Severity: database.UnknownSeverity, - } - - noti, ok, err := tx.FindVulnerabilityNotification("test", 1, "", "") - oldPage := database.PagedVulnerableAncestries{ - Vulnerability: vOld, - Limit: 1, - Affected: make(map[int]string), - End: true, - } - - newPage1 := database.PagedVulnerableAncestries{ - Vulnerability: vNew, - Limit: 1, - Affected: map[int]string{3: "ancestry-3"}, - End: false, - } +type findVulnerabilityNotificationIn struct { + notificationName string + pageSize int + oldAffectedAncestryPage pagination.Token + newAffectedAncestryPage pagination.Token +} - newPage2 := database.PagedVulnerableAncestries{ - Vulnerability: vNew, - Limit: 1, - Affected: map[int]string{4: "ancestry-4"}, - Next: "", - End: true, - } +type findVulnerabilityNotificationOut struct { + notification *database.VulnerabilityNotificationWithVulnerable + ok bool + err string +} - if assert.Nil(t, err) && assert.True(t, ok) { - assert.Equal(t, "test", noti.Name) - if assert.NotNil(t, noti.Old) && assert.NotNil(t, noti.New) { - var oldPage Page - err := tx.key.UnmarshalToken(noti.Old.Current, &oldPage) - if !assert.Nil(t, err) { - assert.FailNow(t, "") - } +var findVulnerabilityNotificationTests = []struct { + title string + in findVulnerabilityNotificationIn + out findVulnerabilityNotificationOut +}{ + { + title: "find notification with invalid page", + in: findVulnerabilityNotificationIn{ + notificationName: "test", + pageSize: 1, + oldAffectedAncestryPage: pagination.FirstPageToken, + newAffectedAncestryPage: pagination.Token("random non sense"), + }, + out: findVulnerabilityNotificationOut{ + err: pagination.ErrInvalidToken.Error(), + }, + }, + { + title: "find non-existing notification", + in: findVulnerabilityNotificationIn{ + notificationName: "non-existing", + pageSize: 1, + oldAffectedAncestryPage: pagination.FirstPageToken, + newAffectedAncestryPage: pagination.FirstPageToken, + }, + out: findVulnerabilityNotificationOut{ + ok: false, + }, + }, + { + title: "find existing notification first page", + in: findVulnerabilityNotificationIn{ + notificationName: "test", + pageSize: 1, + oldAffectedAncestryPage: pagination.FirstPageToken, + newAffectedAncestryPage: pagination.FirstPageToken, + }, + out: findVulnerabilityNotificationOut{ + &database.VulnerabilityNotificationWithVulnerable{ + NotificationHook: realNotification[1].NotificationHook, + Old: &database.PagedVulnerableAncestries{ + Vulnerability: realVulnerability[2], + Limit: 1, + Affected: make(map[int]string), + Current: mustMarshalToken(testPaginationKey, Page{0}), + Next: mustMarshalToken(testPaginationKey, Page{0}), + End: true, + }, + New: &database.PagedVulnerableAncestries{ + Vulnerability: realVulnerability[1], + Limit: 1, + Affected: map[int]string{3: "ancestry-3"}, + Current: mustMarshalToken(testPaginationKey, Page{0}), + Next: mustMarshalToken(testPaginationKey, Page{4}), + End: false, + }, + }, - assert.Equal(t, int64(0), oldPage.StartID) - var newPage Page - err = tx.key.UnmarshalToken(noti.New.Current, &newPage) - if !assert.Nil(t, err) { - assert.FailNow(t, "") - } - var newPageNext Page - err = tx.key.UnmarshalToken(noti.New.Next, &newPageNext) - if !assert.Nil(t, err) { - assert.FailNow(t, "") - } - assert.Equal(t, int64(0), newPage.StartID) - assert.Equal(t, int64(4), newPageNext.StartID) - - noti.Old.Current = "" - noti.New.Current = "" - noti.New.Next = "" - assert.Equal(t, oldPage, *noti.Old) - assert.Equal(t, newPage1, *noti.New) - } - } + true, + "", + }, + }, + + { + title: "find existing notification of second page of new affected ancestry", + in: findVulnerabilityNotificationIn{ + notificationName: "test", + pageSize: 1, + oldAffectedAncestryPage: pagination.FirstPageToken, + newAffectedAncestryPage: mustMarshalToken(testPaginationKey, Page{4}), + }, + out: findVulnerabilityNotificationOut{ + &database.VulnerabilityNotificationWithVulnerable{ + NotificationHook: realNotification[1].NotificationHook, + Old: &database.PagedVulnerableAncestries{ + Vulnerability: realVulnerability[2], + Limit: 1, + Affected: make(map[int]string), + Current: mustMarshalToken(testPaginationKey, Page{0}), + Next: mustMarshalToken(testPaginationKey, Page{0}), + End: true, + }, + New: &database.PagedVulnerableAncestries{ + Vulnerability: realVulnerability[1], + Limit: 1, + Affected: map[int]string{4: "ancestry-4"}, + Current: mustMarshalToken(testPaginationKey, Page{4}), + Next: mustMarshalToken(testPaginationKey, Page{0}), + End: true, + }, + }, - pageNum1, err := tx.key.MarshalToken(Page{0}) - if !assert.Nil(t, err) { - assert.FailNow(t, "") - } + true, + "", + }, + }, +} - pageNum2, err := tx.key.MarshalToken(Page{4}) - if !assert.Nil(t, err) { - assert.FailNow(t, "") - } +func TestFindVulnerabilityNotification(t *testing.T) { + datastore, tx := openSessionForTest(t, "pagination", true) + defer closeTest(t, datastore, tx) - noti, ok, err = tx.FindVulnerabilityNotification("test", 1, pageNum1, pageNum2) - if assert.Nil(t, err) && assert.True(t, ok) { - assert.Equal(t, "test", noti.Name) - if assert.NotNil(t, noti.Old) && assert.NotNil(t, noti.New) { - var oldCurrentPage Page - err = tx.key.UnmarshalToken(noti.Old.Current, &oldCurrentPage) - if !assert.Nil(t, err) { - assert.FailNow(t, "") + for _, test := range findVulnerabilityNotificationTests { + t.Run(test.title, func(t *testing.T) { + notification, ok, err := tx.FindVulnerabilityNotification(test.in.notificationName, test.in.pageSize, test.in.oldAffectedAncestryPage, test.in.newAffectedAncestryPage) + if test.out.err != "" { + require.EqualError(t, err, test.out.err) + return } - var newCurrentPage Page - err = tx.key.UnmarshalToken(noti.New.Current, &newCurrentPage) - if !assert.Nil(t, err) { - assert.FailNow(t, "") + require.Nil(t, err) + if !test.out.ok { + require.Equal(t, test.out.ok, ok) + return } - assert.Equal(t, int64(0), oldCurrentPage.StartID) - assert.Equal(t, int64(4), newCurrentPage.StartID) - noti.Old.Current = "" - noti.New.Current = "" - assert.Equal(t, oldPage, *noti.Old) - assert.Equal(t, newPage2, *noti.New) - } + require.True(t, ok) + assertVulnerabilityNotificationWithVulnerableEqual(t, testPaginationKey, test.out.notification, ¬ification) + }) } } diff --git a/database/pgsql/pgsql.go b/database/pgsql/pgsql.go index 9af010b6..4b23e014 100644 --- a/database/pgsql/pgsql.go +++ b/database/pgsql/pgsql.go @@ -270,6 +270,7 @@ func migrateDatabase(db *sql.DB) error { // createDatabase creates a new database. // The source parameter should not contain a dbname. func createDatabase(source, dbName string) error { + log.WithFields(log.Fields{"source": source, "dbName": dbName}).Debug("creating database...") // Open database. db, err := sql.Open("postgres", source) if err != nil { @@ -325,7 +326,7 @@ func handleError(desc string, err error) error { return commonerr.ErrNotFound } - log.WithError(err).WithField("Description", desc).Error("Handled Database Error") + log.WithError(err).WithField("Description", desc).Error("database: handled database error") promErrorsTotal.WithLabelValues(desc).Inc() if _, o := err.(*pq.Error); o || err == sql.ErrTxDone || strings.HasPrefix(err.Error(), "sql:") { diff --git a/database/pgsql/pgsql_test.go b/database/pgsql/pgsql_test.go index e4a8c8b4..863445a5 100644 --- a/database/pgsql/pgsql_test.go +++ b/database/pgsql/pgsql_test.go @@ -37,6 +37,8 @@ var ( withFixtureName, withoutFixtureName string ) +var testPaginationKey = pagination.Must(pagination.NewKey()) + func genTemplateDatabase(name string, loadFixture bool) (sourceURL string, dbName string) { config := generateTestConfig(name, loadFixture, false) source := config.Options["source"].(string) @@ -215,13 +217,15 @@ func generateTestConfig(testName string, loadFixture bool, manageLife bool) data source = fmt.Sprintf(sourceEnv, dbName) } + log.Infof("pagination key for current test: %s", testPaginationKey.String()) + return database.RegistrableComponentConfig{ Options: map[string]interface{}{ "source": source, "cachesize": 0, "managedatabaselifecycle": manageLife, "fixturepath": fixturePath, - "paginationkey": pagination.Must(pagination.NewKey()).String(), + "paginationkey": testPaginationKey.String(), }, } } @@ -247,6 +251,8 @@ func openSessionForTest(t *testing.T, name string, loadFixture bool) (*pgSQL, *p t.Error(err) t.FailNow() } + + log.Infof("transaction pagination key: '%s'", tx.(*pgSession).key.String()) return store, tx.(*pgSession) } diff --git a/database/pgsql/queries.go b/database/pgsql/queries.go index fa137620..2d4b7e99 100644 --- a/database/pgsql/queries.go +++ b/database/pgsql/queries.go @@ -21,235 +21,6 @@ import ( "github.com/lib/pq" ) -const ( - lockVulnerabilityAffects = `LOCK vulnerability_affected_namespaced_feature IN SHARE ROW EXCLUSIVE MODE` - - // keyvalue.go - searchKeyValue = `SELECT value FROM KeyValue WHERE key = $1` - upsertKeyValue = ` - INSERT INTO KeyValue(key, value) - VALUES ($1, $2) - ON CONFLICT ON CONSTRAINT keyvalue_key_key - DO UPDATE SET key=$1, value=$2` - - // namespace.go - - searchNamespaceID = `SELECT id FROM Namespace WHERE name = $1 AND version_format = $2` - - // feature.go - soiNamespacedFeature = ` - WITH new_feature_ns AS ( - INSERT INTO namespaced_feature(feature_id, namespace_id) - SELECT CAST ($1 AS INTEGER), CAST ($2 AS INTEGER) - WHERE NOT EXISTS ( SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2) - RETURNING id - ) - SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2 - UNION - SELECT id FROM new_feature_ns` - - searchPotentialAffectingVulneraibilities = ` - SELECT nf.id, v.id, vaf.affected_version, vaf.id - FROM vulnerability_affected_feature AS vaf, vulnerability AS v, - namespaced_feature AS nf, feature AS f - WHERE nf.id = ANY($1) - AND nf.feature_id = f.id - AND nf.namespace_id = v.namespace_id - AND vaf.feature_name = f.name - AND vaf.vulnerability_id = v.id - AND v.deleted_at IS NULL` - - searchNamespacedFeaturesVulnerabilities = ` - SELECT vanf.namespaced_feature_id, v.name, v.description, v.link, - v.severity, v.metadata, vaf.fixedin, n.name, n.version_format - FROM vulnerability_affected_namespaced_feature AS vanf, - Vulnerability AS v, - vulnerability_affected_feature AS vaf, - namespace AS n - WHERE vanf.namespaced_feature_id = ANY($1) - AND vaf.id = vanf.added_by - AND v.id = vanf.vulnerability_id - AND n.id = v.namespace_id - AND v.deleted_at IS NULL` - - // layer.go - soiLayer = ` - WITH new_layer AS ( - INSERT INTO layer (hash) - SELECT CAST ($1 AS VARCHAR) - WHERE NOT EXISTS (SELECT id FROM layer WHERE hash = $1) - RETURNING id - ) - SELECT id FROM new_Layer - UNION - SELECT id FROM layer WHERE hash = $1` - - searchLayerFeatures = ` - SELECT feature.Name, feature.Version, feature.version_format - FROM feature, layer_feature - WHERE layer_feature.layer_id = $1 - AND layer_feature.feature_id = feature.id` - - searchLayerNamespaces = ` - SELECT namespace.Name, namespace.version_format - FROM namespace, layer_namespace - WHERE layer_namespace.layer_id = $1 - AND layer_namespace.namespace_id = namespace.id` - - searchLayer = `SELECT id FROM layer WHERE hash = $1` - searchLayerDetectors = `SELECT detector FROM layer_detector WHERE layer_id = $1` - searchLayerListers = `SELECT lister FROM layer_lister WHERE layer_id = $1` - - // lock.go - soiLock = `INSERT INTO lock(name, owner, until) VALUES ($1, $2, $3)` - - searchLock = `SELECT owner, until FROM Lock WHERE name = $1` - updateLock = `UPDATE Lock SET until = $3 WHERE name = $1 AND owner = $2` - removeLock = `DELETE FROM Lock WHERE name = $1 AND owner = $2` - removeLockExpired = `DELETE FROM LOCK WHERE until < CURRENT_TIMESTAMP` - - // vulnerability.go - searchVulnerability = ` - SELECT v.id, v.description, v.link, v.severity, v.metadata, n.version_format - FROM vulnerability AS v, namespace AS n - WHERE v.namespace_id = n.id - AND v.name = $1 - AND n.name = $2 - AND v.deleted_at IS NULL - ` - - insertVulnerabilityAffected = ` - INSERT INTO vulnerability_affected_feature(vulnerability_id, feature_name, affected_version, fixedin) - VALUES ($1, $2, $3, $4) - RETURNING ID - ` - - searchVulnerabilityAffected = ` - SELECT vulnerability_id, feature_name, affected_version, fixedin - FROM vulnerability_affected_feature - WHERE vulnerability_id = ANY($1) - ` - - searchVulnerabilityByID = ` - SELECT v.name, v.description, v.link, v.severity, v.metadata, n.name, n.version_format - FROM vulnerability AS v, namespace AS n - WHERE v.namespace_id = n.id - AND v.id = $1` - - searchVulnerabilityPotentialAffected = ` - WITH req AS ( - SELECT vaf.id AS vaf_id, n.id AS n_id, vaf.feature_name AS name, v.id AS vulnerability_id - FROM vulnerability_affected_feature AS vaf, - vulnerability AS v, - namespace AS n - WHERE vaf.vulnerability_id = ANY($1) - AND v.id = vaf.vulnerability_id - AND n.id = v.namespace_id - ) - SELECT req.vulnerability_id, nf.id, f.version, req.vaf_id AS added_by - FROM feature AS f, namespaced_feature AS nf, req - WHERE f.name = req.name - AND nf.namespace_id = req.n_id - AND nf.feature_id = f.id` - - insertVulnerabilityAffectedNamespacedFeature = ` - INSERT INTO vulnerability_affected_namespaced_feature(vulnerability_id, namespaced_feature_id, added_by) - VALUES ($1, $2, $3)` - - insertVulnerability = ` - WITH ns AS ( - SELECT id FROM namespace WHERE name = $6 AND version_format = $7 - ) - INSERT INTO Vulnerability(namespace_id, name, description, link, severity, metadata, created_at) - VALUES((SELECT id FROM ns), $1, $2, $3, $4, $5, CURRENT_TIMESTAMP) - RETURNING id` - - removeVulnerability = ` - UPDATE Vulnerability - SET deleted_at = CURRENT_TIMESTAMP - WHERE namespace_id = (SELECT id FROM Namespace WHERE name = $1) - AND name = $2 - AND deleted_at IS NULL - RETURNING id` - - // notification.go - insertNotification = ` - INSERT INTO Vulnerability_Notification(name, created_at, old_vulnerability_id, new_vulnerability_id) - VALUES ($1, $2, $3, $4)` - - updatedNotificationAsRead = ` - UPDATE Vulnerability_Notification - SET notified_at = CURRENT_TIMESTAMP - WHERE name = $1` - - removeNotification = ` - UPDATE Vulnerability_Notification - SET deleted_at = CURRENT_TIMESTAMP - WHERE name = $1 AND deleted_at IS NULL` - - searchNotificationAvailable = ` - SELECT name, created_at, notified_at, deleted_at - FROM Vulnerability_Notification - WHERE (notified_at IS NULL OR notified_at < $1) - AND deleted_at IS NULL - AND name NOT IN (SELECT name FROM Lock) - ORDER BY Random() - LIMIT 1` - - searchNotification = ` - SELECT created_at, notified_at, deleted_at, old_vulnerability_id, new_vulnerability_id - FROM Vulnerability_Notification - WHERE name = $1` - - searchNotificationVulnerableAncestry = ` - SELECT DISTINCT ON (a.id) - a.id, a.name - FROM vulnerability_affected_namespaced_feature AS vanf, - ancestry_layer AS al, ancestry_feature AS af - WHERE vanf.vulnerability_id = $1 - AND al.ancestry_id >= $2 - AND al.id = af.ancestry_layer_id - AND af.namespaced_feature_id = vanf.namespaced_feature_id - ORDER BY a.id ASC - LIMIT $3;` - - // ancestry.go - persistAncestryLister = ` - INSERT INTO ancestry_lister (ancestry_id, lister) - SELECT CAST ($1 AS INTEGER), CAST ($2 AS TEXT) - WHERE NOT EXISTS (SELECT id FROM ancestry_lister WHERE ancestry_id = $1 AND lister = $2) ON CONFLICT DO NOTHING` - - persistAncestryDetector = ` - INSERT INTO ancestry_detector (ancestry_id, detector) - SELECT CAST ($1 AS INTEGER), CAST ($2 AS TEXT) - WHERE NOT EXISTS (SELECT id FROM ancestry_detector WHERE ancestry_id = $1 AND detector = $2) ON CONFLICT DO NOTHING` - - insertAncestry = `INSERT INTO ancestry (name) VALUES ($1) RETURNING id` - - searchAncestryLayer = ` - SELECT layer.hash, layer.id, ancestry_layer.ancestry_index - FROM layer, ancestry_layer - WHERE ancestry_layer.ancestry_id = $1 - AND ancestry_layer.layer_id = layer.id - ORDER BY ancestry_layer.ancestry_index ASC` - - searchAncestryFeatures = ` - SELECT namespace.name, namespace.version_format, feature.name, feature.version, feature.version_format, ancestry_layer.ancestry_index - FROM namespace, feature, namespaced_feature, ancestry_layer, ancestry_feature - WHERE ancestry_layer.ancestry_id = $1 - AND ancestry_feature.ancestry_layer_id = ancestry_layer.id - AND ancestry_feature.namespaced_feature_id = namespaced_feature.id - AND namespaced_feature.feature_id = feature.id - AND namespaced_feature.namespace_id = namespace.id` - - searchAncestry = `SELECT id FROM ancestry WHERE name = $1` - searchAncestryDetectors = `SELECT detector FROM ancestry_detector WHERE ancestry_id = $1` - searchAncestryListers = `SELECT lister FROM ancestry_lister WHERE ancestry_id = $1` - removeAncestry = `DELETE FROM ancestry WHERE name = $1` - insertAncestryLayer = `INSERT INTO ancestry_layer(ancestry_id, ancestry_index, layer_id) VALUES($1,$2, (SELECT layer.id FROM layer WHERE hash = $3 LIMIT 1)) RETURNING id` - insertAncestryLayerFeature = `INSERT INTO ancestry_feature(ancestry_layer_id, namespaced_feature_id) VALUES ($1, $2)` -) - // NOTE(Sida): Every search query can only have count less than postgres set // stack depth. IN will be resolved to nested OR_s and the parser might exceed // stack depth. TODO(Sida): Generate different queries for different count: if @@ -350,7 +121,8 @@ func queryPersistLayerFeature(count int) string { "layer_feature", "layer_feature_layer_id_feature_id_key", "layer_id", - "feature_id") + "feature_id", + "detector_id") } func queryPersistNamespace(count int) string { @@ -361,28 +133,13 @@ func queryPersistNamespace(count int) string { "version_format") } -func queryPersistLayerListers(count int) string { - return queryPersist(count, - "layer_lister", - "layer_lister_layer_id_lister_key", - "layer_id", - "lister") -} - -func queryPersistLayerDetectors(count int) string { - return queryPersist(count, - "layer_detector", - "layer_detector_layer_id_detector_key", - "layer_id", - "detector") -} - func queryPersistLayerNamespace(count int) string { return queryPersist(count, "layer_namespace", "layer_namespace_layer_id_namespace_id_key", "layer_id", - "namespace_id") + "namespace_id", + "detector_id") } // size of key and array should be both greater than 0 diff --git a/database/pgsql/testdata/data.sql b/database/pgsql/testdata/data.sql index 9d8e0323..e7484209 100644 --- a/database/pgsql/testdata/data.sql +++ b/database/pgsql/testdata/data.sql @@ -1,57 +1,69 @@ +-- initialize entities INSERT INTO namespace (id, name, version_format) VALUES -(1, 'debian:7', 'dpkg'), -(2, 'debian:8', 'dpkg'), -(3, 'fake:1.0', 'rpm'); + (1, 'debian:7', 'dpkg'), + (2, 'debian:8', 'dpkg'), + (3, 'fake:1.0', 'rpm'); INSERT INTO feature (id, name, version, version_format) VALUES -(1, 'wechat', '0.5', 'dpkg'), -(2, 'openssl', '1.0', 'dpkg'), -(3, 'openssl', '2.0', 'dpkg'), -(4, 'fake', '2.0', 'rpm'); + (1, 'ourchat', '0.5', 'dpkg'), + (2, 'openssl', '1.0', 'dpkg'), + (3, 'openssl', '2.0', 'dpkg'), + (4, 'fake', '2.0', 'rpm'); +INSERT INTO namespaced_feature(id, feature_id, namespace_id) VALUES + (1, 1, 1), -- ourchat 0.5, debian:7 + (2, 2, 1), -- openssl 1.0, debian:7 + (3, 2, 2), -- openssl 1.0, debian:8 + (4, 3, 1); -- openssl 2.0, debian:7 + +INSERT INTO detector(id, name, version, dtype) VALUES + (1, 'os-release', '1.0', 'namespace'), + (2, 'dpkg', '1.0', 'feature'), + (3, 'rpm', '1.0', 'feature'), + (4, 'apt-sources', '1.0', 'namespace'); + +-- initialize layers INSERT INTO layer (id, hash) VALUES (1, 'layer-0'), -- blank - (2, 'layer-1'), -- debian:7; wechat 0.5, openssl 1.0 - (3, 'layer-2'), -- debian:7; wechat 0.5, openssl 2.0 + (2, 'layer-1'), -- debian:7; ourchat 0.5, openssl 1.0 + (3, 'layer-2'), -- debian:7; ourchat 0.5, openssl 2.0 (4, 'layer-3a'),-- debian:7; - (5, 'layer-3b'),-- debian:8; wechat 0.5, openssl 1.0 + (5, 'layer-3b'),-- debian:8; ourchat 0.5, openssl 1.0 (6, 'layer-4'); -- debian:7, fake:1.0; openssl 2.0 (debian), fake 2.0 (fake) -INSERT INTO layer_namespace(id, layer_id, namespace_id) VALUES - (1, 2, 1), - (2, 3, 1), - (3, 4, 1), - (4, 5, 2), - (5, 6, 1), - (6, 6, 3); - -INSERT INTO layer_feature(id, layer_id, feature_id) VALUES - (1, 2, 1), - (2, 2, 2), - (3, 3, 1), - (4, 3, 3), - (5, 5, 1), - (6, 5, 2), - (7, 6, 4), - (8, 6, 3); - -INSERT INTO layer_lister(id, layer_id, lister) VALUES - (1, 1, 'dpkg'), - (2, 2, 'dpkg'), - (3, 3, 'dpkg'), - (4, 4, 'dpkg'), - (5, 5, 'dpkg'), - (6, 6, 'dpkg'), - (7, 6, 'rpm'); - -INSERT INTO layer_detector(id, layer_id, detector) VALUES - (1, 1, 'os-release'), - (2, 2, 'os-release'), - (3, 3, 'os-release'), - (4, 4, 'os-release'), - (5, 5, 'os-release'), - (6, 6, 'os-release'), - (7, 6, 'apt-sources'); +INSERT INTO layer_namespace(id, layer_id, namespace_id, detector_id) VALUES + (1, 2, 1, 1), -- layer-1: debian:7 + (2, 3, 1, 1), -- layer-2: debian:7 + (3, 4, 1, 1), -- layer-3a: debian:7 + (4, 5, 2, 1), -- layer-3b: debian:8 + (5, 6, 1, 1), -- layer-4: debian:7 + (6, 6, 3, 4); -- layer-4: fake:1.0 + +INSERT INTO layer_feature(id, layer_id, feature_id, detector_id) VALUES + (1, 2, 1, 2), -- layer-1: ourchat 0.5 + (2, 2, 2, 2), -- layer-1: openssl 1.0 + (3, 3, 1, 2), -- layer-2: ourchat 0.5 + (4, 3, 3, 2), -- layer-2: openssl 2.0 + (5, 5, 1, 2), -- layer-3b: ourchat 0.5 + (6, 5, 2, 2), -- layer-3b: openssl 1.0 + (7, 6, 4, 3), -- layer-4: fake 2.0 + (8, 6, 3, 2); -- layer-4: openssl 2.0 + +INSERT INTO layer_detector(layer_id, detector_id) VALUES + (1, 1), + (2, 1), + (3, 1), + (4, 1), + (5, 1), + (6, 1), + (6, 4), + (1, 2), + (2, 2), + (3, 2), + (4, 2), + (5, 2), + (6, 2), + (6, 3); INSERT INTO ancestry (id, name) VALUES (1, 'ancestry-1'), -- layer-0, layer-1, layer-2, layer-3a @@ -59,32 +71,39 @@ INSERT INTO ancestry (id, name) VALUES (3, 'ancestry-3'), -- layer-0 (4, 'ancestry-4'); -- layer-0 -INSERT INTO ancestry_lister (id, ancestry_id, lister) VALUES - (1, 1, 'dpkg'), - (2, 2, 'dpkg'); - -INSERT INTO ancestry_detector (id, ancestry_id, detector) VALUES - (1, 1, 'os-release'), - (2, 2, 'os-release'); +INSERT INTO ancestry_detector (ancestry_id, detector_id) VALUES + (1, 2), + (2, 2), + (1, 1), + (2, 1); INSERT INTO ancestry_layer (id, ancestry_id, layer_id, ancestry_index) VALUES + -- ancestry-1: layer-0, layer-1, layer-2, layer-3a (1, 1, 1, 0),(2, 1, 2, 1),(3, 1, 3, 2),(4, 1, 4, 3), + -- ancestry-2: layer-0, layer-1, layer-2, layer-3b (5, 2, 1, 0),(6, 2, 2, 1),(7, 2, 3, 2),(8, 2, 5, 3), - (9, 3, 1, 0), - (10, 4, 1, 0); - -INSERT INTO namespaced_feature(id, feature_id, namespace_id) VALUES - (1, 1, 1), -- wechat 0.5, debian:7 - (2, 2, 1), -- openssl 1.0, debian:7 - (3, 2, 2), -- openssl 1.0, debian:8 - (4, 3, 1); -- openssl 2.0, debian:7 + -- ancestry-3: layer-1 + (9, 3, 2, 0), + -- ancestry-4: layer-1 + (10, 4, 2, 0); -- assume that ancestry-3 and ancestry-4 are vulnerable. -INSERT INTO ancestry_feature (id, ancestry_layer_id, namespaced_feature_id) VALUES - (1, 1, 1), (2, 1, 4), -- ancestry-1, layer 0 introduces 1, 4 - (3, 5, 1), (4, 5, 3), -- ancestry-2, layer 0 introduces 1, 3 - (5, 9, 2), -- ancestry-3, layer 0 introduces 2 - (6, 10, 2); -- ancestry-4, layer 0 introduces 2 +INSERT INTO ancestry_feature (id, ancestry_layer_id, namespaced_feature_id, feature_detector_id, namespace_detector_id) VALUES + -- ancestry-1: + -- layer-2: ourchat 0.5 <- detected by dpkg 1.0 (2); debian: 7 <- detected by os-release 1.0 (1) + -- layer-2: openssl 2.0, debian:7 + (1, 3, 1, 2, 1), (2, 3, 4, 2, 1), + -- ancestry 2: + -- 1(ourchat 0.5; debian:7 layer-2) + -- 3(openssl 1.0; debian:8 layer-3b) + (3, 7, 1, 2, 1), (4, 8, 3, 2, 1), + -- ancestry-3: + -- 2(openssl 1.0, debian:7 layer-1) + -- 1(ourchat 0.5, debian:7 layer-1) + (5, 9, 2, 2, 1), (6, 9, 1, 2, 1), -- vulnerable + -- ancestry-4: + -- same as ancestry-3 + (7, 10, 2, 2, 1), (8, 10, 1, 2, 1); -- vulnerable INSERT INTO vulnerability (id, namespace_id, name, description, link, severity) VALUES (1, 1, 'CVE-OPENSSL-1-DEB7', 'A vulnerability affecting OpenSSL < 2.0 on Debian 7.0', 'http://google.com/#q=CVE-OPENSSL-1-DEB7', 'High'), @@ -103,19 +122,23 @@ INSERT INTO vulnerability_affected_namespaced_feature(id, vulnerability_id, name INSERT INTO vulnerability_notification(id, name, created_at, notified_at, deleted_at, old_vulnerability_id, new_vulnerability_id) VALUES (1, 'test', NULL, NULL, NULL, 2, 1); -- 'CVE-NOPE' -> 'CVE-OPENSSL-1-DEB7' +SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('namespace', 'id'), (SELECT MAX(id) FROM namespace)+1); +SELECT pg_catalog.setval(pg_get_serial_sequence('namespaced_feature', 'id'), (SELECT MAX(id) FROM namespaced_feature)+1); +SELECT pg_catalog.setval(pg_get_serial_sequence('detector', 'id'), (SELECT MAX(id) FROM detector)+1); + SELECT pg_catalog.setval(pg_get_serial_sequence('ancestry', 'id'), (SELECT MAX(id) FROM ancestry)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('ancestry_layer', 'id'), (SELECT MAX(id) FROM ancestry_layer)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('ancestry_feature', 'id'), (SELECT MAX(id) FROM ancestry_feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('ancestry_detector', 'id'), (SELECT MAX(id) FROM ancestry_detector)+1); -SELECT pg_catalog.setval(pg_get_serial_sequence('ancestry_lister', 'id'), (SELECT MAX(id) FROM ancestry_lister)+1); -SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1); -SELECT pg_catalog.setval(pg_get_serial_sequence('namespaced_feature', 'id'), (SELECT MAX(id) FROM namespaced_feature)+1); + SELECT pg_catalog.setval(pg_get_serial_sequence('layer', 'id'), (SELECT MAX(id) FROM layer)+1); +SELECT pg_catalog.setval(pg_get_serial_sequence('layer_feature', 'id'), (SELECT MAX(id) FROM layer_feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('layer_namespace', 'id'), (SELECT MAX(id) FROM layer_namespace)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('layer_detector', 'id'), (SELECT MAX(id) FROM layer_detector)+1); -SELECT pg_catalog.setval(pg_get_serial_sequence('layer_lister', 'id'), (SELECT MAX(id) FROM layer_lister)+1); + SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability', 'id'), (SELECT MAX(id) FROM vulnerability)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_affected_feature', 'id'), (SELECT MAX(id) FROM vulnerability_affected_feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_affected_namespaced_feature', 'id'), (SELECT MAX(id) FROM vulnerability_affected_namespaced_feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_notification', 'id'), (SELECT MAX(id) FROM vulnerability_notification)+1); +SELECT pg_catalog.setval(pg_get_serial_sequence('detector', 'id'), (SELECT MAX(id) FROM detector)+1); diff --git a/database/pgsql/testutil.go b/database/pgsql/testutil.go new file mode 100644 index 00000000..1561e59d --- /dev/null +++ b/database/pgsql/testutil.go @@ -0,0 +1,262 @@ +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pgsql + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/pagination" +) + +// int keys must be the consistent with the database ID. +var ( + realFeatures = map[int]database.Feature{ + 1: {"ourchat", "0.5", "dpkg"}, + 2: {"openssl", "1.0", "dpkg"}, + 3: {"openssl", "2.0", "dpkg"}, + 4: {"fake", "2.0", "rpm"}, + } + + realNamespaces = map[int]database.Namespace{ + 1: {"debian:7", "dpkg"}, + 2: {"debian:8", "dpkg"}, + 3: {"fake:1.0", "rpm"}, + } + + realNamespacedFeatures = map[int]database.NamespacedFeature{ + 1: {realFeatures[1], realNamespaces[1]}, + 2: {realFeatures[2], realNamespaces[1]}, + 3: {realFeatures[2], realNamespaces[2]}, + 4: {realFeatures[3], realNamespaces[1]}, + } + + realDetectors = map[int]database.Detector{ + 1: database.NewNamespaceDetector("os-release", "1.0"), + 2: database.NewFeatureDetector("dpkg", "1.0"), + 3: database.NewFeatureDetector("rpm", "1.0"), + 4: database.NewNamespaceDetector("apt-sources", "1.0"), + } + + realLayers = map[int]database.Layer{ + 2: { + Hash: "layer-1", + By: []database.Detector{realDetectors[1], realDetectors[2]}, + Features: []database.LayerFeature{ + {realFeatures[1], realDetectors[2]}, + {realFeatures[2], realDetectors[2]}, + }, + Namespaces: []database.LayerNamespace{ + {realNamespaces[1], realDetectors[1]}, + }, + }, + 6: { + Hash: "layer-4", + By: []database.Detector{realDetectors[1], realDetectors[2], realDetectors[3], realDetectors[4]}, + Features: []database.LayerFeature{ + {realFeatures[4], realDetectors[3]}, + {realFeatures[3], realDetectors[2]}, + }, + Namespaces: []database.LayerNamespace{ + {realNamespaces[1], realDetectors[1]}, + {realNamespaces[3], realDetectors[4]}, + }, + }, + } + + realAncestries = map[int]database.Ancestry{ + 2: { + Name: "ancestry-2", + By: []database.Detector{realDetectors[2], realDetectors[1]}, + Layers: []database.AncestryLayer{ + { + "layer-0", + []database.AncestryFeature{}, + }, + { + "layer-1", + []database.AncestryFeature{}, + }, + { + "layer-2", + []database.AncestryFeature{ + { + realNamespacedFeatures[1], + realDetectors[2], + realDetectors[1], + }, + }, + }, + { + "layer-3b", + []database.AncestryFeature{ + { + realNamespacedFeatures[3], + realDetectors[2], + realDetectors[1], + }, + }, + }, + }, + }, + } + + realVulnerability = map[int]database.Vulnerability{ + 1: { + Name: "CVE-OPENSSL-1-DEB7", + Namespace: realNamespaces[1], + Description: "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", + Link: "http://google.com/#q=CVE-OPENSSL-1-DEB7", + Severity: database.HighSeverity, + }, + 2: { + Name: "CVE-NOPE", + Namespace: realNamespaces[1], + Description: "A vulnerability affecting nothing", + Severity: database.UnknownSeverity, + }, + } + + realNotification = map[int]database.VulnerabilityNotification{ + 1: { + NotificationHook: database.NotificationHook{ + Name: "test", + }, + Old: takeVulnerabilityPointerFromMap(realVulnerability, 2), + New: takeVulnerabilityPointerFromMap(realVulnerability, 1), + }, + } + + fakeFeatures = map[int]database.Feature{ + 1: { + Name: "ourchat", + Version: "0.6", + VersionFormat: "dpkg", + }, + } + + fakeNamespaces = map[int]database.Namespace{ + 1: {"green hat", "rpm"}, + } + fakeNamespacedFeatures = map[int]database.NamespacedFeature{ + 1: { + Feature: fakeFeatures[0], + Namespace: realNamespaces[0], + }, + } + + fakeDetector = map[int]database.Detector{ + 1: { + Name: "fake", + Version: "1.0", + DType: database.FeatureDetectorType, + }, + 2: { + Name: "fake2", + Version: "2.0", + DType: database.NamespaceDetectorType, + }, + } +) + +func takeVulnerabilityPointerFromMap(m map[int]database.Vulnerability, id int) *database.Vulnerability { + x := m[id] + return &x +} + +func takeAncestryPointerFromMap(m map[int]database.Ancestry, id int) *database.Ancestry { + x := m[id] + return &x +} + +func takeLayerPointerFromMap(m map[int]database.Layer, id int) *database.Layer { + x := m[id] + return &x +} + +func listNamespaces(t *testing.T, tx *pgSession) []database.Namespace { + rows, err := tx.Query("SELECT name, version_format FROM namespace") + if err != nil { + t.FailNow() + } + defer rows.Close() + + namespaces := []database.Namespace{} + for rows.Next() { + var ns database.Namespace + err := rows.Scan(&ns.Name, &ns.VersionFormat) + if err != nil { + t.FailNow() + } + namespaces = append(namespaces, ns) + } + + return namespaces +} + +func assertVulnerabilityNotificationWithVulnerableEqual(t *testing.T, key pagination.Key, expected, actual *database.VulnerabilityNotificationWithVulnerable) bool { + if expected == actual { + return true + } + + if expected == nil || actual == nil { + return assert.Equal(t, expected, actual) + } + + return assert.Equal(t, expected.NotificationHook, actual.NotificationHook) && + AssertPagedVulnerableAncestriesEqual(t, key, expected.Old, actual.Old) && + AssertPagedVulnerableAncestriesEqual(t, key, expected.New, actual.New) +} + +func AssertPagedVulnerableAncestriesEqual(t *testing.T, key pagination.Key, expected, actual *database.PagedVulnerableAncestries) bool { + if expected == actual { + return true + } + + if expected == nil || actual == nil { + return assert.Equal(t, expected, actual) + } + + return database.AssertVulnerabilityEqual(t, &expected.Vulnerability, &actual.Vulnerability) && + assert.Equal(t, expected.Limit, actual.Limit) && + assert.Equal(t, mustUnmarshalToken(key, expected.Current), mustUnmarshalToken(key, actual.Current)) && + assert.Equal(t, mustUnmarshalToken(key, expected.Next), mustUnmarshalToken(key, actual.Next)) && + assert.Equal(t, expected.End, actual.End) && + database.AssertIntStringMapEqual(t, expected.Affected, actual.Affected) +} + +func mustUnmarshalToken(key pagination.Key, token pagination.Token) Page { + if token == pagination.FirstPageToken { + return Page{} + } + + p := Page{} + if err := key.UnmarshalToken(token, &p); err != nil { + panic(err) + } + + return p +} + +func mustMarshalToken(key pagination.Key, v interface{}) pagination.Token { + token, err := key.MarshalToken(v) + if err != nil { + panic(err) + } + + return token +} diff --git a/database/pgsql/vulnerability.go b/database/pgsql/vulnerability.go index fb483cfe..93518a87 100644 --- a/database/pgsql/vulnerability.go +++ b/database/pgsql/vulnerability.go @@ -26,6 +26,73 @@ import ( "github.com/coreos/clair/ext/versionfmt" ) +const ( + lockVulnerabilityAffects = `LOCK vulnerability_affected_namespaced_feature IN SHARE ROW EXCLUSIVE MODE` + + searchVulnerability = ` + SELECT v.id, v.description, v.link, v.severity, v.metadata, n.version_format + FROM vulnerability AS v, namespace AS n + WHERE v.namespace_id = n.id + AND v.name = $1 + AND n.name = $2 + AND v.deleted_at IS NULL + ` + + insertVulnerabilityAffected = ` + INSERT INTO vulnerability_affected_feature(vulnerability_id, feature_name, affected_version, fixedin) + VALUES ($1, $2, $3, $4) + RETURNING ID + ` + + searchVulnerabilityAffected = ` + SELECT vulnerability_id, feature_name, affected_version, fixedin + FROM vulnerability_affected_feature + WHERE vulnerability_id = ANY($1) + ` + + searchVulnerabilityByID = ` + SELECT v.name, v.description, v.link, v.severity, v.metadata, n.name, n.version_format + FROM vulnerability AS v, namespace AS n + WHERE v.namespace_id = n.id + AND v.id = $1` + + searchVulnerabilityPotentialAffected = ` + WITH req AS ( + SELECT vaf.id AS vaf_id, n.id AS n_id, vaf.feature_name AS name, v.id AS vulnerability_id + FROM vulnerability_affected_feature AS vaf, + vulnerability AS v, + namespace AS n + WHERE vaf.vulnerability_id = ANY($1) + AND v.id = vaf.vulnerability_id + AND n.id = v.namespace_id + ) + SELECT req.vulnerability_id, nf.id, f.version, req.vaf_id AS added_by + FROM feature AS f, namespaced_feature AS nf, req + WHERE f.name = req.name + AND nf.namespace_id = req.n_id + AND nf.feature_id = f.id` + + insertVulnerabilityAffectedNamespacedFeature = ` + INSERT INTO vulnerability_affected_namespaced_feature(vulnerability_id, namespaced_feature_id, added_by) + VALUES ($1, $2, $3)` + + insertVulnerability = ` + WITH ns AS ( + SELECT id FROM namespace WHERE name = $6 AND version_format = $7 + ) + INSERT INTO Vulnerability(namespace_id, name, description, link, severity, metadata, created_at) + VALUES((SELECT id FROM ns), $1, $2, $3, $4, $5, CURRENT_TIMESTAMP) + RETURNING id` + + removeVulnerability = ` + UPDATE Vulnerability + SET deleted_at = CURRENT_TIMESTAMP + WHERE namespace_id = (SELECT id FROM Namespace WHERE name = $1) + AND name = $2 + AND deleted_at IS NULL + RETURNING id` +) + var ( errVulnerabilityNotFound = errors.New("vulnerability is not in database") ) diff --git a/database/pgsql/vulnerability_test.go b/database/pgsql/vulnerability_test.go index 9fe2c23b..bfa465b2 100644 --- a/database/pgsql/vulnerability_test.go +++ b/database/pgsql/vulnerability_test.go @@ -306,14 +306,14 @@ func TestFindVulnerabilityIDs(t *testing.T) { ids, err := tx.findLatestDeletedVulnerabilityIDs([]database.VulnerabilityID{{Name: "CVE-DELETED", Namespace: "debian:7"}}) if assert.Nil(t, err) { - if !(assert.Len(t, ids, 1) && assert.True(t, ids[0].Valid) && assert.Equal(t, 3, ids[0].Int64)) { + if !(assert.Len(t, ids, 1) && assert.True(t, ids[0].Valid) && assert.Equal(t, 3, int(ids[0].Int64))) { assert.Fail(t, "") } } ids, err = tx.findNotDeletedVulnerabilityIDs([]database.VulnerabilityID{{Name: "CVE-NOPE", Namespace: "debian:7"}}) if assert.Nil(t, err) { - if !(assert.Len(t, ids, 1) && assert.True(t, ids[0].Valid) && assert.Equal(t, 2, ids[0].Int64)) { + if !(assert.Len(t, ids, 1) && assert.True(t, ids[0].Valid) && assert.Equal(t, 2, int(ids[0].Int64))) { assert.Fail(t, "") } } diff --git a/database/testutil.go b/database/testutil.go new file mode 100644 index 00000000..232e9202 --- /dev/null +++ b/database/testutil.go @@ -0,0 +1,297 @@ +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "encoding/json" + "sort" + "testing" + + "github.com/deckarep/golang-set" + "github.com/stretchr/testify/assert" +) + +// AssertDetectorsEqual asserts actual detectors are content wise equal to +// expected detectors regardless of the ordering. +func AssertDetectorsEqual(t *testing.T, expected, actual []Detector) bool { + if len(expected) != len(actual) { + return assert.Fail(t, "detectors are not equal", "expected: '%v', actual: '%v'", expected, actual) + } + + sort.Slice(expected, func(i, j int) bool { + return expected[i].String() < expected[j].String() + }) + + sort.Slice(actual, func(i, j int) bool { + return actual[i].String() < actual[j].String() + }) + + for i := range expected { + if expected[i] != actual[i] { + return assert.Fail(t, "detectors are not equal", "expected: '%v', actual: '%v'", expected, actual) + } + } + + return true +} + +// AssertAncestryEqual asserts actual ancestry equals to expected ancestry +// content wise. +func AssertAncestryEqual(t *testing.T, expected, actual *Ancestry) bool { + if expected == actual { + return true + } + + if actual == nil || expected == nil { + return assert.Equal(t, expected, actual) + } + + if !assert.Equal(t, expected.Name, actual.Name) || !AssertDetectorsEqual(t, expected.By, actual.By) { + return false + } + + if assert.Equal(t, len(expected.Layers), len(actual.Layers)) { + for index := range expected.Layers { + if !AssertAncestryLayerEqual(t, &expected.Layers[index], &actual.Layers[index]) { + return false + } + } + return true + } + return false +} + +// AssertAncestryLayerEqual asserts actual ancestry layer equals to expected +// ancestry layer content wise. +func AssertAncestryLayerEqual(t *testing.T, expected, actual *AncestryLayer) bool { + if !assert.Equal(t, expected.Hash, actual.Hash) { + return false + } + + if !assert.Equal(t, len(expected.Features), len(actual.Features), + "layer: %s\nExpected: %v\n Actual: %v", + expected.Hash, expected.Features, actual.Features, + ) { + return false + } + + // feature -> is in actual layer + hitCounter := map[AncestryFeature]bool{} + for _, f := range expected.Features { + hitCounter[f] = false + } + + // if there's no extra features and no duplicated features, since expected + // and actual have the same length, their result must equal. + for _, f := range actual.Features { + v, ok := hitCounter[f] + assert.True(t, ok, "unexpected feature %s", f) + assert.False(t, v, "duplicated feature %s", f) + hitCounter[f] = true + } + + for f, visited := range hitCounter { + assert.True(t, visited, "missing feature %s", f) + } + + return true +} + +// AssertElementsEqual asserts that content in actual equals to content in +// expected array regardless of ordering. +// +// Note: This function uses interface wise comparison. +func AssertElementsEqual(t *testing.T, expected, actual []interface{}) bool { + counter := map[interface{}]bool{} + for _, f := range expected { + counter[f] = false + } + + for _, f := range actual { + v, ok := counter[f] + if !assert.True(t, ok, "unexpected element %v\nExpected: %v\n Actual: %v\n", f, expected, actual) { + return false + } + + if !assert.False(t, v, "duplicated element %v\nExpected: %v\n Actual: %v\n", f, expected, actual) { + return false + } + + counter[f] = true + } + + for f, visited := range counter { + if !assert.True(t, visited, "missing feature %v\nExpected: %v\n Actual: %v\n", f, expected, actual) { + return false + } + } + + return true +} + +// AssertFeaturesEqual asserts content in actual equals content in expected +// regardless of ordering. +func AssertFeaturesEqual(t *testing.T, expected, actual []Feature) bool { + if assert.Len(t, actual, len(expected)) { + has := map[Feature]bool{} + for _, nf := range expected { + has[nf] = false + } + + for _, nf := range actual { + has[nf] = true + } + + for nf, visited := range has { + if !assert.True(t, visited, nf.Name+" is expected") { + return false + } + return true + } + } + return false +} + +// AssertLayerFeaturesEqual asserts content in actual equals to content in +// expected regardless of ordering. +func AssertLayerFeaturesEqual(t *testing.T, expected, actual []LayerFeature) bool { + if !assert.Len(t, actual, len(expected)) { + return false + } + + expectedInterfaces := []interface{}{} + for _, e := range expected { + expectedInterfaces = append(expectedInterfaces, e) + } + + actualInterfaces := []interface{}{} + for _, a := range actual { + actualInterfaces = append(actualInterfaces, a) + } + + return AssertElementsEqual(t, expectedInterfaces, actualInterfaces) +} + +// AssertNamespacesEqual asserts content in actual equals to content in +// expected regardless of ordering. +func AssertNamespacesEqual(t *testing.T, expected, actual []Namespace) bool { + expectedInterfaces := []interface{}{} + for _, e := range expected { + expectedInterfaces = append(expectedInterfaces, e) + } + + actualInterfaces := []interface{}{} + for _, a := range actual { + actualInterfaces = append(actualInterfaces, a) + } + + return AssertElementsEqual(t, expectedInterfaces, actualInterfaces) +} + +// AssertLayerNamespacesEqual asserts content in actual equals to content in +// expected regardless of ordering. +func AssertLayerNamespacesEqual(t *testing.T, expected, actual []LayerNamespace) bool { + expectedInterfaces := []interface{}{} + for _, e := range expected { + expectedInterfaces = append(expectedInterfaces, e) + } + + actualInterfaces := []interface{}{} + for _, a := range actual { + actualInterfaces = append(actualInterfaces, a) + } + + return AssertElementsEqual(t, expectedInterfaces, actualInterfaces) +} + +// AssertLayerEqual asserts actual layer equals to expected layer content wise. +func AssertLayerEqual(t *testing.T, expected, actual *Layer) bool { + if expected == actual { + return true + } + + if expected == nil || actual == nil { + return assert.Equal(t, expected, actual) + } + + return assert.Equal(t, expected.Hash, actual.Hash) && + AssertDetectorsEqual(t, expected.By, actual.By) && + AssertLayerFeaturesEqual(t, expected.Features, actual.Features) && + AssertLayerNamespacesEqual(t, expected.Namespaces, actual.Namespaces) +} + +// AssertIntStringMapEqual asserts two maps with integer as key and string as +// value are equal. +func AssertIntStringMapEqual(t *testing.T, expected, actual map[int]string) bool { + checked := mapset.NewSet() + for k, v := range expected { + assert.Equal(t, v, actual[k]) + checked.Add(k) + } + + for k := range actual { + if !assert.True(t, checked.Contains(k)) { + return false + } + } + + return true +} + +// AssertVulnerabilityEqual asserts two vulnerabilities are equal. +func AssertVulnerabilityEqual(t *testing.T, expected, actual *Vulnerability) bool { + return assert.Equal(t, expected.Name, actual.Name) && + assert.Equal(t, expected.Link, actual.Link) && + assert.Equal(t, expected.Description, actual.Description) && + assert.Equal(t, expected.Namespace, actual.Namespace) && + assert.Equal(t, expected.Severity, actual.Severity) && + AssertMetadataMapEqual(t, expected.Metadata, actual.Metadata) +} + +func castMetadataMapToInterface(metadata MetadataMap) map[string]interface{} { + content, err := json.Marshal(metadata) + if err != nil { + panic(err) + } + + data := make(map[string]interface{}) + if err := json.Unmarshal(content, &data); err != nil { + panic(err) + } + + return data +} + +// AssertMetadataMapEqual asserts two metadata maps are equal. +func AssertMetadataMapEqual(t *testing.T, expected, actual MetadataMap) bool { + expectedMap := castMetadataMapToInterface(expected) + actualMap := castMetadataMapToInterface(actual) + checked := mapset.NewSet() + for k, v := range expectedMap { + if !assert.Equal(t, v, (actualMap)[k]) { + return false + } + + checked.Add(k) + } + + for k := range actual { + if !assert.True(t, checked.Contains(k)) { + return false + } + } + + return true +} diff --git a/ext/featurefmt/apk/apk.go b/ext/featurefmt/apk/apk.go index 195c8920..eda25cd4 100644 --- a/ext/featurefmt/apk/apk.go +++ b/ext/featurefmt/apk/apk.go @@ -29,7 +29,7 @@ import ( ) func init() { - featurefmt.RegisterLister("apk", dpkg.ParserName, &lister{}) + featurefmt.RegisterLister("apk", "1.0", &lister{}) } type lister struct{} diff --git a/ext/featurefmt/dpkg/dpkg.go b/ext/featurefmt/dpkg/dpkg.go index 6b987cf3..0ac30a90 100644 --- a/ext/featurefmt/dpkg/dpkg.go +++ b/ext/featurefmt/dpkg/dpkg.go @@ -37,7 +37,7 @@ var ( type lister struct{} func init() { - featurefmt.RegisterLister("dpkg", dpkg.ParserName, &lister{}) + featurefmt.RegisterLister("dpkg", "1.0", &lister{}) } func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) { diff --git a/ext/featurefmt/driver.go b/ext/featurefmt/driver.go index 0f48b0e7..6ade140b 100644 --- a/ext/featurefmt/driver.go +++ b/ext/featurefmt/driver.go @@ -31,9 +31,8 @@ import ( ) var ( - listersM sync.RWMutex - listers = make(map[string]Lister) - versionfmtListerName = make(map[string][]string) + listersM sync.RWMutex + listers = make(map[string]lister) ) // Lister represents an ability to list the features present in an image layer. @@ -48,13 +47,19 @@ type Lister interface { RequiredFilenames() []string } +type lister struct { + Lister + + info database.Detector +} + // RegisterLister makes a Lister available by the provided name. // // If called twice with the same name, the name is blank, or if the provided // Lister is nil, this function panics. -func RegisterLister(name string, versionfmt string, l Lister) { - if name == "" { - panic("featurefmt: could not register a Lister with an empty name") +func RegisterLister(name string, version string, l Lister) { + if name == "" || version == "" { + panic("featurefmt: could not register a Lister with an empty name or version") } if l == nil { panic("featurefmt: could not register a nil Lister") @@ -67,51 +72,65 @@ func RegisterLister(name string, versionfmt string, l Lister) { panic("featurefmt: RegisterLister called twice for " + name) } - listers[name] = l - versionfmtListerName[versionfmt] = append(versionfmtListerName[versionfmt], name) + listers[name] = lister{l, database.NewFeatureDetector(name, version)} } // ListFeatures produces the list of Features in an image layer using // every registered Lister. -func ListFeatures(files tarutil.FilesMap, listerNames []string) ([]database.Feature, error) { +func ListFeatures(files tarutil.FilesMap, toUse []database.Detector) ([]database.LayerFeature, error) { listersM.RLock() defer listersM.RUnlock() - var totalFeatures []database.Feature + features := []database.LayerFeature{} + for _, d := range toUse { + // Only use the detector with the same type + if d.DType != database.FeatureDetectorType { + continue + } - for _, name := range listerNames { - if lister, ok := listers[name]; ok { - features, err := lister.ListFeatures(files) + if lister, ok := listers[d.Name]; ok { + fs, err := lister.ListFeatures(files) if err != nil { - return []database.Feature{}, err + return nil, err } - totalFeatures = append(totalFeatures, features...) + + for _, f := range fs { + features = append(features, database.LayerFeature{ + Feature: f, + By: lister.info, + }) + } + } else { - log.WithField("Name", name).Warn("Unknown Lister") + log.WithField("Name", d).Fatal("unknown feature detector") } } - return totalFeatures, nil + return features, nil } -// RequiredFilenames returns the total list of files required for all -// registered Listers. -func RequiredFilenames(listerNames []string) (files []string) { +// RequiredFilenames returns all files required by the give extensions. Any +// extension metadata that has non feature-detector type will be skipped. +func RequiredFilenames(toUse []database.Detector) (files []string) { listersM.RLock() defer listersM.RUnlock() - for _, lister := range listers { - files = append(files, lister.RequiredFilenames()...) + for _, d := range toUse { + if d.DType != database.FeatureDetectorType { + continue + } + + files = append(files, listers[d.Name].RequiredFilenames()...) } return } // ListListers returns the names of all the registered feature listers. -func ListListers() []string { - r := []string{} - for name := range listers { - r = append(r, name) +func ListListers() []database.Detector { + r := []database.Detector{} + for _, d := range listers { + r = append(r, d.info) } return r } diff --git a/ext/featurefmt/rpm/rpm.go b/ext/featurefmt/rpm/rpm.go index 5a0e1fa1..be582384 100644 --- a/ext/featurefmt/rpm/rpm.go +++ b/ext/featurefmt/rpm/rpm.go @@ -35,7 +35,7 @@ import ( type lister struct{} func init() { - featurefmt.RegisterLister("rpm", rpm.ParserName, &lister{}) + featurefmt.RegisterLister("rpm", "1.0", &lister{}) } func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) { diff --git a/ext/featurens/alpinerelease/alpinerelease.go b/ext/featurens/alpinerelease/alpinerelease.go index fafe5c9f..8fe6f808 100644 --- a/ext/featurens/alpinerelease/alpinerelease.go +++ b/ext/featurens/alpinerelease/alpinerelease.go @@ -36,7 +36,7 @@ const ( var versionRegexp = regexp.MustCompile(`^(\d)+\.(\d)+\.(\d)+$`) func init() { - featurens.RegisterDetector("alpine-release", &detector{}) + featurens.RegisterDetector("alpine-release", "1.0", &detector{}) } type detector struct{} diff --git a/ext/featurens/aptsources/aptsources.go b/ext/featurens/aptsources/aptsources.go index c43818e7..287073f4 100644 --- a/ext/featurens/aptsources/aptsources.go +++ b/ext/featurens/aptsources/aptsources.go @@ -32,7 +32,7 @@ import ( type detector struct{} func init() { - featurens.RegisterDetector("apt-sources", &detector{}) + featurens.RegisterDetector("apt-sources", "1.0", &detector{}) } func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { diff --git a/ext/featurens/driver.go b/ext/featurens/driver.go index b7e0ad37..781ba5d4 100644 --- a/ext/featurens/driver.go +++ b/ext/featurens/driver.go @@ -29,7 +29,7 @@ import ( var ( detectorsM sync.RWMutex - detectors = make(map[string]Detector) + detectors = make(map[string]detector) ) // Detector represents an ability to detect a namespace used for organizing @@ -46,13 +46,19 @@ type Detector interface { RequiredFilenames() []string } +type detector struct { + Detector + + info database.Detector +} + // RegisterDetector makes a detector available by the provided name. // // If called twice with the same name, the name is blank, or if the provided // Detector is nil, this function panics. -func RegisterDetector(name string, d Detector) { - if name == "" { - panic("namespace: could not register a Detector with an empty name") +func RegisterDetector(name string, version string, d Detector) { + if name == "" || version == "" { + panic("namespace: could not register a Detector with an empty name or version") } if d == nil { panic("namespace: could not register a nil Detector") @@ -61,60 +67,69 @@ func RegisterDetector(name string, d Detector) { detectorsM.Lock() defer detectorsM.Unlock() - if _, dup := detectors[name]; dup { + if _, ok := detectors[name]; ok { panic("namespace: RegisterDetector called twice for " + name) } - detectors[name] = d + detectors[name] = detector{d, database.NewNamespaceDetector(name, version)} } -// Detect iterators through all registered Detectors and returns all non-nil detected namespaces -func Detect(files tarutil.FilesMap, detectorNames []string) ([]database.Namespace, error) { +// Detect uses detectors specified to retrieve the detect result. +func Detect(files tarutil.FilesMap, toUse []database.Detector) ([]database.LayerNamespace, error) { detectorsM.RLock() defer detectorsM.RUnlock() - namespaces := map[string]*database.Namespace{} - for _, name := range detectorNames { - if detector, ok := detectors[name]; ok { + + namespaces := []database.LayerNamespace{} + for _, d := range toUse { + // Only use the detector with the same type + if d.DType != database.NamespaceDetectorType { + continue + } + + if detector, ok := detectors[d.Name]; ok { namespace, err := detector.Detect(files) if err != nil { - log.WithError(err).WithField("name", name).Warning("failed while attempting to detect namespace") + log.WithError(err).WithField("detector", d).Warning("failed while attempting to detect namespace") return nil, err } if namespace != nil { - log.WithFields(log.Fields{"name": name, "namespace": namespace.Name}).Debug("detected namespace") - namespaces[namespace.Name] = namespace + log.WithFields(log.Fields{"detector": d, "namespace": namespace.Name}).Debug("detected namespace") + namespaces = append(namespaces, database.LayerNamespace{ + Namespace: *namespace, + By: detector.info, + }) } } else { - log.WithField("Name", name).Warn("Unknown namespace detector") + log.WithField("detector", d).Fatal("unknown namespace detector") } } - nslist := []database.Namespace{} - for _, ns := range namespaces { - nslist = append(nslist, *ns) - } - return nslist, nil + return namespaces, nil } -// RequiredFilenames returns the total list of files required for all -// registered Detectors. -func RequiredFilenames(detectorNames []string) (files []string) { +// RequiredFilenames returns all files required by the give extensions. Any +// extension metadata that has non namespace-detector type will be skipped. +func RequiredFilenames(toUse []database.Detector) (files []string) { detectorsM.RLock() defer detectorsM.RUnlock() - for _, detector := range detectors { - files = append(files, detector.RequiredFilenames()...) + for _, d := range toUse { + if d.DType != database.NamespaceDetectorType { + continue + } + + files = append(files, detectors[d.Name].RequiredFilenames()...) } return } -// ListDetectors returns the names of all registered namespace detectors. -func ListDetectors() []string { - r := []string{} - for name := range detectors { - r = append(r, name) +// ListDetectors returns the info of all registered namespace detectors. +func ListDetectors() []database.Detector { + r := make([]database.Detector, 0, len(detectors)) + for _, d := range detectors { + r = append(r, d.info) } return r } diff --git a/ext/featurens/driver_test.go b/ext/featurens/driver_test.go index 0d46b0b6..1f463d45 100644 --- a/ext/featurens/driver_test.go +++ b/ext/featurens/driver_test.go @@ -16,40 +16,14 @@ import ( _ "github.com/coreos/clair/ext/featurens/redhatrelease" ) -type MultipleNamespaceTestData struct { - Files tarutil.FilesMap - ExpectedNamespaces []database.Namespace -} - -func assertnsNameEqual(t *testing.T, nslist_expected, nslist []database.Namespace) { - assert.Equal(t, len(nslist_expected), len(nslist)) - expected := map[string]struct{}{} - input := map[string]struct{}{} - // compare the two sets - for i := range nslist_expected { - expected[nslist_expected[i].Name] = struct{}{} - input[nslist[i].Name] = struct{}{} - } - assert.Equal(t, expected, input) -} - -func testMultipleNamespace(t *testing.T, testData []MultipleNamespaceTestData) { - for _, td := range testData { - nslist, err := featurens.Detect(td.Files, featurens.ListDetectors()) - assert.Nil(t, err) - assertnsNameEqual(t, td.ExpectedNamespaces, nslist) - } -} - -func TestMultipleNamespaceDetector(t *testing.T) { - testData := []MultipleNamespaceTestData{ - { - ExpectedNamespaces: []database.Namespace{ - {Name: "debian:8", VersionFormat: "dpkg"}, - {Name: "alpine:v3.3", VersionFormat: "dpkg"}, - }, - Files: tarutil.FilesMap{ - "etc/os-release": []byte(` +var namespaceDetectorTests = []struct { + in tarutil.FilesMap + out []database.LayerNamespace + err string +}{ + { + in: tarutil.FilesMap{ + "etc/os-release": []byte(` PRETTY_NAME="Debian GNU/Linux 8 (jessie)" NAME="Debian GNU/Linux" VERSION_ID="8" @@ -58,9 +32,23 @@ ID=debian HOME_URL="http://www.debian.org/" SUPPORT_URL="http://www.debian.org/support/" BUG_REPORT_URL="https://bugs.debian.org/"`), - "etc/alpine-release": []byte(`3.3.4`), - }, + "etc/alpine-release": []byte(`3.3.4`), + }, + out: []database.LayerNamespace{ + {database.Namespace{"debian:8", "dpkg"}, database.NewNamespaceDetector("os-release", "1.0")}, + {database.Namespace{"alpine:v3.3", "dpkg"}, database.NewNamespaceDetector("alpine-release", "1.0")}, }, + }, +} + +func TestNamespaceDetector(t *testing.T) { + for _, test := range namespaceDetectorTests { + out, err := featurens.Detect(test.in, featurens.ListDetectors()) + if test.err != "" { + assert.EqualError(t, err, test.err) + return + } + + database.AssertLayerNamespacesEqual(t, test.out, out) } - testMultipleNamespace(t, testData) } diff --git a/ext/featurens/lsbrelease/lsbrelease.go b/ext/featurens/lsbrelease/lsbrelease.go index d883215f..de528327 100644 --- a/ext/featurens/lsbrelease/lsbrelease.go +++ b/ext/featurens/lsbrelease/lsbrelease.go @@ -38,7 +38,7 @@ var ( type detector struct{} func init() { - featurens.RegisterDetector("lsb-release", &detector{}) + featurens.RegisterDetector("lsb-release", "1.0", &detector{}) } func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { diff --git a/ext/featurens/osrelease/osrelease.go b/ext/featurens/osrelease/osrelease.go index 1139739d..8e736743 100644 --- a/ext/featurens/osrelease/osrelease.go +++ b/ext/featurens/osrelease/osrelease.go @@ -45,7 +45,7 @@ var ( type detector struct{} func init() { - featurens.RegisterDetector("os-release", &detector{}) + featurens.RegisterDetector("os-release", "1.0", &detector{}) } func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { diff --git a/ext/featurens/redhatrelease/redhatrelease.go b/ext/featurens/redhatrelease/redhatrelease.go index 0dabc3fa..a8d27081 100644 --- a/ext/featurens/redhatrelease/redhatrelease.go +++ b/ext/featurens/redhatrelease/redhatrelease.go @@ -38,7 +38,7 @@ var ( type detector struct{} func init() { - featurens.RegisterDetector("redhat-release", &detector{}) + featurens.RegisterDetector("redhat-release", "1.0", &detector{}) } func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { diff --git a/ext/imagefmt/driver.go b/ext/imagefmt/driver.go index 46724401..9838efec 100644 --- a/ext/imagefmt/driver.go +++ b/ext/imagefmt/driver.go @@ -33,6 +33,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/strutil" "github.com/coreos/clair/pkg/tarutil" ) @@ -106,7 +107,7 @@ func UnregisterExtractor(name string) { func Extract(format, path string, headers map[string]string, toExtract []string) (tarutil.FilesMap, error) { var layerReader io.ReadCloser if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { - // Create a new HTTP request object. + log.WithField("path", strutil.CleanURL(path)).Debug("start downloading layer blob...") request, err := http.NewRequest("GET", path, nil) if err != nil { return nil, ErrCouldNotFindLayer @@ -127,21 +128,23 @@ func Extract(format, path string, headers map[string]string, toExtract []string) client := &http.Client{Transport: tr} r, err := client.Do(request) if err != nil { - log.WithError(err).Warning("could not download layer") + log.WithError(err).Error("could not download layer") return nil, ErrCouldNotFindLayer } // Fail if we don't receive a 2xx HTTP status code. if math.Floor(float64(r.StatusCode/100)) != 2 { - log.WithField("status code", r.StatusCode).Warning("could not download layer: expected 2XX") + log.WithError(ErrCouldNotFindLayer).WithField("status code", r.StatusCode).Error("could not download layer: expected 2XX") return nil, ErrCouldNotFindLayer } layerReader = r.Body } else { + log.WithField("path", strutil.CleanURL(path)).Debug("start reading layer blob from local file system...") var err error layerReader, err = os.Open(path) if err != nil { + log.WithError(ErrCouldNotFindLayer).Error("could not open layer") return nil, ErrCouldNotFindLayer } } diff --git a/glide.lock b/glide.lock index 73e38f49..b87b6d83 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 9bf7ad53b92119a17068b8724d9b406a7ca84b5bfcd0baba44b08c696a538b14 -updated: 2018-09-06T15:58:19.234504-04:00 +hash: 3fd0e471868863d6ef4cd32bbcdc9b3d061911a15b458e7edd26cfba4faa62db +updated: 2018-09-17T13:13:44.344244-04:00 imports: - name: github.com/beorn7/perks version: 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 @@ -15,6 +15,8 @@ imports: version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 subpackages: - spew +- name: github.com/deckarep/golang-set + version: cbaa98ba5575e67703b32b4b19f73c91f3c4159e - name: github.com/fernet/fernet-go version: 1b2437bc582b3cfbb341ee5a29f8ef5b42912ff2 - name: github.com/golang/protobuf diff --git a/glide.yaml b/glide.yaml index e5956e18..3da576b1 100644 --- a/glide.yaml +++ b/glide.yaml @@ -28,3 +28,5 @@ import: - assert - package: gopkg.in/yaml.v2 - package: github.com/cockroachdb/cmux +- package: github.com/deckarep/golang-set + version: ^1.7.1 diff --git a/pkg/strutil/strutil.go b/pkg/strutil/strutil.go index a8d04f21..bfd8dc01 100644 --- a/pkg/strutil/strutil.go +++ b/pkg/strutil/strutil.go @@ -14,42 +14,46 @@ package strutil -// CompareStringLists returns the strings that are present in X but not in Y. -func CompareStringLists(X, Y []string) []string { - m := make(map[string]bool) +import ( + "regexp" - for _, y := range Y { - m[y] = true - } + set "github.com/deckarep/golang-set" +) + +var urlParametersRegexp = regexp.MustCompile(`(\?|\&)([^=]+)\=([^ &]+)`) - diff := []string{} +func convertToSet(X []string) set.Set { + s := set.NewSet() for _, x := range X { - if m[x] { - continue - } + s.Add(x) + } + return s +} - diff = append(diff, x) - m[x] = true +func setToStringSlice(s set.Set) []string { + strs := make([]string, 0, s.Cardinality()) + for _, str := range s.ToSlice() { + strs = append(strs, str.(string)) } - return diff + return strs } -// CompareStringListsInBoth returns the strings that are present in both X and Y. -func CompareStringListsInBoth(X, Y []string) []string { - m := make(map[string]struct{}) - - for _, y := range Y { - m[y] = struct{}{} - } +// Difference returns the strings that are present in X but not in Y. +func Difference(X, Y []string) []string { + x := convertToSet(X) + y := convertToSet(Y) + return setToStringSlice(x.Difference(y)) +} - diff := []string{} - for _, x := range X { - if _, e := m[x]; e { - diff = append(diff, x) - delete(m, x) - } - } +// Intersect returns the strings that are present in both X and Y. +func Intersect(X, Y []string) []string { + x := convertToSet(X) + y := convertToSet(Y) + return setToStringSlice(x.Intersect(y)) +} - return diff +// CleanURL removes all parameters from an URL. +func CleanURL(str string) string { + return urlParametersRegexp.ReplaceAllString(str, "") } diff --git a/pkg/strutil/strutil_test.go b/pkg/strutil/strutil_test.go index 4cbf1e90..2e81856c 100644 --- a/pkg/strutil/strutil_test.go +++ b/pkg/strutil/strutil_test.go @@ -21,12 +21,12 @@ import ( ) func TestStringComparison(t *testing.T) { - cmp := CompareStringLists([]string{"a", "b", "b", "a"}, []string{"a", "c"}) + cmp := Difference([]string{"a", "b", "b", "a"}, []string{"a", "c"}) assert.Len(t, cmp, 1) assert.NotContains(t, cmp, "a") assert.Contains(t, cmp, "b") - cmp = CompareStringListsInBoth([]string{"a", "a", "b", "c"}, []string{"a", "c", "c"}) + cmp = Intersect([]string{"a", "a", "b", "c"}, []string{"a", "c", "c"}) assert.Len(t, cmp, 2) assert.NotContains(t, cmp, "b") assert.Contains(t, cmp, "a") diff --git a/updater.go b/updater.go index 792e068b..567db2c2 100644 --- a/updater.go +++ b/updater.go @@ -208,7 +208,7 @@ func update(datastore database.Datastore, firstUpdate bool) { namespaces = append(namespaces, ns) } - if err := persistNamespaces(datastore, namespaces); err != nil { + if err := database.PersistNamespacesAndCommit(datastore, namespaces); err != nil { log.WithError(err).Error("Unable to insert namespaces") return } diff --git a/vendor/github.com/deckarep/golang-set/.gitignore b/vendor/github.com/deckarep/golang-set/.gitignore new file mode 100644 index 00000000..00268614 --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/.gitignore @@ -0,0 +1,22 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe diff --git a/vendor/github.com/deckarep/golang-set/.travis.yml b/vendor/github.com/deckarep/golang-set/.travis.yml new file mode 100644 index 00000000..c760d24d --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/.travis.yml @@ -0,0 +1,11 @@ +language: go + +go: + - 1.8 + - 1.9 + - tip + +script: + - go test -race ./... + - go test -bench=. + diff --git a/vendor/github.com/deckarep/golang-set/LICENSE b/vendor/github.com/deckarep/golang-set/LICENSE new file mode 100644 index 00000000..b5768f89 --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/LICENSE @@ -0,0 +1,22 @@ +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/deckarep/golang-set/README.md b/vendor/github.com/deckarep/golang-set/README.md new file mode 100644 index 00000000..c3b50b2c --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/README.md @@ -0,0 +1,95 @@ +[![Build Status](https://travis-ci.org/deckarep/golang-set.svg?branch=master)](https://travis-ci.org/deckarep/golang-set) +[![Go Report Card](https://goreportcard.com/badge/github.com/deckarep/golang-set)](https://goreportcard.com/report/github.com/deckarep/golang-set) +[![GoDoc](https://godoc.org/github.com/deckarep/golang-set?status.svg)](http://godoc.org/github.com/deckarep/golang-set) + +## golang-set + + +The missing set collection for the Go language. Until Go has sets built-in...use this. + +Coming from Python one of the things I miss is the superbly wonderful set collection. This is my attempt to mimic the primary features of the set from Python. +You can of course argue that there is no need for a set in Go, otherwise the creators would have added one to the standard library. To those I say simply ignore this repository +and carry-on and to the rest that find this useful please contribute in helping me make it better by: + +* Helping to make more idiomatic improvements to the code. +* Helping to increase the performance of it. ~~(So far, no attempt has been made, but since it uses a map internally, I expect it to be mostly performant.)~~ +* Helping to make the unit-tests more robust and kick-ass. +* Helping to fill in the [documentation.](http://godoc.org/github.com/deckarep/golang-set) +* Simply offering feedback and suggestions. (Positive, constructive feedback is appreciated.) + +I have to give some credit for helping seed the idea with this post on [stackoverflow.](http://programmers.stackexchange.com/questions/177428/sets-data-structure-in-golang) + +*Update* - as of 3/9/2014, you can use a compile-time generic version of this package in the [gen](http://clipperhouse.github.io/gen/) framework. This framework allows you to use the golang-set in a completely generic and type-safe way by allowing you to generate a supporting .go file based on your custom types. + +## Features (as of 9/22/2014) + +* a CartesianProduct() method has been added with unit-tests: [Read more about the cartesian product](http://en.wikipedia.org/wiki/Cartesian_product) + +## Features (as of 9/15/2014) + +* a PowerSet() method has been added with unit-tests: [Read more about the Power set](http://en.wikipedia.org/wiki/Power_set) + +## Features (as of 4/22/2014) + +* One common interface to both implementations +* Two set implementations to choose from + * a thread-safe implementation designed for concurrent use + * a non-thread-safe implementation designed for performance +* 75 benchmarks for both implementations +* 35 unit tests for both implementations +* 14 concurrent tests for the thread-safe implementation + + + +Please see the unit test file for additional usage examples. The Python set documentation will also do a better job than I can of explaining how a set typically [works.](http://docs.python.org/2/library/sets.html) Please keep in mind +however that the Python set is a built-in type and supports additional features and syntax that make it awesome. + +## Examples but not exhaustive: + +```go +requiredClasses := mapset.NewSet() +requiredClasses.Add("Cooking") +requiredClasses.Add("English") +requiredClasses.Add("Math") +requiredClasses.Add("Biology") + +scienceSlice := []interface{}{"Biology", "Chemistry"} +scienceClasses := mapset.NewSetFromSlice(scienceSlice) + +electiveClasses := mapset.NewSet() +electiveClasses.Add("Welding") +electiveClasses.Add("Music") +electiveClasses.Add("Automotive") + +bonusClasses := mapset.NewSet() +bonusClasses.Add("Go Programming") +bonusClasses.Add("Python Programming") + +//Show me all the available classes I can take +allClasses := requiredClasses.Union(scienceClasses).Union(electiveClasses).Union(bonusClasses) +fmt.Println(allClasses) //Set{Cooking, English, Math, Chemistry, Welding, Biology, Music, Automotive, Go Programming, Python Programming} + + +//Is cooking considered a science class? +fmt.Println(scienceClasses.Contains("Cooking")) //false + +//Show me all classes that are not science classes, since I hate science. +fmt.Println(allClasses.Difference(scienceClasses)) //Set{Music, Automotive, Go Programming, Python Programming, Cooking, English, Math, Welding} + +//Which science classes are also required classes? +fmt.Println(scienceClasses.Intersect(requiredClasses)) //Set{Biology} + +//How many bonus classes do you offer? +fmt.Println(bonusClasses.Cardinality()) //2 + +//Do you have the following classes? Welding, Automotive and English? +fmt.Println(allClasses.IsSuperset(mapset.NewSetFromSlice([]interface{}{"Welding", "Automotive", "English"}))) //true +``` + +Thanks! + +-Ralph + +[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/deckarep/golang-set/trend.png)](https://bitdeli.com/free "Bitdeli Badge") + +[![Analytics](https://ga-beacon.appspot.com/UA-42584447-2/deckarep/golang-set)](https://github.com/igrigorik/ga-beacon) diff --git a/vendor/github.com/deckarep/golang-set/bench_test.go b/vendor/github.com/deckarep/golang-set/bench_test.go new file mode 100644 index 00000000..f893d101 --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/bench_test.go @@ -0,0 +1,674 @@ +package mapset + +import ( + "math/rand" + "testing" +) + +func nrand(n int) []int { + i := make([]int, n) + for ind := range i { + i[ind] = rand.Int() + } + return i +} + +func toInterfaces(i []int) []interface{} { + ifs := make([]interface{}, len(i)) + for ind, v := range i { + ifs[ind] = v + } + return ifs +} + +func benchAdd(b *testing.B, s Set) { + nums := nrand(b.N) + b.ResetTimer() + for _, v := range nums { + s.Add(v) + } +} + +func BenchmarkAddSafe(b *testing.B) { + benchAdd(b, NewSet()) +} + +func BenchmarkAddUnsafe(b *testing.B) { + benchAdd(b, NewThreadUnsafeSet()) +} + +func benchRemove(b *testing.B, s Set) { + nums := nrand(b.N) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for _, v := range nums { + s.Remove(v) + } +} + +func BenchmarkRemoveSafe(b *testing.B) { + benchRemove(b, NewSet()) +} + +func BenchmarkRemoveUnsafe(b *testing.B) { + benchRemove(b, NewThreadUnsafeSet()) +} + +func benchCardinality(b *testing.B, s Set) { + for i := 0; i < b.N; i++ { + s.Cardinality() + } +} + +func BenchmarkCardinalitySafe(b *testing.B) { + benchCardinality(b, NewSet()) +} + +func BenchmarkCardinalityUnsafe(b *testing.B) { + benchCardinality(b, NewThreadUnsafeSet()) +} + +func benchClear(b *testing.B, s Set) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Clear() + } +} + +func BenchmarkClearSafe(b *testing.B) { + benchClear(b, NewSet()) +} + +func BenchmarkClearUnsafe(b *testing.B) { + benchClear(b, NewThreadUnsafeSet()) +} + +func benchClone(b *testing.B, n int, s Set) { + nums := toInterfaces(nrand(n)) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Clone() + } +} + +func BenchmarkClone1Safe(b *testing.B) { + benchClone(b, 1, NewSet()) +} + +func BenchmarkClone1Unsafe(b *testing.B) { + benchClone(b, 1, NewThreadUnsafeSet()) +} + +func BenchmarkClone10Safe(b *testing.B) { + benchClone(b, 10, NewSet()) +} + +func BenchmarkClone10Unsafe(b *testing.B) { + benchClone(b, 10, NewThreadUnsafeSet()) +} + +func BenchmarkClone100Safe(b *testing.B) { + benchClone(b, 100, NewSet()) +} + +func BenchmarkClone100Unsafe(b *testing.B) { + benchClone(b, 100, NewThreadUnsafeSet()) +} + +func benchContains(b *testing.B, n int, s Set) { + nums := toInterfaces(nrand(n)) + for _, v := range nums { + s.Add(v) + } + + nums[n-1] = -1 // Definitely not in s + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Contains(nums...) + } +} + +func BenchmarkContains1Safe(b *testing.B) { + benchContains(b, 1, NewSet()) +} + +func BenchmarkContains1Unsafe(b *testing.B) { + benchContains(b, 1, NewThreadUnsafeSet()) +} + +func BenchmarkContains10Safe(b *testing.B) { + benchContains(b, 10, NewSet()) +} + +func BenchmarkContains10Unsafe(b *testing.B) { + benchContains(b, 10, NewThreadUnsafeSet()) +} + +func BenchmarkContains100Safe(b *testing.B) { + benchContains(b, 100, NewSet()) +} + +func BenchmarkContains100Unsafe(b *testing.B) { + benchContains(b, 100, NewThreadUnsafeSet()) +} + +func benchEqual(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Equal(t) + } +} + +func BenchmarkEqual1Safe(b *testing.B) { + benchEqual(b, 1, NewSet(), NewSet()) +} + +func BenchmarkEqual1Unsafe(b *testing.B) { + benchEqual(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkEqual10Safe(b *testing.B) { + benchEqual(b, 10, NewSet(), NewSet()) +} + +func BenchmarkEqual10Unsafe(b *testing.B) { + benchEqual(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkEqual100Safe(b *testing.B) { + benchEqual(b, 100, NewSet(), NewSet()) +} + +func BenchmarkEqual100Unsafe(b *testing.B) { + benchEqual(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchDifference(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + } + for _, v := range nums[:n/2] { + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Difference(t) + } +} + +func benchIsSubset(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.IsSubset(t) + } +} + +func BenchmarkIsSubset1Safe(b *testing.B) { + benchIsSubset(b, 1, NewSet(), NewSet()) +} + +func BenchmarkIsSubset1Unsafe(b *testing.B) { + benchIsSubset(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsSubset10Safe(b *testing.B) { + benchIsSubset(b, 10, NewSet(), NewSet()) +} + +func BenchmarkIsSubset10Unsafe(b *testing.B) { + benchIsSubset(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsSubset100Safe(b *testing.B) { + benchIsSubset(b, 100, NewSet(), NewSet()) +} + +func BenchmarkIsSubset100Unsafe(b *testing.B) { + benchIsSubset(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchIsSuperset(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.IsSuperset(t) + } +} + +func BenchmarkIsSuperset1Safe(b *testing.B) { + benchIsSuperset(b, 1, NewSet(), NewSet()) +} + +func BenchmarkIsSuperset1Unsafe(b *testing.B) { + benchIsSuperset(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsSuperset10Safe(b *testing.B) { + benchIsSuperset(b, 10, NewSet(), NewSet()) +} + +func BenchmarkIsSuperset10Unsafe(b *testing.B) { + benchIsSuperset(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsSuperset100Safe(b *testing.B) { + benchIsSuperset(b, 100, NewSet(), NewSet()) +} + +func BenchmarkIsSuperset100Unsafe(b *testing.B) { + benchIsSuperset(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchIsProperSubset(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.IsProperSubset(t) + } +} + +func BenchmarkIsProperSubset1Safe(b *testing.B) { + benchIsProperSubset(b, 1, NewSet(), NewSet()) +} + +func BenchmarkIsProperSubset1Unsafe(b *testing.B) { + benchIsProperSubset(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsProperSubset10Safe(b *testing.B) { + benchIsProperSubset(b, 10, NewSet(), NewSet()) +} + +func BenchmarkIsProperSubset10Unsafe(b *testing.B) { + benchIsProperSubset(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsProperSubset100Safe(b *testing.B) { + benchIsProperSubset(b, 100, NewSet(), NewSet()) +} + +func BenchmarkIsProperSubset100Unsafe(b *testing.B) { + benchIsProperSubset(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchIsProperSuperset(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.IsProperSuperset(t) + } +} + +func BenchmarkIsProperSuperset1Safe(b *testing.B) { + benchIsProperSuperset(b, 1, NewSet(), NewSet()) +} + +func BenchmarkIsProperSuperset1Unsafe(b *testing.B) { + benchIsProperSuperset(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsProperSuperset10Safe(b *testing.B) { + benchIsProperSuperset(b, 10, NewSet(), NewSet()) +} + +func BenchmarkIsProperSuperset10Unsafe(b *testing.B) { + benchIsProperSuperset(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIsProperSuperset100Safe(b *testing.B) { + benchIsProperSuperset(b, 100, NewSet(), NewSet()) +} + +func BenchmarkIsProperSuperset100Unsafe(b *testing.B) { + benchIsProperSuperset(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkDifference1Safe(b *testing.B) { + benchDifference(b, 1, NewSet(), NewSet()) +} + +func BenchmarkDifference1Unsafe(b *testing.B) { + benchDifference(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkDifference10Safe(b *testing.B) { + benchDifference(b, 10, NewSet(), NewSet()) +} + +func BenchmarkDifference10Unsafe(b *testing.B) { + benchDifference(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkDifference100Safe(b *testing.B) { + benchDifference(b, 100, NewSet(), NewSet()) +} + +func BenchmarkDifference100Unsafe(b *testing.B) { + benchDifference(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchIntersect(b *testing.B, n int, s, t Set) { + nums := nrand(int(float64(n) * float64(1.5))) + for _, v := range nums[:n] { + s.Add(v) + } + for _, v := range nums[n/2:] { + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Intersect(t) + } +} + +func BenchmarkIntersect1Safe(b *testing.B) { + benchIntersect(b, 1, NewSet(), NewSet()) +} + +func BenchmarkIntersect1Unsafe(b *testing.B) { + benchIntersect(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIntersect10Safe(b *testing.B) { + benchIntersect(b, 10, NewSet(), NewSet()) +} + +func BenchmarkIntersect10Unsafe(b *testing.B) { + benchIntersect(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkIntersect100Safe(b *testing.B) { + benchIntersect(b, 100, NewSet(), NewSet()) +} + +func BenchmarkIntersect100Unsafe(b *testing.B) { + benchIntersect(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchSymmetricDifference(b *testing.B, n int, s, t Set) { + nums := nrand(int(float64(n) * float64(1.5))) + for _, v := range nums[:n] { + s.Add(v) + } + for _, v := range nums[n/2:] { + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.SymmetricDifference(t) + } +} + +func BenchmarkSymmetricDifference1Safe(b *testing.B) { + benchSymmetricDifference(b, 1, NewSet(), NewSet()) +} + +func BenchmarkSymmetricDifference1Unsafe(b *testing.B) { + benchSymmetricDifference(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkSymmetricDifference10Safe(b *testing.B) { + benchSymmetricDifference(b, 10, NewSet(), NewSet()) +} + +func BenchmarkSymmetricDifference10Unsafe(b *testing.B) { + benchSymmetricDifference(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkSymmetricDifference100Safe(b *testing.B) { + benchSymmetricDifference(b, 100, NewSet(), NewSet()) +} + +func BenchmarkSymmetricDifference100Unsafe(b *testing.B) { + benchSymmetricDifference(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchUnion(b *testing.B, n int, s, t Set) { + nums := nrand(n) + for _, v := range nums[:n/2] { + s.Add(v) + } + for _, v := range nums[n/2:] { + t.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Union(t) + } +} + +func BenchmarkUnion1Safe(b *testing.B) { + benchUnion(b, 1, NewSet(), NewSet()) +} + +func BenchmarkUnion1Unsafe(b *testing.B) { + benchUnion(b, 1, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkUnion10Safe(b *testing.B) { + benchUnion(b, 10, NewSet(), NewSet()) +} + +func BenchmarkUnion10Unsafe(b *testing.B) { + benchUnion(b, 10, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func BenchmarkUnion100Safe(b *testing.B) { + benchUnion(b, 100, NewSet(), NewSet()) +} + +func BenchmarkUnion100Unsafe(b *testing.B) { + benchUnion(b, 100, NewThreadUnsafeSet(), NewThreadUnsafeSet()) +} + +func benchEach(b *testing.B, n int, s Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Each(func(elem interface{}) bool { + return false + }) + } +} + +func BenchmarkEach1Safe(b *testing.B) { + benchEach(b, 1, NewSet()) +} + +func BenchmarkEach1Unsafe(b *testing.B) { + benchEach(b, 1, NewThreadUnsafeSet()) +} + +func BenchmarkEach10Safe(b *testing.B) { + benchEach(b, 10, NewSet()) +} + +func BenchmarkEach10Unsafe(b *testing.B) { + benchEach(b, 10, NewThreadUnsafeSet()) +} + +func BenchmarkEach100Safe(b *testing.B) { + benchEach(b, 100, NewSet()) +} + +func BenchmarkEach100Unsafe(b *testing.B) { + benchEach(b, 100, NewThreadUnsafeSet()) +} + +func benchIter(b *testing.B, n int, s Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + c := s.Iter() + for range c { + + } + } +} + +func BenchmarkIter1Safe(b *testing.B) { + benchIter(b, 1, NewSet()) +} + +func BenchmarkIter1Unsafe(b *testing.B) { + benchIter(b, 1, NewThreadUnsafeSet()) +} + +func BenchmarkIter10Safe(b *testing.B) { + benchIter(b, 10, NewSet()) +} + +func BenchmarkIter10Unsafe(b *testing.B) { + benchIter(b, 10, NewThreadUnsafeSet()) +} + +func BenchmarkIter100Safe(b *testing.B) { + benchIter(b, 100, NewSet()) +} + +func BenchmarkIter100Unsafe(b *testing.B) { + benchIter(b, 100, NewThreadUnsafeSet()) +} + +func benchIterator(b *testing.B, n int, s Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + c := s.Iterator().C + for range c { + + } + } +} + +func BenchmarkIterator1Safe(b *testing.B) { + benchIterator(b, 1, NewSet()) +} + +func BenchmarkIterator1Unsafe(b *testing.B) { + benchIterator(b, 1, NewThreadUnsafeSet()) +} + +func BenchmarkIterator10Safe(b *testing.B) { + benchIterator(b, 10, NewSet()) +} + +func BenchmarkIterator10Unsafe(b *testing.B) { + benchIterator(b, 10, NewThreadUnsafeSet()) +} + +func BenchmarkIterator100Safe(b *testing.B) { + benchIterator(b, 100, NewSet()) +} + +func BenchmarkIterator100Unsafe(b *testing.B) { + benchIterator(b, 100, NewThreadUnsafeSet()) +} + +func benchString(b *testing.B, n int, s Set) { + nums := nrand(n) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = s.String() + } +} + +func BenchmarkString1Safe(b *testing.B) { + benchString(b, 1, NewSet()) +} + +func BenchmarkString1Unsafe(b *testing.B) { + benchString(b, 1, NewThreadUnsafeSet()) +} + +func BenchmarkString10Safe(b *testing.B) { + benchString(b, 10, NewSet()) +} + +func BenchmarkString10Unsafe(b *testing.B) { + benchString(b, 10, NewThreadUnsafeSet()) +} + +func BenchmarkString100Safe(b *testing.B) { + benchString(b, 100, NewSet()) +} + +func BenchmarkString100Unsafe(b *testing.B) { + benchString(b, 100, NewThreadUnsafeSet()) +} + +func benchToSlice(b *testing.B, s Set) { + nums := nrand(b.N) + for _, v := range nums { + s.Add(v) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.ToSlice() + } +} + +func BenchmarkToSliceSafe(b *testing.B) { + benchToSlice(b, NewSet()) +} + +func BenchmarkToSliceUnsafe(b *testing.B) { + benchToSlice(b, NewThreadUnsafeSet()) +} diff --git a/vendor/github.com/deckarep/golang-set/iterator.go b/vendor/github.com/deckarep/golang-set/iterator.go new file mode 100644 index 00000000..9dfecade --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/iterator.go @@ -0,0 +1,58 @@ +/* +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package mapset + +// Iterator defines an iterator over a Set, its C channel can be used to range over the Set's +// elements. +type Iterator struct { + C <-chan interface{} + stop chan struct{} +} + +// Stop stops the Iterator, no further elements will be received on C, C will be closed. +func (i *Iterator) Stop() { + // Allows for Stop() to be called multiple times + // (close() panics when called on already closed channel) + defer func() { + recover() + }() + + close(i.stop) + + // Exhaust any remaining elements. + for range i.C { + } +} + +// newIterator returns a new Iterator instance together with its item and stop channels. +func newIterator() (*Iterator, chan<- interface{}, <-chan struct{}) { + itemChan := make(chan interface{}) + stopChan := make(chan struct{}) + return &Iterator{ + C: itemChan, + stop: stopChan, + }, itemChan, stopChan +} diff --git a/vendor/github.com/deckarep/golang-set/iterator_example_test.go b/vendor/github.com/deckarep/golang-set/iterator_example_test.go new file mode 100644 index 00000000..fc4235d7 --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/iterator_example_test.go @@ -0,0 +1,32 @@ +package mapset + +import ( + "fmt" +) + +type YourType struct { + Name string +} + +func ExampleIterator() { + set := NewSetFromSlice([]interface{}{ + &YourType{Name: "Alise"}, + &YourType{Name: "Bob"}, + &YourType{Name: "John"}, + &YourType{Name: "Nick"}, + }) + + var found *YourType + it := set.Iterator() + + for elem := range it.C { + if elem.(*YourType).Name == "John" { + found = elem.(*YourType) + it.Stop() + } + } + + fmt.Printf("Found %+v\n", found) + + // Output: Found &{Name:John} +} diff --git a/vendor/github.com/deckarep/golang-set/set.go b/vendor/github.com/deckarep/golang-set/set.go new file mode 100644 index 00000000..29eb2e5a --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/set.go @@ -0,0 +1,217 @@ +/* +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Package mapset implements a simple and generic set collection. +// Items stored within it are unordered and unique. It supports +// typical set operations: membership testing, intersection, union, +// difference, symmetric difference and cloning. +// +// Package mapset provides two implementations of the Set +// interface. The default implementation is safe for concurrent +// access, but a non-thread-safe implementation is also provided for +// programs that can benefit from the slight speed improvement and +// that can enforce mutual exclusion through other means. +package mapset + +// Set is the primary interface provided by the mapset package. It +// represents an unordered set of data and a large number of +// operations that can be applied to that set. +type Set interface { + // Adds an element to the set. Returns whether + // the item was added. + Add(i interface{}) bool + + // Returns the number of elements in the set. + Cardinality() int + + // Removes all elements from the set, leaving + // the empty set. + Clear() + + // Returns a clone of the set using the same + // implementation, duplicating all keys. + Clone() Set + + // Returns whether the given items + // are all in the set. + Contains(i ...interface{}) bool + + // Returns the difference between this set + // and other. The returned set will contain + // all elements of this set that are not also + // elements of other. + // + // Note that the argument to Difference + // must be of the same type as the receiver + // of the method. Otherwise, Difference will + // panic. + Difference(other Set) Set + + // Determines if two sets are equal to each + // other. If they have the same cardinality + // and contain the same elements, they are + // considered equal. The order in which + // the elements were added is irrelevant. + // + // Note that the argument to Equal must be + // of the same type as the receiver of the + // method. Otherwise, Equal will panic. + Equal(other Set) bool + + // Returns a new set containing only the elements + // that exist only in both sets. + // + // Note that the argument to Intersect + // must be of the same type as the receiver + // of the method. Otherwise, Intersect will + // panic. + Intersect(other Set) Set + + // Determines if every element in this set is in + // the other set but the two sets are not equal. + // + // Note that the argument to IsProperSubset + // must be of the same type as the receiver + // of the method. Otherwise, IsProperSubset + // will panic. + IsProperSubset(other Set) bool + + // Determines if every element in the other set + // is in this set but the two sets are not + // equal. + // + // Note that the argument to IsSuperset + // must be of the same type as the receiver + // of the method. Otherwise, IsSuperset will + // panic. + IsProperSuperset(other Set) bool + + // Determines if every element in this set is in + // the other set. + // + // Note that the argument to IsSubset + // must be of the same type as the receiver + // of the method. Otherwise, IsSubset will + // panic. + IsSubset(other Set) bool + + // Determines if every element in the other set + // is in this set. + // + // Note that the argument to IsSuperset + // must be of the same type as the receiver + // of the method. Otherwise, IsSuperset will + // panic. + IsSuperset(other Set) bool + + // Iterates over elements and executes the passed func against each element. + // If passed func returns true, stop iteration at the time. + Each(func(interface{}) bool) + + // Returns a channel of elements that you can + // range over. + Iter() <-chan interface{} + + // Returns an Iterator object that you can + // use to range over the set. + Iterator() *Iterator + + // Remove a single element from the set. + Remove(i interface{}) + + // Provides a convenient string representation + // of the current state of the set. + String() string + + // Returns a new set with all elements which are + // in either this set or the other set but not in both. + // + // Note that the argument to SymmetricDifference + // must be of the same type as the receiver + // of the method. Otherwise, SymmetricDifference + // will panic. + SymmetricDifference(other Set) Set + + // Returns a new set with all elements in both sets. + // + // Note that the argument to Union must be of the + + // same type as the receiver of the method. + // Otherwise, IsSuperset will panic. + Union(other Set) Set + + // Pop removes and returns an arbitrary item from the set. + Pop() interface{} + + // Returns all subsets of a given set (Power Set). + PowerSet() Set + + // Returns the Cartesian Product of two sets. + CartesianProduct(other Set) Set + + // Returns the members of the set as a slice. + ToSlice() []interface{} +} + +// NewSet creates and returns a reference to an empty set. Operations +// on the resulting set are thread-safe. +func NewSet(s ...interface{}) Set { + set := newThreadSafeSet() + for _, item := range s { + set.Add(item) + } + return &set +} + +// NewSetWith creates and returns a new set with the given elements. +// Operations on the resulting set are thread-safe. +func NewSetWith(elts ...interface{}) Set { + return NewSetFromSlice(elts) +} + +// NewSetFromSlice creates and returns a reference to a set from an +// existing slice. Operations on the resulting set are thread-safe. +func NewSetFromSlice(s []interface{}) Set { + a := NewSet(s...) + return a +} + +// NewThreadUnsafeSet creates and returns a reference to an empty set. +// Operations on the resulting set are not thread-safe. +func NewThreadUnsafeSet() Set { + set := newThreadUnsafeSet() + return &set +} + +// NewThreadUnsafeSetFromSlice creates and returns a reference to a +// set from an existing slice. Operations on the resulting set are +// not thread-safe. +func NewThreadUnsafeSetFromSlice(s []interface{}) Set { + a := NewThreadUnsafeSet() + for _, item := range s { + a.Add(item) + } + return a +} diff --git a/vendor/github.com/deckarep/golang-set/set_test.go b/vendor/github.com/deckarep/golang-set/set_test.go new file mode 100644 index 00000000..6cdf5833 --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/set_test.go @@ -0,0 +1,1200 @@ +/* +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package mapset + +import "testing" + +func makeSet(ints []int) Set { + set := NewSet() + for _, i := range ints { + set.Add(i) + } + return set +} + +func makeUnsafeSet(ints []int) Set { + set := NewThreadUnsafeSet() + for _, i := range ints { + set.Add(i) + } + return set +} + +func assertEqual(a, b Set, t *testing.T) { + if !a.Equal(b) { + t.Errorf("%v != %v\n", a, b) + } +} + +func Test_NewSet(t *testing.T) { + a := NewSet() + if a.Cardinality() != 0 { + t.Error("NewSet should start out as an empty set") + } + + assertEqual(NewSetFromSlice([]interface{}{}), NewSet(), t) + assertEqual(NewSetFromSlice([]interface{}{1}), NewSet(1), t) + assertEqual(NewSetFromSlice([]interface{}{1, 2}), NewSet(1, 2), t) + assertEqual(NewSetFromSlice([]interface{}{"a"}), NewSet("a"), t) + assertEqual(NewSetFromSlice([]interface{}{"a", "b"}), NewSet("a", "b"), t) +} + +func Test_NewUnsafeSet(t *testing.T) { + a := NewThreadUnsafeSet() + + if a.Cardinality() != 0 { + t.Error("NewSet should start out as an empty set") + } +} + +func Test_AddSet(t *testing.T) { + a := makeSet([]int{1, 2, 3}) + + if a.Cardinality() != 3 { + t.Error("AddSet does not have a size of 3 even though 3 items were added to a new set") + } +} + +func Test_AddUnsafeSet(t *testing.T) { + a := makeUnsafeSet([]int{1, 2, 3}) + + if a.Cardinality() != 3 { + t.Error("AddSet does not have a size of 3 even though 3 items were added to a new set") + } +} + +func Test_AddSetNoDuplicate(t *testing.T) { + a := makeSet([]int{7, 5, 3, 7}) + + if a.Cardinality() != 3 { + t.Error("AddSetNoDuplicate set should have 3 elements since 7 is a duplicate") + } + + if !(a.Contains(7) && a.Contains(5) && a.Contains(3)) { + t.Error("AddSetNoDuplicate set should have a 7, 5, and 3 in it.") + } +} + +func Test_AddUnsafeSetNoDuplicate(t *testing.T) { + a := makeUnsafeSet([]int{7, 5, 3, 7}) + + if a.Cardinality() != 3 { + t.Error("AddSetNoDuplicate set should have 3 elements since 7 is a duplicate") + } + + if !(a.Contains(7) && a.Contains(5) && a.Contains(3)) { + t.Error("AddSetNoDuplicate set should have a 7, 5, and 3 in it.") + } +} + +func Test_RemoveSet(t *testing.T) { + a := makeSet([]int{6, 3, 1}) + + a.Remove(3) + + if a.Cardinality() != 2 { + t.Error("RemoveSet should only have 2 items in the set") + } + + if !(a.Contains(6) && a.Contains(1)) { + t.Error("RemoveSet should have only items 6 and 1 in the set") + } + + a.Remove(6) + a.Remove(1) + + if a.Cardinality() != 0 { + t.Error("RemoveSet should be an empty set after removing 6 and 1") + } +} + +func Test_RemoveUnsafeSet(t *testing.T) { + a := makeUnsafeSet([]int{6, 3, 1}) + + a.Remove(3) + + if a.Cardinality() != 2 { + t.Error("RemoveSet should only have 2 items in the set") + } + + if !(a.Contains(6) && a.Contains(1)) { + t.Error("RemoveSet should have only items 6 and 1 in the set") + } + + a.Remove(6) + a.Remove(1) + + if a.Cardinality() != 0 { + t.Error("RemoveSet should be an empty set after removing 6 and 1") + } +} + +func Test_ContainsSet(t *testing.T) { + a := NewSet() + + a.Add(71) + + if !a.Contains(71) { + t.Error("ContainsSet should contain 71") + } + + a.Remove(71) + + if a.Contains(71) { + t.Error("ContainsSet should not contain 71") + } + + a.Add(13) + a.Add(7) + a.Add(1) + + if !(a.Contains(13) && a.Contains(7) && a.Contains(1)) { + t.Error("ContainsSet should contain 13, 7, 1") + } +} + +func Test_ContainsUnsafeSet(t *testing.T) { + a := NewThreadUnsafeSet() + + a.Add(71) + + if !a.Contains(71) { + t.Error("ContainsSet should contain 71") + } + + a.Remove(71) + + if a.Contains(71) { + t.Error("ContainsSet should not contain 71") + } + + a.Add(13) + a.Add(7) + a.Add(1) + + if !(a.Contains(13) && a.Contains(7) && a.Contains(1)) { + t.Error("ContainsSet should contain 13, 7, 1") + } +} + +func Test_ContainsMultipleSet(t *testing.T) { + a := makeSet([]int{8, 6, 7, 5, 3, 0, 9}) + + if !a.Contains(8, 6, 7, 5, 3, 0, 9) { + t.Error("ContainsAll should contain Jenny's phone number") + } + + if a.Contains(8, 6, 11, 5, 3, 0, 9) { + t.Error("ContainsAll should not have all of these numbers") + } +} + +func Test_ContainsMultipleUnsafeSet(t *testing.T) { + a := makeUnsafeSet([]int{8, 6, 7, 5, 3, 0, 9}) + + if !a.Contains(8, 6, 7, 5, 3, 0, 9) { + t.Error("ContainsAll should contain Jenny's phone number") + } + + if a.Contains(8, 6, 11, 5, 3, 0, 9) { + t.Error("ContainsAll should not have all of these numbers") + } +} + +func Test_ClearSet(t *testing.T) { + a := makeSet([]int{2, 5, 9, 10}) + + a.Clear() + + if a.Cardinality() != 0 { + t.Error("ClearSet should be an empty set") + } +} + +func Test_ClearUnsafeSet(t *testing.T) { + a := makeUnsafeSet([]int{2, 5, 9, 10}) + + a.Clear() + + if a.Cardinality() != 0 { + t.Error("ClearSet should be an empty set") + } +} + +func Test_CardinalitySet(t *testing.T) { + a := NewSet() + + if a.Cardinality() != 0 { + t.Error("set should be an empty set") + } + + a.Add(1) + + if a.Cardinality() != 1 { + t.Error("set should have a size of 1") + } + + a.Remove(1) + + if a.Cardinality() != 0 { + t.Error("set should be an empty set") + } + + a.Add(9) + + if a.Cardinality() != 1 { + t.Error("set should have a size of 1") + } + + a.Clear() + + if a.Cardinality() != 0 { + t.Error("set should have a size of 1") + } +} + +func Test_CardinalityUnsafeSet(t *testing.T) { + a := NewThreadUnsafeSet() + + if a.Cardinality() != 0 { + t.Error("set should be an empty set") + } + + a.Add(1) + + if a.Cardinality() != 1 { + t.Error("set should have a size of 1") + } + + a.Remove(1) + + if a.Cardinality() != 0 { + t.Error("set should be an empty set") + } + + a.Add(9) + + if a.Cardinality() != 1 { + t.Error("set should have a size of 1") + } + + a.Clear() + + if a.Cardinality() != 0 { + t.Error("set should have a size of 1") + } +} + +func Test_SetIsSubset(t *testing.T) { + a := makeSet([]int{1, 2, 3, 5, 7}) + + b := NewSet() + b.Add(3) + b.Add(5) + b.Add(7) + + if !b.IsSubset(a) { + t.Error("set b should be a subset of set a") + } + + b.Add(72) + + if b.IsSubset(a) { + t.Error("set b should not be a subset of set a because it contains 72 which is not in the set of a") + } +} + +func Test_SetIsProperSubset(t *testing.T) { + a := makeSet([]int{1, 2, 3, 5, 7}) + b := makeSet([]int{7, 5, 3, 2, 1}) + + if !a.IsSubset(b) { + t.Error("set a should be a subset of set b") + } + if a.IsProperSubset(b) { + t.Error("set a should not be a proper subset of set b (they're equal)") + } + + b.Add(72) + + if !a.IsSubset(b) { + t.Error("set a should be a subset of set b") + } + if !a.IsProperSubset(b) { + t.Error("set a should be a proper subset of set b") + } +} + +func Test_UnsafeSetIsSubset(t *testing.T) { + a := makeUnsafeSet([]int{1, 2, 3, 5, 7}) + + b := NewThreadUnsafeSet() + b.Add(3) + b.Add(5) + b.Add(7) + + if !b.IsSubset(a) { + t.Error("set b should be a subset of set a") + } + + b.Add(72) + + if b.IsSubset(a) { + t.Error("set b should not be a subset of set a because it contains 72 which is not in the set of a") + } +} + +func Test_UnsafeSetIsProperSubset(t *testing.T) { + a := makeUnsafeSet([]int{1, 2, 3, 5, 7}) + b := NewThreadUnsafeSet() + b.Add(7) + b.Add(1) + b.Add(5) + b.Add(3) + b.Add(2) + + if !a.IsSubset(b) { + t.Error("set a should be a subset of set b") + } + if a.IsProperSubset(b) { + t.Error("set a should not be a proper subset of set b (they're equal)") + } + + b.Add(72) + + if !a.IsSubset(b) { + t.Error("set a should be a subset of set b") + } + if !a.IsProperSubset(b) { + t.Error("set a should be a proper subset of set b because set b has 72") + } +} + +func Test_SetIsSuperset(t *testing.T) { + a := NewSet() + a.Add(9) + a.Add(5) + a.Add(2) + a.Add(1) + a.Add(11) + + b := NewSet() + b.Add(5) + b.Add(2) + b.Add(11) + + if !a.IsSuperset(b) { + t.Error("set a should be a superset of set b") + } + + b.Add(42) + + if a.IsSuperset(b) { + t.Error("set a should not be a superset of set b because set b has a 42") + } +} + +func Test_SetIsProperSuperset(t *testing.T) { + a := NewSet() + a.Add(5) + a.Add(2) + a.Add(11) + + b := NewSet() + b.Add(2) + b.Add(5) + b.Add(11) + + if !a.IsSuperset(b) { + t.Error("set a should be a superset of set b") + } + if a.IsProperSuperset(b) { + t.Error("set a should not be a proper superset of set b (they're equal)") + } + + a.Add(9) + + if !a.IsSuperset(b) { + t.Error("set a should be a superset of set b") + } + if !a.IsProperSuperset(b) { + t.Error("set a not be a proper superset of set b because set a has a 9") + } + + b.Add(42) + + if a.IsSuperset(b) { + t.Error("set a should not be a superset of set b because set b has a 42") + } + if a.IsProperSuperset(b) { + t.Error("set a should not be a proper superset of set b because set b has a 42") + } +} + +func Test_UnsafeSetIsSuperset(t *testing.T) { + a := NewThreadUnsafeSet() + a.Add(9) + a.Add(5) + a.Add(2) + a.Add(1) + a.Add(11) + + b := NewThreadUnsafeSet() + b.Add(5) + b.Add(2) + b.Add(11) + + if !a.IsSuperset(b) { + t.Error("set a should be a superset of set b") + } + + b.Add(42) + + if a.IsSuperset(b) { + t.Error("set a should not be a superset of set b because set a has a 42") + } +} + +func Test_UnsafeSetIsProperSuperset(t *testing.T) { + a := NewThreadUnsafeSet() + a.Add(5) + a.Add(2) + a.Add(11) + + b := NewThreadUnsafeSet() + b.Add(2) + b.Add(5) + b.Add(11) + + if !a.IsSuperset(b) { + t.Error("set a should be a superset of set b") + } + if a.IsProperSuperset(b) { + t.Error("set a should not be a proper superset of set b (they're equal)") + } + + a.Add(9) + + if !a.IsSuperset(b) { + t.Error("set a should be a superset of set b") + } + if !a.IsProperSuperset(b) { + t.Error("set a not be a proper superset of set b because set a has a 9") + } + + b.Add(42) + + if a.IsSuperset(b) { + t.Error("set a should not be a superset of set b because set b has a 42") + } + if a.IsProperSuperset(b) { + t.Error("set a should not be a proper superset of set b because set b has a 42") + } +} + +func Test_SetUnion(t *testing.T) { + a := NewSet() + + b := NewSet() + b.Add(1) + b.Add(2) + b.Add(3) + b.Add(4) + b.Add(5) + + c := a.Union(b) + + if c.Cardinality() != 5 { + t.Error("set c is unioned with an empty set and therefore should have 5 elements in it") + } + + d := NewSet() + d.Add(10) + d.Add(14) + d.Add(0) + + e := c.Union(d) + if e.Cardinality() != 8 { + t.Error("set e should should have 8 elements in it after being unioned with set c to d") + } + + f := NewSet() + f.Add(14) + f.Add(3) + + g := f.Union(e) + if g.Cardinality() != 8 { + t.Error("set g should still have 8 elements in it after being unioned with set f that has duplicates") + } +} + +func Test_UnsafeSetUnion(t *testing.T) { + a := NewThreadUnsafeSet() + + b := NewThreadUnsafeSet() + b.Add(1) + b.Add(2) + b.Add(3) + b.Add(4) + b.Add(5) + + c := a.Union(b) + + if c.Cardinality() != 5 { + t.Error("set c is unioned with an empty set and therefore should have 5 elements in it") + } + + d := NewThreadUnsafeSet() + d.Add(10) + d.Add(14) + d.Add(0) + + e := c.Union(d) + if e.Cardinality() != 8 { + t.Error("set e should should have 8 elements in it after being unioned with set c to d") + } + + f := NewThreadUnsafeSet() + f.Add(14) + f.Add(3) + + g := f.Union(e) + if g.Cardinality() != 8 { + t.Error("set g should still have 8 elements in it after being unioned with set f that has duplicates") + } +} + +func Test_SetIntersect(t *testing.T) { + a := NewSet() + a.Add(1) + a.Add(3) + a.Add(5) + + b := NewSet() + a.Add(2) + a.Add(4) + a.Add(6) + + c := a.Intersect(b) + + if c.Cardinality() != 0 { + t.Error("set c should be the empty set because there is no common items to intersect") + } + + a.Add(10) + b.Add(10) + + d := a.Intersect(b) + + if !(d.Cardinality() == 1 && d.Contains(10)) { + t.Error("set d should have a size of 1 and contain the item 10") + } +} + +func Test_UnsafeSetIntersect(t *testing.T) { + a := NewThreadUnsafeSet() + a.Add(1) + a.Add(3) + a.Add(5) + + b := NewThreadUnsafeSet() + a.Add(2) + a.Add(4) + a.Add(6) + + c := a.Intersect(b) + + if c.Cardinality() != 0 { + t.Error("set c should be the empty set because there is no common items to intersect") + } + + a.Add(10) + b.Add(10) + + d := a.Intersect(b) + + if !(d.Cardinality() == 1 && d.Contains(10)) { + t.Error("set d should have a size of 1 and contain the item 10") + } +} + +func Test_SetDifference(t *testing.T) { + a := NewSet() + a.Add(1) + a.Add(2) + a.Add(3) + + b := NewSet() + b.Add(1) + b.Add(3) + b.Add(4) + b.Add(5) + b.Add(6) + b.Add(99) + + c := a.Difference(b) + + if !(c.Cardinality() == 1 && c.Contains(2)) { + t.Error("the difference of set a to b is the set of 1 item: 2") + } +} + +func Test_UnsafeSetDifference(t *testing.T) { + a := NewThreadUnsafeSet() + a.Add(1) + a.Add(2) + a.Add(3) + + b := NewThreadUnsafeSet() + b.Add(1) + b.Add(3) + b.Add(4) + b.Add(5) + b.Add(6) + b.Add(99) + + c := a.Difference(b) + + if !(c.Cardinality() == 1 && c.Contains(2)) { + t.Error("the difference of set a to b is the set of 1 item: 2") + } +} + +func Test_SetSymmetricDifference(t *testing.T) { + a := NewSet() + a.Add(1) + a.Add(2) + a.Add(3) + a.Add(45) + + b := NewSet() + b.Add(1) + b.Add(3) + b.Add(4) + b.Add(5) + b.Add(6) + b.Add(99) + + c := a.SymmetricDifference(b) + + if !(c.Cardinality() == 6 && c.Contains(2) && c.Contains(45) && c.Contains(4) && c.Contains(5) && c.Contains(6) && c.Contains(99)) { + t.Error("the symmetric difference of set a to b is the set of 6 items: 2, 45, 4, 5, 6, 99") + } +} + +func Test_UnsafeSetSymmetricDifference(t *testing.T) { + a := NewThreadUnsafeSet() + a.Add(1) + a.Add(2) + a.Add(3) + a.Add(45) + + b := NewThreadUnsafeSet() + b.Add(1) + b.Add(3) + b.Add(4) + b.Add(5) + b.Add(6) + b.Add(99) + + c := a.SymmetricDifference(b) + + if !(c.Cardinality() == 6 && c.Contains(2) && c.Contains(45) && c.Contains(4) && c.Contains(5) && c.Contains(6) && c.Contains(99)) { + t.Error("the symmetric difference of set a to b is the set of 6 items: 2, 45, 4, 5, 6, 99") + } +} + +func Test_SetEqual(t *testing.T) { + a := NewSet() + b := NewSet() + + if !a.Equal(b) { + t.Error("Both a and b are empty sets, and should be equal") + } + + a.Add(10) + + if a.Equal(b) { + t.Error("a should not be equal to b because b is empty and a has item 1 in it") + } + + b.Add(10) + + if !a.Equal(b) { + t.Error("a is now equal again to b because both have the item 10 in them") + } + + b.Add(8) + b.Add(3) + b.Add(47) + + if a.Equal(b) { + t.Error("b has 3 more elements in it so therefore should not be equal to a") + } + + a.Add(8) + a.Add(3) + a.Add(47) + + if !a.Equal(b) { + t.Error("a and b should be equal with the same number of elements") + } +} + +func Test_UnsafeSetEqual(t *testing.T) { + a := NewThreadUnsafeSet() + b := NewThreadUnsafeSet() + + if !a.Equal(b) { + t.Error("Both a and b are empty sets, and should be equal") + } + + a.Add(10) + + if a.Equal(b) { + t.Error("a should not be equal to b because b is empty and a has item 1 in it") + } + + b.Add(10) + + if !a.Equal(b) { + t.Error("a is now equal again to b because both have the item 10 in them") + } + + b.Add(8) + b.Add(3) + b.Add(47) + + if a.Equal(b) { + t.Error("b has 3 more elements in it so therefore should not be equal to a") + } + + a.Add(8) + a.Add(3) + a.Add(47) + + if !a.Equal(b) { + t.Error("a and b should be equal with the same number of elements") + } +} + +func Test_SetClone(t *testing.T) { + a := NewSet() + a.Add(1) + a.Add(2) + + b := a.Clone() + + if !a.Equal(b) { + t.Error("Clones should be equal") + } + + a.Add(3) + if a.Equal(b) { + t.Error("a contains one more element, they should not be equal") + } + + c := a.Clone() + c.Remove(1) + + if a.Equal(c) { + t.Error("C contains one element less, they should not be equal") + } +} + +func Test_UnsafeSetClone(t *testing.T) { + a := NewThreadUnsafeSet() + a.Add(1) + a.Add(2) + + b := a.Clone() + + if !a.Equal(b) { + t.Error("Clones should be equal") + } + + a.Add(3) + if a.Equal(b) { + t.Error("a contains one more element, they should not be equal") + } + + c := a.Clone() + c.Remove(1) + + if a.Equal(c) { + t.Error("C contains one element less, they should not be equal") + } +} + +func Test_Each(t *testing.T) { + a := NewSet() + + a.Add("Z") + a.Add("Y") + a.Add("X") + a.Add("W") + + b := NewSet() + a.Each(func(elem interface{}) bool { + b.Add(elem) + return false + }) + + if !a.Equal(b) { + t.Error("The sets are not equal after iterating (Each) through the first set") + } + + var count int + a.Each(func(elem interface{}) bool { + if count == 2 { + return true + } + count++ + return false + }) + if count != 2 { + t.Error("Iteration should stop on the way") + } +} + +func Test_Iter(t *testing.T) { + a := NewSet() + + a.Add("Z") + a.Add("Y") + a.Add("X") + a.Add("W") + + b := NewSet() + for val := range a.Iter() { + b.Add(val) + } + + if !a.Equal(b) { + t.Error("The sets are not equal after iterating (Iter) through the first set") + } +} + +func Test_UnsafeIter(t *testing.T) { + a := NewThreadUnsafeSet() + + a.Add("Z") + a.Add("Y") + a.Add("X") + a.Add("W") + + b := NewThreadUnsafeSet() + for val := range a.Iter() { + b.Add(val) + } + + if !a.Equal(b) { + t.Error("The sets are not equal after iterating (Iter) through the first set") + } +} + +func Test_Iterator(t *testing.T) { + a := NewSet() + + a.Add("Z") + a.Add("Y") + a.Add("X") + a.Add("W") + + b := NewSet() + for val := range a.Iterator().C { + b.Add(val) + } + + if !a.Equal(b) { + t.Error("The sets are not equal after iterating (Iterator) through the first set") + } +} + +func Test_UnsafeIterator(t *testing.T) { + a := NewThreadUnsafeSet() + + a.Add("Z") + a.Add("Y") + a.Add("X") + a.Add("W") + + b := NewThreadUnsafeSet() + for val := range a.Iterator().C { + b.Add(val) + } + + if !a.Equal(b) { + t.Error("The sets are not equal after iterating (Iterator) through the first set") + } +} + +func Test_IteratorStop(t *testing.T) { + a := NewSet() + + a.Add("Z") + a.Add("Y") + a.Add("X") + a.Add("W") + + it := a.Iterator() + it.Stop() + for range it.C { + t.Error("The iterating (Iterator) did not stop after Stop() has been called") + } +} + +func Test_PopSafe(t *testing.T) { + a := NewSet() + + a.Add("a") + a.Add("b") + a.Add("c") + a.Add("d") + + captureSet := NewSet() + captureSet.Add(a.Pop()) + captureSet.Add(a.Pop()) + captureSet.Add(a.Pop()) + captureSet.Add(a.Pop()) + finalNil := a.Pop() + + if captureSet.Cardinality() != 4 { + t.Error("unexpected captureSet cardinality; should be 4") + } + + if a.Cardinality() != 0 { + t.Error("unepxected a cardinality; should be zero") + } + + if !captureSet.Contains("c", "a", "d", "b") { + t.Error("unexpected result set; should be a,b,c,d (any order is fine") + } + + if finalNil != nil { + t.Error("when original set is empty, further pops should result in nil") + } +} + +func Test_PopUnsafe(t *testing.T) { + a := NewThreadUnsafeSet() + + a.Add("a") + a.Add("b") + a.Add("c") + a.Add("d") + + captureSet := NewThreadUnsafeSet() + captureSet.Add(a.Pop()) + captureSet.Add(a.Pop()) + captureSet.Add(a.Pop()) + captureSet.Add(a.Pop()) + finalNil := a.Pop() + + if captureSet.Cardinality() != 4 { + t.Error("unexpected captureSet cardinality; should be 4") + } + + if a.Cardinality() != 0 { + t.Error("unepxected a cardinality; should be zero") + } + + if !captureSet.Contains("c", "a", "d", "b") { + t.Error("unexpected result set; should be a,b,c,d (any order is fine") + } + + if finalNil != nil { + t.Error("when original set is empty, further pops should result in nil") + } +} + +func Test_PowerSet(t *testing.T) { + a := NewThreadUnsafeSet() + + a.Add(1) + a.Add("delta") + a.Add("chi") + a.Add(4) + + b := a.PowerSet() + if b.Cardinality() != 16 { + t.Error("unexpected PowerSet cardinality") + } +} + +func Test_PowerSetThreadSafe(t *testing.T) { + set := NewSet().PowerSet() + _, setIsThreadSafe := set.(*threadSafeSet) + if !setIsThreadSafe { + t.Error("result of PowerSet should be thread safe") + } + + subset := set.Pop() + _, subsetIsThreadSafe := subset.(*threadSafeSet) + if !subsetIsThreadSafe { + t.Error("subsets in PowerSet result should be thread safe") + } +} + +func Test_EmptySetProperties(t *testing.T) { + empty := NewSet() + + a := NewSet() + a.Add(1) + a.Add("foo") + a.Add("bar") + + b := NewSet() + b.Add("one") + b.Add("two") + b.Add(3) + b.Add(4) + + if !empty.IsSubset(a) || !empty.IsSubset(b) { + t.Error("The empty set is supposed to be a subset of all sets") + } + + if !a.IsSuperset(empty) || !b.IsSuperset(empty) { + t.Error("All sets are supposed to be a superset of the empty set") + } + + if !empty.IsSubset(empty) || !empty.IsSuperset(empty) { + t.Error("The empty set is supposed to be a subset and a superset of itself") + } + + c := a.Union(empty) + if !c.Equal(a) { + t.Error("The union of any set with the empty set is supposed to be equal to itself") + } + + c = a.Intersect(empty) + if !c.Equal(empty) { + t.Error("The intesection of any set with the empty set is supposed to be the empty set") + } + + c = a.CartesianProduct(empty) + if c.Cardinality() != 0 { + t.Error("Cartesian product of any set and the empty set must be the empty set") + } + + if empty.Cardinality() != 0 { + t.Error("Cardinality of the empty set is supposed to be zero") + } + + c = empty.PowerSet() + if c.Cardinality() != 1 { + t.Error("Cardinality of the power set of the empty set is supposed to be one { {} }") + } +} + +func Test_CartesianProduct(t *testing.T) { + a := NewThreadUnsafeSet() + b := NewThreadUnsafeSet() + empty := NewThreadUnsafeSet() + + a.Add(1) + a.Add(2) + a.Add(3) + + b.Add("one") + b.Add("two") + b.Add("three") + b.Add("alpha") + b.Add("gamma") + + c := a.CartesianProduct(b) + d := b.CartesianProduct(a) + + if c.Cardinality() != d.Cardinality() { + t.Error("Cardinality of AxB must be equal to BxA") + } + + if c.Cardinality() != (a.Cardinality() * b.Cardinality()) { + t.Error("Unexpected cardinality for cartesian product set") + } + + c = a.CartesianProduct(empty) + d = empty.CartesianProduct(b) + + if c.Cardinality() != 0 || d.Cardinality() != 0 { + t.Error("Cartesian product of any set and the empty set Ax0 || 0xA must be the empty set") + } +} + +func Test_ToSliceUnthreadsafe(t *testing.T) { + s := makeUnsafeSet([]int{1, 2, 3}) + setAsSlice := s.ToSlice() + if len(setAsSlice) != s.Cardinality() { + t.Errorf("Set length is incorrect: %v", len(setAsSlice)) + } + + for _, i := range setAsSlice { + if !s.Contains(i) { + t.Errorf("Set is missing element: %v", i) + } + } +} + +func Test_Example(t *testing.T) { + /* + requiredClasses := NewSet() + requiredClasses.Add("Cooking") + requiredClasses.Add("English") + requiredClasses.Add("Math") + requiredClasses.Add("Biology") + + scienceSlice := []interface{}{"Biology", "Chemistry"} + scienceClasses := NewSetFromSlice(scienceSlice) + + electiveClasses := NewSet() + electiveClasses.Add("Welding") + electiveClasses.Add("Music") + electiveClasses.Add("Automotive") + + bonusClasses := NewSet() + bonusClasses.Add("Go Programming") + bonusClasses.Add("Python Programming") + + //Show me all the available classes I can take + allClasses := requiredClasses.Union(scienceClasses).Union(electiveClasses).Union(bonusClasses) + fmt.Println(allClasses) //Set{English, Chemistry, Automotive, Cooking, Math, Biology, Welding, Music, Go Programming} + + //Is cooking considered a science class? + fmt.Println(scienceClasses.Contains("Cooking")) //false + + //Show me all classes that are not science classes, since I hate science. + fmt.Println(allClasses.Difference(scienceClasses)) //Set{English, Automotive, Cooking, Math, Welding, Music, Go Programming} + + //Which science classes are also required classes? + fmt.Println(scienceClasses.Intersect(requiredClasses)) //Set{Biology} + + //How many bonus classes do you offer? + fmt.Println(bonusClasses.Cardinality()) //2 + + //Do you have the following classes? Welding, Automotive and English? + fmt.Println(allClasses.ContainsAll("Welding", "Automotive", "English")) + */ +} diff --git a/vendor/github.com/deckarep/golang-set/threadsafe.go b/vendor/github.com/deckarep/golang-set/threadsafe.go new file mode 100644 index 00000000..269b4ab0 --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/threadsafe.go @@ -0,0 +1,283 @@ +/* +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package mapset + +import "sync" + +type threadSafeSet struct { + s threadUnsafeSet + sync.RWMutex +} + +func newThreadSafeSet() threadSafeSet { + return threadSafeSet{s: newThreadUnsafeSet()} +} + +func (set *threadSafeSet) Add(i interface{}) bool { + set.Lock() + ret := set.s.Add(i) + set.Unlock() + return ret +} + +func (set *threadSafeSet) Contains(i ...interface{}) bool { + set.RLock() + ret := set.s.Contains(i...) + set.RUnlock() + return ret +} + +func (set *threadSafeSet) IsSubset(other Set) bool { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + ret := set.s.IsSubset(&o.s) + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) IsProperSubset(other Set) bool { + o := other.(*threadSafeSet) + + set.RLock() + defer set.RUnlock() + o.RLock() + defer o.RUnlock() + + return set.s.IsProperSubset(&o.s) +} + +func (set *threadSafeSet) IsSuperset(other Set) bool { + return other.IsSubset(set) +} + +func (set *threadSafeSet) IsProperSuperset(other Set) bool { + return other.IsProperSubset(set) +} + +func (set *threadSafeSet) Union(other Set) Set { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + unsafeUnion := set.s.Union(&o.s).(*threadUnsafeSet) + ret := &threadSafeSet{s: *unsafeUnion} + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) Intersect(other Set) Set { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + unsafeIntersection := set.s.Intersect(&o.s).(*threadUnsafeSet) + ret := &threadSafeSet{s: *unsafeIntersection} + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) Difference(other Set) Set { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + unsafeDifference := set.s.Difference(&o.s).(*threadUnsafeSet) + ret := &threadSafeSet{s: *unsafeDifference} + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) SymmetricDifference(other Set) Set { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + unsafeDifference := set.s.SymmetricDifference(&o.s).(*threadUnsafeSet) + ret := &threadSafeSet{s: *unsafeDifference} + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) Clear() { + set.Lock() + set.s = newThreadUnsafeSet() + set.Unlock() +} + +func (set *threadSafeSet) Remove(i interface{}) { + set.Lock() + delete(set.s, i) + set.Unlock() +} + +func (set *threadSafeSet) Cardinality() int { + set.RLock() + defer set.RUnlock() + return len(set.s) +} + +func (set *threadSafeSet) Each(cb func(interface{}) bool) { + set.RLock() + for elem := range set.s { + if cb(elem) { + break + } + } + set.RUnlock() +} + +func (set *threadSafeSet) Iter() <-chan interface{} { + ch := make(chan interface{}) + go func() { + set.RLock() + + for elem := range set.s { + ch <- elem + } + close(ch) + set.RUnlock() + }() + + return ch +} + +func (set *threadSafeSet) Iterator() *Iterator { + iterator, ch, stopCh := newIterator() + + go func() { + set.RLock() + L: + for elem := range set.s { + select { + case <-stopCh: + break L + case ch <- elem: + } + } + close(ch) + set.RUnlock() + }() + + return iterator +} + +func (set *threadSafeSet) Equal(other Set) bool { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + ret := set.s.Equal(&o.s) + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) Clone() Set { + set.RLock() + + unsafeClone := set.s.Clone().(*threadUnsafeSet) + ret := &threadSafeSet{s: *unsafeClone} + set.RUnlock() + return ret +} + +func (set *threadSafeSet) String() string { + set.RLock() + ret := set.s.String() + set.RUnlock() + return ret +} + +func (set *threadSafeSet) PowerSet() Set { + set.RLock() + unsafePowerSet := set.s.PowerSet().(*threadUnsafeSet) + set.RUnlock() + + ret := &threadSafeSet{s: newThreadUnsafeSet()} + for subset := range unsafePowerSet.Iter() { + unsafeSubset := subset.(*threadUnsafeSet) + ret.Add(&threadSafeSet{s: *unsafeSubset}) + } + return ret +} + +func (set *threadSafeSet) Pop() interface{} { + set.Lock() + defer set.Unlock() + return set.s.Pop() +} + +func (set *threadSafeSet) CartesianProduct(other Set) Set { + o := other.(*threadSafeSet) + + set.RLock() + o.RLock() + + unsafeCartProduct := set.s.CartesianProduct(&o.s).(*threadUnsafeSet) + ret := &threadSafeSet{s: *unsafeCartProduct} + set.RUnlock() + o.RUnlock() + return ret +} + +func (set *threadSafeSet) ToSlice() []interface{} { + keys := make([]interface{}, 0, set.Cardinality()) + set.RLock() + for elem := range set.s { + keys = append(keys, elem) + } + set.RUnlock() + return keys +} + +func (set *threadSafeSet) MarshalJSON() ([]byte, error) { + set.RLock() + b, err := set.s.MarshalJSON() + set.RUnlock() + + return b, err +} + +func (set *threadSafeSet) UnmarshalJSON(p []byte) error { + set.RLock() + err := set.s.UnmarshalJSON(p) + set.RUnlock() + + return err +} diff --git a/vendor/github.com/deckarep/golang-set/threadsafe_test.go b/vendor/github.com/deckarep/golang-set/threadsafe_test.go new file mode 100644 index 00000000..5c32fcbd --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/threadsafe_test.go @@ -0,0 +1,524 @@ +/* +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package mapset + +import ( + "encoding/json" + "math/rand" + "runtime" + "sync" + "sync/atomic" + "testing" +) + +const N = 1000 + +func Test_AddConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + + var wg sync.WaitGroup + wg.Add(len(ints)) + for i := 0; i < len(ints); i++ { + go func(i int) { + s.Add(i) + wg.Done() + }(i) + } + + wg.Wait() + for _, i := range ints { + if !s.Contains(i) { + t.Errorf("Set is missing element: %v", i) + } + } +} + +func Test_CardinalityConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + elems := s.Cardinality() + for i := 0; i < N; i++ { + newElems := s.Cardinality() + if newElems < elems { + t.Errorf("Cardinality shrunk from %v to %v", elems, newElems) + } + } + wg.Done() + }() + + for i := 0; i < N; i++ { + s.Add(rand.Int()) + } + wg.Wait() +} + +func Test_ClearConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + + var wg sync.WaitGroup + wg.Add(len(ints)) + for i := 0; i < len(ints); i++ { + go func() { + s.Clear() + wg.Done() + }() + go func(i int) { + s.Add(i) + }(i) + } + + wg.Wait() +} + +func Test_CloneConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + + for _, v := range ints { + s.Add(v) + } + + var wg sync.WaitGroup + wg.Add(len(ints)) + for i := range ints { + go func(i int) { + s.Remove(i) + wg.Done() + }(i) + } + + s.Clone() +} + +func Test_ContainsConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + interfaces := make([]interface{}, 0) + for _, v := range ints { + s.Add(v) + interfaces = append(interfaces, v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.Contains(interfaces...) + wg.Done() + }() + } + wg.Wait() +} + +func Test_DifferenceConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.Difference(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_EqualConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.Equal(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_IntersectConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.Intersect(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_IsSubsetConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.IsSubset(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_IsProperSubsetConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.IsProperSubset(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_IsSupersetConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.IsSuperset(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_IsProperSupersetConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.IsProperSuperset(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_EachConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + concurrent := 10 + + s := NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + } + + var count int64 + wg := new(sync.WaitGroup) + wg.Add(concurrent) + for n := 0; n < concurrent; n++ { + go func() { + defer wg.Done() + s.Each(func(elem interface{}) bool { + atomic.AddInt64(&count, 1) + return false + }) + }() + } + wg.Wait() + + if count != int64(N*concurrent) { + t.Errorf("%v != %v", count, int64(N*concurrent)) + } +} + +func Test_IterConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + } + + cs := make([]<-chan interface{}, 0) + for range ints { + cs = append(cs, s.Iter()) + } + + c := make(chan interface{}) + go func() { + for n := 0; n < len(ints)*N; { + for _, d := range cs { + select { + case <-d: + n++ + c <- nil + default: + } + } + } + close(c) + }() + + for range c { + } +} + +func Test_RemoveConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + } + + var wg sync.WaitGroup + wg.Add(len(ints)) + for _, v := range ints { + go func(i int) { + s.Remove(i) + wg.Done() + }(v) + } + wg.Wait() + + if s.Cardinality() != 0 { + t.Errorf("Expected cardinality 0; got %v", s.Cardinality()) + } +} + +func Test_StringConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + } + + var wg sync.WaitGroup + wg.Add(len(ints)) + for range ints { + go func() { + _ = s.String() + wg.Done() + }() + } + wg.Wait() +} + +func Test_SymmetricDifferenceConcurrent(t *testing.T) { + runtime.GOMAXPROCS(2) + + s, ss := NewSet(), NewSet() + ints := rand.Perm(N) + for _, v := range ints { + s.Add(v) + ss.Add(v) + } + + var wg sync.WaitGroup + for range ints { + wg.Add(1) + go func() { + s.SymmetricDifference(ss) + wg.Done() + }() + } + wg.Wait() +} + +func Test_ToSlice(t *testing.T) { + runtime.GOMAXPROCS(2) + + s := NewSet() + ints := rand.Perm(N) + + var wg sync.WaitGroup + wg.Add(len(ints)) + for i := 0; i < len(ints); i++ { + go func(i int) { + s.Add(i) + wg.Done() + }(i) + } + + wg.Wait() + setAsSlice := s.ToSlice() + if len(setAsSlice) != s.Cardinality() { + t.Errorf("Set length is incorrect: %v", len(setAsSlice)) + } + + for _, i := range setAsSlice { + if !s.Contains(i) { + t.Errorf("Set is missing element: %v", i) + } + } +} + +// Test_ToSliceDeadlock - fixes issue: https://github.com/deckarep/golang-set/issues/36 +// This code reveals the deadlock however it doesn't happen consistently. +func Test_ToSliceDeadlock(t *testing.T) { + runtime.GOMAXPROCS(2) + + var wg sync.WaitGroup + set := NewSet() + workers := 10 + wg.Add(workers) + for i := 1; i <= workers; i++ { + go func() { + for j := 0; j < 1000; j++ { + set.Add(1) + set.ToSlice() + } + wg.Done() + }() + } + wg.Wait() +} + +func Test_UnmarshalJSON(t *testing.T) { + s := []byte(`["test", 1, 2, 3, ["4,5,6"]]`) + expected := NewSetFromSlice( + []interface{}{ + json.Number("1"), + json.Number("2"), + json.Number("3"), + "test", + }, + ) + actual := NewSet() + err := json.Unmarshal(s, actual) + if err != nil { + t.Errorf("Error should be nil: %v", err) + } + + if !expected.Equal(actual) { + t.Errorf("Expected no difference, got: %v", expected.Difference(actual)) + } +} + +func Test_MarshalJSON(t *testing.T) { + expected := NewSetFromSlice( + []interface{}{ + json.Number("1"), + "test", + }, + ) + + b, err := json.Marshal( + NewSetFromSlice( + []interface{}{ + 1, + "test", + }, + ), + ) + if err != nil { + t.Errorf("Error should be nil: %v", err) + } + + actual := NewSet() + err = json.Unmarshal(b, actual) + if err != nil { + t.Errorf("Error should be nil: %v", err) + } + + if !expected.Equal(actual) { + t.Errorf("Expected no difference, got: %v", expected.Difference(actual)) + } +} diff --git a/vendor/github.com/deckarep/golang-set/threadunsafe.go b/vendor/github.com/deckarep/golang-set/threadunsafe.go new file mode 100644 index 00000000..10bdd46f --- /dev/null +++ b/vendor/github.com/deckarep/golang-set/threadunsafe.go @@ -0,0 +1,337 @@ +/* +Open Source Initiative OSI - The MIT License (MIT):Licensing + +The MIT License (MIT) +Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package mapset + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strings" +) + +type threadUnsafeSet map[interface{}]struct{} + +// An OrderedPair represents a 2-tuple of values. +type OrderedPair struct { + First interface{} + Second interface{} +} + +func newThreadUnsafeSet() threadUnsafeSet { + return make(threadUnsafeSet) +} + +// Equal says whether two 2-tuples contain the same values in the same order. +func (pair *OrderedPair) Equal(other OrderedPair) bool { + if pair.First == other.First && + pair.Second == other.Second { + return true + } + + return false +} + +func (set *threadUnsafeSet) Add(i interface{}) bool { + _, found := (*set)[i] + if found { + return false //False if it existed already + } + + (*set)[i] = struct{}{} + return true +} + +func (set *threadUnsafeSet) Contains(i ...interface{}) bool { + for _, val := range i { + if _, ok := (*set)[val]; !ok { + return false + } + } + return true +} + +func (set *threadUnsafeSet) IsSubset(other Set) bool { + _ = other.(*threadUnsafeSet) + for elem := range *set { + if !other.Contains(elem) { + return false + } + } + return true +} + +func (set *threadUnsafeSet) IsProperSubset(other Set) bool { + return set.IsSubset(other) && !set.Equal(other) +} + +func (set *threadUnsafeSet) IsSuperset(other Set) bool { + return other.IsSubset(set) +} + +func (set *threadUnsafeSet) IsProperSuperset(other Set) bool { + return set.IsSuperset(other) && !set.Equal(other) +} + +func (set *threadUnsafeSet) Union(other Set) Set { + o := other.(*threadUnsafeSet) + + unionedSet := newThreadUnsafeSet() + + for elem := range *set { + unionedSet.Add(elem) + } + for elem := range *o { + unionedSet.Add(elem) + } + return &unionedSet +} + +func (set *threadUnsafeSet) Intersect(other Set) Set { + o := other.(*threadUnsafeSet) + + intersection := newThreadUnsafeSet() + // loop over smaller set + if set.Cardinality() < other.Cardinality() { + for elem := range *set { + if other.Contains(elem) { + intersection.Add(elem) + } + } + } else { + for elem := range *o { + if set.Contains(elem) { + intersection.Add(elem) + } + } + } + return &intersection +} + +func (set *threadUnsafeSet) Difference(other Set) Set { + _ = other.(*threadUnsafeSet) + + difference := newThreadUnsafeSet() + for elem := range *set { + if !other.Contains(elem) { + difference.Add(elem) + } + } + return &difference +} + +func (set *threadUnsafeSet) SymmetricDifference(other Set) Set { + _ = other.(*threadUnsafeSet) + + aDiff := set.Difference(other) + bDiff := other.Difference(set) + return aDiff.Union(bDiff) +} + +func (set *threadUnsafeSet) Clear() { + *set = newThreadUnsafeSet() +} + +func (set *threadUnsafeSet) Remove(i interface{}) { + delete(*set, i) +} + +func (set *threadUnsafeSet) Cardinality() int { + return len(*set) +} + +func (set *threadUnsafeSet) Each(cb func(interface{}) bool) { + for elem := range *set { + if cb(elem) { + break + } + } +} + +func (set *threadUnsafeSet) Iter() <-chan interface{} { + ch := make(chan interface{}) + go func() { + for elem := range *set { + ch <- elem + } + close(ch) + }() + + return ch +} + +func (set *threadUnsafeSet) Iterator() *Iterator { + iterator, ch, stopCh := newIterator() + + go func() { + L: + for elem := range *set { + select { + case <-stopCh: + break L + case ch <- elem: + } + } + close(ch) + }() + + return iterator +} + +func (set *threadUnsafeSet) Equal(other Set) bool { + _ = other.(*threadUnsafeSet) + + if set.Cardinality() != other.Cardinality() { + return false + } + for elem := range *set { + if !other.Contains(elem) { + return false + } + } + return true +} + +func (set *threadUnsafeSet) Clone() Set { + clonedSet := newThreadUnsafeSet() + for elem := range *set { + clonedSet.Add(elem) + } + return &clonedSet +} + +func (set *threadUnsafeSet) String() string { + items := make([]string, 0, len(*set)) + + for elem := range *set { + items = append(items, fmt.Sprintf("%v", elem)) + } + return fmt.Sprintf("Set{%s}", strings.Join(items, ", ")) +} + +// String outputs a 2-tuple in the form "(A, B)". +func (pair OrderedPair) String() string { + return fmt.Sprintf("(%v, %v)", pair.First, pair.Second) +} + +func (set *threadUnsafeSet) Pop() interface{} { + for item := range *set { + delete(*set, item) + return item + } + return nil +} + +func (set *threadUnsafeSet) PowerSet() Set { + powSet := NewThreadUnsafeSet() + nullset := newThreadUnsafeSet() + powSet.Add(&nullset) + + for es := range *set { + u := newThreadUnsafeSet() + j := powSet.Iter() + for er := range j { + p := newThreadUnsafeSet() + if reflect.TypeOf(er).Name() == "" { + k := er.(*threadUnsafeSet) + for ek := range *(k) { + p.Add(ek) + } + } else { + p.Add(er) + } + p.Add(es) + u.Add(&p) + } + + powSet = powSet.Union(&u) + } + + return powSet +} + +func (set *threadUnsafeSet) CartesianProduct(other Set) Set { + o := other.(*threadUnsafeSet) + cartProduct := NewThreadUnsafeSet() + + for i := range *set { + for j := range *o { + elem := OrderedPair{First: i, Second: j} + cartProduct.Add(elem) + } + } + + return cartProduct +} + +func (set *threadUnsafeSet) ToSlice() []interface{} { + keys := make([]interface{}, 0, set.Cardinality()) + for elem := range *set { + keys = append(keys, elem) + } + + return keys +} + +// MarshalJSON creates a JSON array from the set, it marshals all elements +func (set *threadUnsafeSet) MarshalJSON() ([]byte, error) { + items := make([]string, 0, set.Cardinality()) + + for elem := range *set { + b, err := json.Marshal(elem) + if err != nil { + return nil, err + } + + items = append(items, string(b)) + } + + return []byte(fmt.Sprintf("[%s]", strings.Join(items, ","))), nil +} + +// UnmarshalJSON recreates a set from a JSON array, it only decodes +// primitive types. Numbers are decoded as json.Number. +func (set *threadUnsafeSet) UnmarshalJSON(b []byte) error { + var i []interface{} + + d := json.NewDecoder(bytes.NewReader(b)) + d.UseNumber() + err := d.Decode(&i) + if err != nil { + return err + } + + for _, v := range i { + switch t := v.(type) { + case []interface{}, map[string]interface{}: + continue + default: + set.Add(t) + } + } + + return nil +} diff --git a/worker.go b/worker.go index 106a4c3e..07a547b3 100644 --- a/worker.go +++ b/worker.go @@ -16,9 +16,9 @@ package clair import ( "errors" - "regexp" "sync" + "github.com/deckarep/golang-set" log "github.com/sirupsen/logrus" "github.com/coreos/clair/database" @@ -27,10 +27,7 @@ import ( "github.com/coreos/clair/ext/imagefmt" "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/pkg/strutil" -) - -const ( - logLayerName = "layer" + "github.com/coreos/clair/pkg/tarutil" ) var ( @@ -38,25 +35,10 @@ var ( // manager is not supported. ErrUnsupported = commonerr.NewBadRequestError("worker: OS and/or package manager are not supported") - // ErrParentUnknown is the error that should be raised when a parent layer - // has yet to be processed for the current layer. - ErrParentUnknown = commonerr.NewBadRequestError("worker: parent layer is unknown, it must be processed first") - - urlParametersRegexp = regexp.MustCompile(`(\?|\&)([^=]+)\=([^ &]+)`) - - // Processors contain the names of namespace detectors and feature listers - // enabled in this instance of Clair. - // - // Processors are initialized during booting and configured in the - // configuration file. - Processors database.Processors + // EnabledDetectors are detectors to be used to scan the layers. + EnabledDetectors []database.Detector ) -type WorkerConfig struct { - EnabledDetectors []string `yaml:"namespace_detectors"` - EnabledListers []string `yaml:"feature_listers"` -} - // LayerRequest represents all information necessary to download and process a // layer. type LayerRequest struct { @@ -65,296 +47,176 @@ type LayerRequest struct { Headers map[string]string } -// partialLayer stores layer's content detected by `processedBy` processors. -type partialLayer struct { - hash string - processedBy database.Processors - namespaces []database.Namespace - features []database.Feature - - err error +type processResult struct { + existingLayer *database.Layer + newLayerContent *database.Layer + err error } -// processRequest stores parameters used for processing layers. +// processRequest stores parameters used for processing a layer. type processRequest struct { - request LayerRequest - // notProcessedBy represents a set of processors used to process the - // request. - notProcessedBy database.Processors + LayerRequest + + existingLayer *database.Layer + detectors []database.Detector } -// cleanURL removes all parameters from an URL. -func cleanURL(str string) string { - return urlParametersRegexp.ReplaceAllString(str, "") +type introducedFeature struct { + feature database.AncestryFeature + layerIndex int } -// processLayers in parallel processes a set of requests for unique set of layers +// processRequests in parallel processes a set of requests for unique set of layers // and returns sets of unique namespaces, features and layers to be inserted // into the database. -func processRequests(imageFormat string, toDetect []processRequest) ([]database.Namespace, []database.Feature, map[string]partialLayer, error) { +func processRequests(imageFormat string, toDetect map[string]*processRequest) (map[string]*processResult, error) { wg := &sync.WaitGroup{} wg.Add(len(toDetect)) - results := make([]partialLayer, len(toDetect)) + + results := map[string]*processResult{} + for i := range toDetect { + results[i] = nil + } + for i := range toDetect { - go func(req *processRequest, res *partialLayer) { - res.hash = req.request.Hash - res.processedBy = req.notProcessedBy - res.namespaces, res.features, res.err = detectContent(imageFormat, req.request.Hash, req.request.Path, req.request.Headers, req.notProcessedBy) + result := processResult{} + results[i] = &result + go func(req *processRequest, res *processResult) { + *res = *detectContent(imageFormat, req) wg.Done() - }(&toDetect[i], &results[i]) + }(toDetect[i], &result) } - wg.Wait() - distinctNS := map[database.Namespace]struct{}{} - distinctF := map[database.Feature]struct{}{} + wg.Wait() errs := []error{} for _, r := range results { errs = append(errs, r.err) } if err := commonerr.CombineErrors(errs...); err != nil { - return nil, nil, nil, err - } - - updates := map[string]partialLayer{} - for _, r := range results { - for _, ns := range r.namespaces { - distinctNS[ns] = struct{}{} - } - - for _, f := range r.features { - distinctF[f] = struct{}{} - } - - if _, ok := updates[r.hash]; !ok { - updates[r.hash] = r - } else { - return nil, nil, nil, errors.New("Duplicated updates is not allowed") - } - } - - namespaces := make([]database.Namespace, 0, len(distinctNS)) - features := make([]database.Feature, 0, len(distinctF)) - - for ns := range distinctNS { - namespaces = append(namespaces, ns) + return nil, err } - for f := range distinctF { - features = append(features, f) - } - return namespaces, features, updates, nil + return results, nil } -func getLayer(datastore database.Datastore, req LayerRequest) (layer database.Layer, preq *processRequest, err error) { - var ( - tx database.Session - ok bool - ) - - if tx, err = datastore.Begin(); err != nil { - return - } - - defer tx.Rollback() - - if layer, ok, err = tx.FindLayer(req.Hash); err != nil { +func getProcessRequest(datastore database.Datastore, req LayerRequest) (preq *processRequest, err error) { + layer, ok, err := database.FindLayerAndRollback(datastore, req.Hash) + if err != nil { return } if !ok { - layer = database.Layer{ - LayerMetadata: database.LayerMetadata{ - Hash: req.Hash, - }, - } - + log.WithField("layer", req.Hash).Debug("found no existing layer in database") preq = &processRequest{ - request: req, - notProcessedBy: Processors, + LayerRequest: req, + existingLayer: &database.Layer{Hash: req.Hash}, + detectors: EnabledDetectors, } } else { - notProcessed := getNotProcessedBy(layer.ProcessedBy) - if !(len(notProcessed.Detectors) == 0 && len(notProcessed.Listers) == 0 && ok) { - preq = &processRequest{ - request: req, - notProcessedBy: notProcessed, - } + log.WithFields(log.Fields{ + "layer": layer.Hash, + "detectors": layer.By, + "feature count": len(layer.Features), + "namespace count": len(layer.Namespaces), + }).Debug("found existing layer in database") + + preq = &processRequest{ + LayerRequest: req, + existingLayer: &layer, + detectors: database.DiffDetectors(EnabledDetectors, layer.By), } } return } -// processLayers processes a set of post layer requests, stores layers and -// returns an ordered list of processed layers with detected features and -// namespaces. -func processLayers(datastore database.Datastore, imageFormat string, requests []LayerRequest) ([]database.Layer, error) { - toDetect := []processRequest{} - layers := map[string]database.Layer{} - for _, req := range requests { - if _, ok := layers[req.Hash]; ok { - continue - } - layer, preq, err := getLayer(datastore, req) - if err != nil { - return nil, err - } - layers[req.Hash] = layer - if preq != nil { - toDetect = append(toDetect, *preq) - } - } - - namespaces, features, partialLayers, err := processRequests(imageFormat, toDetect) - if err != nil { - return nil, err +func persistProcessResult(datastore database.Datastore, results map[string]*processResult) error { + features := []database.Feature{} + namespaces := []database.Namespace{} + for _, r := range results { + features = append(features, r.newLayerContent.GetFeatures()...) + namespaces = append(namespaces, r.newLayerContent.GetNamespaces()...) } - // Store partial results. - if err := persistNamespaces(datastore, namespaces); err != nil { - return nil, err + features = database.DeduplicateFeatures(features...) + namespaces = database.DeduplicateNamespaces(namespaces...) + if err := database.PersistNamespacesAndCommit(datastore, namespaces); err != nil { + return err } - if err := persistFeatures(datastore, features); err != nil { - return nil, err + if err := database.PersistFeaturesAndCommit(datastore, features); err != nil { + return err } - for _, layer := range partialLayers { - if err := persistPartialLayer(datastore, layer); err != nil { - return nil, err - } - - log.WithFields(log.Fields{ - "Hash": layer.hash, - "namespace count": len(layer.namespaces), - "feature count": len(layer.features), - "namespace detectors": layer.processedBy.Detectors, - "feature listers": layer.processedBy.Listers, - }).Debug("saved layer") - } - - // NOTE(Sida): The full layers are computed using partially - // processed layers in current database session. If any other instances of - // Clair are changing some layers in this set of layers, it might generate - // different results especially when the other Clair is with different - // processors. - completeLayers := []database.Layer{} - for _, req := range requests { - if partialLayer, ok := partialLayers[req.Hash]; ok { - completeLayers = append(completeLayers, combineLayers(layers[req.Hash], partialLayer)) - } else { - completeLayers = append(completeLayers, layers[req.Hash]) + for _, layer := range results { + if err := database.PersistPartialLayerAndCommit(datastore, layer.newLayerContent); err != nil { + return err } } - return completeLayers, nil + return nil } -func persistPartialLayer(datastore database.Datastore, layer partialLayer) error { - tx, err := datastore.Begin() - if err != nil { - return err - } - defer tx.Rollback() +// processLayers processes a set of post layer requests, stores layers and +// returns an ordered list of processed layers with detected features and +// namespaces. +func processLayers(datastore database.Datastore, imageFormat string, requests []LayerRequest) ([]database.Layer, error) { + var ( + reqMap = make(map[string]*processRequest) + err error + ) - if err := tx.PersistLayer(layer.hash, layer.namespaces, layer.features, layer.processedBy); err != nil { - return err + for _, r := range requests { + reqMap[r.Hash], err = getProcessRequest(datastore, r) + if err != nil { + return nil, err + } } - return tx.Commit() -} - -func persistFeatures(datastore database.Datastore, features []database.Feature) error { - tx, err := datastore.Begin() + results, err := processRequests(imageFormat, reqMap) if err != nil { - return err - } - defer tx.Rollback() - - if err := tx.PersistFeatures(features); err != nil { - return err + return nil, err } - return tx.Commit() -} -func persistNamespaces(datastore database.Datastore, namespaces []database.Namespace) error { - tx, err := datastore.Begin() - if err != nil { - return err + if err := persistProcessResult(datastore, results); err != nil { + return nil, err } - defer tx.Rollback() - if err := tx.PersistNamespaces(namespaces); err != nil { - return err + completeLayers := getProcessResultLayers(results) + layers := make([]database.Layer, 0, len(requests)) + for _, r := range requests { + layers = append(layers, completeLayers[r.Hash]) } - return tx.Commit() + return layers, nil } -// combineLayers merges `layer` and `partial` without duplicated content. -func combineLayers(layer database.Layer, partial partialLayer) database.Layer { - mapF := map[database.Feature]struct{}{} - mapNS := map[database.Namespace]struct{}{} - for _, f := range layer.Features { - mapF[f] = struct{}{} - } - for _, ns := range layer.Namespaces { - mapNS[ns] = struct{}{} - } - for _, f := range partial.features { - mapF[f] = struct{}{} - } - for _, ns := range partial.namespaces { - mapNS[ns] = struct{}{} - } - features := make([]database.Feature, 0, len(mapF)) - namespaces := make([]database.Namespace, 0, len(mapNS)) - for f := range mapF { - features = append(features, f) - } - for ns := range mapNS { - namespaces = append(namespaces, ns) +func getProcessResultLayers(results map[string]*processResult) map[string]database.Layer { + layers := map[string]database.Layer{} + for name, r := range results { + layers[name] = *database.MergeLayers(r.existingLayer, r.newLayerContent) } - layer.ProcessedBy.Detectors = append(layer.ProcessedBy.Detectors, strutil.CompareStringLists(partial.processedBy.Detectors, layer.ProcessedBy.Detectors)...) - layer.ProcessedBy.Listers = append(layer.ProcessedBy.Listers, strutil.CompareStringLists(partial.processedBy.Listers, layer.ProcessedBy.Listers)...) - return database.Layer{ - LayerMetadata: database.LayerMetadata{ - Hash: layer.Hash, - ProcessedBy: layer.ProcessedBy, - }, - Features: features, - Namespaces: namespaces, - } + return layers } func isAncestryProcessed(datastore database.Datastore, name string) (bool, error) { - tx, err := datastore.Begin() - if err != nil { - return false, err - } - defer tx.Rollback() - ancestry, ok, err := tx.FindAncestry(name) - if err != nil { - return false, err - } - if !ok { - return false, nil + ancestry, ok, err := database.FindAncestryAndRollback(datastore, name) + if err != nil || !ok { + return ok, err } - notProcessed := getNotProcessedBy(ancestry.ProcessedBy) - return len(notProcessed.Detectors) == 0 && len(notProcessed.Listers) == 0, nil + return len(database.DiffDetectors(EnabledDetectors, ancestry.By)) == 0, nil } // ProcessAncestry downloads and scans an ancestry if it's not scanned by all // enabled processors in this instance of Clair. func ProcessAncestry(datastore database.Datastore, imageFormat, name string, layerRequest []LayerRequest) error { var ( - err error - ok bool - layers []database.Layer - commonProcessors database.Processors + err error + ok bool + layers []database.Layer ) if name == "" { @@ -365,10 +227,12 @@ func ProcessAncestry(datastore database.Datastore, imageFormat, name string, lay return commonerr.NewBadRequestError("could not process a layer which does not have a format") } + log.WithField("ancestry", name).Debug("start processing ancestry...") if ok, err = isAncestryProcessed(datastore, name); err != nil { + log.WithError(err).Error("could not determine if ancestry is processed") return err } else if ok { - log.WithField("name", name).Debug("ancestry is already processed") + log.WithField("ancestry", name).Debug("ancestry is already processed") return nil } @@ -376,155 +240,100 @@ func ProcessAncestry(datastore database.Datastore, imageFormat, name string, lay return err } - if commonProcessors, err = getProcessors(layers); err != nil { - return err - } - - return processAncestry(datastore, name, layers, commonProcessors) -} - -// getNamespacedFeatures extracts the namespaced features introduced in each -// layer into one array. -func getNamespacedFeatures(layers []database.AncestryLayer) []database.NamespacedFeature { - features := []database.NamespacedFeature{} - for _, layer := range layers { - features = append(features, layer.DetectedFeatures...) - } - return features + return processAncestry(datastore, name, layers) } -func processAncestry(datastore database.Datastore, name string, layers []database.Layer, commonProcessors database.Processors) error { +func processAncestry(datastore database.Datastore, name string, layers []database.Layer) error { var ( - ancestry database.Ancestry + ancestry = database.Ancestry{Name: name} err error ) - ancestry.Name = name - ancestry.ProcessedBy = commonProcessors - ancestry.Layers, err = computeAncestryLayers(layers, commonProcessors) + ancestry.Layers, ancestry.By, err = computeAncestryLayers(layers) if err != nil { return err } - ancestryFeatures := getNamespacedFeatures(ancestry.Layers) + ancestryFeatures := database.GetAncestryFeatures(ancestry) log.WithFields(log.Fields{ - "ancestry": name, - "number of features": len(ancestryFeatures), - "processed by": Processors, - "number of layers": len(ancestry.Layers), + "ancestry": name, + "processed by": EnabledDetectors, + "features count": len(ancestryFeatures), + "layer count": len(ancestry.Layers), }).Debug("compute ancestry features") - if err := persistNamespacedFeatures(datastore, ancestryFeatures); err != nil { + if err := database.PersistNamespacedFeaturesAndCommit(datastore, ancestryFeatures); err != nil { + log.WithField("ancestry", name).WithError(err).Error("could not persist namespaced features for ancestry") return err } - tx, err := datastore.Begin() - if err != nil { + if err := database.CacheRelatedVulnerabilityAndCommit(datastore, ancestryFeatures); err != nil { + log.WithField("ancestry", name).WithError(err).Error("failed to cache feature related vulnerability") return err } - err = tx.UpsertAncestry(ancestry) - if err != nil { - tx.Rollback() + if err := database.UpsertAncestryAndCommit(datastore, ancestry); err != nil { + log.WithField("ancestry", name).WithError(err).Error("could not upsert ancestry") return err } - err = tx.Commit() - if err != nil { - return err - } return nil } -func persistNamespacedFeatures(datastore database.Datastore, features []database.NamespacedFeature) error { - tx, err := datastore.Begin() - if err != nil { - return err - } - - if err := tx.PersistNamespacedFeatures(features); err != nil { - tx.Rollback() - return err - } - - if err := tx.Commit(); err != nil { - return err - } - - tx, err = datastore.Begin() - if err != nil { - return err - } - - if err := tx.CacheAffectedNamespacedFeatures(features); err != nil { - tx.Rollback() - return err - } - - return tx.Commit() -} - -// getProcessors retrieves common subset of the processors of each layer. -func getProcessors(layers []database.Layer) (database.Processors, error) { - if len(layers) == 0 { - return database.Processors{}, nil +func getCommonDetectors(layers []database.Layer) mapset.Set { + // find the common detector for all layers and filter the namespaces and + // features based on that. + commonDetectors := mapset.NewSet() + for _, d := range layers[0].By { + commonDetectors.Add(d) } - detectors := layers[0].ProcessedBy.Detectors - listers := layers[0].ProcessedBy.Listers - - detectorsLen := len(detectors) - listersLen := len(listers) - - for _, l := range layers[1:] { - detectors := strutil.CompareStringListsInBoth(detectors, l.ProcessedBy.Detectors) - listers := strutil.CompareStringListsInBoth(listers, l.ProcessedBy.Listers) - - if len(detectors) != detectorsLen || len(listers) != listersLen { - // This error might be triggered because of multiple workers are - // processing the same instance with different processors. - // TODO(sidchen): Once the features can be associated with - // Detectors/Listers, we can support dynamically generating ancestry's - // detector/lister based on the layers. - return database.Processors{}, errors.New("processing layers with different Clair instances is currently unsupported") + for _, l := range layers { + detectors := mapset.NewSet() + for _, d := range l.By { + detectors.Add(d) } + + commonDetectors = commonDetectors.Intersect(detectors) } - return database.Processors{ - Detectors: detectors, - Listers: listers, - }, nil -} -type introducedFeature struct { - feature database.NamespacedFeature - layerIndex int + return commonDetectors } // computeAncestryLayers computes ancestry's layers along with what features are // introduced. -func computeAncestryLayers(layers []database.Layer, commonProcessors database.Processors) ([]database.AncestryLayer, error) { - // TODO(sidchen): Once the features are linked to specific processor, we - // will use commonProcessors to filter out the features for this ancestry. +func computeAncestryLayers(layers []database.Layer) ([]database.AncestryLayer, []database.Detector, error) { + if len(layers) == 0 { + return nil, nil, nil + } + commonDetectors := getCommonDetectors(layers) // version format -> namespace - namespaces := map[string]database.Namespace{} + namespaces := map[string]database.LayerNamespace{} // version format -> feature ID -> feature features := map[string]map[string]introducedFeature{} ancestryLayers := []database.AncestryLayer{} for index, layer := range layers { - // Initialize the ancestry Layer - initializedLayer := database.AncestryLayer{LayerMetadata: layer.LayerMetadata, DetectedFeatures: []database.NamespacedFeature{}} + initializedLayer := database.AncestryLayer{Hash: layer.Hash} ancestryLayers = append(ancestryLayers, initializedLayer) // Precondition: namespaces and features contain the result from union // of all parents. for _, ns := range layer.Namespaces { + if !commonDetectors.Contains(ns.By) { + continue + } + namespaces[ns.VersionFormat] = ns } // version format -> feature ID -> feature currentFeatures := map[string]map[string]introducedFeature{} for _, f := range layer.Features { + if !commonDetectors.Contains(f.By) { + continue + } + if ns, ok := namespaces[f.VersionFormat]; ok { var currentMap map[string]introducedFeature if currentMap, ok = currentFeatures[f.VersionFormat]; !ok { @@ -542,16 +351,20 @@ func computeAncestryLayers(layers []database.Layer, commonProcessors database.Pr if !inherited { currentMap[f.Name+":"+f.Version] = introducedFeature{ - feature: database.NamespacedFeature{ - Feature: f, - Namespace: ns, + feature: database.AncestryFeature{ + NamespacedFeature: database.NamespacedFeature{ + Feature: f.Feature, + Namespace: ns.Namespace, + }, + NamespaceBy: ns.By, + FeatureBy: f.By, }, layerIndex: index, } } } else { - return nil, errors.New("No corresponding version format") + return nil, nil, errors.New("No corresponding version format") } } @@ -569,57 +382,97 @@ func computeAncestryLayers(layers []database.Layer, commonProcessors database.Pr for _, featureMap := range features { for _, feature := range featureMap { - ancestryLayers[feature.layerIndex].DetectedFeatures = append( - ancestryLayers[feature.layerIndex].DetectedFeatures, + ancestryLayers[feature.layerIndex].Features = append( + ancestryLayers[feature.layerIndex].Features, feature.feature, ) } } - return ancestryLayers, nil + detectors := make([]database.Detector, 0, commonDetectors.Cardinality()) + for d := range commonDetectors.Iter() { + detectors = append(detectors, d.(database.Detector)) + } + + return ancestryLayers, detectors, nil } -// getNotProcessedBy returns a processors, which contains the detectors and -// listers not in `processedBy` but implemented in the current clair instance. -func getNotProcessedBy(processedBy database.Processors) database.Processors { - notProcessedLister := strutil.CompareStringLists(Processors.Listers, processedBy.Listers) - notProcessedDetector := strutil.CompareStringLists(Processors.Detectors, processedBy.Detectors) - return database.Processors{ - Listers: notProcessedLister, - Detectors: notProcessedDetector, +func extractRequiredFiles(imageFormat string, req *processRequest) (tarutil.FilesMap, error) { + requiredFiles := append(featurefmt.RequiredFilenames(req.detectors), featurens.RequiredFilenames(req.detectors)...) + if len(requiredFiles) == 0 { + log.WithFields(log.Fields{ + "layer": req.Hash, + "detectors": req.detectors, + }).Info("layer requires no file to extract") + return make(tarutil.FilesMap), nil } -} -// detectContent downloads a layer and detects all features and namespaces. -func detectContent(imageFormat, name, path string, headers map[string]string, toProcess database.Processors) (namespaces []database.Namespace, featureVersions []database.Feature, err error) { - log.WithFields(log.Fields{"Hash": name}).Debug("Process Layer") - totalRequiredFiles := append(featurefmt.RequiredFilenames(toProcess.Listers), featurens.RequiredFilenames(toProcess.Detectors)...) - files, err := imagefmt.Extract(imageFormat, path, headers, totalRequiredFiles) + files, err := imagefmt.Extract(imageFormat, req.Path, req.Headers, requiredFiles) if err != nil { log.WithError(err).WithFields(log.Fields{ - logLayerName: name, - "path": cleanURL(path), + "layer": req.Hash, + "path": strutil.CleanURL(req.Path), }).Error("failed to extract data from path") + return nil, err + } + + return files, err +} + +// detectContent downloads a layer and detects all features and namespaces. +func detectContent(imageFormat string, req *processRequest) (res *processResult) { + var ( + files tarutil.FilesMap + layer = database.Layer{Hash: req.Hash, By: req.detectors} + ) + + res = &processResult{req.existingLayer, &layer, nil} + log.WithFields(log.Fields{ + "layer": req.Hash, + "detectors": req.detectors, + }).Info("detecting layer content...") + + files, res.err = extractRequiredFiles(imageFormat, req) + if res.err != nil { return } - namespaces, err = featurens.Detect(files, toProcess.Detectors) - if err != nil { + if layer.Namespaces, res.err = featurens.Detect(files, req.detectors); res.err != nil { return } - if len(featureVersions) > 0 { - log.WithFields(log.Fields{logLayerName: name, "count": len(namespaces)}).Debug("detected layer namespaces") + if layer.Features, res.err = featurefmt.ListFeatures(files, req.detectors); res.err != nil { + return } - featureVersions, err = featurefmt.ListFeatures(files, toProcess.Listers) - if err != nil { + log.WithFields(log.Fields{ + "layer": req.Hash, + "detectors": req.detectors, + "namespace count": len(layer.Namespaces), + "feature count": len(layer.Features), + }).Info("processed layer") + + return +} + +// InitWorker initializes the worker. +func InitWorker(datastore database.Datastore) { + if len(EnabledDetectors) == 0 { + log.Warn("no enabled detector, and therefore, no ancestry will be processed.") return } - if len(featureVersions) > 0 { - log.WithFields(log.Fields{logLayerName: name, "count": len(featureVersions)}).Debug("detected layer features") + tx, err := datastore.Begin() + if err != nil { + log.WithError(err).Fatal("cannot connect to database to initialize worker") + } + + defer tx.Rollback() + if err := tx.PersistDetectors(EnabledDetectors); err != nil { + log.WithError(err).Fatal("cannot insert detectors to initialize worker") } - return + if err := tx.Commit(); err != nil { + log.WithError(err).Fatal("cannot commit detector changes to initialize worker") + } } diff --git a/worker_test.go b/worker_test.go index 2df212e0..aef37985 100644 --- a/worker_test.go +++ b/worker_test.go @@ -22,12 +22,12 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/featurefmt" "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/pkg/strutil" // Register the required detectors. _ "github.com/coreos/clair/ext/featurefmt/dpkg" @@ -58,55 +58,27 @@ type mockSession struct { func copyDatastore(md *mockDatastore) mockDatastore { layers := map[string]database.Layer{} for k, l := range md.layers { - features := append([]database.Feature(nil), l.Features...) - namespaces := append([]database.Namespace(nil), l.Namespaces...) - listers := append([]string(nil), l.ProcessedBy.Listers...) - detectors := append([]string(nil), l.ProcessedBy.Detectors...) layers[k] = database.Layer{ - LayerMetadata: database.LayerMetadata{ - Hash: l.Hash, - ProcessedBy: database.Processors{ - Listers: listers, - Detectors: detectors, - }, - }, - Features: features, - Namespaces: namespaces, + Hash: l.Hash, + By: append([]database.Detector{}, l.By...), + Features: append([]database.LayerFeature{}, l.Features...), + Namespaces: append([]database.LayerNamespace{}, l.Namespaces...), } } ancestry := map[string]database.Ancestry{} for k, a := range md.ancestry { ancestryLayers := []database.AncestryLayer{} - layers := []database.LayerMetadata{} - for _, layer := range a.Layers { - layers = append(layers, database.LayerMetadata{ - Hash: layer.Hash, - ProcessedBy: database.Processors{ - Detectors: append([]string(nil), layer.LayerMetadata.ProcessedBy.Detectors...), - Listers: append([]string(nil), layer.LayerMetadata.ProcessedBy.Listers...), - }, - }) - ancestryLayers = append(ancestryLayers, database.AncestryLayer{ - LayerMetadata: database.LayerMetadata{ - Hash: layer.Hash, - ProcessedBy: database.Processors{ - Detectors: append([]string(nil), layer.LayerMetadata.ProcessedBy.Detectors...), - Listers: append([]string(nil), layer.LayerMetadata.ProcessedBy.Listers...), - }, - }, - DetectedFeatures: append([]database.NamespacedFeature(nil), layer.DetectedFeatures...), + Hash: layer.Hash, + Features: append([]database.AncestryFeature{}, layer.Features...), }) } ancestry[k] = database.Ancestry{ - Name: a.Name, - ProcessedBy: database.Processors{ - Detectors: append([]string(nil), a.ProcessedBy.Detectors...), - Listers: append([]string(nil), a.ProcessedBy.Listers...), - }, + Name: a.Name, + By: append([]database.Detector{}, a.By...), Layers: ancestryLayers, } } @@ -125,6 +97,7 @@ func copyDatastore(md *mockDatastore) mockDatastore { for k, f := range md.namespacedFeatures { namespacedFeatures[k] = f } + return mockDatastore{ layers: layers, ancestry: ancestry, @@ -194,10 +167,7 @@ func newMockDatastore() *mockDatastore { return errSessionDone } for _, n := range ns { - _, ok := session.copy.namespaces[n.Name] - if !ok { - session.copy.namespaces[n.Name] = n - } + session.copy.namespaces[NamespaceKey(&n)] = n } return nil } @@ -207,63 +177,36 @@ func newMockDatastore() *mockDatastore { return errSessionDone } for _, f := range fs { - key := FeatureKey(&f) - _, ok := session.copy.features[key] - if !ok { - session.copy.features[key] = f - } + session.copy.features[FeatureKey(&f)] = f } + return nil } - session.FctPersistLayer = func(hash string, namespaces []database.Namespace, features []database.Feature, processedBy database.Processors) error { + session.FctPersistLayer = func(hash string, features []database.LayerFeature, namespaces []database.LayerNamespace, by []database.Detector) error { if session.terminated { return errSessionDone } - // update the layer - _, ok := session.copy.layers[hash] - if !ok { - session.copy.layers[hash] = database.Layer{} - } - - layer, ok := session.copy.layers[hash] - if !ok { - return errors.New("Failed to insert layer") - } - - layerFeatures := map[string]database.Feature{} - layerNamespaces := map[string]database.Namespace{} - for _, f := range layer.Features { - layerFeatures[FeatureKey(&f)] = f - } - for _, n := range layer.Namespaces { - layerNamespaces[n.Name] = n - } - - // ensure that all the namespaces, features are in the database for _, ns := range namespaces { - if _, ok := session.copy.namespaces[ns.Name]; !ok { - return errors.New("Namespaces should be in the database") - } - if _, ok := layerNamespaces[ns.Name]; !ok { - layer.Namespaces = append(layer.Namespaces, ns) - layerNamespaces[ns.Name] = ns + if _, ok := session.copy.namespaces[NamespaceKey(&ns.Namespace)]; !ok { + panic("") } } for _, f := range features { - if _, ok := session.copy.features[FeatureKey(&f)]; !ok { - return errors.New("Namespaces should be in the database") - } - if _, ok := layerFeatures[FeatureKey(&f)]; !ok { - layer.Features = append(layer.Features, f) - layerFeatures[FeatureKey(&f)] = f + if _, ok := session.copy.features[FeatureKey(&f.Feature)]; !ok { + panic("") } } - layer.ProcessedBy.Detectors = append(layer.ProcessedBy.Detectors, strutil.CompareStringLists(processedBy.Detectors, layer.ProcessedBy.Detectors)...) - layer.ProcessedBy.Listers = append(layer.ProcessedBy.Listers, strutil.CompareStringLists(processedBy.Listers, layer.ProcessedBy.Listers)...) + layer, _ := session.copy.layers[hash] + database.MergeLayers(&layer, &database.Layer{ + Hash: hash, + By: by, + Namespaces: namespaces, + Features: features, + }) session.copy.layers[hash] = layer return nil @@ -274,11 +217,12 @@ func newMockDatastore() *mockDatastore { return errSessionDone } - features := getNamespacedFeatures(ancestry.Layers) - // ensure features are in the database - for _, f := range features { - if _, ok := session.copy.namespacedFeatures[NamespacedFeatureKey(&f)]; !ok { - return errors.New("namespaced feature not in db") + // ensure the namespaces features are in the code base + for _, l := range ancestry.Layers { + for _, f := range l.GetFeatures() { + if _, ok := session.copy.namespacedFeatures[NamespacedFeatureKey(&f)]; !ok { + panic("") + } } } @@ -288,6 +232,14 @@ func newMockDatastore() *mockDatastore { session.FctPersistNamespacedFeatures = func(namespacedFeatures []database.NamespacedFeature) error { for i, f := range namespacedFeatures { + if _, ok := session.copy.features[FeatureKey(&f.Feature)]; !ok { + panic("") + } + + if _, ok := session.copy.namespaces[NamespaceKey(&f.Namespace)]; !ok { + panic("") + } + session.copy.namespacedFeatures[NamespacedFeatureKey(&f)] = namespacedFeatures[i] } return nil @@ -304,10 +256,7 @@ func newMockDatastore() *mockDatastore { } func TestMain(m *testing.M) { - Processors = database.Processors{ - Listers: featurefmt.ListListers(), - Detectors: featurens.ListDetectors(), - } + EnabledDetectors = append(featurefmt.ListListers(), featurens.ListDetectors()...) m.Run() } @@ -315,11 +264,16 @@ func FeatureKey(f *database.Feature) string { return strings.Join([]string{f.Name, f.VersionFormat, f.Version}, "__") } +func NamespaceKey(ns *database.Namespace) string { + return strings.Join([]string{ns.Name, ns.VersionFormat}, "__") +} + func NamespacedFeatureKey(f *database.NamespacedFeature) string { return strings.Join([]string{f.Name, f.Namespace.Name}, "__") } func TestProcessAncestryWithDistUpgrade(t *testing.T) { + // TODO(sidac): Change to use table driven tests. // Create the list of Features that should not been upgraded from one layer to another. nonUpgradedFeatures := []database.Feature{ {Name: "libtext-wrapi18n-perl", Version: "0.06-7"}, @@ -358,7 +312,12 @@ func TestProcessAncestryWithDistUpgrade(t *testing.T) { assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock", layers)) // check the ancestry features - features := getNamespacedFeatures(datastore.ancestry["Mock"].Layers) + features := []database.AncestryFeature{} + for i, l := range datastore.ancestry["Mock"].Layers { + assert.Equal(t, layers[i].Hash, l.Hash) + features = append(features, l.Features...) + } + assert.Len(t, features, 74) for _, f := range features { if _, ok := nonUpgradedMap[f.Feature]; ok { @@ -367,12 +326,6 @@ func TestProcessAncestryWithDistUpgrade(t *testing.T) { assert.Equal(t, "debian:8", f.Namespace.Name) } } - - assert.Equal(t, []database.LayerMetadata{ - {Hash: "blank"}, - {Hash: "wheezy"}, - {Hash: "jessie"}, - }, datastore.ancestry["Mock"].Layers) } func TestProcessLayers(t *testing.T) { @@ -404,8 +357,7 @@ func TestProcessLayers(t *testing.T) { // Ensure each layer has expected namespaces and features detected if blank, ok := datastore.layers["blank"]; ok { - assert.Equal(t, blank.ProcessedBy.Detectors, Processors.Detectors) - assert.Equal(t, blank.ProcessedBy.Listers, Processors.Listers) + database.AssertDetectorsEqual(t, EnabledDetectors, blank.By) assert.Len(t, blank.Namespaces, 0) assert.Len(t, blank.Features, 0) } else { @@ -414,9 +366,11 @@ func TestProcessLayers(t *testing.T) { } if wheezy, ok := datastore.layers["wheezy"]; ok { - assert.Equal(t, wheezy.ProcessedBy.Detectors, Processors.Detectors) - assert.Equal(t, wheezy.ProcessedBy.Listers, Processors.Listers) - assert.Equal(t, wheezy.Namespaces, []database.Namespace{{Name: "debian:7", VersionFormat: dpkg.ParserName}}) + database.AssertDetectorsEqual(t, EnabledDetectors, wheezy.By) + assert.Equal(t, []database.LayerNamespace{ + {database.Namespace{"debian:7", dpkg.ParserName}, database.NewNamespaceDetector("os-release", "1.0")}, + }, wheezy.Namespaces) + assert.Len(t, wheezy.Features, 52) } else { assert.Fail(t, "wheezy is not stored") @@ -424,9 +378,10 @@ func TestProcessLayers(t *testing.T) { } if jessie, ok := datastore.layers["jessie"]; ok { - assert.Equal(t, jessie.ProcessedBy.Detectors, Processors.Detectors) - assert.Equal(t, jessie.ProcessedBy.Listers, Processors.Listers) - assert.Equal(t, jessie.Namespaces, []database.Namespace{{Name: "debian:8", VersionFormat: dpkg.ParserName}}) + database.AssertDetectorsEqual(t, EnabledDetectors, jessie.By) + assert.Equal(t, []database.LayerNamespace{ + {database.Namespace{"debian:8", dpkg.ParserName}, database.NewNamespaceDetector("os-release", "1.0")}, + }, jessie.Namespaces) assert.Len(t, jessie.Features, 74) } else { assert.Fail(t, "jessie is not stored") @@ -434,157 +389,124 @@ func TestProcessLayers(t *testing.T) { } } -// TestUpgradeClair checks if a clair is upgraded and certain ancestry's -// features should not change. We assume that Clair should only upgrade -func TestClairUpgrade(t *testing.T) { - _, f, _, _ := runtime.Caller(0) - testDataPath := filepath.Join(filepath.Dir(f)) + "/testdata/DistUpgrade/" - - datastore := newMockDatastore() - - // suppose there are two ancestries. - layers := []LayerRequest{ - {Hash: "blank", Path: testDataPath + "blank.tar.gz"}, - {Hash: "wheezy", Path: testDataPath + "wheezy.tar.gz"}, - {Hash: "jessie", Path: testDataPath + "jessie.tar.gz"}, +func getFeatures(a database.Ancestry) []database.AncestryFeature { + features := []database.AncestryFeature{} + for _, l := range a.Layers { + features = append(features, l.Features...) } - layers2 := []LayerRequest{ - {Hash: "blank", Path: testDataPath + "blank.tar.gz"}, - {Hash: "wheezy", Path: testDataPath + "wheezy.tar.gz"}, - } - - // Suppose user scan an ancestry with an old instance of Clair. - Processors = database.Processors{ - Detectors: []string{"os-release"}, - Listers: []string{"rpm"}, - } - - assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock", layers)) - assert.Len(t, getNamespacedFeatures(datastore.ancestry["Mock"].Layers), 0) - - assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock2", layers2)) - assert.Len(t, getNamespacedFeatures(datastore.ancestry["Mock2"].Layers), 0) - - // Clair is upgraded to use a new namespace detector. The expected - // behavior is that all layers will be rescanned with "apt-sources" and - // the ancestry's features are recalculated. - Processors = database.Processors{ - Detectors: []string{"os-release", "apt-sources"}, - Listers: []string{"rpm"}, - } - - // Even though Clair processors are upgraded, the ancestry's features should - // not be upgraded without posting the ancestry to Clair again. - assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock", layers)) - assert.Len(t, getNamespacedFeatures(datastore.ancestry["Mock"].Layers), 0) - - // Clair is upgraded to use a new feature lister. The expected behavior is - // that all layers will be rescanned with "dpkg" and the ancestry's features - // are invalidated and recalculated. - Processors = database.Processors{ - Detectors: []string{"os-release", "apt-sources"}, - Listers: []string{"rpm", "dpkg"}, - } - - assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock", layers)) - assert.Len(t, getNamespacedFeatures(datastore.ancestry["Mock"].Layers), 74) - assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock2", layers2)) - assert.Len(t, getNamespacedFeatures(datastore.ancestry["Mock2"].Layers), 52) - - // check the namespaces are correct - for _, f := range getNamespacedFeatures(datastore.ancestry["Mock"].Layers) { - if !assert.NotEqual(t, database.Namespace{}, f.Namespace) { - assert.Fail(t, "Every feature should have a namespace attached") - } - } - - for _, f := range getNamespacedFeatures(datastore.ancestry["Mock2"].Layers) { - if !assert.NotEqual(t, database.Namespace{}, f.Namespace) { - assert.Fail(t, "Every feature should have a namespace attached") - } - } + return features } -// TestMultipleNamespaces tests computing ancestry features func TestComputeAncestryFeatures(t *testing.T) { vf1 := "format 1" vf2 := "format 2" - ns1a := database.Namespace{ - Name: "namespace 1:a", - VersionFormat: vf1, - } - - ns1b := database.Namespace{ - Name: "namespace 1:b", - VersionFormat: vf1, - } - - ns2a := database.Namespace{ - Name: "namespace 2:a", - VersionFormat: vf2, - } - - ns2b := database.Namespace{ - Name: "namespace 2:b", - VersionFormat: vf2, - } - - f1 := database.Feature{ - Name: "feature 1", - Version: "0.1", - VersionFormat: vf1, - } - - f2 := database.Feature{ + nd1 := database.NewNamespaceDetector("apk", "1.0") + fd1 := database.NewFeatureDetector("fd1", "1.0") + // this detector only scans one layer with one extra feature, this one + // should be omitted. + fd2 := database.NewFeatureDetector("fd2", "1.0") + + ns1a := database.LayerNamespace{ + database.Namespace{ + Name: "namespace 1:a", + VersionFormat: vf1, + }, nd1, + } + + ns1b := database.LayerNamespace{ + database.Namespace{ + Name: "namespace 1:b", + VersionFormat: vf1, + }, nd1} + + ns2a := database.LayerNamespace{ + database.Namespace{ + Name: "namespace 2:a", + VersionFormat: vf2, + }, nd1} + + ns2b := database.LayerNamespace{ + database.Namespace{ + Name: "namespace 2:b", + VersionFormat: vf2, + }, nd1} + + f1 := database.LayerFeature{ + database.Feature{ + Name: "feature 1", + Version: "0.1", + VersionFormat: vf1, + }, fd1} + + f2 := database.LayerFeature{database.Feature{ Name: "feature 2", Version: "0.2", VersionFormat: vf1, - } - - f3 := database.Feature{ - Name: "feature 1", - Version: "0.3", - VersionFormat: vf2, - } - - f4 := database.Feature{ - Name: "feature 2", - Version: "0.3", - VersionFormat: vf2, + }, fd2} + + f3 := database.LayerFeature{ + database.Feature{ + Name: "feature 1", + Version: "0.3", + VersionFormat: vf2, + }, fd1} + + f4 := database.LayerFeature{ + database.Feature{ + Name: "feature 2", + Version: "0.3", + VersionFormat: vf2, + }, fd1} + + f5 := database.LayerFeature{ + database.Feature{ + Name: "feature 3", + Version: "0.3", + VersionFormat: vf2, + }, + fd2, } // Suppose Clair is watching two files for namespaces one containing ns1 // changes e.g. os-release and the other one containing ns2 changes e.g. // node. - blank := database.Layer{LayerMetadata: database.LayerMetadata{Hash: "blank"}} + blank := database.Layer{ + Hash: "blank", + By: []database.Detector{nd1, fd1, fd1}, + } initNS1a := database.Layer{ - LayerMetadata: database.LayerMetadata{Hash: "init ns1a"}, - Namespaces: []database.Namespace{ns1a}, - Features: []database.Feature{f1, f2}, + Hash: "initNS1a", + By: []database.Detector{nd1, fd1, fd1}, + Namespaces: []database.LayerNamespace{ns1a}, + Features: []database.LayerFeature{f1, f2}, } upgradeNS2b := database.Layer{ - LayerMetadata: database.LayerMetadata{Hash: "upgrade ns2b"}, - Namespaces: []database.Namespace{ns2b}, + Hash: "upgradeNS2b", + By: []database.Detector{nd1, fd1, fd1}, + Namespaces: []database.LayerNamespace{ns2b}, } upgradeNS1b := database.Layer{ - LayerMetadata: database.LayerMetadata{Hash: "upgrade ns1b"}, - Namespaces: []database.Namespace{ns1b}, - Features: []database.Feature{f1, f2}, + Hash: "upgradeNS1b", + By: []database.Detector{nd1, fd1, fd1, fd2}, + Namespaces: []database.LayerNamespace{ns1b}, + Features: []database.LayerFeature{f1, f2, f5}, } initNS2a := database.Layer{ - LayerMetadata: database.LayerMetadata{Hash: "init ns2a"}, - Namespaces: []database.Namespace{ns2a}, - Features: []database.Feature{f3, f4}, + Hash: "initNS2a", + By: []database.Detector{nd1, fd1, fd1}, + Namespaces: []database.LayerNamespace{ns2a}, + Features: []database.LayerFeature{f3, f4}, } removeF2 := database.Layer{ - LayerMetadata: database.LayerMetadata{Hash: "remove f2"}, - Features: []database.Feature{f1}, + Hash: "removeF2", + By: []database.Detector{nd1, fd1, fd1}, + Features: []database.LayerFeature{f1}, } // blank -> ns1:a, f1 f2 (init) @@ -597,44 +519,65 @@ func TestComputeAncestryFeatures(t *testing.T) { // -> blank (empty) layers := []database.Layer{ - blank, - initNS1a, - removeF2, - initNS2a, - upgradeNS2b, - blank, - upgradeNS1b, - removeF2, + blank, // empty + initNS1a, // namespace: NS1a, features: f1, f2 + removeF2, // namespace: , features: f1 + initNS2a, // namespace: NS2a, features: f3, f4 ( under NS2a ) + upgradeNS2b, // namespace: NS2b, ( f3, f4 are now under NS2b ) + blank, // empty + upgradeNS1b, // namespace: NS1b, ( f1, f2 are now under NS1b, and they are introduced in this layer. ) + removeF2, // namespace: , features: f1 blank, } - expected := map[database.NamespacedFeature]bool{ + expected := []database.AncestryLayer{ + { + "blank", + []database.AncestryFeature{}, + }, { - Feature: f1, - Namespace: ns1a, - }: false, + "initNS1a", + []database.AncestryFeature{{database.NamespacedFeature{f1.Feature, ns1a.Namespace}, f1.By, ns1a.By}}, + }, { - Feature: f3, - Namespace: ns2a, - }: false, + "removeF2", + []database.AncestryFeature{}, + }, { - Feature: f4, - Namespace: ns2a, - }: false, + "initNS2a", + []database.AncestryFeature{ + {database.NamespacedFeature{f3.Feature, ns2a.Namespace}, f3.By, ns2a.By}, + {database.NamespacedFeature{f4.Feature, ns2a.Namespace}, f4.By, ns2a.By}, + }, + }, + { + "upgradeNS2b", + []database.AncestryFeature{}, + }, + { + "blank", + []database.AncestryFeature{}, + }, + { + "upgradeNS1b", + []database.AncestryFeature{}, + }, + { + "removeF2", + []database.AncestryFeature{}, + }, + { + "blank", + []database.AncestryFeature{}, + }, } - ancestryLayers, err := computeAncestryLayers(layers, database.Processors{}) - assert.Nil(t, err) - features := getNamespacedFeatures(ancestryLayers) - for _, f := range features { - if assert.Contains(t, expected, f) { - if assert.False(t, expected[f]) { - expected[f] = true - } - } - } + expectedDetectors := []database.Detector{nd1, fd1} + ancestryLayers, detectors, err := computeAncestryLayers(layers) + require.Nil(t, err) - for f, visited := range expected { - assert.True(t, visited, "expected feature is missing : "+f.Namespace.Name+":"+f.Name) + database.AssertDetectorsEqual(t, expectedDetectors, detectors) + for i := range expected { + database.AssertAncestryLayerEqual(t, &expected[i], &ancestryLayers[i]) } }