arno
/
clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update Database and Worker implementation for layer-wise feature

Browse Source 
Feature extraction algorithm is changed to associate features with
ancestry layer. Database is updated to keep the relationship.
master
Sida Chen 6 years ago committed by Sida Chen
parent 4b64151330
commit 2827b9342b
17 changed files with 619 additions and 489 deletions
Show Stats Download Patch File Download Diff File

170
api/v3/clairpb/clair.pb.go
Unescape View File

@ -297,8 +297,10 @@ func (m *GetAncestryResponse) GetStatus() *ClairStatus {
}
type GetAncestryResponse_AncestryLayer struct {
Layer *Layer `protobuf:"bytes,1,opt,name=layer" json:"layer,omitempty"`
DetectedFeatures []*Feature `protobuf:"bytes,2,rep,name=detectedFeatures" json:"detectedFeatures,omitempty"`
// The layer's information.
Layer *Layer `protobuf:"bytes,1,opt,name=layer" json:"layer,omitempty"`
// The features detected in this layer.
DetectedFeatures []*Feature `protobuf:"bytes,2,rep,name=detected_features,json=detectedFeatures" json:"detected_features,omitempty"`
}
func (m *GetAncestryResponse_AncestryLayer) Reset() { *m = GetAncestryResponse_AncestryLayer{} }
@ -373,7 +375,7 @@ type PostAncestryRequest struct {
// The format of the image being uploaded.
Format string `protobuf:"bytes,2,opt,name=format" json:"format,omitempty"`
// The layers to be scanned for this Ancestry, ordered in the way that i th
// layer is the i + 1 th layer's parent.
// layer is the parent of i + 1 th layer.
Layers []*PostAncestryRequest_PostLayer `protobuf:"bytes,3,rep,name=layers" json:"layers,omitempty"`
}
@ -1023,85 +1025,85 @@ var _StatusService_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("api/v3/clairpb/clair.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 1269 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x57, 0x4b, 0x6f, 0x1b, 0xd5,
0x17, 0xd7, 0xd8, 0x71, 0x6c, 0x1f, 0xdb, 0x49, 0x7a, 0xed, 0xa6, 0x93, 0x49, 0x1f, 0xc9, 0xfc,
0xff, 0x55, 0x4b, 0x8b, 0x6c, 0xd5, 0x65, 0x51, 0xca, 0x02, 0xa5, 0x8f, 0x84, 0x4a, 0xa5, 0xaa,
0xa6, 0x90, 0x05, 0x08, 0x59, 0xd7, 0x33, 0xc7, 0xc9, 0x28, 0xe3, 0x19, 0x33, 0xf7, 0xda, 0xa9,
0x55, 0x75, 0xc3, 0x96, 0x15, 0xb0, 0xe0, 0x33, 0xb0, 0xe1, 0x1b, 0xb0, 0x62, 0x8b, 0x84, 0x04,
0x5b, 0xd8, 0xb1, 0xe0, 0x0b, 0xb0, 0x47, 0xf7, 0x31, 0x93, 0x19, 0xc7, 0x79, 0xb4, 0xac, 0x3c,
0xe7, 0xfd, 0xfa, 0xdd, 0x73, 0x12, 0xb0, 0xe8, 0xc8, 0xef, 0x4c, 0xee, 0x76, 0xdc, 0x80, 0xfa,
0xf1, 0xa8, 0xaf, 0x7e, 0xdb, 0xa3, 0x38, 0xe2, 0x11, 0xa9, 0xbb, 0x51, 0x8c, 0x11, 0x6b, 0x4b,
0x9e, 0x75, 0x6d, 0x2f, 0x8a, 0xf6, 0x02, 0xec, 0x48, 0x59, 0x7f, 0x3c, 0xe8, 0x70, 0x7f, 0x88,
0x8c, 0xd3, 0xe1, 0x48, 0xa9, 0x5b, 0x97, 0xb5, 0x82, 0xf0, 0x48, 0xc3, 0x30, 0xe2, 0x94, 0xfb,
0x51, 0xc8, 0x94, 0xd4, 0xfe, 0xbe, 0x00, 0x8d, 0xdd, 0x71, 0x10, 0x62, 0x4c, 0xfb, 0x7e, 0xe0,
0xf3, 0x29, 0x21, 0xb0, 0x10, 0xd2, 0x21, 0x9a, 0xc6, 0x86, 0x71, 0xb3, 0xea, 0xc8, 0x6f, 0x72,
0x1d, 0x96, 0xc4, 0x2f, 0x1b, 0x51, 0x17, 0x7b, 0x52, 0x5a, 0x90, 0xd2, 0x46, 0xca, 0x7d, 0x26,
0xd4, 0x36, 0xa0, 0xe6, 0x21, 0x73, 0x63, 0x7f, 0x24, 0x42, 0x98, 0x45, 0xa9, 0x93, 0x65, 0x09,
0xe7, 0x81, 0x1f, 0x1e, 0x98, 0x0b, 0xca, 0xb9, 0xf8, 0x26, 0x16, 0x54, 0x18, 0x4e, 0x30, 0xf6,
0xf9, 0xd4, 0x2c, 0x49, 0x7e, 0x4a, 0x0b, 0xd9, 0x10, 0x39, 0xf5, 0x28, 0xa7, 0xe6, 0xa2, 0x92,
0x25, 0x34, 0x59, 0x83, 0xca, 0xc0, 0x7f, 0x89, 0x5e, 0xaf, 0x3f, 0x35, 0xcb, 0x52, 0x56, 0x96,
0xf4, 0x83, 0x29, 0x79, 0x00, 0x17, 0xe8, 0x60, 0x80, 0x2e, 0x47, 0xaf, 0x37, 0xc1, 0x98, 0x89,
0x82, 0xcd, 0xca, 0x46, 0xf1, 0x66, 0xad, 0x7b, 0xb1, 0x9d, 0x6d, 0x5f, 0x7b, 0x1b, 0x29, 0x1f,
0xc7, 0xe8, 0xac, 0x24, 0xfa, 0xbb, 0x5a, 0xdd, 0xfe, 0xc5, 0x80, 0xb2, 0x96, 0xfe, 0x97, 0x9e,
0x98, 0x50, 0xd6, 0x19, 0xe8, 0x7e, 0x24, 0xa4, 0x70, 0xa0, 0x3f, 0x7b, 0x83, 0x28, 0x1e, 0x52,
0xae, 0xbb, 0xd2, 0xd0, 0xdc, 0x6d, 0xc9, 0x24, 0x8f, 0x61, 0x79, 0x92, 0x19, 0x90, 0x8f, 0xcc,
0x2c, 0xc9, 0x4a, 0xd6, 0xf3, 0x95, 0xe4, 0xa6, 0xe8, 0xcc, 0xda, 0xd8, 0xeb, 0x50, 0x7a, 0x4a,
0xa7, 0x18, 0x8b, 0x5a, 0xf6, 0x29, 0xdb, 0x4f, 0x6a, 0x11, 0xdf, 0xf6, 0xd7, 0x06, 0xd4, 0x1e,
0x0a, 0x2f, 0x2f, 0x38, 0xe5, 0x63, 0x26, 0x92, 0x0e, 0x7c, 0xc6, 0x31, 0x66, 0xa6, 0xb1, 0x51,
0x14, 0x49, 0x6b, 0x92, 0x5c, 0x86, 0xaa, 0x87, 0x1c, 0x5d, 0x1e, 0xc5, 0xcc, 0x2c, 0x48, 0xd9,
0x11, 0x83, 0x3c, 0x82, 0x95, 0x80, 0x32, 0xde, 0x1b, 0x8f, 0x3c, 0xca, 0xb1, 0x27, 0xa0, 0x28,
0xab, 0xae, 0x75, 0xad, 0xb6, 0x82, 0x61, 0x3b, 0xc1, 0x69, 0xfb, 0x93, 0x04, 0xa7, 0xce, 0x92,
0xb0, 0xf9, 0x54, 0x9a, 0x08, 0xa6, 0xfd, 0x8d, 0x01, 0x64, 0x07, 0xf9, 0x56, 0xe8, 0x22, 0xe3,
0xf1, 0xd4, 0xc1, 0x2f, 0xc7, 0xc8, 0x38, 0xf9, 0x1f, 0x34, 0xa8, 0x66, 0xf5, 0x32, 0xd3, 0xa8,
0x27, 0x4c, 0xd9, 0xee, 0x3b, 0xd0, 0x3a, 0xf4, 0xf9, 0x7e, 0x6f, 0xb6, 0x65, 0x62, 0x36, 0x15,
0xa7, 0x29, 0x64, 0xbb, 0x79, 0x91, 0xf0, 0x2b, 0x4d, 0x06, 0x6a, 0xd8, 0x4c, 0x66, 0x5c, 0x71,
0xea, 0x82, 0xa9, 0x01, 0xc0, 0xec, 0x5f, 0x8b, 0xd0, 0xcc, 0xe5, 0xc4, 0x46, 0x51, 0xc8, 0x90,
0x6c, 0x43, 0x25, 0x89, 0x2f, 0xf3, 0xa9, 0x75, 0x6f, 0xe5, 0xc7, 0x32, 0xc7, 0xa8, 0x9d, 0x32,
0x52, 0x5b, 0x72, 0x07, 0x16, 0x99, 0xec, 0xbd, 0xcc, 0xb4, 0xd6, 0x5d, 0xcb, 0x7b, 0xc9, 0x0c,
0xc7, 0xd1, 0x8a, 0xd6, 0x6b, 0x68, 0x24, 0x8e, 0xd4, 0x64, 0xdf, 0x81, 0x52, 0x20, 0x3e, 0x74,
0x22, 0xcd, 0xbc, 0x0b, 0xa9, 0xe3, 0x28, 0x0d, 0xb2, 0x05, 0x2b, 0x6a, 0x6a, 0xe8, 0x25, 0x25,
0xca, 0x69, 0x9e, 0xfc, 0x3e, 0x66, 0xd5, 0xad, 0x9f, 0x0c, 0xa8, 0x24, 0xf1, 0xe7, 0x3e, 0x90,
0x1b, 0xb0, 0xcc, 0x5c, 0x1a, 0x86, 0xe8, 0xf5, 0x12, 0x30, 0x2d, 0x48, 0xc0, 0x2c, 0x69, 0xf6,
0x53, 0x8d, 0xa9, 0xdb, 0x70, 0x21, 0x51, 0x3c, 0xc2, 0x56, 0x49, 0xaa, 0xae, 0x68, 0xc1, 0xa3,
0x14, 0x62, 0x3b, 0xb0, 0x28, 0x4b, 0x60, 0xe6, 0xa2, 0xcc, 0xb7, 0x73, 0xfe, 0x76, 0xab, 0x0e,
0x68, 0x73, 0xfb, 0xcf, 0x02, 0x34, 0x9f, 0x47, 0xec, 0xed, 0x60, 0xb6, 0x0a, 0x8b, 0xfa, 0xcd,
0xaa, 0x47, 0xaf, 0x29, 0xf2, 0x30, 0xcd, 0xae, 0x28, 0xb3, 0xbb, 0x9d, 0xcf, 0x6e, 0x4e, 0x3c,
0xc9, 0xcb, 0x65, 0x66, 0xfd, 0x6c, 0x40, 0x35, 0xe5, 0xce, 0x7b, 0xaf, 0x82, 0x37, 0xa2, 0x7c,
0x5f, 0x07, 0x97, 0xdf, 0xc4, 0x81, 0xf2, 0x3e, 0x52, 0xef, 0x28, 0xf6, 0xbd, 0x37, 0x88, 0xdd,
0xfe, 0x48, 0x99, 0x3e, 0x0e, 0x85, 0x34, 0x71, 0x64, 0xdd, 0x87, 0x7a, 0x56, 0x40, 0x56, 0xa0,
0x78, 0x80, 0x53, 0x9d, 0x8a, 0xf8, 0x24, 0x2d, 0x28, 0x4d, 0x68, 0x30, 0x4e, 0x96, 0x9f, 0x22,
0xee, 0x17, 0xee, 0x19, 0xf6, 0x13, 0x68, 0xe5, 0x43, 0xea, 0x17, 0x73, 0x84, 0x74, 0xe3, 0x9c,
0x48, 0xb7, 0x7f, 0x34, 0x60, 0x75, 0x07, 0xf9, 0xb3, 0x88, 0xfb, 0x03, 0xdf, 0x95, 0xf7, 0x2b,
0x99, 0xd6, 0x7b, 0xb0, 0x1a, 0x05, 0x5e, 0xee, 0xb9, 0x4f, 0x7b, 0x23, 0xba, 0x97, 0x8c, 0xad,
0x15, 0x05, 0x5e, 0x6e, 0x33, 0x3e, 0xa7, 0x7b, 0x28, 0xac, 0x42, 0x3c, 0x9c, 0x67, 0xa5, 0xca,
0x68, 0x85, 0x78, 0x78, 0xdc, 0xaa, 0x05, 0xa5, 0xc0, 0x1f, 0xfa, 0x5c, 0x2e, 0x88, 0x92, 0xa3,
0x88, 0x14, 0xfa, 0x0b, 0x47, 0xd0, 0xb7, 0xff, 0x28, 0xc0, 0xa5, 0x63, 0x09, 0xeb, 0xfa, 0x77,
0xa1, 0x1e, 0x66, 0xf8, 0xba, 0x0b, 0xdd, 0x63, 0x30, 0x9e, 0x67, 0xdc, 0xce, 0x31, 0x73, 0x7e,
0xac, 0xbf, 0x0d, 0xa8, 0x67, 0xc5, 0x73, 0xdf, 0xa4, 0x09, 0x65, 0x37, 0x46, 0xca, 0xd1, 0xd3,
0x95, 0x26, 0xa4, 0xb8, 0xb4, 0xca, 0x1d, 0x7a, 0xfa, 0x50, 0xa5, 0xb4, 0xb0, 0xf2, 0x30, 0x40,
0x61, 0xa5, 0xaa, 0x4c, 0x48, 0xf2, 0x3e, 0x14, 0xa3, 0xc0, 0x93, 0x67, 0xbb, 0xd6, 0xbd, 0x31,
0x03, 0x38, 0xba, 0x87, 0x69, 0xef, 0x03, 0xd4, 0x40, 0xf0, 0x91, 0x39, 0xc2, 0x46, 0x98, 0x86,
0x78, 0x28, 0xaf, 0xfa, 0x9b, 0x98, 0x86, 0x78, 0x68, 0xff, 0x56, 0x80, 0xb5, 0x13, 0x55, 0xc8,
0x26, 0xd4, 0xdd, 0x71, 0x1c, 0x63, 0xc8, 0xb3, 0x40, 0xa8, 0x69, 0x9e, 0x9c, 0xe4, 0x3a, 0x54,
0x43, 0x7c, 0xc9, 0xb3, 0x23, 0xaf, 0x08, 0xc6, 0x29, 0x63, 0xde, 0x82, 0x46, 0x0e, 0x2e, 0xb2,
0x13, 0x67, 0x1c, 0xe1, 0xbc, 0x05, 0xf9, 0x1c, 0x80, 0xa6, 0x69, 0xea, 0x23, 0xfe, 0xc1, 0x39,
0x0b, 0x6f, 0x3f, 0x09, 0x3d, 0x7c, 0x89, 0xde, 0x56, 0x66, 0x0b, 0x39, 0x19, 0x77, 0xd6, 0x87,
0xd0, 0x9c, 0xa3, 0x22, 0x8a, 0xf1, 0x05, 0x5b, 0x76, 0xa1, 0xe4, 0x28, 0x22, 0x85, 0x46, 0x21,
0x83, 0xd9, 0xbb, 0x70, 0xe5, 0x63, 0x1a, 0x1f, 0x64, 0x21, 0xb4, 0xc5, 0x1c, 0xa4, 0x5e, 0xf2,
0xd4, 0xe6, 0xe0, 0xc9, 0xde, 0x80, 0xab, 0x27, 0x19, 0x29, 0xc4, 0xda, 0x04, 0x56, 0x76, 0x90,
0xeb, 0x07, 0xad, 0x3c, 0xd9, 0xdb, 0x70, 0x21, 0xc3, 0x7b, 0xeb, 0xbd, 0xd0, 0xfd, 0xc7, 0x80,
0xe5, 0xa4, 0xda, 0x17, 0x18, 0x4f, 0x7c, 0x17, 0xc9, 0x18, 0x6a, 0x99, 0x1b, 0x40, 0x36, 0x4e,
0x39, 0x0f, 0x32, 0x19, 0x6b, 0xf3, 0xcc, 0x03, 0x62, 0x6f, 0x7e, 0xf5, 0xfb, 0x5f, 0xdf, 0x15,
0xd6, 0xc9, 0x5a, 0x27, 0x39, 0x02, 0x9d, 0x57, 0xb9, 0x1b, 0xf1, 0x9a, 0x1c, 0x40, 0x3d, 0xbb,
0xed, 0xc8, 0xe6, 0x99, 0xcb, 0xd7, 0xb2, 0x4f, 0x53, 0xd1, 0x91, 0x5b, 0x32, 0xf2, 0x92, 0x5d,
0x4d, 0x23, 0xdf, 0x37, 0x6e, 0x75, 0x7f, 0x28, 0x40, 0x33, 0xdb, 0xf2, 0xa4, 0xf6, 0xd7, 0xb0,
0x3c, 0xb3, 0x38, 0xc8, 0xff, 0xcf, 0xd8, 0x2b, 0x2a, 0x95, 0xeb, 0xe7, 0xda, 0x3e, 0xf6, 0x15,
0x99, 0xcd, 0x25, 0x72, 0xb1, 0x93, 0xdd, 0x3c, 0xac, 0xf3, 0x4a, 0xf5, 0xe0, 0x5b, 0x03, 0x56,
0xe7, 0xa3, 0x81, 0xcc, 0xdc, 0xc1, 0x53, 0x81, 0x66, 0xbd, 0x7b, 0x3e, 0xe5, 0x7c, 0x52, 0xb7,
0xe6, 0x27, 0xd5, 0x0d, 0xa1, 0xa1, 0x50, 0x93, 0x34, 0xe9, 0x0b, 0xa8, 0xa6, 0xe0, 0x23, 0x57,
0x8f, 0x15, 0x9e, 0x43, 0xaa, 0x75, 0xed, 0x44, 0xb9, 0x8e, 0xbe, 0x2c, 0xa3, 0x57, 0x49, 0xb9,
0xa3, 0x30, 0xf9, 0xe0, 0x2a, 0x34, 0xdd, 0x68, 0x98, 0x37, 0x1b, 0xf5, 0x3f, 0x2b, 0xeb, 0xff,
0xe4, 0xfa, 0x8b, 0xf2, 0x0f, 0xe0, 0xbb, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x9d, 0x0c, 0x7d,
0x93, 0xe2, 0x0d, 0x00, 0x00,
// 1268 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x57, 0x4b, 0x6f, 0x1b, 0x55,
0x14, 0xd6, 0x38, 0x71, 0x6c, 0x1f, 0xdb, 0x49, 0x7a, 0xed, 0xa6, 0x93, 0x49, 0x1f, 0xc9, 0x40,
0xd5, 0xd2, 0x22, 0x5b, 0x75, 0x59, 0x94, 0xb2, 0x40, 0xe9, 0x23, 0xa1, 0x52, 0xa9, 0xaa, 0x29,
0x64, 0x01, 0x42, 0xd6, 0xcd, 0xcc, 0x71, 0x32, 0xca, 0x78, 0xc6, 0xcc, 0xbd, 0x4e, 0x6a, 0x55,
0x65, 0xc1, 0x96, 0x15, 0xb0, 0xe0, 0x37, 0xb0, 0xe1, 0x1f, 0xb0, 0x62, 0xcb, 0x02, 0xc1, 0x16,
0x76, 0x2c, 0xf8, 0x03, 0xec, 0xd1, 0x7d, 0x4d, 0x66, 0x12, 0x37, 0x49, 0xcb, 0xca, 0x73, 0xde,
0xaf, 0xef, 0x9e, 0x93, 0x80, 0x43, 0x47, 0x61, 0x77, 0xff, 0x76, 0xd7, 0x8f, 0x68, 0x98, 0x8e,
0xb6, 0xd5, 0x6f, 0x67, 0x94, 0x26, 0x3c, 0x21, 0x0d, 0x3f, 0x49, 0x31, 0x61, 0x1d, 0xc9, 0x73,
0xae, 0xec, 0x24, 0xc9, 0x4e, 0x84, 0x5d, 0x29, 0xdb, 0x1e, 0x0f, 0xba, 0x3c, 0x1c, 0x22, 0xe3,
0x74, 0x38, 0x52, 0xea, 0xce, 0x45, 0xad, 0x20, 0x3c, 0xd2, 0x38, 0x4e, 0x38, 0xe5, 0x61, 0x12,
0x33, 0x25, 0x75, 0x7f, 0x28, 0x41, 0x73, 0x6b, 0x1c, 0xc5, 0x98, 0xd2, 0xed, 0x30, 0x0a, 0xf9,
0x84, 0x10, 0x98, 0x8d, 0xe9, 0x10, 0x6d, 0x6b, 0xd5, 0xba, 0x5e, 0xf3, 0xe4, 0x37, 0xb9, 0x0a,
0xf3, 0xe2, 0x97, 0x8d, 0xa8, 0x8f, 0x7d, 0x29, 0x2d, 0x49, 0x69, 0x33, 0xe3, 0x3e, 0x11, 0x6a,
0xab, 0x50, 0x0f, 0x90, 0xf9, 0x69, 0x38, 0x12, 0x21, 0xec, 0x19, 0xa9, 0x93, 0x67, 0x09, 0xe7,
0x51, 0x18, 0xef, 0xd9, 0xb3, 0xca, 0xb9, 0xf8, 0x26, 0x0e, 0x54, 0x19, 0xee, 0x63, 0x1a, 0xf2,
0x89, 0x5d, 0x96, 0xfc, 0x8c, 0x16, 0xb2, 0x21, 0x72, 0x1a, 0x50, 0x4e, 0xed, 0x39, 0x25, 0x33,
0x34, 0x59, 0x86, 0xea, 0x20, 0x7c, 0x8e, 0x41, 0x7f, 0x7b, 0x62, 0x57, 0xa4, 0xac, 0x22, 0xe9,
0x7b, 0x13, 0x72, 0x0f, 0xce, 0xd1, 0xc1, 0x00, 0x7d, 0x8e, 0x41, 0x7f, 0x1f, 0x53, 0x26, 0x0a,
0xb6, 0xab, 0xab, 0x33, 0xd7, 0xeb, 0xbd, 0xf3, 0x9d, 0x7c, 0xfb, 0x3a, 0x1b, 0x48, 0xf9, 0x38,
0x45, 0x6f, 0xd1, 0xe8, 0x6f, 0x69, 0x75, 0xf7, 0x57, 0x0b, 0x2a, 0x5a, 0xfa, 0x7f, 0x7a, 0x62,
0x43, 0x45, 0x67, 0xa0, 0xfb, 0x61, 0x48, 0xe1, 0x40, 0x7f, 0xf6, 0x07, 0x49, 0x3a, 0xa4, 0x5c,
0x77, 0xa5, 0xa9, 0xb9, 0x1b, 0x92, 0x49, 0x1e, 0xc2, 0xc2, 0x7e, 0x6e, 0x40, 0x21, 0x32, 0xbb,
0x2c, 0x2b, 0x59, 0x29, 0x56, 0x52, 0x98, 0xa2, 0x77, 0xd4, 0xc6, 0x5d, 0x81, 0xf2, 0x63, 0x3a,
0xc1, 0x54, 0xd4, 0xb2, 0x4b, 0xd9, 0xae, 0xa9, 0x45, 0x7c, 0xbb, 0xdf, 0x58, 0x50, 0xbf, 0x2f,
0xbc, 0x3c, 0xe3, 0x94, 0x8f, 0x99, 0x48, 0x3a, 0x0a, 0x19, 0xc7, 0x94, 0xd9, 0xd6, 0xea, 0x8c,
0x48, 0x5a, 0x93, 0xe4, 0x22, 0xd4, 0x02, 0xe4, 0xe8, 0xf3, 0x24, 0x65, 0x76, 0x49, 0xca, 0x0e,
0x19, 0xe4, 0x01, 0x2c, 0x46, 0x94, 0xf1, 0xfe, 0x78, 0x14, 0x50, 0x8e, 0x7d, 0x01, 0x45, 0x59,
0x75, 0xbd, 0xe7, 0x74, 0x14, 0x0c, 0x3b, 0x06, 0xa7, 0x9d, 0x4f, 0x0c, 0x4e, 0xbd, 0x79, 0x61,
0xf3, 0xa9, 0x34, 0x11, 0x4c, 0xf7, 0x5b, 0x0b, 0xc8, 0x26, 0xf2, 0xf5, 0xd8, 0x47, 0xc6, 0xd3,
0x89, 0x87, 0x5f, 0x8e, 0x91, 0x71, 0xf2, 0x16, 0x34, 0xa9, 0x66, 0xf5, 0x73, 0xd3, 0x68, 0x18,
0xa6, 0x6c, 0xf7, 0x2d, 0x68, 0x1f, 0x84, 0x7c, 0xb7, 0x7f, 0xb4, 0x65, 0x62, 0x36, 0x55, 0xaf,
0x25, 0x64, 0x5b, 0x45, 0x91, 0xf0, 0x2b, 0x4d, 0x06, 0x6a, 0xd8, 0x4c, 0x66, 0x5c, 0xf5, 0x1a,
0x82, 0xa9, 0x01, 0xc0, 0xdc, 0xdf, 0x66, 0xa0, 0x55, 0xc8, 0x89, 0x8d, 0x92, 0x98, 0x21, 0xd9,
0x80, 0xaa, 0x89, 0x2f, 0xf3, 0xa9, 0xf7, 0x6e, 0x14, 0xc7, 0x32, 0xc5, 0xa8, 0x93, 0x31, 0x32,
0x5b, 0x72, 0x0b, 0xe6, 0x98, 0xec, 0xbd, 0xcc, 0xb4, 0xde, 0x5b, 0x2e, 0x7a, 0xc9, 0x0d, 0xc7,
0xd3, 0x8a, 0xce, 0x57, 0xd0, 0x34, 0x8e, 0xd4, 0x64, 0xdf, 0x81, 0x72, 0x24, 0x3e, 0x74, 0x22,
0xad, 0xa2, 0x0b, 0xa9, 0xe3, 0x29, 0x0d, 0xf1, 0x40, 0xd4, 0xd4, 0x30, 0x38, 0xac, 0xbb, 0x74,
0xe2, 0x03, 0x31, 0xfa, 0xa6, 0x25, 0xce, 0xcf, 0x16, 0x54, 0x4d, 0x02, 0x53, 0x5f, 0xc8, 0x35,
0x58, 0x60, 0x3e, 0x8d, 0x63, 0x0c, 0xfa, 0x06, 0x4d, 0xb3, 0x12, 0x31, 0xf3, 0x9a, 0xfd, 0x58,
0x83, 0xea, 0x26, 0x9c, 0x33, 0x8a, 0x87, 0xe0, 0x2a, 0x4b, 0xd5, 0x45, 0x2d, 0x78, 0x90, 0x61,
0x6c, 0x13, 0xe6, 0x64, 0x0d, 0xcc, 0x9e, 0x93, 0xf9, 0x76, 0xcf, 0xde, 0x6f, 0xd5, 0x02, 0x6d,
0xee, 0xfe, 0x55, 0x82, 0xd6, 0xd3, 0x84, 0xbd, 0x19, 0xce, 0x96, 0x60, 0x4e, 0x3f, 0x5a, 0xf5,
0xea, 0x35, 0x45, 0xee, 0x67, 0xd9, 0xcd, 0xc8, 0xec, 0x6e, 0x16, 0xb3, 0x9b, 0x12, 0x4f, 0xf2,
0x0a, 0x99, 0x39, 0xbf, 0x58, 0x50, 0xcb, 0xb8, 0xd3, 0x1e, 0xac, 0xe0, 0x8d, 0x28, 0xdf, 0xd5,
0xc1, 0xe5, 0x37, 0xf1, 0xa0, 0xb2, 0x8b, 0x34, 0x38, 0x8c, 0x7d, 0xe7, 0x35, 0x62, 0x77, 0x3e,
0x52, 0xa6, 0x0f, 0x63, 0x21, 0x35, 0x8e, 0x9c, 0xbb, 0xd0, 0xc8, 0x0b, 0xc8, 0x22, 0xcc, 0xec,
0xe1, 0x44, 0xa7, 0x22, 0x3e, 0x49, 0x1b, 0xca, 0xfb, 0x34, 0x1a, 0x9b, 0xed, 0xa7, 0x88, 0xbb,
0xa5, 0x3b, 0x96, 0xfb, 0x08, 0xda, 0xc5, 0x90, 0xfa, 0xc9, 0x1c, 0x42, 0xdd, 0x3a, 0x23, 0xd4,
0xdd, 0x9f, 0x2c, 0x58, 0xda, 0x44, 0xfe, 0x24, 0xe1, 0xe1, 0x20, 0xf4, 0xe5, 0x01, 0x33, 0xd3,
0x7a, 0x0f, 0x96, 0x92, 0x28, 0x28, 0xbc, 0xf7, 0x49, 0x7f, 0x44, 0x77, 0xcc, 0xd8, 0xda, 0x49,
0x14, 0x14, 0x56, 0xe3, 0x53, 0xba, 0x83, 0xc2, 0x2a, 0xc6, 0x83, 0x69, 0x56, 0xaa, 0x8c, 0x76,
0x8c, 0x07, 0xc7, 0xad, 0xda, 0x50, 0x8e, 0xc2, 0x61, 0xc8, 0xe5, 0x86, 0x28, 0x7b, 0x8a, 0xc8,
0xa0, 0x3f, 0x7b, 0x08, 0x7d, 0xf7, 0xcf, 0x12, 0x5c, 0x38, 0x96, 0xb0, 0xae, 0x7f, 0x0b, 0x1a,
0x71, 0x8e, 0xaf, 0xbb, 0xd0, 0x3b, 0x06, 0xe3, 0x69, 0xc6, 0x9d, 0x02, 0xb3, 0xe0, 0xc7, 0xf9,
0xc7, 0x82, 0x46, 0x5e, 0x3c, 0xf5, 0x4d, 0xda, 0x50, 0xf1, 0x53, 0xa4, 0x1c, 0x03, 0x5d, 0xa9,
0x21, 0xc5, 0xa9, 0x55, 0xee, 0x30, 0xd0, 0x97, 0x2a, 0xa3, 0x85, 0x55, 0x80, 0x11, 0x0a, 0x2b,
0x55, 0xa5, 0x21, 0xc9, 0xfb, 0x30, 0x93, 0x44, 0x81, 0xbc, 0xdb, 0xf5, 0xde, 0xb5, 0x23, 0x80,
0xa3, 0x3b, 0x98, 0xf5, 0x3e, 0x42, 0x0d, 0x84, 0x10, 0x99, 0x27, 0x6c, 0x84, 0x69, 0x8c, 0x07,
0xf2, 0xac, 0xbf, 0x8e, 0x69, 0x8c, 0x07, 0xee, 0xef, 0x25, 0x58, 0x7e, 0xa5, 0x0a, 0x59, 0x83,
0x86, 0x3f, 0x4e, 0x53, 0x8c, 0x79, 0x1e, 0x08, 0x75, 0xcd, 0x93, 0x93, 0x5c, 0x81, 0x5a, 0x8c,
0xcf, 0x79, 0x7e, 0xe4, 0x55, 0xc1, 0x38, 0x61, 0xcc, 0xeb, 0xd0, 0x2c, 0xc0, 0x45, 0x76, 0xe2,
0x94, 0x2b, 0x5c, 0xb4, 0x20, 0x9f, 0x03, 0xd0, 0x2c, 0x4d, 0x7d, 0xc5, 0x3f, 0x38, 0x63, 0xe1,
0x9d, 0x47, 0x71, 0x80, 0xcf, 0x31, 0x58, 0xcf, 0x6d, 0x21, 0x2f, 0xe7, 0xce, 0xf9, 0x10, 0x5a,
0x53, 0x54, 0x44, 0x31, 0xa1, 0x60, 0xcb, 0x2e, 0x94, 0x3d, 0x45, 0x64, 0xd0, 0x28, 0xe5, 0x30,
0x7b, 0x1b, 0x2e, 0x7d, 0x4c, 0xd3, 0xbd, 0x3c, 0x84, 0xd6, 0x99, 0x87, 0x34, 0x30, 0x4f, 0x6d,
0x0a, 0x9e, 0xdc, 0x55, 0xb8, 0xfc, 0x2a, 0x23, 0x85, 0x58, 0x97, 0xc0, 0xe2, 0x26, 0x72, 0xfd,
0xa0, 0x95, 0x27, 0x77, 0x03, 0xce, 0xe5, 0x78, 0x6f, 0xbc, 0x17, 0x7a, 0xff, 0x5a, 0xb0, 0x60,
0xaa, 0x7d, 0x86, 0xe9, 0x7e, 0xe8, 0x23, 0x19, 0x43, 0x3d, 0x77, 0x03, 0xc8, 0xea, 0x09, 0xe7,
0x41, 0x26, 0xe3, 0xac, 0x9d, 0x7a, 0x40, 0xdc, 0xb5, 0xaf, 0xff, 0xf8, 0xfb, 0xfb, 0xd2, 0x0a,
0x59, 0xee, 0x9a, 0x23, 0xd0, 0x7d, 0x51, 0xb8, 0x11, 0x2f, 0xc9, 0x1e, 0x34, 0xf2, 0xdb, 0x8e,
0xac, 0x9d, 0xba, 0x7c, 0x1d, 0xf7, 0x24, 0x15, 0x1d, 0xb9, 0x2d, 0x23, 0xcf, 0xbb, 0xb5, 0x2c,
0xf2, 0x5d, 0xeb, 0x46, 0xef, 0xc7, 0x12, 0xb4, 0xf2, 0x2d, 0x37, 0xb5, 0xbf, 0x84, 0x85, 0x23,
0x8b, 0x83, 0xbc, 0x7d, 0xca, 0x5e, 0x51, 0xa9, 0x5c, 0x3d, 0xd3, 0xf6, 0x71, 0x2f, 0xc9, 0x6c,
0x2e, 0x90, 0xf3, 0xdd, 0xfc, 0xe6, 0x61, 0xdd, 0x17, 0xaa, 0x07, 0xdf, 0x59, 0xb0, 0x34, 0x1d,
0x0d, 0xe4, 0xc8, 0x1d, 0x3c, 0x11, 0x68, 0xce, 0xbb, 0x67, 0x53, 0x2e, 0x26, 0x75, 0x63, 0x7a,
0x52, 0xbd, 0x18, 0x9a, 0x0a, 0x35, 0xa6, 0x49, 0x5f, 0x40, 0x2d, 0x03, 0x1f, 0xb9, 0x7c, 0xac,
0xf0, 0x02, 0x52, 0x9d, 0x2b, 0xaf, 0x94, 0xeb, 0xe8, 0x0b, 0x32, 0x7a, 0x8d, 0x54, 0xba, 0x0a,
0x93, 0xf7, 0x2e, 0x43, 0xcb, 0x4f, 0x86, 0x45, 0xb3, 0xd1, 0xf6, 0x67, 0x15, 0xfd, 0xaf, 0xdc,
0xf6, 0x9c, 0xfc, 0x0b, 0xf8, 0xf6, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xc7, 0xaf, 0x17, 0x4a,
0xe3, 0x0d, 0x00, 0x00,
}

3
api/v3/clairpb/clair.proto
Unescape View File

@ -99,9 +99,8 @@ message GetAncestryResponse {
// The layer's information.
Layer layer = 1;
// The features detected in this layer.
repeated Feature detectedFeatures = 2;
repeated Feature detected_features = 2;
}
message Ancestry {
// The name of the desired ancestry.
string name = 1;

10
api/v3/clairpb/clair.swagger.json
Unescape View File

@ -206,13 +206,15 @@
"type": "object",
"properties": {
"layer": {
"$ref": "#/definitions/clairLayer"
"$ref": "#/definitions/clairLayer",
"description": "The layer's information."
},
"detectedFeatures": {
"detected_features": {
"type": "array",
"items": {
"$ref": "#/definitions/clairFeature"
}
},
"description": "The features detected in this layer."
}
}
},
@ -419,7 +421,7 @@
"items": {
"$ref": "#/definitions/PostAncestryRequestPostLayer"
},
"description": "The layers to be scanned for this Ancestry, ordered in the way that i th\nlayer is the i + 1 th layer's parent."
"description": "The layers to be scanned for this Ancestry, ordered in the way that i th\nlayer is the parent of i + 1 th layer."
}
}
},

14
api/v3/clairpb/convert.go
Unescape View File

@ -124,11 +124,17 @@ func VulnerabilityWithFixedInFromDatabaseModel(dbVuln database.VulnerabilityWith
// AncestryFromDatabaseModel converts database ancestry to api ancestry.
func AncestryFromDatabaseModel(dbAncestry database.Ancestry) *GetAncestryResponse_Ancestry {
ancestry := &GetAncestryResponse_Ancestry{Name: dbAncestry.Name}
ancestry := &GetAncestryResponse_Ancestry{
Name: dbAncestry.Name,
ScannedDetectors: dbAncestry.ProcessedBy.Detectors,
ScannedListers: dbAncestry.ProcessedBy.Listers,
}
for _, layer := range dbAncestry.Layers {
ancestryLayer := &GetAncestryResponse_AncestryLayer{}
ancestryLayer.Layer = LayerFromDatabaseModel(layer)
ancestry.Layers = append(ancestry.Layers, ancestryLayer)
ancestry.Layers = append(ancestry.Layers,
&GetAncestryResponse_AncestryLayer{
Layer: LayerFromDatabaseModel(layer),
})
}
return ancestry

15
api/v3/rpc.go
Unescape View File

@ -130,13 +130,18 @@ func (s *AncestryServer) GetAncestry(ctx context.Context, req *pb.GetAncestryReq
return nil, status.Error(codes.NotFound, fmt.Sprintf("requested ancestry '%s' is not found", req.GetAncestryName()))
}
respAncestry = &pb.GetAncestryResponse_Ancestry{Name: name}
respAncestry.ScannedDetectors = ancestry.ProcessedBy.Detectors
respAncestry.ScannedListers = ancestry.ProcessedBy.Listers
respAncestry.Layers = []*pb.GetAncestryResponse_AncestryLayer{}
respAncestry = &pb.GetAncestryResponse_Ancestry{
Name: name,
ScannedDetectors: ancestry.ProcessedBy.Detectors,
ScannedListers: ancestry.ProcessedBy.Listers,
}
for _, layer := range ancestry.Layers {
ancestryLayer := &pb.GetAncestryResponse_AncestryLayer{}
ancestryLayer := &pb.GetAncestryResponse_AncestryLayer{
Layer: &pb.Layer{
Hash: layer.Hash,
},
}
if req.GetWithVulnerabilities() {
featureVulnerabilities, err := tx.FindAffectedNamespacedFeatures(layer.DetectedFeatures)

18
database/database.go
Unescape View File

@ -91,18 +91,18 @@ type Session interface {
// UpsertAncestry inserts or replaces an ancestry and its namespaced
// features and processors used to scan the ancestry.
UpsertAncestry(ancestry Ancestry, features []NamespacedFeature, processedBy Processors) error
UpsertAncestry(AncestryWithContent) error
// FindAncestry retrieves an ancestry with processors used to scan the
// ancestry. If the ancestry is not found, return false.
//
// The ancestry's processors are returned to short cut processing ancestry
// if it has been processed by all processors in the current Clair instance.
FindAncestry(name string) (ancestry Ancestry, processedBy Processors, found bool, err error)
FindAncestry(name string) (ancestry Ancestry, found bool, err error)
// FindAncestryFeatures retrieves an ancestry with all detected namespaced
// features. If the ancestry is not found, return false.
FindAncestryFeatures(name string) (ancestry AncestryWithFeatures, found bool, err error)
// FindAncestryWithContent retrieves an ancestry with all detected
// namespaced features. If the ancestry is not found, return false.
FindAncestryWithContent(name string) (ancestry AncestryWithContent, found bool, err error)
// PersistFeatures inserts a set of features if not in the database.
PersistFeatures(features []Feature) error
@ -125,8 +125,8 @@ type Session interface {
// PersistNamespaces inserts a set of namespaces if not in the database.
PersistNamespaces([]Namespace) error
// PersistLayer inserts a layer if not in the datastore.
PersistLayer(Layer) error
// PersistLayer creates a layer using the blob Sum hash.
PersistLayer(hash string) error
// PersistLayerContent persists a layer's content in the database. The given
// namespaces and features can be partial content of this layer.
@ -135,8 +135,8 @@ type Session interface {
// in the database.
PersistLayerContent(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error
// FindLayer retrieves a layer and the processors scanned the layer.
FindLayer(hash string) (layer Layer, processedBy Processors, found bool, err error)
// FindLayer retrieves the metadata of a layer.
FindLayer(hash string) (layer Layer, found bool, err error)
// FindLayerWithContent returns a layer with all detected features and
// namespaces.

26
database/mock.go
Unescape View File

@ -21,17 +21,17 @@ import "time"
type MockSession struct {
FctCommit func() error
FctRollback func() error
FctUpsertAncestry func(Ancestry, []NamespacedFeature, Processors) error
FctFindAncestry func(name string) (Ancestry, Processors, bool, error)
FctFindAncestryFeatures func(name string) (AncestryWithFeatures, bool, error)
FctUpsertAncestry func(AncestryWithContent) error
FctFindAncestry func(name string) (Ancestry, bool, error)
FctFindAncestryWithContent func(name string) (AncestryWithContent, bool, error)
FctFindAffectedNamespacedFeatures func(features []NamespacedFeature) ([]NullableAffectedNamespacedFeature, error)
FctPersistNamespaces func([]Namespace) error
FctPersistFeatures func([]Feature) error
FctPersistNamespacedFeatures func([]NamespacedFeature) error
FctCacheAffectedNamespacedFeatures func([]NamespacedFeature) error
FctPersistLayer func(Layer) error
FctPersistLayer func(hash string) error
FctPersistLayerContent func(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error
FctFindLayer func(name string) (Layer, Processors, bool, error)
FctFindLayer func(name string) (Layer, bool, error)
FctFindLayerWithContent func(name string) (LayerWithContent, bool, error)
FctInsertVulnerabilities func([]VulnerabilityWithAffected) error
FctFindVulnerabilities func([]VulnerabilityID) ([]NullableVulnerability, error)
@ -63,23 +63,23 @@ func (ms *MockSession) Rollback() error {
panic("required mock function not implemented")
}
func (ms *MockSession) UpsertAncestry(ancestry Ancestry, features []NamespacedFeature, processedBy Processors) error {
func (ms *MockSession) UpsertAncestry(ancestry AncestryWithContent) error {
if ms.FctUpsertAncestry != nil {
return ms.FctUpsertAncestry(ancestry, features, processedBy)
return ms.FctUpsertAncestry(ancestry)
}
panic("required mock function not implemented")
}
func (ms *MockSession) FindAncestry(name string) (Ancestry, Processors, bool, error) {
func (ms *MockSession) FindAncestry(name string) (Ancestry, bool, error) {
if ms.FctFindAncestry != nil {
return ms.FctFindAncestry(name)
}
panic("required mock function not implemented")
}
func (ms *MockSession) FindAncestryFeatures(name string) (AncestryWithFeatures, bool, error) {
if ms.FctFindAncestryFeatures != nil {
return ms.FctFindAncestryFeatures(name)
func (ms *MockSession) FindAncestryWithContent(name string) (AncestryWithContent, bool, error) {
if ms.FctFindAncestryWithContent != nil {
return ms.FctFindAncestryWithContent(name)
}
panic("required mock function not implemented")
}
@ -119,7 +119,7 @@ func (ms *MockSession) CacheAffectedNamespacedFeatures(namespacedFeatures []Name
panic("required mock function not implemented")
}
func (ms *MockSession) PersistLayer(layer Layer) error {
func (ms *MockSession) PersistLayer(layer string) error {
if ms.FctPersistLayer != nil {
return ms.FctPersistLayer(layer)
}
@ -133,7 +133,7 @@ func (ms *MockSession) PersistLayerContent(hash string, namespaces []Namespace,
panic("required mock function not implemented")
}
func (ms *MockSession) FindLayer(name string) (Layer, Processors, bool, error) {
func (ms *MockSession) FindLayer(name string) (Layer, bool, error) {
if ms.FctFindLayer != nil {
return ms.FctFindLayer(name)
}

35
database/models.go
Unescape View File

@ -20,7 +20,7 @@ import (
"time"
)
// Processors are extentions to scan layer's content.
// Processors are extentions to scan a layer's content.
type Processors struct {
Listers []string
Detectors []string
@ -29,24 +29,39 @@ type Processors struct {
// Ancestry is a manifest that keeps all layers in an image in order.
type Ancestry struct {
Name string
// ProcessedBy contains the processors that are used when computing the
// content of this ancestry.
ProcessedBy Processors
// Layers should be ordered and i_th layer is the parent of i+1_th layer in
// the slice.
Layers []Layer
}
// AncestryWithFeatures is an ancestry with namespaced features detected in the
// ancestry, which is processed by `ProcessedBy`.
type AncestryWithFeatures struct {
// AncestryWithContent has the ancestry's name and the Ancestry Layers
// associated with it.
type AncestryWithContent struct {
Ancestry
ProcessedBy Processors
Features []NamespacedFeature
// TODO(sidchen) deduplicate the Layers here and the Layers in
// Ancestry.Layers.
// AncestryLayers should have the same order as Ancestry.Layers.
Layers []AncestryLayer
}
// AncestryLayer is a layer with all detected namespaced features.
type AncestryLayer struct {
Layer
// DetectedFeatures are the features introduced by this layer.
DetectedFeatures []NamespacedFeature
}
// Layer corresponds to a layer in an image processed by `ProcessedBy`.
// Layer contains the metadata of a layer.
type Layer struct {
// Hash is content hash of the layer.
Hash string
// ProcessedBy contains the processors that processed this layer.
ProcessedBy Processors
}
// LayerWithContent is a layer with its detected namespaces and features by
@ -54,9 +69,8 @@ type Layer struct {
type LayerWithContent struct {
Layer
ProcessedBy Processors
Namespaces []Namespace
Features []Feature
Namespaces []Namespace
Features []Feature
}
// Namespace is the contextual information around features.
@ -198,6 +212,7 @@ type VulnerabilityNotificationWithVulnerable struct {
// PageNumber is used to do pagination.
type PageNumber string
// MetadataMap is for storing the metadata returned by vulnerability database.
type MetadataMap map[string]interface{}
// NullableAffectedNamespacedFeature is an affectednamespacedfeature with

201
database/pgsql/ancestry.go
Unescape View File

@ -3,48 +3,37 @@ package pgsql
import (
"database/sql"
"errors"
"fmt"
"strings"
"github.com/lib/pq"
log "github.com/sirupsen/logrus"
"github.com/coreos/clair/database"
"github.com/coreos/clair/pkg/commonerr"
)
func (tx *pgSession) UpsertAncestry(ancestry database.Ancestry, features []database.NamespacedFeature, processedBy database.Processors) error {
func (tx *pgSession) UpsertAncestry(ancestry database.AncestryWithContent) error {
if ancestry.Name == "" {
log.Warning("Empty ancestry name is not allowed")
log.Error("Empty ancestry name is not allowed")
return commonerr.NewBadRequestError("could not insert an ancestry with empty name")
}
if len(ancestry.Layers) == 0 {
log.Warning("Empty ancestry is not allowed")
log.Error("Empty ancestry is not allowed")
return commonerr.NewBadRequestError("could not insert an ancestry with 0 layers")
}
err := tx.deleteAncestry(ancestry.Name)
if err != nil {
if err := tx.deleteAncestry(ancestry.Name); err != nil {
return err
}
var ancestryID int64
err = tx.QueryRow(insertAncestry, ancestry.Name).Scan(&ancestryID)
if err != nil {
if err := tx.QueryRow(insertAncestry, ancestry.Name).Scan(&ancestryID); err != nil {
if isErrUniqueViolation(err) {
return handleError("insertAncestry", errors.New("Other Go-routine is processing this ancestry (skip)."))
return handleError("insertAncestry", errors.New("other Go-routine is processing this ancestry (skip)"))
}
return handleError("insertAncestry", err)
}
err = tx.insertAncestryLayers(ancestryID, ancestry.Layers)
if err != nil {
return err
}
err = tx.insertAncestryFeatures(ancestryID, features)
if err != nil {
if err := tx.insertAncestryLayers(ancestryID, ancestry.Layers); err != nil {
return err
}
@ -52,71 +41,82 @@ func (tx *pgSession) UpsertAncestry(ancestry database.Ancestry, features []datab
"persistAncestryLister",
persistAncestryDetector,
"persistAncestryDetector",
ancestryID, processedBy)
ancestryID, ancestry.ProcessedBy)
}
func (tx *pgSession) FindAncestry(name string) (database.Ancestry, database.Processors, bool, error) {
ancestry := database.Ancestry{Name: name}
processed := database.Processors{}
func (tx *pgSession) FindAncestry(name string) (database.Ancestry, bool, error) {
var (
ancestryID int64
ancestry = database.Ancestry{Name: name}
err error
)
var ancestryID int64
err := tx.QueryRow(searchAncestry, name).Scan(&ancestryID)
if err != nil {
if err = tx.QueryRow(searchAncestry, name).Scan(&ancestryID); err != nil {
if err == sql.ErrNoRows {
return ancestry, processed, false, nil
return ancestry, false, nil
}
return ancestry, processed, false, handleError("searchAncestry", err)
return ancestry, false, handleError("searchAncestry", err)
}
ancestry.Layers, err = tx.findAncestryLayers(ancestryID)
if err != nil {
return ancestry, processed, false, err
if ancestry.Layers, err = tx.findAncestryLayers(ancestryID); err != nil {
return ancestry, false, err
}
processed.Detectors, err = tx.findProcessors(searchAncestryDetectors, "searchAncestryDetectors", "detector", ancestryID)
if err != nil {
return ancestry, processed, false, err
if ancestry.ProcessedBy.Detectors, err = tx.findProcessors(searchAncestryDetectors, "searchAncestryDetectors", "detector", ancestryID); err != nil {
return ancestry, false, err
}
processed.Listers, err = tx.findProcessors(searchAncestryListers, "searchAncestryListers", "lister", ancestryID)
if err != nil {
return ancestry, processed, false, err
if ancestry.ProcessedBy.Listers, err = tx.findProcessors(searchAncestryListers, "searchAncestryListers", "lister", ancestryID); err != nil {
return ancestry, false, err
}
return ancestry, processed, true, nil
return ancestry, true, nil
}
func (tx *pgSession) FindAncestryFeatures(name string) (database.AncestryWithFeatures, bool, error) {
func (tx *pgSession) FindAncestryWithContent(name string) (database.AncestryWithContent, bool, error) {
var (
awf database.AncestryWithFeatures
ok bool
err error
ancestryContent database.AncestryWithContent
isValid bool
err error
)
awf.Ancestry, awf.ProcessedBy, ok, err = tx.FindAncestry(name)
if err != nil {
return awf, false, err
}
if !ok {
return awf, false, nil
if ancestryContent.Ancestry, isValid, err = tx.FindAncestry(name); err != nil || !isValid {
return ancestryContent, isValid, err
}
rows, err := tx.Query(searchAncestryFeatures, name)
if err != nil {
return awf, false, handleError("searchAncestryFeatures", err)
return ancestryContent, false, handleError("searchAncestryFeatures", err)
}
features := map[int][]database.NamespacedFeature{}
for rows.Next() {
nf := database.NamespacedFeature{}
err := rows.Scan(&nf.Namespace.Name, &nf.Namespace.VersionFormat, &nf.Feature.Name, &nf.Feature.Version)
if err != nil {
return awf, false, handleError("searchAncestryFeatures", err)
var (
feature database.NamespacedFeature
// layerIndex is used to determine which layer the namespaced feature belongs to.
layerIndex sql.NullInt64
)
if err := rows.Scan(&feature.Namespace.Name,
&feature.Namespace.VersionFormat,
&feature.Feature.Name, &feature.Feature.Version,
&layerIndex); err != nil {
return ancestryContent, false, handleError("searchAncestryFeatures", err)
}
nf.Feature.VersionFormat = nf.Namespace.VersionFormat
awf.Features = append(awf.Features, nf)
feature.Feature.VersionFormat = feature.Namespace.VersionFormat // This looks strange.
features[int(layerIndex.Int64)] = append(features[int(layerIndex.Int64)], feature)
}
// By the assumption of Ancestry Layer Index, we have the ancestry's layer
// index corresponding to the index in the array.
for index, layer := range ancestryContent.Ancestry.Layers {
ancestryLayer := database.AncestryLayer{Layer: layer}
ancestryLayer.DetectedFeatures, _ = features[index]
ancestryContent.Layers = append(ancestryContent.Layers, ancestryLayer)
}
return awf, true, nil
return ancestryContent, true, nil
}
func (tx *pgSession) deleteAncestry(name string) error {
@ -164,97 +164,62 @@ func (tx *pgSession) findAncestryLayers(ancestryID int64) ([]database.Layer, err
if err != nil {
return nil, handleError("searchAncestryLayer", err)
}
layers := []database.Layer{}
for rows.Next() {
var layer database.Layer
err := rows.Scan(&layer.Hash)
if err != nil {
if err := rows.Scan(&layer.Hash); err != nil {
return nil, handleError("searchAncestryLayer", err)
}
layers = append(layers, layer)
}
return layers, nil
}
func (tx *pgSession) insertAncestryLayers(ancestryID int64, layers []database.Layer) error {
layerIDs := map[string]sql.NullInt64{}
for _, l := range layers {
layerIDs[l.Hash] = sql.NullInt64{}
}
layerHashes := []string{}
for hash := range layerIDs {
layerHashes = append(layerHashes, hash)
}
rows, err := tx.Query(searchLayerIDs, pq.Array(layerHashes))
if err != nil {
return handleError("searchLayerIDs", err)
}
for rows.Next() {
var (
layerID sql.NullInt64
layerName string
)
err := rows.Scan(&layerID, &layerName)
if err != nil {
return handleError("searchLayerIDs", err)
}
layerIDs[layerName] = layerID
}
notFound := []string{}
for hash, id := range layerIDs {
if !id.Valid {
notFound = append(notFound, hash)
}
}
if len(notFound) > 0 {
return handleError("searchLayerIDs", fmt.Errorf("Layer %s is not found in database", strings.Join(notFound, ",")))
}
// insertAncestryLayers inserts the ancestry layers along with its content into
// the database. The layers are 0 based indexed in the original order.
func (tx *pgSession) insertAncestryLayers(ancestryID int64, layers []database.AncestryLayer) error {
//TODO(Sida): use bulk insert.
stmt, err := tx.Prepare(insertAncestryLayer)
if err != nil {
return handleError("insertAncestryLayer", err)
}
defer stmt.Close()
ancestryLayerIDs := []sql.NullInt64{}
for index, layer := range layers {
_, err := stmt.Exec(ancestryID, index, layerIDs[layer.Hash].Int64)
if err != nil {
var ancestryLayerID sql.NullInt64
if err := stmt.QueryRow(ancestryID, index, layer.Hash).Scan(&ancestryLayerID); err != nil {
return handleError("insertAncestryLayer", commonerr.CombineErrors(err, stmt.Close()))
}
}
return nil
}
func (tx *pgSession) insertAncestryFeatures(ancestryID int64, features []database.NamespacedFeature) error {
featureIDs, err := tx.findNamespacedFeatureIDs(features)
if err != nil {
return err
ancestryLayerIDs = append(ancestryLayerIDs, ancestryLayerID)
}
//TODO(Sida): use bulk insert.
stmtFeatures, err := tx.Prepare(insertAncestryFeature)
if err != nil {
return handleError("insertAncestryFeature", err)
if err := stmt.Close(); err != nil {
return handleError("Failed to close insertAncestryLayer statement", err)
}
defer stmtFeatures.Close()
stmt, err = tx.Prepare(insertAncestryLayerFeature)
defer stmt.Close()
for _, id := range featureIDs {
if !id.Valid {
return errors.New("requested namespaced feature is not in database")
for i, layer := range layers {
var (
nsFeatureIDs []sql.NullInt64
layerID = ancestryLayerIDs[i]
)
if nsFeatureIDs, err = tx.findNamespacedFeatureIDs(layer.DetectedFeatures); err != nil {
return err
}
_, err := stmtFeatures.Exec(ancestryID, id)
if err != nil {
return handleError("insertAncestryFeature", err)
for _, id := range nsFeatureIDs {
if _, err := stmt.Exec(layerID, id); err != nil {
return handleError("insertAncestryLayerFeature", commonerr.CombineErrors(err, stmt.Close()))
}
}
}
return nil

183
database/pgsql/ancestry_test.go
Unescape View File

@ -26,26 +26,53 @@ import (
func TestUpsertAncestry(t *testing.T) {
store, tx := openSessionForTest(t, "UpsertAncestry", true)
defer closeTest(t, store, tx)
a1 := database.Ancestry{
Name: "a1",
Layers: []database.Layer{
{Hash: "layer-N"},
a1 := database.AncestryWithContent{
Ancestry: database.Ancestry{
Name: "a1",
Layers: []database.Layer{
{Hash: "layer-N"},
},
},
Layers: []database.AncestryLayer{
{
Layer: database.Layer{
Hash: "layer-N",
},
},
},
}
a2 := database.Ancestry{}
a2 := database.AncestryWithContent{}
a3 := database.Ancestry{
Name: "a",
Layers: []database.Layer{
{Hash: "layer-0"},
a3 := database.AncestryWithContent{
Ancestry: database.Ancestry{
Name: "a",
Layers: []database.Layer{
{Hash: "layer-0"},
},
},
Layers: []database.AncestryLayer{
{
Layer: database.Layer{
Hash: "layer-0",
},
},
},
}
a4 := database.Ancestry{
Name: "a",
Layers: []database.Layer{
{Hash: "layer-1"},
a4 := database.AncestryWithContent{
Ancestry: database.Ancestry{
Name: "a",
Layers: []database.Layer{
{Hash: "layer-1"},
},
},
Layers: []database.AncestryLayer{
{
Layer: database.Layer{
Hash: "layer-1",
},
},
},
}
@ -83,17 +110,20 @@ func TestUpsertAncestry(t *testing.T) {
Feature: f2,
}
a4.ProcessedBy = p
// invalid case
assert.NotNil(t, tx.UpsertAncestry(a1, nil, database.Processors{}))
assert.NotNil(t, tx.UpsertAncestry(a2, nil, database.Processors{}))
assert.NotNil(t, tx.UpsertAncestry(a1))
assert.NotNil(t, tx.UpsertAncestry(a2))
// valid case
assert.Nil(t, tx.UpsertAncestry(a3, nil, database.Processors{}))
assert.Nil(t, tx.UpsertAncestry(a3))
a4.Layers[0].DetectedFeatures = []database.NamespacedFeature{nsf1, nsf2}
// replace invalid case
assert.NotNil(t, tx.UpsertAncestry(a4, []database.NamespacedFeature{nsf1, nsf2}, p))
assert.NotNil(t, tx.UpsertAncestry(a4))
a4.Layers[0].DetectedFeatures = []database.NamespacedFeature{nsf1}
// replace valid case
assert.Nil(t, tx.UpsertAncestry(a4, []database.NamespacedFeature{nsf1}, p))
assert.Nil(t, tx.UpsertAncestry(a4))
// validate
ancestry, ok, err := tx.FindAncestryFeatures("a")
ancestry, ok, err := tx.FindAncestryWithContent("a")
assert.Nil(t, err)
assert.True(t, ok)
assert.Equal(t, a4, ancestry.Ancestry)
@ -111,8 +141,7 @@ func TestFindAncestry(t *testing.T) {
store, tx := openSessionForTest(t, "FindAncestry", true)
defer closeTest(t, store, tx)
// not found
_, _, ok, err := tx.FindAncestry("ancestry-non")
_, ok, err := tx.FindAncestry("ancestry-non")
assert.Nil(t, err)
assert.False(t, ok)
@ -124,41 +153,52 @@ func TestFindAncestry(t *testing.T) {
{Hash: "layer-2"},
{Hash: "layer-3a"},
},
ProcessedBy: database.Processors{
Detectors: []string{"os-release"},
Listers: []string{"dpkg"},
},
}
expectedProcessors := database.Processors{
Detectors: []string{"os-release"},
Listers: []string{"dpkg"},
}
// found
a, p, ok2, err := tx.FindAncestry("ancestry-1")
a, ok2, err := tx.FindAncestry("ancestry-1")
if assert.Nil(t, err) && assert.True(t, ok2) {
assertAncestryEqual(t, expected, a)
assertProcessorsEqual(t, expectedProcessors, p)
}
}
func assertAncestryWithFeatureEqual(t *testing.T, expected database.AncestryWithFeatures, actual database.AncestryWithFeatures) bool {
return assertAncestryEqual(t, expected.Ancestry, actual.Ancestry) &&
assertNamespacedFeatureEqual(t, expected.Features, actual.Features) &&
assertProcessorsEqual(t, expected.ProcessedBy, actual.ProcessedBy)
func assertAncestryWithFeatureEqual(t *testing.T, expected database.AncestryWithContent, actual database.AncestryWithContent) bool {
if assertAncestryEqual(t, expected.Ancestry, actual.Ancestry) && assert.Equal(t, len(expected.Layers), len(actual.Layers)) {
for index, layer := range expected.Layers {
if !assertAncestryLayerEqual(t, layer, actual.Layers[index]) {
return false
}
}
return true
}
return false
}
func assertAncestryLayerEqual(t *testing.T, expected database.AncestryLayer, actual database.AncestryLayer) bool {
return assertLayerEqual(t, expected.Layer, actual.Layer) &&
assertNamespacedFeatureEqual(t, expected.DetectedFeatures, actual.DetectedFeatures)
}
func assertAncestryEqual(t *testing.T, expected database.Ancestry, actual database.Ancestry) bool {
return assert.Equal(t, expected.Name, actual.Name) && assert.Equal(t, expected.Layers, actual.Layers)
return assert.Equal(t, expected.Name, actual.Name) &&
assert.Equal(t, expected.Layers, actual.Layers) &&
assertProcessorsEqual(t, expected.ProcessedBy, actual.ProcessedBy)
}
func TestFindAncestryFeatures(t *testing.T) {
store, tx := openSessionForTest(t, "FindAncestryFeatures", true)
func TestFindAncestryWithContent(t *testing.T) {
store, tx := openSessionForTest(t, "FindAncestryWithContent", true)
defer closeTest(t, store, tx)
// invalid
_, ok, err := tx.FindAncestryFeatures("ancestry-non")
_, ok, err := tx.FindAncestryWithContent("ancestry-non")
if assert.Nil(t, err) {
assert.False(t, ok)
}
expected := database.AncestryWithFeatures{
expected := database.AncestryWithContent{
Ancestry: database.Ancestry{
Name: "ancestry-2",
Layers: []database.Layer{
@ -167,41 +207,62 @@ func TestFindAncestryFeatures(t *testing.T) {
{Hash: "layer-2"},
{Hash: "layer-3b"},
},
ProcessedBy: database.Processors{
Detectors: []string{"os-release"},
Listers: []string{"dpkg"},
},
},
ProcessedBy: database.Processors{
Detectors: []string{"os-release"},
Listers: []string{"dpkg"},
},
Features: []database.NamespacedFeature{
Layers: []database.AncestryLayer{
{
Namespace: database.Namespace{
Name: "debian:7",
VersionFormat: "dpkg",
Layer: database.Layer{
Hash: "layer-0",
},
DetectedFeatures: []database.NamespacedFeature{
{
Namespace: database.Namespace{
Name: "debian:7",
VersionFormat: "dpkg",
},
Feature: database.Feature{
Name: "wechat",
Version: "0.5",
VersionFormat: "dpkg",
},
},
{
Namespace: database.Namespace{
Name: "debian:8",
VersionFormat: "dpkg",
},
Feature: database.Feature{
Name: "openssl",
Version: "1.0",
VersionFormat: "dpkg",
},
},
},
Feature: database.Feature{
Name: "wechat",
Version: "0.5",
VersionFormat: "dpkg",
},
{
Layer: database.Layer{
Hash: "layer-1",
},
},
{
Namespace: database.Namespace{
Name: "debian:8",
VersionFormat: "dpkg",
Layer: database.Layer{
Hash: "layer-2",
},
Feature: database.Feature{
Name: "openssl",
Version: "1.0",
VersionFormat: "dpkg",
},
{
Layer: database.Layer{
Hash: "layer-3b",
},
},
},
}
// valid
ancestry, ok, err := tx.FindAncestryFeatures("ancestry-2")
ancestry, ok, err := tx.FindAncestryWithContent("ancestry-2")
if assert.Nil(t, err) && assert.True(t, ok) {
assertAncestryEqual(t, expected.Ancestry, ancestry.Ancestry)
assertNamespacedFeatureEqual(t, expected.Features, ancestry.Features)
assertProcessorsEqual(t, expected.ProcessedBy, ancestry.ProcessedBy)
assertAncestryWithFeatureEqual(t, expected, ancestry)
}
}

37
database/pgsql/layer.go
Unescape View File

@ -22,9 +22,9 @@ import (
"github.com/coreos/clair/pkg/commonerr"
)
func (tx *pgSession) FindLayer(hash string) (database.Layer, database.Processors, bool, error) {
l, p, _, ok, err := tx.findLayer(hash)
return l, p, ok, err
func (tx *pgSession) FindLayer(hash string) (database.Layer, bool, error) {
layer, _, ok, err := tx.findLayer(hash)
return layer, ok, err
}
func (tx *pgSession) FindLayerWithContent(hash string) (database.LayerWithContent, bool, error) {
@ -35,7 +35,7 @@ func (tx *pgSession) FindLayerWithContent(hash string) (database.LayerWithConten
err error
)
layer.Layer, layer.ProcessedBy, layerID, ok, err = tx.findLayer(hash)
layer.Layer, layerID, ok, err = tx.findLayer(hash)
if err != nil {
return layer, false, err
}
@ -49,12 +49,12 @@ func (tx *pgSession) FindLayerWithContent(hash string) (database.LayerWithConten
return layer, true, nil
}
func (tx *pgSession) PersistLayer(layer database.Layer) error {
if layer.Hash == "" {
func (tx *pgSession) PersistLayer(hash string) error {
if hash == "" {
return commonerr.NewBadRequestError("Empty Layer Hash is not allowed")
}
_, err := tx.Exec(queryPersistLayer(1), layer.Hash)
_, err := tx.Exec(queryPersistLayer(1), hash)
if err != nil {
return handleError("queryPersistLayer", err)
}
@ -275,34 +275,33 @@ func (tx *pgSession) findLayerFeatures(layerID int64) ([]database.Feature, error
return features, nil
}
func (tx *pgSession) findLayer(hash string) (database.Layer, database.Processors, int64, bool, error) {
func (tx *pgSession) findLayer(hash string) (database.Layer, int64, bool, error) {
var (
layerID int64
layer = database.Layer{Hash: hash}
processors database.Processors
layerID int64
layer = database.Layer{Hash: hash, ProcessedBy: database.Processors{}}
)
if hash == "" {
return layer, processors, layerID, false, commonerr.NewBadRequestError("Empty Layer Hash is not allowed")
return layer, layerID, false, commonerr.NewBadRequestError("Empty Layer Hash is not allowed")
}
err := tx.QueryRow(searchLayer, hash).Scan(&layerID)
if err != nil {
if err == sql.ErrNoRows {
return layer, processors, layerID, false, nil
return layer, layerID, false, nil
}
return layer, processors, layerID, false, err
return layer, layerID, false, err
}
processors.Detectors, err = tx.findProcessors(searchLayerDetectors, "searchLayerDetectors", "detector", layerID)
layer.ProcessedBy.Detectors, err = tx.findProcessors(searchLayerDetectors, "searchLayerDetectors", "detector", layerID)
if err != nil {
return layer, processors, layerID, false, err
return layer, layerID, false, err
}
processors.Listers, err = tx.findProcessors(searchLayerListers, "searchLayerListers", "lister", layerID)
layer.ProcessedBy.Listers, err = tx.findProcessors(searchLayerListers, "searchLayerListers", "lister", layerID)
if err != nil {
return layer, processors, layerID, false, err
return layer, layerID, false, err
}
return layer, processors, layerID, true, nil
return layer, layerID, true, nil
}

39
database/pgsql/layer_test.go
Unescape View File

@ -26,8 +26,8 @@ func TestPersistLayer(t *testing.T) {
datastore, tx := openSessionForTest(t, "PersistLayer", false)
defer closeTest(t, datastore, tx)
l1 := database.Layer{}
l2 := database.Layer{Hash: "HESOYAM"}
l1 := ""
l2 := "HESOYAM"
// invalid
assert.NotNil(t, tx.PersistLayer(l1))
@ -51,24 +51,25 @@ func TestFindLayer(t *testing.T) {
datastore, tx := openSessionForTest(t, "FindLayer", true)
defer closeTest(t, datastore, tx)
expected := database.Layer{Hash: "layer-4"}
expectedProcessors := database.Processors{
Detectors: []string{"os-release", "apt-sources"},
Listers: []string{"dpkg", "rpm"},
expected := database.Layer{
Hash: "layer-4",
ProcessedBy: database.Processors{
Detectors: []string{"os-release", "apt-sources"},
Listers: []string{"dpkg", "rpm"},
},
}
// invalid
_, _, _, err := tx.FindLayer("")
_, _, err := tx.FindLayer("")
assert.NotNil(t, err)
_, _, ok, err := tx.FindLayer("layer-non")
_, ok, err := tx.FindLayer("layer-non")
assert.Nil(t, err)
assert.False(t, ok)
// valid
layer, processors, ok2, err := tx.FindLayer("layer-4")
layer, ok2, err := tx.FindLayer("layer-4")
if assert.Nil(t, err) && assert.True(t, ok2) {
assert.Equal(t, expected, layer)
assertProcessorsEqual(t, expectedProcessors, processors)
assertLayerEqual(t, expected, layer)
}
}
@ -85,6 +86,10 @@ func TestFindLayerWithContent(t *testing.T) {
expectedL := database.LayerWithContent{
Layer: database.Layer{
Hash: "layer-4",
ProcessedBy: database.Processors{
Detectors: []string{"os-release", "apt-sources"},
Listers: []string{"dpkg", "rpm"},
},
},
Features: []database.Feature{
{Name: "fake", Version: "2.0", VersionFormat: "rpm"},
@ -94,10 +99,6 @@ func TestFindLayerWithContent(t *testing.T) {
{Name: "debian:7", VersionFormat: "dpkg"},
{Name: "fake:1.0", VersionFormat: "rpm"},
},
ProcessedBy: database.Processors{