From 5d725e67b08d4fc9805a1d13eac43fa82fa27785 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Fri, 7 Sep 2018 11:31:35 -0400 Subject: [PATCH] Replace Ancestry with AncestryWithContent struct in database models As one of the steps to simplifies the codebase, the AncestryWithContent struct is renamed to Ancestry, and Ancestry is removed. It will cause the PostAncestry request to be slower. --- api/v3/clairpb/clair.pb.go | 178 ++++++++++++--------------- api/v3/clairpb/clair.pb.gw.go | 8 -- api/v3/clairpb/clair.proto | 4 - api/v3/clairpb/clair.swagger.json | 16 --- api/v3/clairpb/convert.go | 18 --- api/v3/rpc.go | 93 ++++---------- api/v3/util.go | 44 +++++++ database/database.go | 13 +- database/mock.go | 12 +- database/models.go | 14 +-- database/pgsql/ancestry.go | 195 +++++++++++++++++++----------- database/pgsql/ancestry_test.go | 102 ++++------------ database/pgsql/layer.go | 22 ++-- database/pgsql/queries.go | 9 +- worker.go | 2 +- worker_test.go | 23 ++-- 16 files changed, 331 insertions(+), 422 deletions(-) diff --git a/api/v3/clairpb/clair.pb.go b/api/v3/clairpb/clair.pb.go index c537ab23..ac441ec0 100644 --- a/api/v3/clairpb/clair.pb.go +++ b/api/v3/clairpb/clair.pb.go @@ -238,10 +238,6 @@ func (m *ClairStatus) GetLastUpdateTime() *google_protobuf.Timestamp { type GetAncestryRequest struct { // The name of the desired ancestry. AncestryName string `protobuf:"bytes,1,opt,name=ancestry_name,json=ancestryName" json:"ancestry_name,omitempty"` - // Whether to include vulnerabilities or not in the response. - WithVulnerabilities bool `protobuf:"varint,2,opt,name=with_vulnerabilities,json=withVulnerabilities" json:"with_vulnerabilities,omitempty"` - // Whether to include features or not in the response. - WithFeatures bool `protobuf:"varint,3,opt,name=with_features,json=withFeatures" json:"with_features,omitempty"` } func (m *GetAncestryRequest) Reset() { *m = GetAncestryRequest{} } @@ -256,20 +252,6 @@ func (m *GetAncestryRequest) GetAncestryName() string { return "" } -func (m *GetAncestryRequest) GetWithVulnerabilities() bool { - if m != nil { - return m.WithVulnerabilities - } - return false -} - -func (m *GetAncestryRequest) GetWithFeatures() bool { - if m != nil { - return m.WithFeatures - } - return false -} - type GetAncestryResponse struct { // The ancestry requested. Ancestry *GetAncestryResponse_Ancestry `protobuf:"bytes,1,opt,name=ancestry" json:"ancestry,omitempty"` @@ -1025,85 +1007,83 @@ var _StatusService_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("api/v3/clairpb/clair.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1268 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x57, 0x4b, 0x6f, 0x1b, 0x55, - 0x14, 0xd6, 0x38, 0x71, 0x6c, 0x1f, 0xdb, 0x49, 0x7a, 0xed, 0xa6, 0x93, 0x49, 0x1f, 0xc9, 0x40, - 0xd5, 0xd2, 0x22, 0x5b, 0x75, 0x59, 0x94, 0xb2, 0x40, 0xe9, 0x23, 0xa1, 0x52, 0xa9, 0xaa, 0x29, - 0x64, 0x01, 0x42, 0xd6, 0xcd, 0xcc, 0x71, 0x32, 0xca, 0x78, 0xc6, 0xcc, 0xbd, 0x4e, 0x6a, 0x55, - 0x65, 0xc1, 0x96, 0x15, 0xb0, 0xe0, 0x37, 0xb0, 0xe1, 0x1f, 0xb0, 0x62, 0xcb, 0x02, 0xc1, 0x16, - 0x76, 0x2c, 0xf8, 0x03, 0xec, 0xd1, 0x7d, 0x4d, 0x66, 0x12, 0x37, 0x49, 0xcb, 0xca, 0x73, 0xde, - 0xaf, 0xef, 0x9e, 0x93, 0x80, 0x43, 0x47, 0x61, 0x77, 0xff, 0x76, 0xd7, 0x8f, 0x68, 0x98, 0x8e, - 0xb6, 0xd5, 0x6f, 0x67, 0x94, 0x26, 0x3c, 0x21, 0x0d, 0x3f, 0x49, 0x31, 0x61, 0x1d, 0xc9, 0x73, - 0xae, 0xec, 0x24, 0xc9, 0x4e, 0x84, 0x5d, 0x29, 0xdb, 0x1e, 0x0f, 0xba, 0x3c, 0x1c, 0x22, 0xe3, - 0x74, 0x38, 0x52, 0xea, 0xce, 0x45, 0xad, 0x20, 0x3c, 0xd2, 0x38, 0x4e, 0x38, 0xe5, 0x61, 0x12, - 0x33, 0x25, 0x75, 0x7f, 0x28, 0x41, 0x73, 0x6b, 0x1c, 0xc5, 0x98, 0xd2, 0xed, 0x30, 0x0a, 0xf9, - 0x84, 0x10, 0x98, 0x8d, 0xe9, 0x10, 0x6d, 0x6b, 0xd5, 0xba, 0x5e, 0xf3, 0xe4, 0x37, 0xb9, 0x0a, - 0xf3, 0xe2, 0x97, 0x8d, 0xa8, 0x8f, 0x7d, 0x29, 0x2d, 0x49, 0x69, 0x33, 0xe3, 0x3e, 0x11, 0x6a, - 0xab, 0x50, 0x0f, 0x90, 0xf9, 0x69, 0x38, 0x12, 0x21, 0xec, 0x19, 0xa9, 0x93, 0x67, 0x09, 0xe7, - 0x51, 0x18, 0xef, 0xd9, 0xb3, 0xca, 0xb9, 0xf8, 0x26, 0x0e, 0x54, 0x19, 0xee, 0x63, 0x1a, 0xf2, - 0x89, 0x5d, 0x96, 0xfc, 0x8c, 0x16, 0xb2, 0x21, 0x72, 0x1a, 0x50, 0x4e, 0xed, 0x39, 0x25, 0x33, - 0x34, 0x59, 0x86, 0xea, 0x20, 0x7c, 0x8e, 0x41, 0x7f, 0x7b, 0x62, 0x57, 0xa4, 0xac, 0x22, 0xe9, - 0x7b, 0x13, 0x72, 0x0f, 0xce, 0xd1, 0xc1, 0x00, 0x7d, 0x8e, 0x41, 0x7f, 0x1f, 0x53, 0x26, 0x0a, - 0xb6, 0xab, 0xab, 0x33, 0xd7, 0xeb, 0xbd, 0xf3, 0x9d, 0x7c, 0xfb, 0x3a, 0x1b, 0x48, 0xf9, 0x38, - 0x45, 0x6f, 0xd1, 0xe8, 0x6f, 0x69, 0x75, 0xf7, 0x57, 0x0b, 0x2a, 0x5a, 0xfa, 0x7f, 0x7a, 0x62, - 0x43, 0x45, 0x67, 0xa0, 0xfb, 0x61, 0x48, 0xe1, 0x40, 0x7f, 0xf6, 0x07, 0x49, 0x3a, 0xa4, 0x5c, - 0x77, 0xa5, 0xa9, 0xb9, 0x1b, 0x92, 0x49, 0x1e, 0xc2, 0xc2, 0x7e, 0x6e, 0x40, 0x21, 0x32, 0xbb, - 0x2c, 0x2b, 0x59, 0x29, 0x56, 0x52, 0x98, 0xa2, 0x77, 0xd4, 0xc6, 0x5d, 0x81, 0xf2, 0x63, 0x3a, - 0xc1, 0x54, 0xd4, 0xb2, 0x4b, 0xd9, 0xae, 0xa9, 0x45, 0x7c, 0xbb, 0xdf, 0x58, 0x50, 0xbf, 0x2f, - 0xbc, 0x3c, 0xe3, 0x94, 0x8f, 0x99, 0x48, 0x3a, 0x0a, 0x19, 0xc7, 0x94, 0xd9, 0xd6, 0xea, 0x8c, - 0x48, 0x5a, 0x93, 0xe4, 0x22, 0xd4, 0x02, 0xe4, 0xe8, 0xf3, 0x24, 0x65, 0x76, 0x49, 0xca, 0x0e, - 0x19, 0xe4, 0x01, 0x2c, 0x46, 0x94, 0xf1, 0xfe, 0x78, 0x14, 0x50, 0x8e, 0x7d, 0x01, 0x45, 0x59, - 0x75, 0xbd, 0xe7, 0x74, 0x14, 0x0c, 0x3b, 0x06, 0xa7, 0x9d, 0x4f, 0x0c, 0x4e, 0xbd, 0x79, 0x61, - 0xf3, 0xa9, 0x34, 0x11, 0x4c, 0xf7, 0x5b, 0x0b, 0xc8, 0x26, 0xf2, 0xf5, 0xd8, 0x47, 0xc6, 0xd3, - 0x89, 0x87, 0x5f, 0x8e, 0x91, 0x71, 0xf2, 0x16, 0x34, 0xa9, 0x66, 0xf5, 0x73, 0xd3, 0x68, 0x18, - 0xa6, 0x6c, 0xf7, 0x2d, 0x68, 0x1f, 0x84, 0x7c, 0xb7, 0x7f, 0xb4, 0x65, 0x62, 0x36, 0x55, 0xaf, - 0x25, 0x64, 0x5b, 0x45, 0x91, 0xf0, 0x2b, 0x4d, 0x06, 0x6a, 0xd8, 0x4c, 0x66, 0x5c, 0xf5, 0x1a, - 0x82, 0xa9, 0x01, 0xc0, 0xdc, 0xdf, 0x66, 0xa0, 0x55, 0xc8, 0x89, 0x8d, 0x92, 0x98, 0x21, 0xd9, - 0x80, 0xaa, 0x89, 0x2f, 0xf3, 0xa9, 0xf7, 0x6e, 0x14, 0xc7, 0x32, 0xc5, 0xa8, 0x93, 0x31, 0x32, - 0x5b, 0x72, 0x0b, 0xe6, 0x98, 0xec, 0xbd, 0xcc, 0xb4, 0xde, 0x5b, 0x2e, 0x7a, 0xc9, 0x0d, 0xc7, - 0xd3, 0x8a, 0xce, 0x57, 0xd0, 0x34, 0x8e, 0xd4, 0x64, 0xdf, 0x81, 0x72, 0x24, 0x3e, 0x74, 0x22, - 0xad, 0xa2, 0x0b, 0xa9, 0xe3, 0x29, 0x0d, 0xf1, 0x40, 0xd4, 0xd4, 0x30, 0x38, 0xac, 0xbb, 0x74, - 0xe2, 0x03, 0x31, 0xfa, 0xa6, 0x25, 0xce, 0xcf, 0x16, 0x54, 0x4d, 0x02, 0x53, 0x5f, 0xc8, 0x35, - 0x58, 0x60, 0x3e, 0x8d, 0x63, 0x0c, 0xfa, 0x06, 0x4d, 0xb3, 0x12, 0x31, 0xf3, 0x9a, 0xfd, 0x58, - 0x83, 0xea, 0x26, 0x9c, 0x33, 0x8a, 0x87, 0xe0, 0x2a, 0x4b, 0xd5, 0x45, 0x2d, 0x78, 0x90, 0x61, - 0x6c, 0x13, 0xe6, 0x64, 0x0d, 0xcc, 0x9e, 0x93, 0xf9, 0x76, 0xcf, 0xde, 0x6f, 0xd5, 0x02, 0x6d, - 0xee, 0xfe, 0x55, 0x82, 0xd6, 0xd3, 0x84, 0xbd, 0x19, 0xce, 0x96, 0x60, 0x4e, 0x3f, 0x5a, 0xf5, - 0xea, 0x35, 0x45, 0xee, 0x67, 0xd9, 0xcd, 0xc8, 0xec, 0x6e, 0x16, 0xb3, 0x9b, 0x12, 0x4f, 0xf2, - 0x0a, 0x99, 0x39, 0xbf, 0x58, 0x50, 0xcb, 0xb8, 0xd3, 0x1e, 0xac, 0xe0, 0x8d, 0x28, 0xdf, 0xd5, - 0xc1, 0xe5, 0x37, 0xf1, 0xa0, 0xb2, 0x8b, 0x34, 0x38, 0x8c, 0x7d, 0xe7, 0x35, 0x62, 0x77, 0x3e, - 0x52, 0xa6, 0x0f, 0x63, 0x21, 0x35, 0x8e, 0x9c, 0xbb, 0xd0, 0xc8, 0x0b, 0xc8, 0x22, 0xcc, 0xec, - 0xe1, 0x44, 0xa7, 0x22, 0x3e, 0x49, 0x1b, 0xca, 0xfb, 0x34, 0x1a, 0x9b, 0xed, 0xa7, 0x88, 0xbb, - 0xa5, 0x3b, 0x96, 0xfb, 0x08, 0xda, 0xc5, 0x90, 0xfa, 0xc9, 0x1c, 0x42, 0xdd, 0x3a, 0x23, 0xd4, - 0xdd, 0x9f, 0x2c, 0x58, 0xda, 0x44, 0xfe, 0x24, 0xe1, 0xe1, 0x20, 0xf4, 0xe5, 0x01, 0x33, 0xd3, - 0x7a, 0x0f, 0x96, 0x92, 0x28, 0x28, 0xbc, 0xf7, 0x49, 0x7f, 0x44, 0x77, 0xcc, 0xd8, 0xda, 0x49, - 0x14, 0x14, 0x56, 0xe3, 0x53, 0xba, 0x83, 0xc2, 0x2a, 0xc6, 0x83, 0x69, 0x56, 0xaa, 0x8c, 0x76, - 0x8c, 0x07, 0xc7, 0xad, 0xda, 0x50, 0x8e, 0xc2, 0x61, 0xc8, 0xe5, 0x86, 0x28, 0x7b, 0x8a, 0xc8, - 0xa0, 0x3f, 0x7b, 0x08, 0x7d, 0xf7, 0xcf, 0x12, 0x5c, 0x38, 0x96, 0xb0, 0xae, 0x7f, 0x0b, 0x1a, - 0x71, 0x8e, 0xaf, 0xbb, 0xd0, 0x3b, 0x06, 0xe3, 0x69, 0xc6, 0x9d, 0x02, 0xb3, 0xe0, 0xc7, 0xf9, - 0xc7, 0x82, 0x46, 0x5e, 0x3c, 0xf5, 0x4d, 0xda, 0x50, 0xf1, 0x53, 0xa4, 0x1c, 0x03, 0x5d, 0xa9, - 0x21, 0xc5, 0xa9, 0x55, 0xee, 0x30, 0xd0, 0x97, 0x2a, 0xa3, 0x85, 0x55, 0x80, 0x11, 0x0a, 0x2b, - 0x55, 0xa5, 0x21, 0xc9, 0xfb, 0x30, 0x93, 0x44, 0x81, 0xbc, 0xdb, 0xf5, 0xde, 0xb5, 0x23, 0x80, - 0xa3, 0x3b, 0x98, 0xf5, 0x3e, 0x42, 0x0d, 0x84, 0x10, 0x99, 0x27, 0x6c, 0x84, 0x69, 0x8c, 0x07, - 0xf2, 0xac, 0xbf, 0x8e, 0x69, 0x8c, 0x07, 0xee, 0xef, 0x25, 0x58, 0x7e, 0xa5, 0x0a, 0x59, 0x83, - 0x86, 0x3f, 0x4e, 0x53, 0x8c, 0x79, 0x1e, 0x08, 0x75, 0xcd, 0x93, 0x93, 0x5c, 0x81, 0x5a, 0x8c, - 0xcf, 0x79, 0x7e, 0xe4, 0x55, 0xc1, 0x38, 0x61, 0xcc, 0xeb, 0xd0, 0x2c, 0xc0, 0x45, 0x76, 0xe2, - 0x94, 0x2b, 0x5c, 0xb4, 0x20, 0x9f, 0x03, 0xd0, 0x2c, 0x4d, 0x7d, 0xc5, 0x3f, 0x38, 0x63, 0xe1, - 0x9d, 0x47, 0x71, 0x80, 0xcf, 0x31, 0x58, 0xcf, 0x6d, 0x21, 0x2f, 0xe7, 0xce, 0xf9, 0x10, 0x5a, - 0x53, 0x54, 0x44, 0x31, 0xa1, 0x60, 0xcb, 0x2e, 0x94, 0x3d, 0x45, 0x64, 0xd0, 0x28, 0xe5, 0x30, - 0x7b, 0x1b, 0x2e, 0x7d, 0x4c, 0xd3, 0xbd, 0x3c, 0x84, 0xd6, 0x99, 0x87, 0x34, 0x30, 0x4f, 0x6d, - 0x0a, 0x9e, 0xdc, 0x55, 0xb8, 0xfc, 0x2a, 0x23, 0x85, 0x58, 0x97, 0xc0, 0xe2, 0x26, 0x72, 0xfd, - 0xa0, 0x95, 0x27, 0x77, 0x03, 0xce, 0xe5, 0x78, 0x6f, 0xbc, 0x17, 0x7a, 0xff, 0x5a, 0xb0, 0x60, - 0xaa, 0x7d, 0x86, 0xe9, 0x7e, 0xe8, 0x23, 0x19, 0x43, 0x3d, 0x77, 0x03, 0xc8, 0xea, 0x09, 0xe7, - 0x41, 0x26, 0xe3, 0xac, 0x9d, 0x7a, 0x40, 0xdc, 0xb5, 0xaf, 0xff, 0xf8, 0xfb, 0xfb, 0xd2, 0x0a, - 0x59, 0xee, 0x9a, 0x23, 0xd0, 0x7d, 0x51, 0xb8, 0x11, 0x2f, 0xc9, 0x1e, 0x34, 0xf2, 0xdb, 0x8e, - 0xac, 0x9d, 0xba, 0x7c, 0x1d, 0xf7, 0x24, 0x15, 0x1d, 0xb9, 0x2d, 0x23, 0xcf, 0xbb, 0xb5, 0x2c, - 0xf2, 0x5d, 0xeb, 0x46, 0xef, 0xc7, 0x12, 0xb4, 0xf2, 0x2d, 0x37, 0xb5, 0xbf, 0x84, 0x85, 0x23, - 0x8b, 0x83, 0xbc, 0x7d, 0xca, 0x5e, 0x51, 0xa9, 0x5c, 0x3d, 0xd3, 0xf6, 0x71, 0x2f, 0xc9, 0x6c, - 0x2e, 0x90, 0xf3, 0xdd, 0xfc, 0xe6, 0x61, 0xdd, 0x17, 0xaa, 0x07, 0xdf, 0x59, 0xb0, 0x34, 0x1d, - 0x0d, 0xe4, 0xc8, 0x1d, 0x3c, 0x11, 0x68, 0xce, 0xbb, 0x67, 0x53, 0x2e, 0x26, 0x75, 0x63, 0x7a, - 0x52, 0xbd, 0x18, 0x9a, 0x0a, 0x35, 0xa6, 0x49, 0x5f, 0x40, 0x2d, 0x03, 0x1f, 0xb9, 0x7c, 0xac, - 0xf0, 0x02, 0x52, 0x9d, 0x2b, 0xaf, 0x94, 0xeb, 0xe8, 0x0b, 0x32, 0x7a, 0x8d, 0x54, 0xba, 0x0a, - 0x93, 0xf7, 0x2e, 0x43, 0xcb, 0x4f, 0x86, 0x45, 0xb3, 0xd1, 0xf6, 0x67, 0x15, 0xfd, 0xaf, 0xdc, - 0xf6, 0x9c, 0xfc, 0x0b, 0xf8, 0xf6, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xc7, 0xaf, 0x17, 0x4a, - 0xe3, 0x0d, 0x00, 0x00, + // 1237 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x57, 0x4b, 0x6f, 0x1b, 0xd5, + 0x17, 0xd7, 0xd8, 0x71, 0x1c, 0x1f, 0xdb, 0x49, 0x7a, 0x93, 0xa6, 0x93, 0x49, 0x1f, 0xc9, 0xfc, + 0xff, 0x55, 0x4b, 0x8b, 0x6c, 0xe1, 0xb2, 0x68, 0xcb, 0x02, 0xa5, 0x8f, 0x84, 0x4a, 0xa5, 0xaa, + 0xa6, 0xd0, 0x05, 0x08, 0x59, 0xd7, 0x33, 0xc7, 0xc9, 0x28, 0xe3, 0x19, 0x33, 0xf7, 0xda, 0x89, + 0x55, 0x95, 0x05, 0x5b, 0x76, 0xb0, 0xe0, 0x33, 0xb0, 0xe1, 0x1b, 0xb0, 0x62, 0xcb, 0x02, 0xc1, + 0x16, 0x76, 0x2c, 0xf8, 0x02, 0xec, 0xd1, 0x7d, 0x4d, 0x66, 0x12, 0xe7, 0xd1, 0xb2, 0xf2, 0x9c, + 0xf7, 0xeb, 0x77, 0xcf, 0x49, 0xc0, 0xa1, 0xc3, 0xb0, 0x3d, 0xbe, 0xd3, 0xf6, 0x23, 0x1a, 0xa6, + 0xc3, 0x9e, 0xfa, 0x6d, 0x0d, 0xd3, 0x84, 0x27, 0xa4, 0xe1, 0x27, 0x29, 0x26, 0xac, 0x25, 0x79, + 0xce, 0xb5, 0x9d, 0x24, 0xd9, 0x89, 0xb0, 0x2d, 0x65, 0xbd, 0x51, 0xbf, 0xcd, 0xc3, 0x01, 0x32, + 0x4e, 0x07, 0x43, 0xa5, 0xee, 0x5c, 0xd6, 0x0a, 0xc2, 0x23, 0x8d, 0xe3, 0x84, 0x53, 0x1e, 0x26, + 0x31, 0x53, 0x52, 0xf7, 0xfb, 0x12, 0x34, 0x5f, 0x8e, 0xa2, 0x18, 0x53, 0xda, 0x0b, 0xa3, 0x90, + 0x4f, 0x08, 0x81, 0x99, 0x98, 0x0e, 0xd0, 0xb6, 0xd6, 0xad, 0x9b, 0x35, 0x4f, 0x7e, 0x93, 0xeb, + 0x30, 0x2f, 0x7e, 0xd9, 0x90, 0xfa, 0xd8, 0x95, 0xd2, 0x92, 0x94, 0x36, 0x33, 0xee, 0x33, 0xa1, + 0xb6, 0x0e, 0xf5, 0x00, 0x99, 0x9f, 0x86, 0x43, 0x11, 0xc2, 0x2e, 0x4b, 0x9d, 0x3c, 0x4b, 0x38, + 0x8f, 0xc2, 0x78, 0xcf, 0x9e, 0x51, 0xce, 0xc5, 0x37, 0x71, 0x60, 0x8e, 0xe1, 0x18, 0xd3, 0x90, + 0x4f, 0xec, 0x8a, 0xe4, 0x67, 0xb4, 0x90, 0x0d, 0x90, 0xd3, 0x80, 0x72, 0x6a, 0xcf, 0x2a, 0x99, + 0xa1, 0xc9, 0x2a, 0xcc, 0xf5, 0xc3, 0x03, 0x0c, 0xba, 0xbd, 0x89, 0x5d, 0x95, 0xb2, 0xaa, 0xa4, + 0x1f, 0x4c, 0xc8, 0x03, 0xb8, 0x40, 0xfb, 0x7d, 0xf4, 0x39, 0x06, 0xdd, 0x31, 0xa6, 0x4c, 0x14, + 0x6c, 0xcf, 0xad, 0x97, 0x6f, 0xd6, 0x3b, 0x17, 0x5b, 0xf9, 0xf6, 0xb5, 0xb6, 0x90, 0xf2, 0x51, + 0x8a, 0xde, 0xa2, 0xd1, 0x7f, 0xa9, 0xd5, 0xdd, 0x5f, 0x2c, 0xa8, 0x6a, 0xe9, 0x7f, 0xe9, 0x89, + 0x0d, 0x55, 0x9d, 0x81, 0xee, 0x87, 0x21, 0x85, 0x03, 0xfd, 0xd9, 0xed, 0x27, 0xe9, 0x80, 0x72, + 0xdd, 0x95, 0xa6, 0xe6, 0x6e, 0x49, 0x26, 0x79, 0x0c, 0x0b, 0xe3, 0xdc, 0x80, 0x42, 0x64, 0x76, + 0x45, 0x56, 0xb2, 0x56, 0xac, 0xa4, 0x30, 0x45, 0xef, 0xa8, 0x8d, 0xbb, 0x06, 0x95, 0xa7, 0x74, + 0x82, 0xa9, 0xa8, 0x65, 0x97, 0xb2, 0x5d, 0x53, 0x8b, 0xf8, 0x76, 0xbf, 0xb1, 0xa0, 0xfe, 0x50, + 0x78, 0x79, 0xc1, 0x29, 0x1f, 0x31, 0x91, 0x74, 0x14, 0x32, 0x8e, 0x29, 0xb3, 0xad, 0xf5, 0xb2, + 0x48, 0x5a, 0x93, 0xe4, 0x32, 0xd4, 0x02, 0xe4, 0xe8, 0xf3, 0x24, 0x65, 0x76, 0x49, 0xca, 0x0e, + 0x19, 0xe4, 0x11, 0x2c, 0x46, 0x94, 0xf1, 0xee, 0x68, 0x18, 0x50, 0x8e, 0x5d, 0x01, 0x45, 0x59, + 0x75, 0xbd, 0xe3, 0xb4, 0x14, 0x0c, 0x5b, 0x06, 0xa7, 0xad, 0x4f, 0x0c, 0x4e, 0xbd, 0x79, 0x61, + 0xf3, 0xa9, 0x34, 0x11, 0x4c, 0xf7, 0x1e, 0x90, 0x6d, 0xe4, 0x9b, 0xb1, 0x8f, 0x8c, 0xa7, 0x13, + 0x0f, 0xbf, 0x1c, 0x21, 0xe3, 0xe4, 0x7f, 0xd0, 0xa4, 0x9a, 0xd5, 0xcd, 0x0d, 0xa3, 0x61, 0x98, + 0xa2, 0xdb, 0xee, 0xaf, 0x65, 0x58, 0x2a, 0xd8, 0xb2, 0x61, 0x12, 0x33, 0x24, 0x5b, 0x30, 0x67, + 0xf4, 0xa4, 0x5d, 0xbd, 0x73, 0xab, 0xd8, 0xbd, 0x29, 0x46, 0xad, 0x8c, 0x91, 0xd9, 0x92, 0xf7, + 0x60, 0x96, 0xc9, 0x16, 0xc9, 0x61, 0xd7, 0x3b, 0xab, 0x45, 0x2f, 0xb9, 0x1e, 0x7a, 0x5a, 0xd1, + 0xf9, 0x0a, 0x9a, 0xc6, 0x91, 0x1a, 0xc0, 0x3b, 0x50, 0x89, 0xc4, 0x87, 0x4e, 0x64, 0xa9, 0xe8, + 0x42, 0xea, 0x78, 0x4a, 0x43, 0xe0, 0x58, 0x35, 0x17, 0x83, 0x6e, 0x5f, 0x61, 0x51, 0x75, 0xfd, + 0x64, 0x1c, 0x1b, 0x7d, 0xcd, 0x60, 0xce, 0x4f, 0x16, 0xcc, 0x99, 0x04, 0xa6, 0x02, 0xf9, 0x06, + 0x2c, 0x30, 0x9f, 0xc6, 0x31, 0x06, 0x5d, 0x33, 0xf4, 0x19, 0x39, 0xd8, 0x79, 0xcd, 0x7e, 0xaa, + 0x67, 0x7f, 0x1b, 0x2e, 0x18, 0xc5, 0x43, 0x0c, 0x54, 0xa4, 0xea, 0xa2, 0x16, 0x3c, 0xca, 0xa0, + 0xb0, 0x0d, 0xb3, 0xb2, 0x06, 0x66, 0xcf, 0xca, 0x7c, 0xdb, 0xe7, 0xef, 0xb7, 0x6a, 0x81, 0x36, + 0x77, 0xff, 0x2c, 0xc1, 0xd2, 0xf3, 0x84, 0xbd, 0x15, 0x1e, 0xc8, 0x0a, 0xcc, 0xea, 0xb7, 0xa5, + 0x1e, 0xa7, 0xa6, 0xc8, 0xc3, 0x2c, 0xbb, 0xb2, 0xcc, 0xee, 0x76, 0x31, 0xbb, 0x29, 0xf1, 0x24, + 0xaf, 0x90, 0x99, 0xf3, 0xb3, 0x05, 0xb5, 0x8c, 0x3b, 0xed, 0x5d, 0x09, 0xde, 0x90, 0xf2, 0x5d, + 0x1d, 0x5c, 0x7e, 0x13, 0x0f, 0xaa, 0xbb, 0x48, 0x83, 0xc3, 0xd8, 0x77, 0xdf, 0x20, 0x76, 0xeb, + 0x23, 0x65, 0xfa, 0x38, 0x16, 0x52, 0xe3, 0xc8, 0xb9, 0x0f, 0x8d, 0xbc, 0x80, 0x2c, 0x42, 0x79, + 0x0f, 0x27, 0x3a, 0x15, 0xf1, 0x49, 0x96, 0xa1, 0x32, 0xa6, 0xd1, 0xc8, 0x2c, 0x29, 0x45, 0xdc, + 0x2f, 0xdd, 0xb5, 0xdc, 0x27, 0xb0, 0x5c, 0x0c, 0xa9, 0x9f, 0xcc, 0x21, 0xd4, 0xad, 0x73, 0x42, + 0xdd, 0xfd, 0xd1, 0x82, 0x95, 0x6d, 0xe4, 0xcf, 0x12, 0x1e, 0xf6, 0x43, 0x5f, 0xde, 0x19, 0x33, + 0xad, 0xf7, 0x61, 0x25, 0x89, 0x82, 0x6e, 0x7e, 0x2b, 0x4d, 0xba, 0x43, 0xba, 0x63, 0xc6, 0xb6, + 0x9c, 0x44, 0x41, 0x61, 0x83, 0x3d, 0xa7, 0x3b, 0x28, 0xac, 0x62, 0xdc, 0x9f, 0x66, 0xa5, 0xca, + 0x58, 0x8e, 0x71, 0xff, 0xb8, 0xd5, 0x32, 0x54, 0xa2, 0x70, 0x10, 0x72, 0xb9, 0x7a, 0x2a, 0x9e, + 0x22, 0x32, 0xe8, 0xcf, 0x1c, 0x42, 0xdf, 0xfd, 0xa3, 0x04, 0x97, 0x8e, 0x25, 0xac, 0xeb, 0x7f, + 0x09, 0x8d, 0x38, 0xc7, 0xd7, 0x5d, 0xe8, 0x1c, 0x83, 0xf1, 0x34, 0xe3, 0x56, 0x81, 0x59, 0xf0, + 0xe3, 0xfc, 0x6d, 0x41, 0x23, 0x2f, 0x9e, 0xfa, 0x26, 0x6d, 0xa8, 0xfa, 0x29, 0x52, 0x8e, 0x81, + 0xae, 0xd4, 0x90, 0xe2, 0x22, 0x2a, 0x77, 0x18, 0xe8, 0x83, 0x92, 0xd1, 0xc2, 0x2a, 0xc0, 0x08, + 0x85, 0x95, 0xaa, 0xd2, 0x90, 0xe4, 0x1e, 0x94, 0x93, 0x28, 0x90, 0xe7, 0xb5, 0xde, 0xb9, 0x71, + 0x04, 0x70, 0x74, 0x07, 0xb3, 0xde, 0x47, 0xa8, 0x81, 0x10, 0x22, 0xf3, 0x84, 0x8d, 0x30, 0x8d, + 0x71, 0x5f, 0x5e, 0xdf, 0x37, 0x31, 0x8d, 0x71, 0xdf, 0xfd, 0xad, 0x04, 0xab, 0x27, 0xaa, 0x90, + 0x0d, 0x68, 0xf8, 0xa3, 0x34, 0xc5, 0x98, 0xe7, 0x81, 0x50, 0xd7, 0x3c, 0x39, 0xc9, 0x35, 0xa8, + 0xc5, 0x78, 0xc0, 0xf3, 0x23, 0x9f, 0x13, 0x8c, 0x53, 0xc6, 0xbc, 0x09, 0xcd, 0x02, 0x5c, 0x64, + 0x27, 0xce, 0x38, 0x96, 0x45, 0x0b, 0xf2, 0x39, 0x00, 0xcd, 0xd2, 0xd4, 0xc7, 0xf6, 0x83, 0x73, + 0x16, 0xde, 0x7a, 0x12, 0x07, 0x78, 0x80, 0xc1, 0x66, 0x6e, 0x0b, 0x79, 0x39, 0x77, 0xce, 0x87, + 0xb0, 0x34, 0x45, 0x45, 0x14, 0x13, 0x0a, 0xb6, 0xec, 0x42, 0xc5, 0x53, 0x44, 0x06, 0x8d, 0x52, + 0x0e, 0xb3, 0x77, 0xe0, 0xca, 0xc7, 0x34, 0xdd, 0xcb, 0x43, 0x68, 0x93, 0x79, 0x48, 0x03, 0xf3, + 0xd4, 0xa6, 0xe0, 0xc9, 0x5d, 0x87, 0xab, 0x27, 0x19, 0x29, 0xc4, 0xba, 0x04, 0x16, 0xb7, 0x91, + 0xeb, 0x07, 0xad, 0x3c, 0xb9, 0x5b, 0x70, 0x21, 0xc7, 0x7b, 0xeb, 0xbd, 0xd0, 0xf9, 0xc7, 0x82, + 0x05, 0x53, 0xed, 0x0b, 0x4c, 0xc7, 0xa1, 0x8f, 0x64, 0x04, 0xf5, 0xdc, 0x0d, 0x20, 0xeb, 0xa7, + 0x9c, 0x07, 0x99, 0x8c, 0xb3, 0x71, 0xe6, 0x01, 0x71, 0x37, 0xbe, 0xfe, 0xfd, 0xaf, 0xef, 0x4a, + 0x6b, 0x64, 0xb5, 0x6d, 0x8e, 0x40, 0xfb, 0x55, 0xe1, 0x46, 0xbc, 0x26, 0x7b, 0xd0, 0xc8, 0x6f, + 0x3b, 0xb2, 0x71, 0xe6, 0xf2, 0x75, 0xdc, 0xd3, 0x54, 0x74, 0xe4, 0x65, 0x19, 0x79, 0xde, 0xad, + 0x65, 0x91, 0xef, 0x5b, 0xb7, 0x3a, 0x3f, 0x94, 0x60, 0x29, 0xdf, 0x72, 0x53, 0xfb, 0x6b, 0x58, + 0x38, 0xb2, 0x38, 0xc8, 0xff, 0xcf, 0xd8, 0x2b, 0x2a, 0x95, 0xeb, 0xe7, 0xda, 0x3e, 0xee, 0x15, + 0x99, 0xcd, 0x25, 0x72, 0xb1, 0x9d, 0xdf, 0x3c, 0xac, 0xfd, 0x4a, 0xf5, 0xe0, 0x5b, 0x0b, 0x56, + 0xa6, 0xa3, 0x81, 0x1c, 0xb9, 0x83, 0xa7, 0x02, 0xcd, 0x79, 0xf7, 0x7c, 0xca, 0xc5, 0xa4, 0x6e, + 0x4d, 0x4f, 0xaa, 0x13, 0x43, 0x53, 0xa1, 0xc6, 0x34, 0xe9, 0x0b, 0xa8, 0x65, 0xe0, 0x23, 0x57, + 0x8f, 0x15, 0x5e, 0x40, 0xaa, 0x73, 0xed, 0x44, 0xb9, 0x8e, 0xbe, 0x20, 0xa3, 0xd7, 0x48, 0xb5, + 0xad, 0x30, 0xf9, 0xe0, 0x2a, 0x2c, 0xf9, 0xc9, 0xa0, 0x68, 0x36, 0xec, 0x7d, 0x56, 0xd5, 0xff, + 0x71, 0xf5, 0x66, 0xe5, 0x1f, 0xaa, 0x77, 0xfe, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x7b, 0xd2, 0x74, + 0xfa, 0x8a, 0x0d, 0x00, 0x00, } diff --git a/api/v3/clairpb/clair.pb.gw.go b/api/v3/clairpb/clair.pb.gw.go index e077fe46..16300c1d 100644 --- a/api/v3/clairpb/clair.pb.gw.go +++ b/api/v3/clairpb/clair.pb.gw.go @@ -28,10 +28,6 @@ var _ status.Status var _ = runtime.String var _ = utilities.NewDoubleArray -var ( - filter_AncestryService_GetAncestry_0 = &utilities.DoubleArray{Encoding: map[string]int{"ancestry_name": 0}, Base: []int{1, 1, 0}, Check: []int{0, 1, 2}} -) - func request_AncestryService_GetAncestry_0(ctx context.Context, marshaler runtime.Marshaler, client AncestryServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var protoReq GetAncestryRequest var metadata runtime.ServerMetadata @@ -54,10 +50,6 @@ func request_AncestryService_GetAncestry_0(ctx context.Context, marshaler runtim return nil, metadata, status.Errorf(codes.InvalidArgument, "type mismatch, parameter: %s, error: %v", "ancestry_name", err) } - if err := runtime.PopulateQueryParameters(&protoReq, req.URL.Query(), filter_AncestryService_GetAncestry_0); err != nil { - return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) - } - msg, err := client.GetAncestry(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return msg, metadata, err diff --git a/api/v3/clairpb/clair.proto b/api/v3/clairpb/clair.proto index aad4e658..c0e3ff8a 100644 --- a/api/v3/clairpb/clair.proto +++ b/api/v3/clairpb/clair.proto @@ -88,10 +88,6 @@ message ClairStatus { message GetAncestryRequest { // The name of the desired ancestry. string ancestry_name = 1; - // Whether to include vulnerabilities or not in the response. - bool with_vulnerabilities = 2; - // Whether to include features or not in the response. - bool with_features = 3; } message GetAncestryResponse { diff --git a/api/v3/clairpb/clair.swagger.json b/api/v3/clairpb/clair.swagger.json index 9bc7d0ee..32e00e53 100644 --- a/api/v3/clairpb/clair.swagger.json +++ b/api/v3/clairpb/clair.swagger.json @@ -60,22 +60,6 @@ "in": "path", "required": true, "type": "string" - }, - { - "name": "with_vulnerabilities", - "description": "Whether to include vulnerabilities or not in the response.", - "in": "query", - "required": false, - "type": "boolean", - "format": "boolean" - }, - { - "name": "with_features", - "description": "Whether to include features or not in the response.", - "in": "query", - "required": false, - "type": "boolean", - "format": "boolean" } ], "tags": [ diff --git a/api/v3/clairpb/convert.go b/api/v3/clairpb/convert.go index 364781e2..a7d2172b 100644 --- a/api/v3/clairpb/convert.go +++ b/api/v3/clairpb/convert.go @@ -122,24 +122,6 @@ func VulnerabilityWithFixedInFromDatabaseModel(dbVuln database.VulnerabilityWith return vuln, nil } -// AncestryFromDatabaseModel converts database ancestry to api ancestry. -func AncestryFromDatabaseModel(dbAncestry database.Ancestry) *GetAncestryResponse_Ancestry { - ancestry := &GetAncestryResponse_Ancestry{ - Name: dbAncestry.Name, - ScannedDetectors: dbAncestry.ProcessedBy.Detectors, - ScannedListers: dbAncestry.ProcessedBy.Listers, - } - - for _, layer := range dbAncestry.Layers { - ancestry.Layers = append(ancestry.Layers, - &GetAncestryResponse_AncestryLayer{ - Layer: LayerFromDatabaseModel(layer), - }) - } - - return ancestry -} - // LayerFromDatabaseModel converts database layer to api layer. func LayerFromDatabaseModel(dbLayer database.Layer) *Layer { layer := Layer{Hash: dbLayer.Hash} diff --git a/api/v3/rpc.go b/api/v3/rpc.go index 9c893bf2..b91da65e 100644 --- a/api/v3/rpc.go +++ b/api/v3/rpc.go @@ -44,11 +44,12 @@ type StatusServer struct { // GetStatus implements getting the current status of Clair via the Clair service. func (s *StatusServer) GetStatus(ctx context.Context, req *pb.GetStatusRequest) (*pb.GetStatusResponse, error) { - if clairStatus, err := GetClairStatus(s.Store); err != nil { + clairStatus, err := GetClairStatus(s.Store) + if err != nil { return nil, status.Error(codes.Internal, err.Error()) - } else { - return &pb.GetStatusResponse{Status: clairStatus}, nil } + + return &pb.GetStatusResponse{Status: clairStatus}, nil } // PostAncestry implements posting an ancestry via the Clair gRPC service. @@ -105,11 +106,7 @@ func (s *AncestryServer) PostAncestry(ctx context.Context, req *pb.PostAncestryR // GetAncestry implements retrieving an ancestry via the Clair gRPC service. func (s *AncestryServer) GetAncestry(ctx context.Context, req *pb.GetAncestryRequest) (*pb.GetAncestryResponse, error) { - var ( - respAncestry *pb.GetAncestryResponse_Ancestry - name = req.GetAncestryName() - ) - + name := req.GetAncestryName() if name == "" { return nil, status.Errorf(codes.InvalidArgument, "ancestry name should not be empty") } @@ -118,79 +115,41 @@ func (s *AncestryServer) GetAncestry(ctx context.Context, req *pb.GetAncestryReq if err != nil { return nil, status.Error(codes.Internal, err.Error()) } + defer tx.Rollback() - if req.GetWithFeatures() || req.GetWithVulnerabilities() { - ancestry, ok, err := tx.FindAncestryWithContent(name) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } + ancestry, ok, err := tx.FindAncestry(name) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } - if !ok { - return nil, status.Error(codes.NotFound, fmt.Sprintf("requested ancestry '%s' is not found", req.GetAncestryName())) - } + if !ok { + return nil, status.Error(codes.NotFound, fmt.Sprintf("requested ancestry '%s' is not found", req.GetAncestryName())) + } - respAncestry = &pb.GetAncestryResponse_Ancestry{ - Name: name, - ScannedDetectors: ancestry.ProcessedBy.Detectors, - ScannedListers: ancestry.ProcessedBy.Listers, - } + pbAncestry := &pb.GetAncestryResponse_Ancestry{ + Name: ancestry.Name, + ScannedDetectors: ancestry.ProcessedBy.Detectors, + ScannedListers: ancestry.ProcessedBy.Listers, + } - for _, layer := range ancestry.Layers { - ancestryLayer := &pb.GetAncestryResponse_AncestryLayer{ - Layer: &pb.Layer{ - Hash: layer.Hash, - }, - } - - if req.GetWithVulnerabilities() { - featureVulnerabilities, err := tx.FindAffectedNamespacedFeatures(layer.DetectedFeatures) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - for _, fv := range featureVulnerabilities { - // Ensure that every feature can be found. - if !fv.Valid { - return nil, status.Error(codes.Internal, "ancestry feature is not found") - } - - feature := pb.NamespacedFeatureFromDatabaseModel(fv.NamespacedFeature) - for _, v := range fv.AffectedBy { - vuln, err := pb.VulnerabilityWithFixedInFromDatabaseModel(v) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - feature.Vulnerabilities = append(feature.Vulnerabilities, vuln) - } - ancestryLayer.DetectedFeatures = append(ancestryLayer.DetectedFeatures, feature) - } - } else { - for _, dbFeature := range layer.DetectedFeatures { - ancestryLayer.DetectedFeatures = append(ancestryLayer.DetectedFeatures, pb.NamespacedFeatureFromDatabaseModel(dbFeature)) - } - } - - respAncestry.Layers = append(respAncestry.Layers, ancestryLayer) - } - } else { - dbAncestry, ok, err := tx.FindAncestry(name) + for _, layer := range ancestry.Layers { + pbLayer, err := GetPbAncestryLayer(tx, layer) if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } else if !ok { - return nil, status.Error(codes.NotFound, fmt.Sprintf("requested ancestry '%s' is not found", req.GetAncestryName())) + return nil, err } - respAncestry = pb.AncestryFromDatabaseModel(dbAncestry) + + pbAncestry.Layers = append(pbAncestry.Layers, pbLayer) } - clairStatus, err := GetClairStatus(s.Store) + pbClairStatus, err := GetClairStatus(s.Store) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } return &pb.GetAncestryResponse{ - Status: clairStatus, - Ancestry: respAncestry, + Status: pbClairStatus, + Ancestry: pbAncestry, }, nil } diff --git a/api/v3/util.go b/api/v3/util.go index ecf2cb12..6b6baa0a 100644 --- a/api/v3/util.go +++ b/api/v3/util.go @@ -5,6 +5,8 @@ import ( pb "github.com/coreos/clair/api/v3/clairpb" "github.com/coreos/clair/database" "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // GetClairStatus retrieves the current status of Clair and wrap it inside @@ -29,3 +31,45 @@ func GetClairStatus(store database.Datastore) (*pb.ClairStatus, error) { } return status, nil } + +// GetPbAncestryLayer retrieves an ancestry layer with vulnerabilities and +// features in an ancestry based on the provided database layer. +func GetPbAncestryLayer(session database.Session, layer database.AncestryLayer) (*pb.GetAncestryResponse_AncestryLayer, error) { + pbLayer := &pb.GetAncestryResponse_AncestryLayer{ + Layer: &pb.Layer{ + Hash: layer.Hash, + }, + } + + var ( + features []database.NullableAffectedNamespacedFeature + err error + ) + + if features, err = session.FindAffectedNamespacedFeatures(layer.DetectedFeatures); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + for _, feature := range features { + if !feature.Valid { + return nil, status.Error(codes.Internal, "ancestry feature is not found") + } + + var ( + pbFeature = pb.NamespacedFeatureFromDatabaseModel(feature.NamespacedFeature) + pbVuln *pb.Vulnerability + err error + ) + for _, vuln := range feature.AffectedBy { + if pbVuln, err = pb.VulnerabilityWithFixedInFromDatabaseModel(vuln); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + pbFeature.Vulnerabilities = append(pbFeature.Vulnerabilities, pbVuln) + } + + pbLayer.DetectedFeatures = append(pbLayer.DetectedFeatures, pbFeature) + } + + return pbLayer, nil +} diff --git a/database/database.go b/database/database.go index 5c427494..54e63b35 100644 --- a/database/database.go +++ b/database/database.go @@ -91,18 +91,11 @@ type Session interface { // UpsertAncestry inserts or replaces an ancestry and its namespaced // features and processors used to scan the ancestry. - UpsertAncestry(AncestryWithContent) error + UpsertAncestry(Ancestry) error - // FindAncestry retrieves an ancestry with processors used to scan the - // ancestry. If the ancestry is not found, return false. - // - // The ancestry's processors are returned to short cut processing ancestry - // if it has been processed by all processors in the current Clair instance. - FindAncestry(name string) (ancestry Ancestry, found bool, err error) - - // FindAncestryWithContent retrieves an ancestry with all detected + // FindAncestry retrieves an ancestry with all detected // namespaced features. If the ancestry is not found, return false. - FindAncestryWithContent(name string) (ancestry AncestryWithContent, found bool, err error) + FindAncestry(name string) (ancestry Ancestry, found bool, err error) // PersistFeatures inserts a set of features if not in the database. PersistFeatures(features []Feature) error diff --git a/database/mock.go b/database/mock.go index ed6e8f16..e28b7401 100644 --- a/database/mock.go +++ b/database/mock.go @@ -21,9 +21,8 @@ import "time" type MockSession struct { FctCommit func() error FctRollback func() error - FctUpsertAncestry func(AncestryWithContent) error + FctUpsertAncestry func(Ancestry) error FctFindAncestry func(name string) (Ancestry, bool, error) - FctFindAncestryWithContent func(name string) (AncestryWithContent, bool, error) FctFindAffectedNamespacedFeatures func(features []NamespacedFeature) ([]NullableAffectedNamespacedFeature, error) FctPersistNamespaces func([]Namespace) error FctPersistFeatures func([]Feature) error @@ -63,7 +62,7 @@ func (ms *MockSession) Rollback() error { panic("required mock function not implemented") } -func (ms *MockSession) UpsertAncestry(ancestry AncestryWithContent) error { +func (ms *MockSession) UpsertAncestry(ancestry Ancestry) error { if ms.FctUpsertAncestry != nil { return ms.FctUpsertAncestry(ancestry) } @@ -77,13 +76,6 @@ func (ms *MockSession) FindAncestry(name string) (Ancestry, bool, error) { panic("required mock function not implemented") } -func (ms *MockSession) FindAncestryWithContent(name string) (AncestryWithContent, bool, error) { - if ms.FctFindAncestryWithContent != nil { - return ms.FctFindAncestryWithContent(name) - } - panic("required mock function not implemented") -} - func (ms *MockSession) FindAffectedNamespacedFeatures(features []NamespacedFeature) ([]NullableAffectedNamespacedFeature, error) { if ms.FctFindAffectedNamespacedFeatures != nil { return ms.FctFindAffectedNamespacedFeatures(features) diff --git a/database/models.go b/database/models.go index e5a82358..79b061ee 100644 --- a/database/models.go +++ b/database/models.go @@ -34,17 +34,6 @@ type Ancestry struct { ProcessedBy Processors // Layers should be ordered and i_th layer is the parent of i+1_th layer in // the slice. - Layers []Layer -} - -// AncestryWithContent has the ancestry's name and the Ancestry Layers -// associated with it. -type AncestryWithContent struct { - Ancestry - - // TODO(sidchen) deduplicate the Layers here and the Layers in - // Ancestry.Layers. - // AncestryLayers should have the same order as Ancestry.Layers. Layers []AncestryLayer } @@ -52,7 +41,8 @@ type AncestryWithContent struct { type AncestryLayer struct { Layer - // DetectedFeatures are the features introduced by this layer. + // DetectedFeatures are the features introduced by this layer when it was + // processed. DetectedFeatures []NamespacedFeature } diff --git a/database/pgsql/ancestry.go b/database/pgsql/ancestry.go index b5678bc5..20d93992 100644 --- a/database/pgsql/ancestry.go +++ b/database/pgsql/ancestry.go @@ -10,7 +10,13 @@ import ( "github.com/coreos/clair/pkg/commonerr" ) -func (tx *pgSession) UpsertAncestry(ancestry database.AncestryWithContent) error { +type ancestryLayerWithID struct { + database.AncestryLayer + + layerID int64 +} + +func (tx *pgSession) UpsertAncestry(ancestry database.Ancestry) error { if ancestry.Name == "" { log.Error("Empty ancestry name is not allowed") return commonerr.NewBadRequestError("could not insert an ancestry with empty name") @@ -44,79 +50,56 @@ func (tx *pgSession) UpsertAncestry(ancestry database.AncestryWithContent) error ancestryID, ancestry.ProcessedBy) } -func (tx *pgSession) FindAncestry(name string) (database.Ancestry, bool, error) { - var ( - ancestryID int64 - ancestry = database.Ancestry{Name: name} - err error - ) - - if err = tx.QueryRow(searchAncestry, name).Scan(&ancestryID); err != nil { +func (tx *pgSession) findAncestryID(name string) (int64, bool, error) { + var id sql.NullInt64 + if err := tx.QueryRow(searchAncestry, name).Scan(&id); err != nil { if err == sql.ErrNoRows { - return ancestry, false, nil + return 0, false, nil } - return ancestry, false, handleError("searchAncestry", err) - } - - if ancestry.Layers, err = tx.findAncestryLayers(ancestryID); err != nil { - return ancestry, false, err - } - - if ancestry.ProcessedBy.Detectors, err = tx.findProcessors(searchAncestryDetectors, "searchAncestryDetectors", "detector", ancestryID); err != nil { - return ancestry, false, err - } - if ancestry.ProcessedBy.Listers, err = tx.findProcessors(searchAncestryListers, "searchAncestryListers", "lister", ancestryID); err != nil { - return ancestry, false, err + return 0, false, handleError("searchAncestry", err) } - return ancestry, true, nil + return id.Int64, true, nil } -func (tx *pgSession) FindAncestryWithContent(name string) (database.AncestryWithContent, bool, error) { +func (tx *pgSession) findAncestryProcessors(id int64) (database.Processors, error) { var ( - ancestryContent database.AncestryWithContent - isValid bool - err error + processors database.Processors + err error ) - if ancestryContent.Ancestry, isValid, err = tx.FindAncestry(name); err != nil || !isValid { - return ancestryContent, isValid, err + if processors.Detectors, err = tx.findProcessors(searchAncestryDetectors, id); err != nil { + return processors, handleError("searchAncestryDetectors", err) } - rows, err := tx.Query(searchAncestryFeatures, name) - if err != nil { - return ancestryContent, false, handleError("searchAncestryFeatures", err) + if processors.Listers, err = tx.findProcessors(searchAncestryListers, id); err != nil { + return processors, handleError("searchAncestryListers", err) } - features := map[int][]database.NamespacedFeature{} - for rows.Next() { - var ( - feature database.NamespacedFeature - // layerIndex is used to determine which layer the namespaced feature belongs to. - layerIndex sql.NullInt64 - ) + return processors, err +} - if err := rows.Scan(&feature.Namespace.Name, - &feature.Namespace.VersionFormat, - &feature.Feature.Name, &feature.Feature.Version, - &layerIndex); err != nil { - return ancestryContent, false, handleError("searchAncestryFeatures", err) - } +func (tx *pgSession) FindAncestry(name string) (database.Ancestry, bool, error) { + var ( + ancestry = database.Ancestry{Name: name} + err error + ) - feature.Feature.VersionFormat = feature.Namespace.VersionFormat // This looks strange. - features[int(layerIndex.Int64)] = append(features[int(layerIndex.Int64)], feature) + id, ok, err := tx.findAncestryID(name) + if !ok || err != nil { + return ancestry, ok, err } - // By the assumption of Ancestry Layer Index, we have the ancestry's layer - // index corresponding to the index in the array. - for index, layer := range ancestryContent.Ancestry.Layers { - ancestryLayer := database.AncestryLayer{Layer: layer} - ancestryLayer.DetectedFeatures, _ = features[index] - ancestryContent.Layers = append(ancestryContent.Layers, ancestryLayer) + if ancestry.ProcessedBy, err = tx.findAncestryProcessors(id); err != nil { + return ancestry, false, err } - return ancestryContent, true, nil + if ancestry.Layers, err = tx.findAncestryLayers(id); err != nil { + return ancestry, false, err + } + + return ancestry, true, nil } func (tx *pgSession) deleteAncestry(name string) error { @@ -133,49 +116,115 @@ func (tx *pgSession) deleteAncestry(name string) error { return nil } -func (tx *pgSession) findProcessors(query, queryName, processorType string, id int64) ([]string, error) { +func (tx *pgSession) findProcessors(query string, id int64) ([]string, error) { + var ( + processors []string + processor string + ) + rows, err := tx.Query(query, id) if err != nil { if err == sql.ErrNoRows { - log.Warning("No " + processorType + " are used") return nil, nil } - return nil, handleError(queryName, err) - } - var ( - processors []string - processor string - ) + return nil, err + } for rows.Next() { - err := rows.Scan(&processor) - if err != nil { - return nil, handleError(queryName, err) + if err := rows.Scan(&processor); err != nil { + return nil, err } + processors = append(processors, processor) } return processors, nil } -func (tx *pgSession) findAncestryLayers(ancestryID int64) ([]database.Layer, error) { - rows, err := tx.Query(searchAncestryLayer, ancestryID) - if err != nil { +func (tx *pgSession) findAncestryLayers(id int64) ([]database.AncestryLayer, error) { + var ( + err error + rows *sql.Rows + // layer index -> Ancestry Layer + Layer ID + layers = map[int64]ancestryLayerWithID{} + // layer index -> layer-wise features + features = map[int64][]database.NamespacedFeature{} + ancestryLayers []database.AncestryLayer + ) + + // retrieve ancestry layer metadata + if rows, err = tx.Query(searchAncestryLayer, id); err != nil { return nil, handleError("searchAncestryLayer", err) } - layers := []database.Layer{} for rows.Next() { - var layer database.Layer - if err := rows.Scan(&layer.Hash); err != nil { + var ( + layer database.AncestryLayer + index sql.NullInt64 + id sql.NullInt64 + ) + + if err = rows.Scan(&layer.Hash, &id, &index); err != nil { return nil, handleError("searchAncestryLayer", err) } - layers = append(layers, layer) + if !index.Valid || !id.Valid { + return nil, commonerr.ErrNotFound + } + + if _, ok := layers[index.Int64]; ok { + // one ancestry index should correspond to only one layer + return nil, database.ErrInconsistent + } + + layers[index.Int64] = ancestryLayerWithID{layer, id.Int64} + } + + for _, layer := range layers { + if layer.ProcessedBy, err = tx.findLayerProcessors(layer.layerID); err != nil { + return nil, err + } + } + + // retrieve ancestry layer's namespaced features + if rows, err = tx.Query(searchAncestryFeatures, id); err != nil { + return nil, handleError("searchAncestryFeatures", err) + } + + for rows.Next() { + var ( + feature database.NamespacedFeature + // index is used to determine which layer the feature belongs to. + index sql.NullInt64 + ) + + if err := rows.Scan( + &feature.Namespace.Name, + &feature.Namespace.VersionFormat, + &feature.Feature.Name, + &feature.Feature.Version, + &feature.Feature.VersionFormat, + &index, + ); err != nil { + return nil, handleError("searchAncestryFeatures", err) + } + + if feature.Feature.VersionFormat != feature.Namespace.VersionFormat { + // Feature must have the same version format as the associated + // namespace version format. + return nil, database.ErrInconsistent + } + + features[index.Int64] = append(features[index.Int64], feature) + } + + for index, layer := range layers { + layer.DetectedFeatures = features[index] + ancestryLayers = append(ancestryLayers, layer.AncestryLayer) } - return layers, nil + return ancestryLayers, nil } // insertAncestryLayers inserts the ancestry layers along with its content into diff --git a/database/pgsql/ancestry_test.go b/database/pgsql/ancestry_test.go index 2453f28d..ccc6855c 100644 --- a/database/pgsql/ancestry_test.go +++ b/database/pgsql/ancestry_test.go @@ -26,13 +26,8 @@ import ( func TestUpsertAncestry(t *testing.T) { store, tx := openSessionForTest(t, "UpsertAncestry", true) defer closeTest(t, store, tx) - a1 := database.AncestryWithContent{ - Ancestry: database.Ancestry{ - Name: "a1", - Layers: []database.Layer{ - {Hash: "layer-N"}, - }, - }, + a1 := database.Ancestry{ + Name: "a1", Layers: []database.AncestryLayer{ { Layer: database.Layer{ @@ -42,15 +37,10 @@ func TestUpsertAncestry(t *testing.T) { }, } - a2 := database.AncestryWithContent{} + a2 := database.Ancestry{} - a3 := database.AncestryWithContent{ - Ancestry: database.Ancestry{ - Name: "a", - Layers: []database.Layer{ - {Hash: "layer-0"}, - }, - }, + a3 := database.Ancestry{ + Name: "a", Layers: []database.AncestryLayer{ { Layer: database.Layer{ @@ -60,13 +50,8 @@ func TestUpsertAncestry(t *testing.T) { }, } - a4 := database.AncestryWithContent{ - Ancestry: database.Ancestry{ - Name: "a", - Layers: []database.Layer{ - {Hash: "layer-1"}, - }, - }, + a4 := database.Ancestry{ + Name: "a", Layers: []database.AncestryLayer{ { Layer: database.Layer{ @@ -123,10 +108,10 @@ func TestUpsertAncestry(t *testing.T) { // replace valid case assert.Nil(t, tx.UpsertAncestry(a4)) // validate - ancestry, ok, err := tx.FindAncestryWithContent("a") + ancestry, ok, err := tx.FindAncestry("a") assert.Nil(t, err) assert.True(t, ok) - assert.Equal(t, a4, ancestry.Ancestry) + assertAncestryEqual(t, a4, ancestry) } func assertProcessorsEqual(t *testing.T, expected database.Processors, actual database.Processors) bool { @@ -137,36 +122,10 @@ func assertProcessorsEqual(t *testing.T, expected database.Processors, actual da return assert.Equal(t, expected.Detectors, actual.Detectors) && assert.Equal(t, expected.Listers, actual.Listers) } -func TestFindAncestry(t *testing.T) { - store, tx := openSessionForTest(t, "FindAncestry", true) - defer closeTest(t, store, tx) - - _, ok, err := tx.FindAncestry("ancestry-non") - assert.Nil(t, err) - assert.False(t, ok) - - expected := database.Ancestry{ - Name: "ancestry-1", - Layers: []database.Layer{ - {Hash: "layer-0"}, - {Hash: "layer-1"}, - {Hash: "layer-2"}, - {Hash: "layer-3a"}, - }, - ProcessedBy: database.Processors{ - Detectors: []string{"os-release"}, - Listers: []string{"dpkg"}, - }, - } - - a, ok2, err := tx.FindAncestry("ancestry-1") - if assert.Nil(t, err) && assert.True(t, ok2) { - assertAncestryEqual(t, expected, a) - } -} - -func assertAncestryWithFeatureEqual(t *testing.T, expected database.AncestryWithContent, actual database.AncestryWithContent) bool { - if assertAncestryEqual(t, expected.Ancestry, actual.Ancestry) && assert.Equal(t, len(expected.Layers), len(actual.Layers)) { +func assertAncestryEqual(t *testing.T, expected database.Ancestry, actual database.Ancestry) bool { + assert.Equal(t, expected.Name, actual.Name) + assertProcessorsEqual(t, expected.ProcessedBy, actual.ProcessedBy) + if assert.Equal(t, len(expected.Layers), len(actual.Layers)) { for index, layer := range expected.Layers { if !assertAncestryLayerEqual(t, layer, actual.Layers[index]) { return false @@ -182,37 +141,22 @@ func assertAncestryLayerEqual(t *testing.T, expected database.AncestryLayer, act assertNamespacedFeatureEqual(t, expected.DetectedFeatures, actual.DetectedFeatures) } -func assertAncestryEqual(t *testing.T, expected database.Ancestry, actual database.Ancestry) bool { - return assert.Equal(t, expected.Name, actual.Name) && - assert.Equal(t, expected.Layers, actual.Layers) && - assertProcessorsEqual(t, expected.ProcessedBy, actual.ProcessedBy) -} - -func TestFindAncestryWithContent(t *testing.T) { - store, tx := openSessionForTest(t, "FindAncestryWithContent", true) +func TestFindAncestry(t *testing.T) { + store, tx := openSessionForTest(t, "FindAncestry", true) defer closeTest(t, store, tx) // invalid - _, ok, err := tx.FindAncestryWithContent("ancestry-non") + _, ok, err := tx.FindAncestry("ancestry-non") if assert.Nil(t, err) { assert.False(t, ok) } - expected := database.AncestryWithContent{ - Ancestry: database.Ancestry{ - Name: "ancestry-2", - Layers: []database.Layer{ - {Hash: "layer-0"}, - {Hash: "layer-1"}, - {Hash: "layer-2"}, - {Hash: "layer-3b"}, - }, - ProcessedBy: database.Processors{ - Detectors: []string{"os-release"}, - Listers: []string{"dpkg"}, - }, + expected := database.Ancestry{ + Name: "ancestry-2", + ProcessedBy: database.Processors{ + Detectors: []string{"os-release"}, + Listers: []string{"dpkg"}, }, - Layers: []database.AncestryLayer{ { Layer: database.Layer{ @@ -261,8 +205,8 @@ func TestFindAncestryWithContent(t *testing.T) { }, } // valid - ancestry, ok, err := tx.FindAncestryWithContent("ancestry-2") + ancestry, ok, err := tx.FindAncestry("ancestry-2") if assert.Nil(t, err) && assert.True(t, ok) { - assertAncestryWithFeatureEqual(t, expected, ancestry) + assertAncestryEqual(t, expected, ancestry) } } diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go index daa6a704..93e5a97f 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer.go @@ -293,15 +293,23 @@ func (tx *pgSession) findLayer(hash string) (database.Layer, int64, bool, error) return layer, layerID, false, err } - layer.ProcessedBy.Detectors, err = tx.findProcessors(searchLayerDetectors, "searchLayerDetectors", "detector", layerID) - if err != nil { - return layer, layerID, false, err + layer.ProcessedBy, err = tx.findLayerProcessors(layerID) + return layer, layerID, true, err +} + +func (tx *pgSession) findLayerProcessors(id int64) (database.Processors, error) { + var ( + err error + processors database.Processors + ) + + if processors.Detectors, err = tx.findProcessors(searchLayerDetectors, id); err != nil { + return processors, handleError("searchLayerDetectors", err) } - layer.ProcessedBy.Listers, err = tx.findProcessors(searchLayerListers, "searchLayerListers", "lister", layerID) - if err != nil { - return layer, layerID, false, err + if processors.Listers, err = tx.findProcessors(searchLayerListers, id); err != nil { + return processors, handleError("searchLayerListers", err) } - return layer, layerID, true, nil + return processors, nil } diff --git a/database/pgsql/queries.go b/database/pgsql/queries.go index dc91f10b..0a99365d 100644 --- a/database/pgsql/queries.go +++ b/database/pgsql/queries.go @@ -218,17 +218,16 @@ const ( insertAncestry = `INSERT INTO ancestry (name) VALUES ($1) RETURNING id` searchAncestryLayer = ` - SELECT layer.hash + SELECT layer.hash, layer.id, ancestry_layer.ancestry_index FROM layer, ancestry_layer WHERE ancestry_layer.ancestry_id = $1 AND ancestry_layer.layer_id = layer.id ORDER BY ancestry_layer.ancestry_index ASC` searchAncestryFeatures = ` - SELECT namespace.name, namespace.version_format, feature.name, feature.version, ancestry_layer.ancestry_index - FROM namespace, feature, ancestry, namespaced_feature, ancestry_layer, ancestry_feature - WHERE ancestry.name = $1 - AND ancestry.id = ancestry_layer.ancestry_id + SELECT namespace.name, namespace.version_format, feature.name, feature.version, feature.version_format, ancestry_layer.ancestry_index + FROM namespace, feature, namespaced_feature, ancestry_layer, ancestry_feature + WHERE ancestry_layer.ancestry_id = $1 AND ancestry_feature.ancestry_layer_id = ancestry_layer.id AND ancestry_feature.namespaced_feature_id = namespaced_feature.id AND namespaced_feature.feature_id = feature.id diff --git a/worker.go b/worker.go index 66a7f7fc..3fd25ba8 100644 --- a/worker.go +++ b/worker.go @@ -388,7 +388,7 @@ func getNamespacedFeatures(layers []database.AncestryLayer) []database.Namespace func processAncestry(datastore database.Datastore, name string, layers []database.LayerWithContent, commonProcessors database.Processors) error { var ( - ancestry database.AncestryWithContent + ancestry database.Ancestry err error ) diff --git a/worker_test.go b/worker_test.go index e78430a6..13fccf9b 100644 --- a/worker_test.go +++ b/worker_test.go @@ -41,7 +41,7 @@ type mockDatastore struct { database.MockDatastore layers map[string]database.LayerWithContent - ancestry map[string]database.AncestryWithContent + ancestry map[string]database.Ancestry namespaces map[string]database.Namespace features map[string]database.Feature namespacedFeatures map[string]database.NamespacedFeature @@ -75,7 +75,7 @@ func copyDatastore(md *mockDatastore) mockDatastore { } } - ancestry := map[string]database.AncestryWithContent{} + ancestry := map[string]database.Ancestry{} for k, a := range md.ancestry { ancestryLayers := []database.AncestryLayer{} layers := []database.Layer{} @@ -101,14 +101,11 @@ func copyDatastore(md *mockDatastore) mockDatastore { }) } - ancestry[k] = database.AncestryWithContent{ - Ancestry: database.Ancestry{ - Name: a.Name, - Layers: layers, - ProcessedBy: database.Processors{ - Detectors: append([]string(nil), a.ProcessedBy.Detectors...), - Listers: append([]string(nil), a.ProcessedBy.Listers...), - }, + ancestry[k] = database.Ancestry{ + Name: a.Name, + ProcessedBy: database.Processors{ + Detectors: append([]string(nil), a.ProcessedBy.Detectors...), + Listers: append([]string(nil), a.ProcessedBy.Listers...), }, Layers: ancestryLayers, } @@ -141,7 +138,7 @@ func newMockDatastore() *mockDatastore { errSessionDone := errors.New("Session Done") md := &mockDatastore{ layers: make(map[string]database.LayerWithContent), - ancestry: make(map[string]database.AncestryWithContent), + ancestry: make(map[string]database.Ancestry), namespaces: make(map[string]database.Namespace), features: make(map[string]database.Feature), namespacedFeatures: make(map[string]database.NamespacedFeature), @@ -181,7 +178,7 @@ func newMockDatastore() *mockDatastore { return database.Ancestry{}, false, errSessionDone } ancestry, ok := session.copy.ancestry[name] - return ancestry.Ancestry, ok, nil + return ancestry, ok, nil } session.FctFindLayer = func(name string) (database.Layer, bool, error) { @@ -285,7 +282,7 @@ func newMockDatastore() *mockDatastore { return nil } - session.FctUpsertAncestry = func(ancestry database.AncestryWithContent) error { + session.FctUpsertAncestry = func(ancestry database.Ancestry) error { if session.terminated { return errSessionDone }