arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #741 from KeyboardNerd/parallel_download

Browse Source 
v3: Analyze layer content in parallel
This commit is contained in:
Sida Chen 2019-03-20 12:50:15 -04:00 committed by GitHub
commit 1234a8d2f0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 43 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
api/v3/rpc.go
View File

@ -15,7 +15,10 @@
package v3
import (
"sync"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@ -83,32 +86,47 @@ func (s *AncestryServer) PostAncestry(ctx context.Context, req *pb.PostAncestryR
}
builder := clair.NewAncestryBuilder(clair.EnabledDetectors())
layerMap := map[string]*database.Layer{}
layerMapLock := sync.RWMutex{}
g, analyzerCtx := errgroup.WithContext(ctx)
for i := range req.Layers {
layer := req.Layers[i]
if _, ok := layerMap[layer.Hash]; !ok {
layerMap[layer.Hash] = nil
if layer == nil {
err := status.Error(codes.InvalidArgument, "ancestry layer is invalid")
return nil, err
}
if layer.GetHash() == "" {
return nil, status.Error(codes.InvalidArgument, "ancestry layer hash should not be empty")
}
if layer.GetPath() == "" {
return nil, status.Error(codes.InvalidArgument, "ancestry layer path should not be empty")
}
g.Go(func() error {
clairLayer, err := clair.AnalyzeLayer(analyzerCtx, s.Store, layer.Hash, req.Format, layer.Path, layer.Headers)
if err != nil {
return err
}
layerMapLock.Lock()
layerMap[layer.Hash] = clairLayer
layerMapLock.Unlock()
return nil
})
}
}
if err = g.Wait(); err != nil {
return nil, newRPCErrorWithClairError(codes.Internal, err)
}
for _, layer := range req.Layers {
if layer == nil {
err := status.Error(codes.InvalidArgument, "ancestry layer is invalid")
return nil, err
}
if layer.GetHash() == "" {
return nil, status.Error(codes.InvalidArgument, "ancestry layer hash should not be empty")
}
if layer.GetPath() == "" {
return nil, status.Error(codes.InvalidArgument, "ancestry layer path should not be empty")
}
// TODO(sidac): make AnalyzeLayer to be async to ensure
// non-blocking downloads.
// We'll need to deal with two layers post by the same or different
// requests that may have the same hash. In that case, since every
// layer/feature/namespace is unique in the database, it may introduce
// deadlock.
clairLayer, err := clair.AnalyzeLayer(ctx, s.Store, layer.Hash, req.Format, layer.Path, layer.Headers)
if err != nil {
return nil, newRPCErrorWithClairError(codes.Internal, err)
}
builder.AddLeafLayer(clairLayer)
builder.AddLeafLayer(layerMap[layer.Hash])
}
if err := clair.SaveAncestry(s.Store, builder.Ancestry(req.AncestryName)); err != nil {