clair: Use builder pattern for constructing ancestry

- Add Ancestry builder
- Change RPC to use the ancestry builder
master
Sida Chen 5 years ago
parent 891ce1697d
commit 5b2376498b

@ -0,0 +1,145 @@
// Copyright 2019 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clair
import (
"context"
log "github.com/sirupsen/logrus"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/ext/featurens"
"github.com/coreos/clair/ext/imagefmt"
)
// AnalyzeError represents an failure when analyzing layer or constructing
// ancestry.
type AnalyzeError string
func (e AnalyzeError) Error() string {
return string(e)
}
var (
// StorageError represents an analyze error caused by the storage
StorageError = AnalyzeError("failed to query the database.")
// RetrieveBlobError represents an analyze error caused by failure of
// downloading or extracting layer blobs.
RetrieveBlobError = AnalyzeError("failed to download layer blob.")
// ExtractBlobError represents an analyzer error caused by failure of
// extracting a layer blob by imagefmt.
ExtractBlobError = AnalyzeError("failed to extract files from layer blob.")
// FeatureDetectorError is an error caused by failure of feature listing by
// featurefmt.
FeatureDetectorError = AnalyzeError("failed to scan feature from layer blob files.")
// NamespaceDetectorError is an error caused by failure of namespace
// detection by featurens.
NamespaceDetectorError = AnalyzeError("failed to scan namespace from layer blob files.")
)
// AnalyzeLayer retrieves the clair layer with all extracted features and namespaces.
// If a layer is already scanned by all enabled detectors in the Clair instance, it returns directly.
// Otherwise, it re-download the layer blob and scan the features and namespaced again.
func AnalyzeLayer(ctx context.Context, store database.Datastore, blobSha256 string, blobFormat string, downloadURI string, downloadHeaders map[string]string) (*database.Layer, error) {
layer, found, err := database.FindLayerAndRollback(store, blobSha256)
logFields := log.Fields{"layer.Hash": blobSha256}
if err != nil {
log.WithError(err).WithFields(logFields).Error("failed to find layer in the storage")
return nil, StorageError
}
var scannedBy []database.Detector
if found {
scannedBy = layer.By
}
// layer will be scanned by detectors not scanned the layer already.
toScan := database.DiffDetectors(EnabledDetectors(), scannedBy)
if len(toScan) != 0 {
log.WithFields(logFields).Debug("scan layer blob not already scanned")
newLayerScanResult := &database.Layer{Hash: blobSha256, By: toScan}
blob, err := retrieveLayerBlob(ctx, downloadURI, downloadHeaders)
if err != nil {
log.WithError(err).WithFields(logFields).Error("failed to retrieve layer blob")
return nil, RetrieveBlobError
}
defer func() {
if err := blob.Close(); err != nil {
log.WithFields(logFields).Error("failed to close layer blob reader")
}
}()
files := append(featurefmt.RequiredFilenames(toScan), featurens.RequiredFilenames(toScan)...)
fileMap, err := imagefmt.Extract(blobFormat, blob, files)
if err != nil {
log.WithFields(logFields).WithError(err).Error("failed to extract layer blob")
return nil, ExtractBlobError
}
newLayerScanResult.Features, err = featurefmt.ListFeatures(fileMap, toScan)
if err != nil {
log.WithFields(logFields).WithError(err).Error("failed to detect features")
return nil, FeatureDetectorError
}
newLayerScanResult.Namespaces, err = featurens.Detect(fileMap, toScan)
if err != nil {
log.WithFields(logFields).WithError(err).Error("failed to detect namespaces")
return nil, NamespaceDetectorError
}
if err = saveLayerChange(store, newLayerScanResult); err != nil {
log.WithFields(logFields).WithError(err).Error("failed to store layer change")
return nil, StorageError
}
layer = database.MergeLayers(layer, newLayerScanResult)
} else {
log.WithFields(logFields).Debug("found scanned layer blob")
}
return layer, nil
}
// EnabledDetectors retrieves a list of all detectors installed in the Clair
// instance.
func EnabledDetectors() []database.Detector {
return append(featurefmt.ListListers(), featurens.ListDetectors()...)
}
// RegisterConfiguredDetectors populates the database with registered detectors.
func RegisterConfiguredDetectors(store database.Datastore) {
if err := database.PersistDetectorsAndCommit(store, EnabledDetectors()); err != nil {
panic("failed to initialize Clair analyzer")
}
}
func saveLayerChange(store database.Datastore, layer *database.Layer) error {
if err := database.PersistFeaturesAndCommit(store, layer.GetFeatures()); err != nil {
return err
}
if err := database.PersistNamespacesAndCommit(store, layer.GetNamespaces()); err != nil {
return err
}
if err := database.PersistPartialLayerAndCommit(store, layer); err != nil {
return err
}
return nil
}

@ -0,0 +1,290 @@
// Copyright 2019 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clair
import (
"crypto/sha256"
"encoding/hex"
"strings"
log "github.com/sirupsen/logrus"
"github.com/coreos/clair/database"
)
type layerIndexedFeature struct {
feature *database.LayerFeature
namespace *layerIndexedNamespace
introducedIn int
}
type layerIndexedNamespace struct {
namespace database.LayerNamespace
introducedIn int
}
// AncestryBuilder builds an Ancestry, which contains an ordered list of layers
// and their features.
type AncestryBuilder struct {
layerIndex int
layerNames []string
detectors []database.Detector
namespaces map[database.Detector]*layerIndexedNamespace
features map[database.Detector][]layerIndexedFeature
}
// NewAncestryBuilder creates a new ancestry builder.
//
// ancestry builder takes in the extracted layer information and produce a set of
// namespaces, features, and the relation between features for the whole image.
func NewAncestryBuilder(detectors []database.Detector) *AncestryBuilder {
return &AncestryBuilder{
layerIndex: 0,
detectors: detectors,
namespaces: make(map[database.Detector]*layerIndexedNamespace),
features: make(map[database.Detector][]layerIndexedFeature),
}
}
// AddLeafLayer adds a leaf layer to the ancestry builder, and computes the
// namespaced features.
func (b *AncestryBuilder) AddLeafLayer(layer *database.Layer) {
b.layerNames = append(b.layerNames, layer.Hash)
for i := range layer.Namespaces {
b.updateNamespace(&layer.Namespaces[i])
}
allFeatureMap := map[database.Detector][]database.LayerFeature{}
for i := range layer.Features {
layerFeature := layer.Features[i]
allFeatureMap[layerFeature.By] = append(allFeatureMap[layerFeature.By], layerFeature)
}
// we only care about the ones specified by builder's detectors
featureMap := map[database.Detector][]database.LayerFeature{}
for i := range b.detectors {
detector := b.detectors[i]
featureMap[detector] = allFeatureMap[detector]
}
for detector := range featureMap {
b.addLayerFeatures(detector, featureMap[detector])
}
b.layerIndex++
}
// Every detector inspects a set of files for the features
// therefore, if that set of files gives a different set of features, it
// should replace the existing features.
func (b *AncestryBuilder) addLayerFeatures(detector database.Detector, features []database.LayerFeature) {
if len(features) == 0 {
// TODO(sidac): we need to differentiate if the detector finds that all
// features are removed ( a file change ), or the package installer is
// removed ( a file deletion ), or there's no change in the file ( file
// does not exist in the blob ) Right now, we're just assuming that no
// change in the file because that's the most common case.
return
}
existingFeatures := b.features[detector]
currentFeatures := make([]layerIndexedFeature, 0, len(features))
// Features that are not in the current layer should be removed.
for i := range existingFeatures {
feature := existingFeatures[i]
for j := range features {
if features[j] == *feature.feature {
currentFeatures = append(currentFeatures, feature)
break
}
}
}
// Features that newly introduced in the current layer should be added.
for i := range features {
found := false
for j := range existingFeatures {
if *existingFeatures[j].feature == features[i] {
found = true
break
}
}
if !found {
namespace, found := b.lookupNamespace(&features[i])
if !found {
log.WithField("Layer Hashes", b.layerNames).Error("skip, could not find the proper namespace for feature")
continue
}
currentFeatures = append(currentFeatures, b.createLayerIndexedFeature(namespace, &features[i]))
}
}
b.features[detector] = currentFeatures
}
// updateNamespace update the namespaces for the ancestry. It does the following things:
// 1. when a detector detects a new namespace, it's added to the ancestry.
// 2. when a detector detects a difference in the detected namespace, it
// replaces the namespace, and also move all features under that namespace to
// the new namespace.
func (b *AncestryBuilder) updateNamespace(layerNamespace *database.LayerNamespace) {
var (
previous *layerIndexedNamespace
ok bool
)
if previous, ok = b.namespaces[layerNamespace.By]; !ok {
b.namespaces[layerNamespace.By] = &layerIndexedNamespace{
namespace: *layerNamespace,
introducedIn: b.layerIndex,
}
return
}
// All features referencing to this namespace are now pointing to the new namespace.
// Also those features are now treated as introduced in the same layer as
// when this new namespace is introduced.
previous.namespace = *layerNamespace
previous.introducedIn = b.layerIndex
for _, features := range b.features {
for i, feature := range features {
if feature.namespace == previous {
features[i].introducedIn = previous.introducedIn
}
}
}
}
func (b *AncestryBuilder) createLayerIndexedFeature(namespace *layerIndexedNamespace, feature *database.LayerFeature) layerIndexedFeature {
return layerIndexedFeature{
feature: feature,
namespace: namespace,
introducedIn: b.layerIndex,
}
}
func (b *AncestryBuilder) lookupNamespace(feature *database.LayerFeature) (*layerIndexedNamespace, bool) {
for _, namespace := range b.namespaces {
if namespace.namespace.VersionFormat == feature.VersionFormat {
return namespace, true
}
}
return nil, false
}
func (b *AncestryBuilder) ancestryFeatures(index int) []database.AncestryFeature {
ancestryFeatures := []database.AncestryFeature{}
for detector, features := range b.features {
for _, feature := range features {
if feature.introducedIn == index {
ancestryFeatures = append(ancestryFeatures, database.AncestryFeature{
NamespacedFeature: database.NamespacedFeature{
Feature: feature.feature.Feature,
Namespace: feature.namespace.namespace.Namespace,
},
FeatureBy: detector,
NamespaceBy: feature.namespace.namespace.By,
})
}
}
}
return ancestryFeatures
}
func (b *AncestryBuilder) ancestryLayers() []database.AncestryLayer {
layers := make([]database.AncestryLayer, 0, b.layerIndex)
for i := 0; i < b.layerIndex; i++ {
layers = append(layers, database.AncestryLayer{
Hash: b.layerNames[i],
Features: b.ancestryFeatures(i),
})
}
return layers
}
// Ancestry produces an Ancestry from the builder.
func (b *AncestryBuilder) Ancestry(name string) *database.Ancestry {
if name == "" {
// TODO(sidac): we'll use the computed ancestry name in the future.
// During the transition, it still requires the user to use the correct
// ancestry name.
name = ancestryName(b.layerNames)
log.WithField("ancestry.Name", name).Warn("generated ancestry name since it's not specified")
}
return &database.Ancestry{
Name: name,
By: b.detectors,
Layers: b.ancestryLayers(),
}
}
// SaveAncestry saves an ancestry to the datastore.
func SaveAncestry(store database.Datastore, ancestry *database.Ancestry) error {
log.WithField("ancestry.Name", ancestry.Name).Debug("saving ancestry")
features := []database.NamespacedFeature{}
for _, layer := range ancestry.Layers {
features = append(features, layer.GetFeatures()...)
}
if err := database.PersistNamespacedFeaturesAndCommit(store, features); err != nil {
return StorageError
}
if err := database.UpsertAncestryAndCommit(store, ancestry); err != nil {
return StorageError
}
if err := database.CacheRelatedVulnerabilityAndCommit(store, features); err != nil {
return StorageError
}
return nil
}
// IsAncestryCached checks if the ancestry is already cached in the database with the current set of detectors.
func IsAncestryCached(store database.Datastore, name string, layerHashes []string) (bool, error) {
if name == "" {
// TODO(sidac): we'll use the computed ancestry name in the future.
// During the transition, it still requires the user to use the correct
// ancestry name.
name = ancestryName(layerHashes)
log.WithField("ancestry.Name", name).Warn("generated ancestry name since it's not specified")
}
ancestry, found, err := database.FindAncestryAndRollback(store, name)
if err != nil {
log.WithError(err).WithField("ancestry.Name", name).Error("failed to query ancestry in database")
return false, StorageError
}
if found {
log.WithField("ancestry.Name", name).Debug("found cached ancestry")
}
return found && len(database.DiffDetectors(EnabledDetectors(), ancestry.By)) == 0, nil
}
func ancestryName(layerHashes []string) string {
tag := sha256.Sum256([]byte(strings.Join(layerHashes, ",")))
return hex.EncodeToString(tag[:])
}

@ -0,0 +1,267 @@
// Copyright 2019 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clair
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/coreos/clair/database"
)
var (
dpkg = database.NewFeatureDetector("dpkg", "1.0")
rpm = database.NewFeatureDetector("rpm", "1.0")
pip = database.NewFeatureDetector("pip", "1.0")
python = database.NewNamespaceDetector("python", "1.0")
osrelease = database.NewNamespaceDetector("os-release", "1.0")
ubuntu = *database.NewNamespace("ubuntu:14.04", "dpkg")
ubuntu16 = *database.NewNamespace("ubuntu:16.04", "dpkg")
python2 = *database.NewNamespace("python:2", "pip")
sed = *database.NewSourcePackage("sed", "4.4-2", "dpkg")
sedBin = *database.NewBinaryPackage("sed", "4.4-2", "dpkg")
tar = *database.NewBinaryPackage("tar", "1.29b-2", "dpkg")
scipy = *database.NewSourcePackage("scipy", "3.0.0", "pip")
detectors = []database.Detector{dpkg, osrelease, rpm}
multinamespaceDetectors = []database.Detector{dpkg, osrelease, pip}
)
// layerBuilder is for helping constructing the layer test artifacts.
type layerBuilder struct {
layer *database.Layer
}
func newLayerBuilder(hash string) *layerBuilder {
return &layerBuilder{&database.Layer{Hash: hash, By: detectors}}
}
func (b *layerBuilder) addNamespace(detector database.Detector, ns database.Namespace) *layerBuilder {
b.layer.Namespaces = append(b.layer.Namespaces, database.LayerNamespace{
Namespace: ns,
By: detector,
})
return b
}
func (b *layerBuilder) addFeature(detector database.Detector, f database.Feature) *layerBuilder {
b.layer.Features = append(b.layer.Features, database.LayerFeature{
Feature: f,
By: detector,
})
return b
}
var testImage = []*database.Layer{
// empty layer
newLayerBuilder("0").layer,
// ubuntu namespace
newLayerBuilder("1").addNamespace(osrelease, ubuntu).layer,
// install sed
newLayerBuilder("2").addFeature(dpkg, sed).layer,
// install tar
newLayerBuilder("3").addFeature(dpkg, sed).addFeature(dpkg, tar).layer,
// remove tar
newLayerBuilder("4").addFeature(dpkg, sed).layer,
// upgrade ubuntu
newLayerBuilder("5").addNamespace(osrelease, ubuntu16).layer,
// no change to the detectable files
newLayerBuilder("6").layer,
// change to the package installer database but no features are affected.
newLayerBuilder("7").addFeature(dpkg, sed).layer,
}
var clairLimit = []*database.Layer{
// TODO(sidac): how about install rpm package under ubuntu?
newLayerBuilder("1").addNamespace(osrelease, ubuntu).layer,
newLayerBuilder("2").addFeature(rpm, sed).layer,
}
var multipleNamespace = []*database.Layer{
// TODO(sidac): support for multiple namespaces
}
var invalidNamespace = []*database.Layer{
// add package without namespace, this indicates that the namespace detector
// could not detect the namespace.
newLayerBuilder("0").addFeature(dpkg, sed).layer,
}
var multiplePackagesOnFirstLayer = []*database.Layer{
newLayerBuilder("0").addFeature(dpkg, sed).addFeature(dpkg, tar).addFeature(dpkg, sedBin).addNamespace(osrelease, ubuntu16).layer,
}
func TestAddLayer(t *testing.T) {
cases := []struct {
title string
image []*database.Layer
expectedAncestry database.Ancestry
}{
{
title: "empty image",
expectedAncestry: database.Ancestry{Name: ancestryName([]string{}), By: detectors},
},
{
title: "empty layer",
image: testImage[:1],
expectedAncestry: database.Ancestry{Name: ancestryName([]string{"0"}), By: detectors, Layers: []database.AncestryLayer{{Hash: "0"}}},
},
{
title: "ubuntu",
image: testImage[:2],
expectedAncestry: database.Ancestry{
Name: ancestryName([]string{"0", "1"}),
By: detectors,
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}},
},
},
{
title: "ubuntu install sed",
image: testImage[:3],
expectedAncestry: database.Ancestry{
Name: ancestryName([]string{"0", "1", "2"}),
By: detectors,
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2", Features: []database.AncestryFeature{
{
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu},
FeatureBy: dpkg,
NamespaceBy: osrelease,
},
}}},
},
},
{
title: "ubuntu install tar",
image: testImage[:4],
expectedAncestry: database.Ancestry{
Name: ancestryName([]string{"0", "1", "2", "3"}),
By: detectors,
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2", Features: []database.AncestryFeature{
{
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu},
FeatureBy: dpkg,
NamespaceBy: osrelease,
},
}}, {
Hash: "3", Features: []database.AncestryFeature{
{
NamespacedFeature: database.NamespacedFeature{Feature: tar, Namespace: ubuntu},
FeatureBy: dpkg,
NamespaceBy: osrelease,
},
},
}},
},
}, {
title: "ubuntu uninstall tar",
image: testImage[:5],
expectedAncestry: database.Ancestry{
Name: ancestryName([]string{"0", "1", "2", "3", "4"}),
By: detectors,
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2", Features: []database.AncestryFeature{
{
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu},
FeatureBy: dpkg,
NamespaceBy: osrelease,
},
}}, {Hash: "3"}, {Hash: "4"}},
},
}, {
title: "ubuntu upgrade",
image: testImage[:6],
expectedAncestry: database.Ancestry{
Name: ancestryName([]string{"0", "1", "2", "3", "4", "5"}),
By: detectors,
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2"}, {Hash: "3"}, {Hash: "4"}, {Hash: "5", Features: []database.AncestryFeature{
{
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
FeatureBy: dpkg,
NamespaceBy: osrelease,
}}},
},
},
}, {
title: "no change to the detectable files",
image: testImage[:7],
expectedAncestry: database.Ancestry{
Name: ancestryName([]string{"0", "1", "2", "3", "4", "5", "6"}),
By: detectors,
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2"}, {Hash: "3"}, {Hash: "4"}, {Hash: "5", Features: []database.AncestryFeature{
{
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
FeatureBy: dpkg,
NamespaceBy: osrelease,
}}}, {Hash: "6"}},
},
}, {
title: "change to the package installer database but no features are affected.",
image: testImage[:8],
expectedAncestry: database.Ancestry{
Name: ancestryName([]string{"0", "1", "2", "3", "4", "5", "6", "7"}),
By: detectors,
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2"}, {Hash: "3"}, {Hash: "4"}, {Hash: "5", Features: []database.AncestryFeature{
{
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
FeatureBy: dpkg,
NamespaceBy: osrelease,
}}}, {Hash: "6"}, {Hash: "7"}},
},
}, {
title: "layers with features and namespace.",
image: multiplePackagesOnFirstLayer,
expectedAncestry: database.Ancestry{
Name: ancestryName([]string{"0"}),
By: detectors,
Layers: []database.AncestryLayer{
{
Hash: "0",
Features: []database.AncestryFeature{
{
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
FeatureBy: dpkg,
NamespaceBy: osrelease,
},
{
NamespacedFeature: database.NamespacedFeature{Feature: sedBin, Namespace: ubuntu16},
FeatureBy: dpkg,
NamespaceBy: osrelease,
},
{
NamespacedFeature: database.NamespacedFeature{Feature: tar, Namespace: ubuntu16},
FeatureBy: dpkg,
NamespaceBy: osrelease,
},
},
},
},
},
},
}
for _, test := range cases {
t.Run(test.title, func(t *testing.T) {
builder := NewAncestryBuilder(detectors)
for _, layer := range test.image {
builder.AddLeafLayer(layer)
}
ancestry := builder.Ancestry("")
require.True(t, database.AssertAncestryEqual(t, &test.expectedAncestry, ancestry))
})
}
}

@ -17,6 +17,8 @@ package v3
import (
"fmt"
"github.com/coreos/clair/ext/imagefmt"
"golang.org/x/net/context"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@ -28,6 +30,10 @@ import (
"github.com/coreos/clair/pkg/pagination"
)
func newRPCErrorWithClairError(code codes.Code, err error) error {
return status.Errorf(code, "clair error reason: '%s'", err.Error())
}
// NotificationServer implements NotificationService interface for serving RPC.
type NotificationServer struct {
Store database.Datastore
@ -55,23 +61,34 @@ func (s *StatusServer) GetStatus(ctx context.Context, req *pb.GetStatusRequest)
// PostAncestry implements posting an ancestry via the Clair gRPC service.
func (s *AncestryServer) PostAncestry(ctx context.Context, req *pb.PostAncestryRequest) (*pb.PostAncestryResponse, error) {
ancestryName := req.GetAncestryName()
if ancestryName == "" {
return nil, status.Error(codes.InvalidArgument, "ancestry name should not be empty")
// validate request
blobFormat := req.GetFormat()
if !imagefmt.IsSupported(blobFormat) {
return nil, status.Error(codes.InvalidArgument, "image blob format is not supported")
}
clairStatus, err := GetClairStatus(s.Store)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
// check if the ancestry is already processed, if not we build the ancestry again.
layerHashes := make([]string, len(req.Layers))
for i, layer := range req.Layers {
layerHashes[i] = layer.GetHash()
}
layers := req.GetLayers()
if len(layers) == 0 {
return nil, status.Error(codes.InvalidArgument, "ancestry should have at least one layer")
found, err := clair.IsAncestryCached(s.Store, req.AncestryName, layerHashes)
if err != nil {
return nil, newRPCErrorWithClairError(codes.Internal, err)
}
ancestryFormat := req.GetFormat()
if ancestryFormat == "" {
return nil, status.Error(codes.InvalidArgument, "ancestry format should not be empty")
if found {
return &pb.PostAncestryResponse{Status: clairStatus}, nil
}
ancestryLayers := []clair.LayerRequest{}
for _, layer := range layers {
builder := clair.NewAncestryBuilder(clair.EnabledDetectors())
for _, layer := range req.Layers {
if layer == nil {
err := status.Error(codes.InvalidArgument, "ancestry layer is invalid")
return nil, err
@ -85,21 +102,22 @@ func (s *AncestryServer) PostAncestry(ctx context.Context, req *pb.PostAncestryR
return nil, status.Error(codes.InvalidArgument, "ancestry layer path should not be empty")
}
ancestryLayers = append(ancestryLayers, clair.LayerRequest{
Hash: layer.Hash,
Headers: layer.Headers,
Path: layer.Path,
})
}
// TODO(sidac): make AnalyzeLayer to be async to ensure
// non-blocking downloads.
// We'll need to deal with two layers post by the same or different
// requests that may have the same hash. In that case, since every
// layer/feature/namespace is unique in the database, it may introduce
// deadlock.
clairLayer, err := clair.AnalyzeLayer(ctx, s.Store, layer.Hash, req.Format, layer.Path, layer.Headers)
if err != nil {
return nil, newRPCErrorWithClairError(codes.Internal, err)
}
err := clair.ProcessAncestry(s.Store, ancestryFormat, ancestryName, ancestryLayers)
if err != nil {
return nil, status.Error(codes.Internal, "ancestry is failed to be processed: "+err.Error())
builder.AddLeafLayer(clairLayer)
}
clairStatus, err := GetClairStatus(s.Store)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
if err := clair.SaveAncestry(s.Store, builder.Ancestry(req.AncestryName)); err != nil {
return nil, newRPCErrorWithClairError(codes.Internal, err)
}
return &pb.PostAncestryResponse{Status: clairStatus}, nil

@ -13,7 +13,7 @@ import (
// protobuf struct.
func GetClairStatus(store database.Datastore) (*pb.ClairStatus, error) {
status := &pb.ClairStatus{
Detectors: pb.DetectorsFromDatabaseModel(clair.EnabledDetectors),
Detectors: pb.DetectorsFromDatabaseModel(clair.EnabledDetectors()),
}
t, firstUpdate, err := clair.GetLastUpdateTime(store)

@ -1,61 +1,43 @@
// Copyright 2019 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clair
import (
"context"
"crypto/tls"
"io"
"net/http"
"os"
"strings"
log "github.com/sirupsen/logrus"
"github.com/coreos/clair/pkg/httputil"
)
func retrieveLayerBlob(ctx context.Context, blobSha256 string, path string, headers map[string]string) (io.ReadCloser, error) {
func retrieveLayerBlob(ctx context.Context, path string, headers map[string]string) (io.ReadCloser, error) {
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
return downloadLayerBlob(ctx, blobSha256, path, headers)
}
return loadLayerBlobFromFS(blobSha256)
}
func downloadLayerBlob(ctx context.Context, blobSha256 string, uri string, headers map[string]string) (io.ReadCloser, error) {
request, err := http.NewRequest("GET", uri, nil)
if err != nil {
return nil, RetrieveBlobError
}
if headers != nil {
for k, v := range headers {
request.Header.Set(k, v)
httpHeaders := make(http.Header)
for key, value := range headers {
httpHeaders[key] = []string{value}
}
}
tr := &http.Transport{
TLSClientConfig: &tls.Config{},
Proxy: http.ProxyFromEnvironment,
}
client := &http.Client{Transport: tr}
r, err := client.Do(request)
if err != nil {
log.WithError(err).Error("could not download layer")
return nil, RetrieveBlobError
}
reader, err := httputil.GetWithContext(ctx, path, httpHeaders)
if err != nil {
return nil, err
}
// Fail if we don't receive a 2xx HTTP status code.
if is2xx(r.StatusCode) {
log.WithField("status", r.StatusCode).Error("could not download layer: expected 2XX")
return nil, RetrieveBlobError
return reader, nil
}
return r.Body, nil
}
func is2xx(statusCode int) bool {
return statusCode >= 200 && statusCode < 300
}
func loadLayerBlobFromFS(path string) (io.ReadCloser, error) {
return os.Open(path)
}

@ -30,9 +30,6 @@ import (
"github.com/coreos/clair"
"github.com/coreos/clair/api"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/ext/featurens"
"github.com/coreos/clair/ext/imagefmt"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/pkg/formatter"
"github.com/coreos/clair/pkg/stopper"
@ -103,11 +100,10 @@ func stopCPUProfiling(f *os.File) {
}
func configClairVersion(config *Config) {
clair.EnabledDetectors = append(featurefmt.ListListers(), featurens.ListDetectors()...)
clair.EnabledUpdaters = strutil.Intersect(config.Updater.EnabledUpdaters, vulnsrc.ListUpdaters())
log.WithFields(log.Fields{
"Detectors": database.SerializeDetectors(clair.EnabledDetectors),
"Detectors": database.SerializeDetectors(clair.EnabledDetectors()),
"Updaters": clair.EnabledUpdaters,
}).Info("enabled Clair extensions")
}
@ -134,7 +130,8 @@ func Boot(config *Config) {
defer db.Close()
clair.InitWorker(db)
clair.RegisterConfiguredDetectors(db)
// Start notifier
st.Begin()
go clair.RunNotifier(config.Notifier, db, st)
@ -173,7 +170,6 @@ func main() {
flagConfigPath := flag.String("config", "/etc/clair/config.yaml", "Load configuration from the specified file.")
flagCPUProfilePath := flag.String("cpu-profile", "", "Write a CPU profile to the specified file before exiting.")
flagLogLevel := flag.String("log-level", "info", "Define the logging level.")
flagInsecureTLS := flag.Bool("insecure-tls", false, "Disable TLS server's certificate chain and hostname verification when pulling layers.")
flag.Parse()
configureLogger(flagLogLevel)
@ -195,12 +191,6 @@ func main() {
defer stopCPUProfiling(startCPUProfiling(*flagCPUProfilePath))
}
// Enable TLS server's certificate chain and hostname verification
// when pulling layers if specified
if *flagInsecureTLS {
imagefmt.SetInsecureTLS(*flagInsecureTLS)
}
// configure updater and worker
configClairVersion(config)

@ -15,8 +15,11 @@
package database
import (
"encoding/json"
"time"
log "github.com/sirupsen/logrus"
"github.com/deckarep/golang-set"
)
@ -94,8 +97,11 @@ func PersistFeaturesAndCommit(datastore Datastore, features []Feature) error {
defer tx.Rollback()
if err := tx.PersistFeatures(features); err != nil {
serialized, _ := json.Marshal(features)
log.WithError(err).WithField("feature", string(serialized)).Error("failed to store features")
return err
}
return tx.Commit()
}
@ -129,14 +135,18 @@ func FindAncestryAndRollback(datastore Datastore, name string) (Ancestry, bool,
}
// FindLayerAndRollback wraps session FindLayer function with begin and rollback.
func FindLayerAndRollback(datastore Datastore, hash string) (layer Layer, ok bool, err error) {
func FindLayerAndRollback(datastore Datastore, hash string) (layer *Layer, ok bool, err error) {
var tx Session
if tx, err = datastore.Begin(); err != nil {
return
}
defer tx.Rollback()
layer, ok, err = tx.FindLayer(hash)
// TODO(sidac): In order to make the session interface more idiomatic, we'll
// return the pointer value in the future.
var dereferencedLayer Layer
dereferencedLayer, ok, err = tx.FindLayer(hash)
layer = &dereferencedLayer
return
}
@ -168,13 +178,17 @@ func GetAncestryFeatures(ancestry Ancestry) []NamespacedFeature {
}
// UpsertAncestryAndCommit wraps session UpsertAncestry function with begin and commit.
func UpsertAncestryAndCommit(datastore Datastore, ancestry Ancestry) error {
func UpsertAncestryAndCommit(datastore Datastore, ancestry *Ancestry) error {
tx, err := datastore.Begin()
if err != nil {
return err
}
if err = tx.UpsertAncestry(ancestry); err != nil {
if err = tx.UpsertAncestry(*ancestry); err != nil {
log.WithError(err).Error("failed to upsert the ancestry")
serialized, _ := json.Marshal(ancestry)
log.Debug(string(serialized))
tx.Rollback()
return err
}
@ -350,3 +364,22 @@ func ReleaseLock(datastore Datastore, name, owner string) {
return
}
}
// PersistDetectorsAndCommit stores the detectors in the data store.
func PersistDetectorsAndCommit(store Datastore, detectors []Detector) error {
tx, err := store.Begin()
if err != nil {
return err
}
defer tx.Rollback()
if err := tx.PersistDetectors(detectors); err != nil {
return err
}
if err := tx.Commit(); err != nil {
return err
}
return nil
}

@ -93,12 +93,12 @@ func (s DetectorType) Valid() bool {
type Detector struct {
// Name of an extension should be non-empty and uniquely identifies the
// extension.
Name string
Name string `json:"name"`
// Version of an extension should be non-empty.
Version string
Version string `json:"version"`
// DType is the type of the extension and should be one of the types in
// DetectorTypes.
DType DetectorType
DType DetectorType `json:"dtype"`
}
// Valid checks if all fields in the detector satisfies the spec.

@ -26,13 +26,13 @@ import (
type Ancestry struct {
// Name is a globally unique value for a set of layers. This is often the
// sha256 digest of an OCI/Docker manifest.
Name string
Name string `json:"name"`
// By contains the processors that are used when computing the
// content of this ancestry.
By []Detector
By []Detector `json:"by"`
// Layers should be ordered and i_th layer is the parent of i+1_th layer in
// the slice.
Layers []AncestryLayer
Layers []AncestryLayer `json:"layers"`
}
// Valid checks if the ancestry is compliant to spec.
@ -63,10 +63,10 @@ func (a *Ancestry) Valid() bool {
// AncestryLayer is a layer with all detected namespaced features.
type AncestryLayer struct {
// Hash is the sha-256 tarsum on the layer's blob content.
Hash string
Hash string `json:"hash"`
// Features are the features introduced by this layer when it was
// processed.
Features []AncestryFeature
Features []AncestryFeature `json:"features"`
}
// Valid checks if the Ancestry Layer is compliant to the spec.
@ -95,22 +95,22 @@ func (l *AncestryLayer) GetFeatures() []NamespacedFeature {
// AncestryFeature is a namespaced feature with the detectors used to
// find this feature.
type AncestryFeature struct {
NamespacedFeature
NamespacedFeature `json:"namespacedFeature"`
// FeatureBy is the detector that detected the feature.
FeatureBy Detector
FeatureBy Detector `json:"featureBy"`
// NamespaceBy is the detector that detected the namespace.
NamespaceBy Detector
NamespaceBy Detector `json:"namespaceBy"`
}
// Layer is a layer with all the detected features and namespaces.
type Layer struct {
// Hash is the sha-256 tarsum on the layer's blob content.
Hash string
Hash string `json:"hash"`
// By contains a list of detectors scanned this Layer.
By []Detector
Namespaces []LayerNamespace
Features []LayerFeature
By []Detector `json:"by"`
Namespaces []LayerNamespace `json:"namespaces"`
Features []LayerFeature `json:"features"`
}
func (l *Layer) GetFeatures() []Feature {
@ -133,26 +133,26 @@ func (l *Layer) GetNamespaces() []Namespace {
// LayerNamespace is a namespace with detection information.
type LayerNamespace struct {
Namespace
Namespace `json:"namespace"`
// By is the detector found the namespace.
By Detector
By Detector `json:"by"`
}
// LayerFeature is a feature with detection information.
type LayerFeature struct {
Feature
Feature `json:"feature"`
// By is the detector found the feature.
By Detector
By Detector `json:"by"`
}
// Namespace is the contextual information around features.
//
// e.g. Debian:7, NodeJS.
type Namespace struct {
Name string
VersionFormat string
Name string `json:"name"`
VersionFormat string `json:"versionFormat"`
}
func NewNamespace(name string, versionFormat string) *Namespace {
@ -166,10 +166,10 @@ func NewNamespace(name string, versionFormat string) *Namespace {
// dpkg is the version format of the installer package manager, which in this
// case could be dpkg or apk.
type Feature struct {
Name string
Version string
VersionFormat string
Type FeatureType
Name string `json:"name"`
Version string `json:"version"`
VersionFormat string `json:"versionFormat"`
Type FeatureType `json:"type"`
}
func NewFeature(name string, version string, versionFormat string, featureType FeatureType) *Feature {
@ -189,9 +189,9 @@ func NewSourcePackage(name string, version string, versionFormat string) *Featur
//
// e.g. OpenSSL 1.0 dpkg Debian:7.
type NamespacedFeature struct {
Feature
Feature `json:"feature"`
Namespace Namespace
Namespace Namespace `json:"namespace"`
}
func NewNamespacedFeature(namespace *Namespace, feature *Feature) *NamespacedFeature {

@ -16,6 +16,10 @@
package httputil
import (
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
"strings"
@ -62,6 +66,38 @@ func GetClientAddr(r *http.Request) string {
return addr
}
// GetWithContext do HTTP GET to the URI with headers and returns response blob
// reader.
func GetWithContext(ctx context.Context, uri string, headers http.Header) (io.ReadCloser, error) {
request, err := http.NewRequest("GET", uri, nil)
if err != nil {
return nil, err
}
if headers != nil {
request.Header = headers
}
tr := &http.Transport{
TLSClientConfig: &tls.Config{},
Proxy: http.ProxyFromEnvironment,
}
client := &http.Client{Transport: tr}
request = request.WithContext(ctx)
r, err := client.Do(request)
if err != nil {
return nil, err
}
// Fail if we don't receive a 2xx HTTP status code.
if !Status2xx(r) {
return nil, fmt.Errorf("failed HTTP GET: expected 2XX, got %d", r.StatusCode)
}
return r.Body, nil
}
// Status2xx returns true if the response's status code is success (2xx)
func Status2xx(resp *http.Response) bool {
return resp.StatusCode/100 == 2

@ -1,478 +0,0 @@
// Copyright 2018 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clair
import (
"errors"
"sync"
"github.com/deckarep/golang-set"
log "github.com/sirupsen/logrus"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/ext/featurens"
"github.com/coreos/clair/ext/imagefmt"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/pkg/strutil"
"github.com/coreos/clair/pkg/tarutil"
)
var (
// ErrUnsupported is the error that should be raised when an OS or package
// manager is not supported.
ErrUnsupported = commonerr.NewBadRequestError("worker: OS and/or package manager are not supported")
// EnabledDetectors are detectors to be used to scan the layers.
EnabledDetectors []database.Detector
)
// LayerRequest represents all information necessary to download and process a
// layer.
type LayerRequest struct {
Hash string
Path string
Headers map[string]string
}
type processResult struct {
existingLayer *database.Layer
newLayerContent *database.Layer
err error
}
// processRequest stores parameters used for processing a layer.
type processRequest struct {
LayerRequest
existingLayer *database.Layer
detectors []database.Detector
}
type introducedFeature struct {
feature database.AncestryFeature
layerIndex int
}
// processRequests in parallel processes a set of requests for unique set of layers
// and returns sets of unique namespaces, features and layers to be inserted
// into the database.
func processRequests(imageFormat string, toDetect map[string]*processRequest) (map[string]*processResult, error) {
wg := &sync.WaitGroup{}
wg.Add(len(toDetect))
results := map[string]*processResult{}
for i := range toDetect {
results[i] = nil
}
for i := range toDetect {
result := processResult{}
results[i] = &result
go func(req *processRequest, res *processResult) {
*res = *detectContent(imageFormat, req)
wg.Done()
}(toDetect[i], &result)
}
wg.Wait()
errs := []error{}
for _, r := range results {
errs = append(errs, r.err)
}
if err := commonerr.CombineErrors(errs...); err != nil {
return nil, err
}
return results, nil
}
func getProcessRequest(datastore database.Datastore, req LayerRequest) (preq *processRequest, err error) {
layer, ok, err := database.FindLayerAndRollback(datastore, req.Hash)
if err != nil {
return
}
if !ok {
log.WithField("layer", req.Hash).Debug("found no existing layer in database")
preq = &processRequest{
LayerRequest: req,
existingLayer: &database.Layer{Hash: req.Hash},
detectors: EnabledDetectors,
}
} else {
log.WithFields(log.Fields{
"layer": layer.Hash,
"detectors": layer.By,
"feature count": len(layer.Features),
"namespace count": len(layer.Namespaces),
}).Debug("found existing layer in database")
preq = &processRequest{
LayerRequest: req,
existingLayer: &layer,
detectors: database.DiffDetectors(EnabledDetectors, layer.By),
}
}
return
}
func persistProcessResult(datastore database.Datastore, results map[string]*processResult) error {
features := []database.Feature{}
namespaces := []database.Namespace{}
for _, r := range results {
features = append(features, r.newLayerContent.GetFeatures()...)
namespaces = append(namespaces, r.newLayerContent.GetNamespaces()...)
}
features = database.DeduplicateFeatures(features...)
namespaces = database.DeduplicateNamespaces(namespaces...)
if err := database.PersistNamespacesAndCommit(datastore, namespaces); err != nil {
return err
}
if err := database.PersistFeaturesAndCommit(datastore, features); err != nil {
return err
}
for _, layer := range results {
if err := database.PersistPartialLayerAndCommit(datastore, layer.newLayerContent); err != nil {
return err
}
}
return nil
}
// processLayers processes a set of post layer requests, stores layers and
// returns an ordered list of processed layers with detected features and
// namespaces.
func processLayers(datastore database.Datastore, imageFormat string, requests []LayerRequest) ([]database.Layer, error) {
var (
reqMap = make(map[string]*processRequest)
err error
)
for _, r := range requests {
reqMap[r.Hash], err = getProcessRequest(datastore, r)
if err != nil {
return nil, err
}
}
results, err := processRequests(imageFormat, reqMap)
if err != nil {
return nil, err
}
if err := persistProcessResult(datastore, results); err != nil {
return nil, err
}
completeLayers := getProcessResultLayers(results)
layers := make([]database.Layer, 0, len(requests))
for _, r := range requests {
layers = append(layers, completeLayers[r.Hash])
}
return layers, nil
}
func getProcessResultLayers(results map[string]*processResult) map[string]database.Layer {
layers := map[string]database.Layer{}
for name, r := range results {
layers[name] = *database.MergeLayers(r.existingLayer, r.newLayerContent)