clair: Use builder pattern for constructing ancestry
- Add Ancestry builder - Change RPC to use the ancestry builder
This commit is contained in:
parent
891ce1697d
commit
5b2376498b
145
analyzer.go
Normal file
145
analyzer.go
Normal file
@ -0,0 +1,145 @@
|
||||
// Copyright 2019 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package clair
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/coreos/clair/database"
|
||||
"github.com/coreos/clair/ext/featurefmt"
|
||||
"github.com/coreos/clair/ext/featurens"
|
||||
"github.com/coreos/clair/ext/imagefmt"
|
||||
)
|
||||
|
||||
// AnalyzeError represents an failure when analyzing layer or constructing
|
||||
// ancestry.
|
||||
type AnalyzeError string
|
||||
|
||||
func (e AnalyzeError) Error() string {
|
||||
return string(e)
|
||||
}
|
||||
|
||||
var (
|
||||
// StorageError represents an analyze error caused by the storage
|
||||
StorageError = AnalyzeError("failed to query the database.")
|
||||
// RetrieveBlobError represents an analyze error caused by failure of
|
||||
// downloading or extracting layer blobs.
|
||||
RetrieveBlobError = AnalyzeError("failed to download layer blob.")
|
||||
// ExtractBlobError represents an analyzer error caused by failure of
|
||||
// extracting a layer blob by imagefmt.
|
||||
ExtractBlobError = AnalyzeError("failed to extract files from layer blob.")
|
||||
// FeatureDetectorError is an error caused by failure of feature listing by
|
||||
// featurefmt.
|
||||
FeatureDetectorError = AnalyzeError("failed to scan feature from layer blob files.")
|
||||
// NamespaceDetectorError is an error caused by failure of namespace
|
||||
// detection by featurens.
|
||||
NamespaceDetectorError = AnalyzeError("failed to scan namespace from layer blob files.")
|
||||
)
|
||||
|
||||
// AnalyzeLayer retrieves the clair layer with all extracted features and namespaces.
|
||||
// If a layer is already scanned by all enabled detectors in the Clair instance, it returns directly.
|
||||
// Otherwise, it re-download the layer blob and scan the features and namespaced again.
|
||||
func AnalyzeLayer(ctx context.Context, store database.Datastore, blobSha256 string, blobFormat string, downloadURI string, downloadHeaders map[string]string) (*database.Layer, error) {
|
||||
layer, found, err := database.FindLayerAndRollback(store, blobSha256)
|
||||
logFields := log.Fields{"layer.Hash": blobSha256}
|
||||
if err != nil {
|
||||
log.WithError(err).WithFields(logFields).Error("failed to find layer in the storage")
|
||||
return nil, StorageError
|
||||
}
|
||||
|
||||
var scannedBy []database.Detector
|
||||
if found {
|
||||
scannedBy = layer.By
|
||||
}
|
||||
|
||||
// layer will be scanned by detectors not scanned the layer already.
|
||||
toScan := database.DiffDetectors(EnabledDetectors(), scannedBy)
|
||||
if len(toScan) != 0 {
|
||||
log.WithFields(logFields).Debug("scan layer blob not already scanned")
|
||||
newLayerScanResult := &database.Layer{Hash: blobSha256, By: toScan}
|
||||
blob, err := retrieveLayerBlob(ctx, downloadURI, downloadHeaders)
|
||||
if err != nil {
|
||||
log.WithError(err).WithFields(logFields).Error("failed to retrieve layer blob")
|
||||
return nil, RetrieveBlobError
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := blob.Close(); err != nil {
|
||||
log.WithFields(logFields).Error("failed to close layer blob reader")
|
||||
}
|
||||
}()
|
||||
|
||||
files := append(featurefmt.RequiredFilenames(toScan), featurens.RequiredFilenames(toScan)...)
|
||||
fileMap, err := imagefmt.Extract(blobFormat, blob, files)
|
||||
if err != nil {
|
||||
log.WithFields(logFields).WithError(err).Error("failed to extract layer blob")
|
||||
return nil, ExtractBlobError
|
||||
}
|
||||
|
||||
newLayerScanResult.Features, err = featurefmt.ListFeatures(fileMap, toScan)
|
||||
if err != nil {
|
||||
log.WithFields(logFields).WithError(err).Error("failed to detect features")
|
||||
return nil, FeatureDetectorError
|
||||
}
|
||||
|
||||
newLayerScanResult.Namespaces, err = featurens.Detect(fileMap, toScan)
|
||||
if err != nil {
|
||||
log.WithFields(logFields).WithError(err).Error("failed to detect namespaces")
|
||||
return nil, NamespaceDetectorError
|
||||
}
|
||||
|
||||
if err = saveLayerChange(store, newLayerScanResult); err != nil {
|
||||
log.WithFields(logFields).WithError(err).Error("failed to store layer change")
|
||||
return nil, StorageError
|
||||
}
|
||||
|
||||
layer = database.MergeLayers(layer, newLayerScanResult)
|
||||
} else {
|
||||
log.WithFields(logFields).Debug("found scanned layer blob")
|
||||
}
|
||||
|
||||
return layer, nil
|
||||
}
|
||||
|
||||
// EnabledDetectors retrieves a list of all detectors installed in the Clair
|
||||
// instance.
|
||||
func EnabledDetectors() []database.Detector {
|
||||
return append(featurefmt.ListListers(), featurens.ListDetectors()...)
|
||||
}
|
||||
|
||||
// RegisterConfiguredDetectors populates the database with registered detectors.
|
||||
func RegisterConfiguredDetectors(store database.Datastore) {
|
||||
if err := database.PersistDetectorsAndCommit(store, EnabledDetectors()); err != nil {
|
||||
panic("failed to initialize Clair analyzer")
|
||||
}
|
||||
}
|
||||
|
||||
func saveLayerChange(store database.Datastore, layer *database.Layer) error {
|
||||
if err := database.PersistFeaturesAndCommit(store, layer.GetFeatures()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := database.PersistNamespacesAndCommit(store, layer.GetNamespaces()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := database.PersistPartialLayerAndCommit(store, layer); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
290
ancestry.go
Normal file
290
ancestry.go
Normal file
@ -0,0 +1,290 @@
|
||||
// Copyright 2019 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package clair
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/coreos/clair/database"
|
||||
)
|
||||
|
||||
type layerIndexedFeature struct {
|
||||
feature *database.LayerFeature
|
||||
namespace *layerIndexedNamespace
|
||||
introducedIn int
|
||||
}
|
||||
|
||||
type layerIndexedNamespace struct {
|
||||
namespace database.LayerNamespace
|
||||
introducedIn int
|
||||
}
|
||||
|
||||
// AncestryBuilder builds an Ancestry, which contains an ordered list of layers
|
||||
// and their features.
|
||||
type AncestryBuilder struct {
|
||||
layerIndex int
|
||||
layerNames []string
|
||||
detectors []database.Detector
|
||||
namespaces map[database.Detector]*layerIndexedNamespace
|
||||
features map[database.Detector][]layerIndexedFeature
|
||||
}
|
||||
|
||||
// NewAncestryBuilder creates a new ancestry builder.
|
||||
//
|
||||
// ancestry builder takes in the extracted layer information and produce a set of
|
||||
// namespaces, features, and the relation between features for the whole image.
|
||||
func NewAncestryBuilder(detectors []database.Detector) *AncestryBuilder {
|
||||
return &AncestryBuilder{
|
||||
layerIndex: 0,
|
||||
detectors: detectors,
|
||||
namespaces: make(map[database.Detector]*layerIndexedNamespace),
|
||||
features: make(map[database.Detector][]layerIndexedFeature),
|
||||
}
|
||||
}
|
||||
|
||||
// AddLeafLayer adds a leaf layer to the ancestry builder, and computes the
|
||||
// namespaced features.
|
||||
func (b *AncestryBuilder) AddLeafLayer(layer *database.Layer) {
|
||||
b.layerNames = append(b.layerNames, layer.Hash)
|
||||
for i := range layer.Namespaces {
|
||||
b.updateNamespace(&layer.Namespaces[i])
|
||||
}
|
||||
|
||||
allFeatureMap := map[database.Detector][]database.LayerFeature{}
|
||||
for i := range layer.Features {
|
||||
layerFeature := layer.Features[i]
|
||||
allFeatureMap[layerFeature.By] = append(allFeatureMap[layerFeature.By], layerFeature)
|
||||
}
|
||||
|
||||
// we only care about the ones specified by builder's detectors
|
||||
featureMap := map[database.Detector][]database.LayerFeature{}
|
||||
for i := range b.detectors {
|
||||
detector := b.detectors[i]
|
||||
featureMap[detector] = allFeatureMap[detector]
|
||||
}
|
||||
|
||||
for detector := range featureMap {
|
||||
b.addLayerFeatures(detector, featureMap[detector])
|
||||
}
|
||||
|
||||
b.layerIndex++
|
||||
}
|
||||
|
||||
// Every detector inspects a set of files for the features
|
||||
// therefore, if that set of files gives a different set of features, it
|
||||
// should replace the existing features.
|
||||
func (b *AncestryBuilder) addLayerFeatures(detector database.Detector, features []database.LayerFeature) {
|
||||
if len(features) == 0 {
|
||||
// TODO(sidac): we need to differentiate if the detector finds that all
|
||||
// features are removed ( a file change ), or the package installer is
|
||||
// removed ( a file deletion ), or there's no change in the file ( file
|
||||
// does not exist in the blob ) Right now, we're just assuming that no
|
||||
// change in the file because that's the most common case.
|
||||
return
|
||||
}
|
||||
|
||||
existingFeatures := b.features[detector]
|
||||
currentFeatures := make([]layerIndexedFeature, 0, len(features))
|
||||
// Features that are not in the current layer should be removed.
|
||||
for i := range existingFeatures {
|
||||
feature := existingFeatures[i]
|
||||
for j := range features {
|
||||
if features[j] == *feature.feature {
|
||||
currentFeatures = append(currentFeatures, feature)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Features that newly introduced in the current layer should be added.
|
||||
for i := range features {
|
||||
found := false
|
||||
for j := range existingFeatures {
|
||||
if *existingFeatures[j].feature == features[i] {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
namespace, found := b.lookupNamespace(&features[i])
|
||||
if !found {
|
||||
log.WithField("Layer Hashes", b.layerNames).Error("skip, could not find the proper namespace for feature")
|
||||
continue
|
||||
}
|
||||
|
||||
currentFeatures = append(currentFeatures, b.createLayerIndexedFeature(namespace, &features[i]))
|
||||
}
|
||||
}
|
||||
|
||||
b.features[detector] = currentFeatures
|
||||
}
|
||||
|
||||
// updateNamespace update the namespaces for the ancestry. It does the following things:
|
||||
// 1. when a detector detects a new namespace, it's added to the ancestry.
|
||||
// 2. when a detector detects a difference in the detected namespace, it
|
||||
// replaces the namespace, and also move all features under that namespace to
|
||||
// the new namespace.
|
||||
func (b *AncestryBuilder) updateNamespace(layerNamespace *database.LayerNamespace) {
|
||||
var (
|
||||
previous *layerIndexedNamespace
|
||||
ok bool
|
||||
)
|
||||
|
||||
if previous, ok = b.namespaces[layerNamespace.By]; !ok {
|
||||
b.namespaces[layerNamespace.By] = &layerIndexedNamespace{
|
||||
namespace: *layerNamespace,
|
||||
introducedIn: b.layerIndex,
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// All features referencing to this namespace are now pointing to the new namespace.
|
||||
// Also those features are now treated as introduced in the same layer as
|
||||
// when this new namespace is introduced.
|
||||
previous.namespace = *layerNamespace
|
||||
previous.introducedIn = b.layerIndex
|
||||
|
||||
for _, features := range b.features {
|
||||
for i, feature := range features {
|
||||
if feature.namespace == previous {
|
||||
features[i].introducedIn = previous.introducedIn
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *AncestryBuilder) createLayerIndexedFeature(namespace *layerIndexedNamespace, feature *database.LayerFeature) layerIndexedFeature {
|
||||
return layerIndexedFeature{
|
||||
feature: feature,
|
||||
namespace: namespace,
|
||||
introducedIn: b.layerIndex,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *AncestryBuilder) lookupNamespace(feature *database.LayerFeature) (*layerIndexedNamespace, bool) {
|
||||
for _, namespace := range b.namespaces {
|
||||
if namespace.namespace.VersionFormat == feature.VersionFormat {
|
||||
return namespace, true
|
||||
}
|
||||
}
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (b *AncestryBuilder) ancestryFeatures(index int) []database.AncestryFeature {
|
||||
ancestryFeatures := []database.AncestryFeature{}
|
||||
for detector, features := range b.features {
|
||||
for _, feature := range features {
|
||||
if feature.introducedIn == index {
|
||||
ancestryFeatures = append(ancestryFeatures, database.AncestryFeature{
|
||||
NamespacedFeature: database.NamespacedFeature{
|
||||
Feature: feature.feature.Feature,
|
||||
Namespace: feature.namespace.namespace.Namespace,
|
||||
},
|
||||
FeatureBy: detector,
|
||||
NamespaceBy: feature.namespace.namespace.By,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ancestryFeatures
|
||||
}
|
||||
|
||||
func (b *AncestryBuilder) ancestryLayers() []database.AncestryLayer {
|
||||
layers := make([]database.AncestryLayer, 0, b.layerIndex)
|
||||
for i := 0; i < b.layerIndex; i++ {
|
||||
layers = append(layers, database.AncestryLayer{
|
||||
Hash: b.layerNames[i],
|
||||
Features: b.ancestryFeatures(i),
|
||||
})
|
||||
}
|
||||
|
||||
return layers
|
||||
}
|
||||
|
||||
// Ancestry produces an Ancestry from the builder.
|
||||
func (b *AncestryBuilder) Ancestry(name string) *database.Ancestry {
|
||||
if name == "" {
|
||||
// TODO(sidac): we'll use the computed ancestry name in the future.
|
||||
// During the transition, it still requires the user to use the correct
|
||||
// ancestry name.
|
||||
name = ancestryName(b.layerNames)
|
||||
log.WithField("ancestry.Name", name).Warn("generated ancestry name since it's not specified")
|
||||
}
|
||||
|
||||
return &database.Ancestry{
|
||||
Name: name,
|
||||
By: b.detectors,
|
||||
Layers: b.ancestryLayers(),
|
||||
}
|
||||
}
|
||||
|
||||
// SaveAncestry saves an ancestry to the datastore.
|
||||
func SaveAncestry(store database.Datastore, ancestry *database.Ancestry) error {
|
||||
log.WithField("ancestry.Name", ancestry.Name).Debug("saving ancestry")
|
||||
features := []database.NamespacedFeature{}
|
||||
for _, layer := range ancestry.Layers {
|
||||
features = append(features, layer.GetFeatures()...)
|
||||
}
|
||||
|
||||
if err := database.PersistNamespacedFeaturesAndCommit(store, features); err != nil {
|
||||
return StorageError
|
||||
}
|
||||
|
||||
if err := database.UpsertAncestryAndCommit(store, ancestry); err != nil {
|
||||
return StorageError
|
||||
}
|
||||
|
||||
if err := database.CacheRelatedVulnerabilityAndCommit(store, features); err != nil {
|
||||
return StorageError
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsAncestryCached checks if the ancestry is already cached in the database with the current set of detectors.
|
||||
func IsAncestryCached(store database.Datastore, name string, layerHashes []string) (bool, error) {
|
||||
if name == "" {
|
||||
// TODO(sidac): we'll use the computed ancestry name in the future.
|
||||
// During the transition, it still requires the user to use the correct
|
||||
// ancestry name.
|
||||
name = ancestryName(layerHashes)
|
||||
log.WithField("ancestry.Name", name).Warn("generated ancestry name since it's not specified")
|
||||
}
|
||||
|
||||
ancestry, found, err := database.FindAncestryAndRollback(store, name)
|
||||
if err != nil {
|
||||
log.WithError(err).WithField("ancestry.Name", name).Error("failed to query ancestry in database")
|
||||
return false, StorageError
|
||||
}
|
||||
|
||||
if found {
|
||||
log.WithField("ancestry.Name", name).Debug("found cached ancestry")
|
||||
}
|
||||
|
||||
return found && len(database.DiffDetectors(EnabledDetectors(), ancestry.By)) == 0, nil
|
||||
}
|
||||
|
||||
func ancestryName(layerHashes []string) string {
|
||||
tag := sha256.Sum256([]byte(strings.Join(layerHashes, ",")))
|
||||
return hex.EncodeToString(tag[:])
|
||||
}
|
267
ancestry_test.go
Normal file
267
ancestry_test.go
Normal file
@ -0,0 +1,267 @@
|
||||
// Copyright 2019 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package clair
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/coreos/clair/database"
|
||||
)
|
||||
|
||||
var (
|
||||
dpkg = database.NewFeatureDetector("dpkg", "1.0")
|
||||
rpm = database.NewFeatureDetector("rpm", "1.0")
|
||||
pip = database.NewFeatureDetector("pip", "1.0")
|
||||
python = database.NewNamespaceDetector("python", "1.0")
|
||||
osrelease = database.NewNamespaceDetector("os-release", "1.0")
|
||||
ubuntu = *database.NewNamespace("ubuntu:14.04", "dpkg")
|
||||
ubuntu16 = *database.NewNamespace("ubuntu:16.04", "dpkg")
|
||||
python2 = *database.NewNamespace("python:2", "pip")
|
||||
sed = *database.NewSourcePackage("sed", "4.4-2", "dpkg")
|
||||
sedBin = *database.NewBinaryPackage("sed", "4.4-2", "dpkg")
|
||||
tar = *database.NewBinaryPackage("tar", "1.29b-2", "dpkg")
|
||||
scipy = *database.NewSourcePackage("scipy", "3.0.0", "pip")
|
||||
|
||||
detectors = []database.Detector{dpkg, osrelease, rpm}
|
||||
multinamespaceDetectors = []database.Detector{dpkg, osrelease, pip}
|
||||
)
|
||||
|
||||
// layerBuilder is for helping constructing the layer test artifacts.
|
||||
type layerBuilder struct {
|
||||
layer *database.Layer
|
||||
}
|
||||
|
||||
func newLayerBuilder(hash string) *layerBuilder {
|
||||
return &layerBuilder{&database.Layer{Hash: hash, By: detectors}}
|
||||
}
|
||||
|
||||
func (b *layerBuilder) addNamespace(detector database.Detector, ns database.Namespace) *layerBuilder {
|
||||
b.layer.Namespaces = append(b.layer.Namespaces, database.LayerNamespace{
|
||||
Namespace: ns,
|
||||
By: detector,
|
||||
})
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *layerBuilder) addFeature(detector database.Detector, f database.Feature) *layerBuilder {
|
||||
b.layer.Features = append(b.layer.Features, database.LayerFeature{
|
||||
Feature: f,
|
||||
By: detector,
|
||||
})
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
var testImage = []*database.Layer{
|
||||
// empty layer
|
||||
newLayerBuilder("0").layer,
|
||||
// ubuntu namespace
|
||||
newLayerBuilder("1").addNamespace(osrelease, ubuntu).layer,
|
||||
// install sed
|
||||
newLayerBuilder("2").addFeature(dpkg, sed).layer,
|
||||
// install tar
|
||||
newLayerBuilder("3").addFeature(dpkg, sed).addFeature(dpkg, tar).layer,
|
||||
// remove tar
|
||||
newLayerBuilder("4").addFeature(dpkg, sed).layer,
|
||||
// upgrade ubuntu
|
||||
newLayerBuilder("5").addNamespace(osrelease, ubuntu16).layer,
|
||||
// no change to the detectable files
|
||||
newLayerBuilder("6").layer,
|
||||
// change to the package installer database but no features are affected.
|
||||
newLayerBuilder("7").addFeature(dpkg, sed).layer,
|
||||
}
|
||||
|
||||
var clairLimit = []*database.Layer{
|
||||
// TODO(sidac): how about install rpm package under ubuntu?
|
||||
newLayerBuilder("1").addNamespace(osrelease, ubuntu).layer,
|
||||
newLayerBuilder("2").addFeature(rpm, sed).layer,
|
||||
}
|
||||
|
||||
var multipleNamespace = []*database.Layer{
|
||||
// TODO(sidac): support for multiple namespaces
|
||||
}
|
||||
|
||||
var invalidNamespace = []*database.Layer{
|
||||
// add package without namespace, this indicates that the namespace detector
|
||||
// could not detect the namespace.
|
||||
newLayerBuilder("0").addFeature(dpkg, sed).layer,
|
||||
}
|
||||
|
||||
var multiplePackagesOnFirstLayer = []*database.Layer{
|
||||
newLayerBuilder("0").addFeature(dpkg, sed).addFeature(dpkg, tar).addFeature(dpkg, sedBin).addNamespace(osrelease, ubuntu16).layer,
|
||||
}
|
||||
|
||||
func TestAddLayer(t *testing.T) {
|
||||
cases := []struct {
|
||||
title string
|
||||
image []*database.Layer
|
||||
|
||||
expectedAncestry database.Ancestry
|
||||
}{
|
||||
{
|
||||
title: "empty image",
|
||||
expectedAncestry: database.Ancestry{Name: ancestryName([]string{}), By: detectors},
|
||||
},
|
||||
{
|
||||
title: "empty layer",
|
||||
image: testImage[:1],
|
||||
expectedAncestry: database.Ancestry{Name: ancestryName([]string{"0"}), By: detectors, Layers: []database.AncestryLayer{{Hash: "0"}}},
|
||||
},
|
||||
{
|
||||
title: "ubuntu",
|
||||
image: testImage[:2],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "ubuntu install sed",
|
||||
image: testImage[:3],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1", "2"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
}}},
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "ubuntu install tar",
|
||||
image: testImage[:4],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1", "2", "3"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
}}, {
|
||||
Hash: "3", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: tar, Namespace: ubuntu},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
}, {
|
||||
title: "ubuntu uninstall tar",
|
||||
image: testImage[:5],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1", "2", "3", "4"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
}}, {Hash: "3"}, {Hash: "4"}},
|
||||
},
|
||||
}, {
|
||||
title: "ubuntu upgrade",
|
||||
image: testImage[:6],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1", "2", "3", "4", "5"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2"}, {Hash: "3"}, {Hash: "4"}, {Hash: "5", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
}}},
|
||||
},
|
||||
},
|
||||
}, {
|
||||
title: "no change to the detectable files",
|
||||
image: testImage[:7],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1", "2", "3", "4", "5", "6"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2"}, {Hash: "3"}, {Hash: "4"}, {Hash: "5", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
}}}, {Hash: "6"}},
|
||||
},
|
||||
}, {
|
||||
title: "change to the package installer database but no features are affected.",
|
||||
image: testImage[:8],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1", "2", "3", "4", "5", "6", "7"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2"}, {Hash: "3"}, {Hash: "4"}, {Hash: "5", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
}}}, {Hash: "6"}, {Hash: "7"}},
|
||||
},
|
||||
}, {
|
||||
title: "layers with features and namespace.",
|
||||
image: multiplePackagesOnFirstLayer,
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{
|
||||
{
|
||||
Hash: "0",
|
||||
Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sedBin, Namespace: ubuntu16},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: tar, Namespace: ubuntu16},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range cases {
|
||||
t.Run(test.title, func(t *testing.T) {
|
||||
builder := NewAncestryBuilder(detectors)
|
||||
for _, layer := range test.image {
|
||||
builder.AddLeafLayer(layer)
|
||||
}
|
||||
|
||||
ancestry := builder.Ancestry("")
|
||||
require.True(t, database.AssertAncestryEqual(t, &test.expectedAncestry, ancestry))
|
||||
})
|
||||
}
|
||||
}
|
@ -17,6 +17,8 @@ package v3
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/coreos/clair/ext/imagefmt"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
@ -28,6 +30,10 @@ import (
|
||||
"github.com/coreos/clair/pkg/pagination"
|
||||
)
|
||||
|
||||
func newRPCErrorWithClairError(code codes.Code, err error) error {
|
||||
return status.Errorf(code, "clair error reason: '%s'", err.Error())
|
||||
}
|
||||
|
||||
// NotificationServer implements NotificationService interface for serving RPC.
|
||||
type NotificationServer struct {
|
||||
Store database.Datastore
|
||||
@ -55,23 +61,34 @@ func (s *StatusServer) GetStatus(ctx context.Context, req *pb.GetStatusRequest)
|
||||
|
||||
// PostAncestry implements posting an ancestry via the Clair gRPC service.
|
||||
func (s *AncestryServer) PostAncestry(ctx context.Context, req *pb.PostAncestryRequest) (*pb.PostAncestryResponse, error) {
|
||||
ancestryName := req.GetAncestryName()
|
||||
if ancestryName == "" {
|
||||
return nil, status.Error(codes.InvalidArgument, "ancestry name should not be empty")
|
||||
// validate request
|
||||
blobFormat := req.GetFormat()
|
||||
if !imagefmt.IsSupported(blobFormat) {
|
||||
return nil, status.Error(codes.InvalidArgument, "image blob format is not supported")
|
||||
}
|
||||
|
||||
layers := req.GetLayers()
|
||||
if len(layers) == 0 {
|
||||
return nil, status.Error(codes.InvalidArgument, "ancestry should have at least one layer")
|
||||
clairStatus, err := GetClairStatus(s.Store)
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
ancestryFormat := req.GetFormat()
|
||||
if ancestryFormat == "" {
|
||||
return nil, status.Error(codes.InvalidArgument, "ancestry format should not be empty")
|
||||
// check if the ancestry is already processed, if not we build the ancestry again.
|
||||
layerHashes := make([]string, len(req.Layers))
|
||||
for i, layer := range req.Layers {
|
||||
layerHashes[i] = layer.GetHash()
|
||||
}
|
||||
|
||||
ancestryLayers := []clair.LayerRequest{}
|
||||
for _, layer := range layers {
|
||||
found, err := clair.IsAncestryCached(s.Store, req.AncestryName, layerHashes)
|
||||
if err != nil {
|
||||
return nil, newRPCErrorWithClairError(codes.Internal, err)
|
||||
}
|
||||
|
||||
if found {
|
||||
return &pb.PostAncestryResponse{Status: clairStatus}, nil
|
||||
}
|
||||
|
||||
builder := clair.NewAncestryBuilder(clair.EnabledDetectors())
|
||||
for _, layer := range req.Layers {
|
||||
if layer == nil {
|
||||
err := status.Error(codes.InvalidArgument, "ancestry layer is invalid")
|
||||
return nil, err
|
||||
@ -85,21 +102,22 @@ func (s *AncestryServer) PostAncestry(ctx context.Context, req *pb.PostAncestryR
|
||||
return nil, status.Error(codes.InvalidArgument, "ancestry layer path should not be empty")
|
||||
}
|
||||
|
||||
ancestryLayers = append(ancestryLayers, clair.LayerRequest{
|
||||
Hash: layer.Hash,
|
||||
Headers: layer.Headers,
|
||||
Path: layer.Path,
|
||||
})
|
||||
// TODO(sidac): make AnalyzeLayer to be async to ensure
|
||||
// non-blocking downloads.
|
||||
// We'll need to deal with two layers post by the same or different
|
||||
// requests that may have the same hash. In that case, since every
|
||||
// layer/feature/namespace is unique in the database, it may introduce
|
||||
// deadlock.
|
||||
clairLayer, err := clair.AnalyzeLayer(ctx, s.Store, layer.Hash, req.Format, layer.Path, layer.Headers)
|
||||
if err != nil {
|
||||
return nil, newRPCErrorWithClairError(codes.Internal, err)
|
||||
}
|
||||
|
||||
builder.AddLeafLayer(clairLayer)
|
||||
}
|
||||
|
||||
err := clair.ProcessAncestry(s.Store, ancestryFormat, ancestryName, ancestryLayers)
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.Internal, "ancestry is failed to be processed: "+err.Error())
|
||||
}
|
||||
|
||||
clairStatus, err := GetClairStatus(s.Store)
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
if err := clair.SaveAncestry(s.Store, builder.Ancestry(req.AncestryName)); err != nil {
|
||||
return nil, newRPCErrorWithClairError(codes.Internal, err)
|
||||
}
|
||||
|
||||
return &pb.PostAncestryResponse{Status: clairStatus}, nil
|
||||
|
@ -13,7 +13,7 @@ import (
|
||||
// protobuf struct.
|
||||
func GetClairStatus(store database.Datastore) (*pb.ClairStatus, error) {
|
||||
status := &pb.ClairStatus{
|
||||
Detectors: pb.DetectorsFromDatabaseModel(clair.EnabledDetectors),
|
||||
Detectors: pb.DetectorsFromDatabaseModel(clair.EnabledDetectors()),
|
||||
}
|
||||
|
||||
t, firstUpdate, err := clair.GetLastUpdateTime(store)
|
||||
|
70
blob.go
70
blob.go
@ -1,61 +1,43 @@
|
||||
// Copyright 2019 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package clair
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/coreos/clair/pkg/httputil"
|
||||
)
|
||||
|
||||
func retrieveLayerBlob(ctx context.Context, blobSha256 string, path string, headers map[string]string) (io.ReadCloser, error) {
|
||||
func retrieveLayerBlob(ctx context.Context, path string, headers map[string]string) (io.ReadCloser, error) {
|
||||
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
|
||||
return downloadLayerBlob(ctx, blobSha256, path, headers)
|
||||
}
|
||||
|
||||
return loadLayerBlobFromFS(blobSha256)
|
||||
}
|
||||
|
||||
func downloadLayerBlob(ctx context.Context, blobSha256 string, uri string, headers map[string]string) (io.ReadCloser, error) {
|
||||
request, err := http.NewRequest("GET", uri, nil)
|
||||
if err != nil {
|
||||
return nil, RetrieveBlobError
|
||||
}
|
||||
|
||||
if headers != nil {
|
||||
for k, v := range headers {
|
||||
request.Header.Set(k, v)
|
||||
httpHeaders := make(http.Header)
|
||||
for key, value := range headers {
|
||||
httpHeaders[key] = []string{value}
|
||||
}
|
||||
|
||||
reader, err := httputil.GetWithContext(ctx, path, httpHeaders)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{},
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
}
|
||||
|
||||
client := &http.Client{Transport: tr}
|
||||
r, err := client.Do(request)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("could not download layer")
|
||||
return nil, RetrieveBlobError
|
||||
}
|
||||
|
||||
// Fail if we don't receive a 2xx HTTP status code.
|
||||
if is2xx(r.StatusCode) {
|
||||
log.WithField("status", r.StatusCode).Error("could not download layer: expected 2XX")
|
||||
return nil, RetrieveBlobError
|
||||
}
|
||||
|
||||
return r.Body, nil
|
||||
}
|
||||
|
||||
func is2xx(statusCode int) bool {
|
||||
return statusCode >= 200 && statusCode < 300
|
||||
}
|
||||
|
||||
func loadLayerBlobFromFS(path string) (io.ReadCloser, error) {
|
||||
return os.Open(path)
|
||||
}
|
||||
|
@ -30,9 +30,6 @@ import (
|
||||
"github.com/coreos/clair"
|
||||
"github.com/coreos/clair/api"
|
||||
"github.com/coreos/clair/database"
|
||||
"github.com/coreos/clair/ext/featurefmt"
|
||||
"github.com/coreos/clair/ext/featurens"
|
||||
"github.com/coreos/clair/ext/imagefmt"
|
||||
"github.com/coreos/clair/ext/vulnsrc"
|
||||
"github.com/coreos/clair/pkg/formatter"
|
||||
"github.com/coreos/clair/pkg/stopper"
|
||||
@ -103,11 +100,10 @@ func stopCPUProfiling(f *os.File) {
|
||||
}
|
||||
|
||||
func configClairVersion(config *Config) {
|
||||
clair.EnabledDetectors = append(featurefmt.ListListers(), featurens.ListDetectors()...)
|
||||
clair.EnabledUpdaters = strutil.Intersect(config.Updater.EnabledUpdaters, vulnsrc.ListUpdaters())
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"Detectors": database.SerializeDetectors(clair.EnabledDetectors),
|
||||
"Detectors": database.SerializeDetectors(clair.EnabledDetectors()),
|
||||
"Updaters": clair.EnabledUpdaters,
|
||||
}).Info("enabled Clair extensions")
|
||||
}
|
||||
@ -134,7 +130,8 @@ func Boot(config *Config) {
|
||||
|
||||
defer db.Close()
|
||||
|
||||
clair.InitWorker(db)
|
||||
clair.RegisterConfiguredDetectors(db)
|
||||
|
||||
// Start notifier
|
||||
st.Begin()
|
||||
go clair.RunNotifier(config.Notifier, db, st)
|
||||
@ -173,7 +170,6 @@ func main() {
|
||||
flagConfigPath := flag.String("config", "/etc/clair/config.yaml", "Load configuration from the specified file.")
|
||||
flagCPUProfilePath := flag.String("cpu-profile", "", "Write a CPU profile to the specified file before exiting.")
|
||||
flagLogLevel := flag.String("log-level", "info", "Define the logging level.")
|
||||
flagInsecureTLS := flag.Bool("insecure-tls", false, "Disable TLS server's certificate chain and hostname verification when pulling layers.")
|
||||
flag.Parse()
|
||||
|
||||
configureLogger(flagLogLevel)
|
||||
@ -195,12 +191,6 @@ func main() {
|
||||
defer stopCPUProfiling(startCPUProfiling(*flagCPUProfilePath))
|
||||
}
|
||||
|
||||
// Enable TLS server's certificate chain and hostname verification
|
||||
// when pulling layers if specified
|
||||
if *flagInsecureTLS {
|
||||
imagefmt.SetInsecureTLS(*flagInsecureTLS)
|
||||
}
|
||||
|
||||
// configure updater and worker
|
||||
configClairVersion(config)
|
||||
|
||||
|
@ -15,8 +15,11 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/deckarep/golang-set"
|
||||
)
|
||||
|
||||
@ -94,8 +97,11 @@ func PersistFeaturesAndCommit(datastore Datastore, features []Feature) error {
|
||||
defer tx.Rollback()
|
||||
|
||||
if err := tx.PersistFeatures(features); err != nil {
|
||||
serialized, _ := json.Marshal(features)
|
||||
log.WithError(err).WithField("feature", string(serialized)).Error("failed to store features")
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@ -129,14 +135,18 @@ func FindAncestryAndRollback(datastore Datastore, name string) (Ancestry, bool,
|
||||
}
|
||||
|
||||
// FindLayerAndRollback wraps session FindLayer function with begin and rollback.
|
||||
func FindLayerAndRollback(datastore Datastore, hash string) (layer Layer, ok bool, err error) {
|
||||
func FindLayerAndRollback(datastore Datastore, hash string) (layer *Layer, ok bool, err error) {
|
||||
var tx Session
|
||||
if tx, err = datastore.Begin(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
defer tx.Rollback()
|
||||
layer, ok, err = tx.FindLayer(hash)
|
||||
// TODO(sidac): In order to make the session interface more idiomatic, we'll
|
||||
// return the pointer value in the future.
|
||||
var dereferencedLayer Layer
|
||||
dereferencedLayer, ok, err = tx.FindLayer(hash)
|
||||
layer = &dereferencedLayer
|
||||
return
|
||||
}
|
||||
|
||||
@ -168,13 +178,17 @@ func GetAncestryFeatures(ancestry Ancestry) []NamespacedFeature {
|
||||
}
|
||||
|
||||
// UpsertAncestryAndCommit wraps session UpsertAncestry function with begin and commit.
|
||||
func UpsertAncestryAndCommit(datastore Datastore, ancestry Ancestry) error {
|
||||
func UpsertAncestryAndCommit(datastore Datastore, ancestry *Ancestry) error {
|
||||
tx, err := datastore.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = tx.UpsertAncestry(ancestry); err != nil {
|
||||
if err = tx.UpsertAncestry(*ancestry); err != nil {
|
||||
log.WithError(err).Error("failed to upsert the ancestry")
|
||||
serialized, _ := json.Marshal(ancestry)
|
||||
log.Debug(string(serialized))
|
||||
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
@ -350,3 +364,22 @@ func ReleaseLock(datastore Datastore, name, owner string) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// PersistDetectorsAndCommit stores the detectors in the data store.
|
||||
func PersistDetectorsAndCommit(store Datastore, detectors []Detector) error {
|
||||
tx, err := store.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer tx.Rollback()
|
||||
if err := tx.PersistDetectors(detectors); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -93,12 +93,12 @@ func (s DetectorType) Valid() bool {
|
||||
type Detector struct {
|
||||
// Name of an extension should be non-empty and uniquely identifies the
|
||||
// extension.
|
||||
Name string
|
||||
Name string `json:"name"`
|
||||
// Version of an extension should be non-empty.
|
||||
Version string
|
||||
Version string `json:"version"`
|
||||
// DType is the type of the extension and should be one of the types in
|
||||
// DetectorTypes.
|
||||
DType DetectorType
|
||||
DType DetectorType `json:"dtype"`
|
||||
}
|
||||
|
||||
// Valid checks if all fields in the detector satisfies the spec.
|
||||
|
@ -26,13 +26,13 @@ import (
|
||||
type Ancestry struct {
|
||||
// Name is a globally unique value for a set of layers. This is often the
|
||||
// sha256 digest of an OCI/Docker manifest.
|
||||
Name string
|
||||
Name string `json:"name"`
|
||||
// By contains the processors that are used when computing the
|
||||
// content of this ancestry.
|
||||
By []Detector
|
||||
By []Detector `json:"by"`
|
||||
// Layers should be ordered and i_th layer is the parent of i+1_th layer in
|
||||
// the slice.
|
||||
Layers []AncestryLayer
|
||||
Layers []AncestryLayer `json:"layers"`
|
||||
}
|
||||
|
||||
// Valid checks if the ancestry is compliant to spec.
|
||||
@ -63,10 +63,10 @@ func (a *Ancestry) Valid() bool {
|
||||
// AncestryLayer is a layer with all detected namespaced features.
|
||||
type AncestryLayer struct {
|
||||
// Hash is the sha-256 tarsum on the layer's blob content.
|
||||
Hash string
|
||||
Hash string `json:"hash"`
|
||||
// Features are the features introduced by this layer when it was
|
||||
// processed.
|
||||
Features []AncestryFeature
|
||||
Features []AncestryFeature `json:"features"`
|
||||
}
|
||||
|
||||
// Valid checks if the Ancestry Layer is compliant to the spec.
|
||||
@ -95,22 +95,22 @@ func (l *AncestryLayer) GetFeatures() []NamespacedFeature {
|
||||
// AncestryFeature is a namespaced feature with the detectors used to
|
||||
// find this feature.
|
||||
type AncestryFeature struct {
|
||||
NamespacedFeature
|
||||
NamespacedFeature `json:"namespacedFeature"`
|
||||
|
||||
// FeatureBy is the detector that detected the feature.
|
||||
FeatureBy Detector
|
||||
FeatureBy Detector `json:"featureBy"`
|
||||
// NamespaceBy is the detector that detected the namespace.
|
||||
NamespaceBy Detector
|
||||
NamespaceBy Detector `json:"namespaceBy"`
|
||||
}
|
||||
|
||||
// Layer is a layer with all the detected features and namespaces.
|
||||
type Layer struct {
|
||||
// Hash is the sha-256 tarsum on the layer's blob content.
|
||||
Hash string
|
||||
Hash string `json:"hash"`
|
||||
// By contains a list of detectors scanned this Layer.
|
||||
By []Detector
|
||||
Namespaces []LayerNamespace
|
||||
Features []LayerFeature
|
||||
By []Detector `json:"by"`
|
||||
Namespaces []LayerNamespace `json:"namespaces"`
|
||||
Features []LayerFeature `json:"features"`
|
||||
}
|
||||
|
||||
func (l *Layer) GetFeatures() []Feature {
|
||||
@ -133,26 +133,26 @@ func (l *Layer) GetNamespaces() []Namespace {
|
||||
|
||||
// LayerNamespace is a namespace with detection information.
|
||||
type LayerNamespace struct {
|
||||
Namespace
|
||||
Namespace `json:"namespace"`
|
||||
|
||||
// By is the detector found the namespace.
|
||||
By Detector
|
||||
By Detector `json:"by"`
|
||||
}
|
||||
|
||||
// LayerFeature is a feature with detection information.
|
||||
type LayerFeature struct {
|
||||
Feature
|
||||
Feature `json:"feature"`
|
||||
|
||||
// By is the detector found the feature.
|
||||
By Detector
|
||||
By Detector `json:"by"`
|
||||
}
|
||||
|
||||
// Namespace is the contextual information around features.
|
||||
//
|
||||
// e.g. Debian:7, NodeJS.
|
||||
type Namespace struct {
|
||||
Name string
|
||||
VersionFormat string
|
||||
Name string `json:"name"`
|
||||
VersionFormat string `json:"versionFormat"`
|
||||
}
|
||||
|
||||
func NewNamespace(name string, versionFormat string) *Namespace {
|
||||
@ -166,10 +166,10 @@ func NewNamespace(name string, versionFormat string) *Namespace {
|
||||
// dpkg is the version format of the installer package manager, which in this
|
||||
// case could be dpkg or apk.
|
||||
type Feature struct {
|
||||
Name string
|
||||
Version string
|
||||
VersionFormat string
|
||||
Type FeatureType
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
VersionFormat string `json:"versionFormat"`
|
||||
Type FeatureType `json:"type"`
|
||||
}
|
||||
|
||||
func NewFeature(name string, version string, versionFormat string, featureType FeatureType) *Feature {
|
||||
@ -189,9 +189,9 @@ func NewSourcePackage(name string, version string, versionFormat string) *Featur
|
||||
//
|
||||
// e.g. OpenSSL 1.0 dpkg Debian:7.
|
||||
type NamespacedFeature struct {
|
||||
Feature
|
||||
Feature `json:"feature"`
|
||||
|
||||
Namespace Namespace
|
||||
Namespace Namespace `json:"namespace"`
|
||||
}
|
||||
|
||||
func NewNamespacedFeature(namespace *Namespace, feature *Feature) *NamespacedFeature {
|
||||
|
@ -16,6 +16,10 @@
|
||||
package httputil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
@ -62,6 +66,38 @@ func GetClientAddr(r *http.Request) string {
|
||||
return addr
|
||||
}
|
||||
|
||||
// GetWithContext do HTTP GET to the URI with headers and returns response blob
|
||||
// reader.
|
||||
func GetWithContext(ctx context.Context, uri string, headers http.Header) (io.ReadCloser, error) {
|
||||
request, err := http.NewRequest("GET", uri, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if headers != nil {
|
||||
request.Header = headers
|
||||
}
|
||||
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{},
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
}
|
||||
|
||||
client := &http.Client{Transport: tr}
|
||||
request = request.WithContext(ctx)
|
||||
r, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Fail if we don't receive a 2xx HTTP status code.
|
||||
if !Status2xx(r) {
|
||||
return nil, fmt.Errorf("failed HTTP GET: expected 2XX, got %d", r.StatusCode)
|
||||
}
|
||||
|
||||
return r.Body, nil
|
||||
}
|
||||
|
||||
// Status2xx returns true if the response's status code is success (2xx)
|
||||
func Status2xx(resp *http.Response) bool {
|
||||
return resp.StatusCode/100 == 2
|
||||
|
478
worker.go
478
worker.go
@ -1,478 +0,0 @@
|
||||
// Copyright 2018 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package clair
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
|
||||
"github.com/deckarep/golang-set"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/coreos/clair/database"
|
||||
"github.com/coreos/clair/ext/featurefmt"
|
||||
"github.com/coreos/clair/ext/featurens"
|
||||
"github.com/coreos/clair/ext/imagefmt"
|
||||
"github.com/coreos/clair/pkg/commonerr"
|
||||
"github.com/coreos/clair/pkg/strutil"
|
||||
"github.com/coreos/clair/pkg/tarutil"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrUnsupported is the error that should be raised when an OS or package
|
||||
// manager is not supported.
|
||||
ErrUnsupported = commonerr.NewBadRequestError("worker: OS and/or package manager are not supported")
|
||||
|
||||
// EnabledDetectors are detectors to be used to scan the layers.
|
||||
EnabledDetectors []database.Detector
|
||||
)
|
||||
|
||||
// LayerRequest represents all information necessary to download and process a
|
||||
// layer.
|
||||
type LayerRequest struct {
|
||||
Hash string
|
||||
Path string
|
||||
Headers map[string]string
|
||||
}
|
||||
|
||||
type processResult struct {
|
||||
existingLayer *database.Layer
|
||||
newLayerContent *database.Layer
|
||||
err error
|
||||
}
|
||||
|
||||
// processRequest stores parameters used for processing a layer.
|
||||
type processRequest struct {
|
||||
LayerRequest
|
||||
|
||||
existingLayer *database.Layer
|
||||
detectors []database.Detector
|
||||
}
|
||||
|
||||
type introducedFeature struct {
|
||||
feature database.AncestryFeature
|
||||
layerIndex int
|
||||
}
|
||||
|
||||
// processRequests in parallel processes a set of requests for unique set of layers
|
||||
// and returns sets of unique namespaces, features and layers to be inserted
|
||||
// into the database.
|
||||
func processRequests(imageFormat string, toDetect map[string]*processRequest) (map[string]*processResult, error) {
|
||||
wg := &sync.WaitGroup{}
|
||||
wg.Add(len(toDetect))
|
||||
|
||||
results := map[string]*processResult{}
|
||||
for i := range toDetect {
|
||||
results[i] = nil
|
||||
}
|
||||
|
||||
for i := range toDetect {
|
||||
result := processResult{}
|
||||
results[i] = &result
|
||||
go func(req *processRequest, res *processResult) {
|
||||
*res = *detectContent(imageFormat, req)
|
||||
wg.Done()
|
||||
}(toDetect[i], &result)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
errs := []error{}
|
||||
for _, r := range results {
|
||||
errs = append(errs, r.err)
|
||||
}
|
||||
|
||||
if err := commonerr.CombineErrors(errs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func getProcessRequest(datastore database.Datastore, req LayerRequest) (preq *processRequest, err error) {
|
||||
layer, ok, err := database.FindLayerAndRollback(datastore, req.Hash)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if !ok {
|
||||
log.WithField("layer", req.Hash).Debug("found no existing layer in database")
|
||||
preq = &processRequest{
|
||||
LayerRequest: req,
|
||||
existingLayer: &database.Layer{Hash: req.Hash},
|
||||
detectors: EnabledDetectors,
|
||||
}
|
||||
} else {
|
||||
log.WithFields(log.Fields{
|
||||
"layer": layer.Hash,
|
||||
"detectors": layer.By,
|
||||
"feature count": len(layer.Features),
|
||||
"namespace count": len(layer.Namespaces),
|
||||
}).Debug("found existing layer in database")
|
||||
|
||||
preq = &processRequest{
|
||||
LayerRequest: req,
|
||||
existingLayer: &layer,
|
||||
detectors: database.DiffDetectors(EnabledDetectors, layer.By),
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func persistProcessResult(datastore database.Datastore, results map[string]*processResult) error {
|
||||
features := []database.Feature{}
|
||||
namespaces := []database.Namespace{}
|
||||
for _, r := range results {
|
||||
features = append(features, r.newLayerContent.GetFeatures()...)
|
||||
namespaces = append(namespaces, r.newLayerContent.GetNamespaces()...)
|
||||
}
|
||||
|
||||
features = database.DeduplicateFeatures(features...)
|
||||
namespaces = database.DeduplicateNamespaces(namespaces...)
|
||||
if err := database.PersistNamespacesAndCommit(datastore, namespaces); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := database.PersistFeaturesAndCommit(datastore, features); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, layer := range results {
|
||||
if err := database.PersistPartialLayerAndCommit(datastore, layer.newLayerContent); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// processLayers processes a set of post layer requests, stores layers and
|
||||
// returns an ordered list of processed layers with detected features and
|
||||
// namespaces.
|
||||
func processLayers(datastore database.Datastore, imageFormat string, requests []LayerRequest) ([]database.Layer, error) {
|
||||
var (
|
||||
reqMap = make(map[string]*processRequest)
|
||||
err error
|
||||
)
|
||||
|
||||
for _, r := range requests {
|
||||
reqMap[r.Hash], err = getProcessRequest(datastore, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
results, err := processRequests(imageFormat, reqMap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := persistProcessResult(datastore, results); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
completeLayers := getProcessResultLayers(results)
|
||||
layers := make([]database.Layer, 0, len(requests))
|
||||
for _, r := range requests {
|
||||
layers = append(layers, completeLayers[r.Hash])
|
||||
}
|
||||
|
||||
return layers, nil
|
||||