- Add Ancestry builder - Change RPC to use the ancestry buildermaster
Sida Chen
5 years ago
parent
891ce1697d
commit
5b2376498b
@ -0,0 +1,145 @@
|
||||
// Copyright 2019 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package clair
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/coreos/clair/database"
|
||||
"github.com/coreos/clair/ext/featurefmt"
|
||||
"github.com/coreos/clair/ext/featurens"
|
||||
"github.com/coreos/clair/ext/imagefmt"
|
||||
)
|
||||
|
||||
// AnalyzeError represents an failure when analyzing layer or constructing
|
||||
// ancestry.
|
||||
type AnalyzeError string
|
||||
|
||||
func (e AnalyzeError) Error() string {
|
||||
return string(e)
|
||||
}
|
||||
|
||||
var (
|
||||
// StorageError represents an analyze error caused by the storage
|
||||
StorageError = AnalyzeError("failed to query the database.")
|
||||
// RetrieveBlobError represents an analyze error caused by failure of
|
||||
// downloading or extracting layer blobs.
|
||||
RetrieveBlobError = AnalyzeError("failed to download layer blob.")
|
||||
// ExtractBlobError represents an analyzer error caused by failure of
|
||||
// extracting a layer blob by imagefmt.
|
||||
ExtractBlobError = AnalyzeError("failed to extract files from layer blob.")
|
||||
// FeatureDetectorError is an error caused by failure of feature listing by
|
||||
// featurefmt.
|
||||
FeatureDetectorError = AnalyzeError("failed to scan feature from layer blob files.")
|
||||
// NamespaceDetectorError is an error caused by failure of namespace
|
||||
// detection by featurens.
|
||||
NamespaceDetectorError = AnalyzeError("failed to scan namespace from layer blob files.")
|
||||
)
|
||||
|
||||
// AnalyzeLayer retrieves the clair layer with all extracted features and namespaces.
|
||||
// If a layer is already scanned by all enabled detectors in the Clair instance, it returns directly.
|
||||
// Otherwise, it re-download the layer blob and scan the features and namespaced again.
|
||||
func AnalyzeLayer(ctx context.Context, store database.Datastore, blobSha256 string, blobFormat string, downloadURI string, downloadHeaders map[string]string) (*database.Layer, error) {
|
||||
layer, found, err := database.FindLayerAndRollback(store, blobSha256)
|
||||
logFields := log.Fields{"layer.Hash": blobSha256}
|
||||
if err != nil {
|
||||
log.WithError(err).WithFields(logFields).Error("failed to find layer in the storage")
|
||||
return nil, StorageError
|
||||
}
|
||||
|
||||
var scannedBy []database.Detector
|
||||
if found {
|
||||
scannedBy = layer.By
|
||||
}
|
||||
|
||||
// layer will be scanned by detectors not scanned the layer already.
|
||||
toScan := database.DiffDetectors(EnabledDetectors(), scannedBy)
|
||||
if len(toScan) != 0 {
|
||||
log.WithFields(logFields).Debug("scan layer blob not already scanned")
|
||||
newLayerScanResult := &database.Layer{Hash: blobSha256, By: toScan}
|
||||
blob, err := retrieveLayerBlob(ctx, downloadURI, downloadHeaders)
|
||||
if err != nil {
|
||||
log.WithError(err).WithFields(logFields).Error("failed to retrieve layer blob")
|
||||
return nil, RetrieveBlobError
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := blob.Close(); err != nil {
|
||||
log.WithFields(logFields).Error("failed to close layer blob reader")
|
||||
}
|
||||
}()
|
||||
|
||||
files := append(featurefmt.RequiredFilenames(toScan), featurens.RequiredFilenames(toScan)...)
|
||||
fileMap, err := imagefmt.Extract(blobFormat, blob, files)
|
||||
if err != nil {
|
||||
log.WithFields(logFields).WithError(err).Error("failed to extract layer blob")
|
||||
return nil, ExtractBlobError
|
||||
}
|
||||
|
||||
newLayerScanResult.Features, err = featurefmt.ListFeatures(fileMap, toScan)
|
||||
if err != nil {
|
||||
log.WithFields(logFields).WithError(err).Error("failed to detect features")
|
||||
return nil, FeatureDetectorError
|
||||
}
|
||||
|
||||
newLayerScanResult.Namespaces, err = featurens.Detect(fileMap, toScan)
|
||||
if err != nil {
|
||||
log.WithFields(logFields).WithError(err).Error("failed to detect namespaces")
|
||||
return nil, NamespaceDetectorError
|
||||
}
|
||||
|
||||
if err = saveLayerChange(store, newLayerScanResult); err != nil {
|
||||
log.WithFields(logFields).WithError(err).Error("failed to store layer change")
|
||||
return nil, StorageError
|
||||
}
|
||||
|
||||
layer = database.MergeLayers(layer, newLayerScanResult)
|
||||
} else {
|
||||
log.WithFields(logFields).Debug("found scanned layer blob")
|
||||
}
|
||||
|
||||
return layer, nil
|
||||
}
|
||||
|
||||
// EnabledDetectors retrieves a list of all detectors installed in the Clair
|
||||
// instance.
|
||||
func EnabledDetectors() []database.Detector {
|
||||
return append(featurefmt.ListListers(), featurens.ListDetectors()...)
|
||||
}
|
||||
|
||||
// RegisterConfiguredDetectors populates the database with registered detectors.
|
||||
func RegisterConfiguredDetectors(store database.Datastore) {
|
||||
if err := database.PersistDetectorsAndCommit(store, EnabledDetectors()); err != nil {
|
||||
panic("failed to initialize Clair analyzer")
|
||||
}
|
||||
}
|
||||
|
||||
func saveLayerChange(store database.Datastore, layer *database.Layer) error {
|
||||
if err := database.PersistFeaturesAndCommit(store, layer.GetFeatures()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := database.PersistNamespacesAndCommit(store, layer.GetNamespaces()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := database.PersistPartialLayerAndCommit(store, layer); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,290 @@
|
||||
// Copyright 2019 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package clair
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/coreos/clair/database"
|
||||
)
|
||||
|
||||
type layerIndexedFeature struct {
|
||||
feature *database.LayerFeature
|
||||
namespace *layerIndexedNamespace
|
||||
introducedIn int
|
||||
}
|
||||
|
||||
type layerIndexedNamespace struct {
|
||||
namespace database.LayerNamespace
|
||||
introducedIn int
|
||||
}
|
||||
|
||||
// AncestryBuilder builds an Ancestry, which contains an ordered list of layers
|
||||
// and their features.
|
||||
type AncestryBuilder struct {
|
||||
layerIndex int
|
||||
layerNames []string
|
||||
detectors []database.Detector
|
||||
namespaces map[database.Detector]*layerIndexedNamespace
|
||||
features map[database.Detector][]layerIndexedFeature
|
||||
}
|
||||
|
||||
// NewAncestryBuilder creates a new ancestry builder.
|
||||
//
|
||||
// ancestry builder takes in the extracted layer information and produce a set of
|
||||
// namespaces, features, and the relation between features for the whole image.
|
||||
func NewAncestryBuilder(detectors []database.Detector) *AncestryBuilder {
|
||||
return &AncestryBuilder{
|
||||
layerIndex: 0,
|
||||
detectors: detectors,
|
||||
namespaces: make(map[database.Detector]*layerIndexedNamespace),
|
||||
features: make(map[database.Detector][]layerIndexedFeature),
|
||||
}
|
||||
}
|
||||
|
||||
// AddLeafLayer adds a leaf layer to the ancestry builder, and computes the
|
||||
// namespaced features.
|
||||
func (b *AncestryBuilder) AddLeafLayer(layer *database.Layer) {
|
||||
b.layerNames = append(b.layerNames, layer.Hash)
|
||||
for i := range layer.Namespaces {
|
||||
b.updateNamespace(&layer.Namespaces[i])
|
||||
}
|
||||
|
||||
allFeatureMap := map[database.Detector][]database.LayerFeature{}
|
||||
for i := range layer.Features {
|
||||
layerFeature := layer.Features[i]
|
||||
allFeatureMap[layerFeature.By] = append(allFeatureMap[layerFeature.By], layerFeature)
|
||||
}
|
||||
|
||||
// we only care about the ones specified by builder's detectors
|
||||
featureMap := map[database.Detector][]database.LayerFeature{}
|
||||
for i := range b.detectors {
|
||||
detector := b.detectors[i]
|
||||
featureMap[detector] = allFeatureMap[detector]
|
||||
}
|
||||
|
||||
for detector := range featureMap {
|
||||
b.addLayerFeatures(detector, featureMap[detector])
|
||||
}
|
||||
|
||||
b.layerIndex++
|
||||
}
|
||||
|
||||
// Every detector inspects a set of files for the features
|
||||
// therefore, if that set of files gives a different set of features, it
|
||||
// should replace the existing features.
|
||||
func (b *AncestryBuilder) addLayerFeatures(detector database.Detector, features []database.LayerFeature) {
|
||||
if len(features) == 0 {
|
||||
// TODO(sidac): we need to differentiate if the detector finds that all
|
||||
// features are removed ( a file change ), or the package installer is
|
||||
// removed ( a file deletion ), or there's no change in the file ( file
|
||||
// does not exist in the blob ) Right now, we're just assuming that no
|
||||
// change in the file because that's the most common case.
|
||||
return
|
||||
}
|
||||
|
||||
existingFeatures := b.features[detector]
|
||||
currentFeatures := make([]layerIndexedFeature, 0, len(features))
|
||||
// Features that are not in the current layer should be removed.
|
||||
for i := range existingFeatures {
|
||||
feature := existingFeatures[i]
|
||||
for j := range features {
|
||||
if features[j] == *feature.feature {
|
||||
currentFeatures = append(currentFeatures, feature)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Features that newly introduced in the current layer should be added.
|
||||
for i := range features {
|
||||
found := false
|
||||
for j := range existingFeatures {
|
||||
if *existingFeatures[j].feature == features[i] {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
namespace, found := b.lookupNamespace(&features[i])
|
||||
if !found {
|
||||
log.WithField("Layer Hashes", b.layerNames).Error("skip, could not find the proper namespace for feature")
|
||||
continue
|
||||
}
|
||||
|
||||
currentFeatures = append(currentFeatures, b.createLayerIndexedFeature(namespace, &features[i]))
|
||||
}
|
||||
}
|
||||
|
||||
b.features[detector] = currentFeatures
|
||||
}
|
||||
|
||||
// updateNamespace update the namespaces for the ancestry. It does the following things:
|
||||
// 1. when a detector detects a new namespace, it's added to the ancestry.
|
||||
// 2. when a detector detects a difference in the detected namespace, it
|
||||
// replaces the namespace, and also move all features under that namespace to
|
||||
// the new namespace.
|
||||
func (b *AncestryBuilder) updateNamespace(layerNamespace *database.LayerNamespace) {
|
||||
var (
|
||||
previous *layerIndexedNamespace
|
||||
ok bool
|
||||
)
|
||||
|
||||
if previous, ok = b.namespaces[layerNamespace.By]; !ok {
|
||||
b.namespaces[layerNamespace.By] = &layerIndexedNamespace{
|
||||
namespace: *layerNamespace,
|
||||
introducedIn: b.layerIndex,
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// All features referencing to this namespace are now pointing to the new namespace.
|
||||
// Also those features are now treated as introduced in the same layer as
|
||||
// when this new namespace is introduced.
|
||||
previous.namespace = *layerNamespace
|
||||
previous.introducedIn = b.layerIndex
|
||||
|
||||
for _, features := range b.features {
|
||||
for i, feature := range features {
|
||||
if feature.namespace == previous {
|
||||
features[i].introducedIn = previous.introducedIn
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *AncestryBuilder) createLayerIndexedFeature(namespace *layerIndexedNamespace, feature *database.LayerFeature) layerIndexedFeature {
|
||||
return layerIndexedFeature{
|
||||
feature: feature,
|
||||
namespace: namespace,
|
||||
introducedIn: b.layerIndex,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *AncestryBuilder) lookupNamespace(feature *database.LayerFeature) (*layerIndexedNamespace, bool) {
|
||||
for _, namespace := range b.namespaces {
|
||||
if namespace.namespace.VersionFormat == feature.VersionFormat {
|
||||
return namespace, true
|
||||
}
|
||||
}
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (b *AncestryBuilder) ancestryFeatures(index int) []database.AncestryFeature {
|
||||
ancestryFeatures := []database.AncestryFeature{}
|
||||
for detector, features := range b.features {
|
||||
for _, feature := range features {
|
||||
if feature.introducedIn == index {
|
||||
ancestryFeatures = append(ancestryFeatures, database.AncestryFeature{
|
||||
NamespacedFeature: database.NamespacedFeature{
|
||||
Feature: feature.feature.Feature,
|
||||
Namespace: feature.namespace.namespace.Namespace,
|
||||
},
|
||||
FeatureBy: detector,
|
||||
NamespaceBy: feature.namespace.namespace.By,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ancestryFeatures
|
||||
}
|
||||
|
||||
func (b *AncestryBuilder) ancestryLayers() []database.AncestryLayer {
|
||||
layers := make([]database.AncestryLayer, 0, b.layerIndex)
|
||||
for i := 0; i < b.layerIndex; i++ {
|
||||
layers = append(layers, database.AncestryLayer{
|
||||
Hash: b.layerNames[i],
|
||||
Features: b.ancestryFeatures(i),
|
||||
})
|
||||
}
|
||||
|
||||
return layers
|
||||
}
|
||||
|
||||
// Ancestry produces an Ancestry from the builder.
|
||||
func (b *AncestryBuilder) Ancestry(name string) *database.Ancestry {
|
||||
if name == "" {
|
||||
// TODO(sidac): we'll use the computed ancestry name in the future.
|
||||
// During the transition, it still requires the user to use the correct
|
||||
// ancestry name.
|
||||
name = ancestryName(b.layerNames)
|
||||
log.WithField("ancestry.Name", name).Warn("generated ancestry name since it's not specified")
|
||||
}
|
||||
|
||||
return &database.Ancestry{
|
||||
Name: name,
|
||||
By: b.detectors,
|
||||
Layers: b.ancestryLayers(),
|
||||
}
|
||||
}
|
||||
|
||||
// SaveAncestry saves an ancestry to the datastore.
|
||||
func SaveAncestry(store database.Datastore, ancestry *database.Ancestry) error {
|
||||
log.WithField("ancestry.Name", ancestry.Name).Debug("saving ancestry")
|
||||
features := []database.NamespacedFeature{}
|
||||
for _, layer := range ancestry.Layers {
|
||||
features = append(features, layer.GetFeatures()...)
|
||||
}
|
||||
|
||||
if err := database.PersistNamespacedFeaturesAndCommit(store, features); err != nil {
|
||||
return StorageError
|
||||
}
|
||||
|
||||
if err := database.UpsertAncestryAndCommit(store, ancestry); err != nil {
|
||||
return StorageError
|
||||
}
|
||||
|
||||
if err := database.CacheRelatedVulnerabilityAndCommit(store, features); err != nil {
|
||||
return StorageError
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsAncestryCached checks if the ancestry is already cached in the database with the current set of detectors.
|
||||
func IsAncestryCached(store database.Datastore, name string, layerHashes []string) (bool, error) {
|
||||
if name == "" {
|
||||
// TODO(sidac): we'll use the computed ancestry name in the future.
|
||||
// During the transition, it still requires the user to use the correct
|
||||
// ancestry name.
|
||||
name = ancestryName(layerHashes)
|
||||
log.WithField("ancestry.Name", name).Warn("generated ancestry name since it's not specified")
|
||||
}
|
||||
|
||||
ancestry, found, err := database.FindAncestryAndRollback(store, name)
|
||||
if err != nil {
|
||||
log.WithError(err).WithField("ancestry.Name", name).Error("failed to query ancestry in database")
|
||||
return false, StorageError
|
||||
}
|
||||
|
||||
if found {
|
||||
log.WithField("ancestry.Name", name).Debug("found cached ancestry")
|
||||
}
|
||||
|
||||
return found && len(database.DiffDetectors(EnabledDetectors(), ancestry.By)) == 0, nil
|
||||
}
|
||||
|
||||
func ancestryName(layerHashes []string) string {
|
||||
tag := sha256.Sum256([]byte(strings.Join(layerHashes, ",")))
|
||||
return hex.EncodeToString(tag[:])
|
||||
}
|
||||
@ -0,0 +1,267 @@
|
||||
// Copyright 2019 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package clair
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/coreos/clair/database"
|
||||
)
|
||||
|
||||
var (
|
||||
dpkg = database.NewFeatureDetector("dpkg", "1.0")
|
||||
rpm = database.NewFeatureDetector("rpm", "1.0")
|
||||
pip = database.NewFeatureDetector("pip", "1.0")
|
||||
python = database.NewNamespaceDetector("python", "1.0")
|
||||
osrelease = database.NewNamespaceDetector("os-release", "1.0")
|
||||
ubuntu = *database.NewNamespace("ubuntu:14.04", "dpkg")
|
||||
ubuntu16 = *database.NewNamespace("ubuntu:16.04", "dpkg")
|
||||
python2 = *database.NewNamespace("python:2", "pip")
|
||||
sed = *database.NewSourcePackage("sed", "4.4-2", "dpkg")
|
||||
sedBin = *database.NewBinaryPackage("sed", "4.4-2", "dpkg")
|
||||
tar = *database.NewBinaryPackage("tar", "1.29b-2", "dpkg")
|
||||
scipy = *database.NewSourcePackage("scipy", "3.0.0", "pip")
|
||||
|
||||
detectors = []database.Detector{dpkg, osrelease, rpm}
|
||||
multinamespaceDetectors = []database.Detector{dpkg, osrelease, pip}
|
||||
)
|
||||
|
||||
// layerBuilder is for helping constructing the layer test artifacts.
|
||||
type layerBuilder struct {
|
||||
layer *database.Layer
|
||||
}
|
||||
|
||||
func newLayerBuilder(hash string) *layerBuilder {
|
||||
return &layerBuilder{&database.Layer{Hash: hash, By: detectors}}
|
||||
}
|
||||
|
||||
func (b *layerBuilder) addNamespace(detector database.Detector, ns database.Namespace) *layerBuilder {
|
||||
b.layer.Namespaces = append(b.layer.Namespaces, database.LayerNamespace{
|
||||
Namespace: ns,
|
||||
By: detector,
|
||||
})
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *layerBuilder) addFeature(detector database.Detector, f database.Feature) *layerBuilder {
|
||||
b.layer.Features = append(b.layer.Features, database.LayerFeature{
|
||||
Feature: f,
|
||||
By: detector,
|
||||
})
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
var testImage = []*database.Layer{
|
||||
// empty layer
|
||||
newLayerBuilder("0").layer,
|
||||
// ubuntu namespace
|
||||
newLayerBuilder("1").addNamespace(osrelease, ubuntu).layer,
|
||||
// install sed
|
||||
newLayerBuilder("2").addFeature(dpkg, sed).layer,
|
||||
// install tar
|
||||
newLayerBuilder("3").addFeature(dpkg, sed).addFeature(dpkg, tar).layer,
|
||||
// remove tar
|
||||
newLayerBuilder("4").addFeature(dpkg, sed).layer,
|
||||
// upgrade ubuntu
|
||||
newLayerBuilder("5").addNamespace(osrelease, ubuntu16).layer,
|
||||
// no change to the detectable files
|
||||
newLayerBuilder("6").layer,
|
||||
// change to the package installer database but no features are affected.
|
||||
newLayerBuilder("7").addFeature(dpkg, sed).layer,
|
||||
}
|
||||
|
||||
var clairLimit = []*database.Layer{
|
||||
// TODO(sidac): how about install rpm package under ubuntu?
|
||||
newLayerBuilder("1").addNamespace(osrelease, ubuntu).layer,
|
||||
newLayerBuilder("2").addFeature(rpm, sed).layer,
|
||||
}
|
||||
|
||||
var multipleNamespace = []*database.Layer{
|
||||
// TODO(sidac): support for multiple namespaces
|
||||
}
|
||||
|
||||
var invalidNamespace = []*database.Layer{
|
||||
// add package without namespace, this indicates that the namespace detector
|
||||
// could not detect the namespace.
|
||||
newLayerBuilder("0").addFeature(dpkg, sed).layer,
|
||||
}
|
||||
|
||||
var multiplePackagesOnFirstLayer = []*database.Layer{
|
||||
newLayerBuilder("0").addFeature(dpkg, sed).addFeature(dpkg, tar).addFeature(dpkg, sedBin).addNamespace(osrelease, ubuntu16).layer,
|
||||
}
|
||||
|
||||
func TestAddLayer(t *testing.T) {
|
||||
cases := []struct {
|
||||
title string
|
||||
image []*database.Layer
|
||||
|
||||
expectedAncestry database.Ancestry
|
||||
}{
|
||||
{
|
||||
title: "empty image",
|
||||
expectedAncestry: database.Ancestry{Name: ancestryName([]string{}), By: detectors},
|
||||
},
|
||||
{
|
||||
title: "empty layer",
|
||||
image: testImage[:1],
|
||||
expectedAncestry: database.Ancestry{Name: ancestryName([]string{"0"}), By: detectors, Layers: []database.AncestryLayer{{Hash: "0"}}},
|
||||
},
|
||||
{
|
||||
title: "ubuntu",
|
||||
image: testImage[:2],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "ubuntu install sed",
|
||||
image: testImage[:3],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1", "2"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
}}},
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "ubuntu install tar",
|
||||
image: testImage[:4],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1", "2", "3"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
}}, {
|
||||
Hash: "3", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: tar, Namespace: ubuntu},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
}, {
|
||||
title: "ubuntu uninstall tar",
|
||||
image: testImage[:5],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1", "2", "3", "4"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
}}, {Hash: "3"}, {Hash: "4"}},
|
||||
},
|
||||
}, {
|
||||
title: "ubuntu upgrade",
|
||||
image: testImage[:6],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1", "2", "3", "4", "5"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2"}, {Hash: "3"}, {Hash: "4"}, {Hash: "5", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
}}},
|
||||
},
|
||||
},
|
||||
}, {
|
||||
title: "no change to the detectable files",
|
||||
image: testImage[:7],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1", "2", "3", "4", "5", "6"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2"}, {Hash: "3"}, {Hash: "4"}, {Hash: "5", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
}}}, {Hash: "6"}},
|
||||
},
|
||||
}, {
|
||||
title: "change to the package installer database but no features are affected.",
|
||||
image: testImage[:8],
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0", "1", "2", "3", "4", "5", "6", "7"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2"}, {Hash: "3"}, {Hash: "4"}, {Hash: "5", Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
}}}, {Hash: "6"}, {Hash: "7"}},
|
||||
},
|
||||
}, {
|
||||
title: "layers with features and namespace.",
|
||||
image: multiplePackagesOnFirstLayer,
|
||||
expectedAncestry: database.Ancestry{
|
||||
Name: ancestryName([]string{"0"}),
|
||||
By: detectors,
|
||||
Layers: []database.AncestryLayer{
|
||||
{
|
||||
Hash: "0",
|
||||
Features: []database.AncestryFeature{
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: sedBin, Namespace: ubuntu16},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
{
|
||||
NamespacedFeature: database.NamespacedFeature{Feature: tar, Namespace: ubuntu16},
|
||||
FeatureBy: dpkg,
|
||||
NamespaceBy: osrelease,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range cases {
|
||||
t.Run(test.title, func(t *testing.T) {
|
||||
builder := NewAncestryBuilder(detectors)
|
||||
for _, layer := range test.image {
|
||||
builder.AddLeafLayer(layer)
|
||||
}
|
||||
|
||||
ancestry := builder.Ancestry("")
|
||||
require.True(t, database.AssertAncestryEqual(t, &test.expectedAncestry, ancestry))
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1,61 +1,43 @@
|
||||
// Copyright 2019 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package clair
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/coreos/clair/pkg/httputil"
|
||||
)
|
||||
|
||||
func retrieveLayerBlob(ctx context.Context, blobSha256 string, path string, headers map[string]string) (io.ReadCloser, error) {
|
||||
func retrieveLayerBlob(ctx context.Context, path string, headers map[string]string) (io.ReadCloser, error) {
|
||||
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
|
||||
return downloadLayerBlob(ctx, blobSha256, path, headers)
|
||||
}
|
||||
|
||||
return loadLayerBlobFromFS(blobSha256)
|
||||
}
|
||||
|
||||
func downloadLayerBlob(ctx context.Context, blobSha256 string, uri string, headers map[string]string) (io.ReadCloser, error) {
|
||||
request, err := http.NewRequest("GET", uri, nil)
|
||||
if err != nil {
|
||||
return nil, RetrieveBlobError
|
||||
}
|
||||
|
||||
if headers != nil {
|
||||
for k, v := range headers {
|
||||
request.Header.Set(k, v)
|
||||
httpHeaders := make(http.Header)
|
||||
for key, value := range headers {
|
||||
httpHeaders[key] = []string{value}
|
||||
}
|
||||
}
|
||||
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{},
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
}
|
||||
|
||||
client := &http.Client{Transport: tr}
|
||||
r, err := client.Do(request)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("could not download layer")
|
||||
return nil, RetrieveBlobError
|
||||
}
|
||||
reader, err := httputil.GetWithContext(ctx, path, httpHeaders)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Fail if we don't receive a 2xx HTTP status code.
|
||||
if is2xx(r.StatusCode) {
|
||||
log.WithField("status", r.StatusCode).Error("could not download layer: expected 2XX")
|
||||
return nil, RetrieveBlobError
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
return r.Body, nil
|
||||
}
|
||||
|
||||
func is2xx(statusCode int) bool {
|
||||
return statusCode >= 200 && statusCode < 300
|
||||
}
|
||||
|
||||
func loadLayerBlobFromFS(path string) (io.ReadCloser, error) {
|
||||
return os.Open(path)
|
||||
}
|
||||
|
||||
@ -1,478 +0,0 @@
|
||||
// Copyright 2018 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package clair
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
|
||||
"github.com/deckarep/golang-set"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/coreos/clair/database"
|
||||
"github.com/coreos/clair/ext/featurefmt"
|
||||
"github.com/coreos/clair/ext/featurens"
|
||||
"github.com/coreos/clair/ext/imagefmt"
|
||||
"github.com/coreos/clair/pkg/commonerr"
|
||||
"github.com/coreos/clair/pkg/strutil"
|
||||
"github.com/coreos/clair/pkg/tarutil"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrUnsupported is the error that should be raised when an OS or package
|
||||
// manager is not supported.
|
||||
ErrUnsupported = commonerr.NewBadRequestError("worker: OS and/or package manager are not supported")
|
||||
|
||||
// EnabledDetectors are detectors to be used to scan the layers.
|
||||
EnabledDetectors []database.Detector
|
||||
)
|
||||
|
||||
// LayerRequest represents all information necessary to download and process a
|
||||
// layer.
|
||||
type LayerRequest struct {
|
||||
Hash string
|
||||
Path string
|
||||
Headers map[string]string
|
||||
}
|
||||
|
||||
type processResult struct {
|
||||
existingLayer *database.Layer
|
||||
newLayerContent *database.Layer
|
||||
err error
|
||||
}
|
||||
|
||||
// processRequest stores parameters used for processing a layer.
|
||||
type processRequest struct {
|
||||
LayerRequest
|
||||
|
||||
existingLayer *database.Layer
|
||||
detectors []database.Detector
|
||||
}
|
||||
|
||||
type introducedFeature struct {
|
||||
feature database.AncestryFeature
|
||||
layerIndex int
|
||||
}
|
||||
|
||||
// processRequests in parallel processes a set of requests for unique set of layers
|
||||
// and returns sets of unique namespaces, features and layers to be inserted
|
||||
// into the database.
|
||||
func processRequests(imageFormat string, toDetect map[string]*processRequest) (map[string]*processResult, error) {
|
||||
wg := &sync.WaitGroup{}
|
||||
wg.Add(len(toDetect))
|
||||
|
||||
results := map[string]*processResult{}
|
||||
for i := range toDetect {
|
||||
results[i] = nil
|
||||
}
|
||||
|
||||
for i := range toDetect {
|
||||
result := processResult{}
|
||||
results[i] = &result
|
||||
go func(req *processRequest, res *processResult) {
|
||||
*res = *detectContent(imageFormat, req)
|
||||
wg.Done()
|
||||
}(toDetect[i], &result)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
errs := []error{}
|
||||
for _, r := range results {
|
||||
errs = append(errs, r.err)
|
||||
}
|
||||
|
||||
if err := commonerr.CombineErrors(errs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func getProcessRequest(datastore database.Datastore, req LayerRequest) (preq *processRequest, err error) {
|
||||
layer, ok, err := database.FindLayerAndRollback(datastore, req.Hash)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if !ok {
|
||||
log.WithField("layer", req.Hash).Debug("found no existing layer in database")
|
||||
preq = &processRequest{
|
||||
LayerRequest: req,
|
||||
existingLayer: &database.Layer{Hash: req.Hash},
|
||||
detectors: EnabledDetectors,
|
||||
}
|
||||
} else {
|
||||
log.WithFields(log.Fields{
|
||||
"layer": layer.Hash,
|
||||
"detectors": layer.By,
|
||||
"feature count": len(layer.Features),
|
||||
"namespace count": len(layer.Namespaces),
|
||||
}).Debug("found existing layer in database")
|
||||
|
||||
preq = &processRequest{
|
||||
LayerRequest: req,
|
||||
existingLayer: &layer,
|
||||
detectors: database.DiffDetectors(EnabledDetectors, layer.By),
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func persistProcessResult(datastore database.Datastore, results map[string]*processResult) error {
|
||||
features := []database.Feature{}
|
||||
namespaces := []database.Namespace{}
|
||||
for _, r := range results {
|
||||
features = append(features, r.newLayerContent.GetFeatures()...)
|
||||
namespaces = append(namespaces, r.newLayerContent.GetNamespaces()...)
|
||||
}
|
||||
|
||||
features = database.DeduplicateFeatures(features...)
|
||||
namespaces = database.DeduplicateNamespaces(namespaces...)
|
||||
if err := database.PersistNamespacesAndCommit(datastore, namespaces); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := database.PersistFeaturesAndCommit(datastore, features); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, layer := range results {
|
||||
if err := database.PersistPartialLayerAndCommit(datastore, layer.newLayerContent); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// processLayers processes a set of post layer requests, stores layers and
|
||||
// returns an ordered list of processed layers with detected features and
|
||||
// namespaces.
|
||||
func processLayers(datastore database.Datastore, imageFormat string, requests []LayerRequest) ([]database.Layer, error) {
|
||||
var (
|
||||
reqMap = make(map[string]*processRequest)
|
||||
err error
|
||||
)
|
||||
|
||||
for _, r := range requests {
|
||||
reqMap[r.Hash], err = getProcessRequest(datastore, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
results, err := processRequests(imageFormat, reqMap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := persistProcessResult(datastore, results); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
completeLayers := getProcessResultLayers(results)
|
||||
layers := make([]database.Layer, 0, len(requests))
|
||||
for _, r := range requests {
|
||||
layers = append(layers, completeLayers[r.Hash])
|
||||
}
|
||||
|
||||
return layers, nil
|
||||
}
|
||||
|
||||
func getProcessResultLayers(results map[string]*processResult) map[string]database.Layer {
|
||||
layers := map[string]database.Layer{}
|
||||
for name, r := range results {
|
||||
layers[name] = *database.MergeLayers(r.existingLayer, r.newLayerContent)
|
||||
}
|
||||
|
||||
return layers
|
||||
}
|
||||
|
||||
func isAncestryProcessed(datastore database.Datastore, name string) (bool, error) {
|
||||
ancestry, ok, err := database.FindAncestryAndRollback(datastore, name)
|
||||
if err != nil || !ok {
|
||||
return ok, err
|
||||
}
|
||||
|
||||
return len(database.DiffDetectors(EnabledDetectors, ancestry.By)) == 0, nil
|
||||
}
|
||||
|
||||
// ProcessAncestry downloads and scans an ancestry if it's not scanned by all
|
||||
// enabled processors in this instance of Clair.
|
||||
func ProcessAncestry(datastore database.Datastore, imageFormat, name string, layerRequest []LayerRequest) error {
|
||||
var (
|
||||
err error
|
||||
ok bool
|
||||
layers []database.Layer
|
||||
)
|
||||
|
||||
if name == "" {
|
||||
return commonerr.NewBadRequestError("could not process a layer which does not have a name")
|
||||
}
|
||||
|
||||
if imageFormat == "" {
|
||||
return commonerr.NewBadRequestError("could not process a layer which does not have a format")
|
||||
}
|
||||
|
||||
log.WithField("ancestry", name).Debug("start processing ancestry...")
|
||||
if ok, err = isAncestryProcessed(datastore, name); err != nil {
|
||||
log.WithError(err).Error("could not determine if ancestry is processed")
|
||||
return err
|
||||
} else if ok {
|
||||
log.WithField("ancestry", name).Debug("ancestry is already processed")
|
||||
return nil
|
||||
}
|
||||
|
||||
if layers, err = processLayers(datastore, imageFormat, layerRequest); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return processAncestry(datastore, name, layers)
|
||||
}
|
||||
|
||||
func processAncestry(datastore database.Datastore, name string, layers []database.Layer) error {
|
||||
var (
|
||||
ancestry = database.Ancestry{Name: name}
|
||||
err error
|
||||
)
|
||||
|
||||
ancestry.Layers, ancestry.By, err = computeAncestryLayers(layers)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ancestryFeatures := database.GetAncestryFeatures(ancestry)
|
||||
log.WithFields(log.Fields{
|
||||
"ancestry": name,
|
||||
"processed by": EnabledDetectors,
|
||||
"features count": len(ancestryFeatures),
|
||||
"layer count": len(ancestry.Layers),
|
||||
}).Debug("compute ancestry features")
|
||||
|
||||
if err := database.PersistNamespacedFeaturesAndCommit(datastore, ancestryFeatures); err != nil {
|
||||
log.WithField("ancestry", name).WithError(err).Error("could not persist namespaced features for ancestry")
|
||||
return err
|
||||
}
|
||||
|
||||
if err := database.CacheRelatedVulnerabilityAndCommit(datastore, ancestryFeatures); err != nil {
|
||||
log.WithField("ancestry", name).WithError(err).Error("failed to cache feature related vulnerability")
|
||||
return err
|
||||
}
|
||||
|
||||
if err := database.UpsertAncestryAndCommit(datastore, ancestry); err != nil {
|
||||
log.WithField("ancestry", name).WithError(err).Error("could not upsert ancestry")
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getCommonDetectors(layers []database.Layer) mapset.Set {
|
||||
// find the common detector for all layers and filter the namespaces and
|
||||
// features based on that.
|
||||
commonDetectors := mapset.NewSet()
|
||||
for _, d := range layers[0].By {
|
||||
commonDetectors.Add(d)
|
||||
}
|
||||
|
||||
for _, l := range layers {
|
||||
detectors := mapset.NewSet()
|
||||
for _, d := range l.By {
|
||||
detectors.Add(d)
|
||||
}
|
||||
|
||||
commonDetectors = commonDetectors.Intersect(detectors)
|
||||
}
|
||||
|
||||
return commonDetectors
|
||||
}
|
||||
|
||||
// computeAncestryLayers computes ancestry's layers along with what features are
|
||||
// introduced.
|
||||
func computeAncestryLayers(layers []database.Layer) ([]database.AncestryLayer, []database.Detector, error) {
|
||||
if len(layers) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
commonDetectors := getCommonDetectors(layers)
|
||||
// version format -> namespace
|
||||
namespaces := map[string]database.LayerNamespace{}
|
||||
// version format -> feature ID -> feature
|
||||
features := map[string]map[string]introducedFeature{}
|
||||
ancestryLayers := []database.AncestryLayer{}
|
||||
for index, layer := range layers {
|
||||
initializedLayer := database.AncestryLayer{Hash: layer.Hash}
|
||||
ancestryLayers = append(ancestryLayers, initializedLayer)
|
||||
|
||||
// Precondition: namespaces and features contain the result from union
|
||||
// of all parents.
|
||||
for _, ns := range layer.Namespaces {
|
||||
if !commonDetectors.Contains(ns.By) {
|
||||
continue
|
||||
}
|
||||
|
||||
namespaces[ns.VersionFormat] = ns
|
||||
}
|
||||
|
||||
// version format -> feature ID -> feature
|
||||
currentFeatures := map[string]map[string]introducedFeature{}
|
||||
for _, f := range layer.Features {
|
||||
if !commonDetectors.Contains(f.By) {
|
||||
continue
|
||||
}
|
||||
|
||||
if ns, ok := namespaces[f.VersionFormat]; ok {
|
||||
var currentMap map[string]introducedFeature
|
||||
if currentMap, ok = currentFeatures[f.VersionFormat]; !ok {
|
||||
currentFeatures[f.VersionFormat] = make(map[string]introducedFeature)
|
||||
currentMap = currentFeatures[f.VersionFormat]
|
||||
}
|
||||
|
||||
inherited := false
|
||||
if mapF, ok := features[f.VersionFormat]; ok {
|
||||
if parentFeature, ok := mapF[f.Name+":"+f.Version]; ok {
|
||||
currentMap[f.Name+":"+f.Version] = parentFeature
|
||||
inherited = true
|
||||
}
|
||||
}
|
||||
|
||||
if !inherited {
|
||||
currentMap[f.Name+":"+f.Version] = introducedFeature{
|
||||
feature: database.AncestryFeature{
|
||||
NamespacedFeature: database.NamespacedFeature{
|
||||
Feature: f.Feature,
|
||||
Namespace: ns.Namespace,
|
||||
},
|
||||
NamespaceBy: ns.By,
|
||||
FeatureBy: f.By,
|
||||
},
|
||||
layerIndex: index,
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
return nil, nil, errors.New("No corresponding version format")
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE(Sida): we update the feature map in some version format
|
||||
// only if there's at least one feature with that version format. This
|
||||
// approach won't differentiate feature file removed vs all detectable
|
||||
// features removed from that file vs feature file not changed.
|
||||
//
|
||||
// One way to differentiate (feature file removed or not changed) vs
|
||||
// all detectable features removed is to pass in the file status.
|
||||
for vf, mapF := range currentFeatures {
|
||||
features[vf] = mapF
|
||||
}
|
||||
}
|
||||
|
||||
for _, featureMap := range features {
|
||||
for _, feature := range featureMap {
|
||||
ancestryLayers[feature.layerIndex].Features = append(
|
||||
ancestryLayers[feature.layerIndex].Features,
|
||||
feature.feature,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
detectors := make([]database.Detector, 0, commonDetectors.Cardinality())
|
||||
for d := range commonDetectors.Iter() {
|
||||
detectors = append(detectors, d.(database.Detector))
|
||||
}
|
||||
|
||||
return ancestryLayers, detectors, nil
|
||||
}
|
||||
|
||||
func extractRequiredFiles(imageFormat string, req *processRequest) (tarutil.FilesMap, error) {
|
||||
requiredFiles := append(featurefmt.RequiredFilenames(req.detectors), featurens.RequiredFilenames(req.detectors)...)
|
||||
if len(requiredFiles) == 0 {
|
||||
log.WithFields(log.Fields{
|
||||
"layer": req.Hash,
|
||||
"detectors": req.detectors,
|
||||
}).Info("layer requires no file to extract")
|
||||
return make(tarutil.FilesMap), nil
|
||||
}
|
||||
|
||||
files, err := imagefmt.Extract(imageFormat, req.Path, req.Headers, requiredFiles)
|
||||
if err != nil {
|
||||
log.WithError(err).WithFields(log.Fields{
|
||||
"layer": req.Hash,
|
||||
"path": strutil.CleanURL(req.Path),
|
||||
}).Error("failed to extract data from path")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return files, err
|
||||
}
|
||||
|
||||
// detectContent downloads a layer and detects all features and namespaces.
|
||||
func detectContent(imageFormat string, req *processRequest) (res *processResult) {
|
||||
var (
|
||||
files tarutil.FilesMap
|
||||
layer = database.Layer{Hash: req.Hash, By: req.detectors}
|
||||
)
|
||||
|
||||
res = &processResult{req.existingLayer, &layer, nil}
|
||||
log.WithFields(log.Fields{
|
||||
"layer": req.Hash,
|
||||
"detectors": req.detectors,
|
||||
}).Info("detecting layer content...")
|
||||
|
||||
files, res.err = extractRequiredFiles(imageFormat, req)
|
||||
if res.err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if layer.Namespaces, res.err = featurens.Detect(files, req.detectors); res.err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if layer.Features, res.err = featurefmt.ListFeatures(files, req.detectors); res.err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"layer": req.Hash,
|
||||
"detectors": req.detectors,
|
||||
"namespace count": len(layer.Namespaces),
|
||||
"feature count": len(layer.Features),
|
||||
}).Info("processed layer")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// InitWorker initializes the worker.
|
||||
func InitWorker(datastore database.Datastore) {
|
||||
if len(EnabledDetectors) == 0 {
|
||||
log.Warn("no enabled detector, and therefore, no ancestry will be processed.")
|
||||
return
|
||||
}
|
||||
|
||||
tx, err := datastore.Begin()
|
||||
if err != nil {
|
||||
log.WithError(err).Fatal("cannot connect to database to initialize worker")
|
||||
}
|
||||
|
||||
defer tx.Rollback()
|
||||
if err := tx.PersistDetectors(EnabledDetectors); err != nil {
|
||||
log.WithError(err).Fatal("cannot insert detectors to initialize worker")
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
log.WithError(err).Fatal("cannot commit detector changes to initialize worker")
|
||||
}
|
||||
}
|
||||
@ -1,587 +0,0 @@
|
||||
// Copyright 2017 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package clair
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/coreos/clair/database"
|
||||
"github.com/coreos/clair/ext/featurefmt"
|
||||
"github.com/coreos/clair/ext/featurens"
|
||||
"github.com/coreos/clair/ext/versionfmt/dpkg"
|
||||
|
||||
// Register the required detectors.
|
||||
_ "github.com/coreos/clair/ext/featurefmt/dpkg"
|
||||
_ "github.com/coreos/clair/ext/featurefmt/rpm"
|
||||
_ "github.com/coreos/clair/ext/featurens/aptsources"
|
||||
_ "github.com/coreos/clair/ext/featurens/osrelease"
|
||||
_ "github.com/coreos/clair/ext/imagefmt/docker"
|
||||
)
|
||||
|
||||
type mockDatastore struct {
|
||||
database.MockDatastore
|
||||
|
||||
layers map[string]database.Layer
|
||||
ancestry map[string]database.Ancestry
|
||||
namespaces map[string]database.Namespace
|
||||
features map[string]database.Feature
|
||||
namespacedFeatures map[string]database.NamespacedFeature
|
||||
}
|
||||
|
||||
type mockSession struct {
|
||||
database.MockSession
|
||||
|
||||
store *mockDatastore
|
||||
copy mockDatastore
|
||||
terminated bool
|
||||
}
|
||||
|
||||
func copyDatastore(md *mockDatastore) mockDatastore {
|
||||
layers := map[string]database.Layer{}
|
||||
for k, l := range md.layers {
|
||||
layers[k] = database.Layer{
|
||||
Hash: l.Hash,
|
||||
By: append([]database.Detector{}, l.By...),
|
||||
Features: append([]database.LayerFeature{}, l.Features...),
|
||||
Namespaces: append([]database.LayerNamespace{}, l.Namespaces...),
|
||||
}
|
||||
}
|
||||
|
||||
ancestry := map[string]database.Ancestry{}
|
||||
for k, a := range md.ancestry {
|
||||
ancestryLayers := []database.AncestryLayer{}
|
||||
for _, layer := range a.Layers {
|
||||
ancestryLayers = append(ancestryLayers, database.AncestryLayer{
|
||||
Hash: layer.Hash,
|
||||
Features: append([]database.AncestryFeature{}, layer.Features...),
|
||||
})
|
||||
}
|
||||
|
||||
ancestry[k] = database.Ancestry{
|
||||
Name: a.Name,
|
||||
By: append([]database.Detector{}, a.By...),
|
||||
Layers: ancestryLayers,
|
||||
}
|
||||
}
|
||||
|
||||
namespaces := map[string]database.Namespace{}
|
||||
for k, n := range md.namespaces {
|
||||
namespaces[k] = n
|
||||
}
|
||||
|
||||
features := map[string]database.Feature{}
|
||||
for k, f := range md.features {
|
||||
features[k] = f
|
||||
}
|
||||
|
||||
namespacedFeatures := map[string]database.NamespacedFeature{}
|
||||
for k, f := range md.namespacedFeatures {
|
||||
namespacedFeatures[k] = f
|
||||
}
|
||||
|
||||
return mockDatastore{
|
||||
layers: layers,
|
||||
ancestry: ancestry,
|
||||
namespaces: namespaces,
|
||||
namespacedFeatures: namespacedFeatures,
|
||||
features: features,
|
||||
}
|
||||
}
|
||||
|
||||
func newMockDatastore() *mockDatastore {
|
||||
errSessionDone := errors.New("Session Done")
|
||||
md := &mockDatastore{
|
||||
layers: make(map[string]database.Layer),
|
||||
ancestry: make(map[string]database.Ancestry),
|
||||
namespaces: make(map[string]database.Namespace),
|
||||
features: make(map[string]database.Feature),
|
||||
namespacedFeatures: make(map[string]database.NamespacedFeature),
|
||||
}
|
||||
|
||||
md.FctBegin = func() (database.Session, error) {
|
||||
session := &mockSession{
|
||||
store: md,
|
||||
copy: copyDatastore(md),
|
||||
terminated: false,
|
||||
}
|
||||
|
||||
session.FctCommit = func() error {
|
||||
if session.terminated {
|
||||
return nil
|
||||
}
|
||||
session.store.layers = session.copy.layers
|
||||
session.store.ancestry = session.copy.ancestry
|
||||
session.store.namespaces = session.copy.namespaces
|
||||
session.store.features = session.copy.features
|
||||
session.store.namespacedFeatures = session.copy.namespacedFeatures
|
||||
session.terminated = true
|
||||
return nil
|
||||
}
|
||||
|
||||
session.FctRollback = func() error {
|
||||
if session.terminated {
|
||||
return nil
|
||||
}
|
||||
session.terminated = true
|
||||
session.copy = mockDatastore{}
|
||||
return nil
|
||||
}
|
||||
|
||||
session.FctFindAncestry = func(name string) (database.Ancestry, bool, error) {
|
||||
if session.terminated {
|
||||
return database.Ancestry{}, false, errSessionDone
|
||||
}
|
||||
ancestry, ok := session.copy.ancestry[name]
|
||||
return ancestry, ok, nil
|
||||
}
|
||||
|
||||
session.FctFindLayer = func(name string) (database.Layer, bool, error) {
|
||||
if session.terminated {
|
||||
return database.Layer{}, false, errSessionDone
|
||||
}
|
||||
layer, ok := session.copy.layers[name]
|
||||
return layer, ok, nil
|
||||
}
|
||||
|
||||
session.FctPersistNamespaces = func(ns []database.Namespace) error {
|
||||
if session.terminated {
|
||||
return errSessionDone
|
||||
}
|
||||
for _, n := range ns {
|
||||
session.copy.namespaces[NamespaceKey(&n)] = n
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
session.FctPersistFeatures = func(fs []database.Feature) error {
|
||||
if session.terminated {
|
||||
return errSessionDone
|
||||
}
|
||||
for _, f := range fs {
|
||||
session.copy.features[FeatureKey(&f)] = f
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
session.FctPersistLayer = func(hash string, features []database.LayerFeature, namespaces []database.LayerNamespace, by []database.Detector) error {
|
||||
if session.terminated {
|
||||
return errSessionDone
|
||||
}
|
||||
|
||||
for _, ns := range namespaces {
|
||||
if _, ok := session.copy.namespaces[NamespaceKey(&ns.Namespace)]; !ok {
|
||||
panic("")
|
||||
}
|
||||
}
|
||||
|
||||
for _, f := range features {
|
||||
if _, ok := session.copy.features[FeatureKey(&f.Feature)]; !ok {
|
||||
panic("")
|
||||
}
|
||||
}
|
||||
|
||||
layer, _ := session.copy.layers[hash]
|
||||
database.MergeLayers(&layer, &database.Layer{
|
||||
Hash: hash,
|
||||
By: by,
|
||||
Namespaces: namespaces,
|
||||
Features: features,
|
||||
})
|
||||
|
||||
session.copy.layers[hash] = layer
|
||||
return nil
|
||||
}
|
||||
|
||||
session.FctUpsertAncestry = func(ancestry database.Ancestry) error {
|
||||
if session.terminated {
|
||||
return errSessionDone
|
||||
}
|
||||
|
||||
// ensure the namespaces features are in the code base
|
||||
for _, l := range ancestry.Layers {
|
||||
for _, f := range l.GetFeatures() {
|
||||
if _, ok := session.copy.namespacedFeatures[NamespacedFeatureKey(&f)]; !ok {
|
||||
panic("")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
session.copy.ancestry[ancestry.Name] = ancestry
|
||||
return nil
|
||||
}
|
||||
|
||||
session.FctPersistNamespacedFeatures = func(namespacedFeatures []database.NamespacedFeature) error {
|
||||
for i, f := range namespacedFeatures {
|
||||
if _, ok := session.copy.features[FeatureKey(&f.Feature)]; !ok {
|
||||
panic("")
|
||||
}
|
||||
|
||||
if _, ok := session.copy.namespaces[NamespaceKey(&f.Namespace)]; !ok {
|
||||
panic("")
|
||||
}
|
||||
|
||||
session.copy.namespacedFeatures[NamespacedFeatureKey(&f)] = namespacedFeatures[i]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
session.FctCacheAffectedNamespacedFeatures = func(namespacedFeatures []database.NamespacedFeature) error {
|
||||
// The function does nothing because we don't care about the vulnerability cache in worker_test.
|
||||
return nil
|
||||
}
|
||||
|
||||
return session, nil
|
||||
}
|
||||
return md
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
EnabledDetectors = append(featurefmt.ListListers(), featurens.ListDetectors()...)
|
||||
m.Run()
|
||||
}
|
||||
|
||||
func FeatureKey(f *database.Feature) string {
|
||||
return strings.Join([]string{f.Name, f.VersionFormat, f.Version}, "__")
|
||||
}
|
||||
|
||||
func NamespaceKey(ns *database.Namespace) string {
|
||||
return strings.Join([]string{ns.Name, ns.VersionFormat}, "__")
|
||||
}
|
||||
|
||||
func NamespacedFeatureKey(f *database.NamespacedFeature) string {
|
||||
return strings.Join([]string{f.Name, f.Namespace.Name}, "__")
|
||||
}
|
||||
|
||||
func TestProcessAncestryWithDistUpgrade(t *testing.T) {
|
||||
// TODO(sidac): Change to use table driven tests.
|
||||
// Create the list of Features that should not been upgraded from one layer to another.
|
||||
nonUpgradedFeatures := []database.Feature{
|
||||
{Name: "libtext-wrapi18n-perl", Version: "0.06-7"},
|
||||
{Name: "libtext-charwidth-perl", Version: "0.04-7"},
|
||||
{Name: "libtext-iconv-perl", Version: "1.7-5"},
|
||||
{Name: "mawk", Version: "1.3.3-17"},
|
||||
{Name: "insserv", Version: "1.14.0-5"},
|
||||
{Name: "db", Version: "5.1.29-5"},
|
||||
{Name: "ustr", Version: "1.0.4-3"},
|
||||
{Name: "xz-utils", Version: "5.1.1alpha+20120614-2"},
|
||||
{Name: "libdb5.1", Version: "5.1.29-5"},
|
||||
}
|
||||
|
||||
nonUpgradedMap := map[database.Feature]struct{}{}
|
||||
for _, f := range nonUpgradedFeatures {
|
||||
f.VersionFormat = "dpkg"
|
||||
f.Type = database.SourcePackage
|
||||
nonUpgradedMap[f] = struct{}{}
|
||||
f.Type = database.BinaryPackage
|
||||
nonUpgradedMap[f] = struct{}{}
|
||||
}
|
||||
|
||||
// Process test layers.
|
||||
//
|
||||
// blank.tar: MAINTAINER Quentin MACHU <quentin.machu.fr>
|
||||
// wheezy.tar: FROM debian:wheezy
|
||||
// jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update &&
|
||||
// apt-get -y dist-upgrade
|
||||
_, f, _, _ := runtime.Caller(0)
|
||||
testDataPath := filepath.Join(filepath.Dir(f)) + "/testdata/DistUpgrade/"
|
||||
|
||||
datastore := newMockDatastore()
|
||||
|
||||
layers := []LayerRequest{
|
||||
{Hash: "blank", Path: testDataPath + "blank.tar.gz"},
|
||||
{Hash: "wheezy", Path: testDataPath + "wheezy.tar.gz"},
|
||||
{Hash: "jessie", Path: testDataPath + "jessie.tar.gz"},
|
||||
}
|
||||
|
||||
assert.Nil(t, ProcessAncestry(datastore, "Docker", "Mock", layers))
|
||||
|
||||
// check the ancestry features
|
||||
features := []database.AncestryFeature{}
|
||||
for i, l := range datastore.ancestry["Mock"].Layers {
|
||||
assert.Equal(t, layers[i].Hash, l.Hash)
|
||||
features = append(features, l.Features...)
|
||||
}
|
||||
|
||||
assert.Len(t, features, 161)
|
||||
for _, f := range features {
|
||||
if _, ok := nonUpgradedMap[f.Feature]; ok {
|
||||
assert.Equal(t, "debian:7", f.Namespace.Name, "%#v", f)
|
||||
} else {
|
||||
assert.Equal(t, "debian:8", f.Namespace.Name, "#%v", f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessLayers(t *testing.T) {
|
||||
_, f, _, _ := runtime.Caller(0)
|
||||
testDataPath := filepath.Join(filepath.Dir(f)) + "/testdata/DistUpgrade/"
|
||||
|
||||
datastore := newMockDatastore()
|
||||
|
||||
layers := []LayerRequest{
|
||||
{Hash: "blank", Path: testDataPath + "blank.tar.gz"},
|
||||
{Hash: "wheezy", Path: testDataPath + "wheezy.tar.gz"},
|
||||
{Hash: "jessie", Path: testDataPath + "jessie.tar.gz"},
|
||||
}
|
||||
|
||||
LayerWithContents, err := processLayers(datastore, "Docker", layers)
|
||||
assert.Nil(t, err)
|
||||
assert.Len(t, LayerWithContents, 3)
|
||||
// ensure resubmit won't break the stuff
|
||||
LayerWithContents, err = processLayers(datastore, "Docker", layers)
|
||||
assert.Nil(t, err)
|
||||
assert.Len(t, LayerWithContents, 3)
|
||||
// Ensure each processed layer is correct
|
||||
assert.Len(t, LayerWithContents[0].Namespaces, 0)
|
||||
assert.Len(t, LayerWithContents[1].Namespaces, 1)
|
||||
assert.Len(t, LayerWithContents[2].Namespaces, 1)
|
||||
assert.Len(t, LayerWithContents[0].Features, 0)
|
||||
assert.Len(t, LayerWithContents[1].Features, 132)
|
||||
assert.Len(t, LayerWithContents[2].Features, 191)
|
||||
|
||||
// Ensure each layer has expected namespaces and features detected
|
||||
if blank, ok := datastore.layers["blank"]; ok {
|
||||
database.AssertDetectorsEqual(t, EnabledDetectors, blank.By)
|
||||
assert.Len(t, blank.Namespaces, 0)
|
||||
assert.Len(t, blank.Features, 0)
|
||||
} else {
|
||||
assert.Fail(t, "blank is not stored")
|
||||
return
|
||||
}
|
||||
|
||||
if wheezy, ok := datastore.layers["wheezy"]; ok {
|
||||
database.AssertDetectorsEqual(t, EnabledDetectors, wheezy.By)
|
||||
assert.Equal(t, []database.LayerNamespace{
|
||||
{database.Namespace{"debian:7", dpkg.ParserName}, database.NewNamespaceDetector("os-release", "1.0")},
|
||||
}, wheezy.Namespaces)
|
||||
|
||||
assert.Len(t, wheezy.Features, 132)
|
||||
} else {
|
||||
assert.Fail(t, "wheezy is not stored")
|
||||
return
|
||||
}
|
||||
|
||||
if jessie, ok := datastore.layers["jessie"]; ok {
|
||||
database.AssertDetectorsEqual(t, EnabledDetectors, jessie.By)
|
||||
assert.Equal(t, []database.LayerNamespace{
|
||||
{database.Namespace{"debian:8", dpkg.ParserName}, database.NewNamespaceDetector("os-release", "1.0")},
|
||||
}, jessie.Namespaces)
|
||||
assert.Len(t, jessie.Features, 191)
|
||||
} else {
|
||||
assert.Fail(t, "jessie is not stored")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func getFeatures(a database.Ancestry) []database.AncestryFeature {
|
||||
features := []database.AncestryFeature{}
|
||||
for _, l := range a.Layers {
|
||||
features = append(features, l.Features...)
|
||||
}
|
||||
|
||||
return features
|
||||
}
|
||||
|
||||
func TestComputeAncestryFeatures(t *testing.T) {
|
||||
vf1 := "format 1"
|
||||
vf2 := "format 2"
|
||||
|
||||
nd1 := database.NewNamespaceDetector("apk", "1.0")
|
||||
fd1 := database.NewFeatureDetector("fd1", "1.0")
|
||||
// this detector only scans one layer with one extra feature, this one
|
||||
// should be omitted.
|
||||
fd2 := database.NewFeatureDetector("fd2", "1.0")
|
||||
|
||||
ns1a := database.LayerNamespace{
|
||||
database.Namespace{
|
||||
Name: "namespace 1:a",
|
||||
VersionFormat: vf1,
|
||||
}, nd1,
|
||||
}
|
||||
|
||||
ns1b := database.LayerNamespace{
|
||||
database.Namespace{
|
||||
Name: "namespace 1:b",
|
||||
VersionFormat: vf1,
|
||||
}, nd1}
|
||||
|
||||
ns2a := database.LayerNamespace{
|
||||
database.Namespace{
|
||||
Name: "namespace 2:a",
|
||||
VersionFormat: vf2,
|
||||
}, nd1}
|
||||
|
||||
ns2b := database.LayerNamespace{
|
||||
database.Namespace{
|
||||
Name: "namespace 2:b",
|
||||
VersionFormat: vf2,
|
||||
}, nd1}
|
||||
|
||||
f1 := database.LayerFeature{
|
||||
database.Feature{
|
||||
Name: "feature 1",
|
||||
Version: "0.1",
|
||||
VersionFormat: vf1,
|
||||
}, fd1}
|
||||
|
||||
f2 := database.LayerFeature{database.Feature{
|
||||
Name: "feature 2",
|
||||
Version: "0.2",
|
||||
VersionFormat: vf1,
|
||||
}, fd2}
|
||||
|
||||
f3 := database.LayerFeature{
|
||||
database.Feature{
|
||||
Name: "feature 1",
|
||||
Version: "0.3",
|
||||
VersionFormat: vf2,
|
||||
}, fd1}
|
||||
|
||||
f4 := database.LayerFeature{
|
||||
database.Feature{
|
||||
Name: "feature 2",
|
||||
Version: "0.3",
|
||||
VersionFormat: vf2,
|
||||
}, fd1}
|
||||
|
||||
f5 := database.LayerFeature{
|
||||
database.Feature{
|
||||
Name: "feature 3",
|
||||
Version: "0.3",
|
||||
VersionFormat: vf2,
|
||||
},
|
||||
fd2,
|
||||
}
|
||||
|
||||
// Suppose Clair is watching two files for namespaces one containing ns1
|
||||
// changes e.g. os-release and the other one containing ns2 changes e.g.
|
||||
// node.
|
||||
blank := database.Layer{
|
||||
Hash: "blank",
|
||||
By: []database.Detector{nd1, fd1, fd1},
|
||||
}
|
||||
initNS1a := database.Layer{
|
||||
Hash: "initNS1a",
|
||||
By: []database.Detector{nd1, fd1, fd1},
|
||||
Namespaces: []database.LayerNamespace{ns1a},
|
||||
Features: []database.LayerFeature{f1, f2},
|
||||
}
|
||||
|
||||
upgradeNS2b := database.Layer{
|
||||
Hash: "upgradeNS2b",
|
||||
By: []database.Detector{nd1, fd1, fd1},
|
||||
Namespaces: []database.LayerNamespace{ns2b},
|
||||
}
|
||||
|
||||
upgradeNS1b := database.Layer{
|
||||
Hash: "upgradeNS1b",
|
||||
By: []database.Detector{nd1, fd1, fd1, fd2},
|
||||
Namespaces: []database.LayerNamespace{ns1b},
|
||||
Features: []database.LayerFeature{f1, f2, f5},
|
||||
}
|
||||
|
||||
initNS2a := database.Layer{
|
||||
Hash: "initNS2a",
|
||||
By: []database.Detector{nd1, fd1, fd1},
|
||||
Namespaces: []database.LayerNamespace{ns2a},
|
||||
Features: []database.LayerFeature{f3, f4},
|
||||
}
|
||||
|
||||
removeF2 := database.Layer{
|
||||
Hash: "removeF2",
|
||||
By: []database.Detector{nd1, fd1, fd1},
|
||||
Features: []database.LayerFeature{f1},
|
||||
}
|
||||
|
||||
// blank -> ns1:a, f1 f2 (init)
|
||||
// -> f1 (feature change)
|
||||
// -> ns2:a, f3, f4 (init ns2a)
|
||||
// -> ns2:b (ns2 upgrade without changing features)
|
||||
// -> blank (empty)
|
||||
// -> ns1:b, f1 f2 (ns1 upgrade and add f2)
|
||||
// -> f1 (remove f2)
|
||||
// -> blank (empty)
|
||||
|
||||
layers := []database.Layer{
|
||||
blank, // empty
|
||||
initNS1a, // namespace: NS1a, features: f1, f2
|
||||
removeF2, // namespace: , features: f1
|
||||
initNS2a, // namespace: NS2a, features: f3, f4 ( under NS2a )
|
||||
upgradeNS2b, // namespace: NS2b, ( f3, f4 are now under NS2b )
|
||||
blank, // empty
|
||||
upgradeNS1b, // namespace: NS1b, ( f1, f2 are now under NS1b, and they are introduced in this layer. )
|
||||
removeF2, // namespace: , features: f1
|
||||
blank,
|
||||
}
|
||||
|
||||
expected := []database.AncestryLayer{
|
||||
{
|
||||
"blank",
|
||||
[]database.AncestryFeature{},
|
||||
},
|
||||
{
|
||||
"initNS1a",
|
||||
[]database.AncestryFeature{{database.NamespacedFeature{f1.Feature, ns1a.Namespace}, f1.By, ns1a.By}},
|
||||
},
|
||||
{
|
||||
"removeF2",
|
||||
[]database.AncestryFeature{},
|
||||
},
|
||||
{
|
||||
"initNS2a",
|
||||
[]database.AncestryFeature{
|
||||
{database.NamespacedFeature{f3.Feature, ns2a.Namespace}, f3.By, ns2a.By},
|
||||
{database.NamespacedFeature{f4.Feature, ns2a.Namespace}, f4.By, ns2a.By},
|
||||
},
|
||||
},
|
||||
{
|
||||
"upgradeNS2b",
|
||||
[]database.AncestryFeature{},
|
||||
},
|
||||
{
|
||||
"blank",
|
||||
[]database.AncestryFeature{},
|
||||
},
|
||||
{
|
||||
"upgradeNS1b",
|
||||
[]database.AncestryFeature{},
|
||||
},
|
||||
{
|
||||
"removeF2",
|
||||
[]database.AncestryFeature{},
|
||||
},
|
||||
{
|
||||
"blank",
|
||||
[]database.AncestryFeature{},
|
||||
},
|
||||
}
|
||||
|
||||
expectedDetectors := []database.Detector{nd1, fd1}
|
||||
ancestryLayers, detectors, err := computeAncestryLayers(layers)
|
||||
require.Nil(t, err)
|
||||
|
||||
database.AssertDetectorsEqual(t, expectedDetectors, detectors)
|
||||
for i := range expected {
|
||||
database.AssertAncestryLayerEqual(t, &expected[i], &ancestryLayers[i])
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue