clair: Use builder pattern for constructing ancestry

- Add Ancestry builder
- Change RPC to use the ancestry builder

analyzer.go

@@ -0,0 +1,145 @@
+// Copyright 2019 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clair
+
+import (
+	"context"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/ext/featurefmt"
+	"github.com/coreos/clair/ext/featurens"
+	"github.com/coreos/clair/ext/imagefmt"
+)
+
+// AnalyzeError represents an failure when analyzing layer or constructing
+// ancestry.
+type AnalyzeError string
+
+func (e AnalyzeError) Error() string {
+	return string(e)
+}
+
+var (
+	// StorageError represents an analyze error caused by the storage
+	StorageError = AnalyzeError("failed to query the database.")
+	// RetrieveBlobError represents an analyze error caused by failure of
+	// downloading or extracting layer blobs.
+	RetrieveBlobError = AnalyzeError("failed to download layer blob.")
+	// ExtractBlobError represents an analyzer error caused by failure of
+	// extracting a layer blob by imagefmt.
+	ExtractBlobError = AnalyzeError("failed to extract files from layer blob.")
+	// FeatureDetectorError is an error caused by failure of feature listing by
+	// featurefmt.
+	FeatureDetectorError = AnalyzeError("failed to scan feature from layer blob files.")
+	// NamespaceDetectorError is an error caused by failure of namespace
+	// detection by featurens.
+	NamespaceDetectorError = AnalyzeError("failed to scan namespace from layer blob files.")
+)
+
+// AnalyzeLayer retrieves the clair layer with all extracted features and namespaces.
+// If a layer is already scanned by all enabled detectors in the Clair instance, it returns directly.
+// Otherwise, it re-download the layer blob and scan the features and namespaced again.
+func AnalyzeLayer(ctx context.Context, store database.Datastore, blobSha256 string, blobFormat string, downloadURI string, downloadHeaders map[string]string) (*database.Layer, error) {
+	layer, found, err := database.FindLayerAndRollback(store, blobSha256)
+	logFields := log.Fields{"layer.Hash": blobSha256}
+	if err != nil {
+		log.WithError(err).WithFields(logFields).Error("failed to find layer in the storage")
+		return nil, StorageError
+	}
+
+	var scannedBy []database.Detector
+	if found {
+		scannedBy = layer.By
+	}
+
+	// layer will be scanned by detectors not scanned the layer already.
+	toScan := database.DiffDetectors(EnabledDetectors(), scannedBy)
+	if len(toScan) != 0 {
+		log.WithFields(logFields).Debug("scan layer blob not already scanned")
+		newLayerScanResult := &database.Layer{Hash: blobSha256, By: toScan}
+		blob, err := retrieveLayerBlob(ctx, downloadURI, downloadHeaders)
+		if err != nil {
+			log.WithError(err).WithFields(logFields).Error("failed to retrieve layer blob")
+			return nil, RetrieveBlobError
+		}
+
+		defer func() {
+			if err := blob.Close(); err != nil {
+				log.WithFields(logFields).Error("failed to close layer blob reader")
+			}
+		}()
+
+		files := append(featurefmt.RequiredFilenames(toScan), featurens.RequiredFilenames(toScan)...)
+		fileMap, err := imagefmt.Extract(blobFormat, blob, files)
+		if err != nil {
+			log.WithFields(logFields).WithError(err).Error("failed to extract layer blob")
+			return nil, ExtractBlobError
+		}
+
+		newLayerScanResult.Features, err = featurefmt.ListFeatures(fileMap, toScan)
+		if err != nil {
+			log.WithFields(logFields).WithError(err).Error("failed to detect features")
+			return nil, FeatureDetectorError
+		}
+
+		newLayerScanResult.Namespaces, err = featurens.Detect(fileMap, toScan)
+		if err != nil {
+			log.WithFields(logFields).WithError(err).Error("failed to detect namespaces")
+			return nil, NamespaceDetectorError
+		}
+
+		if err = saveLayerChange(store, newLayerScanResult); err != nil {
+			log.WithFields(logFields).WithError(err).Error("failed to store layer change")
+			return nil, StorageError
+		}
+
+		layer = database.MergeLayers(layer, newLayerScanResult)
+	} else {
+		log.WithFields(logFields).Debug("found scanned layer blob")
+	}
+
+	return layer, nil
+}
+
+// EnabledDetectors retrieves a list of all detectors installed in the Clair
+// instance.
+func EnabledDetectors() []database.Detector {
+	return append(featurefmt.ListListers(), featurens.ListDetectors()...)
+}
+
+// RegisterConfiguredDetectors populates the database with registered detectors.
+func RegisterConfiguredDetectors(store database.Datastore) {
+	if err := database.PersistDetectorsAndCommit(store, EnabledDetectors()); err != nil {
+		panic("failed to initialize Clair analyzer")
+	}
+}
+
+func saveLayerChange(store database.Datastore, layer *database.Layer) error {
+	if err := database.PersistFeaturesAndCommit(store, layer.GetFeatures()); err != nil {
+		return err
+	}
+
+	if err := database.PersistNamespacesAndCommit(store, layer.GetNamespaces()); err != nil {
+		return err
+	}
+
+	if err := database.PersistPartialLayerAndCommit(store, layer); err != nil {
+		return err
+	}
+
+	return nil
+}

ancestry.go

@@ -0,0 +1,290 @@
+// Copyright 2019 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clair
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/coreos/clair/database"
+)
+
+type layerIndexedFeature struct {
+	feature      *database.LayerFeature
+	namespace    *layerIndexedNamespace
+	introducedIn int
+}
+
+type layerIndexedNamespace struct {
+	namespace    database.LayerNamespace
+	introducedIn int
+}
+
+// AncestryBuilder builds an Ancestry, which contains an ordered list of layers
+// and their features.
+type AncestryBuilder struct {
+	layerIndex int
+	layerNames []string
+	detectors  []database.Detector
+	namespaces map[database.Detector]*layerIndexedNamespace
+	features   map[database.Detector][]layerIndexedFeature
+}
+
+// NewAncestryBuilder creates a new ancestry builder.
+//
+// ancestry builder takes in the extracted layer information and produce a set of
+// namespaces, features, and the relation between features for the whole image.
+func NewAncestryBuilder(detectors []database.Detector) *AncestryBuilder {
+	return &AncestryBuilder{
+		layerIndex: 0,
+		detectors:  detectors,
+		namespaces: make(map[database.Detector]*layerIndexedNamespace),
+		features:   make(map[database.Detector][]layerIndexedFeature),
+	}
+}
+
+// AddLeafLayer adds a leaf layer to the ancestry builder, and computes the
+// namespaced features.
+func (b *AncestryBuilder) AddLeafLayer(layer *database.Layer) {
+	b.layerNames = append(b.layerNames, layer.Hash)
+	for i := range layer.Namespaces {
+		b.updateNamespace(&layer.Namespaces[i])
+	}
+
+	allFeatureMap := map[database.Detector][]database.LayerFeature{}
+	for i := range layer.Features {
+		layerFeature := layer.Features[i]
+		allFeatureMap[layerFeature.By] = append(allFeatureMap[layerFeature.By], layerFeature)
+	}
+
+	// we only care about the ones specified by builder's detectors
+	featureMap := map[database.Detector][]database.LayerFeature{}
+	for i := range b.detectors {
+		detector := b.detectors[i]
+		featureMap[detector] = allFeatureMap[detector]
+	}
+
+	for detector := range featureMap {
+		b.addLayerFeatures(detector, featureMap[detector])
+	}
+
+	b.layerIndex++
+}
+
+// Every detector inspects a set of files for the features
+// therefore, if that set of files gives a different set of features, it
+// should replace the existing features.
+func (b *AncestryBuilder) addLayerFeatures(detector database.Detector, features []database.LayerFeature) {
+	if len(features) == 0 {
+		// TODO(sidac): we need to differentiate if the detector finds that all
+		// features are removed ( a file change ), or the package installer is
+		// removed ( a file deletion ), or there's no change in the file ( file
+		// does not exist in the blob ) Right now, we're just assuming that no
+		// change in the file because that's the most common case.
+		return
+	}
+
+	existingFeatures := b.features[detector]
+	currentFeatures := make([]layerIndexedFeature, 0, len(features))
+	// Features that are not in the current layer should be removed.
+	for i := range existingFeatures {
+		feature := existingFeatures[i]
+		for j := range features {
+			if features[j] == *feature.feature {
+				currentFeatures = append(currentFeatures, feature)
+				break
+			}
+		}
+	}
+
+	// Features that newly introduced in the current layer should be added.
+	for i := range features {
+		found := false
+		for j := range existingFeatures {
+			if *existingFeatures[j].feature == features[i] {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			namespace, found := b.lookupNamespace(&features[i])
+			if !found {
+				log.WithField("Layer Hashes", b.layerNames).Error("skip, could not find the proper namespace for feature")
+				continue
+			}
+
+			currentFeatures = append(currentFeatures, b.createLayerIndexedFeature(namespace, &features[i]))
+		}
+	}
+
+	b.features[detector] = currentFeatures
+}
+
+// updateNamespace update the namespaces for the ancestry. It does the following things:
+// 1. when a detector detects a new namespace, it's added to the ancestry.
+// 2. when a detector detects a difference in the detected namespace, it
+// replaces the namespace, and also move all features under that namespace to
+// the new namespace.
+func (b *AncestryBuilder) updateNamespace(layerNamespace *database.LayerNamespace) {
+	var (
+		previous *layerIndexedNamespace
+		ok       bool
+	)
+
+	if previous, ok = b.namespaces[layerNamespace.By]; !ok {
+		b.namespaces[layerNamespace.By] = &layerIndexedNamespace{
+			namespace:    *layerNamespace,
+			introducedIn: b.layerIndex,
+		}
+
+		return
+	}
+
+	// All features referencing to this namespace are now pointing to the new namespace.
+	// Also those features are now treated as introduced in the same layer as
+	// when this new namespace is introduced.
+	previous.namespace = *layerNamespace
+	previous.introducedIn = b.layerIndex
+
+	for _, features := range b.features {
+		for i, feature := range features {
+			if feature.namespace == previous {
+				features[i].introducedIn = previous.introducedIn
+			}
+		}
+	}
+}
+
+func (b *AncestryBuilder) createLayerIndexedFeature(namespace *layerIndexedNamespace, feature *database.LayerFeature) layerIndexedFeature {
+	return layerIndexedFeature{
+		feature:      feature,
+		namespace:    namespace,
+		introducedIn: b.layerIndex,
+	}
+}
+
+func (b *AncestryBuilder) lookupNamespace(feature *database.LayerFeature) (*layerIndexedNamespace, bool) {
+	for _, namespace := range b.namespaces {
+		if namespace.namespace.VersionFormat == feature.VersionFormat {
+			return namespace, true
+		}
+	}
+
+	return nil, false
+}
+
+func (b *AncestryBuilder) ancestryFeatures(index int) []database.AncestryFeature {
+	ancestryFeatures := []database.AncestryFeature{}
+	for detector, features := range b.features {
+		for _, feature := range features {
+			if feature.introducedIn == index {
+				ancestryFeatures = append(ancestryFeatures, database.AncestryFeature{
+					NamespacedFeature: database.NamespacedFeature{
+						Feature:   feature.feature.Feature,
+						Namespace: feature.namespace.namespace.Namespace,
+					},
+					FeatureBy:   detector,
+					NamespaceBy: feature.namespace.namespace.By,
+				})
+			}
+		}
+	}
+
+	return ancestryFeatures
+}
+
+func (b *AncestryBuilder) ancestryLayers() []database.AncestryLayer {
+	layers := make([]database.AncestryLayer, 0, b.layerIndex)
+	for i := 0; i < b.layerIndex; i++ {
+		layers = append(layers, database.AncestryLayer{
+			Hash:     b.layerNames[i],
+			Features: b.ancestryFeatures(i),
+		})
+	}
+
+	return layers
+}
+
+// Ancestry produces an Ancestry from the builder.
+func (b *AncestryBuilder) Ancestry(name string) *database.Ancestry {
+	if name == "" {
+		// TODO(sidac): we'll use the computed ancestry name in the future.
+		// During the transition, it still requires the user to use the correct
+		// ancestry name.
+		name = ancestryName(b.layerNames)
+		log.WithField("ancestry.Name", name).Warn("generated ancestry name since it's not specified")
+	}
+
+	return &database.Ancestry{
+		Name:   name,
+		By:     b.detectors,
+		Layers: b.ancestryLayers(),
+	}
+}
+
+// SaveAncestry saves an ancestry to the datastore.
+func SaveAncestry(store database.Datastore, ancestry *database.Ancestry) error {
+	log.WithField("ancestry.Name", ancestry.Name).Debug("saving ancestry")
+	features := []database.NamespacedFeature{}
+	for _, layer := range ancestry.Layers {
+		features = append(features, layer.GetFeatures()...)
+	}
+
+	if err := database.PersistNamespacedFeaturesAndCommit(store, features); err != nil {
+		return StorageError
+	}
+
+	if err := database.UpsertAncestryAndCommit(store, ancestry); err != nil {
+		return StorageError
+	}
+
+	if err := database.CacheRelatedVulnerabilityAndCommit(store, features); err != nil {
+		return StorageError
+	}
+
+	return nil
+}
+
+// IsAncestryCached checks if the ancestry is already cached in the database with the current set of detectors.
+func IsAncestryCached(store database.Datastore, name string, layerHashes []string) (bool, error) {
+	if name == "" {
+		// TODO(sidac): we'll use the computed ancestry name in the future.
+		// During the transition, it still requires the user to use the correct
+		// ancestry name.
+		name = ancestryName(layerHashes)
+		log.WithField("ancestry.Name", name).Warn("generated ancestry name since it's not specified")
+	}
+
+	ancestry, found, err := database.FindAncestryAndRollback(store, name)
+	if err != nil {
+		log.WithError(err).WithField("ancestry.Name", name).Error("failed to query ancestry in database")
+		return false, StorageError
+	}
+
+	if found {
+		log.WithField("ancestry.Name", name).Debug("found cached ancestry")
+	}
+
+	return found && len(database.DiffDetectors(EnabledDetectors(), ancestry.By)) == 0, nil
+}
+
+func ancestryName(layerHashes []string) string {
+	tag := sha256.Sum256([]byte(strings.Join(layerHashes, ",")))
+	return hex.EncodeToString(tag[:])
+}

ancestry_test.go

@@ -0,0 +1,267 @@
+// Copyright 2019 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clair
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coreos/clair/database"
+)
+
+var (
+	dpkg      = database.NewFeatureDetector("dpkg", "1.0")
+	rpm       = database.NewFeatureDetector("rpm", "1.0")
+	pip       = database.NewFeatureDetector("pip", "1.0")
+	python    = database.NewNamespaceDetector("python", "1.0")
+	osrelease = database.NewNamespaceDetector("os-release", "1.0")
+	ubuntu    = *database.NewNamespace("ubuntu:14.04", "dpkg")
+	ubuntu16  = *database.NewNamespace("ubuntu:16.04", "dpkg")
+	python2   = *database.NewNamespace("python:2", "pip")
+	sed       = *database.NewSourcePackage("sed", "4.4-2", "dpkg")
+	sedBin    = *database.NewBinaryPackage("sed", "4.4-2", "dpkg")
+	tar       = *database.NewBinaryPackage("tar", "1.29b-2", "dpkg")
+	scipy     = *database.NewSourcePackage("scipy", "3.0.0", "pip")
+
+	detectors               = []database.Detector{dpkg, osrelease, rpm}
+	multinamespaceDetectors = []database.Detector{dpkg, osrelease, pip}
+)
+
+// layerBuilder is for helping constructing the layer test artifacts.
+type layerBuilder struct {
+	layer *database.Layer
+}
+
+func newLayerBuilder(hash string) *layerBuilder {
+	return &layerBuilder{&database.Layer{Hash: hash, By: detectors}}
+}
+
+func (b *layerBuilder) addNamespace(detector database.Detector, ns database.Namespace) *layerBuilder {
+	b.layer.Namespaces = append(b.layer.Namespaces, database.LayerNamespace{
+		Namespace: ns,
+		By:        detector,
+	})
+	return b
+}
+
+func (b *layerBuilder) addFeature(detector database.Detector, f database.Feature) *layerBuilder {
+	b.layer.Features = append(b.layer.Features, database.LayerFeature{
+		Feature: f,
+		By:      detector,
+	})
+
+	return b
+}
+
+var testImage = []*database.Layer{
+	// empty layer
+	newLayerBuilder("0").layer,
+	// ubuntu namespace
+	newLayerBuilder("1").addNamespace(osrelease, ubuntu).layer,
+	// install sed
+	newLayerBuilder("2").addFeature(dpkg, sed).layer,
+	// install tar
+	newLayerBuilder("3").addFeature(dpkg, sed).addFeature(dpkg, tar).layer,
+	// remove tar
+	newLayerBuilder("4").addFeature(dpkg, sed).layer,
+	// upgrade ubuntu
+	newLayerBuilder("5").addNamespace(osrelease, ubuntu16).layer,
+	// no change to the detectable files
+	newLayerBuilder("6").layer,
+	// change to the package installer database but no features are affected.
+	newLayerBuilder("7").addFeature(dpkg, sed).layer,
+}
+
+var clairLimit = []*database.Layer{
+	// TODO(sidac): how about install rpm package under ubuntu?
+	newLayerBuilder("1").addNamespace(osrelease, ubuntu).layer,
+	newLayerBuilder("2").addFeature(rpm, sed).layer,
+}
+
+var multipleNamespace = []*database.Layer{
+	// TODO(sidac): support for multiple namespaces
+}
+
+var invalidNamespace = []*database.Layer{
+	// add package without namespace, this indicates that the namespace detector
+	// could not detect the namespace.
+	newLayerBuilder("0").addFeature(dpkg, sed).layer,
+}
+
+var multiplePackagesOnFirstLayer = []*database.Layer{
+	newLayerBuilder("0").addFeature(dpkg, sed).addFeature(dpkg, tar).addFeature(dpkg, sedBin).addNamespace(osrelease, ubuntu16).layer,
+}
+
+func TestAddLayer(t *testing.T) {
+	cases := []struct {
+		title string
+		image []*database.Layer
+
+		expectedAncestry database.Ancestry
+	}{
+		{
+			title:            "empty image",
+			expectedAncestry: database.Ancestry{Name: ancestryName([]string{}), By: detectors},
+		},
+		{
+			title:            "empty layer",
+			image:            testImage[:1],
+			expectedAncestry: database.Ancestry{Name: ancestryName([]string{"0"}), By: detectors, Layers: []database.AncestryLayer{{Hash: "0"}}},
+		},
+		{
+			title: "ubuntu",
+			image: testImage[:2],
+			expectedAncestry: database.Ancestry{
+				Name:   ancestryName([]string{"0", "1"}),
+				By:     detectors,
+				Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}},
+			},
+		},
+		{
+			title: "ubuntu install sed",
+			image: testImage[:3],
+			expectedAncestry: database.Ancestry{
+				Name: ancestryName([]string{"0", "1", "2"}),
+				By:   detectors,
+				Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2", Features: []database.AncestryFeature{
+					{
+						NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu},
+						FeatureBy:         dpkg,
+						NamespaceBy:       osrelease,
+					},
+				}}},
+			},
+		},
+		{
+			title: "ubuntu install tar",
+			image: testImage[:4],
+			expectedAncestry: database.Ancestry{
+				Name: ancestryName([]string{"0", "1", "2", "3"}),
+				By:   detectors,
+				Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2", Features: []database.AncestryFeature{
+					{
+						NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu},
+						FeatureBy:         dpkg,
+						NamespaceBy:       osrelease,
+					},
+				}}, {
+					Hash: "3", Features: []database.AncestryFeature{
+						{
+							NamespacedFeature: database.NamespacedFeature{Feature: tar, Namespace: ubuntu},
+							FeatureBy:         dpkg,
+							NamespaceBy:       osrelease,
+						},
+					},
+				}},
+			},
+		}, {
+			title: "ubuntu uninstall tar",
+			image: testImage[:5],
+			expectedAncestry: database.Ancestry{
+				Name: ancestryName([]string{"0", "1", "2", "3", "4"}),
+				By:   detectors,
+				Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2", Features: []database.AncestryFeature{
+					{
+						NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu},
+						FeatureBy:         dpkg,
+						NamespaceBy:       osrelease,
+					},
+				}}, {Hash: "3"}, {Hash: "4"}},
+			},
+		}, {
+			title: "ubuntu upgrade",
+			image: testImage[:6],
+			expectedAncestry: database.Ancestry{
+				Name: ancestryName([]string{"0", "1", "2", "3", "4", "5"}),
+				By:   detectors,
+				Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2"}, {Hash: "3"}, {Hash: "4"}, {Hash: "5", Features: []database.AncestryFeature{
+					{
+						NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
+						FeatureBy:         dpkg,
+						NamespaceBy:       osrelease,
+					}}},
+				},
+			},
+		}, {
+			title: "no change to the detectable files",
+			image: testImage[:7],
+			expectedAncestry: database.Ancestry{
+				Name: ancestryName([]string{"0", "1", "2", "3", "4", "5", "6"}),
+				By:   detectors,
+				Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2"}, {Hash: "3"}, {Hash: "4"}, {Hash: "5", Features: []database.AncestryFeature{
+					{
+						NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
+						FeatureBy:         dpkg,
+						NamespaceBy:       osrelease,
+					}}}, {Hash: "6"}},
+			},
+		}, {
+			title: "change to the package installer database but no features are affected.",
+			image: testImage[:8],
+			expectedAncestry: database.Ancestry{
+				Name: ancestryName([]string{"0", "1", "2", "3", "4", "5", "6", "7"}),
+				By:   detectors,
+				Layers: []database.AncestryLayer{{Hash: "0"}, {Hash: "1"}, {Hash: "2"}, {Hash: "3"}, {Hash: "4"}, {Hash: "5", Features: []database.AncestryFeature{
+					{
+						NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
+						FeatureBy:         dpkg,
+						NamespaceBy:       osrelease,
+					}}}, {Hash: "6"}, {Hash: "7"}},
+			},
+		}, {
+			title: "layers with features and namespace.",
+			image: multiplePackagesOnFirstLayer,
+			expectedAncestry: database.Ancestry{
+				Name: ancestryName([]string{"0"}),
+				By:   detectors,
+				Layers: []database.AncestryLayer{
+					{
+						Hash: "0",
+						Features: []database.AncestryFeature{
+							{
+								NamespacedFeature: database.NamespacedFeature{Feature: sed, Namespace: ubuntu16},
+								FeatureBy:         dpkg,
+								NamespaceBy:       osrelease,
+							},
+							{
+								NamespacedFeature: database.NamespacedFeature{Feature: sedBin, Namespace: ubuntu16},
+								FeatureBy:         dpkg,
+								NamespaceBy:       osrelease,
+							},
+							{
+								NamespacedFeature: database.NamespacedFeature{Feature: tar, Namespace: ubuntu16},
+								FeatureBy:         dpkg,
+								NamespaceBy:       osrelease,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range cases {
+		t.Run(test.title, func(t *testing.T) {
+			builder := NewAncestryBuilder(detectors)
+			for _, layer := range test.image {
+				builder.AddLeafLayer(layer)
+			}
+
+			ancestry := builder.Ancestry("")
+			require.True(t, database.AssertAncestryEqual(t, &test.expectedAncestry, ancestry))
+		})
+	}
+}

api/v3/rpc.go

@@ -17,6 +17,8 @@ package v3
 import (
 	"fmt"
 
+	"github.com/coreos/clair/ext/imagefmt"
+
 	"golang.org/x/net/context"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -28,6 +30,10 @@ import (
 	"github.com/coreos/clair/pkg/pagination"
 )
 
+func newRPCErrorWithClairError(code codes.Code, err error) error {
+	return status.Errorf(code, "clair error reason: '%s'", err.Error())
+}
+
 // NotificationServer implements NotificationService interface for serving RPC.
 type NotificationServer struct {
 	Store database.Datastore
@@ -55,23 +61,34 @@ func (s *StatusServer) GetStatus(ctx context.Context, req *pb.GetStatusRequest)
 
 // PostAncestry implements posting an ancestry via the Clair gRPC service.
 func (s *AncestryServer) PostAncestry(ctx context.Context, req *pb.PostAncestryRequest) (*pb.PostAncestryResponse, error) {
-	ancestryName := req.GetAncestryName()
-	if ancestryName == "" {
-		return nil, status.Error(codes.InvalidArgument, "ancestry name should not be empty")
+	// validate request
+	blobFormat := req.GetFormat()
+	if !imagefmt.IsSupported(blobFormat) {
+		return nil, status.Error(codes.InvalidArgument, "image blob format is not supported")
 	}
 
-	layers := req.GetLayers()
-	if len(layers) == 0 {
-		return nil, status.Error(codes.InvalidArgument, "ancestry should have at least one layer")
+	clairStatus, err := GetClairStatus(s.Store)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
 	}
 
-	ancestryFormat := req.GetFormat()
-	if ancestryFormat == "" {
-		return nil, status.Error(codes.InvalidArgument, "ancestry format should not be empty")
+	// check if the ancestry is already processed, if not we build the ancestry again.
+	layerHashes := make([]string, len(req.Layers))
+	for i, layer := range req.Layers {
+		layerHashes[i] = layer.GetHash()
 	}
 
-	ancestryLayers := []clair.LayerRequest{}
-	for _, layer := range layers {
+	found, err := clair.IsAncestryCached(s.Store, req.AncestryName, layerHashes)
+	if err != nil {
+		return nil, newRPCErrorWithClairError(codes.Internal, err)
+	}
+
+	if found {
+		return &pb.PostAncestryResponse{Status: clairStatus}, nil
+	}
+
+	builder := clair.NewAncestryBuilder(clair.EnabledDetectors())
+	for _, layer := range req.Layers {
 		if layer == nil {
 			err := status.Error(codes.InvalidArgument, "ancestry layer is invalid")
 			return nil, err
@@ -85,21 +102,22 @@ func (s *AncestryServer) PostAncestry(ctx context.Context, req *pb.PostAncestryR
 			return nil, status.Error(codes.InvalidArgument, "ancestry layer path should not be empty")
 		}
 
-		ancestryLayers = append(ancestryLayers, clair.LayerRequest{
-			Hash:    layer.Hash,
-			Headers: layer.Headers,
-			Path:    layer.Path,
-		})
+		// TODO(sidac): make AnalyzeLayer to be async to ensure
+		// non-blocking downloads.
+		// We'll need to deal with two layers post by the same or different
+		// requests that may have the same hash. In that case, since every
+		// layer/feature/namespace is unique in the database, it may introduce
+		// deadlock.
+		clairLayer, err := clair.AnalyzeLayer(ctx, s.Store, layer.Hash, req.Format, layer.Path, layer.Headers)
+		if err != nil {
+			return nil, newRPCErrorWithClairError(codes.Internal, err)
+		}
+
+		builder.AddLeafLayer(clairLayer)
 	}
 
-	err := clair.ProcessAncestry(s.Store, ancestryFormat, ancestryName, ancestryLayers)
-	if err != nil {
-		return nil, status.Error(codes.Internal, "ancestry is failed to be processed: "+err.Error())
-	}
-
-	clairStatus, err := GetClairStatus(s.Store)
-	if err != nil {
-		return nil, status.Error(codes.Internal, err.Error())
+	if err := clair.SaveAncestry(s.Store, builder.Ancestry(req.AncestryName)); err != nil {
+		return nil, newRPCErrorWithClairError(codes.Internal, err)
 	}
 
 	return &pb.PostAncestryResponse{Status: clairStatus}, nil

api/v3/util.go

@@ -13,7 +13,7 @@ import (
 // protobuf struct.
 func GetClairStatus(store database.Datastore) (*pb.ClairStatus, error) {
 	status := &pb.ClairStatus{
-		Detectors: pb.DetectorsFromDatabaseModel(clair.EnabledDetectors),
+		Detectors: pb.DetectorsFromDatabaseModel(clair.EnabledDetectors()),
 	}
 
 	t, firstUpdate, err := clair.GetLastUpdateTime(store)

blob.go

@@ -1,61 +1,43 @@
+// Copyright 2019 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package clair
 
 import (
 	"context"
-	"crypto/tls"
 	"io"
 	"net/http"
 	"os"
 	"strings"
 
-	log "github.com/sirupsen/logrus"
+	"github.com/coreos/clair/pkg/httputil"
 )
 
-func retrieveLayerBlob(ctx context.Context, blobSha256 string, path string, headers map[string]string) (io.ReadCloser, error) {
+func retrieveLayerBlob(ctx context.Context, path string, headers map[string]string) (io.ReadCloser, error) {
 	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
-		return downloadLayerBlob(ctx, blobSha256, path, headers)
-	}
-
-	return loadLayerBlobFromFS(blobSha256)
-}
-
-func downloadLayerBlob(ctx context.Context, blobSha256 string, uri string, headers map[string]string) (io.ReadCloser, error) {
-	request, err := http.NewRequest("GET", uri, nil)
-	if err != nil {
-		return nil, RetrieveBlobError
-	}
-
-	if headers != nil {
-		for k, v := range headers {
-			request.Header.Set(k, v)
+		httpHeaders := make(http.Header)
+		for key, value := range headers {
+			httpHeaders[key] = []string{value}
 		}
+
+		reader, err := httputil.GetWithContext(ctx, path, httpHeaders)
+		if err != nil {
+			return nil, err
+		}
+
+		return reader, nil
 	}
 
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{},
-		Proxy:           http.ProxyFromEnvironment,
-	}
-
-	client := &http.Client{Transport: tr}
-	r, err := client.Do(request)
-	if err != nil {
-		log.WithError(err).Error("could not download layer")
-		return nil, RetrieveBlobError
-	}
-
-	// Fail if we don't receive a 2xx HTTP status code.
-	if is2xx(r.StatusCode) {
-		log.WithField("status", r.StatusCode).Error("could not download layer: expected 2XX")
-		return nil, RetrieveBlobError
-	}
-
-	return r.Body, nil
-}
-
-func is2xx(statusCode int) bool {
-	return statusCode >= 200 && statusCode < 300
-}
-
-func loadLayerBlobFromFS(path string) (io.ReadCloser, error) {
 	return os.Open(path)
 }

cmd/clair/main.go

@@ -30,9 +30,6 @@ import (
 	"github.com/coreos/clair"
 	"github.com/coreos/clair/api"
 	"github.com/coreos/clair/database"
-	"github.com/coreos/clair/ext/featurefmt"
-	"github.com/coreos/clair/ext/featurens"
-	"github.com/coreos/clair/ext/imagefmt"
 	"github.com/coreos/clair/ext/vulnsrc"
 	"github.com/coreos/clair/pkg/formatter"
 	"github.com/coreos/clair/pkg/stopper"
@@ -103,11 +100,10 @@ func stopCPUProfiling(f *os.File) {
 }
 
 func configClairVersion(config *Config) {
-	clair.EnabledDetectors = append(featurefmt.ListListers(), featurens.ListDetectors()...)
 	clair.EnabledUpdaters = strutil.Intersect(config.Updater.EnabledUpdaters, vulnsrc.ListUpdaters())
 
 	log.WithFields(log.Fields{
-		"Detectors": database.SerializeDetectors(clair.EnabledDetectors),
+		"Detectors": database.SerializeDetectors(clair.EnabledDetectors()),
 		"Updaters":  clair.EnabledUpdaters,
 	}).Info("enabled Clair extensions")
 }
@@ -134,7 +130,8 @@ func Boot(config *Config) {
 
 	defer db.Close()
 
-	clair.InitWorker(db)
+	clair.RegisterConfiguredDetectors(db)
+
 	// Start notifier
 	st.Begin()
 	go clair.RunNotifier(config.Notifier, db, st)
@@ -173,7 +170,6 @@ func main() {
 	flagConfigPath := flag.String("config", "/etc/clair/config.yaml", "Load configuration from the specified file.")
 	flagCPUProfilePath := flag.String("cpu-profile", "", "Write a CPU profile to the specified file before exiting.")
 	flagLogLevel := flag.String("log-level", "info", "Define the logging level.")
-	flagInsecureTLS := flag.Bool("insecure-tls", false, "Disable TLS server's certificate chain and hostname verification when pulling layers.")
 	flag.Parse()
 
 	configureLogger(flagLogLevel)
@@ -195,12 +191,6 @@ func main() {
 		defer stopCPUProfiling(startCPUProfiling(*flagCPUProfilePath))
 	}
 
-	// Enable TLS server's certificate chain and hostname verification
-	// when pulling layers if specified
-	if *flagInsecureTLS {
-		imagefmt.SetInsecureTLS(*flagInsecureTLS)
-	}
-
 	// configure updater and worker
 	configClairVersion(config)
 

database/dbutil.go

@@ -15,8 +15,11 @@
 package database
 
 import (
+	"encoding/json"
 	"time"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/deckarep/golang-set"
 )
 
@@ -94,8 +97,11 @@ func PersistFeaturesAndCommit(datastore Datastore, features []Feature) error {
 	defer tx.Rollback()
 
 	if err := tx.PersistFeatures(features); err != nil {
+		serialized, _ := json.Marshal(features)
+		log.WithError(err).WithField("feature", string(serialized)).Error("failed to store features")
 		return err
 	}
+
 	return tx.Commit()
 }
 
@@ -129,14 +135,18 @@ func FindAncestryAndRollback(datastore Datastore, name string) (Ancestry, bool,
 }
 
 // FindLayerAndRollback wraps session FindLayer function with begin and rollback.
-func FindLayerAndRollback(datastore Datastore, hash string) (layer Layer, ok bool, err error) {
+func FindLayerAndRollback(datastore Datastore, hash string) (layer *Layer, ok bool, err error) {
 	var tx Session
 	if tx, err = datastore.Begin(); err != nil {
 		return
 	}
 
 	defer tx.Rollback()
-	layer, ok, err = tx.FindLayer(hash)
+	// TODO(sidac): In order to make the session interface more idiomatic, we'll
+	// return the pointer value in the future.
+	var dereferencedLayer Layer
+	dereferencedLayer, ok, err = tx.FindLayer(hash)
+	layer = &dereferencedLayer
 	return
 }
 
@@ -168,13 +178,17 @@ func GetAncestryFeatures(ancestry Ancestry) []NamespacedFeature {
 }
 
 // UpsertAncestryAndCommit wraps session UpsertAncestry function with begin and commit.
-func UpsertAncestryAndCommit(datastore Datastore, ancestry Ancestry) error {
+func UpsertAncestryAndCommit(datastore Datastore, ancestry *Ancestry) error {
 	tx, err := datastore.Begin()
 	if err != nil {
 		return err
 	}
 
-	if err = tx.UpsertAncestry(ancestry); err != nil {
+	if err = tx.UpsertAncestry(*ancestry); err != nil {
+		log.WithError(err).Error("failed to upsert the ancestry")
+		serialized, _ := json.Marshal(ancestry)
+		log.Debug(string(serialized))
+
 		tx.Rollback()
 		return err
 	}
@@ -350,3 +364,22 @@ func ReleaseLock(datastore Datastore, name, owner string) {
 		return
 	}
 }
+
+// PersistDetectorsAndCommit stores the detectors in the data store.
+func PersistDetectorsAndCommit(store Datastore, detectors []Detector) error {
+	tx, err := store.Begin()
+	if err != nil {
+		return err
+	}
+
+	defer tx.Rollback()
+	if err := tx.PersistDetectors(detectors); err != nil {
+		return err
+	}
+
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+
+	return nil
+}

database/detector.go

@@ -93,12 +93,12 @@ func (s DetectorType) Valid() bool {
 type Detector struct {
 	// Name of an extension should be non-empty and uniquely identifies the
 	// extension.
-	Name string
+	Name string `json:"name"`
 	// Version of an extension should be non-empty.
-	Version string
+	Version string `json:"version"`
 	// DType is the type of the extension and should be one of the types in
 	// DetectorTypes.
-	DType DetectorType
+	DType DetectorType `json:"dtype"`
 }
 
 // Valid checks if all fields in the detector satisfies the spec.

database/models.go

@@ -26,13 +26,13 @@ import (
 type Ancestry struct {
 	// Name is a globally unique value for a set of layers. This is often the
 	// sha256 digest of an OCI/Docker manifest.
-	Name string
+	Name string `json:"name"`
 	// By contains the processors that are used when computing the
 	// content of this ancestry.
-	By []Detector
+	By []Detector `json:"by"`
 	// Layers should be ordered and i_th layer is the parent of i+1_th layer in
 	// the slice.
-	Layers []AncestryLayer
+	Layers []AncestryLayer `json:"layers"`
 }
 
 // Valid checks if the ancestry is compliant to spec.
@@ -63,10 +63,10 @@ func (a *Ancestry) Valid() bool {
 // AncestryLayer is a layer with all detected namespaced features.
 type AncestryLayer struct {
 	// Hash is the sha-256 tarsum on the layer's blob content.
-	Hash string
+	Hash string `json:"hash"`
 	// Features are the features introduced by this layer when it was
 	// processed.
-	Features []AncestryFeature
+	Features []AncestryFeature `json:"features"`
 }
 
 // Valid checks if the Ancestry Layer is compliant to the spec.
@@ -95,22 +95,22 @@ func (l *AncestryLayer) GetFeatures() []NamespacedFeature {
 // AncestryFeature is a namespaced feature with the detectors used to
 // find this feature.
 type AncestryFeature struct {
-	NamespacedFeature
+	NamespacedFeature `json:"namespacedFeature"`
 
 	// FeatureBy is the detector that detected the feature.
-	FeatureBy Detector
+	FeatureBy Detector `json:"featureBy"`
 	// NamespaceBy is the detector that detected the namespace.
-	NamespaceBy Detector
+	NamespaceBy Detector `json:"namespaceBy"`
 }
 
 // Layer is a layer with all the detected features and namespaces.
 type Layer struct {
 	// Hash is the sha-256 tarsum on the layer's blob content.
-	Hash string
+	Hash string `json:"hash"`
 	// By contains a list of detectors scanned this Layer.
-	By         []Detector
-	Namespaces []LayerNamespace
-	Features   []LayerFeature
+	By         []Detector       `json:"by"`
+	Namespaces []LayerNamespace `json:"namespaces"`
+	Features   []LayerFeature   `json:"features"`
 }
 
 func (l *Layer) GetFeatures() []Feature {
@@ -133,26 +133,26 @@ func (l *Layer) GetNamespaces() []Namespace {
 
 // LayerNamespace is a namespace with detection information.
 type LayerNamespace struct {
-	Namespace
+	Namespace `json:"namespace"`
 
 	// By is the detector found the namespace.
-	By Detector
+	By Detector `json:"by"`
 }
 
 // LayerFeature is a feature with detection information.
 type LayerFeature struct {
-	Feature
+	Feature `json:"feature"`
 
 	// By is the detector found the feature.
-	By Detector
+	By Detector `json:"by"`
 }
 
 // Namespace is the contextual information around features.
 //
 // e.g. Debian:7, NodeJS.
 type Namespace struct {
-	Name          string
-	VersionFormat string
+	Name          string `json:"name"`
+	VersionFormat string `json:"versionFormat"`
 }
 
 func NewNamespace(name string, versionFormat string) *Namespace {
@@ -166,10 +166,10 @@ func NewNamespace(name string, versionFormat string) *Namespace {
 // dpkg is the version format of the installer package manager, which in this
 // case could be dpkg or apk.
 type Feature struct {
-	Name          string
-	Version       string
-	VersionFormat string
-	Type          FeatureType
+	Name          string      `json:"name"`
+	Version       string      `json:"version"`
+	VersionFormat string      `json:"versionFormat"`
+	Type          FeatureType `json:"type"`
 }
 
 func NewFeature(name string, version string, versionFormat string, featureType FeatureType) *Feature {
@@ -189,9 +189,9 @@ func NewSourcePackage(name string, version string, versionFormat string) *Featur
 //
 // e.g. OpenSSL 1.0 dpkg Debian:7.
 type NamespacedFeature struct {
-	Feature
+	Feature `json:"feature"`
 
-	Namespace Namespace
+	Namespace Namespace `json:"namespace"`
 }
 
 func NewNamespacedFeature(namespace *Namespace, feature *Feature) *NamespacedFeature {

pkg/httputil/httputil.go

@@ -16,6 +16,10 @@
 package httputil
 
 import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"strings"
@@ -62,6 +66,38 @@ func GetClientAddr(r *http.Request) string {
 	return addr
 }
 
+// GetWithContext do HTTP GET to the URI with headers and returns response blob
+// reader.
+func GetWithContext(ctx context.Context, uri string, headers http.Header) (io.ReadCloser, error) {
+	request, err := http.NewRequest("GET", uri, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if headers != nil {
+		request.Header = headers
+	}
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{},
+		Proxy:           http.ProxyFromEnvironment,
+	}
+
+	client := &http.Client{Transport: tr}
+	request = request.WithContext(ctx)
+	r, err := client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+
+	// Fail if we don't receive a 2xx HTTP status code.
+	if !Status2xx(r) {
+		return nil, fmt.Errorf("failed HTTP GET: expected 2XX, got %d", r.StatusCode)
+	}
+
+	return r.Body, nil
+}
+
 // Status2xx returns true if the response's status code is success (2xx)
 func Status2xx(resp *http.Response) bool {
 	return resp.StatusCode/100 == 2

worker.go

@@ -1,478 +0,0 @@
-// Copyright 2018 clair authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package clair
-
-import (
-	"errors"
-	"sync"
-
-	"github.com/deckarep/golang-set"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/coreos/clair/database"
-	"github.com/coreos/clair/ext/featurefmt"
-	"github.com/coreos/clair/ext/featurens"
-	"github.com/coreos/clair/ext/imagefmt"
-	"github.com/coreos/clair/pkg/commonerr"
-	"github.com/coreos/clair/pkg/strutil"
-	"github.com/coreos/clair/pkg/tarutil"
-)
-
-var (
-	// ErrUnsupported is the error that should be raised when an OS or package
-	// manager is not supported.
-	ErrUnsupported = commonerr.NewBadRequestError("worker: OS and/or package manager are not supported")
-
-	// EnabledDetectors are detectors to be used to scan the layers.
-	EnabledDetectors []database.Detector
-)
-
-// LayerRequest represents all information necessary to download and process a
-// layer.
-type LayerRequest struct {
-	Hash    string
-	Path    string
-	Headers map[string]string
-}
-
-type processResult struct {
-	existingLayer   *database.Layer
-	newLayerContent *database.Layer
-	err             error
-}
-
-// processRequest stores parameters used for processing a layer.
-type processRequest struct {
-	LayerRequest
-
-	existingLayer *database.Layer
-	detectors     []database.Detector
-}
-
-type introducedFeature struct {
-	feature    database.AncestryFeature
-	layerIndex int
-}
-
-// processRequests in parallel processes a set of requests for unique set of layers
-// and returns sets of unique namespaces, features and layers to be inserted
-// into the database.
-func processRequests(imageFormat string, toDetect map[string]*processRequest) (map[string]*processResult, error) {
-	wg := &sync.WaitGroup{}
-	wg.Add(len(toDetect))
-
-	results := map[string]*processResult{}
-	for i := range toDetect {
-		results[i] = nil
-	}
-
-	for i := range toDetect {
-		result := processResult{}
-		results[i] = &result
-		go func(req *processRequest, res *processResult) {
-			*res = *detectContent(imageFormat, req)
-			wg.Done()
-		}(toDetect[i], &result)
-	}
-
-	wg.Wait()
-	errs := []error{}
-	for _, r := range results {
-		errs = append(errs, r.err)
-	}
-
-	if err := commonerr.CombineErrors(errs...); err != nil {
-		return nil, err
-	}
-
-	return results, nil
-}
-
-func getProcessRequest(datastore database.Datastore, req LayerRequest) (preq *processRequest, err error) {
-	layer, ok, err := database.FindLayerAndRollback(datastore, req.Hash)
-	if err != nil {
-		return
-	}
-
-	if !ok {
-		log.WithField("layer", req.Hash).Debug("found no existing layer in database")
-		preq = &processRequest{
-			LayerRequest:  req,
-			existingLayer: &database.Layer{Hash: req.Hash},
-			detectors:     EnabledDetectors,
-		}
-	} else {
-		log.WithFields(log.Fields{
-			"layer":           layer.Hash,
-			"detectors":       layer.By,
-			"feature count":   len(layer.Features),
-			"namespace count": len(layer.Namespaces),
-		}).Debug("found existing layer in database")
-
-		preq = &processRequest{
-			LayerRequest:  req,
-			existingLayer: &layer,
-			detectors:     database.DiffDetectors(EnabledDetectors, layer.By),
-		}
-	}
-
-	return
-}
-
-func persistProcessResult(datastore database.Datastore, results map[string]*processResult) error {
-	features := []database.Feature{}
-	namespaces := []database.Namespace{}
-	for _, r := range results {
-		features = append(features, r.newLayerContent.GetFeatures()...)
-		namespaces = append(namespaces, r.newLayerContent.GetNamespaces()...)
-	}
-
-	features = database.DeduplicateFeatures(features...)
-	namespaces = database.DeduplicateNamespaces(namespaces...)
-	if err := database.PersistNamespacesAndCommit(datastore, namespaces); err != nil {
-		return err
-	}
-
-	if err := database.PersistFeaturesAndCommit(datastore, features); err != nil {
-		return err
-	}
-
-	for _, layer := range results {
-		if err := database.PersistPartialLayerAndCommit(datastore, layer.newLayerContent); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// processLayers processes a set of post layer requests, stores layers and
-// returns an ordered list of processed layers with detected features and
-// namespaces.
-func processLayers(datastore database.Datastore, imageFormat string, requests []LayerRequest) ([]database.Layer, error) {
-	var (
-		reqMap = make(map[string]*processRequest)
-		err    error
-	)
-
-	for _, r := range requests {
-		reqMap[r.Hash], err = getProcessRequest(datastore, r)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	results, err := processRequests(imageFormat, reqMap)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := persistProcessResult(datastore, results); err != nil {
-		return nil, err
-	}
-
-	completeLayers := getProcessResultLayers(results)
-	layers := make([]database.Layer, 0, len(requests))
-	for _, r := range requests {
-		layers = append(layers, completeLayers[r.Hash])
-	}
-
-	return layers, nil