arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ext: feature detector -> featurefmt

Browse Source
This commit is contained in:
Jimmy Zelinskie 2017-01-13 18:49:02 -05:00
parent 71a8b542f9
commit cda3d4819c
15 changed files with 191 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/clair/main.go
View File

@ -25,7 +25,13 @@ import (
"github.com/coreos/clair"
"github.com/coreos/clair/config"
// Register database driver.
_ "github.com/coreos/clair/database/pgsql"
// Register extensions.
_ "github.com/coreos/clair/ext/featurefmt/apk"
_ "github.com/coreos/clair/ext/featurefmt/dpkg"
_ "github.com/coreos/clair/ext/featurefmt/rpm"
_ "github.com/coreos/clair/ext/featurens/alpinerelease"
_ "github.com/coreos/clair/ext/featurens/aptsources"
_ "github.com/coreos/clair/ext/featurens/lsbrelease"
@ -40,12 +46,6 @@ import (
_ "github.com/coreos/clair/ext/vulnsrc/oracle"
_ "github.com/coreos/clair/ext/vulnsrc/rhel"
_ "github.com/coreos/clair/ext/vulnsrc/ubuntu"
_ "github.com/coreos/clair/worker/detectors/feature/apk"
_ "github.com/coreos/clair/worker/detectors/feature/dpkg"
_ "github.com/coreos/clair/worker/detectors/feature/rpm"
_ "github.com/coreos/clair/database/pgsql"
)
var log = capnslog.NewPackageLogger("github.com/coreos/clair/cmd/clair", "main")

18
worker/detectors/feature/apk/apk.go → ext/featurefmt/apk/apk.go
View File

@ -1,4 +1,4 @@
// Copyright 2016 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package apk implements a featurefmt.Lister for APK packages.
package apk
import (
@ -21,21 +22,22 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/worker/detectors"
"github.com/coreos/clair/pkg/tarutil"
)
var log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker/detectors/packages")
var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/featurefmt/apk")
func init() {
detectors.RegisterFeaturesDetector("apk", &detector{})
featurefmt.RegisterLister("apk", &lister{})
}
type detector struct{}
type lister struct{}
func (d *detector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) {
file, exists := data["lib/apk/db/installed"]
func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, error) {
file, exists := files["lib/apk/db/installed"]
if !exists {
return []database.FeatureVersion{}, nil
}
@ -83,6 +85,6 @@ func (d *detector) Detect(data map[string][]byte) ([]database.FeatureVersion, er
return pkgs, nil
}
func (d *detector) GetRequiredFiles() []string {
func (l lister) RequiredFilenames() []string {
return []string{"lib/apk/db/installed"}
}

13
worker/detectors/feature/apk/apk_test.go → ext/featurefmt/apk/apk_test.go
View File

@ -1,4 +1,4 @@
// Copyright 2016 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -18,11 +18,12 @@ import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/worker/detectors/feature"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/pkg/tarutil"
)
func TestAPKFeatureDetection(t *testing.T) {
testData := []feature.TestData{
testData := []featurefmt.TestData{
{
FeatureVersions: []database.FeatureVersion{
{
@ -70,10 +71,10 @@ func TestAPKFeatureDetection(t *testing.T) {
Version: "0.7-r0",
},
},
Data: map[string][]byte{
"lib/apk/db/installed": feature.LoadFileForTest("apk/testdata/installed"),
Files: tarutil.FilesMap{
"lib/apk/db/installed": featurefmt.LoadFileForTest("apk/testdata/installed"),
},
},
}
feature.TestDetector(t, &detector{}, testData)
featurefmt.TestLister(t, &lister{}, testData)
}

0
worker/detectors/feature/apk/testdata/installed → ext/featurefmt/apk/testdata/installed vendored
View File

22
worker/detectors/feature/dpkg/dpkg.go → ext/featurefmt/dpkg/dpkg.go
View File

@ -1,4 +1,4 @@
// Copyright 2015 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package dpkg implements a featurefmt.Lister for dpkg packages.
package dpkg
import (
@ -22,28 +23,27 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/worker/detectors"
"github.com/coreos/clair/pkg/tarutil"
)
var (
log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker/detectors/packages")
log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/featurefmt/dpkg")
dpkgSrcCaptureRegexp = regexp.MustCompile(`Source: (?P<name>[^\s]*)( \((?P<version>.*)\))?`)
dpkgSrcCaptureRegexpNames = dpkgSrcCaptureRegexp.SubexpNames()
)
// DpkgFeaturesDetector implements FeaturesDetector and detects dpkg packages
type DpkgFeaturesDetector struct{}
type lister struct{}
func init() {
detectors.RegisterFeaturesDetector("dpkg", &DpkgFeaturesDetector{})
featurefmt.RegisterLister("dpkg", &lister{})
}
// Detect detects packages using var/lib/dpkg/status from the input data
func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) {
f, hasFile := data["var/lib/dpkg/status"]
func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, error) {
f, hasFile := files["var/lib/dpkg/status"]
if !hasFile {
return []database.FeatureVersion{}, nil
}
@ -116,8 +116,6 @@ func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database
return packages, nil
}
// GetRequiredFiles returns the list of files required for Detect, without
// leading /
func (detector *DpkgFeaturesDetector) GetRequiredFiles() []string {
func (l lister) RequiredFilenames() []string {
return []string{"var/lib/dpkg/status"}
}

13
worker/detectors/feature/dpkg/dpkg_test.go → ext/featurefmt/dpkg/dpkg_test.go
View File

@ -1,4 +1,4 @@
// Copyright 2015 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -18,11 +18,12 @@ import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/worker/detectors/feature"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/pkg/tarutil"
)
func TestDpkgFeatureDetection(t *testing.T) {
testData := []feature.TestData{
testData := []featurefmt.TestData{
// Test an Ubuntu dpkg status file
{
FeatureVersions: []database.FeatureVersion{
@ -40,11 +41,11 @@ func TestDpkgFeatureDetection(t *testing.T) {
Version: "5.1.1-12ubuntu1", // The version comes from the "Source:" line
},
},
Data: map[string][]byte{
"var/lib/dpkg/status": feature.LoadFileForTest("dpkg/testdata/status"),
Files: tarutil.FilesMap{
"var/lib/dpkg/status": featurefmt.LoadFileForTest("dpkg/testdata/status"),
},
},
}
feature.TestDetector(t, &DpkgFeaturesDetector{}, testData)
featurefmt.TestLister(t, &lister{}, testData)
}

0
worker/detectors/feature/dpkg/testdata/status → ext/featurefmt/dpkg/testdata/status vendored
View File

126
ext/featurefmt/driver.go Normal file
View File

@ -0,0 +1,126 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package featurefmt exposes functions to dynamically register methods for
// determining the features present in an image layer.
package featurefmt
import (
"io/ioutil"
"path/filepath"
"runtime"
"sync"
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/pkg/tarutil"
"github.com/stretchr/testify/assert"
)
var (
listersM sync.RWMutex
listers = make(map[string]Lister)
)
// Lister represents an ability to list the features present in an image layer.
type Lister interface {
// ListFeatures produces a list of FeatureVersions present in an image layer.
ListFeatures(tarutil.FilesMap) ([]database.FeatureVersion, error)
// RequiredFilenames returns the list of files required to be in the FilesMap
// provided to the ListFeatures method.
//
// Filenames must not begin with "/".
RequiredFilenames() []string
}
// RegisterLister makes a Lister available by the provided name.
//
// If called twice with the same name, the name is blank, or if the provided
// Lister is nil, this function panics.
func RegisterLister(name string, l Lister) {
if name == "" {
panic("featurefmt: could not register a Lister with an empty name")
}
if l == nil {
panic("featurefmt: could not register a nil Lister")
}
listersM.Lock()
defer listersM.Unlock()
if _, dup := listers[name]; dup {
panic("featurefmt: RegisterLister called twice for " + name)
}
listers[name] = l
}
// ListFeatures produces the list of FeatureVersions in an image layer using
// every registered Lister.
func ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, error) {
listersM.RLock()
defer listersM.RUnlock()
var totalFeatures []database.FeatureVersion
for _, lister := range listers {
features, err := lister.ListFeatures(files)
if err != nil {
return []database.FeatureVersion{}, err
}
totalFeatures = append(totalFeatures, features...)
}
return totalFeatures, nil
}
// RequiredFilenames returns the total list of files required for all
// registered Listers.
func RequiredFilenames() (files []string) {
listersM.RLock()
defer listersM.RUnlock()
for _, lister := range listers {
files = append(files, lister.RequiredFilenames()...)
}
return
}
// TestData represents the data used to test an implementation of Lister.
type TestData struct {
Files tarutil.FilesMap
FeatureVersions []database.FeatureVersion
}
// LoadFileForTest can be used in order to obtain the []byte contents of a file
// that is meant to be used for test data.
func LoadFileForTest(name string) []byte {
_, filename, _, _ := runtime.Caller(0)
d, _ := ioutil.ReadFile(filepath.Join(filepath.Dir(filename)) + "/" + name)
return d
}
// TestLister runs a Lister on each provided instance of TestData and asserts
// the ouput to be equal to the expected output.
func TestLister(t *testing.T, l Lister, testData []TestData) {
for _, td := range testData {
featureVersions, err := l.ListFeatures(td.Files)
if assert.Nil(t, err) && assert.Len(t, featureVersions, len(td.FeatureVersions)) {
for _, expectedFeatureVersion := range td.FeatureVersions {
assert.Contains(t, featureVersions, expectedFeatureVersion)
}
}
}
}

21
worker/detectors/feature/rpm/rpm.go → ext/featurefmt/rpm/rpm.go
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package rpm implements a featurefmt.Lister for rpm packages.
package rpm
import (
@ -23,26 +24,24 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/rpm"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/pkg/tarutil"
"github.com/coreos/clair/utils"
"github.com/coreos/clair/worker/detectors"
)
var log = capnslog.NewPackageLogger("github.com/coreos/clair", "rpm")
var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/featurefmt/rpm")
// RpmFeaturesDetector implements FeaturesDetector and detects rpm packages
// It requires the "rpm" binary to be in the PATH
type RpmFeaturesDetector struct{}
type lister struct{}
func init() {
detectors.RegisterFeaturesDetector("rpm", &RpmFeaturesDetector{})
featurefmt.RegisterLister("rpm", &lister{})
}
// Detect detects packages using var/lib/rpm/Packages from the input data
func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) {
f, hasFile := data["var/lib/rpm/Packages"]
func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, error) {
f, hasFile := files["var/lib/rpm/Packages"]
if !hasFile {
return []database.FeatureVersion{}, nil
}
@ -116,8 +115,6 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.
return packages, nil
}
// GetRequiredFiles returns the list of files required for Detect, without
// leading /
func (detector *RpmFeaturesDetector) GetRequiredFiles() []string {
func (l lister) RequiredFilenames() []string {
return []string{"var/lib/rpm/Packages"}
}

13
worker/detectors/feature/rpm/rpm_test.go → ext/featurefmt/rpm/rpm_test.go
View File

@ -1,4 +1,4 @@
// Copyright 2016 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -18,11 +18,12 @@ import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/worker/detectors/feature"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/pkg/tarutil"
)
func TestRpmFeatureDetection(t *testing.T) {
testData := []feature.TestData{
testData := []featurefmt.TestData{
// Test a CentOS 7 RPM database
// Memo: Use the following command on a RPM-based system to shrink a database: rpm -qa --qf "%{NAME}\n" |tail -n +3| xargs rpm -e --justdb
{
@ -38,11 +39,11 @@ func TestRpmFeatureDetection(t *testing.T) {
Version: "3.2-18.el7",
},
},
Data: map[string][]byte{
"var/lib/rpm/Packages": feature.LoadFileForTest("rpm/testdata/Packages"),
Files: tarutil.FilesMap{
"var/lib/rpm/Packages": featurefmt.LoadFileForTest("rpm/testdata/Packages"),
},
},
}
feature.TestDetector(t, &RpmFeaturesDetector{}, testData)
featurefmt.TestLister(t, &lister{}, testData)
}

0
worker/detectors/feature/rpm/testdata/Packages → ext/featurefmt/rpm/testdata/Packages vendored
View File

56
worker/detectors/feature/test.go
View File

@ -1,56 +0,0 @@
// Copyright 2016 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package feature implements utilities common to implementations of
// FeatureDetector.
package feature
import (
"io/ioutil"
"path/filepath"
"runtime"
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/worker/detectors"
"github.com/stretchr/testify/assert"
)
// TestData represents the data used to test an implementation of
// FeatureDetector.
type TestData struct {
Data map[string][]byte
FeatureVersions []database.FeatureVersion
}
// LoadFileForTest can be used in order to obtain the []byte contents of a file
// that is meant to be used for test data.
func LoadFileForTest(name string) []byte {
_, filename, _, _ := runtime.Caller(0)
d, _ := ioutil.ReadFile(filepath.Join(filepath.Dir(filename)) + "/" + name)
return d
}
// TestDetector runs a detector on each provided instance of TestData and
// asserts the ouput to be equal to the expected output.
func TestDetector(t *testing.T, detector detectors.FeaturesDetector, testData []TestData) {
for _, td := range testData {
featureVersions, err := detector.Detect(td.Data)
if assert.Nil(t, err) && assert.Len(t, featureVersions, len(td.FeatureVersions)) {
for _, expectedFeatureVersion := range td.FeatureVersions {
assert.Contains(t, featureVersions, expectedFeatureVersion)
}
}
}
}

79
worker/detectors/features.go
View File

@ -1,79 +0,0 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package detectors
import (
"fmt"
"sync"
"github.com/coreos/clair/database"
)
// The FeaturesDetector interface defines a way to detect packages from input data.
type FeaturesDetector interface {
// Detect detects a list of FeatureVersion from the input data.
Detect(map[string][]byte) ([]database.FeatureVersion, error)
// GetRequiredFiles returns the list of files required for Detect, without
// leading /.
GetRequiredFiles() []string
}
var (
featuresDetectorsLock sync.Mutex
featuresDetectors = make(map[string]FeaturesDetector)
)
// RegisterFeaturesDetector makes a FeaturesDetector available for DetectFeatures.
func RegisterFeaturesDetector(name string, f FeaturesDetector) {
if name == "" {
panic("Could not register a FeaturesDetector with an empty name")
}
if f == nil {
panic("Could not register a nil FeaturesDetector")
}
featuresDetectorsLock.Lock()
defer featuresDetectorsLock.Unlock()
if _, alreadyExists := featuresDetectors[name]; alreadyExists {
panic(fmt.Sprintf("Detector '%s' is already registered", name))
}
featuresDetectors[name] = f
}
// DetectFeatures detects a list of FeatureVersion using every registered FeaturesDetector.
func DetectFeatures(data map[string][]byte) ([]database.FeatureVersion, error) {
var packages []database.FeatureVersion
for _, detector := range featuresDetectors {
pkgs, err := detector.Detect(data)
if err != nil {
return []database.FeatureVersion{}, err
}
packages = append(packages, pkgs...)
}
return packages, nil
}
// GetRequiredFilesFeatures returns the list of files required for Detect for every
// registered FeaturesDetector, without leading /.
func GetRequiredFilesFeatures() (files []string) {
for _, detector := range featuresDetectors {
files = append(files, detector.GetRequiredFiles()...)
}
return
}

11
worker/worker.go
View File

@ -20,12 +20,12 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/ext/featurens"
"github.com/coreos/clair/ext/imagefmt"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/pkg/tarutil"
"github.com/coreos/clair/utils"
"github.com/coreos/clair/worker/detectors"
)
const (
@ -115,7 +115,7 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s
// detectContent downloads a layer's archive and extracts its Namespace and
// Features.
func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) {
totalRequiredFiles := append(detectors.GetRequiredFilesFeatures(), featurens.RequiredFilenames()...)
totalRequiredFiles := append(featurefmt.RequiredFilenames(), featurens.RequiredFilenames()...)
files, err := imagefmt.Extract(imageFormat, path, headers, totalRequiredFiles)
if err != nil {
log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
@ -128,8 +128,7 @@ func detectContent(imageFormat, name, path string, headers map[string]string, pa
}
// Detect features.
data := map[string][]byte(files)
featureVersions, err = detectFeatureVersions(name, data, namespace, parent)
featureVersions, err = detectFeatureVersions(name, files, namespace, parent)
if err != nil {
return
}
@ -162,12 +161,12 @@ func detectNamespace(name string, files tarutil.FilesMap, parent *database.Layer
return
}
func detectFeatureVersions(name string, data map[string][]byte, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) {
func detectFeatureVersions(name string, files tarutil.FilesMap, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) {
// TODO(Quentin-M): We need to pass the parent image to DetectFeatures because it's possible that
// some detectors would need it in order to produce the entire feature list (if they can only
// detect a diff). Also, we should probably pass the detected namespace so detectors could
// make their own decision.
features, err = detectors.DetectFeatures(data)
features, err = featurefmt.ListFeatures(files)
if err != nil {
return
}

8
worker/worker_test.go
View File

@ -26,10 +26,10 @@ import (
"github.com/coreos/clair/pkg/commonerr"
// Register the required detectors.
_ "github.com/coreos/clair/worker/detectors/data/docker"
_ "github.com/coreos/clair/worker/detectors/feature/dpkg"
_ "github.com/coreos/clair/worker/detectors/namespace/aptsources"
_ "github.com/coreos/clair/worker/detectors/namespace/osrelease"
_ "github.com/coreos/clair/ext/featurefmt/dpkg"
_ "github.com/coreos/clair/ext/featurens/aptsources"
_ "github.com/coreos/clair/ext/featurens/osrelease"
_ "github.com/coreos/clair/ext/imagefmt/docker"
)
type mockDatastore struct {