arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ext: data detector -> imagefmt

Browse Source
This commit is contained in:
Jimmy Zelinskie 2017-01-13 03:07:35 -05:00
parent f9b319089d
commit d9be34c3c4
8 changed files with 229 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/clair/main.go
View File

@ -1,4 +1,4 @@
// Copyright 2015 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -26,6 +26,8 @@ import (
"github.com/coreos/clair/config"
// Register extensions.
_ "github.com/coreos/clair/ext/imagefmt/aci"
_ "github.com/coreos/clair/ext/imagefmt/docker"
_ "github.com/coreos/clair/ext/notification/webhook"
_ "github.com/coreos/clair/ext/vulnmdsrc/nvd"
_ "github.com/coreos/clair/ext/vulnsrc/alpine"
@ -34,9 +36,6 @@ import (
_ "github.com/coreos/clair/ext/vulnsrc/rhel"
_ "github.com/coreos/clair/ext/vulnsrc/ubuntu"
_ "github.com/coreos/clair/worker/detectors/data/aci"
_ "github.com/coreos/clair/worker/detectors/data/docker"
_ "github.com/coreos/clair/worker/detectors/feature/apk"
_ "github.com/coreos/clair/worker/detectors/feature/dpkg"
_ "github.com/coreos/clair/worker/detectors/feature/rpm"

40
ext/imagefmt/aci/aci.go Normal file
View File

@ -0,0 +1,40 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package aci
import (
"io"
"path/filepath"
"github.com/coreos/clair/ext/imagefmt"
"github.com/coreos/clair/pkg/tarutil"
)
type format struct{}
func init() {
imagefmt.RegisterExtractor("aci", &format{})
}
func (f format) ExtractFiles(layerReader io.ReadCloser, toExtract []string) (tarutil.FilesMap, error) {
// All contents are inside a "rootfs" directory, so this needs to be
// prepended to each filename.
var filenames []string
for _, filename := range toExtract {
filenames = append(filenames, filepath.Join("rootfs/", filename))
}
return tarutil.ExtractFiles(layerReader, filenames)
}

32
ext/imagefmt/docker/docker.go Normal file
View File

@ -0,0 +1,32 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package docker
import (
"io"
"github.com/coreos/clair/ext/imagefmt"
"github.com/coreos/clair/pkg/tarutil"
)
type format struct{}
func init() {
imagefmt.RegisterExtractor("docker", &format{})
}
func (f format) ExtractFiles(layerReader io.ReadCloser, toExtract []string) (tarutil.FilesMap, error) {
return tarutil.ExtractFiles(layerReader, toExtract)
}

150
ext/imagefmt/driver.go Normal file
View File

@ -0,0 +1,150 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package notification fetches notifications from the database and informs the
// specified remote handler about their existences, inviting the third party to
// actively query the API about it.
// Package imagefmt exposes functions to dynamically register methods to
// detect different types of container image formats.
package imagefmt
import (
"fmt"
"io"
"math"
"net/http"
"os"
"strings"
"sync"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/pkg/tarutil"
"github.com/coreos/pkg/capnslog"
)
var (
// ErrCouldNotFindLayer is returned when we could not download or open the layer file.
ErrCouldNotFindLayer = commonerr.NewBadRequestError("could not find layer")
log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/imagefmt")
extractorsM sync.RWMutex
extractors = make(map[string]Extractor)
)
// Extractor represents an ability to extract files from a particular container
// image format.
type Extractor interface {
// ExtractFiles produces a tarutil.FilesMap from a image layer.
ExtractFiles(layer io.ReadCloser, filenames []string) (tarutil.FilesMap, error)
}
// RegisterExtractor makes a extractor available by the provided name.
//
// If called twice with the same name, the name is blank, or if the provided
// Extractor is nil, this function panics.
func RegisterExtractor(name string, d Extractor) {
extractorsM.Lock()
defer extractorsM.Unlock()
if name == "" {
panic("imagefmt: could not register a extractor with an empty name")
}
if d == nil {
panic("imagefmt: could not register a nil extractor")
}
// Enforce lowercase names, so that they can be reliably be found in a map.
name = strings.ToLower(name)
if _, dup := extractors[name]; dup {
panic("imagefmt: RegisterExtractor called twice for " + name)
}
extractors[name] = d
}
// Extractors returns the list of the registered extractors.
func Extractors() map[string]Extractor {
extractorsM.RLock()
defer extractorsM.RUnlock()
ret := make(map[string]Extractor)
for k, v := range extractors {
ret[k] = v
}
return ret
}
// UnregisterExtractor removes a Extractor with a particular name from the list.
func UnregisterExtractor(name string) {
extractorsM.Lock()
defer extractorsM.Unlock()
delete(extractors, name)
}
// Extract streams an image layer from disk or over HTTP, determines the
// image format, then extracts the files specified.
func Extract(format, path string, headers map[string]string, toExtract []string) (tarutil.FilesMap, error) {
var layerReader io.ReadCloser
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
// Create a new HTTP request object.
request, err := http.NewRequest("GET", path, nil)
if err != nil {
return nil, ErrCouldNotFindLayer
}
// Set any provided HTTP Headers.
if headers != nil {
for k, v := range headers {
request.Header.Set(k, v)
}
}
// Send the request and handle the response.
r, err := http.DefaultClient.Do(request)
if err != nil {
log.Warningf("could not download layer: %s", err)
return nil, ErrCouldNotFindLayer
}
// Fail if we don't receive a 2xx HTTP status code.
if math.Floor(float64(r.StatusCode/100)) != 2 {
log.Warningf("could not download layer: got status code %d, expected 2XX", r.StatusCode)
return nil, ErrCouldNotFindLayer
}
layerReader = r.Body
} else {
var err error
layerReader, err = os.Open(path)
if err != nil {
return nil, ErrCouldNotFindLayer
}
}
defer layerReader.Close()
if extractor, exists := Extractors()[strings.ToLower(format)]; exists {
files, err := extractor.ExtractFiles(layerReader, toExtract)
if err != nil {
return nil, err
}
return files, nil
}
return nil, commonerr.NewBadRequestError(fmt.Sprintf("unsupported image format '%s'", format))
}

122
worker/detectors/data.go
View File

@ -1,122 +0,0 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package detectors exposes functions to register and use container
// information extractors.
package detectors
import (
"fmt"
"io"
"math"
"net/http"
"os"
"strings"
"sync"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/pkg/capnslog"
)
// The DataDetector interface defines a way to detect the required data from input path
type DataDetector interface {
//Support check if the input path and format are supported by the underling detector
Supported(path string, format string) bool
// Detect detects the required data from input path
Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (data map[string][]byte, err error)
}
var (
dataDetectorsLock sync.Mutex
dataDetectors = make(map[string]DataDetector)
log = capnslog.NewPackageLogger("github.com/coreos/clair", "detectors")
// ErrCouldNotFindLayer is returned when we could not download or open the layer file.
ErrCouldNotFindLayer = cerrors.NewBadRequestError("could not find layer")
)
// RegisterDataDetector provides a way to dynamically register an implementation of a
// DataDetector.
//
// If RegisterDataDetector is called twice with the same name if DataDetector is nil,
// or if the name is blank, it panics.
func RegisterDataDetector(name string, f DataDetector) {
if name == "" {
panic("Could not register a DataDetector with an empty name")
}
if f == nil {
panic("Could not register a nil DataDetector")
}
dataDetectorsLock.Lock()
defer dataDetectorsLock.Unlock()
if _, alreadyExists := dataDetectors[name]; alreadyExists {
panic(fmt.Sprintf("Detector '%s' is already registered", name))
}
dataDetectors[name] = f
}
// DetectData finds the Data of the layer by using every registered DataDetector
func DetectData(format, path string, headers map[string]string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) {
var layerReader io.ReadCloser
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
// Create a new HTTP request object.
request, err := http.NewRequest("GET", path, nil)
if err != nil {
return nil, ErrCouldNotFindLayer
}
// Set any provided HTTP Headers.
if headers != nil {
for k, v := range headers {
request.Header.Set(k, v)
}
}
// Send the request and handle the response.
r, err := http.DefaultClient.Do(request)
if err != nil {
log.Warningf("could not download layer: %s", err)
return nil, ErrCouldNotFindLayer
}
// Fail if we don't receive a 2xx HTTP status code.
if math.Floor(float64(r.StatusCode/100)) != 2 {
log.Warningf("could not download layer: got status code %d, expected 2XX", r.StatusCode)
return nil, ErrCouldNotFindLayer
}
layerReader = r.Body
} else {
layerReader, err = os.Open(path)
if err != nil {
return nil, ErrCouldNotFindLayer
}
}
defer layerReader.Close()
for _, detector := range dataDetectors {
if detector.Supported(path, format) {
data, err = detector.Detect(layerReader, toExtract, maxFileSize)
if err != nil {
return nil, err
}
return data, nil
}
}
return nil, cerrors.NewBadRequestError(fmt.Sprintf("unsupported image format '%s'", format))
}

41
worker/detectors/data/aci/aci.go
View File

@ -1,41 +0,0 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package aci
import (
"io"
"strings"
"github.com/coreos/clair/utils"
"github.com/coreos/clair/worker/detectors"
)
// ACIDataDetector implements DataDetector and detects layer data in 'aci' format
type ACIDataDetector struct{}
func init() {
detectors.RegisterDataDetector("aci", &ACIDataDetector{})
}
func (detector *ACIDataDetector) Supported(path string, format string) bool {
if strings.EqualFold(format, "ACI") {
return true
}
return false
}
func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
return utils.SelectivelyExtractArchive(layerReader, "rootfs/", toExtract, maxFileSize)
}

41
worker/detectors/data/docker/docker.go
View File

@ -1,41 +0,0 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package docker
import (
"io"
"strings"
"github.com/coreos/clair/utils"
"github.com/coreos/clair/worker/detectors"
)
// DockerDataDetector implements DataDetector and detects layer data in 'Docker' format
type DockerDataDetector struct{}
func init() {
detectors.RegisterDataDetector("Docker", &DockerDataDetector{})
}
func (detector *DockerDataDetector) Supported(path string, format string) bool {
if strings.EqualFold(format, "Docker") {
return true
}
return false
}
func (detector *DockerDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
return utils.SelectivelyExtractArchive(layerReader, "", toExtract, maxFileSize)
}

10
worker/worker.go
View File

@ -20,6 +20,7 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/imagefmt"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/utils"
"github.com/coreos/clair/worker/detectors"
@ -29,11 +30,6 @@ const (
// Version (integer) represents the worker version.
// Increased each time the engine changes.
Version = 3
// maxFileSize enforces a maximum size of a single file within a tarball that
// will be extracted. This protects against malicious layers that may contain
// extremely large package database files.
maxFileSize = 200 * 1024 * 1024 // 200 MiB
)
var (
@ -116,12 +112,14 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s
// detectContent downloads a layer's archive and extracts its Namespace and Features.
func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) {
data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize)
files, err := imagefmt.Extract(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...))
if err != nil {
log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
return
}
data := map[string][]byte(files)
// Detect namespace.
namespace = detectNamespace(name, data, parent)