ext: data detector -> imagefmt

cmd/clair/main.go

@@ -1,4 +1,4 @@
-// Copyright 2015 clair authors
+// Copyright 2017 clair authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -26,6 +26,8 @@ import (
 	"github.com/coreos/clair/config"
 
 	// Register extensions.
+	_ "github.com/coreos/clair/ext/imagefmt/aci"
+	_ "github.com/coreos/clair/ext/imagefmt/docker"
 	_ "github.com/coreos/clair/ext/notification/webhook"
 	_ "github.com/coreos/clair/ext/vulnmdsrc/nvd"
 	_ "github.com/coreos/clair/ext/vulnsrc/alpine"
@@ -34,9 +36,6 @@ import (
 	_ "github.com/coreos/clair/ext/vulnsrc/rhel"
 	_ "github.com/coreos/clair/ext/vulnsrc/ubuntu"
 
-	_ "github.com/coreos/clair/worker/detectors/data/aci"
-	_ "github.com/coreos/clair/worker/detectors/data/docker"
-
 	_ "github.com/coreos/clair/worker/detectors/feature/apk"
 	_ "github.com/coreos/clair/worker/detectors/feature/dpkg"
 	_ "github.com/coreos/clair/worker/detectors/feature/rpm"

ext/imagefmt/aci/aci.go

@@ -0,0 +1,40 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aci
+
+import (
+	"io"
+	"path/filepath"
+
+	"github.com/coreos/clair/ext/imagefmt"
+	"github.com/coreos/clair/pkg/tarutil"
+)
+
+type format struct{}
+
+func init() {
+	imagefmt.RegisterExtractor("aci", &format{})
+}
+
+func (f format) ExtractFiles(layerReader io.ReadCloser, toExtract []string) (tarutil.FilesMap, error) {
+	// All contents are inside a "rootfs" directory, so this needs to be
+	// prepended to each filename.
+	var filenames []string
+	for _, filename := range toExtract {
+		filenames = append(filenames, filepath.Join("rootfs/", filename))
+	}
+
+	return tarutil.ExtractFiles(layerReader, filenames)
+}

ext/imagefmt/docker/docker.go

@@ -0,0 +1,32 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package docker
+
+import (
+	"io"
+
+	"github.com/coreos/clair/ext/imagefmt"
+	"github.com/coreos/clair/pkg/tarutil"
+)
+
+type format struct{}
+
+func init() {
+	imagefmt.RegisterExtractor("docker", &format{})
+}
+
+func (f format) ExtractFiles(layerReader io.ReadCloser, toExtract []string) (tarutil.FilesMap, error) {
+	return tarutil.ExtractFiles(layerReader, toExtract)
+}

ext/imagefmt/driver.go

@@ -0,0 +1,150 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package notification fetches notifications from the database and informs the
+// specified remote handler about their existences, inviting the third party to
+// actively query the API about it.
+
+// Package imagefmt exposes functions to dynamically register methods to
+// detect different types of container image formats.
+package imagefmt
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/coreos/clair/pkg/commonerr"
+	"github.com/coreos/clair/pkg/tarutil"
+	"github.com/coreos/pkg/capnslog"
+)
+
+var (
+	// ErrCouldNotFindLayer is returned when we could not download or open the layer file.
+	ErrCouldNotFindLayer = commonerr.NewBadRequestError("could not find layer")
+
+	log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/imagefmt")
+
+	extractorsM sync.RWMutex
+	extractors  = make(map[string]Extractor)
+)
+
+// Extractor represents an ability to extract files from a particular container
+// image format.
+type Extractor interface {
+	// ExtractFiles produces a tarutil.FilesMap from a image layer.
+	ExtractFiles(layer io.ReadCloser, filenames []string) (tarutil.FilesMap, error)
+}
+
+// RegisterExtractor makes a extractor available by the provided name.
+//
+// If called twice with the same name, the name is blank, or if the provided
+// Extractor is nil, this function panics.
+func RegisterExtractor(name string, d Extractor) {
+	extractorsM.Lock()
+	defer extractorsM.Unlock()
+
+	if name == "" {
+		panic("imagefmt: could not register a extractor with an empty name")
+	}
+
+	if d == nil {
+		panic("imagefmt: could not register a nil extractor")
+	}
+
+	// Enforce lowercase names, so that they can be reliably be found in a map.
+	name = strings.ToLower(name)
+
+	if _, dup := extractors[name]; dup {
+		panic("imagefmt: RegisterExtractor called twice for " + name)
+	}
+
+	extractors[name] = d
+}
+
+// Extractors returns the list of the registered extractors.
+func Extractors() map[string]Extractor {
+	extractorsM.RLock()
+	defer extractorsM.RUnlock()
+
+	ret := make(map[string]Extractor)
+	for k, v := range extractors {
+		ret[k] = v
+	}
+
+	return ret
+}
+
+// UnregisterExtractor removes a Extractor with a particular name from the list.
+func UnregisterExtractor(name string) {
+	extractorsM.Lock()
+	defer extractorsM.Unlock()
+	delete(extractors, name)
+}
+
+// Extract streams an image layer from disk or over HTTP, determines the
+// image format, then extracts the files specified.
+func Extract(format, path string, headers map[string]string, toExtract []string) (tarutil.FilesMap, error) {
+	var layerReader io.ReadCloser
+	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
+		// Create a new HTTP request object.
+		request, err := http.NewRequest("GET", path, nil)
+		if err != nil {
+			return nil, ErrCouldNotFindLayer
+		}
+
+		// Set any provided HTTP Headers.
+		if headers != nil {
+			for k, v := range headers {
+				request.Header.Set(k, v)
+			}
+		}
+
+		// Send the request and handle the response.
+		r, err := http.DefaultClient.Do(request)
+		if err != nil {
+			log.Warningf("could not download layer: %s", err)
+			return nil, ErrCouldNotFindLayer
+		}
+
+		// Fail if we don't receive a 2xx HTTP status code.
+		if math.Floor(float64(r.StatusCode/100)) != 2 {
+			log.Warningf("could not download layer: got status code %d, expected 2XX", r.StatusCode)
+			return nil, ErrCouldNotFindLayer
+		}
+
+		layerReader = r.Body
+	} else {
+		var err error
+		layerReader, err = os.Open(path)
+		if err != nil {
+			return nil, ErrCouldNotFindLayer
+		}
+	}
+	defer layerReader.Close()
+
+	if extractor, exists := Extractors()[strings.ToLower(format)]; exists {
+		files, err := extractor.ExtractFiles(layerReader, toExtract)
+		if err != nil {
+			return nil, err
+		}
+		return files, nil
+	}
+
+	return nil, commonerr.NewBadRequestError(fmt.Sprintf("unsupported image format '%s'", format))
+}

worker/detectors/data.go

@@ -1,122 +0,0 @@
-// Copyright 2015 clair authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package detectors exposes functions to register and use container
-// information extractors.
-package detectors
-
-import (
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"os"
-	"strings"
-	"sync"
-
-	cerrors "github.com/coreos/clair/utils/errors"
-	"github.com/coreos/pkg/capnslog"
-)
-
-// The DataDetector interface defines a way to detect the required data from input path
-type DataDetector interface {
-	//Support check if the input path and format are supported by the underling detector
-	Supported(path string, format string) bool
-	// Detect detects the required data from input path
-	Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (data map[string][]byte, err error)
-}
-
-var (
-	dataDetectorsLock sync.Mutex
-	dataDetectors     = make(map[string]DataDetector)
-
-	log = capnslog.NewPackageLogger("github.com/coreos/clair", "detectors")
-
-	// ErrCouldNotFindLayer is returned when we could not download or open the layer file.
-	ErrCouldNotFindLayer = cerrors.NewBadRequestError("could not find layer")
-)
-
-// RegisterDataDetector provides a way to dynamically register an implementation of a
-// DataDetector.
-//
-// If RegisterDataDetector is called twice with the same name if DataDetector is nil,
-// or if the name is blank, it panics.
-func RegisterDataDetector(name string, f DataDetector) {
-	if name == "" {
-		panic("Could not register a DataDetector with an empty name")
-	}
-	if f == nil {
-		panic("Could not register a nil DataDetector")
-	}
-
-	dataDetectorsLock.Lock()
-	defer dataDetectorsLock.Unlock()
-
-	if _, alreadyExists := dataDetectors[name]; alreadyExists {
-		panic(fmt.Sprintf("Detector '%s' is already registered", name))
-	}
-	dataDetectors[name] = f
-}
-
-// DetectData finds the Data of the layer by using every registered DataDetector
-func DetectData(format, path string, headers map[string]string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) {
-	var layerReader io.ReadCloser
-	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
-		// Create a new HTTP request object.
-		request, err := http.NewRequest("GET", path, nil)
-		if err != nil {
-			return nil, ErrCouldNotFindLayer
-		}
-
-		// Set any provided HTTP Headers.
-		if headers != nil {
-			for k, v := range headers {
-				request.Header.Set(k, v)
-			}
-		}
-
-		// Send the request and handle the response.
-		r, err := http.DefaultClient.Do(request)
-		if err != nil {
-			log.Warningf("could not download layer: %s", err)
-			return nil, ErrCouldNotFindLayer
-		}
-
-		// Fail if we don't receive a 2xx HTTP status code.
-		if math.Floor(float64(r.StatusCode/100)) != 2 {
-			log.Warningf("could not download layer: got status code %d, expected 2XX", r.StatusCode)
-			return nil, ErrCouldNotFindLayer
-		}
-
-		layerReader = r.Body
-	} else {
-		layerReader, err = os.Open(path)
-		if err != nil {
-			return nil, ErrCouldNotFindLayer
-		}
-	}
-	defer layerReader.Close()
-
-	for _, detector := range dataDetectors {
-		if detector.Supported(path, format) {
-			data, err = detector.Detect(layerReader, toExtract, maxFileSize)
-			if err != nil {
-				return nil, err
-			}
-			return data, nil
-		}
-	}
-
-	return nil, cerrors.NewBadRequestError(fmt.Sprintf("unsupported image format '%s'", format))
-}

worker/detectors/data/aci/aci.go

@@ -1,41 +0,0 @@
-// Copyright 2015 clair authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package aci
-
-import (
-	"io"
-	"strings"
-
-	"github.com/coreos/clair/utils"
-	"github.com/coreos/clair/worker/detectors"
-)
-
-// ACIDataDetector implements DataDetector and detects layer data in 'aci' format
-type ACIDataDetector struct{}
-
-func init() {
-	detectors.RegisterDataDetector("aci", &ACIDataDetector{})
-}
-
-func (detector *ACIDataDetector) Supported(path string, format string) bool {
-	if strings.EqualFold(format, "ACI") {
-		return true
-	}
-	return false
-}
-
-func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
-	return utils.SelectivelyExtractArchive(layerReader, "rootfs/", toExtract, maxFileSize)
-}

worker/detectors/data/docker/docker.go

@@ -1,41 +0,0 @@
-// Copyright 2015 clair authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package docker
-
-import (
-	"io"
-	"strings"
-
-	"github.com/coreos/clair/utils"
-	"github.com/coreos/clair/worker/detectors"
-)
-
-// DockerDataDetector implements DataDetector and detects layer data in 'Docker' format
-type DockerDataDetector struct{}
-
-func init() {
-	detectors.RegisterDataDetector("Docker", &DockerDataDetector{})
-}
-
-func (detector *DockerDataDetector) Supported(path string, format string) bool {
-	if strings.EqualFold(format, "Docker") {
-		return true
-	}
-	return false
-}
-
-func (detector *DockerDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
-	return utils.SelectivelyExtractArchive(layerReader, "", toExtract, maxFileSize)
-}

worker/worker.go

@@ -20,6 +20,7 @@ import (
 	"github.com/coreos/pkg/capnslog"
 
 	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/ext/imagefmt"
 	"github.com/coreos/clair/pkg/commonerr"
 	"github.com/coreos/clair/utils"
 	"github.com/coreos/clair/worker/detectors"
@@ -29,11 +30,6 @@ const (
 	// Version (integer) represents the worker version.
 	// Increased each time the engine changes.
 	Version = 3
-
-	// maxFileSize enforces a maximum size of a single file within a tarball that
-	// will be extracted. This protects against malicious layers that may contain
-	// extremely large package database files.
-	maxFileSize = 200 * 1024 * 1024 // 200 MiB
 )
 
 var (
@@ -116,12 +112,14 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s
 
 // detectContent downloads a layer's archive and extracts its Namespace and Features.
 func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) {
-	data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize)
+	files, err := imagefmt.Extract(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...))
 	if err != nil {
 		log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
 		return
 	}
 
+	data := map[string][]byte(files)
+
 	// Detect namespace.
 	namespace = detectNamespace(name, data, parent)