Merge pull request #49 from liangchenye/master

Add DataDetector to support ACI and other layout format

Dockerfile

@@ -1,7 +1,7 @@
 FROM golang:1.5
 MAINTAINER Quentin Machu <quentin.machu@coreos.com>
 
-RUN apt-get update && apt-get install -y bzr rpm && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN apt-get update && apt-get install -y bzr rpm xz && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 RUN mkdir         /db
 VOLUME            /db

api/logic/layers.go

@@ -30,7 +30,7 @@ import (
 
 // POSTLayersParameters represents the expected parameters for POSTLayers.
 type POSTLayersParameters struct {
-	ID, Path, ParentID string
+	ID, Path, ParentID, ImageFormat string
 }
 
 // POSTLayers analyzes a layer and returns the engine version that has been used
@@ -43,7 +43,7 @@ func POSTLayers(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 	}
 
 	// Process data.
-	if err := worker.Process(parameters.ID, parameters.ParentID, parameters.Path); err != nil {
+	if err := worker.Process(parameters.ID, parameters.ParentID, parameters.Path, parameters.ImageFormat); err != nil {
 		httputils.WriteHTTPError(w, 0, err)
 		return
 	}

cmd/clair/main.go

@@ -28,6 +28,7 @@ import (
 	// Register components
 	_ "github.com/coreos/clair/notifier/notifiers"
 	_ "github.com/coreos/clair/updater/fetchers"
+	_ "github.com/coreos/clair/worker/detectors/data"
 	_ "github.com/coreos/clair/worker/detectors/os"
 	_ "github.com/coreos/clair/worker/detectors/packages"
 )

contrib/analyze-local-images/main.go

@@ -197,7 +197,7 @@ func history(imageName string) ([]string, error) {
 }
 
 func analyzeLayer(endpoint, path, layerID, parentLayerID string) error {
-	payload := struct{ ID, Path, ParentID string }{ID: layerID, Path: path, ParentID: parentLayerID}
+	payload := struct{ ID, Path, ParentID, ImageFormat string }{ID: layerID, Path: path, ParentID: parentLayerID, ImageFormat: "Docker"}
 	jsonPayload, err := json.Marshal(payload)
 	if err != nil {
 		return err

contrib/check-openvz-mirror-with-clair/main.go

@@ -51,6 +51,7 @@ type AddLayoutRequestAPI struct {
 	ID       string `json:"ID"`
 	Path     string `json:"Path"`
 	ParantID string `json:"ParantID"`
+	ImageFormat string `json:"ImageFormat"`
 }
 
 type VulnerabilityItem struct {
@@ -223,7 +224,7 @@ func (clair ClairAPI) AddLayer(openvzMirror string, templateName string) error {
 		client = httpClient
 	}
 
-	jsonRequest, err := json.Marshal(AddLayoutRequestAPI{ID: templateName, Path: openvzMirror + "/" + templateName + ".tar.gz"})
+	jsonRequest, err := json.Marshal(AddLayoutRequestAPI{ID: templateName, Path: openvzMirror + "/" + templateName + ".tar.gz", ImageFormat: "Docker"})
 	if err != nil {
 		log.Println("Cannot convert to json request with error: ", err)
 		return err

docs/API.md

@@ -112,7 +112,8 @@ It processes and inserts a new Layer in the database.
 |------|-----|-------------|
 |ID|String|Unique ID of the Layer|
 |Path|String|Absolute path or HTTP link pointing to the Layer's tar file|
-|ParentID|String|(Optionnal) Unique ID of the Layer's parent
+|ParentID|String|(Optional) Unique ID of the Layer's parent|
+|ImageFormat|String|Image format of the Layer ('Docker' or 'ACI')|
 
 If the Layer has not parent, the ParentID field should be omitted or empty.
 
@@ -346,7 +347,7 @@ It returns the lists of vulnerabilities which affect a given Layer.
 |Name|Type|Description|
 |------|-----|-------------|
 |ID|String|Unique ID of the Layer|
-|minimumPriority|Priority|(Optionnal) The minimum priority of the returned vulnerabilities. Defaults to High|
+|minimumPriority|Priority|(Optional) The minimum priority of the returned vulnerabilities. Defaults to High|
 
 ### Example
 
@@ -389,7 +390,7 @@ It returns the lists of vulnerabilities which are introduced and removed by the
 |Name|Type|Description|
 |------|-----|-------------|
 |ID|String|Unique ID of the Layer|
-|minimumPriority|Priority|(Optionnal) The minimum priority of the returned vulnerabilities|
+|minimumPriority|Priority|(Optional) The minimum priority of the returned vulnerabilities|
 
 ### Example
 
@@ -436,7 +437,7 @@ Counterintuitively, this request is actually a POST to be able to pass a lot of
 |Name|Type|Description|
 |------|-----|-------------|
 |LayersIDs|Array of strings|Unique IDs of Layers|
-|minimumPriority|Priority|(Optionnal) The minimum priority of the returned vulnerabilities. Defaults to High|
+|minimumPriority|Priority|(Optional) The minimum priority of the returned vulnerabilities. Defaults to High|
 
 ### Example
 

utils/tar.go

@@ -18,10 +18,12 @@ import (
 	"archive/tar"
 	"bufio"
 	"bytes"
+	"compress/bzip2"
 	"compress/gzip"
 	"errors"
 	"io"
 	"io/ioutil"
+	"os/exec"
 	"strings"
 )
 
@@ -32,19 +34,75 @@ var (
 	// ErrExtractedFileTooBig occurs when a file to extract is too big.
 	ErrExtractedFileTooBig = errors.New("utils: could not extract one or more files from the archive: file too big")
 
-	gzipHeader = []byte{0x1f, 0x8b}
+	readLen = 6 // max bytes to sniff
+
+	gzipHeader  = []byte{0x1f, 0x8b}
+	bzip2Header = []byte{0x42, 0x5a, 0x68}
+	xzHeader    = []byte{0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00}
 )
 
+// XzReader is an io.ReadCloser which decompresses xz compressed data.
+type XzReader struct {
+	io.ReadCloser
+	cmd     *exec.Cmd
+	closech chan error
+}
+
+// NewXzReader shells out to a command line xz executable (if
+// available) to decompress the given io.Reader using the xz
+// compression format and returns an *XzReader.
+// It is the caller's responsibility to call Close on the XzReader when done.
+func NewXzReader(r io.Reader) (*XzReader, error) {
+	rpipe, wpipe := io.Pipe()
+	ex, err := exec.LookPath("xz")
+	if err != nil {
+		return nil, err
+	}
+	cmd := exec.Command(ex, "--decompress", "--stdout")
+
+	closech := make(chan error)
+
+	cmd.Stdin = r
+	cmd.Stdout = wpipe
+
+	go func() {
+		err := cmd.Run()
+		wpipe.CloseWithError(err)
+		closech <- err
+	}()
+
+	return &XzReader{rpipe, cmd, closech}, nil
+}
+
+func (r *XzReader) Close() error {
+	r.ReadCloser.Close()
+	r.cmd.Process.Kill()
+	return <-r.closech
+}
+
+// TarReadCloser embeds a *tar.Reader and the related io.Closer
+// It is the caller's responsibility to call Close on TarReadCloser when
+// done.
+type TarReadCloser struct {
+	*tar.Reader
+	io.Closer
+}
+
+func (r *TarReadCloser) Close() error {
+	return r.Closer.Close()
+}
+
 // SelectivelyExtractArchive extracts the specified files and folders
 // from targz data read from the given reader and store them in a map indexed by file paths
-func SelectivelyExtractArchive(r io.Reader, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
+func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
 	data := make(map[string][]byte)
 
-	// Create a tar or tar/tar-gzip reader
+	// Create a tar or tar/tar-gzip/tar-bzip2/tar-xz reader
 	tr, err := getTarReader(r)
 	if err != nil {
 		return data, ErrCouldNotExtract
 	}
+	defer tr.Close()
 
 	// For each element in the archive
 	for {
@@ -59,6 +117,9 @@ func SelectivelyExtractArchive(r io.Reader, toExtract []string, maxFileSize int6
 		// Get element filename
 		filename := hdr.Name
 		filename = strings.TrimPrefix(filename, "./")
+		if prefix != "" {
+			filename = strings.TrimPrefix(filename, prefix)
+		}
 
 		// Determine if we should extract the element
 		toBeExtracted := false
@@ -86,22 +147,35 @@ func SelectivelyExtractArchive(r io.Reader, toExtract []string, maxFileSize int6
 	return data, nil
 }
 
-// getTarReader returns a tar.Reader associated with the specified io.Reader,
+// getTarReader returns a TarReaderCloser associated with the specified io.Reader.
-// optionally backed by a gzip.Reader if gzip compression is detected.
 //
-// Gzip detection is done by using the magic numbers defined in the RFC1952 :
+// Gzip/Bzip2/XZ detection is done by using the magic numbers:
-// the first two bytes should be 0x1f and 0x8b..
+// Gzip: the first two bytes should be 0x1f and 0x8b. Defined in the RFC1952.
-func getTarReader(r io.Reader) (*tar.Reader, error) {
+// Bzip2: the first three bytes should be 0x42, 0x5a and 0x68. No RFC.
+// XZ: the first three bytes should be 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00. No RFC.
+func getTarReader(r io.Reader) (*TarReadCloser, error) {
 	br := bufio.NewReader(r)
-	header, err := br.Peek(2)
+	header, err := br.Peek(readLen)
-
+	if err == nil {
-	if err == nil && bytes.Equal(header, gzipHeader) {
+		switch {
-		gr, err := gzip.NewReader(br)
+		case bytes.HasPrefix(header, gzipHeader):
-		if err != nil {
+			gr, err := gzip.NewReader(br)
-			return nil, err
+			if err != nil {
+				return nil, err
+			}
+			return &TarReadCloser{tar.NewReader(gr), gr}, nil
+		case bytes.HasPrefix(header, bzip2Header):
+			bzip2r := ioutil.NopCloser(bzip2.NewReader(br))
+			return &TarReadCloser{tar.NewReader(bzip2r), bzip2r}, nil
+		case bytes.HasPrefix(header, xzHeader):
+			xzr, err := NewXzReader(br)
+			if err != nil {
+				return nil, err
+			}
+			return &TarReadCloser{tar.NewReader(xzr), xzr}, nil
 		}
-		return tar.NewReader(gr), nil
 	}
 
-	return tar.NewReader(br), nil
+	dr := ioutil.NopCloser(br)
+	return &TarReadCloser{tar.NewReader(dr), dr}, nil
 }

utils/utils_test.go

@@ -60,18 +60,18 @@ func TestTar(t *testing.T) {
 	var err error
 	var data map[string][]byte
 	_, filepath, _, _ := runtime.Caller(0)
-
+	testDataDir := "/testdata"
-	for _, filename := range []string{"/testdata/utils_test.tar.gz", "/testdata/utils_test.tar"} {
+	for _, filename := range []string{"utils_test.tar.gz", "utils_test.tar.bz2", "utils_test.tar.xz", "utils_test.tar"} {
-		testArchivePath := path.Join(path.Dir(filepath)) + filename
+		testArchivePath := path.Join(path.Dir(filepath), testDataDir, filename)
 
 		// Extract non compressed data
-		data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), []string{}, 0)
+		data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), "", []string{}, 0)
 		assert.Error(t, err, "Extracting non compressed data should return an error")
 
 		// Extract an archive
 		f, _ := os.Open(testArchivePath)
 		defer f.Close()
-		data, err = SelectivelyExtractArchive(f, []string{"test/"}, 0)
+		data, err = SelectivelyExtractArchive(f, "", []string{"test/"}, 0)
 		assert.Nil(t, err)
 
 		if c, n := data["test/test.txt"]; !n {
@@ -86,7 +86,7 @@ func TestTar(t *testing.T) {
 		// File size limit
 		f, _ = os.Open(testArchivePath)
 		defer f.Close()
-		data, err = SelectivelyExtractArchive(f, []string{"test"}, 50)
+		data, err = SelectivelyExtractArchive(f, "", []string{"test"}, 50)
 		assert.Equal(t, ErrExtractedFileTooBig, err)
 	}
 }

worker/detectors/data.go

@@ -0,0 +1,91 @@
+// Copyright 2015 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package detectors exposes functions to register and use container
+// information extractors.
+package detectors
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+
+	cerrors "github.com/coreos/clair/utils/errors"
+)
+
+// The DataDetector interface defines a way to detect the required data from input path
+type DataDetector interface {
+	//Support check if the input path and format are supported by the underling detector
+	Supported(path string, format string) bool
+	// Detect detects the required data from input path
+	Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (data map[string][]byte, err error)
+}
+
+var (
+	dataDetectorsLock sync.Mutex
+	dataDetectors     = make(map[string]DataDetector)
+)
+
+// RegisterDataDetector provides a way to dynamically register an implementation of a
+// DataDetector.
+//
+// If RegisterDataDetector is called twice with the same name if DataDetector is nil,
+// or if the name is blank, it panics.
+func RegisterDataDetector(name string, f DataDetector) {
+	if name == "" {
+		panic("Could not register a DataDetector with an empty name")
+	}
+	if f == nil {
+		panic("Could not register a nil DataDetector")
+	}
+
+	dataDetectorsLock.Lock()
+	defer dataDetectorsLock.Unlock()
+
+	if _, alreadyExists := dataDetectors[name]; alreadyExists {
+		panic(fmt.Sprintf("Detector '%s' is already registered", name))
+	}
+	dataDetectors[name] = f
+}
+
+// DetectData finds the Data of the layer by using every registered DataDetector
+func DetectData(path string, format string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) {
+	var layerReader io.ReadCloser
+	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
+		r, err := http.Get(path)
+		if err != nil {
+			return nil, cerrors.ErrCouldNotDownload
+		}
+		layerReader = r.Body
+	} else {
+		layerReader, err = os.Open(path)
+		if err != nil {
+			return nil, cerrors.ErrNotFound
+		}
+	}
+	defer layerReader.Close()
+
+	for _, detector := range dataDetectors {
+		if detector.Supported(path, format) {
+			if data, err = detector.Detect(layerReader, toExtract, maxFileSize); err == nil {
+				return data, nil
+			}
+		}
+	}
+
+	return nil, nil
+}

worker/detectors/data/aci.go

@@ -0,0 +1,41 @@
+// Copyright 2015 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package data
+
+import (
+	"io"
+	"strings"
+
+	"github.com/coreos/clair/utils"
+	"github.com/coreos/clair/worker/detectors"
+)
+
+// ACIDataDetector implements DataDetector and detects layer data in 'aci' format
+type ACIDataDetector struct{}
+
+func init() {
+	detectors.RegisterDataDetector("aci", &ACIDataDetector{})
+}
+
+func (detector *ACIDataDetector) Supported(path string, format string) bool {
+	if strings.EqualFold(format, "ACI") {
+		return true
+	}
+	return false
+}
+
+func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
+	return utils.SelectivelyExtractArchive(layerReader, "rootfs/", toExtract, maxFileSize)
+}

worker/detectors/data/docker.go

@@ -0,0 +1,41 @@
+// Copyright 2015 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package data
+
+import (
+	"io"
+	"strings"
+
+	"github.com/coreos/clair/utils"
+	"github.com/coreos/clair/worker/detectors"
+)
+
+// DockerDataDetector implements DataDetector and detects layer data in 'Docker' format
+type DockerDataDetector struct{}
+
+func init() {
+	detectors.RegisterDataDetector("Docker", &DockerDataDetector{})
+}
+
+func (detector *DockerDataDetector) Supported(path string, format string) bool {
+	if strings.EqualFold(format, "Docker") {
+		return true
+	}
+	return false
+}
+
+func (detector *DockerDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
+	return utils.SelectivelyExtractArchive(layerReader, "", toExtract, maxFileSize)
+}

worker/worker.go

@@ -18,9 +18,6 @@ package worker
 
 import (
 	"errors"
-	"io"
-	"net/http"
-	"os"
 	"strings"
 
 	"github.com/coreos/clair/database"
@@ -52,19 +49,36 @@ var (
 
 	// SupportedOS is the list of operating system names that the worker supports.
 	SupportedOS = []string{"debian", "ubuntu", "centos"}
+
+	// SupportedImageFormat is the list of image formats that the worker supports.
+	SupportedImageFormat = []string{"Docker", "ACI"}
 )
 
 // Process detects the OS of a layer, the packages it installs/removes, and
 // then stores everything in the database.
-func Process(ID, parentID, path string) error {
+func Process(ID, parentID, path string, imageFormat string) error {
 	if ID == "" {
 		return cerrors.NewBadRequestError("could not process a layer which does not have ID")
 	}
 	if path == "" {
 		return cerrors.NewBadRequestError("could not process a layer which does not have a path")
 	}
+	if imageFormat == "" {
+		return cerrors.NewBadRequestError("could not process a layer which does not have a specified format")
+	} else {
+		isSupported := false
+		for _, format := range SupportedImageFormat {
+			if strings.EqualFold(imageFormat, format) {
+				isSupported = true
+				break
+			}
+		}
+		if !isSupported {
+			return cerrors.NewBadRequestError("could not process a layer which does not have a supported format")
+		}
+	}
 
-	log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s)", ID, utils.CleanURL(path), Version, parentID)
+	log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, Format: %s)", ID, utils.CleanURL(path), Version, parentID, imageFormat)
 
 	// Check to see if the layer is already in the database.
 	layer, err := database.FindOneLayerByID(ID, []string{database.FieldLayerEngineVersion})
@@ -101,7 +115,7 @@ func Process(ID, parentID, path string) error {
 	}
 
 	// Analyze the content.
-	layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent)
+	layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent, imageFormat)
 	if err != nil {
 		return err
 	}
@@ -114,8 +128,8 @@ func Process(ID, parentID, path string) error {
 //
 // If parent is not nil, database.FieldLayerOS, database.FieldLayerPackages fields must be
 // has been selectioned.
-func detectContent(ID, path string, parent *database.Layer) (OS string, installedPackagesNodes, removedPackagesNodes []string, err error) {
+func detectContent(ID, path string, parent *database.Layer, imageFormat string) (OS string, installedPackagesNodes, removedPackagesNodes []string, err error) {
-	data, err := getLayerData(path)
+	data, err := getLayerData(path, imageFormat)
 	if err != nil {
 		log.Errorf("layer %s: failed to extract data from %s: %s", ID, utils.CleanURL(path), err)
 		return
@@ -182,23 +196,8 @@ func detectContent(ID, path string, parent *database.Layer) (OS string, installe
 }
 
 // getLayerData downloads/opens a layer archive and extracts it into memory.
-func getLayerData(path string) (data map[string][]byte, err error) {
+func getLayerData(path string, imageFormat string) (data map[string][]byte, err error) {
-	var layerReader io.ReadCloser
+	data, err = detectors.DetectData(path, imageFormat, append(detectors.GetRequiredFilesPackages(), detectors.GetRequiredFilesOS()...), maxFileSize)
-	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
-		r, err := http.Get(path)
-		if err != nil {
-			return nil, cerrors.ErrCouldNotDownload
-		}
-		layerReader = r.Body
-	} else {
-		layerReader, err = os.Open(path)
-		if err != nil {
-			return nil, cerrors.ErrNotFound
-		}
-	}
-	defer layerReader.Close()
-
-	data, err = utils.SelectivelyExtractArchive(layerReader, append(detectors.GetRequiredFilesPackages(), detectors.GetRequiredFilesOS()...), maxFileSize)
 	if err != nil {
 		return nil, err
 	}

worker/worker_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	// Register detectors
+	_ "github.com/coreos/clair/worker/detectors/data"
 	_ "github.com/coreos/clair/worker/detectors/os"
 	_ "github.com/coreos/clair/worker/detectors/packages"
 )
@@ -25,9 +26,15 @@ func TestDistUpgrade(t *testing.T) {
 	// blank.tar: MAINTAINER Quentin MACHU <quentin.machu.fr>
 	// wheezy.tar: FROM debian:wheezy
 	// jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update && apt-get -y dist-upgrade
-	assert.Nil(t, Process("blank", "", path+"blank.tar.gz"))
+	assert.Nil(t, Process("blank", "", path+"blank.tar.gz", "Docker"))
-	assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz"))
+	assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz", "Docker"))
-	assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz"))
+	assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz", "Docker"))
+
+	err := Process("blank", "", path+"blank.tar.gz", "")
+	assert.Error(t, err, "could not process a layer which does not have a specified format")
+
+	err = Process("blank", "", path+"blank.tar.gz", "invalid")
+	assert.Error(t, err, "could not process a layer which does not have a supported format")
 
 	wheezy, err := database.FindOneLayerByID("wheezy", database.FieldLayerAll)
 	if assert.Nil(t, err) {