add data detector to support ACI and other format in the future

Signed-off-by: liangchenye <liangchenye@huawei.com>

api/logic/layers.go

@@ -30,7 +30,7 @@ import (
 
 // POSTLayersParameters represents the expected parameters for POSTLayers.
 type POSTLayersParameters struct {
-	ID, Path, ParentID string
+	ID, Path, ParentID, Type string
 }
 
 // POSTLayers analyzes a layer and returns the engine version that has been used
@@ -43,7 +43,7 @@ func POSTLayers(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 	}
 
 	// Process data.
-	if err := worker.Process(parameters.ID, parameters.ParentID, parameters.Path); err != nil {
+	if err := worker.Process(parameters.ID, parameters.ParentID, parameters.Path, parameters.Type); err != nil {
 		httputils.WriteHTTPError(w, 0, err)
 		return
 	}

cmd/clair/main.go

@@ -27,6 +27,7 @@ import (
 
 	// Register components
 	_ "github.com/coreos/clair/updater/fetchers"
+	_ "github.com/coreos/clair/worker/detectors/data"
 	_ "github.com/coreos/clair/worker/detectors/os"
 	_ "github.com/coreos/clair/worker/detectors/packages"
 )

utils/tar.go

@@ -37,7 +37,7 @@ var (
 
 // SelectivelyExtractArchive extracts the specified files and folders
 // from targz data read from the given reader and store them in a map indexed by file paths
-func SelectivelyExtractArchive(r io.Reader, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
+func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
 	data := make(map[string][]byte)
 
 	// Create a tar or tar/tar-gzip reader
@@ -58,7 +58,7 @@ func SelectivelyExtractArchive(r io.Reader, toExtract []string, maxFileSize int6
 
 		// Get element filename
 		filename := hdr.Name
-		filename = strings.TrimPrefix(filename, "./")
+		filename = strings.TrimPrefix(filename, prefix)
 
 		// Determine if we should extract the element
 		toBeExtracted := false

utils/utils_test.go

@@ -65,13 +65,13 @@ func TestTar(t *testing.T) {
 		testArchivePath := path.Join(path.Dir(filepath)) + filename
 
 		// Extract non compressed data
-		data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), []string{}, 0)
+		data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), "./", []string{}, 0)
 		assert.Error(t, err, "Extracting non compressed data should return an error")
 
 		// Extract an archive
 		f, _ := os.Open(testArchivePath)
 		defer f.Close()
-		data, err = SelectivelyExtractArchive(f, []string{"test/"}, 0)
+		data, err = SelectivelyExtractArchive(f, "./", []string{"test/"}, 0)
 		assert.Nil(t, err)
 
 		if c, n := data["test/test.txt"]; !n {
@@ -86,7 +86,7 @@ func TestTar(t *testing.T) {
 		// File size limit
 		f, _ = os.Open(testArchivePath)
 		defer f.Close()
-		data, err = SelectivelyExtractArchive(f, []string{"test"}, 50)
+		data, err = SelectivelyExtractArchive(f, "./", []string{"test"}, 50)
 		assert.Equal(t, ErrExtractedFileTooBig, err)
 	}
 }

worker/detectors/data.go

@@ -0,0 +1,91 @@
+// Copyright 2015 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package detectors exposes functions to register and use container
+// information extractors.
+package detectors
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+
+	cerrors "github.com/coreos/clair/utils/errors"
+)
+
+// The DataDetector interface defines a way to detect the required data from input path
+type DataDetector interface {
+	//Support check if the input path and format are supported by the underling detector
+	Supported(path string, format string) bool
+	// Detect detects the required data from input path
+	Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (data map[string][]byte, err error)
+}
+
+var (
+	dataDetectorsLock sync.Mutex
+	dataDetectors     = make(map[string]DataDetector)
+)
+
+// RegisterDataDetector provides a way to dynamically register an implementation of a
+// DataDetector.
+//
+// If RegisterDataDetector is called twice with the same name if DataDetector is nil,
+// or if the name is blank, it panics.
+func RegisterDataDetector(name string, f DataDetector) {
+	if name == "" {
+		panic("Could not register a DataDetector with an empty name")
+	}
+	if f == nil {
+		panic("Could not register a nil DataDetector")
+	}
+
+	dataDetectorsLock.Lock()
+	defer dataDetectorsLock.Unlock()
+
+	if _, alreadyExists := dataDetectors[name]; alreadyExists {
+		panic(fmt.Sprintf("Detector '%s' is already registered", name))
+	}
+	dataDetectors[name] = f
+}
+
+// DetectData finds the Data of the layer by using every registered DataDetector
+func DetectData(path string, format string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) {
+	var layerReader io.ReadCloser
+	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
+		r, err := http.Get(path)
+		if err != nil {
+			return nil, cerrors.ErrCouldNotDownload
+		}
+		layerReader = r.Body
+	} else {
+		layerReader, err = os.Open(path)
+		if err != nil {
+			return nil, cerrors.ErrNotFound
+		}
+	}
+	defer layerReader.Close()
+
+	for _, detector := range dataDetectors {
+		if detector.Supported(path, format) {
+			if data, err = detector.Detect(layerReader, toExtract, maxFileSize); err == nil {
+				return data, nil
+			}
+		}
+	}
+
+	return nil, nil
+}

worker/detectors/data/aci.go

@@ -0,0 +1,46 @@
+// Copyright 2015 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package data
+
+import (
+	"io"
+	"strings"
+
+	"github.com/coreos/clair/utils"
+	"github.com/coreos/clair/worker/detectors"
+)
+
+// ACIDataDetector implements DataDetector and detects layer data in 'aci' format
+type ACIDataDetector struct{}
+
+func init() {
+	detectors.RegisterDataDetector("aci", &ACIDataDetector{})
+}
+
+func (detector *ACIDataDetector) Supported(path string, format string) bool {
+	switch format {
+	case "":
+		if strings.HasSuffix(path, ".aci") {
+			return true
+		}
+	case "aci":
+		return true
+	}
+	return false
+}
+
+func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
+	return utils.SelectivelyExtractArchive(layerReader, "./rootfs/", toExtract, maxFileSize)
+}

worker/detectors/data/tar.go

@@ -0,0 +1,46 @@
+// Copyright 2015 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package data
+
+import (
+	"io"
+	"strings"
+
+	"github.com/coreos/clair/utils"
+	"github.com/coreos/clair/worker/detectors"
+)
+
+// TarDataDetector implements DataDetector and detects layer data in 'tar' format
+type TarDataDetector struct{}
+
+func init() {
+	detectors.RegisterDataDetector("tar", &TarDataDetector{})
+}
+
+func (detector *TarDataDetector) Supported(path string, format string) bool {
+	switch format {
+	case "":
+		if strings.HasSuffix(path, ".tar") || strings.HasSuffix(path, ".tar.gz") {
+			return true
+		}
+	case "tar":
+		return true
+	}
+	return false
+}
+
+func (detector *TarDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
+	return utils.SelectivelyExtractArchive(layerReader, "./", toExtract, maxFileSize)
+}

worker/worker.go

@@ -18,9 +18,6 @@ package worker
 
 import (
 	"errors"
-	"io"
-	"net/http"
-	"os"
 	"strings"
 
 	"github.com/coreos/clair/database"
@@ -56,7 +53,7 @@ var (
 
 // Process detects the OS of a layer, the packages it installs/removes, and
 // then stores everything in the database.
-func Process(ID, parentID, path string) error {
+func Process(ID, parentID, path string, imageFormat string) error {
 	if ID == "" {
 		return cerrors.NewBadRequestError("could not process a layer which does not have ID")
 	}
@@ -64,7 +61,7 @@ func Process(ID, parentID, path string) error {
 		return cerrors.NewBadRequestError("could not process a layer which does not have a path")
 	}
 
-	log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s)", ID, utils.CleanURL(path), Version, parentID)
+	log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, format: %s)", ID, utils.CleanURL(path), Version, parentID, imageFormat)
 
 	// Check to see if the layer is already in the database.
 	layer, err := database.FindOneLayerByID(ID, []string{database.FieldLayerEngineVersion})
@@ -101,7 +98,7 @@ func Process(ID, parentID, path string) error {
 	}
 
 	// Analyze the content.
-	layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent)
+	layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent, imageFormat)
 	if err != nil {
 		return err
 	}
@@ -114,8 +111,8 @@ func Process(ID, parentID, path string) error {
 //
 // If parent is not nil, database.FieldLayerOS, database.FieldLayerPackages fields must be
 // has been selectioned.
-func detectContent(ID, path string, parent *database.Layer) (OS string, installedPackagesNodes, removedPackagesNodes []string, err error) {
-	data, err := getLayerData(path)
+func detectContent(ID, path string, parent *database.Layer, imageFormat string) (OS string, installedPackagesNodes, removedPackagesNodes []string, err error) {
+	data, err := getLayerData(path, imageFormat)
 	if err != nil {
 		log.Errorf("layer %s: failed to extract data from %s: %s", ID, utils.CleanURL(path), err)
 		return
@@ -182,23 +179,8 @@ func detectContent(ID, path string, parent *database.Layer) (OS string, installe
 }
 
 // getLayerData downloads/opens a layer archive and extracts it into memory.
-func getLayerData(path string) (data map[string][]byte, err error) {
-	var layerReader io.ReadCloser
-	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
-		r, err := http.Get(path)
-		if err != nil {
-			return nil, cerrors.ErrCouldNotDownload
-		}
-		layerReader = r.Body
-	} else {
-		layerReader, err = os.Open(path)
-		if err != nil {
-			return nil, cerrors.ErrNotFound
-		}
-	}
-	defer layerReader.Close()
-
-	data, err = utils.SelectivelyExtractArchive(layerReader, append(detectors.GetRequiredFilesPackages(), detectors.GetRequiredFilesOS()...), maxFileSize)
+func getLayerData(path string, imageFormat string) (data map[string][]byte, err error) {
+	data, err = detectors.DetectData(path, imageFormat, append(detectors.GetRequiredFilesPackages(), detectors.GetRequiredFilesOS()...), maxFileSize)
 	if err != nil {
 		return nil, err
 	}

worker/worker_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	// Register detectors
+	_ "github.com/coreos/clair/worker/detectors/data"
 	_ "github.com/coreos/clair/worker/detectors/os"
 	_ "github.com/coreos/clair/worker/detectors/packages"
 )
@@ -25,9 +26,9 @@ func TestDistUpgrade(t *testing.T) {
 	// blank.tar: MAINTAINER Quentin MACHU <quentin.machu.fr>
 	// wheezy.tar: FROM debian:wheezy
 	// jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update && apt-get -y dist-upgrade
-	assert.Nil(t, Process("blank", "", path+"blank.tar.gz"))
-	assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz"))
-	assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz"))
+	assert.Nil(t, Process("blank", "", path+"blank.tar.gz", ""))
+	assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz", ""))
+	assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz", ""))
 
 	wheezy, err := database.FindOneLayerByID("wheezy", database.FieldLayerAll)
 	if assert.Nil(t, err) {