diff --git a/api/logic/layers.go b/api/logic/layers.go index 6e24ebe4..a8655987 100644 --- a/api/logic/layers.go +++ b/api/logic/layers.go @@ -30,7 +30,7 @@ import ( // POSTLayersParameters represents the expected parameters for POSTLayers. type POSTLayersParameters struct { - ID, Path, ParentID string + ID, Path, ParentID, Type string } // POSTLayers analyzes a layer and returns the engine version that has been used @@ -43,7 +43,7 @@ func POSTLayers(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { } // Process data. - if err := worker.Process(parameters.ID, parameters.ParentID, parameters.Path); err != nil { + if err := worker.Process(parameters.ID, parameters.ParentID, parameters.Path, parameters.Type); err != nil { httputils.WriteHTTPError(w, 0, err) return } diff --git a/cmd/clair/main.go b/cmd/clair/main.go index c08546cf..8d89c11e 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -27,6 +27,7 @@ import ( // Register components _ "github.com/coreos/clair/updater/fetchers" + _ "github.com/coreos/clair/worker/detectors/data" _ "github.com/coreos/clair/worker/detectors/os" _ "github.com/coreos/clair/worker/detectors/packages" ) diff --git a/utils/tar.go b/utils/tar.go index 66d40ea5..477aa9cc 100644 --- a/utils/tar.go +++ b/utils/tar.go @@ -37,7 +37,7 @@ var ( // SelectivelyExtractArchive extracts the specified files and folders // from targz data read from the given reader and store them in a map indexed by file paths -func SelectivelyExtractArchive(r io.Reader, toExtract []string, maxFileSize int64) (map[string][]byte, error) { +func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []string, maxFileSize int64) (map[string][]byte, error) { data := make(map[string][]byte) // Create a tar or tar/tar-gzip reader @@ -58,7 +58,7 @@ func SelectivelyExtractArchive(r io.Reader, toExtract []string, maxFileSize int6 // Get element filename filename := hdr.Name - filename = strings.TrimPrefix(filename, "./") + filename = strings.TrimPrefix(filename, prefix) // Determine if we should extract the element toBeExtracted := false diff --git a/utils/utils_test.go b/utils/utils_test.go index f9c877c8..e543c1c1 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -65,13 +65,13 @@ func TestTar(t *testing.T) { testArchivePath := path.Join(path.Dir(filepath)) + filename // Extract non compressed data - data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), []string{}, 0) + data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), "./", []string{}, 0) assert.Error(t, err, "Extracting non compressed data should return an error") // Extract an archive f, _ := os.Open(testArchivePath) defer f.Close() - data, err = SelectivelyExtractArchive(f, []string{"test/"}, 0) + data, err = SelectivelyExtractArchive(f, "./", []string{"test/"}, 0) assert.Nil(t, err) if c, n := data["test/test.txt"]; !n { @@ -86,7 +86,7 @@ func TestTar(t *testing.T) { // File size limit f, _ = os.Open(testArchivePath) defer f.Close() - data, err = SelectivelyExtractArchive(f, []string{"test"}, 50) + data, err = SelectivelyExtractArchive(f, "./", []string{"test"}, 50) assert.Equal(t, ErrExtractedFileTooBig, err) } } diff --git a/worker/detectors/data.go b/worker/detectors/data.go new file mode 100644 index 00000000..23f50e11 --- /dev/null +++ b/worker/detectors/data.go @@ -0,0 +1,91 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package detectors exposes functions to register and use container +// information extractors. +package detectors + +import ( + "fmt" + "io" + "net/http" + "os" + "strings" + "sync" + + cerrors "github.com/coreos/clair/utils/errors" +) + +// The DataDetector interface defines a way to detect the required data from input path +type DataDetector interface { + //Support check if the input path and format are supported by the underling detector + Supported(path string, format string) bool + // Detect detects the required data from input path + Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) +} + +var ( + dataDetectorsLock sync.Mutex + dataDetectors = make(map[string]DataDetector) +) + +// RegisterDataDetector provides a way to dynamically register an implementation of a +// DataDetector. +// +// If RegisterDataDetector is called twice with the same name if DataDetector is nil, +// or if the name is blank, it panics. +func RegisterDataDetector(name string, f DataDetector) { + if name == "" { + panic("Could not register a DataDetector with an empty name") + } + if f == nil { + panic("Could not register a nil DataDetector") + } + + dataDetectorsLock.Lock() + defer dataDetectorsLock.Unlock() + + if _, alreadyExists := dataDetectors[name]; alreadyExists { + panic(fmt.Sprintf("Detector '%s' is already registered", name)) + } + dataDetectors[name] = f +} + +// DetectData finds the Data of the layer by using every registered DataDetector +func DetectData(path string, format string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) { + var layerReader io.ReadCloser + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + r, err := http.Get(path) + if err != nil { + return nil, cerrors.ErrCouldNotDownload + } + layerReader = r.Body + } else { + layerReader, err = os.Open(path) + if err != nil { + return nil, cerrors.ErrNotFound + } + } + defer layerReader.Close() + + for _, detector := range dataDetectors { + if detector.Supported(path, format) { + if data, err = detector.Detect(layerReader, toExtract, maxFileSize); err == nil { + return data, nil + } + } + } + + return nil, nil +} diff --git a/worker/detectors/data/aci.go b/worker/detectors/data/aci.go new file mode 100644 index 00000000..aa9526cd --- /dev/null +++ b/worker/detectors/data/aci.go @@ -0,0 +1,46 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package data + +import ( + "io" + "strings" + + "github.com/coreos/clair/utils" + "github.com/coreos/clair/worker/detectors" +) + +// ACIDataDetector implements DataDetector and detects layer data in 'aci' format +type ACIDataDetector struct{} + +func init() { + detectors.RegisterDataDetector("aci", &ACIDataDetector{}) +} + +func (detector *ACIDataDetector) Supported(path string, format string) bool { + switch format { + case "": + if strings.HasSuffix(path, ".aci") { + return true + } + case "aci": + return true + } + return false +} + +func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) { + return utils.SelectivelyExtractArchive(layerReader, "./rootfs/", toExtract, maxFileSize) +} diff --git a/worker/detectors/data/tar.go b/worker/detectors/data/tar.go new file mode 100644 index 00000000..ae1cc5a2 --- /dev/null +++ b/worker/detectors/data/tar.go @@ -0,0 +1,46 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package data + +import ( + "io" + "strings" + + "github.com/coreos/clair/utils" + "github.com/coreos/clair/worker/detectors" +) + +// TarDataDetector implements DataDetector and detects layer data in 'tar' format +type TarDataDetector struct{} + +func init() { + detectors.RegisterDataDetector("tar", &TarDataDetector{}) +} + +func (detector *TarDataDetector) Supported(path string, format string) bool { + switch format { + case "": + if strings.HasSuffix(path, ".tar") || strings.HasSuffix(path, ".tar.gz") { + return true + } + case "tar": + return true + } + return false +} + +func (detector *TarDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) { + return utils.SelectivelyExtractArchive(layerReader, "./", toExtract, maxFileSize) +} diff --git a/worker/worker.go b/worker/worker.go index d501da14..08c77dec 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -18,9 +18,6 @@ package worker import ( "errors" - "io" - "net/http" - "os" "strings" "github.com/coreos/clair/database" @@ -56,7 +53,7 @@ var ( // Process detects the OS of a layer, the packages it installs/removes, and // then stores everything in the database. -func Process(ID, parentID, path string) error { +func Process(ID, parentID, path string, imageFormat string) error { if ID == "" { return cerrors.NewBadRequestError("could not process a layer which does not have ID") } @@ -64,7 +61,7 @@ func Process(ID, parentID, path string) error { return cerrors.NewBadRequestError("could not process a layer which does not have a path") } - log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s)", ID, utils.CleanURL(path), Version, parentID) + log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, format: %s)", ID, utils.CleanURL(path), Version, parentID, imageFormat) // Check to see if the layer is already in the database. layer, err := database.FindOneLayerByID(ID, []string{database.FieldLayerEngineVersion}) @@ -101,7 +98,7 @@ func Process(ID, parentID, path string) error { } // Analyze the content. - layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent) + layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent, imageFormat) if err != nil { return err } @@ -114,8 +111,8 @@ func Process(ID, parentID, path string) error { // // If parent is not nil, database.FieldLayerOS, database.FieldLayerPackages fields must be // has been selectioned. -func detectContent(ID, path string, parent *database.Layer) (OS string, installedPackagesNodes, removedPackagesNodes []string, err error) { - data, err := getLayerData(path) +func detectContent(ID, path string, parent *database.Layer, imageFormat string) (OS string, installedPackagesNodes, removedPackagesNodes []string, err error) { + data, err := getLayerData(path, imageFormat) if err != nil { log.Errorf("layer %s: failed to extract data from %s: %s", ID, utils.CleanURL(path), err) return @@ -182,23 +179,8 @@ func detectContent(ID, path string, parent *database.Layer) (OS string, installe } // getLayerData downloads/opens a layer archive and extracts it into memory. -func getLayerData(path string) (data map[string][]byte, err error) { - var layerReader io.ReadCloser - if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { - r, err := http.Get(path) - if err != nil { - return nil, cerrors.ErrCouldNotDownload - } - layerReader = r.Body - } else { - layerReader, err = os.Open(path) - if err != nil { - return nil, cerrors.ErrNotFound - } - } - defer layerReader.Close() - - data, err = utils.SelectivelyExtractArchive(layerReader, append(detectors.GetRequiredFilesPackages(), detectors.GetRequiredFilesOS()...), maxFileSize) +func getLayerData(path string, imageFormat string) (data map[string][]byte, err error) { + data, err = detectors.DetectData(path, imageFormat, append(detectors.GetRequiredFilesPackages(), detectors.GetRequiredFilesOS()...), maxFileSize) if err != nil { return nil, err } diff --git a/worker/worker_test.go b/worker/worker_test.go index 1be861b7..8d59d8cd 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" // Register detectors + _ "github.com/coreos/clair/worker/detectors/data" _ "github.com/coreos/clair/worker/detectors/os" _ "github.com/coreos/clair/worker/detectors/packages" ) @@ -25,9 +26,9 @@ func TestDistUpgrade(t *testing.T) { // blank.tar: MAINTAINER Quentin MACHU // wheezy.tar: FROM debian:wheezy // jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update && apt-get -y dist-upgrade - assert.Nil(t, Process("blank", "", path+"blank.tar.gz")) - assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz")) - assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz")) + assert.Nil(t, Process("blank", "", path+"blank.tar.gz", "")) + assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz", "")) + assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz", "")) wheezy, err := database.FindOneLayerByID("wheezy", database.FieldLayerAll) if assert.Nil(t, err) {