From b1775ed3dc1177df90b4d1bbefa8d6fe97ff6079 Mon Sep 17 00:00:00 2001 From: liangchenye Date: Fri, 11 Dec 2015 19:37:25 +0800 Subject: [PATCH 1/6] add data detector to support ACI and other format in the future Signed-off-by: liangchenye --- api/logic/layers.go | 4 +- cmd/clair/main.go | 1 + utils/tar.go | 4 +- utils/utils_test.go | 6 +-- worker/detectors/data.go | 91 ++++++++++++++++++++++++++++++++++++ worker/detectors/data/aci.go | 46 ++++++++++++++++++ worker/detectors/data/tar.go | 46 ++++++++++++++++++ worker/worker.go | 32 +++---------- worker/worker_test.go | 7 +-- 9 files changed, 202 insertions(+), 35 deletions(-) create mode 100644 worker/detectors/data.go create mode 100644 worker/detectors/data/aci.go create mode 100644 worker/detectors/data/tar.go diff --git a/api/logic/layers.go b/api/logic/layers.go index 6e24ebe4..a8655987 100644 --- a/api/logic/layers.go +++ b/api/logic/layers.go @@ -30,7 +30,7 @@ import ( // POSTLayersParameters represents the expected parameters for POSTLayers. type POSTLayersParameters struct { - ID, Path, ParentID string + ID, Path, ParentID, Type string } // POSTLayers analyzes a layer and returns the engine version that has been used @@ -43,7 +43,7 @@ func POSTLayers(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { } // Process data. - if err := worker.Process(parameters.ID, parameters.ParentID, parameters.Path); err != nil { + if err := worker.Process(parameters.ID, parameters.ParentID, parameters.Path, parameters.Type); err != nil { httputils.WriteHTTPError(w, 0, err) return } diff --git a/cmd/clair/main.go b/cmd/clair/main.go index c08546cf..8d89c11e 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -27,6 +27,7 @@ import ( // Register components _ "github.com/coreos/clair/updater/fetchers" + _ "github.com/coreos/clair/worker/detectors/data" _ "github.com/coreos/clair/worker/detectors/os" _ "github.com/coreos/clair/worker/detectors/packages" ) diff --git a/utils/tar.go b/utils/tar.go index 66d40ea5..477aa9cc 100644 --- a/utils/tar.go +++ b/utils/tar.go @@ -37,7 +37,7 @@ var ( // SelectivelyExtractArchive extracts the specified files and folders // from targz data read from the given reader and store them in a map indexed by file paths -func SelectivelyExtractArchive(r io.Reader, toExtract []string, maxFileSize int64) (map[string][]byte, error) { +func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []string, maxFileSize int64) (map[string][]byte, error) { data := make(map[string][]byte) // Create a tar or tar/tar-gzip reader @@ -58,7 +58,7 @@ func SelectivelyExtractArchive(r io.Reader, toExtract []string, maxFileSize int6 // Get element filename filename := hdr.Name - filename = strings.TrimPrefix(filename, "./") + filename = strings.TrimPrefix(filename, prefix) // Determine if we should extract the element toBeExtracted := false diff --git a/utils/utils_test.go b/utils/utils_test.go index f9c877c8..e543c1c1 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -65,13 +65,13 @@ func TestTar(t *testing.T) { testArchivePath := path.Join(path.Dir(filepath)) + filename // Extract non compressed data - data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), []string{}, 0) + data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), "./", []string{}, 0) assert.Error(t, err, "Extracting non compressed data should return an error") // Extract an archive f, _ := os.Open(testArchivePath) defer f.Close() - data, err = SelectivelyExtractArchive(f, []string{"test/"}, 0) + data, err = SelectivelyExtractArchive(f, "./", []string{"test/"}, 0) assert.Nil(t, err) if c, n := data["test/test.txt"]; !n { @@ -86,7 +86,7 @@ func TestTar(t *testing.T) { // File size limit f, _ = os.Open(testArchivePath) defer f.Close() - data, err = SelectivelyExtractArchive(f, []string{"test"}, 50) + data, err = SelectivelyExtractArchive(f, "./", []string{"test"}, 50) assert.Equal(t, ErrExtractedFileTooBig, err) } } diff --git a/worker/detectors/data.go b/worker/detectors/data.go new file mode 100644 index 00000000..23f50e11 --- /dev/null +++ b/worker/detectors/data.go @@ -0,0 +1,91 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package detectors exposes functions to register and use container +// information extractors. +package detectors + +import ( + "fmt" + "io" + "net/http" + "os" + "strings" + "sync" + + cerrors "github.com/coreos/clair/utils/errors" +) + +// The DataDetector interface defines a way to detect the required data from input path +type DataDetector interface { + //Support check if the input path and format are supported by the underling detector + Supported(path string, format string) bool + // Detect detects the required data from input path + Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) +} + +var ( + dataDetectorsLock sync.Mutex + dataDetectors = make(map[string]DataDetector) +) + +// RegisterDataDetector provides a way to dynamically register an implementation of a +// DataDetector. +// +// If RegisterDataDetector is called twice with the same name if DataDetector is nil, +// or if the name is blank, it panics. +func RegisterDataDetector(name string, f DataDetector) { + if name == "" { + panic("Could not register a DataDetector with an empty name") + } + if f == nil { + panic("Could not register a nil DataDetector") + } + + dataDetectorsLock.Lock() + defer dataDetectorsLock.Unlock() + + if _, alreadyExists := dataDetectors[name]; alreadyExists { + panic(fmt.Sprintf("Detector '%s' is already registered", name)) + } + dataDetectors[name] = f +} + +// DetectData finds the Data of the layer by using every registered DataDetector +func DetectData(path string, format string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) { + var layerReader io.ReadCloser + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + r, err := http.Get(path) + if err != nil { + return nil, cerrors.ErrCouldNotDownload + } + layerReader = r.Body + } else { + layerReader, err = os.Open(path) + if err != nil { + return nil, cerrors.ErrNotFound + } + } + defer layerReader.Close() + + for _, detector := range dataDetectors { + if detector.Supported(path, format) { + if data, err = detector.Detect(layerReader, toExtract, maxFileSize); err == nil { + return data, nil + } + } + } + + return nil, nil +} diff --git a/worker/detectors/data/aci.go b/worker/detectors/data/aci.go new file mode 100644 index 00000000..aa9526cd --- /dev/null +++ b/worker/detectors/data/aci.go @@ -0,0 +1,46 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package data + +import ( + "io" + "strings" + + "github.com/coreos/clair/utils" + "github.com/coreos/clair/worker/detectors" +) + +// ACIDataDetector implements DataDetector and detects layer data in 'aci' format +type ACIDataDetector struct{} + +func init() { + detectors.RegisterDataDetector("aci", &ACIDataDetector{}) +} + +func (detector *ACIDataDetector) Supported(path string, format string) bool { + switch format { + case "": + if strings.HasSuffix(path, ".aci") { + return true + } + case "aci": + return true + } + return false +} + +func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) { + return utils.SelectivelyExtractArchive(layerReader, "./rootfs/", toExtract, maxFileSize) +} diff --git a/worker/detectors/data/tar.go b/worker/detectors/data/tar.go new file mode 100644 index 00000000..ae1cc5a2 --- /dev/null +++ b/worker/detectors/data/tar.go @@ -0,0 +1,46 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package data + +import ( + "io" + "strings" + + "github.com/coreos/clair/utils" + "github.com/coreos/clair/worker/detectors" +) + +// TarDataDetector implements DataDetector and detects layer data in 'tar' format +type TarDataDetector struct{} + +func init() { + detectors.RegisterDataDetector("tar", &TarDataDetector{}) +} + +func (detector *TarDataDetector) Supported(path string, format string) bool { + switch format { + case "": + if strings.HasSuffix(path, ".tar") || strings.HasSuffix(path, ".tar.gz") { + return true + } + case "tar": + return true + } + return false +} + +func (detector *TarDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) { + return utils.SelectivelyExtractArchive(layerReader, "./", toExtract, maxFileSize) +} diff --git a/worker/worker.go b/worker/worker.go index d501da14..08c77dec 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -18,9 +18,6 @@ package worker import ( "errors" - "io" - "net/http" - "os" "strings" "github.com/coreos/clair/database" @@ -56,7 +53,7 @@ var ( // Process detects the OS of a layer, the packages it installs/removes, and // then stores everything in the database. -func Process(ID, parentID, path string) error { +func Process(ID, parentID, path string, imageFormat string) error { if ID == "" { return cerrors.NewBadRequestError("could not process a layer which does not have ID") } @@ -64,7 +61,7 @@ func Process(ID, parentID, path string) error { return cerrors.NewBadRequestError("could not process a layer which does not have a path") } - log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s)", ID, utils.CleanURL(path), Version, parentID) + log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, format: %s)", ID, utils.CleanURL(path), Version, parentID, imageFormat) // Check to see if the layer is already in the database. layer, err := database.FindOneLayerByID(ID, []string{database.FieldLayerEngineVersion}) @@ -101,7 +98,7 @@ func Process(ID, parentID, path string) error { } // Analyze the content. - layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent) + layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent, imageFormat) if err != nil { return err } @@ -114,8 +111,8 @@ func Process(ID, parentID, path string) error { // // If parent is not nil, database.FieldLayerOS, database.FieldLayerPackages fields must be // has been selectioned. -func detectContent(ID, path string, parent *database.Layer) (OS string, installedPackagesNodes, removedPackagesNodes []string, err error) { - data, err := getLayerData(path) +func detectContent(ID, path string, parent *database.Layer, imageFormat string) (OS string, installedPackagesNodes, removedPackagesNodes []string, err error) { + data, err := getLayerData(path, imageFormat) if err != nil { log.Errorf("layer %s: failed to extract data from %s: %s", ID, utils.CleanURL(path), err) return @@ -182,23 +179,8 @@ func detectContent(ID, path string, parent *database.Layer) (OS string, installe } // getLayerData downloads/opens a layer archive and extracts it into memory. -func getLayerData(path string) (data map[string][]byte, err error) { - var layerReader io.ReadCloser - if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { - r, err := http.Get(path) - if err != nil { - return nil, cerrors.ErrCouldNotDownload - } - layerReader = r.Body - } else { - layerReader, err = os.Open(path) - if err != nil { - return nil, cerrors.ErrNotFound - } - } - defer layerReader.Close() - - data, err = utils.SelectivelyExtractArchive(layerReader, append(detectors.GetRequiredFilesPackages(), detectors.GetRequiredFilesOS()...), maxFileSize) +func getLayerData(path string, imageFormat string) (data map[string][]byte, err error) { + data, err = detectors.DetectData(path, imageFormat, append(detectors.GetRequiredFilesPackages(), detectors.GetRequiredFilesOS()...), maxFileSize) if err != nil { return nil, err } diff --git a/worker/worker_test.go b/worker/worker_test.go index 1be861b7..8d59d8cd 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" // Register detectors + _ "github.com/coreos/clair/worker/detectors/data" _ "github.com/coreos/clair/worker/detectors/os" _ "github.com/coreos/clair/worker/detectors/packages" ) @@ -25,9 +26,9 @@ func TestDistUpgrade(t *testing.T) { // blank.tar: MAINTAINER Quentin MACHU // wheezy.tar: FROM debian:wheezy // jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update && apt-get -y dist-upgrade - assert.Nil(t, Process("blank", "", path+"blank.tar.gz")) - assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz")) - assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz")) + assert.Nil(t, Process("blank", "", path+"blank.tar.gz", "")) + assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz", "")) + assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz", "")) wheezy, err := database.FindOneLayerByID("wheezy", database.FieldLayerAll) if assert.Nil(t, err) { From d402ae818e8f9dbf6678dc00c7d4b4aab10567c8 Mon Sep 17 00:00:00 2001 From: liangchenye Date: Fri, 11 Dec 2015 19:41:32 +0800 Subject: [PATCH 2/6] use imageFormat in params for consistence Signed-off-by: liangchenye --- api/logic/layers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/logic/layers.go b/api/logic/layers.go index a8655987..e69d236f 100644 --- a/api/logic/layers.go +++ b/api/logic/layers.go @@ -30,7 +30,7 @@ import ( // POSTLayersParameters represents the expected parameters for POSTLayers. type POSTLayersParameters struct { - ID, Path, ParentID, Type string + ID, Path, ParentID, ImageFormat string } // POSTLayers analyzes a layer and returns the engine version that has been used @@ -43,7 +43,7 @@ func POSTLayers(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { } // Process data. - if err := worker.Process(parameters.ID, parameters.ParentID, parameters.Path, parameters.Type); err != nil { + if err := worker.Process(parameters.ID, parameters.ParentID, parameters.Path, parameters.ImageFormat); err != nil { httputils.WriteHTTPError(w, 0, err) return } From 354c4b3672a4b1663d71dbe05ed3b15e71b874d0 Mon Sep 17 00:00:00 2001 From: liangchenye Date: Tue, 15 Dec 2015 11:34:22 +0800 Subject: [PATCH 3/6] always trim './' Signed-off-by: liangchenye --- utils/tar.go | 5 ++++- utils/utils_test.go | 6 +++--- worker/detectors/data/tar.go | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/utils/tar.go b/utils/tar.go index 477aa9cc..26a47a50 100644 --- a/utils/tar.go +++ b/utils/tar.go @@ -58,7 +58,10 @@ func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []string, m // Get element filename filename := hdr.Name - filename = strings.TrimPrefix(filename, prefix) + filename = strings.TrimPrefix(filename, "./") + if prefix != "" { + filename = strings.TrimPrefix(filename, prefix) + } // Determine if we should extract the element toBeExtracted := false diff --git a/utils/utils_test.go b/utils/utils_test.go index e543c1c1..6b46b055 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -65,13 +65,13 @@ func TestTar(t *testing.T) { testArchivePath := path.Join(path.Dir(filepath)) + filename // Extract non compressed data - data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), "./", []string{}, 0) + data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), "", []string{}, 0) assert.Error(t, err, "Extracting non compressed data should return an error") // Extract an archive f, _ := os.Open(testArchivePath) defer f.Close() - data, err = SelectivelyExtractArchive(f, "./", []string{"test/"}, 0) + data, err = SelectivelyExtractArchive(f, "", []string{"test/"}, 0) assert.Nil(t, err) if c, n := data["test/test.txt"]; !n { @@ -86,7 +86,7 @@ func TestTar(t *testing.T) { // File size limit f, _ = os.Open(testArchivePath) defer f.Close() - data, err = SelectivelyExtractArchive(f, "./", []string{"test"}, 50) + data, err = SelectivelyExtractArchive(f, "", []string{"test"}, 50) assert.Equal(t, ErrExtractedFileTooBig, err) } } diff --git a/worker/detectors/data/tar.go b/worker/detectors/data/tar.go index ae1cc5a2..93a457d1 100644 --- a/worker/detectors/data/tar.go +++ b/worker/detectors/data/tar.go @@ -42,5 +42,5 @@ func (detector *TarDataDetector) Supported(path string, format string) bool { } func (detector *TarDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) { - return utils.SelectivelyExtractArchive(layerReader, "./", toExtract, maxFileSize) + return utils.SelectivelyExtractArchive(layerReader, "", toExtract, maxFileSize) } From 8b649af666f77d40512bfaae1ac37bd0e39ea60b Mon Sep 17 00:00:00 2001 From: liangchenye Date: Tue, 15 Dec 2015 16:08:27 +0800 Subject: [PATCH 4/6] detect bzip2/xz; add test data Signed-off-by: liangchenye --- utils/tar.go | 101 +++++++++++++++++++++++++----- utils/testdata/utils_test.tar.bz2 | Bin 0 -> 644 bytes utils/testdata/utils_test.tar.xz | Bin 0 -> 608 bytes utils/utils_test.go | 6 +- 4 files changed, 89 insertions(+), 18 deletions(-) create mode 100644 utils/testdata/utils_test.tar.bz2 create mode 100644 utils/testdata/utils_test.tar.xz diff --git a/utils/tar.go b/utils/tar.go index 26a47a50..f2cff669 100644 --- a/utils/tar.go +++ b/utils/tar.go @@ -18,10 +18,12 @@ import ( "archive/tar" "bufio" "bytes" + "compress/bzip2" "compress/gzip" "errors" "io" "io/ioutil" + "os/exec" "strings" ) @@ -32,19 +34,75 @@ var ( // ErrExtractedFileTooBig occurs when a file to extract is too big. ErrExtractedFileTooBig = errors.New("utils: could not extract one or more files from the archive: file too big") - gzipHeader = []byte{0x1f, 0x8b} + readLen = 6 // max bytes to sniff + + gzipHeader = []byte{0x1f, 0x8b} + bzip2Header = []byte{0x42, 0x5a, 0x68} + xzHeader = []byte{0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00} ) +// XzReader is an io.ReadCloser which decompresses xz compressed data. +type XzReader struct { + io.ReadCloser + cmd *exec.Cmd + closech chan error +} + +// NewXzReader shells out to a command line xz executable (if +// available) to decompress the given io.Reader using the xz +// compression format and returns an *XzReader. +// It is the caller's responsibility to call Close on the XzReader when done. +func NewXzReader(r io.Reader) (*XzReader, error) { + rpipe, wpipe := io.Pipe() + ex, err := exec.LookPath("xz") + if err != nil { + return nil, err + } + cmd := exec.Command(ex, "--decompress", "--stdout") + + closech := make(chan error) + + cmd.Stdin = r + cmd.Stdout = wpipe + + go func() { + err := cmd.Run() + wpipe.CloseWithError(err) + closech <- err + }() + + return &XzReader{rpipe, cmd, closech}, nil +} + +func (r *XzReader) Close() error { + r.ReadCloser.Close() + r.cmd.Process.Kill() + return <-r.closech +} + +// TarReadCloser embeds a *tar.Reader and the related io.Closer +// It is the caller's responsibility to call Close on TarReadCloser when +// done. +type TarReadCloser struct { + *tar.Reader + io.Closer +} + +func (r *TarReadCloser) Close() error { + return r.Closer.Close() +} + // SelectivelyExtractArchive extracts the specified files and folders // from targz data read from the given reader and store them in a map indexed by file paths func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []string, maxFileSize int64) (map[string][]byte, error) { data := make(map[string][]byte) - // Create a tar or tar/tar-gzip reader + // Create a tar or tar/tar-gzip/tar-bzip2/tar-xz reader tr, err := getTarReader(r) if err != nil { return data, ErrCouldNotExtract } + defer tr.Close() // For each element in the archive for { @@ -89,22 +147,35 @@ func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []string, m return data, nil } -// getTarReader returns a tar.Reader associated with the specified io.Reader, -// optionally backed by a gzip.Reader if gzip compression is detected. +// getTarReader returns a TarReaderCloser associated with the specified io.Reader. // -// Gzip detection is done by using the magic numbers defined in the RFC1952 : -// the first two bytes should be 0x1f and 0x8b.. -func getTarReader(r io.Reader) (*tar.Reader, error) { +// Gzip/Bzip2/XZ detection is done by using the magic numbers: +// Gzip: the first two bytes should be 0x1f and 0x8b. Defined in the RFC1952. +// Bzip2: the first three bytes should be 0x42, 0x5a and 0x68. No RFC. +// XZ: the first three bytes should be 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00. No RFC. +func getTarReader(r io.Reader) (*TarReadCloser, error) { br := bufio.NewReader(r) - header, err := br.Peek(2) - - if err == nil && bytes.Equal(header, gzipHeader) { - gr, err := gzip.NewReader(br) - if err != nil { - return nil, err + header, err := br.Peek(readLen) + if err == nil { + switch { + case bytes.HasPrefix(header, gzipHeader): + gr, err := gzip.NewReader(br) + if err != nil { + return nil, err + } + return &TarReadCloser{tar.NewReader(gr), gr}, nil + case bytes.HasPrefix(header, bzip2Header): + bzip2r := ioutil.NopCloser(bzip2.NewReader(br)) + return &TarReadCloser{tar.NewReader(bzip2r), bzip2r}, nil + case bytes.HasPrefix(header, xzHeader): + xzr, err := NewXzReader(br) + if err != nil { + return nil, err + } + return &TarReadCloser{tar.NewReader(xzr), xzr}, nil } - return tar.NewReader(gr), nil } - return tar.NewReader(br), nil + dr := ioutil.NopCloser(br) + return &TarReadCloser{tar.NewReader(dr), dr}, nil } diff --git a/utils/testdata/utils_test.tar.bz2 b/utils/testdata/utils_test.tar.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..a359247228a1dbff32f83675fcf96336d686458d GIT binary patch literal 644 zcmV-~0(<>JT4*^jL0KkKStl@5v;YDZfAs(O&EP@>|A2P}|6%{9-oQWr03ZgS1ONd@ zKmvu;N=7s!l{QDHiK7q=0MG!?0009fLuv^U1cfxn$+a?iM$%$!CN!8sO*CROJs{Dc zz%Yy^n1%rW$N_|DlO_>>hJjKEnwP{)?*!6{HFL{f{=hms*AF`o5~eDVy+L*1Zk#O)KoNg9T{Q!g zokt#rWsU4sv$6L1WAXdDb2-?c`@g%j$H!0CrQyl3#b%PIs_03YNN97Vh*c2UL%9lu ztp5G~%U1^mv^49XXl%pLcxI^w^Fq>@R1rB4wj*jq$&|~rg4l2(D}IO?S6+af(1NHe zg0eKxWTsGT6v9cTx-`&}i3Cy^ub`O*ELgfWXuDe? zl2XKEs)EEd5J_4vK*d+AP-B$TK`ZjL2y9?x!(Epy%#Nfrp_hI#ZJeBSV95%G8bx%i eXOC?dUn-OqkX1e4iishAi@744C`c2SDp~;aHzgbZ literal 0 HcmV?d00001 diff --git a/utils/testdata/utils_test.tar.xz b/utils/testdata/utils_test.tar.xz new file mode 100644 index 0000000000000000000000000000000000000000..a530986b9f3ab77cac0e2bb84434fee2235d543b GIT binary patch literal 608 zcmV-m0-yc;H+ooF000E$*0e?f03iVu0001VFXf})JO2V6T>uvgyc~T2m4X!T9%rf# z@pd6uzQ%FkEmUW8SYFUdkMvASY#Q*>+E^1VvHqc9}-KHl*>DfEYThGbS-?#+6N!yly(PAfbnW`P?nUyaFYk3uu4 zh+nvx!vUK7KTs)9CvnWtNI8PfE2rhaD;z*oRq%ubIl0VHV*LcPMc_1K0P;$JF_Lp7xa@g(> zTz;16;`_Aem4*3tXui0K*bQ&^(FAa{z@w%#D%}%vBfbv%Wi+0BeK2YJi;!qlJXo(>lUhbBO*Qd z7tTo}0QzLW>q%yq->CR1tc7(b_Ztt~Ir?5=v%X19EKsB4PW-0lEZ$cmM$1%|AM^#Ao{g000001X)@Wv?2=t literal 0 HcmV?d00001 diff --git a/utils/utils_test.go b/utils/utils_test.go index 6b46b055..aafcc2dc 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -60,9 +60,9 @@ func TestTar(t *testing.T) { var err error var data map[string][]byte _, filepath, _, _ := runtime.Caller(0) - - for _, filename := range []string{"/testdata/utils_test.tar.gz", "/testdata/utils_test.tar"} { - testArchivePath := path.Join(path.Dir(filepath)) + filename + testDataDir := "/testdata" + for _, filename := range []string{"utils_test.tar.gz", "utils_test.tar.bz2", "utils_test.tar.xz", "utils_test.tar"} { + testArchivePath := path.Join(path.Dir(filepath), testDataDir, filename) // Extract non compressed data data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), "", []string{}, 0) From 41509ccd3e60bc1aa7b9a025cca06f783b7b4687 Mon Sep 17 00:00:00 2001 From: Liang Chenye Date: Tue, 15 Dec 2015 20:29:53 -0800 Subject: [PATCH 5/6] add imageFormt to API.md; add xz to Dockerfile; fix bugs Signed-off-by: Liang Chenye --- Dockerfile | 2 +- docs/API.md | 9 +++++---- worker/detectors/data/aci.go | 9 ++------- worker/detectors/data/{tar.go => docker.go} | 17 ++++++----------- worker/worker.go | 19 ++++++++++++++++++- 5 files changed, 32 insertions(+), 24 deletions(-) rename worker/detectors/data/{tar.go => docker.go} (61%) diff --git a/Dockerfile b/Dockerfile index ac350b97..10e63569 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.5 MAINTAINER Quentin Machu -RUN apt-get update && apt-get install -y bzr rpm && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +RUN apt-get update && apt-get install -y bzr rpm xz && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN mkdir /db VOLUME /db diff --git a/docs/API.md b/docs/API.md index 26db2e68..4f6cea55 100644 --- a/docs/API.md +++ b/docs/API.md @@ -112,7 +112,8 @@ It processes and inserts a new Layer in the database. |------|-----|-------------| |ID|String|Unique ID of the Layer| |Path|String|Absolute path or HTTP link pointing to the Layer's tar file| -|ParentID|String|(Optionnal) Unique ID of the Layer's parent +|ParentID|String|(Optional) Unique ID of the Layer's parent| +|ImageFormat|String|Image format of the Layer ('Docker' or 'ACI')| If the Layer has not parent, the ParentID field should be omitted or empty. @@ -346,7 +347,7 @@ It returns the lists of vulnerabilities which affect a given Layer. |Name|Type|Description| |------|-----|-------------| |ID|String|Unique ID of the Layer| -|minimumPriority|Priority|(Optionnal) The minimum priority of the returned vulnerabilities. Defaults to High| +|minimumPriority|Priority|(Optional) The minimum priority of the returned vulnerabilities. Defaults to High| ### Example @@ -389,7 +390,7 @@ It returns the lists of vulnerabilities which are introduced and removed by the |Name|Type|Description| |------|-----|-------------| |ID|String|Unique ID of the Layer| -|minimumPriority|Priority|(Optionnal) The minimum priority of the returned vulnerabilities| +|minimumPriority|Priority|(Optional) The minimum priority of the returned vulnerabilities| ### Example @@ -436,7 +437,7 @@ Counterintuitively, this request is actually a POST to be able to pass a lot of |Name|Type|Description| |------|-----|-------------| |LayersIDs|Array of strings|Unique IDs of Layers| -|minimumPriority|Priority|(Optionnal) The minimum priority of the returned vulnerabilities. Defaults to High| +|minimumPriority|Priority|(Optional) The minimum priority of the returned vulnerabilities. Defaults to High| ### Example diff --git a/worker/detectors/data/aci.go b/worker/detectors/data/aci.go index aa9526cd..43a7a187 100644 --- a/worker/detectors/data/aci.go +++ b/worker/detectors/data/aci.go @@ -30,17 +30,12 @@ func init() { } func (detector *ACIDataDetector) Supported(path string, format string) bool { - switch format { - case "": - if strings.HasSuffix(path, ".aci") { - return true - } - case "aci": + if strings.EqualFold(format, "ACI") { return true } return false } func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) { - return utils.SelectivelyExtractArchive(layerReader, "./rootfs/", toExtract, maxFileSize) + return utils.SelectivelyExtractArchive(layerReader, "rootfs/", toExtract, maxFileSize) } diff --git a/worker/detectors/data/tar.go b/worker/detectors/data/docker.go similarity index 61% rename from worker/detectors/data/tar.go rename to worker/detectors/data/docker.go index 93a457d1..d1b32ad4 100644 --- a/worker/detectors/data/tar.go +++ b/worker/detectors/data/docker.go @@ -22,25 +22,20 @@ import ( "github.com/coreos/clair/worker/detectors" ) -// TarDataDetector implements DataDetector and detects layer data in 'tar' format -type TarDataDetector struct{} +// DockerDataDetector implements DataDetector and detects layer data in 'Docker' format +type DockerDataDetector struct{} func init() { - detectors.RegisterDataDetector("tar", &TarDataDetector{}) + detectors.RegisterDataDetector("Docker", &DockerDataDetector{}) } -func (detector *TarDataDetector) Supported(path string, format string) bool { - switch format { - case "": - if strings.HasSuffix(path, ".tar") || strings.HasSuffix(path, ".tar.gz") { - return true - } - case "tar": +func (detector *DockerDataDetector) Supported(path string, format string) bool { + if strings.EqualFold(format, "Docker") { return true } return false } -func (detector *TarDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) { +func (detector *DockerDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) { return utils.SelectivelyExtractArchive(layerReader, "", toExtract, maxFileSize) } diff --git a/worker/worker.go b/worker/worker.go index 08c77dec..2f00a5fe 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -49,6 +49,9 @@ var ( // SupportedOS is the list of operating system names that the worker supports. SupportedOS = []string{"debian", "ubuntu", "centos"} + + // SupportedImageFormat is the list of image formats that the worker supports. + SupportedImageFormat = []string{"Docker", "ACI"} ) // Process detects the OS of a layer, the packages it installs/removes, and @@ -60,8 +63,22 @@ func Process(ID, parentID, path string, imageFormat string) error { if path == "" { return cerrors.NewBadRequestError("could not process a layer which does not have a path") } + if imageFormat == "" { + return cerrors.NewBadRequestError("could not process a layer which does not have a specified format") + } else { + isSupported := false + for _, format := range SupportedImageFormat { + if strings.EqualFold(imageFormat, format) { + isSupported = true + break + } + } + if !isSupported { + return cerrors.NewBadRequestError("could not process a layer which does not have a supported format") + } + } - log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, format: %s)", ID, utils.CleanURL(path), Version, parentID, imageFormat) + log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, Format: %s)", ID, utils.CleanURL(path), Version, parentID, imageFormat) // Check to see if the layer is already in the database. layer, err := database.FindOneLayerByID(ID, []string{database.FieldLayerEngineVersion}) From 4b1149106751d510fc6d16f8c4b923e72a054448 Mon Sep 17 00:00:00 2001 From: Liang Chenye Date: Tue, 15 Dec 2015 21:01:22 -0800 Subject: [PATCH 6/6] add ImageFormat to worker_test and programs under contrib Signed-off-by: Liang Chenye --- contrib/analyze-local-images/main.go | 2 +- contrib/check-openvz-mirror-with-clair/main.go | 3 ++- worker/worker_test.go | 12 +++++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/contrib/analyze-local-images/main.go b/contrib/analyze-local-images/main.go index 1fa5219d..777eb89d 100644 --- a/contrib/analyze-local-images/main.go +++ b/contrib/analyze-local-images/main.go @@ -197,7 +197,7 @@ func history(imageName string) ([]string, error) { } func analyzeLayer(endpoint, path, layerID, parentLayerID string) error { - payload := struct{ ID, Path, ParentID string }{ID: layerID, Path: path, ParentID: parentLayerID} + payload := struct{ ID, Path, ParentID, ImageFormat string }{ID: layerID, Path: path, ParentID: parentLayerID, ImageFormat: "Docker"} jsonPayload, err := json.Marshal(payload) if err != nil { return err diff --git a/contrib/check-openvz-mirror-with-clair/main.go b/contrib/check-openvz-mirror-with-clair/main.go index 31b80e25..8b26a524 100644 --- a/contrib/check-openvz-mirror-with-clair/main.go +++ b/contrib/check-openvz-mirror-with-clair/main.go @@ -51,6 +51,7 @@ type AddLayoutRequestAPI struct { ID string `json:"ID"` Path string `json:"Path"` ParantID string `json:"ParantID"` + ImageFormat string `json:"ImageFormat"` } type VulnerabilityItem struct { @@ -223,7 +224,7 @@ func (clair ClairAPI) AddLayer(openvzMirror string, templateName string) error { client = httpClient } - jsonRequest, err := json.Marshal(AddLayoutRequestAPI{ID: templateName, Path: openvzMirror + "/" + templateName + ".tar.gz"}) + jsonRequest, err := json.Marshal(AddLayoutRequestAPI{ID: templateName, Path: openvzMirror + "/" + templateName + ".tar.gz", ImageFormat: "Docker"}) if err != nil { log.Println("Cannot convert to json request with error: ", err) return err diff --git a/worker/worker_test.go b/worker/worker_test.go index 8d59d8cd..5852b09d 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -26,9 +26,15 @@ func TestDistUpgrade(t *testing.T) { // blank.tar: MAINTAINER Quentin MACHU // wheezy.tar: FROM debian:wheezy // jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update && apt-get -y dist-upgrade - assert.Nil(t, Process("blank", "", path+"blank.tar.gz", "")) - assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz", "")) - assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz", "")) + assert.Nil(t, Process("blank", "", path+"blank.tar.gz", "Docker")) + assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz", "Docker")) + assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz", "Docker")) + + err := Process("blank", "", path+"blank.tar.gz", "") + assert.Error(t, err, "could not process a layer which does not have a specified format") + + err = Process("blank", "", path+"blank.tar.gz", "invalid") + assert.Error(t, err, "could not process a layer which does not have a supported format") wheezy, err := database.FindOneLayerByID("wheezy", database.FieldLayerAll) if assert.Nil(t, err) {