Merge pull request #49 from liangchenye/master

Add DataDetector to support ACI and other layout format
pull/65/head
Quentin Machu 8 years ago
commit e834301941

@ -1,7 +1,7 @@
FROM golang:1.5
MAINTAINER Quentin Machu <quentin.machu@coreos.com>
RUN apt-get update && apt-get install -y bzr rpm && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
RUN apt-get update && apt-get install -y bzr rpm xz && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
RUN mkdir /db
VOLUME /db

@ -30,7 +30,7 @@ import (
// POSTLayersParameters represents the expected parameters for POSTLayers.
type POSTLayersParameters struct {
ID, Path, ParentID string
ID, Path, ParentID, ImageFormat string
}
// POSTLayers analyzes a layer and returns the engine version that has been used
@ -43,7 +43,7 @@ func POSTLayers(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
}
// Process data.
if err := worker.Process(parameters.ID, parameters.ParentID, parameters.Path); err != nil {
if err := worker.Process(parameters.ID, parameters.ParentID, parameters.Path, parameters.ImageFormat); err != nil {
httputils.WriteHTTPError(w, 0, err)
return
}

@ -28,6 +28,7 @@ import (
// Register components
_ "github.com/coreos/clair/notifier/notifiers"
_ "github.com/coreos/clair/updater/fetchers"
_ "github.com/coreos/clair/worker/detectors/data"
_ "github.com/coreos/clair/worker/detectors/os"
_ "github.com/coreos/clair/worker/detectors/packages"
)

@ -197,7 +197,7 @@ func history(imageName string) ([]string, error) {
}
func analyzeLayer(endpoint, path, layerID, parentLayerID string) error {
payload := struct{ ID, Path, ParentID string }{ID: layerID, Path: path, ParentID: parentLayerID}
payload := struct{ ID, Path, ParentID, ImageFormat string }{ID: layerID, Path: path, ParentID: parentLayerID, ImageFormat: "Docker"}
jsonPayload, err := json.Marshal(payload)
if err != nil {
return err

@ -51,6 +51,7 @@ type AddLayoutRequestAPI struct {
ID string `json:"ID"`
Path string `json:"Path"`
ParantID string `json:"ParantID"`
ImageFormat string `json:"ImageFormat"`
}
type VulnerabilityItem struct {
@ -223,7 +224,7 @@ func (clair ClairAPI) AddLayer(openvzMirror string, templateName string) error {
client = httpClient
}
jsonRequest, err := json.Marshal(AddLayoutRequestAPI{ID: templateName, Path: openvzMirror + "/" + templateName + ".tar.gz"})
jsonRequest, err := json.Marshal(AddLayoutRequestAPI{ID: templateName, Path: openvzMirror + "/" + templateName + ".tar.gz", ImageFormat: "Docker"})
if err != nil {
log.Println("Cannot convert to json request with error: ", err)
return err

@ -112,7 +112,8 @@ It processes and inserts a new Layer in the database.
|------|-----|-------------|
|ID|String|Unique ID of the Layer|
|Path|String|Absolute path or HTTP link pointing to the Layer's tar file|
|ParentID|String|(Optionnal) Unique ID of the Layer's parent
|ParentID|String|(Optional) Unique ID of the Layer's parent|
|ImageFormat|String|Image format of the Layer ('Docker' or 'ACI')|
If the Layer has not parent, the ParentID field should be omitted or empty.
@ -346,7 +347,7 @@ It returns the lists of vulnerabilities which affect a given Layer.
|Name|Type|Description|
|------|-----|-------------|
|ID|String|Unique ID of the Layer|
|minimumPriority|Priority|(Optionnal) The minimum priority of the returned vulnerabilities. Defaults to High|
|minimumPriority|Priority|(Optional) The minimum priority of the returned vulnerabilities. Defaults to High|
### Example
@ -389,7 +390,7 @@ It returns the lists of vulnerabilities which are introduced and removed by the
|Name|Type|Description|
|------|-----|-------------|
|ID|String|Unique ID of the Layer|
|minimumPriority|Priority|(Optionnal) The minimum priority of the returned vulnerabilities|
|minimumPriority|Priority|(Optional) The minimum priority of the returned vulnerabilities|
### Example
@ -436,7 +437,7 @@ Counterintuitively, this request is actually a POST to be able to pass a lot of
|Name|Type|Description|
|------|-----|-------------|
|LayersIDs|Array of strings|Unique IDs of Layers|
|minimumPriority|Priority|(Optionnal) The minimum priority of the returned vulnerabilities. Defaults to High|
|minimumPriority|Priority|(Optional) The minimum priority of the returned vulnerabilities. Defaults to High|
### Example

@ -18,10 +18,12 @@ import (
"archive/tar"
"bufio"
"bytes"
"compress/bzip2"
"compress/gzip"
"errors"
"io"
"io/ioutil"
"os/exec"
"strings"
)
@ -32,19 +34,75 @@ var (
// ErrExtractedFileTooBig occurs when a file to extract is too big.
ErrExtractedFileTooBig = errors.New("utils: could not extract one or more files from the archive: file too big")
gzipHeader = []byte{0x1f, 0x8b}
readLen = 6 // max bytes to sniff
gzipHeader = []byte{0x1f, 0x8b}
bzip2Header = []byte{0x42, 0x5a, 0x68}
xzHeader = []byte{0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00}
)
// XzReader is an io.ReadCloser which decompresses xz compressed data.
type XzReader struct {
io.ReadCloser
cmd *exec.Cmd
closech chan error
}
// NewXzReader shells out to a command line xz executable (if
// available) to decompress the given io.Reader using the xz
// compression format and returns an *XzReader.
// It is the caller's responsibility to call Close on the XzReader when done.
func NewXzReader(r io.Reader) (*XzReader, error) {
rpipe, wpipe := io.Pipe()
ex, err := exec.LookPath("xz")
if err != nil {
return nil, err
}
cmd := exec.Command(ex, "--decompress", "--stdout")
closech := make(chan error)
cmd.Stdin = r
cmd.Stdout = wpipe
go func() {
err := cmd.Run()
wpipe.CloseWithError(err)
closech <- err
}()
return &XzReader{rpipe, cmd, closech}, nil
}
func (r *XzReader) Close() error {
r.ReadCloser.Close()
r.cmd.Process.Kill()
return <-r.closech
}
// TarReadCloser embeds a *tar.Reader and the related io.Closer
// It is the caller's responsibility to call Close on TarReadCloser when
// done.
type TarReadCloser struct {
*tar.Reader
io.Closer
}
func (r *TarReadCloser) Close() error {
return r.Closer.Close()
}
// SelectivelyExtractArchive extracts the specified files and folders
// from targz data read from the given reader and store them in a map indexed by file paths
func SelectivelyExtractArchive(r io.Reader, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
data := make(map[string][]byte)
// Create a tar or tar/tar-gzip reader
// Create a tar or tar/tar-gzip/tar-bzip2/tar-xz reader
tr, err := getTarReader(r)
if err != nil {
return data, ErrCouldNotExtract
}
defer tr.Close()
// For each element in the archive
for {
@ -59,6 +117,9 @@ func SelectivelyExtractArchive(r io.Reader, toExtract []string, maxFileSize int6
// Get element filename
filename := hdr.Name
filename = strings.TrimPrefix(filename, "./")
if prefix != "" {
filename = strings.TrimPrefix(filename, prefix)
}
// Determine if we should extract the element
toBeExtracted := false
@ -86,22 +147,35 @@ func SelectivelyExtractArchive(r io.Reader, toExtract []string, maxFileSize int6
return data, nil
}
// getTarReader returns a tar.Reader associated with the specified io.Reader,
// optionally backed by a gzip.Reader if gzip compression is detected.
// getTarReader returns a TarReaderCloser associated with the specified io.Reader.
//
// Gzip detection is done by using the magic numbers defined in the RFC1952 :
// the first two bytes should be 0x1f and 0x8b..
func getTarReader(r io.Reader) (*tar.Reader, error) {
// Gzip/Bzip2/XZ detection is done by using the magic numbers:
// Gzip: the first two bytes should be 0x1f and 0x8b. Defined in the RFC1952.
// Bzip2: the first three bytes should be 0x42, 0x5a and 0x68. No RFC.
// XZ: the first three bytes should be 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00. No RFC.
func getTarReader(r io.Reader) (*TarReadCloser, error) {
br := bufio.NewReader(r)
header, err := br.Peek(2)
if err == nil && bytes.Equal(header, gzipHeader) {
gr, err := gzip.NewReader(br)
if err != nil {
return nil, err
header, err := br.Peek(readLen)
if err == nil {
switch {
case bytes.HasPrefix(header, gzipHeader):
gr, err := gzip.NewReader(br)
if err != nil {
return nil, err
}
return &TarReadCloser{tar.NewReader(gr), gr}, nil
case bytes.HasPrefix(header, bzip2Header):
bzip2r := ioutil.NopCloser(bzip2.NewReader(br))
return &TarReadCloser{tar.NewReader(bzip2r), bzip2r}, nil
case bytes.HasPrefix(header, xzHeader):
xzr, err := NewXzReader(br)
if err != nil {
return nil, err
}
return &TarReadCloser{tar.NewReader(xzr), xzr}, nil
}
return tar.NewReader(gr), nil
}
return tar.NewReader(br), nil
dr := ioutil.NopCloser(br)
return &TarReadCloser{tar.NewReader(dr), dr}, nil
}

Binary file not shown.

Binary file not shown.

@ -60,18 +60,18 @@ func TestTar(t *testing.T) {
var err error
var data map[string][]byte
_, filepath, _, _ := runtime.Caller(0)
for _, filename := range []string{"/testdata/utils_test.tar.gz", "/testdata/utils_test.tar"} {
testArchivePath := path.Join(path.Dir(filepath)) + filename
testDataDir := "/testdata"
for _, filename := range []string{"utils_test.tar.gz", "utils_test.tar.bz2", "utils_test.tar.xz", "utils_test.tar"} {
testArchivePath := path.Join(path.Dir(filepath), testDataDir, filename)
// Extract non compressed data
data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), []string{}, 0)
data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), "", []string{}, 0)
assert.Error(t, err, "Extracting non compressed data should return an error")
// Extract an archive
f, _ := os.Open(testArchivePath)
defer f.Close()
data, err = SelectivelyExtractArchive(f, []string{"test/"}, 0)
data, err = SelectivelyExtractArchive(f, "", []string{"test/"}, 0)
assert.Nil(t, err)
if c, n := data["test/test.txt"]; !n {
@ -86,7 +86,7 @@ func TestTar(t *testing.T) {
// File size limit
f, _ = os.Open(testArchivePath)
defer f.Close()
data, err = SelectivelyExtractArchive(f, []string{"test"}, 50)
data, err = SelectivelyExtractArchive(f, "", []string{"test"}, 50)
assert.Equal(t, ErrExtractedFileTooBig, err)
}
}

@ -0,0 +1,91 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package detectors exposes functions to register and use container
// information extractors.
package detectors
import (
"fmt"
"io"
"net/http"
"os"
"strings"
"sync"
cerrors "github.com/coreos/clair/utils/errors"
)
// The DataDetector interface defines a way to detect the required data from input path
type DataDetector interface {
//Support check if the input path and format are supported by the underling detector
Supported(path string, format string) bool
// Detect detects the required data from input path
Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (data map[string][]byte, err error)
}
var (
dataDetectorsLock sync.Mutex
dataDetectors = make(map[string]DataDetector)
)
// RegisterDataDetector provides a way to dynamically register an implementation of a
// DataDetector.
//
// If RegisterDataDetector is called twice with the same name if DataDetector is nil,
// or if the name is blank, it panics.
func RegisterDataDetector(name string, f DataDetector) {
if name == "" {
panic("Could not register a DataDetector with an empty name")
}
if f == nil {
panic("Could not register a nil DataDetector")
}
dataDetectorsLock.Lock()
defer dataDetectorsLock.Unlock()
if _, alreadyExists := dataDetectors[name]; alreadyExists {
panic(fmt.Sprintf("Detector '%s' is already registered", name))
}
dataDetectors[name] = f
}
// DetectData finds the Data of the layer by using every registered DataDetector
func DetectData(path string, format string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) {
var layerReader io.ReadCloser
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
r, err := http.Get(path)
if err != nil {
return nil, cerrors.ErrCouldNotDownload
}
layerReader = r.Body
} else {
layerReader, err = os.Open(path)
if err != nil {
return nil, cerrors.ErrNotFound
}
}
defer layerReader.Close()
for _, detector := range dataDetectors {
if detector.Supported(path, format) {
if data, err = detector.Detect(layerReader, toExtract, maxFileSize); err == nil {
return data, nil
}
}
}
return nil, nil
}

@ -0,0 +1,41 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package data
import (
"io"
"strings"
"github.com/coreos/clair/utils"
"github.com/coreos/clair/worker/detectors"
)
// ACIDataDetector implements DataDetector and detects layer data in 'aci' format
type ACIDataDetector struct{}
func init() {
detectors.RegisterDataDetector("aci", &ACIDataDetector{})
}
func (detector *ACIDataDetector) Supported(path string, format string) bool {
if strings.EqualFold(format, "ACI") {
return true
}
return false
}
func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
return utils.SelectivelyExtractArchive(layerReader, "rootfs/", toExtract, maxFileSize)
}

@ -0,0 +1,41 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package data
import (
"io"
"strings"
"github.com/coreos/clair/utils"
"github.com/coreos/clair/worker/detectors"
)
// DockerDataDetector implements DataDetector and detects layer data in 'Docker' format
type DockerDataDetector struct{}
func init() {
detectors.RegisterDataDetector("Docker", &DockerDataDetector{})
}
func (detector *DockerDataDetector) Supported(path string, format string) bool {
if strings.EqualFold(format, "Docker") {
return true
}
return false
}
func (detector *DockerDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
return utils.SelectivelyExtractArchive(layerReader, "", toExtract, maxFileSize)
}

@ -18,9 +18,6 @@ package worker
import (
"errors"
"io"
"net/http"
"os"
"strings"
"github.com/coreos/clair/database"
@ -52,19 +49,36 @@ var (
// SupportedOS is the list of operating system names that the worker supports.
SupportedOS = []string{"debian", "ubuntu", "centos"}
// SupportedImageFormat is the list of image formats that the worker supports.
SupportedImageFormat = []string{"Docker", "ACI"}
)
// Process detects the OS of a layer, the packages it installs/removes, and
// then stores everything in the database.
func Process(ID, parentID, path string) error {
func Process(ID, parentID, path string, imageFormat string) error {
if ID == "" {
return cerrors.NewBadRequestError("could not process a layer which does not have ID")
}
if path == "" {
return cerrors.NewBadRequestError("could not process a layer which does not have a path")
}
if imageFormat == "" {
return cerrors.NewBadRequestError("could not process a layer which does not have a specified format")
} else {
isSupported := false
for _, format := range SupportedImageFormat {
if strings.EqualFold(imageFormat, format) {
isSupported = true
break
}
}
if !isSupported {
return cerrors.NewBadRequestError("could not process a layer which does not have a supported format")
}
}
log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s)", ID, utils.CleanURL(path), Version, parentID)
log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, Format: %s)", ID, utils.CleanURL(path), Version, parentID, imageFormat)
// Check to see if the layer is already in the database.
layer, err := database.FindOneLayerByID(ID, []string{database.FieldLayerEngineVersion})
@ -101,7 +115,7 @@ func Process(ID, parentID, path string) error {
}
// Analyze the content.
layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent)
layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent, imageFormat)
if err != nil {
return err
}
@ -114,8 +128,8 @@ func Process(ID, parentID, path string) error {
//
// If parent is not nil, database.FieldLayerOS, database.FieldLayerPackages fields must be
// has been selectioned.
func detectContent(ID, path string, parent *database.Layer) (OS string, installedPackagesNodes, removedPackagesNodes []string, err error) {
data, err := getLayerData(path)
func detectContent(ID, path string, parent *database.Layer, imageFormat string) (OS string, installedPackagesNodes, removedPackagesNodes []string, err error) {
data, err := getLayerData(path, imageFormat)
if err != nil {
log.Errorf("layer %s: failed to extract data from %s: %s", ID, utils.CleanURL(path), err)
return
@ -182,23 +196,8 @@ func detectContent(ID, path string, parent *database.Layer) (OS string, installe
}
// getLayerData downloads/opens a layer archive and extracts it into memory.
func getLayerData(path string) (data map[string][]byte, err error) {
var layerReader io.ReadCloser
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
r, err := http.Get(path)
if err != nil {
return nil, cerrors.ErrCouldNotDownload
}
layerReader = r.Body
} else {
layerReader, err = os.Open(path)
if err != nil {
return nil, cerrors.ErrNotFound
}
}
defer layerReader.Close()
data, err = utils.SelectivelyExtractArchive(layerReader, append(detectors.GetRequiredFilesPackages(), detectors.GetRequiredFilesOS()...), maxFileSize)
func getLayerData(path string, imageFormat string) (data map[string][]byte, err error) {
data, err = detectors.DetectData(path, imageFormat, append(detectors.GetRequiredFilesPackages(), detectors.GetRequiredFilesOS()...), maxFileSize)
if err != nil {
return nil, err
}

@ -11,6 +11,7 @@ import (
"github.com/stretchr/testify/assert"
// Register detectors
_ "github.com/coreos/clair/worker/detectors/data"
_ "github.com/coreos/clair/worker/detectors/os"
_ "github.com/coreos/clair/worker/detectors/packages"
)
@ -25,9 +26,15 @@ func TestDistUpgrade(t *testing.T) {
// blank.tar: MAINTAINER Quentin MACHU <quentin.machu.fr>
// wheezy.tar: FROM debian:wheezy
// jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update && apt-get -y dist-upgrade
assert.Nil(t, Process("blank", "", path+"blank.tar.gz"))
assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz"))
assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz"))
assert.Nil(t, Process("blank", "", path+"blank.tar.gz", "Docker"))
assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz", "Docker"))
assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz", "Docker"))
err := Process("blank", "", path+"blank.tar.gz", "")
assert.Error(t, err, "could not process a layer which does not have a specified format")
err = Process("blank", "", path+"blank.tar.gz", "invalid")
assert.Error(t, err, "could not process a layer which does not have a supported format")
wheezy, err := database.FindOneLayerByID("wheezy", database.FieldLayerAll)
if assert.Nil(t, err) {

Loading…
Cancel
Save