diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 8d3e35f5..9ac8a2e4 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,6 +26,8 @@ import ( "github.com/coreos/clair/config" // Register extensions. + _ "github.com/coreos/clair/ext/imagefmt/aci" + _ "github.com/coreos/clair/ext/imagefmt/docker" _ "github.com/coreos/clair/ext/notification/webhook" _ "github.com/coreos/clair/ext/vulnmdsrc/nvd" _ "github.com/coreos/clair/ext/vulnsrc/alpine" @@ -34,9 +36,6 @@ import ( _ "github.com/coreos/clair/ext/vulnsrc/rhel" _ "github.com/coreos/clair/ext/vulnsrc/ubuntu" - _ "github.com/coreos/clair/worker/detectors/data/aci" - _ "github.com/coreos/clair/worker/detectors/data/docker" - _ "github.com/coreos/clair/worker/detectors/feature/apk" _ "github.com/coreos/clair/worker/detectors/feature/dpkg" _ "github.com/coreos/clair/worker/detectors/feature/rpm" diff --git a/ext/imagefmt/aci/aci.go b/ext/imagefmt/aci/aci.go new file mode 100644 index 00000000..38e26217 --- /dev/null +++ b/ext/imagefmt/aci/aci.go @@ -0,0 +1,40 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aci + +import ( + "io" + "path/filepath" + + "github.com/coreos/clair/ext/imagefmt" + "github.com/coreos/clair/pkg/tarutil" +) + +type format struct{} + +func init() { + imagefmt.RegisterExtractor("aci", &format{}) +} + +func (f format) ExtractFiles(layerReader io.ReadCloser, toExtract []string) (tarutil.FilesMap, error) { + // All contents are inside a "rootfs" directory, so this needs to be + // prepended to each filename. + var filenames []string + for _, filename := range toExtract { + filenames = append(filenames, filepath.Join("rootfs/", filename)) + } + + return tarutil.ExtractFiles(layerReader, filenames) +} diff --git a/ext/imagefmt/docker/docker.go b/ext/imagefmt/docker/docker.go new file mode 100644 index 00000000..94f866fd --- /dev/null +++ b/ext/imagefmt/docker/docker.go @@ -0,0 +1,32 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker + +import ( + "io" + + "github.com/coreos/clair/ext/imagefmt" + "github.com/coreos/clair/pkg/tarutil" +) + +type format struct{} + +func init() { + imagefmt.RegisterExtractor("docker", &format{}) +} + +func (f format) ExtractFiles(layerReader io.ReadCloser, toExtract []string) (tarutil.FilesMap, error) { + return tarutil.ExtractFiles(layerReader, toExtract) +} diff --git a/ext/imagefmt/driver.go b/ext/imagefmt/driver.go new file mode 100644 index 00000000..81d8ce3f --- /dev/null +++ b/ext/imagefmt/driver.go @@ -0,0 +1,150 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package notification fetches notifications from the database and informs the +// specified remote handler about their existences, inviting the third party to +// actively query the API about it. + +// Package imagefmt exposes functions to dynamically register methods to +// detect different types of container image formats. +package imagefmt + +import ( + "fmt" + "io" + "math" + "net/http" + "os" + "strings" + "sync" + + "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/tarutil" + "github.com/coreos/pkg/capnslog" +) + +var ( + // ErrCouldNotFindLayer is returned when we could not download or open the layer file. + ErrCouldNotFindLayer = commonerr.NewBadRequestError("could not find layer") + + log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/imagefmt") + + extractorsM sync.RWMutex + extractors = make(map[string]Extractor) +) + +// Extractor represents an ability to extract files from a particular container +// image format. +type Extractor interface { + // ExtractFiles produces a tarutil.FilesMap from a image layer. + ExtractFiles(layer io.ReadCloser, filenames []string) (tarutil.FilesMap, error) +} + +// RegisterExtractor makes a extractor available by the provided name. +// +// If called twice with the same name, the name is blank, or if the provided +// Extractor is nil, this function panics. +func RegisterExtractor(name string, d Extractor) { + extractorsM.Lock() + defer extractorsM.Unlock() + + if name == "" { + panic("imagefmt: could not register a extractor with an empty name") + } + + if d == nil { + panic("imagefmt: could not register a nil extractor") + } + + // Enforce lowercase names, so that they can be reliably be found in a map. + name = strings.ToLower(name) + + if _, dup := extractors[name]; dup { + panic("imagefmt: RegisterExtractor called twice for " + name) + } + + extractors[name] = d +} + +// Extractors returns the list of the registered extractors. +func Extractors() map[string]Extractor { + extractorsM.RLock() + defer extractorsM.RUnlock() + + ret := make(map[string]Extractor) + for k, v := range extractors { + ret[k] = v + } + + return ret +} + +// UnregisterExtractor removes a Extractor with a particular name from the list. +func UnregisterExtractor(name string) { + extractorsM.Lock() + defer extractorsM.Unlock() + delete(extractors, name) +} + +// Extract streams an image layer from disk or over HTTP, determines the +// image format, then extracts the files specified. +func Extract(format, path string, headers map[string]string, toExtract []string) (tarutil.FilesMap, error) { + var layerReader io.ReadCloser + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + // Create a new HTTP request object. + request, err := http.NewRequest("GET", path, nil) + if err != nil { + return nil, ErrCouldNotFindLayer + } + + // Set any provided HTTP Headers. + if headers != nil { + for k, v := range headers { + request.Header.Set(k, v) + } + } + + // Send the request and handle the response. + r, err := http.DefaultClient.Do(request) + if err != nil { + log.Warningf("could not download layer: %s", err) + return nil, ErrCouldNotFindLayer + } + + // Fail if we don't receive a 2xx HTTP status code. + if math.Floor(float64(r.StatusCode/100)) != 2 { + log.Warningf("could not download layer: got status code %d, expected 2XX", r.StatusCode) + return nil, ErrCouldNotFindLayer + } + + layerReader = r.Body + } else { + var err error + layerReader, err = os.Open(path) + if err != nil { + return nil, ErrCouldNotFindLayer + } + } + defer layerReader.Close() + + if extractor, exists := Extractors()[strings.ToLower(format)]; exists { + files, err := extractor.ExtractFiles(layerReader, toExtract) + if err != nil { + return nil, err + } + return files, nil + } + + return nil, commonerr.NewBadRequestError(fmt.Sprintf("unsupported image format '%s'", format)) +} diff --git a/worker/detectors/data.go b/worker/detectors/data.go deleted file mode 100644 index 376a2548..00000000 --- a/worker/detectors/data.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package detectors exposes functions to register and use container -// information extractors. -package detectors - -import ( - "fmt" - "io" - "math" - "net/http" - "os" - "strings" - "sync" - - cerrors "github.com/coreos/clair/utils/errors" - "github.com/coreos/pkg/capnslog" -) - -// The DataDetector interface defines a way to detect the required data from input path -type DataDetector interface { - //Support check if the input path and format are supported by the underling detector - Supported(path string, format string) bool - // Detect detects the required data from input path - Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) -} - -var ( - dataDetectorsLock sync.Mutex - dataDetectors = make(map[string]DataDetector) - - log = capnslog.NewPackageLogger("github.com/coreos/clair", "detectors") - - // ErrCouldNotFindLayer is returned when we could not download or open the layer file. - ErrCouldNotFindLayer = cerrors.NewBadRequestError("could not find layer") -) - -// RegisterDataDetector provides a way to dynamically register an implementation of a -// DataDetector. -// -// If RegisterDataDetector is called twice with the same name if DataDetector is nil, -// or if the name is blank, it panics. -func RegisterDataDetector(name string, f DataDetector) { - if name == "" { - panic("Could not register a DataDetector with an empty name") - } - if f == nil { - panic("Could not register a nil DataDetector") - } - - dataDetectorsLock.Lock() - defer dataDetectorsLock.Unlock() - - if _, alreadyExists := dataDetectors[name]; alreadyExists { - panic(fmt.Sprintf("Detector '%s' is already registered", name)) - } - dataDetectors[name] = f -} - -// DetectData finds the Data of the layer by using every registered DataDetector -func DetectData(format, path string, headers map[string]string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) { - var layerReader io.ReadCloser - if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { - // Create a new HTTP request object. - request, err := http.NewRequest("GET", path, nil) - if err != nil { - return nil, ErrCouldNotFindLayer - } - - // Set any provided HTTP Headers. - if headers != nil { - for k, v := range headers { - request.Header.Set(k, v) - } - } - - // Send the request and handle the response. - r, err := http.DefaultClient.Do(request) - if err != nil { - log.Warningf("could not download layer: %s", err) - return nil, ErrCouldNotFindLayer - } - - // Fail if we don't receive a 2xx HTTP status code. - if math.Floor(float64(r.StatusCode/100)) != 2 { - log.Warningf("could not download layer: got status code %d, expected 2XX", r.StatusCode) - return nil, ErrCouldNotFindLayer - } - - layerReader = r.Body - } else { - layerReader, err = os.Open(path) - if err != nil { - return nil, ErrCouldNotFindLayer - } - } - defer layerReader.Close() - - for _, detector := range dataDetectors { - if detector.Supported(path, format) { - data, err = detector.Detect(layerReader, toExtract, maxFileSize) - if err != nil { - return nil, err - } - return data, nil - } - } - - return nil, cerrors.NewBadRequestError(fmt.Sprintf("unsupported image format '%s'", format)) -} diff --git a/worker/detectors/data/aci/aci.go b/worker/detectors/data/aci/aci.go deleted file mode 100644 index 78551aa7..00000000 --- a/worker/detectors/data/aci/aci.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package aci - -import ( - "io" - "strings" - - "github.com/coreos/clair/utils" - "github.com/coreos/clair/worker/detectors" -) - -// ACIDataDetector implements DataDetector and detects layer data in 'aci' format -type ACIDataDetector struct{} - -func init() { - detectors.RegisterDataDetector("aci", &ACIDataDetector{}) -} - -func (detector *ACIDataDetector) Supported(path string, format string) bool { - if strings.EqualFold(format, "ACI") { - return true - } - return false -} - -func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) { - return utils.SelectivelyExtractArchive(layerReader, "rootfs/", toExtract, maxFileSize) -} diff --git a/worker/detectors/data/docker/docker.go b/worker/detectors/data/docker/docker.go deleted file mode 100644 index d70de3bc..00000000 --- a/worker/detectors/data/docker/docker.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package docker - -import ( - "io" - "strings" - - "github.com/coreos/clair/utils" - "github.com/coreos/clair/worker/detectors" -) - -// DockerDataDetector implements DataDetector and detects layer data in 'Docker' format -type DockerDataDetector struct{} - -func init() { - detectors.RegisterDataDetector("Docker", &DockerDataDetector{}) -} - -func (detector *DockerDataDetector) Supported(path string, format string) bool { - if strings.EqualFold(format, "Docker") { - return true - } - return false -} - -func (detector *DockerDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) { - return utils.SelectivelyExtractArchive(layerReader, "", toExtract, maxFileSize) -} diff --git a/worker/worker.go b/worker/worker.go index 984def88..904155ee 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -20,6 +20,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/imagefmt" "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils" "github.com/coreos/clair/worker/detectors" @@ -29,11 +30,6 @@ const ( // Version (integer) represents the worker version. // Increased each time the engine changes. Version = 3 - - // maxFileSize enforces a maximum size of a single file within a tarball that - // will be extracted. This protects against malicious layers that may contain - // extremely large package database files. - maxFileSize = 200 * 1024 * 1024 // 200 MiB ) var ( @@ -116,12 +112,14 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s // detectContent downloads a layer's archive and extracts its Namespace and Features. func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) { - data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize) + files, err := imagefmt.Extract(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...)) if err != nil { log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err) return } + data := map[string][]byte(files) + // Detect namespace. namespace = detectNamespace(name, data, parent)