From 0b2a9ab12b1f6e1308cad933bb7ad6cf15017011 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 9 Nov 2016 15:51:28 -0500 Subject: [PATCH 01/13] detectors/namespace: support pointers in tests This change adjusts some names of types being exported and adds some documentation. --- .../namespace/aptsources/aptsources_test.go | 24 +++++----- .../namespace/lsbrelease/lsbrelease_test.go | 34 +++++++------- .../namespace/osrelease/osrelease_test.go | 46 +++++++++---------- .../redhatrelease/redhatrelease_test.go | 34 +++++++------- worker/detectors/namespace/test.go | 23 +++++++--- 5 files changed, 86 insertions(+), 75 deletions(-) diff --git a/worker/detectors/namespace/aptsources/aptsources_test.go b/worker/detectors/namespace/aptsources/aptsources_test.go index d502dc48..238d1a9b 100644 --- a/worker/detectors/namespace/aptsources/aptsources_test.go +++ b/worker/detectors/namespace/aptsources/aptsources_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2016 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,22 +21,22 @@ import ( "github.com/coreos/clair/worker/detectors/namespace" ) -var aptSourcesOSTests = []namespace.NamespaceTest{ - { - ExpectedNamespace: database.Namespace{Name: "debian:unstable"}, - Data: map[string][]byte{ - "etc/os-release": []byte( - `PRETTY_NAME="Debian GNU/Linux stretch/sid" +func TestAptSourcesNamespaceDetector(t *testing.T) { + testData := []namespace.TestData{ + { + ExpectedNamespace: &database.Namespace{Name: "debian:unstable"}, + Data: map[string][]byte{ + "etc/os-release": []byte( + `PRETTY_NAME="Debian GNU/Linux stretch/sid" NAME="Debian GNU/Linux" ID=debian HOME_URL="https://www.debian.org/" SUPPORT_URL="https://www.debian.org/support/" BUG_REPORT_URL="https://bugs.debian.org/"`), - "etc/apt/sources.list": []byte(`deb http://httpredir.debian.org/debian unstable main`), + "etc/apt/sources.list": []byte(`deb http://httpredir.debian.org/debian unstable main`), + }, }, - }, -} + } -func TestAptSourcesNamespaceDetector(t *testing.T) { - namespace.TestNamespaceDetector(t, &AptSourcesNamespaceDetector{}, aptSourcesOSTests) + namespace.TestDetector(t, &AptSourcesNamespaceDetector{}, testData) } diff --git a/worker/detectors/namespace/lsbrelease/lsbrelease_test.go b/worker/detectors/namespace/lsbrelease/lsbrelease_test.go index 9aa3b64f..6d24f629 100644 --- a/worker/detectors/namespace/lsbrelease/lsbrelease_test.go +++ b/worker/detectors/namespace/lsbrelease/lsbrelease_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2016 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,29 +21,29 @@ import ( "github.com/coreos/clair/worker/detectors/namespace" ) -var lsbReleaseOSTests = []namespace.NamespaceTest{ - { - ExpectedNamespace: database.Namespace{Name: "ubuntu:12.04"}, - Data: map[string][]byte{ - "etc/lsb-release": []byte( - `DISTRIB_ID=Ubuntu +func TestLsbReleaseNamespaceDetector(t *testing.T) { + testData := []namespace.TestData{ + { + ExpectedNamespace: &database.Namespace{Name: "ubuntu:12.04"}, + Data: map[string][]byte{ + "etc/lsb-release": []byte( + `DISTRIB_ID=Ubuntu DISTRIB_RELEASE=12.04 DISTRIB_CODENAME=precise DISTRIB_DESCRIPTION="Ubuntu 12.04 LTS"`), + }, }, - }, - { // We don't care about the minor version of Debian - ExpectedNamespace: database.Namespace{Name: "debian:7"}, - Data: map[string][]byte{ - "etc/lsb-release": []byte( - `DISTRIB_ID=Debian + { // We don't care about the minor version of Debian + ExpectedNamespace: &database.Namespace{Name: "debian:7"}, + Data: map[string][]byte{ + "etc/lsb-release": []byte( + `DISTRIB_ID=Debian DISTRIB_RELEASE=7.1 DISTRIB_CODENAME=wheezy DISTRIB_DESCRIPTION="Debian 7.1"`), + }, }, - }, -} + } -func TestLsbReleaseNamespaceDetector(t *testing.T) { - namespace.TestNamespaceDetector(t, &LsbReleaseNamespaceDetector{}, lsbReleaseOSTests) + namespace.TestDetector(t, &LsbReleaseNamespaceDetector{}, testData) } diff --git a/worker/detectors/namespace/osrelease/osrelease_test.go b/worker/detectors/namespace/osrelease/osrelease_test.go index 4b08cf33..f02f409a 100644 --- a/worker/detectors/namespace/osrelease/osrelease_test.go +++ b/worker/detectors/namespace/osrelease/osrelease_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2016 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,12 +21,13 @@ import ( "github.com/coreos/clair/worker/detectors/namespace" ) -var osReleaseOSTests = []namespace.NamespaceTest{ - { - ExpectedNamespace: database.Namespace{Name: "debian:8"}, - Data: map[string][]byte{ - "etc/os-release": []byte( - `PRETTY_NAME="Debian GNU/Linux 8 (jessie)" +func TestOsReleaseNamespaceDetector(t *testing.T) { + testData := []namespace.TestData{ + { + ExpectedNamespace: &database.Namespace{Name: "debian:8"}, + Data: map[string][]byte{ + "etc/os-release": []byte( + `PRETTY_NAME="Debian GNU/Linux 8 (jessie)" NAME="Debian GNU/Linux" VERSION_ID="8" VERSION="8 (jessie)" @@ -34,13 +35,13 @@ ID=debian HOME_URL="http://www.debian.org/" SUPPORT_URL="http://www.debian.org/support/" BUG_REPORT_URL="https://bugs.debian.org/"`), + }, }, - }, - { - ExpectedNamespace: database.Namespace{Name: "ubuntu:15.10"}, - Data: map[string][]byte{ - "etc/os-release": []byte( - `NAME="Ubuntu" + { + ExpectedNamespace: &database.Namespace{Name: "ubuntu:15.10"}, + Data: map[string][]byte{ + "etc/os-release": []byte( + `NAME="Ubuntu" VERSION="15.10 (Wily Werewolf)" ID=ubuntu ID_LIKE=debian @@ -49,13 +50,13 @@ VERSION_ID="15.10" HOME_URL="http://www.ubuntu.com/" SUPPORT_URL="http://help.ubuntu.com/" BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"`), + }, }, - }, - { // Doesn't have quotes around VERSION_ID - ExpectedNamespace: database.Namespace{Name: "fedora:20"}, - Data: map[string][]byte{ - "etc/os-release": []byte( - `NAME=Fedora + { // Doesn't have quotes around VERSION_ID + ExpectedNamespace: &database.Namespace{Name: "fedora:20"}, + Data: map[string][]byte{ + "etc/os-release": []byte( + `NAME=Fedora VERSION="20 (Heisenbug)" ID=fedora VERSION_ID=20 @@ -68,10 +69,9 @@ REDHAT_BUGZILLA_PRODUCT="Fedora" REDHAT_BUGZILLA_PRODUCT_VERSION=20 REDHAT_SUPPORT_PRODUCT="Fedora" REDHAT_SUPPORT_PRODUCT_VERSION=20`), + }, }, - }, -} + } -func TestOsReleaseNamespaceDetector(t *testing.T) { - namespace.TestNamespaceDetector(t, &OsReleaseNamespaceDetector{}, osReleaseOSTests) + namespace.TestDetector(t, &OsReleaseNamespaceDetector{}, testData) } diff --git a/worker/detectors/namespace/redhatrelease/redhatrelease_test.go b/worker/detectors/namespace/redhatrelease/redhatrelease_test.go index 25c786ac..75e470f6 100644 --- a/worker/detectors/namespace/redhatrelease/redhatrelease_test.go +++ b/worker/detectors/namespace/redhatrelease/redhatrelease_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2016 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,21 +21,21 @@ import ( "github.com/coreos/clair/worker/detectors/namespace" ) -var redhatReleaseTests = []namespace.NamespaceTest{ - { - ExpectedNamespace: database.Namespace{Name: "centos:6"}, - Data: map[string][]byte{ - "etc/centos-release": []byte(`CentOS release 6.6 (Final)`), - }, - }, - { - ExpectedNamespace: database.Namespace{Name: "centos:7"}, - Data: map[string][]byte{ - "etc/system-release": []byte(`CentOS Linux release 7.1.1503 (Core)`), - }, - }, -} - func TestRedhatReleaseNamespaceDetector(t *testing.T) { - namespace.TestNamespaceDetector(t, &RedhatReleaseNamespaceDetector{}, redhatReleaseTests) + testData := []namespace.TestData{ + { + ExpectedNamespace: &database.Namespace{Name: "centos:6"}, + Data: map[string][]byte{ + "etc/centos-release": []byte(`CentOS release 6.6 (Final)`), + }, + }, + { + ExpectedNamespace: &database.Namespace{Name: "centos:7"}, + Data: map[string][]byte{ + "etc/system-release": []byte(`CentOS Linux release 7.1.1503 (Core)`), + }, + }, + } + + namespace.TestDetector(t, &RedhatReleaseNamespaceDetector{}, testData) } diff --git a/worker/detectors/namespace/test.go b/worker/detectors/namespace/test.go index 679d23db..89e33ec5 100644 --- a/worker/detectors/namespace/test.go +++ b/worker/detectors/namespace/test.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2016 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package namespace implements utilities common to implementations of +// NamespaceDetector. package namespace import ( @@ -22,13 +24,22 @@ import ( "github.com/stretchr/testify/assert" ) -type NamespaceTest struct { +// TestData represents the data used to test an implementation of +// NameSpaceDetector. +type TestData struct { Data map[string][]byte - ExpectedNamespace database.Namespace + ExpectedNamespace *database.Namespace } -func TestNamespaceDetector(t *testing.T, detector detectors.NamespaceDetector, tests []NamespaceTest) { - for _, test := range tests { - assert.Equal(t, test.ExpectedNamespace, *detector.Detect(test.Data)) +// TestDetector runs a detector on each provided instance of TestData and +// asserts the output to be equal to the expected output. +func TestDetector(t *testing.T, detector detectors.NamespaceDetector, testData []TestData) { + for _, td := range testData { + detectedNamespace := detector.Detect(td.Data) + if detectedNamespace == nil { + assert.Equal(t, td.ExpectedNamespace, detectedNamespace) + } else { + assert.Equal(t, td.ExpectedNamespace.Name, detectedNamespace.Name) + } } } From 1d5a9ddd3c2849ecd0346b55212c73a616b1382d Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 9 Nov 2016 15:53:01 -0500 Subject: [PATCH 02/13] detectors/namespace: add alpine-release detector --- .../namespace/alpinerelease/alpinerelease.go | 60 +++++++++++++++++++ .../alpinerelease/alpinerelease_test.go | 43 +++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 worker/detectors/namespace/alpinerelease/alpinerelease.go create mode 100644 worker/detectors/namespace/alpinerelease/alpinerelease_test.go diff --git a/worker/detectors/namespace/alpinerelease/alpinerelease.go b/worker/detectors/namespace/alpinerelease/alpinerelease.go new file mode 100644 index 00000000..7f2765a6 --- /dev/null +++ b/worker/detectors/namespace/alpinerelease/alpinerelease.go @@ -0,0 +1,60 @@ +// Copyright 2016 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package alpinerelease + +import ( + "bufio" + "bytes" + "regexp" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/worker/detectors" +) + +const ( + osName = "alpine" + alpineReleasePath = "/etc/alpine-release" +) + +var versionRegexp = regexp.MustCompile(`^(\d)+\.(\d)+\.(\d)+$`) + +func init() { + detectors.RegisterNamespaceDetector("alpine-release", &detector{}) +} + +// detector implements NamespaceDetector by reading the current version of +// Alpine Linux from /etc/alpine-release. +type detector struct{} + +func (d *detector) Detect(data map[string][]byte) *database.Namespace { + file, exists := data[alpineReleasePath] + if exists { + scanner := bufio.NewScanner(bytes.NewBuffer(file)) + for scanner.Scan() { + line := scanner.Text() + match := versionRegexp.FindStringSubmatch(line) + if len(match) > 0 { + version := match[0] + return &database.Namespace{Name: osName + ":" + version} + } + } + } + + return nil +} + +func (d *detector) GetRequiredFiles() []string { + return []string{alpineReleasePath} +} diff --git a/worker/detectors/namespace/alpinerelease/alpinerelease_test.go b/worker/detectors/namespace/alpinerelease/alpinerelease_test.go new file mode 100644 index 00000000..d56c5b4c --- /dev/null +++ b/worker/detectors/namespace/alpinerelease/alpinerelease_test.go @@ -0,0 +1,43 @@ +// Copyright 2016 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package alpinerelease + +import ( + "testing" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/worker/detectors/namespace" +) + +func TestAlpineReleaseNamespaceDetection(t *testing.T) { + testData := []namespace.TestData{ + { + ExpectedNamespace: &database.Namespace{Name: "alpine:0.3.4"}, + Data: map[string][]byte{"/etc/alpine-release": []byte(`0.3.4`)}, + }, + { + ExpectedNamespace: &database.Namespace{Name: "alpine:0.3.4"}, + Data: map[string][]byte{"/etc/alpine-release": []byte(` +0.3.4 +`)}, + }, + { + ExpectedNamespace: nil, + Data: map[string][]byte{}, + }, + } + + namespace.TestDetector(t, &detector{}, testData) +} From e4b5930f7769083767005d8e1730be81bf9eab8f Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 11 Nov 2016 15:14:37 -0500 Subject: [PATCH 03/13] detectors/feature: consistent naming and godoc --- worker/detectors/feature/dpkg/dpkg_test.go | 44 +++++++++++----------- worker/detectors/feature/rpm/rpm_test.go | 40 ++++++++++---------- worker/detectors/feature/test.go | 24 ++++++++---- 3 files changed, 58 insertions(+), 50 deletions(-) diff --git a/worker/detectors/feature/dpkg/dpkg_test.go b/worker/detectors/feature/dpkg/dpkg_test.go index 94fd34d4..d6f26d8c 100644 --- a/worker/detectors/feature/dpkg/dpkg_test.go +++ b/worker/detectors/feature/dpkg/dpkg_test.go @@ -22,30 +22,30 @@ import ( "github.com/coreos/clair/worker/detectors/feature" ) -var dpkgPackagesTests = []feature.FeatureVersionTest{ - // Test an Ubuntu dpkg status file - { - FeatureVersions: []database.FeatureVersion{ - // Two packages from this source are installed, it should only appear one time - { - Feature: database.Feature{Name: "pam"}, - Version: types.NewVersionUnsafe("1.1.8-3.1ubuntu3"), +func TestDpkgFeatureDetection(t *testing.T) { + testData := []feature.TestData{ + // Test an Ubuntu dpkg status file + { + FeatureVersions: []database.FeatureVersion{ + // Two packages from this source are installed, it should only appear one time + { + Feature: database.Feature{Name: "pam"}, + Version: types.NewVersionUnsafe("1.1.8-3.1ubuntu3"), + }, + { + Feature: database.Feature{Name: "makedev"}, // The source name and the package name are equals + Version: types.NewVersionUnsafe("2.3.1-93ubuntu1"), // The version comes from the "Version:" line + }, + { + Feature: database.Feature{Name: "gcc-5"}, + Version: types.NewVersionUnsafe("5.1.1-12ubuntu1"), // The version comes from the "Source:" line + }, }, - { - Feature: database.Feature{Name: "makedev"}, // The source name and the package name are equals - Version: types.NewVersionUnsafe("2.3.1-93ubuntu1"), // The version comes from the "Version:" line - }, - { - Feature: database.Feature{Name: "gcc-5"}, - Version: types.NewVersionUnsafe("5.1.1-12ubuntu1"), // The version comes from the "Source:" line + Data: map[string][]byte{ + "var/lib/dpkg/status": feature.LoadFileForTest("dpkg/testdata/status"), }, }, - Data: map[string][]byte{ - "var/lib/dpkg/status": feature.LoadFileForTest("dpkg/testdata/status"), - }, - }, -} + } -func TestDpkgFeaturesDetector(t *testing.T) { - feature.TestFeaturesDetector(t, &DpkgFeaturesDetector{}, dpkgPackagesTests) + feature.TestDetector(t, &DpkgFeaturesDetector{}, testData) } diff --git a/worker/detectors/feature/rpm/rpm_test.go b/worker/detectors/feature/rpm/rpm_test.go index adb5adb4..5e359aba 100644 --- a/worker/detectors/feature/rpm/rpm_test.go +++ b/worker/detectors/feature/rpm/rpm_test.go @@ -22,28 +22,28 @@ import ( "github.com/coreos/clair/worker/detectors/feature" ) -var rpmPackagesTests = []feature.FeatureVersionTest{ - // Test a CentOS 7 RPM database - // Memo: Use the following command on a RPM-based system to shrink a database: rpm -qa --qf "%{NAME}\n" |tail -n +3| xargs rpm -e --justdb - { - FeatureVersions: []database.FeatureVersion{ - // Two packages from this source are installed, it should only appear once - { - Feature: database.Feature{Name: "centos-release"}, - Version: types.NewVersionUnsafe("7-1.1503.el7.centos.2.8"), +func TestRpmFeatureDetection(t *testing.T) { + testData := []feature.TestData{ + // Test a CentOS 7 RPM database + // Memo: Use the following command on a RPM-based system to shrink a database: rpm -qa --qf "%{NAME}\n" |tail -n +3| xargs rpm -e --justdb + { + FeatureVersions: []database.FeatureVersion{ + // Two packages from this source are installed, it should only appear once + { + Feature: database.Feature{Name: "centos-release"}, + Version: types.NewVersionUnsafe("7-1.1503.el7.centos.2.8"), + }, + // Two packages from this source are installed, it should only appear once + { + Feature: database.Feature{Name: "filesystem"}, + Version: types.NewVersionUnsafe("3.2-18.el7"), + }, }, - // Two packages from this source are installed, it should only appear once - { - Feature: database.Feature{Name: "filesystem"}, - Version: types.NewVersionUnsafe("3.2-18.el7"), + Data: map[string][]byte{ + "var/lib/rpm/Packages": feature.LoadFileForTest("rpm/testdata/Packages"), }, }, - Data: map[string][]byte{ - "var/lib/rpm/Packages": feature.LoadFileForTest("rpm/testdata/Packages"), - }, - }, -} + } -func TestRpmFeaturesDetector(t *testing.T) { - feature.TestFeaturesDetector(t, &RpmFeaturesDetector{}, rpmPackagesTests) + feature.TestDetector(t, &RpmFeaturesDetector{}, testData) } diff --git a/worker/detectors/feature/test.go b/worker/detectors/feature/test.go index fe9f3d55..45335d7d 100644 --- a/worker/detectors/feature/test.go +++ b/worker/detectors/feature/test.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2016 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package feature implements utilities common to implementations of +// FeatureDetector. package feature import ( @@ -25,22 +27,28 @@ import ( "github.com/stretchr/testify/assert" ) -type FeatureVersionTest struct { - FeatureVersions []database.FeatureVersion +// TestData represents the data used to test an implementation of +// FeatureDetector. +type TestData struct { Data map[string][]byte + FeatureVersions []database.FeatureVersion } +// LoadFileForTest can be used in order to obtain the []byte contents of a file +// that is meant to be used for test data. func LoadFileForTest(name string) []byte { _, filename, _, _ := runtime.Caller(0) d, _ := ioutil.ReadFile(filepath.Join(filepath.Dir(filename)) + "/" + name) return d } -func TestFeaturesDetector(t *testing.T, detector detectors.FeaturesDetector, tests []FeatureVersionTest) { - for _, test := range tests { - featureVersions, err := detector.Detect(test.Data) - if assert.Nil(t, err) && assert.Len(t, featureVersions, len(test.FeatureVersions)) { - for _, expectedFeatureVersion := range test.FeatureVersions { +// TestDetector runs a detector on each provided instance of TestData and +// asserts the ouput to be equal to the expected output. +func TestDetector(t *testing.T, detector detectors.FeaturesDetector, testData []TestData) { + for _, td := range testData { + featureVersions, err := detector.Detect(td.Data) + if assert.Nil(t, err) && assert.Len(t, featureVersions, len(td.FeatureVersions)) { + for _, expectedFeatureVersion := range td.FeatureVersions { assert.Contains(t, featureVersions, expectedFeatureVersion) } } From fc908e65ba6c0ad3edb344cfb263adff8efe6f4e Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 10 Nov 2016 15:05:47 -0500 Subject: [PATCH 04/13] detectors/feature: add apk feature detector --- worker/detectors/feature/apk/apk.go | 84 ++++ worker/detectors/feature/apk/apk_test.go | 80 ++++ .../detectors/feature/apk/testdata/installed | 448 ++++++++++++++++++ 3 files changed, 612 insertions(+) create mode 100644 worker/detectors/feature/apk/apk.go create mode 100644 worker/detectors/feature/apk/apk_test.go create mode 100644 worker/detectors/feature/apk/testdata/installed diff --git a/worker/detectors/feature/apk/apk.go b/worker/detectors/feature/apk/apk.go new file mode 100644 index 00000000..7fd39571 --- /dev/null +++ b/worker/detectors/feature/apk/apk.go @@ -0,0 +1,84 @@ +// Copyright 2016 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apk + +import ( + "bufio" + "bytes" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" + "github.com/coreos/clair/worker/detectors" + "github.com/coreos/pkg/capnslog" +) + +var log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker/detectors/packages") + +func init() { + detectors.RegisterFeaturesDetector("apk", &detector{}) +} + +type detector struct{} + +func (d *detector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) { + file, exists := data["lib/apk/db/installed"] + if !exists { + return []database.FeatureVersion{}, nil + } + + // Iterate over each line in the "installed" file attempting to parse each + // package into a feature that will be stored in a set to guarantee + // uniqueness. + pkgSet := make(map[string]database.FeatureVersion) + ipkg := database.FeatureVersion{} + scanner := bufio.NewScanner(bytes.NewBuffer(file)) + for scanner.Scan() { + line := scanner.Text() + if len(line) < 2 { + continue + } + + // Parse the package name or version. + switch { + case line[:2] == "P:": + ipkg.Feature.Name = line[2:] + case line[:2] == "V:": + var err error + ipkg.Version, err = types.NewVersion(line[2:]) + if err != nil { + log.Warningf("could not parse package version '%s': %s. skipping", line[2:], err.Error()) + } + } + + // If we have a whole feature, store it in the set and try to parse a new + // one. + if ipkg.Feature.Name != "" && ipkg.Version.String() != "" { + pkgSet[ipkg.Feature.Name+"#"+ipkg.Version.String()] = ipkg + ipkg = database.FeatureVersion{} + } + } + + // Convert the map into a slice. + pkgs := make([]database.FeatureVersion, 0, len(pkgSet)) + for _, pkg := range pkgSet { + pkgs = append(pkgs, pkg) + } + + return pkgs, nil +} + +func (d *detector) GetRequiredFiles() []string { + return []string{"lib/apk/db/installed"} +} diff --git a/worker/detectors/feature/apk/apk_test.go b/worker/detectors/feature/apk/apk_test.go new file mode 100644 index 00000000..803a6657 --- /dev/null +++ b/worker/detectors/feature/apk/apk_test.go @@ -0,0 +1,80 @@ +// Copyright 2016 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apk + +import ( + "testing" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" + "github.com/coreos/clair/worker/detectors/feature" +) + +func TestAPKFeatureDetection(t *testing.T) { + testData := []feature.TestData{ + { + FeatureVersions: []database.FeatureVersion{ + { + Feature: database.Feature{Name: "musl"}, + Version: types.NewVersionUnsafe("1.1.14-r10"), + }, + { + Feature: database.Feature{Name: "busybox"}, + Version: types.NewVersionUnsafe("1.24.2-r9"), + }, + { + Feature: database.Feature{Name: "alpine-baselayout"}, + Version: types.NewVersionUnsafe("3.0.3-r0"), + }, + { + Feature: database.Feature{Name: "alpine-keys"}, + Version: types.NewVersionUnsafe("1.1-r0"), + }, + { + Feature: database.Feature{Name: "zlib"}, + Version: types.NewVersionUnsafe("1.2.8-r2"), + }, + { + Feature: database.Feature{Name: "libcrypto1.0"}, + Version: types.NewVersionUnsafe("1.0.2h-r1"), + }, + { + Feature: database.Feature{Name: "libssl1.0"}, + Version: types.NewVersionUnsafe("1.0.2h-r1"), + }, + { + Feature: database.Feature{Name: "apk-tools"}, + Version: types.NewVersionUnsafe("2.6.7-r0"), + }, + { + Feature: database.Feature{Name: "scanelf"}, + Version: types.NewVersionUnsafe("1.1.6-r0"), + }, + { + Feature: database.Feature{Name: "musl-utils"}, + Version: types.NewVersionUnsafe("1.1.14-r10"), + }, + { + Feature: database.Feature{Name: "libc-utils"}, + Version: types.NewVersionUnsafe("0.7-r0"), + }, + }, + Data: map[string][]byte{ + "lib/apk/db/installed": feature.LoadFileForTest("apk/testdata/installed"), + }, + }, + } + feature.TestDetector(t, &detector{}, testData) +} diff --git a/worker/detectors/feature/apk/testdata/installed b/worker/detectors/feature/apk/testdata/installed new file mode 100644 index 00000000..747555d6 --- /dev/null +++ b/worker/detectors/feature/apk/testdata/installed @@ -0,0 +1,448 @@ +C:Q11otALzX1d1D0kVawy06IairTXS0= +P:musl +V:1.1.14-r10 +A:x86_64 +S:445080 +I:569344 +T:the musl c library (libc) implementation +U:http://www.musl-libc.org/ +L:MIT +o:musl +m:Timo Teräs +t:1466181580 +c:e6c226fbe17bb8f856dce5f0d9bc088b333e6225 +p:so:libc.musl-x86_64.so.1=1 +F:lib +R:libc.musl-x86_64.so.1 +a:0:0:777 +Z:Q17yJ3JFNypA4mxhJJr0ou6CzsJVI= +R:ld-musl-x86_64.so.1 +a:0:0:755 +Z:Q1KUwsFGLHn/enpN9+QIpK/FmixtQ= + +C:Q1yhJHGSZ80L7cL0y4UKKGrBPwrUQ= +P:busybox +V:1.24.2-r9 +A:x86_64 +S:642121 +I:909312 +T:Size optimized toolbox of many common UNIX utilities +U:http://busybox.net +L:GPL2 +o:busybox +m:Natanael Copa +t:1466671780 +c:386b639b3917a9d1b8588dd87f09ed446501cddf +D:so:libc.musl-x86_64.so.1 +F:bin +R:busybox +a:0:0:755 +Z:Q1xOlCsdvx4O0gnKWoFCNKjz2quRE= +R:sh +a:0:0:777 +Z:Q1pcfTfDNEbNKQc2s1tia7da05M8Q= +F:etc +R:securetty +Z:Q14VDshgWFleuDbp4jqXk+UNES65Q= +R:udhcpd.conf +Z:Q1PWhOJ+TaEzAXw+XC6kkz/FXI/KA= +F:etc/logrotate.d +R:acpid +Z:Q1bPM2hPZy1LntZ/YdI4ZFJzVl1Y8= +F:etc/network +F:etc/network/if-up.d +F:etc/network/if-post-up.d +F:etc/network/if-pre-up.d +F:etc/network/if-post-down.d +F:etc/network/if-pre-down.d +F:etc/network/if-down.d +F:sbin +F:tmp +M:0:0:1777 +F:usr +F:usr/sbin +F:usr/bin +F:var +F:var/lib +F:var/lib/udhcpd +F:var/cache +F:var/cache/misc + +C:Q1ohSJYVBBHXLdH6/bMtHGxIVczPo= +P:alpine-baselayout +V:3.0.3-r0 +A:x86_64 +S:74697 +I:401408 +T:Alpine base dir structure and init scripts +U:http://git.alpinelinux.org/cgit/aports/tree/main/alpine-baselayout +L:GPL2 +o:alpine-baselayout +m:Natanael Copa +t:1466181584 +c:d0bef446b94220475c60e78f2e081f38390b89ca +D:busybox so:libc.musl-x86_64.so.1 +F:dev +F:dev/shm +F:dev/pts +F:etc +R:hosts +Z:Q1S93L8EsQ/7zGSnfGDfj5I7bjCS4= +R:sysctl.conf +Z:Q14upz3tfnNxZkIEsUhWn7Xoiw96g= +R:group +Z:Q1zNuxdqs1x+nJO8WucpZfQrdiapA= +R:protocols +Z:Q13FqXUnvuOpMDrH/6rehxuYAEE34= +R:fstab +Z:Q11Q7hNe8QpDS531guqCdrXBzoA/o= +R:mtab +a:0:0:777 +Z:Q1kiljhXXH1LlQroHsEJIkPZg2eiw= +R:profile +Z:Q1FrM1yy3WJbQTc9LgnKTn5tRovlE= +R:TZ +Z:Q1uHH18uOLEzp56qFUP843WSoKM9E= +R:shells +Z:Q1ojm2YdpCJ6B/apGDaZ/Sdb2xJkA= +R:motd +Z:Q1MaUHN/Rf32Lf67Owrq1BXQU7usE= +R:inittab +Z:Q1TsthbhW7QzWRe1E/NKwTOuD4pHc= +R:hostname +Z:Q16nVwYVXP/tChvUPdukVD2ifXOmc= +R:modules +Z:Q1C7P4uPQo8B6P6V+O78ybHl0AHhA= +R:services +Z:Q1NBe0rrC8HMzNmVf4ybSENcsdey0= +R:shadow +a:0:42:640 +Z:Q1LG0ii8vP4gQgDmSnK0WBtjtovlg= +R:passwd +Z:Q11HpI0rBp2zsun4+LIIBENg8JQUE= +F:etc/profile.d +R:color_prompt +Z:Q10wL23GuSCVfumMRgakabUI6EsSk= +F:etc/init.d +F:etc/apk +F:etc/sysctl.d +R:00-alpine.conf +Z:Q1kZy9KEvjykp1vCw1kWgdvBuEXvg= +F:etc/modprobe.d +R:i386.conf +Z:Q1pnay/njn6ol9cCssL7KiZZ8etlc= +R:blacklist.conf +Z:Q1+TdC1pulajuYy0ebcos8V/aMeqk= +R:aliases.conf +Z:Q1udaZLaeaalyuCcnBgCKPIybDO08= +R:kms.conf +Z:Q1yH/c6fBvCWn0Huny5Rf/GET2Jbs= +F:etc/modules-load.d +F:etc/opt +F:etc/conf.d +F:etc/crontabs +R:root +a:0:0:600 +Z:Q1vfk1apUWI4yLJGhhNRd0kJixfvY= +F:etc/periodic +F:etc/periodic/hourly +F:etc/periodic/weekly +F:etc/periodic/monthly +F:etc/periodic/15min +F:etc/periodic/daily +F:etc/network +F:etc/network/if-up.d +F:etc/network/if-pre-up.d +F:etc/network/if-post-down.d +F:etc/network/if-down.d +F:home +F:lib +F:lib/mdev +F:lib/firmware +F:media +F:media/floppy +F:media/cdrom +F:media/usb +F:mnt +F:proc +F:root +M:0:0:700 +F:run +F:sbin +R:mkmntdirs +a:0:0:755 +Z:Q1lGBnGMsnB3SEZL/oHeN99F1/ie8= +F:srv +F:sys +F:tmp +M:0:0:1777 +F:usr +F:usr/sbin +F:usr/bin +F:usr/local +F:usr/local/bin +F:usr/local/lib +F:usr/local/share +F:usr/share +F:usr/share/man +F:usr/share/misc +F:var +F:var/lock +F:var/lock/subsys +F:var/tmp +M:0:0:1777 +F:var/log +F:var/lib +F:var/lib/misc +F:var/local +F:var/opt +F:var/cache +F:var/cache/misc +F:var/spool +F:var/spool/cron +R:crontabs +a:0:0:777 +Z:Q1OFZt+ZMp7j0Gny0rqSKuWJyqYmA= +F:var/empty +F:var/run + +C:Q1Te9/+u5q66cAwdYlcDJvdcu4+iU= +P:alpine-keys +V:1.1-r0 +A:x86_64 +S:7787 +I:36864 +T:Public keys for Alpine Linux packages +U:http://alpinelinux.org +L:GPL +o:alpine-keys +m:Natanael Copa +t:1461964035 +c:d0b08b1e17d40d21196df7709fdb95f37165615d +r:alpine-base +F:etc +F:etc/apk +F:etc/apk/keys +R:alpine-devel@lists.alpinelinux.org-4d07755e.rsa.pub +Z:Q1XfH9IG0ZFgbOIssIhpiWqDlSspY= +R:alpine-devel@lists.alpinelinux.org-524d27bb.rsa.pub +Z:Q1BTqS+H/UUyhQuzHwiBl47+BTKuU= +R:alpine-devel@lists.alpinelinux.org-5243ef4b.rsa.pub +Z:Q1v7YWZYzAWoclaLDI45jEguI7YN0= +R:alpine-devel@lists.alpinelinux.org-5261cecb.rsa.pub +Z:Q1NnGuDsdQOx4ZNYfB3N97eLyGPkI= +R:alpine-devel@lists.alpinelinux.org-4a6a0840.rsa.pub +Z:Q1OvCFSO94z97c80mIDCxqGkh2Og4= + +C:Q179BNNNQKqOszFIASc2TCeounYO8= +P:zlib +V:1.2.8-r2 +A:x86_64 +S:71733 +I:98304 +T:A compression/decompression Library +U:http://zlib.net +L:zlib +o:zlib +m:Natanael Copa +t:1461931151 +c:a7329a4de4d10d99206c78c229dc3742880cd042 +D:so:libc.musl-x86_64.so.1 +p:so:libz.so.1=1.2.8 +F:lib +R:libz.so.1.2.8 +a:0:0:755 +Z:Q1bh8VMdNZcPKyIQTtLMtz/0VL2H0= +R:libz.so.1 +a:0:0:777 +Z:Q1x1VqGi0rW5lxmvPQOjnqNapz/OU= + +C:Q1Zchg/48QO2ZPQLqEmj9aUcaci+s= +P:libcrypto1.0 +V:1.0.2h-r1 +A:x86_64 +S:1714530 +I:2527232 +T:Crypto library from openssl +U:http://openssl.org +L:openssl +o:openssl +m:Timo Teras +t:1466620012 +c:38c6e1fd86f4d9cba4c146b8bdcd71f84e1a4ee7 +D:so:libc.musl-x86_64.so.1 so:libz.so.1 +p:so:libcrypto.so.1.0.0=1.0.0 +F:lib +R:libcrypto.so.1.0.0 +a:0:0:555 +Z:Q1DgSoxP0AWq64XJSPXT0yRTjSSBk= +F:usr +F:usr/bin +R:c_rehash +a:0:0:755 +Z:Q1XZt0LbTr44grnBtK+Yihjynh2EE= +F:usr/lib +R:libcrypto.so.1.0.0 +a:0:0:777 +Z:Q1jLDKGBtunzKi5FKmK/QTAqfh6uI= +F:usr/lib/engines +R:libubsec.so +a:0:0:555 +Z:Q1SaoP91RpISCN8KO3AxuB8Tzyc/A= +R:libatalla.so +a:0:0:555 +Z:Q1W31yRhAZE9X5Z5zy9l6mkn1tbcQ= +R:libcapi.so +a:0:0:555 +Z:Q1iriyqA2XunZb8pxmsOeRML2ZsRg= +R:libgost.so +a:0:0:555 +Z:Q1zwS6yHAzzdnrHb9BV8pIsE2yIgg= +R:libcswift.so +a:0:0:555 +Z:Q16/TBTN+WkIFQeFlCPTDc5Xs33bU= +R:libchil.so +a:0:0:555 +Z:Q1fMssNRSfAgg4nZVYm0IzYG2gTLk= +R:libgmp.so +a:0:0:555 +Z:Q1onZiPB/LsF3Xqn8rwH5FcYLkuf4= +R:libnuron.so +a:0:0:555 +Z:Q189GVmg2NFt2nTCqfcSl7Wtoym1o= +R:lib4758cca.so +a:0:0:555 +Z:Q1aNXgEbvxfvZdZpa4frZ9p6eq2Y4= +R:libsureware.so +a:0:0:555 +Z:Q14Z9HkfG+WqvGRIb42iugBuBKOag= +R:libpadlock.so +a:0:0:555 +Z:Q1OAwUUirl7Q1OiqTjB96lKBQYbMc= +R:libaep.so +a:0:0:555 +Z:Q1nrvE4qxl4AXEC/psF1Eh9n6E7Pg= + +C:Q1cSDzN+k7K0xE4PXzGhW2DcZ4yhQ= +P:libssl1.0 +V:1.0.2h-r1 +A:x86_64 +S:274743 +I:442368 +T:SSL shared libraries +U:http://openssl.org +L:openssl +o:openssl +m:Timo Teras +t:1466620012 +c:38c6e1fd86f4d9cba4c146b8bdcd71f84e1a4ee7 +D:so:libc.musl-x86_64.so.1 so:libcrypto.so.1.0.0 +p:so:libssl.so.1.0.0=1.0.0 +F:lib +R:libssl.so.1.0.0 +a:0:0:555 +Z:Q1wdqn4897nQP5l/f3SV4DWf9QOkQ= +F:usr +F:usr/lib +R:libssl.so.1.0.0 +a:0:0:777 +Z:Q1ke5dnHGVWcEyRpOe0/lKEqizHHQ= + +C:Q1CZDArNYrQXWBjDpMxg7RHxTiO9g= +P:apk-tools +V:2.6.7-r0 +A:x86_64 +S:146592 +I:253952 +T:Alpine Package Keeper - package manager for alpine +U:http://git.alpinelinux.org/cgit/apk-tools/ +L:GPL2 +o:apk-tools +m:Natanael Copa +t:1464341138 +c:08b6e2ae43356fbb24ef5c262fb08bfe09d675b0 +D:so:libc.musl-x86_64.so.1 so:libcrypto.so.1.0.0 so:libssl.so.1.0.0 so:libz.so.1 +F:etc +F:etc/apk +F:etc/apk/keys +F:etc/apk/protected_paths.d +F:sbin +R:apk +a:0:0:755 +Z:Q1ozOGvVOspzXfX1bUFjrjZ6ygEB0= +F:usr +F:var +F:var/lib +F:var/lib/apk +F:var/cache +F:var/cache/misc + +C:Q1+bq4F8Wtk+kqYhi8D3WWtlfALZA= +P:scanelf +V:1.1.6-r0 +A:x86_64 +S:53666 +I:90112 +T:Scan ELF binaries for stuff +U:https://wiki.gentoo.org/wiki/Hardened/PaX_Utilities +L:GPL2 +o:pax-utils +m:Natanael Copa +t:1461934341 +c:891f6254fb6e1f11f62ee2c9fd35784fd313b9d1 +D:so:libc.musl-x86_64.so.1 +r:pax-utils +F:usr +F:usr/bin +R:scanelf +a:0:0:755 +Z:Q12S+aDBkRA63GvTmx45HqqbOKBz0= + +C:Q1YIqh3Tnv97xmFdPl4zODdQ+PXsw= +P:musl-utils +V:1.1.14-r10 +A:x86_64 +S:50427 +I:110592 +T:the musl c library (libc) implementation +U:http://www.musl-libc.org/ +L:MIT BSD GPL2+ +o:musl +m:Timo Teräs +t:1466181579 +c:e6c226fbe17bb8f856dce5f0d9bc088b333e6225 +D:!uclibc-utils scanelf musl=1.1.14-r10 so:libc.musl-x86_64.so.1 +r:libiconv uclibc-utils +F:sbin +R:ldconfig +a:0:0:755 +Z:Q1Kja2+POZKxEkUOZqwSjC6kmaED4= +F:usr +F:usr/bin +R:iconv +a:0:0:755 +Z:Q1DmLsMEsBDtwb8S0z1pl0MYH29+o= +R:ldd +a:0:0:777 +Z:Q1a/NMxsyXbxLcmrTyGQtovas5gJk= +R:getconf +a:0:0:755 +Z:Q1iub9vwJRjlaCnu21SWjb504fUqk= +R:getent +a:0:0:755 +Z:Q1Z5dnRfQ7nvRbRRSpM1k0J7UMdng= + +C:Q1kFW8ev12zyZyGYBC9y/Epe1PqWE= +P:libc-utils +V:0.7-r0 +A:x86_64 +S:2804 +I:4096 +T:Meta package to pull in correct libc +U:http://alpinelinux.org +L:GPL +o:libc-dev +m:Natanael Copa +t:1461934274 +c:e3725c0af137717d6883265a92db3838900b5cee +D:musl-utils From 0cb8fc9455905c658732fbc36ee9efe41fb78de5 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 18 Nov 2016 16:24:54 -0500 Subject: [PATCH 05/13] updater/fetchers: add alpine secdb fetcher --- updater/fetchers/alpine/alpine.go | 186 +++++++++++++++++++++ updater/fetchers/alpine/alpine_test.go | 42 +++++ updater/fetchers/alpine/testdata/main.yaml | 69 ++++++++ 3 files changed, 297 insertions(+) create mode 100644 updater/fetchers/alpine/alpine.go create mode 100644 updater/fetchers/alpine/alpine_test.go create mode 100644 updater/fetchers/alpine/testdata/main.yaml diff --git a/updater/fetchers/alpine/alpine.go b/updater/fetchers/alpine/alpine.go new file mode 100644 index 00000000..0a074a57 --- /dev/null +++ b/updater/fetchers/alpine/alpine.go @@ -0,0 +1,186 @@ +// Copyright 2016 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package alpine implements a vulnerability Fetcher using the alpine-secdb +// git repository. +package alpine + +import ( + "errors" + "io" + "io/ioutil" + "os" + "strings" + + "gopkg.in/yaml.v2" + + "github.com/coreos/pkg/capnslog" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/updater" + "github.com/coreos/clair/utils" + cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/utils/types" +) + +const ( + secdbGitURL = "http://git.alpinelinux.org/cgit/alpine-secdb" + updaterFlag = "alpine-secdbUpdater" + nvdURLPrefix = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=" +) + +var ( + // ErrFilesystem is returned when a fetcher fails to interact with the local filesystem. + ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs") + + // ErrGitFailure is returned when a fetcher fails to interact with git. + ErrGitFailure = errors.New("updater/fetchers: something went wrong when interacting with git") + + log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/alpine") +) + +func init() { + updater.RegisterFetcher("alpine", &fetcher{}) +} + +type fetcher struct { + repositoryLocalPath string +} + +func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherResponse, err error) { + log.Info("fetching Alpine vulnerabilities") + + // Pull the master branch. + var commit string + commit, err = f.pullRepository() + if err != nil { + return + } + + // Ask the database for the latest commit we successfully applied. + var dbCommit string + dbCommit, err = db.GetKeyValue(updaterFlag) + if err != nil { + return + } + + // Set the updaterFlag to equal the commit processed. + resp.FlagName = updaterFlag + resp.FlagValue = commit + + // Short-circuit if there have been no updates. + if commit == dbCommit { + log.Debug("no alpine update") + return + } + + // Append any changed vulnerabilities to the response. + for _, namespace := range []string{"v3.3", "v3.4"} { + var file io.ReadCloser + file, err = os.Open(f.repositoryLocalPath + "/" + namespace + "/main.yaml") + if err != nil { + return + } + + var vulns []database.Vulnerability + vulns, err = parseYAML(file) + if err != nil { + return + } + resp.Vulnerabilities = append(resp.Vulnerabilities, vulns...) + file.Close() + } + + return +} + +func (f *fetcher) pullRepository() (commit string, err error) { + if _, pathExists := os.Stat(f.repositoryLocalPath); f.repositoryLocalPath == "" || os.IsNotExist(pathExists) { + if f.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil { + return "", ErrFilesystem + } + + if out, err := utils.Exec(f.repositoryLocalPath, "git", "pull"); err != nil { + f.Clean() + log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out) + return "", cerrors.ErrCouldNotDownload + } + } + + out, err := utils.Exec(f.repositoryLocalPath, "git", "rev-parse", "HEAD") + if err != nil { + return "", ErrGitFailure + } + + commit = strings.TrimSpace(string(out)) + return +} + +func (f *fetcher) Clean() { + if f.repositoryLocalPath != "" { + os.RemoveAll(f.repositoryLocalPath) + } +} + +type secdbFile struct { + Distro string `yaml:"distroversion"` + Packages []struct { + Pkg struct { + Name string `yaml:"name"` + Version string `yaml:"ver"` + Fixes []string `yaml:"fixes"` + } `yaml:"pkg"` + } `yaml:"packages"` +} + +func parseYAML(r io.Reader) (vulns []database.Vulnerability, err error) { + var rBytes []byte + rBytes, err = ioutil.ReadAll(r) + if err != nil { + return + } + + var file secdbFile + err = yaml.Unmarshal(rBytes, &file) + if err != nil { + return + } + for _, pack := range file.Packages { + pkg := pack.Pkg + for _, fix := range pkg.Fixes { + version, err := types.NewVersion(pkg.Version) + if err != nil { + log.Warningf("could not parse package version '%s': %s. skipping", pkg.Version, err.Error()) + continue + } + + var vuln database.Vulnerability + vuln.Name = fix + vuln.Link = nvdURLPrefix + fix + vuln.FixedIn = []database.FeatureVersion{ + { + Feature: database.Feature{ + Namespace: database.Namespace{Name: "alpine:" + file.Distro}, + Name: pkg.Name, + }, + Version: version, + }, + } + + vulns = append(vulns, vuln) + } + } + + return +} diff --git a/updater/fetchers/alpine/alpine_test.go b/updater/fetchers/alpine/alpine_test.go new file mode 100644 index 00000000..b4d61655 --- /dev/null +++ b/updater/fetchers/alpine/alpine_test.go @@ -0,0 +1,42 @@ +// Copyright 2016 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package alpine + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAlpineYAMLParsing(t *testing.T) { + _, filename, _, _ := runtime.Caller(0) + path := filepath.Join(filepath.Dir(filename)) + + testData, _ := os.Open(path + "/testdata/main.yaml") + defer testData.Close() + + vulns, err := parseYAML(testData) + if err != nil { + assert.Nil(t, err) + } + assert.Equal(t, len(vulns), 15) + assert.Equal(t, vulns[0].Name, "CVE-2016-2147") + assert.Equal(t, vulns[0].FixedIn[0].Feature.Namespace.Name, "alpine:v3.3") + assert.Equal(t, vulns[0].FixedIn[0].Feature.Name, "busybox") + assert.Equal(t, vulns[0].Link, "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-2147") +} diff --git a/updater/fetchers/alpine/testdata/main.yaml b/updater/fetchers/alpine/testdata/main.yaml new file mode 100644 index 00000000..21f24668 --- /dev/null +++ b/updater/fetchers/alpine/testdata/main.yaml @@ -0,0 +1,69 @@ +--- +distroversion: v3.3 +reponame: main +archs: + - x86_64 + - x86 + - armhf +urlprefix: http://dl-cdn.alpinelinux.org/alpine +apkurl: "{{urlprefix}}/{{distroversion}}/{{reponame}}/{{arch}}/{{pkg.name}}-${{pkg.ver}}.apk" +packages: + - pkg: + name: busybox + ver: 1.24.2-r0 + fixes: + - CVE-2016-2147 + - CVE-2016-2148 + - pkg: + name: expat + ver: 2.1.1-r1 + fixes: + - CVE-2016-0718 + - pkg: + name: gd + ver: 2.1.1-r1 + fixes: + - CVE-2016-3074 + - pkg: + name: giflib + ver: 5.1.1-r1 + fixes: + - CVE-2016-3977 + - pkg: + name: jq + ver: 1.5-r1 + fixes: + - CVE-2015-8863 + - pkg: + name: libarchive + ver: 3.1.2-r3 + fixes: + - CVE-2016-1541 + - pkg: + name: libssh2 + ver: 1.6.0-r1 + fixes: + - CVE-2016-0787 + - pkg: + name: mercurial + ver: 3.7.3-r1 + fixes: + - CVE-2016-3105 + - pkg: + name: openssl + ver: 1.0.2h-r1 + fixes: + - CVE-2016-2177 + - CVE-2016-2178 + - pkg: + name: pcre + ver: 8.38-r1 + fixes: + - CVE-2016-1283 + - CVE-2016-3191 + - pkg: + name: wpa_supplicant + ver: 2.5-r2 + fixes: + - CVE-2016-4476 + - CVE-2016-4477 From c1e0f618caad4464d90ba20e13baaa1fb1617cb9 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 29 Nov 2016 15:09:35 -0500 Subject: [PATCH 06/13] dockerfile: add git dependency --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3eb1346e..fdc48c0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,10 +17,10 @@ FROM golang:1.6 MAINTAINER Quentin Machu RUN apt-get update && \ - apt-get install -y bzr rpm xz-utils && \ + apt-get install -y git bzr rpm xz-utils && \ apt-get autoremove -y && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # 18MAR2016 + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # 29NOV2016 VOLUME /config From 805f620b4b4e84514ac139b49c0d88cd85a4625e Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 29 Nov 2016 15:15:26 -0500 Subject: [PATCH 07/13] README: add alpine data sources --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index cd403c0d..ada68187 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,7 @@ By indexing the features of an image into the database, images only need to be r | [Debian Security Bug Tracker] | Debian 6, 7, 8, unstable namespaces | [dpkg] | [Debian] | | [Ubuntu CVE Tracker] | Ubuntu 12.04, 12.10, 13.04, 14.04, 14.10, 15.04, 15.10, 16.04 namespaces | [dpkg] | [GPLv2] | | [Red Hat Security Data] | CentOS 5, 6, 7 namespaces | [rpm] | [CVRF] | +| [Alpine SecDB] | Alpine 3.3, Alpine 3.4 namespaces | [apk] | [MIT] | | [NVD] | Generic Vulnerability Metadata | N/A | [Public Domain] | [Debian Security Bug Tracker]: https://security-tracker.debian.org/tracker @@ -173,6 +174,9 @@ By indexing the features of an image into the database, images only need to be r [GPLv2]: https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html [CVRF]: http://www.icasi.org/cvrf-licensing/ [Public Domain]: https://nvd.nist.gov/faq +[Alpine SecDB]: http://git.alpinelinux.org/cgit/alpine-secdb/ +[apk]: http://git.alpinelinux.org/cgit/apk-tools/ +[MIT]: https://gist.github.com/jzelinskie/6da1e2da728424d88518be2adbd76979 ### Customization From 3d90cac427e52a6470f112746fa86a595ffe8717 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 14 Dec 2016 17:01:38 -0500 Subject: [PATCH 08/13] alpine: add support for v3.4 YAML schema --- updater/fetchers/alpine/alpine.go | 73 ++++- updater/fetchers/alpine/alpine_test.go | 34 ++- .../testdata/{main.yaml => v33_main.yaml} | 0 .../fetchers/alpine/testdata/v34_main.yaml | 251 ++++++++++++++++++ 4 files changed, 344 insertions(+), 14 deletions(-) rename updater/fetchers/alpine/testdata/{main.yaml => v33_main.yaml} (100%) create mode 100644 updater/fetchers/alpine/testdata/v34_main.yaml diff --git a/updater/fetchers/alpine/alpine.go b/updater/fetchers/alpine/alpine.go index 0a074a57..7705a3d7 100644 --- a/updater/fetchers/alpine/alpine.go +++ b/updater/fetchers/alpine/alpine.go @@ -92,12 +92,19 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon if err != nil { return } + log.Debug(namespace) var vulns []database.Vulnerability - vulns, err = parseYAML(file) + switch namespace { + case "v3.3": + vulns, err = parse33YAML(file) + case "v3.4": + vulns, err = parse34YAML(file) + } if err != nil { return } + resp.Vulnerabilities = append(resp.Vulnerabilities, vulns...) file.Close() } @@ -111,7 +118,7 @@ func (f *fetcher) pullRepository() (commit string, err error) { return "", ErrFilesystem } - if out, err := utils.Exec(f.repositoryLocalPath, "git", "pull"); err != nil { + if out, err := utils.Exec(f.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil { f.Clean() log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out) return "", cerrors.ErrCouldNotDownload @@ -133,7 +140,7 @@ func (f *fetcher) Clean() { } } -type secdbFile struct { +type secdb33File struct { Distro string `yaml:"distroversion"` Packages []struct { Pkg struct { @@ -144,14 +151,14 @@ type secdbFile struct { } `yaml:"packages"` } -func parseYAML(r io.Reader) (vulns []database.Vulnerability, err error) { +func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) { var rBytes []byte rBytes, err = ioutil.ReadAll(r) if err != nil { return } - var file secdbFile + var file secdb33File err = yaml.Unmarshal(rBytes, &file) if err != nil { return @@ -166,6 +173,7 @@ func parseYAML(r io.Reader) (vulns []database.Vulnerability, err error) { } var vuln database.Vulnerability + vuln.Severity = types.Unknown vuln.Name = fix vuln.Link = nvdURLPrefix + fix vuln.FixedIn = []database.FeatureVersion{ @@ -177,10 +185,63 @@ func parseYAML(r io.Reader) (vulns []database.Vulnerability, err error) { Version: version, }, } - vulns = append(vulns, vuln) } } return } + +type secdb34File struct { + Distro string `yaml:"distroversion"` + Packages []struct { + Pkg struct { + Name string `yaml:"name"` + Fixes map[string][]string `yaml:"secfixes"` + } `yaml:"pkg"` + } `yaml:"packages"` +} + +func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) { + var rBytes []byte + rBytes, err = ioutil.ReadAll(r) + if err != nil { + return + } + + var file secdb34File + err = yaml.Unmarshal(rBytes, &file) + if err != nil { + return + } + + for _, pack := range file.Packages { + pkg := pack.Pkg + for versionStr, vulnStrs := range pkg.Fixes { + version, err := types.NewVersion(versionStr) + if err != nil { + log.Warningf("could not parse package version '%s': %s. skipping", versionStr, err.Error()) + continue + } + + for _, vulnStr := range vulnStrs { + var vuln database.Vulnerability + vuln.Severity = types.Unknown + vuln.Name = vulnStr + vuln.Link = nvdURLPrefix + vulnStr + vuln.FixedIn = []database.FeatureVersion{ + { + Feature: database.Feature{ + Namespace: database.Namespace{Name: "alpine:" + file.Distro}, + Name: pkg.Name, + }, + Version: version, + }, + } + vulns = append(vulns, vuln) + } + } + } + + return +} diff --git a/updater/fetchers/alpine/alpine_test.go b/updater/fetchers/alpine/alpine_test.go index b4d61655..4588d97e 100644 --- a/updater/fetchers/alpine/alpine_test.go +++ b/updater/fetchers/alpine/alpine_test.go @@ -23,20 +23,38 @@ import ( "github.com/stretchr/testify/assert" ) -func TestAlpineYAMLParsing(t *testing.T) { +func TestAlpine33YAMLParsing(t *testing.T) { _, filename, _, _ := runtime.Caller(0) path := filepath.Join(filepath.Dir(filename)) - testData, _ := os.Open(path + "/testdata/main.yaml") + testData, _ := os.Open(path + "/testdata/v33_main.yaml") defer testData.Close() - vulns, err := parseYAML(testData) + vulns, err := parse33YAML(testData) if err != nil { assert.Nil(t, err) } - assert.Equal(t, len(vulns), 15) - assert.Equal(t, vulns[0].Name, "CVE-2016-2147") - assert.Equal(t, vulns[0].FixedIn[0].Feature.Namespace.Name, "alpine:v3.3") - assert.Equal(t, vulns[0].FixedIn[0].Feature.Name, "busybox") - assert.Equal(t, vulns[0].Link, "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-2147") + assert.Equal(t, 15, len(vulns)) + assert.Equal(t, "CVE-2016-2147", vulns[0].Name) + assert.Equal(t, "alpine:v3.3", vulns[0].FixedIn[0].Feature.Namespace.Name) + assert.Equal(t, "busybox", vulns[0].FixedIn[0].Feature.Name) + assert.Equal(t, "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-2147", vulns[0].Link) +} + +func TestAlpine34YAMLParsing(t *testing.T) { + _, filename, _, _ := runtime.Caller(0) + path := filepath.Join(filepath.Dir(filename)) + + testData, _ := os.Open(path + "/testdata/v34_main.yaml") + defer testData.Close() + + vulns, err := parse34YAML(testData) + if err != nil { + assert.Nil(t, err) + } + assert.Equal(t, 105, len(vulns)) + assert.Equal(t, "CVE-2016-5387", vulns[0].Name) + assert.Equal(t, "alpine:v3.4", vulns[0].FixedIn[0].Feature.Namespace.Name) + assert.Equal(t, "apache2", vulns[0].FixedIn[0].Feature.Name) + assert.Equal(t, "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-5387", vulns[0].Link) } diff --git a/updater/fetchers/alpine/testdata/main.yaml b/updater/fetchers/alpine/testdata/v33_main.yaml similarity index 100% rename from updater/fetchers/alpine/testdata/main.yaml rename to updater/fetchers/alpine/testdata/v33_main.yaml diff --git a/updater/fetchers/alpine/testdata/v34_main.yaml b/updater/fetchers/alpine/testdata/v34_main.yaml new file mode 100644 index 00000000..325977eb --- /dev/null +++ b/updater/fetchers/alpine/testdata/v34_main.yaml @@ -0,0 +1,251 @@ +--- +distroversion: v3.4 +reponame: main +archs: + - x86_64 + - x86 + - armhf +urlprefix: http://dl-cdn.alpinelinux.org/alpine +apkurl: "{{urlprefix}}/{{distroversion}}/{{reponame}}/{{arch}}/{{pkg.name}}-{{pkg.ver}}.apk" +packages: + - pkg: + name: apache2 + secfixes: + 2.4.23-r1: + - CVE-2016-5387 + - pkg: + name: busybox + secfixes: + 1.24.2-r0: + - CVE-2016-2147 + - CVE-2016-2148 + - pkg: + name: bzip2 + secfixes: + 1.0.6-r5: + - CVE-2016-3189 + - pkg: + name: c-ares + secfixes: + 1.12.0-r0: + - CVE-2016-5180 + - pkg: + name: collectd + secfixes: + 5.5.2-r0: + - CVE-2016-6254 + - pkg: + name: curl + secfixes: + 7.51.0: + - CVE-2016-8615 + - CVE-2016-8616 + - CVE-2016-8617 + - CVE-2016-8618 + - CVE-2016-8619 + - CVE-2016-8620 + - CVE-2016-8621 + - CVE-2016-8622 + - CVE-2016-8623 + - CVE-2016-8624 + - CVE-2016-8625 + 7.50.3-r0: + - CVE-2016-7167 + 7.50.2-r0: + - CVE-2016-7141 + 7.50.1-r0: + - CVE-2016-5419 + - CVE-2016-5420 + - CVE-2016-5421 + 7.36.0-r0: + - CVE-2014-0138 + - CVE-2014-0139 + - pkg: + name: expat + secfixes: + 2.1.1-r1: + - CVE-2016-0718 + 2.1.1-r2: + - CVE-2016-4472 + - pkg: + name: flex + secfixes: + 2.6.1-r0: + - CVE-2016-6354 + - pkg: + name: gd + secfixes: + 2.2.1-r0: + - CVE-2016-3074 + 2.2.2-r0: + - CVE-2015-8874 + - CVE-2016-5767 + 2.2.3-r0: + - CVE-2016-5766 + - CVE-2016-6128 + - CVE-2016-6132 + - CVE-2016-6207 + - CVE-2016-6214 + 2.2.3-r1: + - CVE-2016-7568 + - pkg: + name: giflib + secfixes: + 5.1.4-r0: + - CVE-2016-3977 + - pkg: + name: guile + secfixes: + 2.0.11-r3: + - CVE-2016-8605 + - CVE-2016-8606 + - pkg: + name: icu + secfixes: + 57.1-r1: + - CVE-2016-6293 + - pkg: + name: imagemagick + secfixes: + 6.9.5.3: + - CVE-2016-5010 + - CVE-2016-5687 + - CVE-2016-5688 + - CVE-2016-5689 + - CVE-2016-5690 + - CVE-2016-5691 + - CVE-2016-5841 + - CVE-2016-5842 + - CVE-2016-6491 + 6.9.5.9-r1: + - CVE-2016-7799 + - CVE-2016-7906 + - pkg: + name: jq + secfixes: + 1.5-r1: + - CVE-2015-8863 + - pkg: + name: krb5 + secfixes: + 1.14-r1: + - CVE-2015-8629 + - CVE-2015-8630 + - CVE-2015-8631 + 1.14-r2: + - CVE-2016-3119 + 1.14.3-r0: + - CVE-2016-3120 + - pkg: + name: libarchive + secfixes: + 3.2.0-r0: + - CVE-2016-1541 + 3.2.1-r0: + - CVE-2015-8934 + - CVE-2016-4300 + - CVE-2016-4302 + - CVE-2016-4809 + - CVE-2016-5844 + - CVE-2016-6250 + - pkg: + name: libbsd + secfixes: + 0.8.2: + - CVE-2016-2090 + - pkg: + name: libidn + secfixes: + 1.33-r0: + - CVE-2015-8948 + - CVE-2016-6261 + - CVE-2016-6262 + - CVE-2016-6263 + - pkg: + name: libssh2 + secfixes: + 1.7.0-r0: + - CVE-2016-0787 + - pkg: + name: openjpeg + secfixes: + 2.1.2-r0: + - CVE-2016-7445 + - pkg: + name: openssh + secfixes: + 7.2_p2-r1: + - CVE-2016-6210 + 7.2_p2-r2: + - CVE-2016-6515 + - pkg: + name: openssl + secfixes: + 1.0.2h-r0: + - CVE-2016-2107 + - CVE-2016-2105 + - CVE-2016-2106 + - CVE-2016-2109 + - CVE-2016-2176 + 1.0.2h-r1: + - CVE-2016-2177 + - CVE-2016-2178 + 1.0.2h-r2: + - CVE-2016-2180 + 1.0.2h-r3: + - CVE-2016-2179 + - CVE-2016-2182 + - CVE-2016-6302 + - CVE-2016-6303 + 1.0.2h-r4: + - CVE-2016-2181 + 1.0.2i-r0: + - CVE-2016-2183 + - CVE-2016-6304 + - CVE-2016-6306 + - pkg: + name: pcre + secfixes: + 8.38-r1: + - CVE-2016-1283 + - CVE-2016-3191 + - pkg: + name: py-django + secfixes: + 1.8.16-r0: + - CVE-2016-9013 + - CVE-2016-9014 + - pkg: + name: tar + secfixes: + 1.29-r1: + - CVE-2016-6321 + - pkg: + name: wget + secfixes: + 1.17.1-r1: + - CVE-2016-4971 + - pkg: + name: wpa_supplicant + secfixes: + 2.5-r3: + - CVE-2016-4476 + - CVE-2016-4477 + - pkg: + name: xen + secfixes: + 4.6.3-r1: + - CVE-2016-6258 XSA-182 + - CVE-2016-6259 XSA-183 + - CVE-2016-5403 XSA-184 + 4.6.3-r2: + - CVE-2016-7092 XSA-185 + - CVE-2016-7093 XSA-186 + - CVE-2016-7094 XSA-187 + 4.6.3-r3: + - CVE-2016-7777 XSA-190 + - pkg: + name: zabbix + secfixes: + 3.0.4-r0: + - ZBX-11023 From f8457b98e7dfe1c6fde7783c59a1c1143823a0e2 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 14 Dec 2016 17:01:59 -0500 Subject: [PATCH 09/13] alpine: compile alpine into clair binary --- cmd/clair/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 732a8923..00f6461f 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -28,6 +28,7 @@ import ( // Register components _ "github.com/coreos/clair/notifier/notifiers" + _ "github.com/coreos/clair/updater/fetchers/alpine" _ "github.com/coreos/clair/updater/fetchers/debian" _ "github.com/coreos/clair/updater/fetchers/opensuse" _ "github.com/coreos/clair/updater/fetchers/rhel" @@ -38,9 +39,11 @@ import ( _ "github.com/coreos/clair/worker/detectors/data/aci" _ "github.com/coreos/clair/worker/detectors/data/docker" + _ "github.com/coreos/clair/worker/detectors/feature/apk" _ "github.com/coreos/clair/worker/detectors/feature/dpkg" _ "github.com/coreos/clair/worker/detectors/feature/rpm" + _ "github.com/coreos/clair/worker/detectors/namespace/alpinerelease" _ "github.com/coreos/clair/worker/detectors/namespace/aptsources" _ "github.com/coreos/clair/worker/detectors/namespace/lsbrelease" _ "github.com/coreos/clair/worker/detectors/namespace/osrelease" From 9be305d19f5fec286492cd09ed71623f356fcdc0 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 14 Dec 2016 18:00:05 -0500 Subject: [PATCH 10/13] alpine: truncate namespace to "vMAJOR.MINOR" --- .../namespace/alpinerelease/alpinerelease.go | 7 ++++--- .../alpinerelease/alpinerelease_test.go | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/worker/detectors/namespace/alpinerelease/alpinerelease.go b/worker/detectors/namespace/alpinerelease/alpinerelease.go index 7f2765a6..ec7d5f2a 100644 --- a/worker/detectors/namespace/alpinerelease/alpinerelease.go +++ b/worker/detectors/namespace/alpinerelease/alpinerelease.go @@ -18,6 +18,7 @@ import ( "bufio" "bytes" "regexp" + "strings" "github.com/coreos/clair/database" "github.com/coreos/clair/worker/detectors" @@ -25,7 +26,7 @@ import ( const ( osName = "alpine" - alpineReleasePath = "/etc/alpine-release" + alpineReleasePath = "etc/alpine-release" ) var versionRegexp = regexp.MustCompile(`^(\d)+\.(\d)+\.(\d)+$`) @@ -46,8 +47,8 @@ func (d *detector) Detect(data map[string][]byte) *database.Namespace { line := scanner.Text() match := versionRegexp.FindStringSubmatch(line) if len(match) > 0 { - version := match[0] - return &database.Namespace{Name: osName + ":" + version} + versionNumbers := strings.Split(match[0], ".") + return &database.Namespace{Name: osName + ":" + "v" + versionNumbers[0] + "." + versionNumbers[1]} } } } diff --git a/worker/detectors/namespace/alpinerelease/alpinerelease_test.go b/worker/detectors/namespace/alpinerelease/alpinerelease_test.go index d56c5b4c..3052ce03 100644 --- a/worker/detectors/namespace/alpinerelease/alpinerelease_test.go +++ b/worker/detectors/namespace/alpinerelease/alpinerelease_test.go @@ -24,12 +24,20 @@ import ( func TestAlpineReleaseNamespaceDetection(t *testing.T) { testData := []namespace.TestData{ { - ExpectedNamespace: &database.Namespace{Name: "alpine:0.3.4"}, - Data: map[string][]byte{"/etc/alpine-release": []byte(`0.3.4`)}, + ExpectedNamespace: &database.Namespace{Name: "alpine:v3.3"}, + Data: map[string][]byte{"etc/alpine-release": []byte(`3.3.4`)}, }, { - ExpectedNamespace: &database.Namespace{Name: "alpine:0.3.4"}, - Data: map[string][]byte{"/etc/alpine-release": []byte(` + ExpectedNamespace: &database.Namespace{Name: "alpine:v3.4"}, + Data: map[string][]byte{"etc/alpine-release": []byte(`3.4.0`)}, + }, + { + ExpectedNamespace: &database.Namespace{Name: "alpine:v0.3"}, + Data: map[string][]byte{"etc/alpine-release": []byte(`0.3.4`)}, + }, + { + ExpectedNamespace: &database.Namespace{Name: "alpine:v0.3"}, + Data: map[string][]byte{"etc/alpine-release": []byte(` 0.3.4 `)}, }, From 59e6c628dcb2b4306ed971a609f7a50973ca2b2c Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 15 Dec 2016 16:36:41 -0500 Subject: [PATCH 11/13] alpine: refactor fetcher & git pull on update --- updater/fetchers/alpine/alpine.go | 67 ++++++++++++++++++------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/updater/fetchers/alpine/alpine.go b/updater/fetchers/alpine/alpine.go index 7705a3d7..73ed87b7 100644 --- a/updater/fetchers/alpine/alpine.go +++ b/updater/fetchers/alpine/alpine.go @@ -35,6 +35,7 @@ import ( ) const ( + // When available, this should be updated to use HTTPS. secdbGitURL = "http://git.alpinelinux.org/cgit/alpine-secdb" updaterFlag = "alpine-secdbUpdater" nvdURLPrefix = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=" @@ -87,32 +88,37 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon // Append any changed vulnerabilities to the response. for _, namespace := range []string{"v3.3", "v3.4"} { - var file io.ReadCloser - file, err = os.Open(f.repositoryLocalPath + "/" + namespace + "/main.yaml") - if err != nil { - return - } - log.Debug(namespace) - var vulns []database.Vulnerability - switch namespace { - case "v3.3": - vulns, err = parse33YAML(file) - case "v3.4": - vulns, err = parse34YAML(file) - } + vulns, err = parseVulnsFromNamespace(f.repositoryLocalPath, namespace) if err != nil { return } - resp.Vulnerabilities = append(resp.Vulnerabilities, vulns...) - file.Close() + } + + return +} + +func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.Vulnerability, err error) { + var file io.ReadCloser + file, err = os.Open(repositoryPath + "/" + namespace + "/main.yaml") + if err != nil { + return + } + defer file.Close() + + switch namespace { + case "v3.3": + vulns, err = parse33YAML(file) + case "v3.4": + vulns, err = parse34YAML(file) } return } func (f *fetcher) pullRepository() (commit string, err error) { + // If the repository doesn't exist, clone it. if _, pathExists := os.Stat(f.repositoryLocalPath); f.repositoryLocalPath == "" || os.IsNotExist(pathExists) { if f.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil { return "", ErrFilesystem @@ -123,6 +129,12 @@ func (f *fetcher) pullRepository() (commit string, err error) { log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out) return "", cerrors.ErrCouldNotDownload } + } else { + // The repository exists and it needs to be refreshed via a pull. + _, err := utils.Exec(f.repositoryLocalPath, "git", "pull") + if err != nil { + return "", ErrGitFailure + } } out, err := utils.Exec(f.repositoryLocalPath, "git", "rev-parse", "HEAD") @@ -172,23 +184,22 @@ func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) { continue } - var vuln database.Vulnerability - vuln.Severity = types.Unknown - vuln.Name = fix - vuln.Link = nvdURLPrefix + fix - vuln.FixedIn = []database.FeatureVersion{ - { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "alpine:" + file.Distro}, - Name: pkg.Name, + vulns = append(vulns, database.Vulnerability{ + Name: fix, + Severity: types.Unknown, + Link: nvdURLPrefix + fix, + FixedIn: []database.FeatureVersion{ + { + Feature: database.Feature{ + Namespace: database.Namespace{Name: "alpine:" + file.Distro}, + Name: pkg.Name, + }, + Version: version, }, - Version: version, }, - } - vulns = append(vulns, vuln) + }) } } - return } From 3be8dfcf99dfde8aaf67c60eb279c17b8a7e83d2 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 16 Dec 2016 16:45:14 -0500 Subject: [PATCH 12/13] fetchers/alpine: auto detect namespaces --- updater/fetchers/alpine/alpine.go | 47 +++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/updater/fetchers/alpine/alpine.go b/updater/fetchers/alpine/alpine.go index 73ed87b7..0324b143 100644 --- a/updater/fetchers/alpine/alpine.go +++ b/updater/fetchers/alpine/alpine.go @@ -86,8 +86,10 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon return } + var namespaces []string + namespaces, err = detectNamespaces(f.repositoryLocalPath) // Append any changed vulnerabilities to the response. - for _, namespace := range []string{"v3.3", "v3.4"} { + for _, namespace := range namespaces { var vulns []database.Vulnerability vulns, err = parseVulnsFromNamespace(f.repositoryLocalPath, namespace) if err != nil { @@ -99,6 +101,40 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon return } +func detectNamespaces(path string) ([]string, error) { + // Open the root directory. + dir, err := os.Open(path) + if err != nil { + return nil, err + } + defer dir.Close() + + // Get a list of the namspaces from the directory names. + names, err := dir.Readdirnames(0) + if err != nil { + return nil, err + } + + var namespaces []string + for _, name := range names { + // Filter out hidden directories like `.git`. + if strings.HasPrefix(name, ".") { + continue + } + + namespaces = append(namespaces, name) + } + + return namespaces, nil +} + +type parserFunc func(io.Reader) ([]database.Vulnerability, error) + +var parsers = map[string]parserFunc{ + "v3.3": parse33YAML, + "v3.4": parse34YAML, +} + func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.Vulnerability, err error) { var file io.ReadCloser file, err = os.Open(repositoryPath + "/" + namespace + "/main.yaml") @@ -107,13 +143,12 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database } defer file.Close() - switch namespace { - case "v3.3": - vulns, err = parse33YAML(file) - case "v3.4": - vulns, err = parse34YAML(file) + parseFunc, exists := parsers[namespace] + if !exists { + return } + vulns, err = parseFunc(file) return } From f74cd352438a710d51f56b9f3a32b77cc403fe32 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 16 Dec 2016 17:42:06 -0500 Subject: [PATCH 13/13] fetchers/alpine: add notes for untracked namespaces --- updater/fetchers/alpine/alpine.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/updater/fetchers/alpine/alpine.go b/updater/fetchers/alpine/alpine.go index 0324b143..49f445f6 100644 --- a/updater/fetchers/alpine/alpine.go +++ b/updater/fetchers/alpine/alpine.go @@ -18,6 +18,7 @@ package alpine import ( "errors" + "fmt" "io" "io/ioutil" "os" @@ -91,10 +92,14 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon // Append any changed vulnerabilities to the response. for _, namespace := range namespaces { var vulns []database.Vulnerability - vulns, err = parseVulnsFromNamespace(f.repositoryLocalPath, namespace) + var note string + vulns, note, err = parseVulnsFromNamespace(f.repositoryLocalPath, namespace) if err != nil { return } + if note != "" { + resp.Notes = append(resp.Notes, note) + } resp.Vulnerabilities = append(resp.Vulnerabilities, vulns...) } @@ -135,7 +140,7 @@ var parsers = map[string]parserFunc{ "v3.4": parse34YAML, } -func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.Vulnerability, err error) { +func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.Vulnerability, note string, err error) { var file io.ReadCloser file, err = os.Open(repositoryPath + "/" + namespace + "/main.yaml") if err != nil { @@ -145,6 +150,7 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database parseFunc, exists := parsers[namespace] if !exists { + note = fmt.Sprintf("The file %s is not mapped to any Alpine version number", namespace) return }