Merge pull request #316 from jzelinskie/fix-alpine

Alpine v3.5 support

config.example.yaml

@@ -20,7 +20,7 @@ clair:
     options:
       # PostgreSQL Connection string
       # https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
-      source:
+      source: host=localhost port=5432 user=postgres sslmode=disable statement_timeout=60000
 
       # Number of elements kept in the cache
       # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.

ext/versionfmt/dpkg/parser.go

@@ -96,10 +96,6 @@ func newVersion(str string) (version, error) {
 		return version{}, errors.New("No version")
 	}
 
-	if !unicode.IsDigit(rune(v.version[0])) {
-		return version{}, errors.New("version does not start with digit")
-	}
-
 	for i := 0; i < len(v.version); i = i + 1 {
 		r := rune(v.version[i])
 		if !unicode.IsDigit(r) && !unicode.IsLetter(r) && !containsRune(versionAllowedSymbols, r) {

ext/versionfmt/dpkg/parser_test.go

@@ -70,8 +70,10 @@ func TestParse(t *testing.T) {
 		// Test invalid characters in epoch
 		{"a:0-0", version{}, true},
 		{"A:0-0", version{}, true},
-		// Test version not starting with a digit
+		// Test version not starting with a digit.
-		{"0:abc3-0", version{}, true},
+		// While recommended by the specification, this is not strictly required and
+		// at least one vulnerable Alpine package deviates from this scheme.
+		{"0:abc3-0", version{epoch: 0, version: "abc3", revision: "0"}, false},
 	}
 	for _, c := range cases {
 		v, err := newVersion(c.str)

ext/vulnsrc/alpine/alpine.go

@@ -17,11 +17,11 @@
 package alpine
 
 import (
-	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 
 	"gopkg.in/yaml.v2"
@@ -79,8 +79,13 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er
 		return
 	}
 
+	// Get the list of namespaces from the repository.
 	var namespaces []string
-	namespaces, err = detectNamespaces(u.repositoryLocalPath)
+	namespaces, err = ls(u.repositoryLocalPath, directoriesOnly)
+	if err != nil {
+		return
+	}
+
 	// Append any changed vulnerabilities to the response.
 	for _, namespace := range namespaces {
 		var vulns []database.Vulnerability
@@ -104,58 +109,70 @@ func (u *updater) Clean() {
 	}
 }
 
-func detectNamespaces(path string) ([]string, error) {
+type lsFilter int
-	// Open the root directory.
+
+const (
+	filesOnly lsFilter = iota
+	directoriesOnly
+)
+
+func ls(path string, filter lsFilter) ([]string, error) {
 	dir, err := os.Open(path)
 	if err != nil {
 		return nil, err
 	}
 	defer dir.Close()
 
-	// Get a list of the namspaces from the directory names.
 	finfos, err := dir.Readdir(0)
 	if err != nil {
 		return nil, err
 	}
 
-	var namespaces []string
+	var files []string
 	for _, info := range finfos {
-		if !info.IsDir() {
+		if filter == directoriesOnly && !info.IsDir() {
 			continue
 		}
-		// Filter out hidden directories like `.git`.
+
+		if filter == filesOnly && info.IsDir() {
+			continue
+		}
+
 		if strings.HasPrefix(info.Name(), ".") {
 			continue
 		}
 
-		namespaces = append(namespaces, info.Name())
+		files = append(files, info.Name())
 	}
 
-	return namespaces, nil
+	return files, nil
-}
-
-type parserFunc func(io.Reader) ([]database.Vulnerability, error)
-
-var parsers = map[string]parserFunc{
-	"v3.3": parse33YAML,
-	"v3.4": parse34YAML,
 }
 
 func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.Vulnerability, note string, err error) {
-	var file io.ReadCloser
+	nsDir := filepath.Join(repositoryPath, namespace)
-	file, err = os.Open(repositoryPath + "/" + namespace + "/main.yaml")
+	var dbFilenames []string
+	dbFilenames, err = ls(nsDir, filesOnly)
 	if err != nil {
 		return
 	}
-	defer file.Close()
 
-	parseFunc, exists := parsers[namespace]
+	for _, filename := range dbFilenames {
-	if !exists {
+		var file io.ReadCloser
-		note = fmt.Sprintf("The file %s is not mapped to any Alpine version number", namespace)
+		file, err = os.Open(filepath.Join(nsDir, filename))
+		if err != nil {
 			return
 		}
 
-	vulns, err = parseFunc(file)
+		var fileVulns []database.Vulnerability
+		fileVulns, err = parseYAML(file)
+		if err != nil {
+			return
+		}
+
+		vulns = append(vulns, fileVulns...)
+		file.Close()
+	}
+
 	return
 }
 
@@ -193,61 +210,7 @@ func (u *updater) pullRepository() (commit string, err error) {
 	return
 }
 
-type secdb33File struct {
+type secDBFile struct {
-	Distro   string `yaml:"distroversion"`
-	Packages []struct {
-		Pkg struct {
-			Name    string   `yaml:"name"`
-			Version string   `yaml:"ver"`
-			Fixes   []string `yaml:"fixes"`
-		} `yaml:"pkg"`
-	} `yaml:"packages"`
-}
-
-func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
-	var rBytes []byte
-	rBytes, err = ioutil.ReadAll(r)
-	if err != nil {
-		return
-	}
-
-	var file secdb33File
-	err = yaml.Unmarshal(rBytes, &file)
-	if err != nil {
-		return
-	}
-	for _, pack := range file.Packages {
-		pkg := pack.Pkg
-		for _, fix := range pkg.Fixes {
-			err = versionfmt.Valid(dpkg.ParserName, pkg.Version)
-			if err != nil {
-				log.Warningf("could not parse package version '%s': %s. skipping", pkg.Version, err.Error())
-				continue
-			}
-
-			vulns = append(vulns, database.Vulnerability{
-				Name:     fix,
-				Severity: database.UnknownSeverity,
-				Link:     nvdURLPrefix + fix,
-				FixedIn: []database.FeatureVersion{
-					{
-						Feature: database.Feature{
-							Namespace: database.Namespace{
-								Name:          "alpine:" + file.Distro,
-								VersionFormat: dpkg.ParserName,
-							},
-							Name: pkg.Name,
-						},
-						Version: pkg.Version,
-					},
-				},
-			})
-		}
-	}
-	return
-}
-
-type secdb34File struct {
 	Distro   string `yaml:"distroversion"`
 	Packages []struct {
 		Pkg struct {
@@ -257,14 +220,14 @@ type secdb34File struct {
 	} `yaml:"packages"`
 }
 
-func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
+func parseYAML(r io.Reader) (vulns []database.Vulnerability, err error) {
 	var rBytes []byte
 	rBytes, err = ioutil.ReadAll(r)
 	if err != nil {
 		return
 	}
 
-	var file secdb34File
+	var file secDBFile
 	err = yaml.Unmarshal(rBytes, &file)
 	if err != nil {
 		return

ext/vulnsrc/alpine/alpine_test.go

@@ -23,32 +23,14 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestAlpine33YAMLParsing(t *testing.T) {
+func TestYAMLParsing(t *testing.T) {
-	_, filename, _, _ := runtime.Caller(0)
-	path := filepath.Join(filepath.Dir(filename))
-
-	testData, _ := os.Open(path + "/testdata/v33_main.yaml")
-	defer testData.Close()
-
-	vulns, err := parse33YAML(testData)
-	if err != nil {
-		assert.Nil(t, err)
-	}
-	assert.Equal(t, 15, len(vulns))
-	assert.Equal(t, "CVE-2016-2147", vulns[0].Name)
-	assert.Equal(t, "alpine:v3.3", vulns[0].FixedIn[0].Feature.Namespace.Name)
-	assert.Equal(t, "busybox", vulns[0].FixedIn[0].Feature.Name)
-	assert.Equal(t, "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-2147", vulns[0].Link)
-}
-
-func TestAlpine34YAMLParsing(t *testing.T) {
 	_, filename, _, _ := runtime.Caller(0)
 	path := filepath.Join(filepath.Dir(filename))
 
 	testData, _ := os.Open(path + "/testdata/v34_main.yaml")
 	defer testData.Close()
 
-	vulns, err := parse34YAML(testData)
+	vulns, err := parseYAML(testData)
 	if err != nil {
 		assert.Nil(t, err)
 	}

ext/vulnsrc/alpine/testdata/v33_main.yaml

@@ -1,69 +0,0 @@
----
-distroversion: v3.3
-reponame: main
-archs:
-  - x86_64
-  - x86
-  - armhf
-urlprefix: http://dl-cdn.alpinelinux.org/alpine
-apkurl: "{{urlprefix}}/{{distroversion}}/{{reponame}}/{{arch}}/{{pkg.name}}-${{pkg.ver}}.apk"
-packages:
-  - pkg:
-      name: busybox
-      ver: 1.24.2-r0
-      fixes:
-        - CVE-2016-2147
-        - CVE-2016-2148
-  - pkg:
-      name: expat
-      ver: 2.1.1-r1
-      fixes:
-        - CVE-2016-0718
-  - pkg:
-      name: gd
-      ver: 2.1.1-r1
-      fixes:
-        - CVE-2016-3074
-  - pkg:
-      name: giflib
-      ver: 5.1.1-r1
-      fixes:
-        - CVE-2016-3977
-  - pkg:
-      name: jq
-      ver: 1.5-r1
-      fixes:
-        - CVE-2015-8863
-  - pkg:
-      name: libarchive
-      ver: 3.1.2-r3
-      fixes:
-        - CVE-2016-1541
-  - pkg:
-      name: libssh2
-      ver:  1.6.0-r1
-      fixes:
-        - CVE-2016-0787
-  - pkg:
-      name: mercurial
-      ver: 3.7.3-r1
-      fixes:
-        - CVE-2016-3105
-  - pkg:
-      name: openssl
-      ver: 1.0.2h-r1
-      fixes:
-        - CVE-2016-2177
-        - CVE-2016-2178
-  - pkg:
-      name: pcre
-      ver: 8.38-r1
-      fixes:
-        - CVE-2016-1283
-        - CVE-2016-3191
-  - pkg:
-      name: wpa_supplicant
-      ver: 2.5-r2
-      fixes:
-        - CVE-2016-4476
-        - CVE-2016-4477