vulnsrc/alpine: unify schema and parse v3.5

HEAD of Alpine SecDB now uses one consistent schema for all of their
vulnerabilities, so the logic around parsing different versions can now
be removed. This change also crawls the directory structure to parse all
files due to the addition of community.yaml tracking community Alpine
Linux packages.

ext/vulnsrc/alpine/alpine.go

@@ -17,11 +17,11 @@
 package alpine
 
 import (
-	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 
 	"gopkg.in/yaml.v2"
@@ -79,8 +79,13 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er
 		return
 	}
 
+	// Get the list of namespaces from the repository.
 	var namespaces []string
-	namespaces, err = detectNamespaces(u.repositoryLocalPath)
+	namespaces, err = ls(u.repositoryLocalPath, directoriesOnly)
+	if err != nil {
+		return
+	}
+
 	// Append any changed vulnerabilities to the response.
 	for _, namespace := range namespaces {
 		var vulns []database.Vulnerability
@@ -104,58 +109,70 @@ func (u *updater) Clean() {
 	}
 }
 
-func detectNamespaces(path string) ([]string, error) {
-	// Open the root directory.
+type lsFilter int
+
+const (
+	filesOnly lsFilter = iota
+	directoriesOnly
+)
+
+func ls(path string, filter lsFilter) ([]string, error) {
 	dir, err := os.Open(path)
 	if err != nil {
 		return nil, err
 	}
 	defer dir.Close()
 
-	// Get a list of the namspaces from the directory names.
 	finfos, err := dir.Readdir(0)
 	if err != nil {
 		return nil, err
 	}
 
-	var namespaces []string
+	var files []string
 	for _, info := range finfos {
-		if !info.IsDir() {
+		if filter == directoriesOnly && !info.IsDir() {
+			continue
+		}
+
+		if filter == filesOnly && info.IsDir() {
 			continue
 		}
-		// Filter out hidden directories like `.git`.
+
 		if strings.HasPrefix(info.Name(), ".") {
 			continue
 		}
 
-		namespaces = append(namespaces, info.Name())
+		files = append(files, info.Name())
 	}
 
-	return namespaces, nil
-}
-
-type parserFunc func(io.Reader) ([]database.Vulnerability, error)
-
-var parsers = map[string]parserFunc{
-	"v3.3": parse33YAML,
-	"v3.4": parse34YAML,
+	return files, nil
 }
 
 func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.Vulnerability, note string, err error) {
-	var file io.ReadCloser
-	file, err = os.Open(repositoryPath + "/" + namespace + "/main.yaml")
+	nsDir := filepath.Join(repositoryPath, namespace)
+	var dbFilenames []string
+	dbFilenames, err = ls(nsDir, filesOnly)
 	if err != nil {
 		return
 	}
-	defer file.Close()
 
-	parseFunc, exists := parsers[namespace]
-	if !exists {
-		note = fmt.Sprintf("The file %s is not mapped to any Alpine version number", namespace)
-		return
+	for _, filename := range dbFilenames {
+		var file io.ReadCloser
+		file, err = os.Open(filepath.Join(nsDir, filename))
+		if err != nil {
+			return
+		}
+
+		var fileVulns []database.Vulnerability
+		fileVulns, err = parseYAML(file)
+		if err != nil {
+			return
+		}
+
+		vulns = append(vulns, fileVulns...)
+		file.Close()
 	}
 
-	vulns, err = parseFunc(file)
 	return
 }
 
@@ -193,61 +210,7 @@ func (u *updater) pullRepository() (commit string, err error) {
 	return
 }
 
-type secdb33File struct {
-	Distro   string `yaml:"distroversion"`
-	Packages []struct {
-		Pkg struct {
-			Name    string   `yaml:"name"`
-			Version string   `yaml:"ver"`
-			Fixes   []string `yaml:"fixes"`
-		} `yaml:"pkg"`
-	} `yaml:"packages"`
-}
-
-func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
-	var rBytes []byte
-	rBytes, err = ioutil.ReadAll(r)
-	if err != nil {
-		return
-	}
-
-	var file secdb33File
-	err = yaml.Unmarshal(rBytes, &file)
-	if err != nil {
-		return
-	}
-	for _, pack := range file.Packages {
-		pkg := pack.Pkg
-		for _, fix := range pkg.Fixes {
-			err = versionfmt.Valid(dpkg.ParserName, pkg.Version)
-			if err != nil {
-				log.Warningf("could not parse package version '%s': %s. skipping", pkg.Version, err.Error())
-				continue
-			}
-
-			vulns = append(vulns, database.Vulnerability{
-				Name:     fix,
-				Severity: database.UnknownSeverity,
-				Link:     nvdURLPrefix + fix,
-				FixedIn: []database.FeatureVersion{
-					{
-						Feature: database.Feature{
-							Namespace: database.Namespace{
-								Name:          "alpine:" + file.Distro,
-								VersionFormat: dpkg.ParserName,
-							},
-							Name: pkg.Name,
-						},
-						Version: pkg.Version,
-					},
-				},
-			})
-		}
-	}
-	return
-}
-
-type secdb34File struct {
+type secDBFile struct {
 	Distro   string `yaml:"distroversion"`
 	Packages []struct {
 		Pkg struct {
@@ -257,14 +220,14 @@ type secdb34File struct {
 	} `yaml:"packages"`
 }
 
-func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
+func parseYAML(r io.Reader) (vulns []database.Vulnerability, err error) {
 	var rBytes []byte
 	rBytes, err = ioutil.ReadAll(r)
 	if err != nil {
 		return
 	}
 
-	var file secdb34File
+	var file secDBFile
 	err = yaml.Unmarshal(rBytes, &file)
 	if err != nil {
 		return

ext/vulnsrc/alpine/alpine_test.go

@@ -23,32 +23,14 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestAlpine33YAMLParsing(t *testing.T) {
-	_, filename, _, _ := runtime.Caller(0)
-	path := filepath.Join(filepath.Dir(filename))
-
-	testData, _ := os.Open(path + "/testdata/v33_main.yaml")
-	defer testData.Close()
-
-	vulns, err := parse33YAML(testData)
-	if err != nil {
-		assert.Nil(t, err)
-	}
-	assert.Equal(t, 15, len(vulns))
-	assert.Equal(t, "CVE-2016-2147", vulns[0].Name)
-	assert.Equal(t, "alpine:v3.3", vulns[0].FixedIn[0].Feature.Namespace.Name)
-	assert.Equal(t, "busybox", vulns[0].FixedIn[0].Feature.Name)
-	assert.Equal(t, "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-2147", vulns[0].Link)
-}
-
-func TestAlpine34YAMLParsing(t *testing.T) {
+func TestYAMLParsing(t *testing.T) {
 	_, filename, _, _ := runtime.Caller(0)
 	path := filepath.Join(filepath.Dir(filename))
 
 	testData, _ := os.Open(path + "/testdata/v34_main.yaml")
 	defer testData.Close()
 
-	vulns, err := parse34YAML(testData)
+	vulns, err := parseYAML(testData)
 	if err != nil {
 		assert.Nil(t, err)
 	}

ext/vulnsrc/alpine/testdata/v33_main.yaml

@@ -1,69 +0,0 @@
----
-distroversion: v3.3
-reponame: main
-archs:
-  - x86_64
-  - x86
-  - armhf
-urlprefix: http://dl-cdn.alpinelinux.org/alpine
-apkurl: "{{urlprefix}}/{{distroversion}}/{{reponame}}/{{arch}}/{{pkg.name}}-${{pkg.ver}}.apk"
-packages:
-  - pkg:
-      name: busybox
-      ver: 1.24.2-r0
-      fixes:
-        - CVE-2016-2147
-        - CVE-2016-2148
-  - pkg:
-      name: expat
-      ver: 2.1.1-r1
-      fixes:
-        - CVE-2016-0718
-  - pkg:
-      name: gd
-      ver: 2.1.1-r1
-      fixes:
-        - CVE-2016-3074
-  - pkg:
-      name: giflib
-      ver: 5.1.1-r1
-      fixes:
-        - CVE-2016-3977
-  - pkg:
-      name: jq
-      ver: 1.5-r1
-      fixes:
-        - CVE-2015-8863
-  - pkg:
-      name: libarchive
-      ver: 3.1.2-r3
-      fixes:
-        - CVE-2016-1541
-  - pkg:
-      name: libssh2
-      ver:  1.6.0-r1
-      fixes:
-        - CVE-2016-0787
-  - pkg:
-      name: mercurial
-      ver: 3.7.3-r1
-      fixes:
-        - CVE-2016-3105
-  - pkg:
-      name: openssl
-      ver: 1.0.2h-r1
-      fixes:
-        - CVE-2016-2177
-        - CVE-2016-2178
-  - pkg:
-      name: pcre
-      ver: 8.38-r1
-      fixes:
-        - CVE-2016-1283
-        - CVE-2016-3191
-  - pkg:
-      name: wpa_supplicant
-      ver: 2.5-r2
-      fixes:
-        - CVE-2016-4476
-        - CVE-2016-4477