From c8622d5f3472698e872b7b6a6ff817da42bbcf07 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 7 Feb 2017 11:48:42 -0800 Subject: [PATCH 1/3] vulnsrc/alpine: unify schema and parse v3.5 HEAD of Alpine SecDB now uses one consistent schema for all of their vulnerabilities, so the logic around parsing different versions can now be removed. This change also crawls the directory structure to parse all files due to the addition of community.yaml tracking community Alpine Linux packages. --- ext/vulnsrc/alpine/alpine.go | 127 ++++++++-------------- ext/vulnsrc/alpine/alpine_test.go | 22 +--- ext/vulnsrc/alpine/testdata/v33_main.yaml | 69 ------------ 3 files changed, 47 insertions(+), 171 deletions(-) delete mode 100644 ext/vulnsrc/alpine/testdata/v33_main.yaml diff --git a/ext/vulnsrc/alpine/alpine.go b/ext/vulnsrc/alpine/alpine.go index 5f3acb22..e99926ba 100644 --- a/ext/vulnsrc/alpine/alpine.go +++ b/ext/vulnsrc/alpine/alpine.go @@ -17,11 +17,11 @@ package alpine import ( - "fmt" "io" "io/ioutil" "os" "os/exec" + "path/filepath" "strings" "gopkg.in/yaml.v2" @@ -79,8 +79,13 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er return } + // Get the list of namespaces from the repository. var namespaces []string - namespaces, err = detectNamespaces(u.repositoryLocalPath) + namespaces, err = ls(u.repositoryLocalPath, directoriesOnly) + if err != nil { + return + } + // Append any changed vulnerabilities to the response. for _, namespace := range namespaces { var vulns []database.Vulnerability @@ -104,58 +109,70 @@ func (u *updater) Clean() { } } -func detectNamespaces(path string) ([]string, error) { - // Open the root directory. +type lsFilter int + +const ( + filesOnly lsFilter = iota + directoriesOnly +) + +func ls(path string, filter lsFilter) ([]string, error) { dir, err := os.Open(path) if err != nil { return nil, err } defer dir.Close() - // Get a list of the namspaces from the directory names. finfos, err := dir.Readdir(0) if err != nil { return nil, err } - var namespaces []string + var files []string for _, info := range finfos { - if !info.IsDir() { + if filter == directoriesOnly && !info.IsDir() { + continue + } + + if filter == filesOnly && info.IsDir() { continue } - // Filter out hidden directories like `.git`. + if strings.HasPrefix(info.Name(), ".") { continue } - namespaces = append(namespaces, info.Name()) + files = append(files, info.Name()) } - return namespaces, nil -} - -type parserFunc func(io.Reader) ([]database.Vulnerability, error) - -var parsers = map[string]parserFunc{ - "v3.3": parse33YAML, - "v3.4": parse34YAML, + return files, nil } func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.Vulnerability, note string, err error) { - var file io.ReadCloser - file, err = os.Open(repositoryPath + "/" + namespace + "/main.yaml") + nsDir := filepath.Join(repositoryPath, namespace) + var dbFilenames []string + dbFilenames, err = ls(nsDir, filesOnly) if err != nil { return } - defer file.Close() - parseFunc, exists := parsers[namespace] - if !exists { - note = fmt.Sprintf("The file %s is not mapped to any Alpine version number", namespace) - return + for _, filename := range dbFilenames { + var file io.ReadCloser + file, err = os.Open(filepath.Join(nsDir, filename)) + if err != nil { + return + } + + var fileVulns []database.Vulnerability + fileVulns, err = parseYAML(file) + if err != nil { + return + } + + vulns = append(vulns, fileVulns...) + file.Close() } - vulns, err = parseFunc(file) return } @@ -193,61 +210,7 @@ func (u *updater) pullRepository() (commit string, err error) { return } -type secdb33File struct { - Distro string `yaml:"distroversion"` - Packages []struct { - Pkg struct { - Name string `yaml:"name"` - Version string `yaml:"ver"` - Fixes []string `yaml:"fixes"` - } `yaml:"pkg"` - } `yaml:"packages"` -} - -func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) { - var rBytes []byte - rBytes, err = ioutil.ReadAll(r) - if err != nil { - return - } - - var file secdb33File - err = yaml.Unmarshal(rBytes, &file) - if err != nil { - return - } - for _, pack := range file.Packages { - pkg := pack.Pkg - for _, fix := range pkg.Fixes { - err = versionfmt.Valid(dpkg.ParserName, pkg.Version) - if err != nil { - log.Warningf("could not parse package version '%s': %s. skipping", pkg.Version, err.Error()) - continue - } - - vulns = append(vulns, database.Vulnerability{ - Name: fix, - Severity: database.UnknownSeverity, - Link: nvdURLPrefix + fix, - FixedIn: []database.FeatureVersion{ - { - Feature: database.Feature{ - Namespace: database.Namespace{ - Name: "alpine:" + file.Distro, - VersionFormat: dpkg.ParserName, - }, - Name: pkg.Name, - }, - Version: pkg.Version, - }, - }, - }) - } - } - return -} - -type secdb34File struct { +type secDBFile struct { Distro string `yaml:"distroversion"` Packages []struct { Pkg struct { @@ -257,14 +220,14 @@ type secdb34File struct { } `yaml:"packages"` } -func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) { +func parseYAML(r io.Reader) (vulns []database.Vulnerability, err error) { var rBytes []byte rBytes, err = ioutil.ReadAll(r) if err != nil { return } - var file secdb34File + var file secDBFile err = yaml.Unmarshal(rBytes, &file) if err != nil { return diff --git a/ext/vulnsrc/alpine/alpine_test.go b/ext/vulnsrc/alpine/alpine_test.go index 4588d97e..ac95f5c5 100644 --- a/ext/vulnsrc/alpine/alpine_test.go +++ b/ext/vulnsrc/alpine/alpine_test.go @@ -23,32 +23,14 @@ import ( "github.com/stretchr/testify/assert" ) -func TestAlpine33YAMLParsing(t *testing.T) { - _, filename, _, _ := runtime.Caller(0) - path := filepath.Join(filepath.Dir(filename)) - - testData, _ := os.Open(path + "/testdata/v33_main.yaml") - defer testData.Close() - - vulns, err := parse33YAML(testData) - if err != nil { - assert.Nil(t, err) - } - assert.Equal(t, 15, len(vulns)) - assert.Equal(t, "CVE-2016-2147", vulns[0].Name) - assert.Equal(t, "alpine:v3.3", vulns[0].FixedIn[0].Feature.Namespace.Name) - assert.Equal(t, "busybox", vulns[0].FixedIn[0].Feature.Name) - assert.Equal(t, "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-2147", vulns[0].Link) -} - -func TestAlpine34YAMLParsing(t *testing.T) { +func TestYAMLParsing(t *testing.T) { _, filename, _, _ := runtime.Caller(0) path := filepath.Join(filepath.Dir(filename)) testData, _ := os.Open(path + "/testdata/v34_main.yaml") defer testData.Close() - vulns, err := parse34YAML(testData) + vulns, err := parseYAML(testData) if err != nil { assert.Nil(t, err) } diff --git a/ext/vulnsrc/alpine/testdata/v33_main.yaml b/ext/vulnsrc/alpine/testdata/v33_main.yaml deleted file mode 100644 index 21f24668..00000000 --- a/ext/vulnsrc/alpine/testdata/v33_main.yaml +++ /dev/null @@ -1,69 +0,0 @@ ---- -distroversion: v3.3 -reponame: main -archs: - - x86_64 - - x86 - - armhf -urlprefix: http://dl-cdn.alpinelinux.org/alpine -apkurl: "{{urlprefix}}/{{distroversion}}/{{reponame}}/{{arch}}/{{pkg.name}}-${{pkg.ver}}.apk" -packages: - - pkg: - name: busybox - ver: 1.24.2-r0 - fixes: - - CVE-2016-2147 - - CVE-2016-2148 - - pkg: - name: expat - ver: 2.1.1-r1 - fixes: - - CVE-2016-0718 - - pkg: - name: gd - ver: 2.1.1-r1 - fixes: - - CVE-2016-3074 - - pkg: - name: giflib - ver: 5.1.1-r1 - fixes: - - CVE-2016-3977 - - pkg: - name: jq - ver: 1.5-r1 - fixes: - - CVE-2015-8863 - - pkg: - name: libarchive - ver: 3.1.2-r3 - fixes: - - CVE-2016-1541 - - pkg: - name: libssh2 - ver: 1.6.0-r1 - fixes: - - CVE-2016-0787 - - pkg: - name: mercurial - ver: 3.7.3-r1 - fixes: - - CVE-2016-3105 - - pkg: - name: openssl - ver: 1.0.2h-r1 - fixes: - - CVE-2016-2177 - - CVE-2016-2178 - - pkg: - name: pcre - ver: 8.38-r1 - fixes: - - CVE-2016-1283 - - CVE-2016-3191 - - pkg: - name: wpa_supplicant - ver: 2.5-r2 - fixes: - - CVE-2016-4476 - - CVE-2016-4477 From 1e9f14ae33963d5dea1ec5217ba9069934e2e655 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 7 Feb 2017 11:51:10 -0800 Subject: [PATCH 2/3] versionfmt/dpkg: remove leading digit requirement This is not strictly a requirement and affects some tracked Alpine Linux packages. --- ext/versionfmt/dpkg/parser.go | 4 ---- ext/versionfmt/dpkg/parser_test.go | 6 ++++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ext/versionfmt/dpkg/parser.go b/ext/versionfmt/dpkg/parser.go index 42fbd45e..2d6eefbc 100644 --- a/ext/versionfmt/dpkg/parser.go +++ b/ext/versionfmt/dpkg/parser.go @@ -96,10 +96,6 @@ func newVersion(str string) (version, error) { return version{}, errors.New("No version") } - if !unicode.IsDigit(rune(v.version[0])) { - return version{}, errors.New("version does not start with digit") - } - for i := 0; i < len(v.version); i = i + 1 { r := rune(v.version[i]) if !unicode.IsDigit(r) && !unicode.IsLetter(r) && !containsRune(versionAllowedSymbols, r) { diff --git a/ext/versionfmt/dpkg/parser_test.go b/ext/versionfmt/dpkg/parser_test.go index e4897211..36e40ff8 100644 --- a/ext/versionfmt/dpkg/parser_test.go +++ b/ext/versionfmt/dpkg/parser_test.go @@ -70,8 +70,10 @@ func TestParse(t *testing.T) { // Test invalid characters in epoch {"a:0-0", version{}, true}, {"A:0-0", version{}, true}, - // Test version not starting with a digit - {"0:abc3-0", version{}, true}, + // Test version not starting with a digit. + // While recommended by the specification, this is not strictly required and + // at least one vulnerable Alpine package deviates from this scheme. + {"0:abc3-0", version{epoch: 0, version: "abc3", revision: "0"}, false}, } for _, c := range cases { v, err := newVersion(c.str) From 8d10d93b177490139ec1f9ce417a9a8acfb3b1b6 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 7 Feb 2017 12:03:22 -0800 Subject: [PATCH 3/3] example config: add localhost postgres --- config.example.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.example.yaml b/config.example.yaml index a4714b63..423068a3 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -20,7 +20,7 @@ clair: options: # PostgreSQL Connection string # https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING - source: + source: host=localhost port=5432 user=postgres sslmode=disable statement_timeout=60000 # Number of elements kept in the cache # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.