vulnsrc: Refactor vulnerability sources to use utility functions

ext/vulnsrc/alpine/alpine.go

@@ -18,10 +18,8 @@ package alpine
 
 import (
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
@@ -30,10 +28,13 @@ import (
 	"github.com/coreos/clair/ext/versionfmt"
 	"github.com/coreos/clair/ext/versionfmt/dpkg"
 	"github.com/coreos/clair/ext/vulnsrc"
+	"github.com/coreos/clair/pkg/fsutil"
 	"github.com/coreos/clair/pkg/gitutil"
 )
 
 const (
+	// This Alpine vulnerability database affects origin packages, which has
+	// `origin` field of itself.
 	secdbGitURL  = "https://github.com/alpinelinux/alpine-secdb"
 	updaterFlag  = "alpine-secdbUpdater"
 	nvdURLPrefix = "https://cve.mitre.org/cgi-bin/cvename.cgi?name="
@@ -51,61 +52,44 @@ type updater struct {
 }
 
 func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
-	log.WithField("package", "Alpine").Info("Start fetching vulnerabilities")
+	log.WithField("package", "Alpine").Info("start fetching vulnerabilities")
-
 	// Pull the master branch.
-	var commit string
+	var (
-	u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(secdbGitURL, u.repositoryLocalPath, updaterFlag)
+		commit         string
-	if err != nil {
+		existingCommit string
-		return
+		foundCommit    bool
-	}
+		namespaces     []string
-
+		vulns          []database.VulnerabilityWithAffected
-	// Open a database transaction.
+	)
-	tx, err := db.Begin()
-	if err != nil {
-		return
-	}
-	defer tx.Rollback()
 
-	// Ask the database for the latest commit we successfully applied.
+	if u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(secdbGitURL, u.repositoryLocalPath, updaterFlag); err != nil {
-	var dbCommit string
-	var ok bool
-	dbCommit, ok, err = tx.FindKeyValue(updaterFlag)
-	if err != nil {
 		return
 	}
-	if !ok {
-		dbCommit = ""
-	}
 
 	// Set the updaterFlag to equal the commit processed.
 	resp.FlagName = updaterFlag
 	resp.FlagValue = commit
+	if existingCommit, foundCommit, err = database.FindKeyValueAndRollback(db, updaterFlag); err != nil {
+		return
+	}
 
 	// Short-circuit if there have been no updates.
-	if commit == dbCommit {
+	if foundCommit && commit == existingCommit {
-		log.WithField("package", "alpine").Debug("no update")
+		log.WithField("package", "alpine").Debug("no update, skip")
 		return
 	}
 
 	// Get the list of namespaces from the repository.
-	var namespaces []string
+	if namespaces, err = fsutil.Readdir(u.repositoryLocalPath, fsutil.DirectoriesOnly); err != nil {
-	namespaces, err = ls(u.repositoryLocalPath, directoriesOnly)
-	if err != nil {
 		return
 	}
 
 	// Append any changed vulnerabilities to the response.
 	for _, namespace := range namespaces {
-		var vulns []database.VulnerabilityWithAffected
+		if vulns, err = parseVulnsFromNamespace(u.repositoryLocalPath, namespace); err != nil {
-		var note string
-		vulns, note, err = parseVulnsFromNamespace(u.repositoryLocalPath, namespace)
-		if err != nil {
 			return
 		}
-		if note != "" {
+
-			resp.Notes = append(resp.Notes, note)
-		}
 		resp.Vulnerabilities = append(resp.Vulnerabilities, vulns...)
 	}
 
@@ -118,74 +102,26 @@ func (u *updater) Clean() {
 	}
 }
 
-type lsFilter int
+func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.VulnerabilityWithAffected, err error) {
-
-const (
-	filesOnly lsFilter = iota
-	directoriesOnly
-)
-
-func ls(path string, filter lsFilter) ([]string, error) {
-	dir, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer dir.Close()
-
-	finfos, err := dir.Readdir(0)
-	if err != nil {
-		return nil, err
-	}
-
-	var files []string
-	for _, info := range finfos {
-		if filter == directoriesOnly && !info.IsDir() {
-			continue
-		}
-
-		if filter == filesOnly && info.IsDir() {
-			continue
-		}
-
-		if strings.HasPrefix(info.Name(), ".") {
-			continue
-		}
-
-		files = append(files, info.Name())
-	}
-
-	return files, nil
-}
-
-func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.VulnerabilityWithAffected, note string, err error) {
 	nsDir := filepath.Join(repositoryPath, namespace)
 	var dbFilenames []string
-	dbFilenames, err = ls(nsDir, filesOnly)
+	if dbFilenames, err = fsutil.Readdir(nsDir, fsutil.FilesOnly); err != nil {
-	if err != nil {
 		return
 	}
 
 	for _, filename := range dbFilenames {
-		var file io.ReadCloser
+		var db *secDB
-		file, err = os.Open(filepath.Join(nsDir, filename))
+		if db, err = newSecDB(filepath.Join(nsDir, filename)); err != nil {
-		if err != nil {
 			return
 		}
 
-		var fileVulns []database.VulnerabilityWithAffected
+		vulns = append(vulns, db.Vulnerabilities()...)
-		fileVulns, err = parseYAML(file)
-		if err != nil {
-			return
-		}
-
-		vulns = append(vulns, fileVulns...)
-		file.Close()
 	}
 
 	return
 }
 
-type secDBFile struct {
+type secDB struct {
 	Distro   string `yaml:"distroversion"`
 	Packages []struct {
 		Pkg struct {
@@ -195,42 +131,54 @@ type secDBFile struct {
 	} `yaml:"packages"`
 }
 
-func parseYAML(r io.Reader) (vulns []database.VulnerabilityWithAffected, err error) {
+func newSecDB(filePath string) (file *secDB, err error) {
-	var rBytes []byte
+	var f io.ReadCloser
-	rBytes, err = ioutil.ReadAll(r)
+	f, err = os.Open(filePath)
 	if err != nil {
 		return
 	}
 
-	var file secDBFile
+	defer f.Close()
-	err = yaml.Unmarshal(rBytes, &file)
+	file = &secDB{}
-	if err != nil {
+	err = yaml.NewDecoder(f).Decode(file)
 	return
 }
 
-	for _, pack := range file.Packages {
+func (file *secDB) Vulnerabilities() (vulns []database.VulnerabilityWithAffected) {
-		pkg := pack.Pkg
+	if file == nil {
-		for version, vulnStrs := range pkg.Fixes {
+		return
-			err := versionfmt.Valid(dpkg.ParserName, version)
+	}
-			if err != nil {
+
-				log.WithError(err).WithField("version", version).Warning("could not parse package version. skipping")
+	namespace := database.Namespace{Name: "alpine:" + file.Distro, VersionFormat: dpkg.ParserName}
+	for _, pkg := range file.Packages {
+		for version, cveNames := range pkg.Pkg.Fixes {
+			if err := versionfmt.Valid(dpkg.ParserName, version); err != nil {
+				log.WithError(err).WithFields(log.Fields{
+					"version":      version,
+					"package name": pkg.Pkg.Name,
+				}).Warning("could not parse package version, skipping")
 				continue
 			}
 
-			for _, vulnStr := range vulnStrs {
+			for _, cve := range cveNames {
-				var vuln database.VulnerabilityWithAffected
+				vuln := database.VulnerabilityWithAffected{
-				vuln.Severity = database.UnknownSeverity
+					Vulnerability: database.Vulnerability{
-				vuln.Name = vulnStr
+						Name:      cve,
-				vuln.Link = nvdURLPrefix + vulnStr
+						Link:      nvdURLPrefix + cve,
+						Severity:  database.UnknownSeverity,
+						Namespace: namespace,
+					},
+				}
 
 				var fixedInVersion string
 				if version != versionfmt.MaxVersion {
 					fixedInVersion = version
 				}
+
 				vuln.Affected = []database.AffectedFeature{
 					{
 						AffectedType:    affectedType,
-						FeatureName:     pkg.Name,
+						FeatureName:     pkg.Pkg.Name,
 						AffectedVersion: version,
 						FixedInVersion:  fixedInVersion,
 						Namespace: database.Namespace{

ext/vulnsrc/alpine/alpine_test.go

@@ -15,28 +15,24 @@
 package alpine
 
 import (
-	"os"
 	"path/filepath"
 	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestYAMLParsing(t *testing.T) {
 	_, filename, _, _ := runtime.Caller(0)
 	path := filepath.Join(filepath.Dir(filename))
+	secdb, err := newSecDB(filepath.Join(path, "/testdata/v34_main.yaml"))
+	require.Nil(t, err)
+	vulns := secdb.Vulnerabilities()
 
-	testData, _ := os.Open(path + "/testdata/v34_main.yaml")
-	defer testData.Close()
-
-	vulns, err := parseYAML(testData)
-	if err != nil {
-		assert.Nil(t, err)
-	}
 	assert.Equal(t, 105, len(vulns))
 	assert.Equal(t, "CVE-2016-5387", vulns[0].Name)
-	assert.Equal(t, "alpine:v3.4", vulns[0].Affected[0].Namespace.Name)
+	assert.Equal(t, "alpine:v3.4", vulns[0].Namespace.Name)
 	assert.Equal(t, "apache2", vulns[0].Affected[0].FeatureName)
 	assert.Equal(t, "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-5387", vulns[0].Link)
 }

ext/vulnsrc/debian/debian.go

@@ -62,22 +62,9 @@ func init() {
 
 func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
 	log.WithField("package", "Debian").Info("Start fetching vulnerabilities")
-
+	latestHash, ok, err := database.FindKeyValueAndRollback(datastore, updaterFlag)
-	tx, err := datastore.Begin()
-	if err != nil {
-		return resp, err
-	}
-
-	// Get the hash of the latest update's JSON data
-	latestHash, ok, err := tx.FindKeyValue(updaterFlag)
 	if err != nil {
-		return resp, err
+		return
-	}
-
-	// NOTE(sida): The transaction won't mutate the database and I want the
-	// transaction to be short.
-	if err := tx.Rollback(); err != nil {
-		return resp, err
 	}
 
 	if !ok {
@@ -136,7 +123,7 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp vulnsrc.U
 	// Calculate the hash and skip updating if the hash has been seen before.
 	hash = hex.EncodeToString(jsonSHA.Sum(nil))
 	if latestKnownHash == hash {
-		log.WithField("package", "Debian").Debug("no update")
+		log.WithField("package", "Debian").Debug("no update, skip")
 		return resp, nil
 	}
 

ext/vulnsrc/debian/debian_test.go

@@ -30,7 +30,7 @@ func TestDebianParser(t *testing.T) {
 	_, filename, _, _ := runtime.Caller(0)
 
 	// Test parsing testdata/fetcher_debian_test.json
-	testFile, _ := os.Open(filepath.Join(filepath.Dir(filename)) + "/testdata/fetcher_debian_test.json")
+	testFile, _ := os.Open(filepath.Join(filepath.Dir(filename), "/testdata/fetcher_debian_test.json"))
 	response, err := buildResponse(testFile, "")
 	if assert.Nil(t, err) && assert.Len(t, response.Vulnerabilities, 2) {
 		for _, vulnerability := range response.Vulnerabilities {

ext/vulnsrc/oracle/oracle.go

@@ -119,15 +119,9 @@ func compareELSA(left, right int) int {
 func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
 	log.WithField("package", "Oracle Linux").Info("Start fetching vulnerabilities")
 	// Get the first ELSA we have to manage.
-	tx, err := datastore.Begin()
+	flagValue, ok, err := database.FindKeyValueAndRollback(datastore, updaterFlag)
 	if err != nil {
-		return resp, err
+		return
-	}
-	defer tx.Rollback()
-
-	flagValue, ok, err := tx.FindKeyValue(updaterFlag)
-	if err != nil {
-		return resp, err
 	}
 
 	if !ok {

ext/vulnsrc/oracle/oracle_test.go

@@ -30,7 +30,7 @@ func TestOracleParser(t *testing.T) {
 	path := filepath.Join(filepath.Dir(filename))
 
 	// Test parsing testdata/fetcher_oracle_test.1.xml
-	testFile, _ := os.Open(path + "/testdata/fetcher_oracle_test.1.xml")
+	testFile, _ := os.Open(filepath.Join(path, "/testdata/fetcher_oracle_test.1.xml"))
 	defer testFile.Close()
 
 	vulnerabilities, err := parseELSA(testFile)
@@ -78,7 +78,7 @@ func TestOracleParser(t *testing.T) {
 		}
 	}
 
-	testFile2, _ := os.Open(path + "/testdata/fetcher_oracle_test.2.xml")
+	testFile2, _ := os.Open(filepath.Join(path, "/testdata/fetcher_oracle_test.2.xml"))
 	defer testFile2.Close()
 
 	vulnerabilities, err = parseELSA(testFile2)

ext/vulnsrc/rhel/rhel.go

@@ -101,21 +101,12 @@ func init() {
 func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
 	log.WithField("package", "RHEL").Info("Start fetching vulnerabilities")
 
-	tx, err := datastore.Begin()
-	if err != nil {
-		return resp, err
-	}
-
 	// Get the first RHSA we have to manage.
-	flagValue, ok, err := tx.FindKeyValue(updaterFlag)
+	flagValue, ok, err := database.FindKeyValueAndRollback(datastore, updaterFlag)
 	if err != nil {
 		return resp, err
 	}
 
-	if err := tx.Rollback(); err != nil {
-		return resp, err
-	}
-
 	if !ok {
 		flagValue = ""
 	}

ext/vulnsrc/rhel/rhel_test.go

@@ -31,7 +31,7 @@ func TestRHELParserMultipleCVE(t *testing.T) {
 	path := filepath.Join(filepath.Dir(filename))
 
 	// Test parsing testdata/fetcher_rhel_test.2.xml
-	testFile, _ := os.Open(path + "/testdata/fetcher_rhel_test.2.xml")
+	testFile, _ := os.Open(filepath.Join(path, "/testdata/fetcher_rhel_test.2.xml"))
 	vulnerabilities, err := parseRHSA(testFile)
 
 	// Expected
@@ -86,7 +86,7 @@ func TestRHELParserOneCVE(t *testing.T) {
 	path := filepath.Join(filepath.Dir(filename))
 
 	// Test parsing testdata/fetcher_rhel_test.1.xml
-	testFile, _ := os.Open(path + "/testdata/fetcher_rhel_test.1.xml")
+	testFile, _ := os.Open(filepath.Join(path, "/testdata/fetcher_rhel_test.1.xml"))
 	vulnerabilities, err := parseRHSA(testFile)
 	if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) {
 		assert.Equal(t, "CVE-2015-0252", vulnerabilities[0].Name)

ext/vulnsrc/ubuntu/ubuntu.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 
@@ -101,12 +102,11 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er
 	defer tx.Rollback()
 
 	// Ask the database for the latest commit we successfully applied.
-	var dbCommit string
+	dbCommit, ok, err := database.FindKeyValueAndRollback(db, updaterFlag)
-	var ok bool
-	dbCommit, ok, err = tx.FindKeyValue(updaterFlag)
 	if err != nil {
 		return
 	}
+
 	if !ok {
 		dbCommit = ""
 	}
@@ -161,7 +161,7 @@ func collectModifiedVulnerabilities(commit, dbCommit, repositoryLocalPath string
 
 func processDirectory(repositoryLocalPath, dirName string, modifiedCVE map[string]struct{}) error {
 	// Open the directory.
-	d, err := os.Open(repositoryLocalPath + "/" + dirName)
+	d, err := os.Open(filepath.Join(repositoryLocalPath, dirName))
 	if err != nil {
 		log.WithError(err).Error("could not open Ubuntu vulnerabilities repository's folder")
 		return vulnsrc.ErrFilesystem
@@ -191,7 +191,7 @@ func collectVulnerabilitiesAndNotes(repositoryLocalPath string, modifiedCVE map[
 
 	for cvePath := range modifiedCVE {
 		// Open the CVE file.
-		file, err := os.Open(repositoryLocalPath + "/" + cvePath)
+		file, err := os.Open(filepath.Join(repositoryLocalPath, cvePath))
 		if err != nil {
 			// This can happen when a file is modified then moved in another commit.
 			continue

ext/vulnsrc/ubuntu/ubuntu_test.go

@@ -32,7 +32,7 @@ func TestUbuntuParser(t *testing.T) {
 	path := filepath.Join(filepath.Dir(filename))
 
 	// Test parsing testdata/fetcher_
-	testData, _ := os.Open(path + "/testdata/fetcher_ubuntu_test.txt")
+	testData, _ := os.Open(filepath.Join(path, "/testdata/fetcher_ubuntu_test.txt"))
 	defer testData.Close()
 	vulnerability, unknownReleases, err := parseUbuntuCVE(testData)
 	if assert.Nil(t, err) {