Merge pull request #530 from meringu/patch-1

Proxy for 2.0

Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.8-alpine
+FROM golang:1.10-alpine
 
 VOLUME /config
 EXPOSE 6060 6061
@@ -20,7 +20,7 @@ EXPOSE 6060 6061
 ADD .   /go/src/github.com/coreos/clair/
 WORKDIR /go/src/github.com/coreos/clair/
 
-RUN apk add --no-cache git bzr rpm xz && \
+RUN apk add --no-cache git rpm xz && \
     go install -v github.com/coreos/clair/cmd/clair && \
     mv /go/bin/clair /clair && \
     rm -rf /go /usr/local/go

README.md

@@ -119,14 +119,12 @@ To build Clair, you need to latest stable version of [Go] and a working [Go envi
 In addition, Clair requires some additional binaries be installed on the system [$PATH] as runtime dependencies:
 
 * [git]
-* [bzr]
 * [rpm]
 * [xz]
 
 [Go]: https://github.com/golang/go/releases
 [Go environment]: https://golang.org/doc/code.html
 [git]: https://git-scm.com
-[bzr]: http://bazaar.canonical.com/en
 [rpm]: http://www.rpm.org
 [xz]: http://tukaani.org/xz
 [$PATH]: https://en.wikipedia.org/wiki/PATH_(variable)

cmd/clair/main.go

@@ -127,7 +127,7 @@ func main() {
 	flag.Parse()
 
 	// Check for dependencies.
-	for _, bin := range []string{"git", "bzr", "rpm", "xz"} {
+	for _, bin := range []string{"git", "rpm", "xz"} {
 		_, err := exec.LookPath(bin)
 		if err != nil {
 			log.WithError(err).WithField("dependency", bin).Fatal("failed to find dependency")

contrib/k8s/config.yaml

@@ -15,13 +15,16 @@
 # The values specified here are the default values that Clair uses if no configuration file is specified or if the keys are not defined.
 clair:
   database:
-    # PostgreSQL Connection string
-    # http://www.postgresql.org/docs/9.4/static/libpq-connect.html
-    source: postgres://postgres:password@postgres:5432/postgres?sslmode=disable
+    # Database driver
+    type: pgsql
+    options:
+      # PostgreSQL Connection string
+      # https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
+      source: postgres://postgres:password@postgres:5432/postgres?sslmode=disable
 
-    # Number of elements kept in the cache
-    # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.
-    cacheSize: 16384
+      # Number of elements kept in the cache
+      # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.
+      cachesize: 16384
 
   api:
     # API server port
@@ -37,7 +40,7 @@ clair:
     # 32-bit URL-safe base64 key used to encrypt pagination tokens
     # If one is not provided, it will be generated.
     # Multiple clair instances in the same cluster need the same value.
-    paginationKey:
+    paginationkey:
 
     # Optional PKI configuration
     # If you want to easily generate client certificates and CAs, try the following projects:
@@ -58,7 +61,7 @@ clair:
     attempts: 3
 
     # Duration before a failed notification is retried
-    renotifyInterval: 2h
+    renotifyinterval: 2h
 
     http:
       # Optional endpoint that will receive notifications via POST requests
@@ -72,3 +75,6 @@ clair:
       cafile:
       keyfile:
       certfile:
+
+      # Optional HTTP Proxy: must be a valid URL (including the scheme).
+      proxy:

database/namespace_mapping.go

@@ -21,13 +21,15 @@ var DebianReleasesMapping = map[string]string{
 	"wheezy":  "7",
 	"jessie":  "8",
 	"stretch": "9",
+	"buster":  "10",
 	"sid":     "unstable",
 
 	// Class names
-	"oldstable": "7",
-	"stable":    "8",
-	"testing":   "9",
-	"unstable":  "unstable",
+	"oldoldstable": "7",
+	"oldstable":    "8",
+	"stable":       "9",
+	"testing":      "10",
+	"unstable":     "unstable",
 }
 
 // UbuntuReleasesMapping translates Ubuntu code names to version numbers
@@ -43,4 +45,5 @@ var UbuntuReleasesMapping = map[string]string{
 	"yakkety": "16.10",
 	"zesty":   "17.04",
 	"artful":  "17.10",
+	"bionic":  "18.04",
 }

ext/featurens/aptsources/aptsources.go

@@ -68,6 +68,8 @@ func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) {
 				OS = "debian"
 				break
 			}
+
+			line[2] = strings.Split(line[2], "/")[0]
 			version, found = database.UbuntuReleasesMapping[line[2]]
 			if found {
 				OS = "ubuntu"

ext/featurens/redhatrelease/redhatrelease.go

@@ -30,7 +30,7 @@ import (
 )
 
 var (
-	oracleReleaseRegexp = regexp.MustCompile(`(?P<os>[^\s]*) (Linux Server release) (?P<version>[\d]+)`)
+	oracleReleaseRegexp = regexp.MustCompile(`(?P<os>Oracle) (Linux Server release) (?P<version>[\d]+)`)
 	centosReleaseRegexp = regexp.MustCompile(`(?P<os>[^\s]*) (Linux release|release) (?P<version>[\d]+)`)
 	redhatReleaseRegexp = regexp.MustCompile(`(?P<os>Red Hat Enterprise Linux) (Client release|Server release|Workstation release) (?P<version>[\d]+)`)
 )

ext/featurens/redhatrelease/redhatrelease_test.go

@@ -42,6 +42,12 @@ func TestDetector(t *testing.T) {
 				"etc/centos-release": []byte(`CentOS release 6.6 (Final)`),
 			},
 		},
+		{
+			ExpectedNamespace: &database.Namespace{Name: "centos:7"},
+			Files: tarutil.FilesMap{
+				"etc/redhat-release": []byte(`Red Hat Enterprise Linux Server release 7.2 (Maipo)`),
+			},
+		},
 		{
 			ExpectedNamespace: &database.Namespace{Name: "centos:7"},
 			Files: tarutil.FilesMap{

ext/imagefmt/driver.go

@@ -122,6 +122,7 @@ func Extract(format, path string, headers map[string]string, toExtract []string)
 		// Send the request and handle the response.
 		tr := &http.Transport{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureTLS},
+			Proxy:           http.ProxyFromEnvironment,
 		}
 		client := &http.Client{Transport: tr}
 		r, err := client.Do(request)

ext/vulnmdsrc/nvd/nvd.go

@@ -38,8 +38,8 @@ import (
 )
 
 const (
-	dataFeedURL     string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz"
-	dataFeedMetaURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.meta"
+	dataFeedURL     string = "https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz"
+	dataFeedMetaURL string = "https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.meta"
 
 	appenderName string = "NVD"
 

ext/vulnsrc/ubuntu/ubuntu.go

@@ -18,14 +18,12 @@ package ubuntu
 
 import (
 	"bufio"
-	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"regexp"
-	"strconv"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
@@ -38,10 +36,9 @@ import (
 )
 
 const (
-	trackerURI        = "https://launchpad.net/ubuntu-cve-tracker"
-	trackerRepository = "https://launchpad.net/ubuntu-cve-tracker"
-	updaterFlag       = "ubuntuUpdater"
-	cveURL            = "http://people.ubuntu.com/~ubuntu-security/cve/%s"
+	trackerGitURL = "https://git.launchpad.net/ubuntu-cve-tracker"
+	updaterFlag   = "ubuntuUpdater"
+	cveURL        = "http://people.ubuntu.com/~ubuntu-security/cve/%s"
 )
 
 var (
@@ -87,25 +84,27 @@ func init() {
 func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
 	log.WithField("package", "Ubuntu").Info("Start fetching vulnerabilities")
 
-	// Pull the bzr repository.
-	if err = u.pullRepository(); err != nil {
-		return resp, err
-	}
-
-	// Get revision number.
-	revisionNumber, err := getRevisionNumber(u.repositoryLocalPath)
+	// Pull the master branch.
+	var commit string
+	commit, err = u.pullRepository()
 	if err != nil {
-		return resp, err
+		return
 	}
 
 	// Get the latest revision number we successfully applied in the database.
-	dbRevisionNumber, err := datastore.GetKeyValue("ubuntuUpdater")
+	dbCommit, err := datastore.GetKeyValue(updaterFlag)
 	if err != nil {
 		return resp, err
 	}
 
+	// Short-circuit if there have been no updates.
+	if commit == dbCommit {
+		log.WithField("package", "ubuntu").Debug("no update")
+		return
+	}
+
 	// Get the list of vulnerabilities that we have to update.
-	modifiedCVE, err := collectModifiedVulnerabilities(revisionNumber, dbRevisionNumber, u.repositoryLocalPath)
+	modifiedCVE, err := collectModifiedVulnerabilities(u.repositoryLocalPath)
 	if err != nil {
 		return resp, err
 	}
@@ -136,8 +135,7 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo
 
 			// If we encountered unknown Ubuntu release, we don't want the revision
 			// number to be considered as managed.
-			dbRevisionNumberInt, _ := strconv.Atoi(dbRevisionNumber)
-			revisionNumber = dbRevisionNumberInt
+			commit = dbCommit
 		}
 
 		// Close the file manually.
@@ -150,7 +148,7 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo
 
 	// Add flag and notes.
 	resp.FlagName = updaterFlag
-	resp.FlagValue = strconv.Itoa(revisionNumber)
+	resp.FlagValue = commit
 	for note := range notes {
 		resp.Notes = append(resp.Notes, note)
 	}
@@ -162,120 +160,76 @@ func (u *updater) Clean() {
 	os.RemoveAll(u.repositoryLocalPath)
 }
 
-func (u *updater) pullRepository() (err error) {
-	// Determine whether we should branch or pull.
+func (u *updater) pullRepository() (commit string, err error) {
+	// If the repository doesn't exist, clone it.
 	if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
-		// Create a temporary folder to store the repository.
 		if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil {
-			return vulnsrc.ErrFilesystem
+			return "", vulnsrc.ErrFilesystem
 		}
 
-		// Branch repository.
-		cmd := exec.Command("bzr", "branch", "--use-existing-dir", trackerRepository, ".")
+		cmd := exec.Command("git", "clone", trackerGitURL, ".")
 		cmd.Dir = u.repositoryLocalPath
 		if out, err := cmd.CombinedOutput(); err != nil {
-			log.WithError(err).WithField("output", string(out)).Error("could not branch Ubuntu repository")
-			return commonerr.ErrCouldNotDownload
+			u.Clean()
+			log.WithError(err).WithField("output", string(out)).Error("could not pull ubuntu-cve-tracker repository")
+			return "", commonerr.ErrCouldNotDownload
+		}
+	} else {
+		// The repository already exists and it needs to be refreshed via a pull.
+		cmd := exec.Command("git", "pull")
+		cmd.Dir = u.repositoryLocalPath
+		if _, err := cmd.CombinedOutput(); err != nil {
+			return "", vulnsrc.ErrGitFailure
 		}
-
-		return nil
 	}
 
-	// Pull repository.
-	cmd := exec.Command("bzr", "pull", "--overwrite")
+	cmd := exec.Command("git", "rev-parse", "HEAD")
 	cmd.Dir = u.repositoryLocalPath
-	if out, err := cmd.CombinedOutput(); err != nil {
-		os.RemoveAll(u.repositoryLocalPath)
-		log.WithError(err).WithField("output", string(out)).Error("could not pull Ubuntu repository")
-		return commonerr.ErrCouldNotDownload
-	}
-
-	return nil
-}
-
-func getRevisionNumber(pathToRepo string) (int, error) {
-	cmd := exec.Command("bzr", "revno")
-	cmd.Dir = pathToRepo
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		log.WithError(err).WithField("output", string(out)).Error("could not get Ubuntu repository's revision number")
-		return 0, commonerr.ErrCouldNotDownload
+		return "", vulnsrc.ErrGitFailure
 	}
 
-	revno, err := strconv.Atoi(strings.TrimSpace(string(out)))
-	if err != nil {
-		log.WithError(err).WithField("output", string(out)).Error("could not parse Ubuntu repository's revision number")
-		return 0, commonerr.ErrCouldNotDownload
-	}
-
-	return revno, nil
+	commit = strings.TrimSpace(string(out))
+	return
 }
 
-func collectModifiedVulnerabilities(revision int, dbRevision, repositoryLocalPath string) (map[string]struct{}, error) {
+func collectModifiedVulnerabilities(repositoryLocalPath string) (map[string]struct{}, error) {
 	modifiedCVE := make(map[string]struct{})
 
 	// Handle a brand new database.
-	if dbRevision == "" {
-		for _, folder := range []string{"active", "retired"} {
-			d, err := os.Open(repositoryLocalPath + "/" + folder)
-			if err != nil {
-				log.WithError(err).Error("could not open Ubuntu vulnerabilities repository's folder")
-				return nil, vulnsrc.ErrFilesystem
-			}
 
-			// Get the FileInfo of all the files in the directory.
-			names, err := d.Readdirnames(-1)
-			if err != nil {
-				log.WithError(err).Error("could not read Ubuntu vulnerabilities repository's folder")
-				return nil, vulnsrc.ErrFilesystem
-			}
-
-			// Add the vulnerabilities to the list.
-			for _, name := range names {
-				if strings.HasPrefix(name, "CVE-") {
-					modifiedCVE[folder+"/"+name] = struct{}{}
-				}
-			}
-
-			// Close the file manually.
-			//
-			// We do that instead of using defer because defer works on a function-level scope.
-			// We would open many files and close them all at once at the end of the function,
-			// which could lead to exceed fs.file-max.
-			d.Close()
+	for _, folder := range []string{"active", "retired"} {
+		d, err := os.Open(repositoryLocalPath + "/" + folder)
+		if err != nil {
+			log.WithError(err).Error("could not open Ubuntu vulnerabilities repository's folder")
+			return nil, vulnsrc.ErrFilesystem
 		}
 
-		return modifiedCVE, nil
-	}
-
-	// Handle an up to date database.
-	dbRevisionInt, _ := strconv.Atoi(dbRevision)
-	if revision == dbRevisionInt {
-		log.WithField("package", "Ubuntu").Debug("no update")
-		return modifiedCVE, nil
-	}
-
-	// Handle a database that needs upgrading.
-	cmd := exec.Command("bzr", "log", "--verbose", "-r"+strconv.Itoa(dbRevisionInt+1)+"..", "-n0")
-	cmd.Dir = repositoryLocalPath
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		log.WithError(err).WithField("output", string(out)).Error("could not get Ubuntu vulnerabilities repository logs")
-		return nil, commonerr.ErrCouldNotDownload
-	}
-
-	scanner := bufio.NewScanner(bytes.NewReader(out))
-	for scanner.Scan() {
-		text := strings.TrimSpace(scanner.Text())
-		if strings.Contains(text, "CVE-") && (strings.HasPrefix(text, "active/") || strings.HasPrefix(text, "retired/")) {
-			if strings.Contains(text, " => ") {
-				text = text[strings.Index(text, " => ")+4:]
-			}
-			modifiedCVE[text] = struct{}{}
+		// Get the FileInfo of all the files in the directory.
+		names, err := d.Readdirnames(-1)
+		if err != nil {
+			log.WithError(err).Error("could not read Ubuntu vulnerabilities repository's folder")
+			return nil, vulnsrc.ErrFilesystem
 		}
+
+		// Add the vulnerabilities to the list.
+		for _, name := range names {
+			if strings.HasPrefix(name, "CVE-") {
+				modifiedCVE[folder+"/"+name] = struct{}{}
+			}
+		}
+
+		// Close the file manually.
+		//
+		// We do that instead of using defer because defer works on a function-level scope.
+		// We would open many files and close them all at once at the end of the function,
+		// which could lead to exceed fs.file-max.
+		d.Close()
 	}
 
 	return modifiedCVE, nil
+
 }
 
 func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability, unknownReleases map[string]struct{}, err error) {
@@ -344,6 +298,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability
 			// Only consider the package if its status is needed, active, deferred, not-affected or
 			// released. Ignore DNE (package does not exist), needs-triage, ignored, pending.
 			if md["status"] == "needed" || md["status"] == "active" || md["status"] == "deferred" || md["status"] == "released" || md["status"] == "not-affected" {
+				md["release"] = strings.Split(md["release"], "/")[0]
 				if _, isReleaseIgnored := ubuntuIgnoredReleases[md["release"]]; isReleaseIgnored {
 					continue
 				}
@@ -392,7 +347,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability
 
 	// If no link has been provided (CVE-2006-NNN0 for instance), add the link to the tracker
 	if vulnerability.Link == "" {
-		vulnerability.Link = trackerURI
+		vulnerability.Link = trackerGitURL
 	}
 
 	// If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown

updater.go

@@ -31,10 +31,11 @@ import (
 )
 
 const (
-	updaterLastFlagName        = "updater/last"
-	updaterLockName            = "updater"
-	updaterLockDuration        = updaterLockRefreshDuration + time.Minute*2
-	updaterLockRefreshDuration = time.Minute * 8
+	updaterLastFlagName              = "updater/last"
+	updaterLockName                  = "updater"
+	updaterLockDuration              = updaterLockRefreshDuration + time.Minute*2
+	updaterLockRefreshDuration       = time.Minute * 8
+	updaterSleepBetweenLoopsDuration = time.Minute
 )
 
 var (
@@ -124,7 +125,14 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
 				if stop {
 					break
 				}
+
+				// Sleep for a short duration to prevent pinning the CPU on a
+				// consistent failure.
+				if stopped := sleepUpdater(time.Now().Add(updaterSleepBetweenLoopsDuration), st); stopped {
+					break
+				}
 				continue
+
 			} else {
 				lockOwner, lockExpiration, err := datastore.FindLock(updaterLockName)
 				if err != nil {
@@ -137,14 +145,8 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
 			}
 		}
 
-		// Sleep, but remain stoppable until approximately the next update time.
-		now := time.Now().UTC()
-		waitUntil := nextUpdate.Add(time.Duration(rand.ExpFloat64()/0.5) * time.Second)
-		log.WithField("scheduled time", waitUntil).Debug("next update attempt scheduled")
-		if !waitUntil.Before(now) {
-			if !st.Sleep(waitUntil.Sub(time.Now())) {
-				break
-			}
+		if stopped := sleepUpdater(nextUpdate, st); stopped {
+			break
 		}
 	}
 
@@ -159,6 +161,19 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
 	log.Info("updater service stopped")
 }
 
+// sleepUpdater sleeps the updater for an approximate duration, but remains
+// able to be cancelled by a stopper.
+func sleepUpdater(approxWakeup time.Time, st *stopper.Stopper) (stopped bool) {
+	waitUntil := approxWakeup.Add(time.Duration(rand.ExpFloat64()/0.5) * time.Second)
+	log.WithField("scheduled time", waitUntil).Debug("updater sleeping")
+	if !waitUntil.Before(time.Now().UTC()) {
+		if !st.Sleep(waitUntil.Sub(time.Now())) {
+			return true
+		}
+	}
+	return false
+}
+
 // update fetches all the vulnerabilities from the registered fetchers, upserts
 // them into the database and then sends notifications.
 func update(datastore database.Datastore, firstUpdate bool) {