Merge pull request #621 from jzelinskie/gitutil

pkg/gitutil: init

ext/vulnsrc/alpine/alpine.go

@@ -1,4 +1,4 @@
-// Copyright 2017 clair authors
+// Copyright 2018 clair authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,7 +20,6 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -31,7 +30,7 @@ import (
 	"github.com/coreos/clair/ext/versionfmt"
 	"github.com/coreos/clair/ext/versionfmt/dpkg"
 	"github.com/coreos/clair/ext/vulnsrc"
-	"github.com/coreos/clair/pkg/commonerr"
+	"github.com/coreos/clair/pkg/gitutil"
 )
 
 const (
@@ -53,7 +52,7 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er
 
 	// Pull the master branch.
 	var commit string
-	commit, err = u.pullRepository()
+	u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(secdbGitURL, u.repositoryLocalPath, updaterFlag)
 	if err != nil {
 		return
 	}
@@ -183,40 +182,6 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database
 	return
 }
 
-func (u *updater) pullRepository() (commit string, err error) {
-	// If the repository doesn't exist, clone it.
-	if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
-		if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
-			return "", vulnsrc.ErrFilesystem
-		}
-
-		cmd := exec.Command("git", "clone", secdbGitURL, ".")
-		cmd.Dir = u.repositoryLocalPath
-		if out, err := cmd.CombinedOutput(); err != nil {
-			u.Clean()
-			log.WithError(err).WithField("output", string(out)).Error("could not clone alpine-secdb repository")
-			return "", commonerr.ErrCouldNotDownload
-		}
-	} else {
-		// The repository already exists and it needs to be refreshed via a pull.
-		cmd := exec.Command("git", "pull")
-		cmd.Dir = u.repositoryLocalPath
-		if _, err := cmd.CombinedOutput(); err != nil {
-			return "", vulnsrc.ErrGitFailure
-		}
-	}
-
-	cmd := exec.Command("git", "rev-parse", "HEAD")
-	cmd.Dir = u.repositoryLocalPath
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return "", vulnsrc.ErrGitFailure
-	}
-
-	commit = strings.TrimSpace(string(out))
-	return
-}
-
 type secDBFile struct {
 	Distro   string `yaml:"distroversion"`
 	Packages []struct {

ext/vulnsrc/driver.go

@@ -27,9 +27,6 @@ var (
 	// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
 	ErrFilesystem = errors.New("vulnsrc: something went wrong when interacting with the fs")
 
-	// ErrGitFailure is returned when a fetcher fails to interact with git.
-	ErrGitFailure = errors.New("vulnsrc: something went wrong when interacting with git")
-
 	updatersM sync.RWMutex
 	updaters  = make(map[string]Updater)
 )

ext/vulnsrc/ubuntu/ubuntu.go

@@ -1,4 +1,4 @@
-// Copyright 2017 clair authors
+// Copyright 2018 clair authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -21,9 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
-	"os/exec"
 	"regexp"
 	"strings"
 
@@ -33,7 +31,7 @@ import (
 	"github.com/coreos/clair/ext/versionfmt"
 	"github.com/coreos/clair/ext/versionfmt/dpkg"
 	"github.com/coreos/clair/ext/vulnsrc"
-	"github.com/coreos/clair/pkg/commonerr"
+	"github.com/coreos/clair/pkg/gitutil"
 )
 
 const (
@@ -89,7 +87,7 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er
 
 	// Pull the master branch.
 	var commit string
-	commit, err = u.pullRepository()
+	u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(trackerURI, u.repositoryLocalPath, updaterFlag)
 	if err != nil {
 		return resp, err
 	}
@@ -150,40 +148,6 @@ func (u *updater) Clean() {
 	}
 }
 
-func (u *updater) pullRepository() (commit string, err error) {
-	// Determine whether we should branch or pull.
-	if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
-		// Create a temporary folder to store the repository.
-		if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil {
-			return "", vulnsrc.ErrFilesystem
-		}
-		cmd := exec.Command("git", "clone", trackerURI, ".")
-		cmd.Dir = u.repositoryLocalPath
-		if out, err := cmd.CombinedOutput(); err != nil {
-			u.Clean()
-			log.WithError(err).WithField("output", string(out)).Error("could not clone ubuntu-cve-tracker repository")
-			return "", commonerr.ErrCouldNotDownload
-		}
-	} else {
-		// The repository already exists and it needs to be refreshed via a pull.
-		cmd := exec.Command("git", "pull")
-		cmd.Dir = u.repositoryLocalPath
-		if _, err := cmd.CombinedOutput(); err != nil {
-			return "", vulnsrc.ErrGitFailure
-		}
-	}
-
-	cmd := exec.Command("git", "rev-parse", "HEAD")
-	cmd.Dir = u.repositoryLocalPath
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return "", vulnsrc.ErrGitFailure
-	}
-
-	commit = strings.TrimSpace(string(out))
-	return
-}
-
 func collectModifiedVulnerabilities(commit, dbCommit, repositoryLocalPath string) (map[string]struct{}, error) {
 	modifiedCVE := make(map[string]struct{})
 	for _, dirName := range []string{"active", "retired"} {

pkg/gitutil/gitutil.go

@@ -0,0 +1,146 @@
+// Copyright 2018 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package gitutil implements an easy way to update a git repository to a local
+// temporary directory.
+package gitutil
+
+import (
+	"errors"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// ErrFailedClone is returned when a git clone is unsuccessful.
+var ErrFailedClone = errors.New("failed to clone git repository")
+
+// ErrFailedRevParse is returned when a git rev-parse is unsuccessful.
+var ErrFailedRevParse = errors.New("failed to rev-parse git repository")
+
+// ErrFailedPull is returned when a git pull is unsuccessful.
+var ErrFailedPull = errors.New("failed to pull git repository")
+
+// pull performs a git pull on the provided path and returns the commit SHA
+// for the HEAD reference.
+func pull(path string) (head string, err error) {
+	// Prepare a command to pull the repository.
+	cmd := exec.Command("git", "pull")
+	cmd.Dir = path
+
+	// Execute the command.
+	var commandOutput []byte
+	commandOutput, err = cmd.CombinedOutput()
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"path":   path,
+			"output": string(commandOutput),
+		}).Error("failed to git rev-parse repository")
+		err = ErrFailedPull
+		return
+	}
+
+	return revParseHead(path)
+}
+
+// CloneOrPull performs a git pull if there is a git repository located at
+// repoPath. Otherwise, it performs a git clone to that path.
+//
+// If repoPath is left empty, a temporary directory is generated with the
+// provided prefix and returned.
+func CloneOrPull(remote, repoPath, tempDirPrefix string) (path, head string, err error) {
+	// Create a temporary directory if the path is unspecified.
+	if repoPath == "" {
+		path, err = ioutil.TempDir(os.TempDir(), tempDirPrefix)
+		if err != nil {
+			return
+		}
+	} else {
+		path = repoPath
+	}
+
+	if _, pathExists := os.Stat(path); os.IsNotExist(pathExists) {
+		head, err = clone(remote, path)
+		return
+	}
+
+	head, err = pull(path)
+	return
+}
+
+// clone performs a git clone to the provided path and returns the commit SHA
+// for the HEAD reference.
+func clone(remote, path string) (head string, err error) {
+	// Handle an invalid path.
+	if path == "" {
+		log.WithField("remote", remote).Error("attempted to git clone repository to empty path")
+		err = ErrFailedClone
+		return
+	}
+
+	// Prepare a command to clone the repository.
+	cmd := exec.Command("git", "clone", remote, ".")
+	cmd.Dir = path
+
+	// Execute the command.
+	var commandOutput []byte
+	commandOutput, err = cmd.CombinedOutput()
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"remote": remote,
+			"path":   path,
+			"output": string(commandOutput),
+		}).Error("failed to git clone repository")
+
+		err = os.RemoveAll(path)
+		if err != nil {
+			log.WithError(err).WithField("path", path).Warn("failed to remove directory of failed clone")
+		}
+		err = ErrFailedClone
+		return
+	}
+
+	return revParseHead(path)
+}
+
+// revParseHead performs a git rev-parse HEAD on the provided path and returns
+// the commit SHA for the HEAD reference.
+func revParseHead(path string) (head string, err error) {
+	// Handle an invalid path.
+	if path == "" {
+		log.Error("attempted to rev-parse repository with empty path")
+		err = ErrFailedRevParse
+		return
+	}
+
+	cmd := exec.Command("git", "rev-parse", "HEAD")
+	cmd.Dir = path
+
+	var commandOutput []byte
+	commandOutput, err = cmd.CombinedOutput()
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"path":   path,
+			"output": string(commandOutput),
+		}).Error("failed to git rev-parse repository")
+		err = ErrFailedRevParse
+		return
+	}
+
+	head = strings.TrimSpace(string(commandOutput))
+	return
+}