From c2d887f9e99184af502aca7abbe2044d2929e789 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 14 Sep 2018 17:25:12 -0400 Subject: [PATCH] pkg/gitutil: init This refactors the code we're using to manage temporary git repositories into a utility package. --- ext/vulnsrc/alpine/alpine.go | 41 +--------- ext/vulnsrc/driver.go | 3 - ext/vulnsrc/ubuntu/ubuntu.go | 42 +--------- pkg/gitutil/gitutil.go | 146 +++++++++++++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 80 deletions(-) create mode 100644 pkg/gitutil/gitutil.go diff --git a/ext/vulnsrc/alpine/alpine.go b/ext/vulnsrc/alpine/alpine.go index e7ec38f8..a12e358b 100644 --- a/ext/vulnsrc/alpine/alpine.go +++ b/ext/vulnsrc/alpine/alpine.go @@ -1,4 +1,4 @@ -// Copyright 2017 clair authors +// Copyright 2018 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,7 +20,6 @@ import ( "io" "io/ioutil" "os" - "os/exec" "path/filepath" "strings" @@ -31,7 +30,7 @@ import ( "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/vulnsrc" - "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/gitutil" ) const ( @@ -53,7 +52,7 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er // Pull the master branch. var commit string - commit, err = u.pullRepository() + u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(secdbGitURL, u.repositoryLocalPath, updaterFlag) if err != nil { return } @@ -183,40 +182,6 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database return } -func (u *updater) pullRepository() (commit string, err error) { - // If the repository doesn't exist, clone it. - if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) { - if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil { - return "", vulnsrc.ErrFilesystem - } - - cmd := exec.Command("git", "clone", secdbGitURL, ".") - cmd.Dir = u.repositoryLocalPath - if out, err := cmd.CombinedOutput(); err != nil { - u.Clean() - log.WithError(err).WithField("output", string(out)).Error("could not clone alpine-secdb repository") - return "", commonerr.ErrCouldNotDownload - } - } else { - // The repository already exists and it needs to be refreshed via a pull. - cmd := exec.Command("git", "pull") - cmd.Dir = u.repositoryLocalPath - if _, err := cmd.CombinedOutput(); err != nil { - return "", vulnsrc.ErrGitFailure - } - } - - cmd := exec.Command("git", "rev-parse", "HEAD") - cmd.Dir = u.repositoryLocalPath - out, err := cmd.CombinedOutput() - if err != nil { - return "", vulnsrc.ErrGitFailure - } - - commit = strings.TrimSpace(string(out)) - return -} - type secDBFile struct { Distro string `yaml:"distroversion"` Packages []struct { diff --git a/ext/vulnsrc/driver.go b/ext/vulnsrc/driver.go index 91b28831..4a119c46 100644 --- a/ext/vulnsrc/driver.go +++ b/ext/vulnsrc/driver.go @@ -27,9 +27,6 @@ var ( // ErrFilesystem is returned when a fetcher fails to interact with the local filesystem. ErrFilesystem = errors.New("vulnsrc: something went wrong when interacting with the fs") - // ErrGitFailure is returned when a fetcher fails to interact with git. - ErrGitFailure = errors.New("vulnsrc: something went wrong when interacting with git") - updatersM sync.RWMutex updaters = make(map[string]Updater) ) diff --git a/ext/vulnsrc/ubuntu/ubuntu.go b/ext/vulnsrc/ubuntu/ubuntu.go index 2f040c0b..ba3a9ac5 100644 --- a/ext/vulnsrc/ubuntu/ubuntu.go +++ b/ext/vulnsrc/ubuntu/ubuntu.go @@ -1,4 +1,4 @@ -// Copyright 2017 clair authors +// Copyright 2018 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,9 +21,7 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" - "os/exec" "regexp" "strings" @@ -33,7 +31,7 @@ import ( "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/vulnsrc" - "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/gitutil" ) const ( @@ -89,7 +87,7 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er // Pull the master branch. var commit string - commit, err = u.pullRepository() + u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(trackerURI, u.repositoryLocalPath, updaterFlag) if err != nil { return resp, err } @@ -150,40 +148,6 @@ func (u *updater) Clean() { } } -func (u *updater) pullRepository() (commit string, err error) { - // Determine whether we should branch or pull. - if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) { - // Create a temporary folder to store the repository. - if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil { - return "", vulnsrc.ErrFilesystem - } - cmd := exec.Command("git", "clone", trackerURI, ".") - cmd.Dir = u.repositoryLocalPath - if out, err := cmd.CombinedOutput(); err != nil { - u.Clean() - log.WithError(err).WithField("output", string(out)).Error("could not clone ubuntu-cve-tracker repository") - return "", commonerr.ErrCouldNotDownload - } - } else { - // The repository already exists and it needs to be refreshed via a pull. - cmd := exec.Command("git", "pull") - cmd.Dir = u.repositoryLocalPath - if _, err := cmd.CombinedOutput(); err != nil { - return "", vulnsrc.ErrGitFailure - } - } - - cmd := exec.Command("git", "rev-parse", "HEAD") - cmd.Dir = u.repositoryLocalPath - out, err := cmd.CombinedOutput() - if err != nil { - return "", vulnsrc.ErrGitFailure - } - - commit = strings.TrimSpace(string(out)) - return -} - func collectModifiedVulnerabilities(commit, dbCommit, repositoryLocalPath string) (map[string]struct{}, error) { modifiedCVE := make(map[string]struct{}) for _, dirName := range []string{"active", "retired"} { diff --git a/pkg/gitutil/gitutil.go b/pkg/gitutil/gitutil.go new file mode 100644 index 00000000..b767db3b --- /dev/null +++ b/pkg/gitutil/gitutil.go @@ -0,0 +1,146 @@ +// Copyright 2018 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gitutil implements an easy way to update a git repository to a local +// temporary directory. +package gitutil + +import ( + "errors" + "io/ioutil" + "os" + "os/exec" + "strings" + + log "github.com/sirupsen/logrus" +) + +// ErrFailedClone is returned when a git clone is unsuccessful. +var ErrFailedClone = errors.New("failed to clone git repository") + +// ErrFailedRevParse is returned when a git rev-parse is unsuccessful. +var ErrFailedRevParse = errors.New("failed to rev-parse git repository") + +// ErrFailedPull is returned when a git pull is unsuccessful. +var ErrFailedPull = errors.New("failed to pull git repository") + +// pull performs a git pull on the provided path and returns the commit SHA +// for the HEAD reference. +func pull(path string) (head string, err error) { + // Prepare a command to pull the repository. + cmd := exec.Command("git", "pull") + cmd.Dir = path + + // Execute the command. + var commandOutput []byte + commandOutput, err = cmd.CombinedOutput() + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "path": path, + "output": string(commandOutput), + }).Error("failed to git rev-parse repository") + err = ErrFailedPull + return + } + + return revParseHead(path) +} + +// CloneOrPull performs a git pull if there is a git repository located at +// repoPath. Otherwise, it performs a git clone to that path. +// +// If repoPath is left empty, a temporary directory is generated with the +// provided prefix and returned. +func CloneOrPull(remote, repoPath, tempDirPrefix string) (path, head string, err error) { + // Create a temporary directory if the path is unspecified. + if repoPath == "" { + path, err = ioutil.TempDir(os.TempDir(), tempDirPrefix) + if err != nil { + return + } + } else { + path = repoPath + } + + if _, pathExists := os.Stat(path); os.IsNotExist(pathExists) { + head, err = clone(remote, path) + return + } + + head, err = pull(path) + return +} + +// clone performs a git clone to the provided path and returns the commit SHA +// for the HEAD reference. +func clone(remote, path string) (head string, err error) { + // Handle an invalid path. + if path == "" { + log.WithField("remote", remote).Error("attempted to git clone repository to empty path") + err = ErrFailedClone + return + } + + // Prepare a command to clone the repository. + cmd := exec.Command("git", "clone", remote, ".") + cmd.Dir = path + + // Execute the command. + var commandOutput []byte + commandOutput, err = cmd.CombinedOutput() + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "remote": remote, + "path": path, + "output": string(commandOutput), + }).Error("failed to git clone repository") + + err = os.RemoveAll(path) + if err != nil { + log.WithError(err).WithField("path", path).Warn("failed to remove directory of failed clone") + } + err = ErrFailedClone + return + } + + return revParseHead(path) +} + +// revParseHead performs a git rev-parse HEAD on the provided path and returns +// the commit SHA for the HEAD reference. +func revParseHead(path string) (head string, err error) { + // Handle an invalid path. + if path == "" { + log.Error("attempted to rev-parse repository with empty path") + err = ErrFailedRevParse + return + } + + cmd := exec.Command("git", "rev-parse", "HEAD") + cmd.Dir = path + + var commandOutput []byte + commandOutput, err = cmd.CombinedOutput() + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "path": path, + "output": string(commandOutput), + }).Error("failed to git rev-parse repository") + err = ErrFailedRevParse + return + } + + head = strings.TrimSpace(string(commandOutput)) + return +}