pkg/gitutil: init
This refactors the code we're using to manage temporary git repositories into a utility package.
This commit is contained in:
Jimmy Zelinskie
parent
f98ff58afd
commit
c2d887f9e9
@ -1,4 +1,4 @@
|
||||
// Copyright 2017 clair authors
|
||||
// Copyright 2018 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@ -20,7 +20,6 @@ import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
@ -31,7 +30,7 @@ import (
|
||||
"github.com/coreos/clair/ext/versionfmt"
|
||||
"github.com/coreos/clair/ext/versionfmt/dpkg"
|
||||
"github.com/coreos/clair/ext/vulnsrc"
|
||||
"github.com/coreos/clair/pkg/commonerr"
|
||||
"github.com/coreos/clair/pkg/gitutil"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -53,7 +52,7 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er
|
||||
|
||||
// Pull the master branch.
|
||||
var commit string
|
||||
commit, err = u.pullRepository()
|
||||
u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(secdbGitURL, u.repositoryLocalPath, updaterFlag)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@ -183,40 +182,6 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database
|
||||
return
|
||||
}
|
||||
|
||||
func (u *updater) pullRepository() (commit string, err error) {
|
||||
// If the repository doesn't exist, clone it.
|
||||
if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
|
||||
if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
|
||||
return "", vulnsrc.ErrFilesystem
|
||||
}
|
||||
|
||||
cmd := exec.Command("git", "clone", secdbGitURL, ".")
|
||||
cmd.Dir = u.repositoryLocalPath
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
u.Clean()
|
||||
log.WithError(err).WithField("output", string(out)).Error("could not clone alpine-secdb repository")
|
||||
return "", commonerr.ErrCouldNotDownload
|
||||
}
|
||||
} else {
|
||||
// The repository already exists and it needs to be refreshed via a pull.
|
||||
cmd := exec.Command("git", "pull")
|
||||
cmd.Dir = u.repositoryLocalPath
|
||||
if _, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", vulnsrc.ErrGitFailure
|
||||
}
|
||||
}
|
||||
|
||||
cmd := exec.Command("git", "rev-parse", "HEAD")
|
||||
cmd.Dir = u.repositoryLocalPath
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return "", vulnsrc.ErrGitFailure
|
||||
}
|
||||
|
||||
commit = strings.TrimSpace(string(out))
|
||||
return
|
||||
}
|
||||
|
||||
type secDBFile struct {
|
||||
Distro string `yaml:"distroversion"`
|
||||
Packages []struct {
|
||||
|
||||
@ -27,9 +27,6 @@ var (
|
||||
// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
|
||||
ErrFilesystem = errors.New("vulnsrc: something went wrong when interacting with the fs")
|
||||
|
||||
// ErrGitFailure is returned when a fetcher fails to interact with git.
|
||||
ErrGitFailure = errors.New("vulnsrc: something went wrong when interacting with git")
|
||||
|
||||
updatersM sync.RWMutex
|
||||
updaters = make(map[string]Updater)
|
||||
)
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
// Copyright 2017 clair authors
|
||||
// Copyright 2018 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@ -21,9 +21,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
@ -33,7 +31,7 @@ import (
|
||||
"github.com/coreos/clair/ext/versionfmt"
|
||||
"github.com/coreos/clair/ext/versionfmt/dpkg"
|
||||
"github.com/coreos/clair/ext/vulnsrc"
|
||||
"github.com/coreos/clair/pkg/commonerr"
|
||||
"github.com/coreos/clair/pkg/gitutil"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -89,7 +87,7 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er
|
||||
|
||||
// Pull the master branch.
|
||||
var commit string
|
||||
commit, err = u.pullRepository()
|
||||
u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(trackerURI, u.repositoryLocalPath, updaterFlag)
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
@ -150,40 +148,6 @@ func (u *updater) Clean() {
|
||||
}
|
||||
}
|
||||
|
||||
func (u *updater) pullRepository() (commit string, err error) {
|
||||
// Determine whether we should branch or pull.
|
||||
if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
|
||||
// Create a temporary folder to store the repository.
|
||||
if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil {
|
||||
return "", vulnsrc.ErrFilesystem
|
||||
}
|
||||
cmd := exec.Command("git", "clone", trackerURI, ".")
|
||||
cmd.Dir = u.repositoryLocalPath
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
u.Clean()
|
||||
log.WithError(err).WithField("output", string(out)).Error("could not clone ubuntu-cve-tracker repository")
|
||||
return "", commonerr.ErrCouldNotDownload
|
||||
}
|
||||
} else {
|
||||
// The repository already exists and it needs to be refreshed via a pull.
|
||||
cmd := exec.Command("git", "pull")
|
||||
cmd.Dir = u.repositoryLocalPath
|
||||
if _, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", vulnsrc.ErrGitFailure
|
||||
}
|
||||
}
|
||||
|
||||
cmd := exec.Command("git", "rev-parse", "HEAD")
|
||||
cmd.Dir = u.repositoryLocalPath
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return "", vulnsrc.ErrGitFailure
|
||||
}
|
||||
|
||||
commit = strings.TrimSpace(string(out))
|
||||
return
|
||||
}
|
||||
|
||||
func collectModifiedVulnerabilities(commit, dbCommit, repositoryLocalPath string) (map[string]struct{}, error) {
|
||||
modifiedCVE := make(map[string]struct{})
|
||||
for _, dirName := range []string{"active", "retired"} {
|
||||
|
||||
146
pkg/gitutil/gitutil.go
Normal file
146
pkg/gitutil/gitutil.go
Normal file
@ -0,0 +1,146 @@
|
||||
// Copyright 2018 clair authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package gitutil implements an easy way to update a git repository to a local
|
||||
// temporary directory.
|
||||
package gitutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// ErrFailedClone is returned when a git clone is unsuccessful.
|
||||
var ErrFailedClone = errors.New("failed to clone git repository")
|
||||
|
||||
// ErrFailedRevParse is returned when a git rev-parse is unsuccessful.
|
||||
var ErrFailedRevParse = errors.New("failed to rev-parse git repository")
|
||||
|
||||
// ErrFailedPull is returned when a git pull is unsuccessful.
|
||||
var ErrFailedPull = errors.New("failed to pull git repository")
|
||||
|
||||
// pull performs a git pull on the provided path and returns the commit SHA
|
||||
// for the HEAD reference.
|
||||
func pull(path string) (head string, err error) {
|
||||
// Prepare a command to pull the repository.
|
||||
cmd := exec.Command("git", "pull")
|
||||
cmd.Dir = path
|
||||
|
||||
// Execute the command.
|
||||
var commandOutput []byte
|
||||
commandOutput, err = cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.WithError(err).WithFields(log.Fields{
|
||||
"path": path,
|
||||
"output": string(commandOutput),
|
||||
}).Error("failed to git rev-parse repository")
|
||||
err = ErrFailedPull
|
||||
return
|
||||
}
|
||||
|
||||
return revParseHead(path)
|
||||
}
|
||||
|
||||
// CloneOrPull performs a git pull if there is a git repository located at
|
||||
// repoPath. Otherwise, it performs a git clone to that path.
|
||||
//
|
||||
// If repoPath is left empty, a temporary directory is generated with the
|
||||
// provided prefix and returned.
|
||||
func CloneOrPull(remote, repoPath, tempDirPrefix string) (path, head string, err error) {
|
||||
// Create a temporary directory if the path is unspecified.
|
||||
if repoPath == "" {
|
||||
path, err = ioutil.TempDir(os.TempDir(), tempDirPrefix)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
path = repoPath
|
||||
}
|
||||
|
||||
if _, pathExists := os.Stat(path); os.IsNotExist(pathExists) {
|
||||
head, err = clone(remote, path)
|
||||
return
|
||||
}
|
||||
|
||||
head, err = pull(path)
|
||||
return
|
||||
}
|
||||
|
||||
// clone performs a git clone to the provided path and returns the commit SHA
|
||||
// for the HEAD reference.
|
||||
func clone(remote, path string) (head string, err error) {
|
||||
// Handle an invalid path.
|
||||
if path == "" {
|
||||
log.WithField("remote", remote).Error("attempted to git clone repository to empty path")
|
||||
err = ErrFailedClone
|
||||
return
|
||||
}
|
||||
|
||||
// Prepare a command to clone the repository.
|
||||
cmd := exec.Command("git", "clone", remote, ".")
|
||||
cmd.Dir = path
|
||||
|
||||
// Execute the command.
|
||||
var commandOutput []byte
|
||||
commandOutput, err = cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.WithError(err).WithFields(log.Fields{
|
||||
"remote": remote,
|
||||
"path": path,
|
||||
"output": string(commandOutput),
|
||||
}).Error("failed to git clone repository")
|
||||
|
||||
err = os.RemoveAll(path)
|
||||
if err != nil {
|
||||
log.WithError(err).WithField("path", path).Warn("failed to remove directory of failed clone")
|
||||
}
|
||||
err = ErrFailedClone
|
||||
return
|
||||
}
|
||||
|
||||
return revParseHead(path)
|
||||
}
|
||||
|
||||
// revParseHead performs a git rev-parse HEAD on the provided path and returns
|
||||
// the commit SHA for the HEAD reference.
|
||||
func revParseHead(path string) (head string, err error) {
|
||||
// Handle an invalid path.
|
||||
if path == "" {
|
||||
log.Error("attempted to rev-parse repository with empty path")
|
||||
err = ErrFailedRevParse
|
||||
return
|
||||
}
|
||||
|
||||
cmd := exec.Command("git", "rev-parse", "HEAD")
|
||||
cmd.Dir = path
|
||||
|
||||
var commandOutput []byte
|
||||
commandOutput, err = cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.WithError(err).WithFields(log.Fields{
|
||||
"path": path,
|
||||
"output": string(commandOutput),
|
||||
}).Error("failed to git rev-parse repository")
|
||||
err = ErrFailedRevParse
|
||||
return
|
||||
}
|
||||
|
||||
head = strings.TrimSpace(string(commandOutput))
|
||||
return
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user