arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #621 from jzelinskie/gitutil

Browse Source 
pkg/gitutil: init
This commit is contained in:
Jimmy Zelinskie 2018-09-26 11:42:35 -04:00 committed by GitHub
commit 0ca9431235
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 152 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
ext/vulnsrc/alpine/alpine.go
View File

@ -1,4 +1,4 @@
// Copyright 2017 clair authors
// Copyright 2018 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -20,7 +20,6 @@ import (
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
@ -31,7 +30,7 @@ import (
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/pkg/gitutil"
)
const (
@ -53,7 +52,7 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er
// Pull the master branch.
var commit string
commit, err = u.pullRepository()
u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(secdbGitURL, u.repositoryLocalPath, updaterFlag)
if err != nil {
return
}
@ -183,40 +182,6 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database
return
}
func (u *updater) pullRepository() (commit string, err error) {
// If the repository doesn't exist, clone it.
if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
return "", vulnsrc.ErrFilesystem
}
cmd := exec.Command("git", "clone", secdbGitURL, ".")
cmd.Dir = u.repositoryLocalPath
if out, err := cmd.CombinedOutput(); err != nil {
u.Clean()
log.WithError(err).WithField("output", string(out)).Error("could not clone alpine-secdb repository")
return "", commonerr.ErrCouldNotDownload
}
} else {
// The repository already exists and it needs to be refreshed via a pull.
cmd := exec.Command("git", "pull")
cmd.Dir = u.repositoryLocalPath
if _, err := cmd.CombinedOutput(); err != nil {
return "", vulnsrc.ErrGitFailure
}
}
cmd := exec.Command("git", "rev-parse", "HEAD")
cmd.Dir = u.repositoryLocalPath
out, err := cmd.CombinedOutput()
if err != nil {
return "", vulnsrc.ErrGitFailure
}
commit = strings.TrimSpace(string(out))
return
}
type secDBFile struct {
Distro string `yaml:"distroversion"`
Packages []struct {

3
ext/vulnsrc/driver.go
View File

@ -27,9 +27,6 @@ var (
// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
ErrFilesystem = errors.New("vulnsrc: something went wrong when interacting with the fs")
// ErrGitFailure is returned when a fetcher fails to interact with git.
ErrGitFailure = errors.New("vulnsrc: something went wrong when interacting with git")
updatersM sync.RWMutex
updaters = make(map[string]Updater)
)

42
ext/vulnsrc/ubuntu/ubuntu.go
View File

@ -1,4 +1,4 @@
// Copyright 2017 clair authors
// Copyright 2018 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -21,9 +21,7 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"regexp"
"strings"
@ -33,7 +31,7 @@ import (
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/pkg/gitutil"
)
const (
@ -89,7 +87,7 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er
// Pull the master branch.
var commit string
commit, err = u.pullRepository()
u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(trackerURI, u.repositoryLocalPath, updaterFlag)
if err != nil {
return resp, err
}
@ -150,40 +148,6 @@ func (u *updater) Clean() {
}
}
func (u *updater) pullRepository() (commit string, err error) {
// Determine whether we should branch or pull.
if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
// Create a temporary folder to store the repository.
if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil {
return "", vulnsrc.ErrFilesystem
}
cmd := exec.Command("git", "clone", trackerURI, ".")
cmd.Dir = u.repositoryLocalPath
if out, err := cmd.CombinedOutput(); err != nil {
u.Clean()
log.WithError(err).WithField("output", string(out)).Error("could not clone ubuntu-cve-tracker repository")
return "", commonerr.ErrCouldNotDownload
}
} else {
// The repository already exists and it needs to be refreshed via a pull.
cmd := exec.Command("git", "pull")
cmd.Dir = u.repositoryLocalPath
if _, err := cmd.CombinedOutput(); err != nil {
return "", vulnsrc.ErrGitFailure
}
}
cmd := exec.Command("git", "rev-parse", "HEAD")
cmd.Dir = u.repositoryLocalPath
out, err := cmd.CombinedOutput()
if err != nil {
return "", vulnsrc.ErrGitFailure
}
commit = strings.TrimSpace(string(out))
return
}
func collectModifiedVulnerabilities(commit, dbCommit, repositoryLocalPath string) (map[string]struct{}, error) {
modifiedCVE := make(map[string]struct{})
for _, dirName := range []string{"active", "retired"} {

146
pkg/gitutil/gitutil.go Normal file
View File

@ -0,0 +1,146 @@
// Copyright 2018 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package gitutil implements an easy way to update a git repository to a local
// temporary directory.
package gitutil
import (
"errors"
"io/ioutil"
"os"
"os/exec"
"strings"
log "github.com/sirupsen/logrus"
)
// ErrFailedClone is returned when a git clone is unsuccessful.
var ErrFailedClone = errors.New("failed to clone git repository")
// ErrFailedRevParse is returned when a git rev-parse is unsuccessful.
var ErrFailedRevParse = errors.New("failed to rev-parse git repository")
// ErrFailedPull is returned when a git pull is unsuccessful.
var ErrFailedPull = errors.New("failed to pull git repository")
// pull performs a git pull on the provided path and returns the commit SHA
// for the HEAD reference.
func pull(path string) (head string, err error) {
// Prepare a command to pull the repository.
cmd := exec.Command("git", "pull")
cmd.Dir = path
// Execute the command.
var commandOutput []byte
commandOutput, err = cmd.CombinedOutput()
if err != nil {
log.WithError(err).WithFields(log.Fields{
"path": path,
"output": string(commandOutput),
}).Error("failed to git rev-parse repository")
err = ErrFailedPull
return
}
return revParseHead(path)
}
// CloneOrPull performs a git pull if there is a git repository located at
// repoPath. Otherwise, it performs a git clone to that path.
//
// If repoPath is left empty, a temporary directory is generated with the
// provided prefix and returned.
func CloneOrPull(remote, repoPath, tempDirPrefix string) (path, head string, err error) {
// Create a temporary directory if the path is unspecified.
if repoPath == "" {
path, err = ioutil.TempDir(os.TempDir(), tempDirPrefix)
if err != nil {
return
}
} else {
path = repoPath
}
if _, pathExists := os.Stat(path); os.IsNotExist(pathExists) {
head, err = clone(remote, path)
return
}
head, err = pull(path)
return
}
// clone performs a git clone to the provided path and returns the commit SHA
// for the HEAD reference.
func clone(remote, path string) (head string, err error) {
// Handle an invalid path.
if path == "" {
log.WithField("remote", remote).Error("attempted to git clone repository to empty path")
err = ErrFailedClone
return
}
// Prepare a command to clone the repository.
cmd := exec.Command("git", "clone", remote, ".")
cmd.Dir = path
// Execute the command.
var commandOutput []byte
commandOutput, err = cmd.CombinedOutput()
if err != nil {
log.WithError(err).WithFields(log.Fields{
"remote": remote,
"path": path,
"output": string(commandOutput),
}).Error("failed to git clone repository")
err = os.RemoveAll(path)
if err != nil {
log.WithError(err).WithField("path", path).Warn("failed to remove directory of failed clone")
}
err = ErrFailedClone
return
}
return revParseHead(path)
}
// revParseHead performs a git rev-parse HEAD on the provided path and returns
// the commit SHA for the HEAD reference.
func revParseHead(path string) (head string, err error) {
// Handle an invalid path.
if path == "" {
log.Error("attempted to rev-parse repository with empty path")
err = ErrFailedRevParse
return
}
cmd := exec.Command("git", "rev-parse", "HEAD")
cmd.Dir = path
var commandOutput []byte
commandOutput, err = cmd.CombinedOutput()
if err != nil {
log.WithError(err).WithFields(log.Fields{
"path": path,
"output": string(commandOutput),
}).Error("failed to git rev-parse repository")
err = ErrFailedRevParse
return
}
head = strings.TrimSpace(string(commandOutput))
return
}