@ -18,14 +18,12 @@ package ubuntu
import (
import (
"bufio"
"bufio"
"bytes"
"fmt"
"fmt"
"io"
"io"
"io/ioutil"
"io/ioutil"
"os"
"os"
"os/exec"
"os/exec"
"regexp"
"regexp"
"strconv"
"strings"
"strings"
log "github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"
@ -38,10 +36,9 @@ import (
)
)
const (
const (
trackerURI = "https://launchpad.net/ubuntu-cve-tracker"
trackerGitURL = "https://git.launchpad.net/ubuntu-cve-tracker"
trackerRepository = "https://launchpad.net/ubuntu-cve-tracker"
updaterFlag = "ubuntuUpdater"
updaterFlag = "ubuntuUpdater"
cveURL = "http://people.ubuntu.com/~ubuntu-security/cve/%s"
cveURL = "http://people.ubuntu.com/~ubuntu-security/cve/%s"
)
)
var (
var (
@ -87,25 +84,27 @@ func init() {
func ( u * updater ) Update ( datastore database . Datastore ) ( resp vulnsrc . UpdateResponse , err error ) {
func ( u * updater ) Update ( datastore database . Datastore ) ( resp vulnsrc . UpdateResponse , err error ) {
log . WithField ( "package" , "Ubuntu" ) . Info ( "Start fetching vulnerabilities" )
log . WithField ( "package" , "Ubuntu" ) . Info ( "Start fetching vulnerabilities" )
// Pull the bzr repository.
// Pull the master branch.
if err = u . pullRepository ( ) ; err != nil {
var commit string
return resp , err
commit , err = u . pullRepository ( )
}
// Get revision number.
revisionNumber , err := getRevisionNumber ( u . repositoryLocalPath )
if err != nil {
if err != nil {
return resp , err
return
}
}
// Get the latest revision number we successfully applied in the database.
// Get the latest revision number we successfully applied in the database.
db RevisionNumber, err := datastore . GetKeyValue ( "ubuntuUpdater" )
db Commit, err := datastore . GetKeyValue ( updaterFlag )
if err != nil {
if err != nil {
return resp , err
return resp , err
}
}
// Short-circuit if there have been no updates.
if commit == dbCommit {
log . WithField ( "package" , "ubuntu" ) . Debug ( "no update" )
return
}
// Get the list of vulnerabilities that we have to update.
// Get the list of vulnerabilities that we have to update.
modifiedCVE , err := collectModifiedVulnerabilities ( revisionNumber , dbRevisionNumber , u . repositoryLocalPath )
modifiedCVE , err := collectModifiedVulnerabilities ( u. repositoryLocalPath )
if err != nil {
if err != nil {
return resp , err
return resp , err
}
}
@ -136,8 +135,7 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo
// If we encountered unknown Ubuntu release, we don't want the revision
// If we encountered unknown Ubuntu release, we don't want the revision
// number to be considered as managed.
// number to be considered as managed.
dbRevisionNumberInt , _ := strconv . Atoi ( dbRevisionNumber )
commit = dbCommit
revisionNumber = dbRevisionNumberInt
}
}
// Close the file manually.
// Close the file manually.
@ -150,7 +148,7 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo
// Add flag and notes.
// Add flag and notes.
resp . FlagName = updaterFlag
resp . FlagName = updaterFlag
resp . FlagValue = strconv. Itoa ( revisionNumber )
resp . FlagValue = commit
for note := range notes {
for note := range notes {
resp . Notes = append ( resp . Notes , note )
resp . Notes = append ( resp . Notes , note )
}
}
@ -162,120 +160,76 @@ func (u *updater) Clean() {
os . RemoveAll ( u . repositoryLocalPath )
os . RemoveAll ( u . repositoryLocalPath )
}
}
func ( u * updater ) pullRepository ( ) ( err error ) {
func ( u * updater ) pullRepository ( ) ( commit string , err error ) {
// Determine whether we should branch or pull .
// If the repository doesn't exist, clone it .
if _ , pathExists := os . Stat ( u . repositoryLocalPath ) ; u . repositoryLocalPath == "" || os . IsNotExist ( pathExists ) {
if _ , pathExists := os . Stat ( u . repositoryLocalPath ) ; u . repositoryLocalPath == "" || os . IsNotExist ( pathExists ) {
// Create a temporary folder to store the repository.
if u . repositoryLocalPath , err = ioutil . TempDir ( os . TempDir ( ) , "ubuntu-cve-tracker" ) ; err != nil {
if u . repositoryLocalPath , err = ioutil . TempDir ( os . TempDir ( ) , "ubuntu-cve-tracker" ) ; err != nil {
return vulnsrc . ErrFilesystem
return "" , vulnsrc . ErrFilesystem
}
}
// Branch repository.
cmd := exec . Command ( "git" , "clone" , trackerGitURL , "." )
cmd := exec . Command ( "bzr" , "branch" , "--use-existing-dir" , trackerRepository , "." )
cmd . Dir = u . repositoryLocalPath
cmd . Dir = u . repositoryLocalPath
if out , err := cmd . CombinedOutput ( ) ; err != nil {
if out , err := cmd . CombinedOutput ( ) ; err != nil {
log . WithError ( err ) . WithField ( "output" , string ( out ) ) . Error ( "could not branch Ubuntu repository" )
u . Clean ( )
return commonerr . ErrCouldNotDownload
log . WithError ( err ) . WithField ( "output" , string ( out ) ) . Error ( "could not pull ubuntu-cve-tracker repository" )
return "" , commonerr . ErrCouldNotDownload
}
} else {
// The repository already exists and it needs to be refreshed via a pull.
cmd := exec . Command ( "git" , "pull" )
cmd . Dir = u . repositoryLocalPath
if _ , err := cmd . CombinedOutput ( ) ; err != nil {
return "" , vulnsrc . ErrGitFailure
}
}
return nil
}
}
// Pull repository.
cmd := exec . Command ( "git" , "rev-parse" , "HEAD" )
cmd := exec . Command ( "bzr" , "pull" , "--overwrite" )
cmd . Dir = u . repositoryLocalPath
cmd . Dir = u . repositoryLocalPath
if out , err := cmd . CombinedOutput ( ) ; err != nil {
os . RemoveAll ( u . repositoryLocalPath )
log . WithError ( err ) . WithField ( "output" , string ( out ) ) . Error ( "could not pull Ubuntu repository" )
return commonerr . ErrCouldNotDownload
}
return nil
}
func getRevisionNumber ( pathToRepo string ) ( int , error ) {
cmd := exec . Command ( "bzr" , "revno" )
cmd . Dir = pathToRepo
out , err := cmd . CombinedOutput ( )
out , err := cmd . CombinedOutput ( )
if err != nil {
if err != nil {
log . WithError ( err ) . WithField ( "output" , string ( out ) ) . Error ( "could not get Ubuntu repository's revision number" )
return "" , vulnsrc . ErrGitFailure
return 0 , commonerr . ErrCouldNotDownload
}
revno , err := strconv . Atoi ( strings . TrimSpace ( string ( out ) ) )
if err != nil {
log . WithError ( err ) . WithField ( "output" , string ( out ) ) . Error ( "could not parse Ubuntu repository's revision number" )
return 0 , commonerr . ErrCouldNotDownload
}
}
return revno , nil
commit = strings . TrimSpace ( string ( out ) )
return
}
}
func collectModifiedVulnerabilities ( re vision int , dbRevision , re positoryLocalPath string ) ( map [ string ] struct { } , error ) {
func collectModifiedVulnerabilities ( repositoryLocalPath string ) ( map [ string ] struct { } , error ) {
modifiedCVE := make ( map [ string ] struct { } )
modifiedCVE := make ( map [ string ] struct { } )
// Handle a brand new database.
// Handle a brand new database.
if dbRevision == "" {
for _ , folder := range [ ] string { "active" , "retired" } {
d , err := os . Open ( repositoryLocalPath + "/" + folder )
if err != nil {
log . WithError ( err ) . Error ( "could not open Ubuntu vulnerabilities repository's folder" )
return nil , vulnsrc . ErrFilesystem
}
// Get the FileInfo of all the files in the directory.
for _ , folder := range [ ] string { "active" , "retired" } {
names , err := d . Readdirnames ( - 1 )
d , err := os . Open ( repositoryLocalPath + "/" + folder )
if err != nil {
if err != nil {
log . WithError ( err ) . Error ( "could not read Ubuntu vulnerabilities repository's folder" )
log . WithError ( err ) . Error ( "could not open Ubuntu vulnerabilities repository's folder" )
return nil , vulnsrc . ErrFilesystem
return nil , vulnsrc . ErrFilesystem
}
// Add the vulnerabilities to the list.
for _ , name := range names {
if strings . HasPrefix ( name , "CVE-" ) {
modifiedCVE [ folder + "/" + name ] = struct { } { }
}
}
// Close the file manually.
//
// We do that instead of using defer because defer works on a function-level scope.
// We would open many files and close them all at once at the end of the function,
// which could lead to exceed fs.file-max.
d . Close ( )
}
}
return modifiedCVE , nil
// Get the FileInfo of all the files in the directory.
}
names , err := d . Readdirnames ( - 1 )
if err != nil {
// Handle an up to date database.
log . WithError ( err ) . Error ( "could not read Ubuntu vulnerabilities repository's folder" )
dbRevisionInt , _ := strconv . Atoi ( dbRevision )
return nil , vulnsrc . ErrFilesystem
if revision == dbRevisionInt {
}
log . WithField ( "package" , "Ubuntu" ) . Debug ( "no update" )
return modifiedCVE , nil
}
// Handle a database that needs upgrading.
cmd := exec . Command ( "bzr" , "log" , "--verbose" , "-r" + strconv . Itoa ( dbRevisionInt + 1 ) + ".." , "-n0" )
cmd . Dir = repositoryLocalPath
out , err := cmd . CombinedOutput ( )
if err != nil {
log . WithError ( err ) . WithField ( "output" , string ( out ) ) . Error ( "could not get Ubuntu vulnerabilities repository logs" )
return nil , commonerr . ErrCouldNotDownload
}
scanner := bufio . NewScanner ( bytes . NewReader ( out ) )
// Add the vulnerabilities to the list.
for scanner . Scan ( ) {
for _ , name := range names {
text := strings . TrimSpace ( scanner . Text ( ) )
if strings . HasPrefix ( name , "CVE-" ) {
if strings . Contains ( text , "CVE-" ) && ( strings . HasPrefix ( text , "active/" ) || strings . HasPrefix ( text , "retired/" ) ) {
modifiedCVE [ folder + "/" + name ] = struct { } { }
if strings . Contains ( text , " => " ) {
text = text [ strings . Index ( text , " => " ) + 4 : ]
}
}
modifiedCVE [ text ] = struct { } { }
}
}
// Close the file manually.
//
// We do that instead of using defer because defer works on a function-level scope.
// We would open many files and close them all at once at the end of the function,
// which could lead to exceed fs.file-max.
d . Close ( )
}
}
return modifiedCVE , nil
return modifiedCVE , nil
}
}
func parseUbuntuCVE ( fileContent io . Reader ) ( vulnerability database . Vulnerability , unknownReleases map [ string ] struct { } , err error ) {
func parseUbuntuCVE ( fileContent io . Reader ) ( vulnerability database . Vulnerability , unknownReleases map [ string ] struct { } , err error ) {
@ -344,6 +298,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability
// Only consider the package if its status is needed, active, deferred, not-affected or
// Only consider the package if its status is needed, active, deferred, not-affected or
// released. Ignore DNE (package does not exist), needs-triage, ignored, pending.
// released. Ignore DNE (package does not exist), needs-triage, ignored, pending.
if md [ "status" ] == "needed" || md [ "status" ] == "active" || md [ "status" ] == "deferred" || md [ "status" ] == "released" || md [ "status" ] == "not-affected" {
if md [ "status" ] == "needed" || md [ "status" ] == "active" || md [ "status" ] == "deferred" || md [ "status" ] == "released" || md [ "status" ] == "not-affected" {
md [ "release" ] = strings . Split ( md [ "release" ] , "/" ) [ 0 ]
if _ , isReleaseIgnored := ubuntuIgnoredReleases [ md [ "release" ] ] ; isReleaseIgnored {
if _ , isReleaseIgnored := ubuntuIgnoredReleases [ md [ "release" ] ] ; isReleaseIgnored {
continue
continue
}
}
@ -392,7 +347,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability
// If no link has been provided (CVE-2006-NNN0 for instance), add the link to the tracker
// If no link has been provided (CVE-2006-NNN0 for instance), add the link to the tracker
if vulnerability . Link == "" {
if vulnerability . Link == "" {
vulnerability . Link = tracker URI
vulnerability . Link = tracker GitURL
}
}
// If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown
// If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown