// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package ubuntu implements a vulnerability source updater using the
// Ubuntu CVE Tracker.
package ubuntu
import (
"bufio"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"regexp"
"strings"
log "github.com/sirupsen/logrus"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/pkg/commonerr"
)
const (
trackerURI = "https://git.launchpad.net/ubuntu-cve-tracker"
updaterFlag = "ubuntuUpdater"
cveURL = "http://people.ubuntu.com/~ubuntu-security/cve/%s"
)
var (
ubuntuIgnoredReleases = map [ string ] struct { } {
"upstream" : { } ,
"devel" : { } ,
"dapper" : { } ,
"edgy" : { } ,
"feisty" : { } ,
"gutsy" : { } ,
"hardy" : { } ,
"intrepid" : { } ,
"jaunty" : { } ,
"karmic" : { } ,
"lucid" : { } ,
"maverick" : { } ,
"natty" : { } ,
"oneiric" : { } ,
"saucy" : { } ,
"vivid/ubuntu-core" : { } ,
"vivid/stable-phone-overlay" : { } ,
// Syntax error
"Patches" : { } ,
// Product
"product" : { } ,
}
affectsCaptureRegexp = regexp . MustCompile ( ` (?P<release>.*)_(?P<package>.*): (?P<status>[^\s]*)( \(+(?P<note>[^()]*)\)+)? ` )
affectsCaptureRegexpNames = affectsCaptureRegexp . SubexpNames ( )
errUnknownRelease = errors . New ( "found packages with CVEs for a verison of Ubuntu that Clair doesn't know about" )
)
type updater struct {
repositoryLocalPath string
}
func init ( ) {
vulnsrc . RegisterUpdater ( "ubuntu" , & updater { } )
}
func ( u * updater ) Update ( db database . Datastore ) ( resp vulnsrc . UpdateResponse , err error ) {
log . WithField ( "package" , "Ubuntu" ) . Info ( "Start fetching vulnerabilities" )
// Pull the master branch.
var commit string
commit , err = u . pullRepository ( )
if err != nil {
return resp , err
}
// Open a database transaction.
tx , err := db . Begin ( )
if err != nil {
return resp , err
}
defer tx . Rollback ( )
// Ask the database for the latest commit we successfully applied.
var dbCommit string
var ok bool
dbCommit , ok , err = tx . FindKeyValue ( updaterFlag )
if err != nil {
return
}
if ! ok {
dbCommit = ""
}
// Set the updaterFlag to equal the commit processed.
resp . FlagName = updaterFlag
resp . FlagValue = commit
// Short-circuit if there have been no updates.
if commit == dbCommit {
log . WithField ( "package" , "ubuntu" ) . Debug ( "no update" )
return
}
// Get the list of vulnerabilities that we have to update.
var modifiedCVE map [ string ] struct { }
modifiedCVE , err = collectModifiedVulnerabilities ( commit , dbCommit , u . repositoryLocalPath )
if err != nil {
return
}
// Get the list of vulnerabilities.
resp . Vulnerabilities , resp . Notes , err = collectVulnerabilitiesAndNotes ( u . repositoryLocalPath , modifiedCVE )
if err != nil {
return
}
// The only notes we take are if we encountered unknown Ubuntu release.
// We don't want the commit to be considered as managed in that case.
if len ( resp . Notes ) != 0 {
resp . FlagValue = dbCommit
}
return
}
func ( u * updater ) Clean ( ) {
if u . repositoryLocalPath != "" {
os . RemoveAll ( u . repositoryLocalPath )
}
}
func ( u * updater ) pullRepository ( ) ( commit string , err error ) {
// Determine whether we should branch or pull.
if _ , pathExists := os . Stat ( u . repositoryLocalPath ) ; u . repositoryLocalPath == "" || os . IsNotExist ( pathExists ) {
// Create a temporary folder to store the repository.
if u . repositoryLocalPath , err = ioutil . TempDir ( os . TempDir ( ) , "ubuntu-cve-tracker" ) ; err != nil {
return "" , vulnsrc . ErrFilesystem
}
cmd := exec . Command ( "git" , "clone" , trackerURI , "." )
cmd . Dir = u . repositoryLocalPath
if out , err := cmd . CombinedOutput ( ) ; err != nil {
u . Clean ( )
log . WithError ( err ) . WithField ( "output" , string ( out ) ) . Error ( "could not clone ubuntu-cve-tracker repository" )
return "" , commonerr . ErrCouldNotDownload
}
} else {
// The repository already exists and it needs to be refreshed via a pull.
cmd := exec . Command ( "git" , "pull" )
cmd . Dir = u . repositoryLocalPath
if _ , err := cmd . CombinedOutput ( ) ; err != nil {
return "" , vulnsrc . ErrGitFailure
}
}
cmd := exec . Command ( "git" , "rev-parse" , "HEAD" )
cmd . Dir = u . repositoryLocalPath
out , err := cmd . CombinedOutput ( )
if err != nil {
return "" , vulnsrc . ErrGitFailure
}
commit = strings . TrimSpace ( string ( out ) )
return
}
func collectModifiedVulnerabilities ( commit , dbCommit , repositoryLocalPath string ) ( map [ string ] struct { } , error ) {
modifiedCVE := make ( map [ string ] struct { } )
for _ , dirName := range [ ] string { "active" , "retired" } {
if err := processDirectory ( repositoryLocalPath , dirName , modifiedCVE ) ; err != nil {
return nil , err
}
}
return modifiedCVE , nil
}
func processDirectory ( repositoryLocalPath , dirName string , modifiedCVE map [ string ] struct { } ) error {
// Open the directory.
d , err := os . Open ( repositoryLocalPath + "/" + dirName )
if err != nil {
log . WithError ( err ) . Error ( "could not open Ubuntu vulnerabilities repository's folder" )
return vulnsrc . ErrFilesystem
}
defer d . Close ( )
// Get the FileInfo of all the files in the directory.
names , err := d . Readdirnames ( - 1 )
if err != nil {
log . WithError ( err ) . Error ( "could not read Ubuntu vulnerabilities repository's folder" )
return vulnsrc . ErrFilesystem
}
// Add the vulnerabilities to the list.
for _ , name := range names {
if strings . HasPrefix ( name , "CVE-" ) {
modifiedCVE [ dirName + "/" + name ] = struct { } { }
}
}
return nil
}
func collectVulnerabilitiesAndNotes ( repositoryLocalPath string , modifiedCVE map [ string ] struct { } ) ( [ ] database . VulnerabilityWithAffected , [ ] string , error ) {
vulns := make ( [ ] database . VulnerabilityWithAffected , 0 )
noteSet := make ( map [ string ] struct { } )
for cvePath := range modifiedCVE {
// Open the CVE file.
file , err := os . Open ( repositoryLocalPath + "/" + cvePath )
if err != nil {
// This can happen when a file is modified then moved in another commit.
continue
}
// Parse the vulnerability.
v , unknownReleases , err := parseUbuntuCVE ( file )
if err != nil {
file . Close ( )
return nil , nil , err
}
// Add the vulnerability to the response.
vulns = append ( vulns , v )
// Store any unknown releases as notes.
for k := range unknownReleases {
noteSet [ errUnknownRelease . Error ( ) + ": " + k ] = struct { } { }
}
file . Close ( )
}
// Convert the note set into a slice.
var notes [ ] string
for note := range noteSet {
notes = append ( notes , note )
}
return vulns , notes , nil
}
func parseUbuntuCVE ( fileContent io . Reader ) ( vulnerability database . VulnerabilityWithAffected , unknownReleases map [ string ] struct { } , err error ) {
unknownReleases = make ( map [ string ] struct { } )
readingDescription := false
scanner := bufio . NewScanner ( fileContent )
// only unique major releases will be considered. All sub releases' (e.g.
// precise/esm) features are considered belong to major releases.
uniqueRelease := map [ string ] struct { } { }
for scanner . Scan ( ) {
line := strings . TrimSpace ( scanner . Text ( ) )
// Skip any comments.
if strings . HasPrefix ( line , "#" ) {
continue
}
// Parse the name.
if strings . HasPrefix ( line , "Candidate:" ) {
vulnerability . Name = strings . TrimSpace ( strings . TrimPrefix ( line , "Candidate:" ) )
vulnerability . Link = fmt . Sprintf ( cveURL , vulnerability . Name )
continue
}
// Parse the priority.
if strings . HasPrefix ( line , "Priority:" ) {
priority := strings . TrimSpace ( strings . TrimPrefix ( line , "Priority:" ) )
// Handle syntax error: Priority: medium (heap-protector)
if strings . Contains ( priority , " " ) {
priority = priority [ : strings . Index ( priority , " " ) ]
}
vulnerability . Severity = SeverityFromPriority ( priority )
continue
}
// Parse the description.
if strings . HasPrefix ( line , "Description:" ) {
readingDescription = true
vulnerability . Description = strings . TrimSpace ( strings . TrimPrefix ( line , "Description:" ) ) // In case there is a formatting error and the description starts on the same line
continue
}
if readingDescription {
if strings . HasPrefix ( line , "Ubuntu-Description:" ) || strings . HasPrefix ( line , "Notes:" ) || strings . HasPrefix ( line , "Bugs:" ) || strings . HasPrefix ( line , "Priority:" ) || strings . HasPrefix ( line , "Discovered-by:" ) || strings . HasPrefix ( line , "Assigned-to:" ) {
readingDescription = false
} else {
vulnerability . Description = vulnerability . Description + " " + line
continue
}
}
// Try to parse the package that the vulnerability affects.
affectsCaptureArr := affectsCaptureRegexp . FindAllStringSubmatch ( line , - 1 )
if len ( affectsCaptureArr ) > 0 {
affectsCapture := affectsCaptureArr [ 0 ]
md := map [ string ] string { }
for i , n := range affectsCapture {
md [ affectsCaptureRegexpNames [ i ] ] = strings . TrimSpace ( n )
}
// Ignore Linux kernels.
if strings . HasPrefix ( md [ "package" ] , "linux" ) {
continue
}
// Only consider the package if its status is needed, active, deferred, not-affected or
// released. Ignore DNE (package does not exist), needs-triage, ignored, pending.
if md [ "status" ] == "needed" || md [ "status" ] == "active" || md [ "status" ] == "deferred" || md [ "status" ] == "released" || md [ "status" ] == "not-affected" {
md [ "release" ] = strings . Split ( md [ "release" ] , "/" ) [ 0 ]
if _ , isReleaseIgnored := ubuntuIgnoredReleases [ md [ "release" ] ] ; isReleaseIgnored {
continue
}
if _ , isReleaseKnown := database . UbuntuReleasesMapping [ md [ "release" ] ] ; ! isReleaseKnown {
unknownReleases [ md [ "release" ] ] = struct { } { }
continue
}
var version string
if md [ "status" ] == "released" {
if md [ "note" ] != "" {
var err error
err = versionfmt . Valid ( dpkg . ParserName , md [ "note" ] )
if err != nil {
log . WithError ( err ) . WithField ( "version" , md [ "note" ] ) . Warning ( "could not parse package version. skipping" )
}
version = md [ "note" ]
}
} else {
version = versionfmt . MaxVersion
}
if version == "" {
continue
}
releaseName := "ubuntu:" + database . UbuntuReleasesMapping [ md [ "release" ] ]
if _ , ok := uniqueRelease [ releaseName + "_:_" + md [ "package" ] ] ; ok {
continue
}
uniqueRelease [ releaseName + "_:_" + md [ "package" ] ] = struct { } { }
var fixedinVersion string
if version == versionfmt . MaxVersion {
fixedinVersion = ""
} else {
fixedinVersion = version
}
// Create and add the new package.
featureVersion := database . AffectedFeature {
Namespace : database . Namespace {
Name : releaseName ,
VersionFormat : dpkg . ParserName ,
} ,
FeatureName : md [ "package" ] ,
AffectedVersion : version ,
FixedInVersion : fixedinVersion ,
}
vulnerability . Affected = append ( vulnerability . Affected , featureVersion )
}
}
}
// Trim extra spaces in the description
vulnerability . Description = strings . TrimSpace ( vulnerability . Description )
// If no link has been provided (CVE-2006-NNN0 for instance), add the link to the tracker
if vulnerability . Link == "" {
vulnerability . Link = trackerURI
}
// If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown
if vulnerability . Severity == "" {
vulnerability . Severity = database . UnknownSeverity
}
return
}
// SeverityFromPriority converts an priority from the Ubuntu CVE Tracker into
// a database.Severity.
func SeverityFromPriority ( priority string ) database . Severity {
switch priority {
case "untriaged" :
return database . UnknownSeverity
case "negligible" :
return database . NegligibleSeverity
case "low" :
return database . LowSeverity
case "medium" :
return database . MediumSeverity
case "high" :
return database . HighSeverity
case "critical" :
return database . CriticalSeverity
default :
log . Warningf ( "could not determine a vulnerability severity from: %s" , priority )
return database . UnknownSeverity
}
}