2018-09-14 21:25:12 +00:00
// Copyright 2018 clair authors
2015-11-13 19:11:28 +00:00
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2017-01-04 02:44:32 +00:00
// Package ubuntu implements a vulnerability source updater using the
// Ubuntu CVE Tracker.
2016-01-13 21:41:00 +00:00
package ubuntu
2015-11-13 19:11:28 +00:00
import (
"bufio"
2018-07-05 23:11:50 +00:00
"errors"
2015-11-13 19:11:28 +00:00
"fmt"
"io"
"os"
"regexp"
"strings"
2017-05-04 17:21:25 +00:00
log "github.com/sirupsen/logrus"
2016-12-28 01:45:11 +00:00
2015-11-13 19:11:28 +00:00
"github.com/coreos/clair/database"
2016-12-28 01:45:11 +00:00
"github.com/coreos/clair/ext/versionfmt"
2017-01-03 21:00:20 +00:00
"github.com/coreos/clair/ext/versionfmt/dpkg"
2017-01-04 02:44:32 +00:00
"github.com/coreos/clair/ext/vulnsrc"
2018-09-14 21:25:12 +00:00
"github.com/coreos/clair/pkg/gitutil"
2015-11-13 19:11:28 +00:00
)
const (
2018-07-05 23:11:50 +00:00
trackerURI = "https://git.launchpad.net/ubuntu-cve-tracker"
updaterFlag = "ubuntuUpdater"
cveURL = "http://people.ubuntu.com/~ubuntu-security/cve/%s"
2015-11-13 19:11:28 +00:00
)
var (
ubuntuIgnoredReleases = map [ string ] struct { } {
2016-02-25 00:29:36 +00:00
"upstream" : { } ,
"devel" : { } ,
2015-11-13 19:11:28 +00:00
2016-02-25 00:29:36 +00:00
"dapper" : { } ,
"edgy" : { } ,
"feisty" : { } ,
"gutsy" : { } ,
"hardy" : { } ,
"intrepid" : { } ,
"jaunty" : { } ,
"karmic" : { } ,
"lucid" : { } ,
"maverick" : { } ,
"natty" : { } ,
"oneiric" : { } ,
"saucy" : { } ,
2015-11-13 19:11:28 +00:00
2016-02-25 00:29:36 +00:00
"vivid/ubuntu-core" : { } ,
"vivid/stable-phone-overlay" : { } ,
2016-01-25 17:58:51 +00:00
2015-11-13 19:11:28 +00:00
// Syntax error
2016-02-25 00:29:36 +00:00
"Patches" : { } ,
2015-11-13 19:11:28 +00:00
// Product
2016-02-25 00:29:36 +00:00
"product" : { } ,
2015-11-13 19:11:28 +00:00
}
affectsCaptureRegexp = regexp . MustCompile ( ` (?P<release>.*)_(?P<package>.*): (?P<status>[^\s]*)( \(+(?P<note>[^()]*)\)+)? ` )
affectsCaptureRegexpNames = affectsCaptureRegexp . SubexpNames ( )
2018-07-05 23:11:50 +00:00
errUnknownRelease = errors . New ( "found packages with CVEs for a verison of Ubuntu that Clair doesn't know about" )
2015-11-13 19:11:28 +00:00
)
2017-01-04 02:44:32 +00:00
type updater struct {
2016-01-29 16:22:54 +00:00
repositoryLocalPath string
}
2015-11-13 19:11:28 +00:00
func init ( ) {
2017-01-04 02:44:32 +00:00
vulnsrc . RegisterUpdater ( "ubuntu" , & updater { } )
2015-11-13 19:11:28 +00:00
}
2018-07-05 23:11:50 +00:00
func ( u * updater ) Update ( db database . Datastore ) ( resp vulnsrc . UpdateResponse , err error ) {
2017-05-04 17:21:25 +00:00
log . WithField ( "package" , "Ubuntu" ) . Info ( "Start fetching vulnerabilities" )
2015-11-13 19:11:28 +00:00
2018-07-05 23:11:50 +00:00
// Pull the master branch.
var commit string
2018-09-14 21:25:12 +00:00
u . repositoryLocalPath , commit , err = gitutil . CloneOrPull ( trackerURI , u . repositoryLocalPath , updaterFlag )
2015-11-16 21:57:53 +00:00
if err != nil {
return resp , err
}
2018-07-05 23:11:50 +00:00
// Open a database transaction.
tx , err := db . Begin ( )
2017-07-26 23:22:29 +00:00
if err != nil {
return resp , err
}
2018-07-05 23:11:50 +00:00
defer tx . Rollback ( )
2017-07-26 23:22:29 +00:00
2018-07-05 23:11:50 +00:00
// Ask the database for the latest commit we successfully applied.
var dbCommit string
var ok bool
dbCommit , ok , err = tx . FindKeyValue ( updaterFlag )
2015-11-13 19:11:28 +00:00
if err != nil {
2018-07-05 23:11:50 +00:00
return
2015-11-13 19:11:28 +00:00
}
2018-07-05 23:11:50 +00:00
if ! ok {
dbCommit = ""
2017-07-26 23:22:29 +00:00
}
2018-07-05 23:11:50 +00:00
// Set the updaterFlag to equal the commit processed.
resp . FlagName = updaterFlag
resp . FlagValue = commit
// Short-circuit if there have been no updates.
if commit == dbCommit {
log . WithField ( "package" , "ubuntu" ) . Debug ( "no update" )
return
2017-07-26 23:22:29 +00:00
}
2015-11-13 19:11:28 +00:00
// Get the list of vulnerabilities that we have to update.
2018-07-05 23:11:50 +00:00
var modifiedCVE map [ string ] struct { }
modifiedCVE , err = collectModifiedVulnerabilities ( commit , dbCommit , u . repositoryLocalPath )
2015-11-13 19:11:28 +00:00
if err != nil {
2018-07-05 23:11:50 +00:00
return
2015-11-13 19:11:28 +00:00
}
2018-07-05 23:11:50 +00:00
// Get the list of vulnerabilities.
resp . Vulnerabilities , resp . Notes , err = collectVulnerabilitiesAndNotes ( u . repositoryLocalPath , modifiedCVE )
if err != nil {
return
2015-11-13 19:11:28 +00:00
}
2018-07-05 23:11:50 +00:00
// The only notes we take are if we encountered unknown Ubuntu release.
// We don't want the commit to be considered as managed in that case.
if len ( resp . Notes ) != 0 {
resp . FlagValue = dbCommit
2016-01-25 18:19:49 +00:00
}
2015-11-13 19:11:28 +00:00
return
}
2017-01-04 02:44:32 +00:00
func ( u * updater ) Clean ( ) {
2018-07-05 23:11:50 +00:00
if u . repositoryLocalPath != "" {
os . RemoveAll ( u . repositoryLocalPath )
}
2017-01-04 02:44:32 +00:00
}
2018-07-05 23:11:50 +00:00
func collectModifiedVulnerabilities ( commit , dbCommit , repositoryLocalPath string ) ( map [ string ] struct { } , error ) {
modifiedCVE := make ( map [ string ] struct { } )
for _ , dirName := range [ ] string { "active" , "retired" } {
if err := processDirectory ( repositoryLocalPath , dirName , modifiedCVE ) ; err != nil {
return nil , err
}
}
return modifiedCVE , nil
}
func processDirectory ( repositoryLocalPath , dirName string , modifiedCVE map [ string ] struct { } ) error {
// Open the directory.
d , err := os . Open ( repositoryLocalPath + "/" + dirName )
2016-05-25 19:41:12 +00:00
if err != nil {
2018-07-05 23:11:50 +00:00
log . WithError ( err ) . Error ( "could not open Ubuntu vulnerabilities repository's folder" )
return vulnsrc . ErrFilesystem
2016-05-25 19:41:12 +00:00
}
2018-07-05 23:11:50 +00:00
defer d . Close ( )
2017-01-18 02:22:20 +00:00
2018-07-05 23:11:50 +00:00
// Get the FileInfo of all the files in the directory.
names , err := d . Readdirnames ( - 1 )
2016-05-25 19:41:12 +00:00
if err != nil {
2018-07-05 23:11:50 +00:00
log . WithError ( err ) . Error ( "could not read Ubuntu vulnerabilities repository's folder" )
return vulnsrc . ErrFilesystem
2016-05-25 19:41:12 +00:00
}
2017-01-18 02:22:20 +00:00
2018-07-05 23:11:50 +00:00
// Add the vulnerabilities to the list.
for _ , name := range names {
if strings . HasPrefix ( name , "CVE-" ) {
modifiedCVE [ dirName + "/" + name ] = struct { } { }
}
}
return nil
2016-05-25 19:41:12 +00:00
}
2018-07-05 23:11:50 +00:00
func collectVulnerabilitiesAndNotes ( repositoryLocalPath string , modifiedCVE map [ string ] struct { } ) ( [ ] database . VulnerabilityWithAffected , [ ] string , error ) {
vulns := make ( [ ] database . VulnerabilityWithAffected , 0 )
noteSet := make ( map [ string ] struct { } )
for cvePath := range modifiedCVE {
// Open the CVE file.
file , err := os . Open ( repositoryLocalPath + "/" + cvePath )
if err != nil {
// This can happen when a file is modified then moved in another commit.
continue
}
2015-11-13 19:11:28 +00:00
2018-07-05 23:11:50 +00:00
// Parse the vulnerability.
v , unknownReleases , err := parseUbuntuCVE ( file )
if err != nil {
file . Close ( )
return nil , nil , err
2015-11-13 19:11:28 +00:00
}
2018-07-05 23:11:50 +00:00
// Add the vulnerability to the response.
vulns = append ( vulns , v )
2015-11-13 19:11:28 +00:00
2018-07-05 23:11:50 +00:00
// Store any unknown releases as notes.
for k := range unknownReleases {
noteSet [ errUnknownRelease . Error ( ) + ": " + k ] = struct { } { }
}
2015-11-13 19:11:28 +00:00
2018-07-05 23:11:50 +00:00
file . Close ( )
2015-11-13 19:11:28 +00:00
}
2018-07-05 23:11:50 +00:00
// Convert the note set into a slice.
var notes [ ] string
for note := range noteSet {
notes = append ( notes , note )
2015-11-13 19:11:28 +00:00
}
2018-07-05 23:11:50 +00:00
return vulns , notes , nil
2015-11-13 19:11:28 +00:00
}
2017-07-26 23:22:29 +00:00
func parseUbuntuCVE ( fileContent io . Reader ) ( vulnerability database . VulnerabilityWithAffected , unknownReleases map [ string ] struct { } , err error ) {
2015-11-13 19:11:28 +00:00
unknownReleases = make ( map [ string ] struct { } )
readingDescription := false
scanner := bufio . NewScanner ( fileContent )
2017-07-26 23:22:29 +00:00
// only unique major releases will be considered. All sub releases' (e.g.
// precise/esm) features are considered belong to major releases.
uniqueRelease := map [ string ] struct { } { }
2015-11-13 19:11:28 +00:00
for scanner . Scan ( ) {
line := strings . TrimSpace ( scanner . Text ( ) )
// Skip any comments.
if strings . HasPrefix ( line , "#" ) {
continue
}
// Parse the name.
if strings . HasPrefix ( line , "Candidate:" ) {
2016-01-19 16:03:27 +00:00
vulnerability . Name = strings . TrimSpace ( strings . TrimPrefix ( line , "Candidate:" ) )
2016-02-08 18:37:52 +00:00
vulnerability . Link = fmt . Sprintf ( cveURL , vulnerability . Name )
2015-11-13 19:11:28 +00:00
continue
}
// Parse the priority.
if strings . HasPrefix ( line , "Priority:" ) {
priority := strings . TrimSpace ( strings . TrimPrefix ( line , "Priority:" ) )
// Handle syntax error: Priority: medium (heap-protector)
if strings . Contains ( priority , " " ) {
priority = priority [ : strings . Index ( priority , " " ) ]
}
2017-01-15 15:52:13 +00:00
vulnerability . Severity = SeverityFromPriority ( priority )
2015-11-13 19:11:28 +00:00
continue
}
// Parse the description.
if strings . HasPrefix ( line , "Description:" ) {
readingDescription = true
vulnerability . Description = strings . TrimSpace ( strings . TrimPrefix ( line , "Description:" ) ) // In case there is a formatting error and the description starts on the same line
continue
}
if readingDescription {
if strings . HasPrefix ( line , "Ubuntu-Description:" ) || strings . HasPrefix ( line , "Notes:" ) || strings . HasPrefix ( line , "Bugs:" ) || strings . HasPrefix ( line , "Priority:" ) || strings . HasPrefix ( line , "Discovered-by:" ) || strings . HasPrefix ( line , "Assigned-to:" ) {
readingDescription = false
} else {
vulnerability . Description = vulnerability . Description + " " + line
continue
}
}
// Try to parse the package that the vulnerability affects.
affectsCaptureArr := affectsCaptureRegexp . FindAllStringSubmatch ( line , - 1 )
if len ( affectsCaptureArr ) > 0 {
affectsCapture := affectsCaptureArr [ 0 ]
md := map [ string ] string { }
for i , n := range affectsCapture {
md [ affectsCaptureRegexpNames [ i ] ] = strings . TrimSpace ( n )
}
// Ignore Linux kernels.
if strings . HasPrefix ( md [ "package" ] , "linux" ) {
continue
}
2016-01-19 16:03:27 +00:00
// Only consider the package if its status is needed, active, deferred, not-affected or
// released. Ignore DNE (package does not exist), needs-triage, ignored, pending.
if md [ "status" ] == "needed" || md [ "status" ] == "active" || md [ "status" ] == "deferred" || md [ "status" ] == "released" || md [ "status" ] == "not-affected" {
2017-07-26 23:22:29 +00:00
md [ "release" ] = strings . Split ( md [ "release" ] , "/" ) [ 0 ]
2015-11-13 19:11:28 +00:00
if _ , isReleaseIgnored := ubuntuIgnoredReleases [ md [ "release" ] ] ; isReleaseIgnored {
continue
}
if _ , isReleaseKnown := database . UbuntuReleasesMapping [ md [ "release" ] ] ; ! isReleaseKnown {
unknownReleases [ md [ "release" ] ] = struct { } { }
continue
}
2016-12-28 01:45:11 +00:00
var version string
2015-11-13 19:11:28 +00:00
if md [ "status" ] == "released" {
if md [ "note" ] != "" {
var err error
2017-01-03 21:00:20 +00:00
err = versionfmt . Valid ( dpkg . ParserName , md [ "note" ] )
2015-11-13 19:11:28 +00:00
if err != nil {
2017-05-04 17:21:25 +00:00
log . WithError ( err ) . WithField ( "version" , md [ "note" ] ) . Warning ( "could not parse package version. skipping" )
2015-11-13 19:11:28 +00:00
}
2016-12-28 01:45:11 +00:00
version = md [ "note" ]
2015-11-13 19:11:28 +00:00
}
} else {
2016-12-28 01:45:11 +00:00
version = versionfmt . MaxVersion
2015-11-13 19:11:28 +00:00
}
2016-12-28 01:45:11 +00:00
if version == "" {
2015-11-13 19:11:28 +00:00
continue
}
2017-07-26 23:22:29 +00:00
releaseName := "ubuntu:" + database . UbuntuReleasesMapping [ md [ "release" ] ]
if _ , ok := uniqueRelease [ releaseName + "_:_" + md [ "package" ] ] ; ok {
continue
}
uniqueRelease [ releaseName + "_:_" + md [ "package" ] ] = struct { } { }
var fixedinVersion string
if version == versionfmt . MaxVersion {
fixedinVersion = ""
} else {
fixedinVersion = version
}
2015-11-13 19:11:28 +00:00
// Create and add the new package.
2017-07-26 23:22:29 +00:00
featureVersion := database . AffectedFeature {
Namespace : database . Namespace {
Name : releaseName ,
VersionFormat : dpkg . ParserName ,
2016-01-19 16:03:27 +00:00
} ,
2017-07-26 23:22:29 +00:00
FeatureName : md [ "package" ] ,
AffectedVersion : version ,
FixedInVersion : fixedinVersion ,
2015-12-01 19:58:17 +00:00
}
2017-07-26 23:22:29 +00:00
vulnerability . Affected = append ( vulnerability . Affected , featureVersion )
2015-11-13 19:11:28 +00:00
}
}
}
// Trim extra spaces in the description
vulnerability . Description = strings . TrimSpace ( vulnerability . Description )
// If no link has been provided (CVE-2006-NNN0 for instance), add the link to the tracker
if vulnerability . Link == "" {
2016-02-08 18:37:52 +00:00
vulnerability . Link = trackerURI
2015-11-13 19:11:28 +00:00
}
// If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown
2016-01-20 00:17:08 +00:00
if vulnerability . Severity == "" {
2017-01-19 18:42:37 +00:00
vulnerability . Severity = database . UnknownSeverity
2015-11-13 19:11:28 +00:00
}
return
}
2017-01-15 15:52:13 +00:00
// SeverityFromPriority converts an priority from the Ubuntu CVE Tracker into
2017-01-19 18:42:37 +00:00
// a database.Severity.
func SeverityFromPriority ( priority string ) database . Severity {
2015-11-13 19:11:28 +00:00
switch priority {
case "untriaged" :
2017-01-19 18:42:37 +00:00
return database . UnknownSeverity
2015-11-13 19:11:28 +00:00
case "negligible" :
2017-01-19 18:42:37 +00:00
return database . NegligibleSeverity
2015-11-13 19:11:28 +00:00
case "low" :
2017-01-19 18:42:37 +00:00
return database . LowSeverity
2015-11-13 19:11:28 +00:00
case "medium" :
2017-01-19 18:42:37 +00:00
return database . MediumSeverity
2015-11-13 19:11:28 +00:00
case "high" :
2017-01-19 18:42:37 +00:00
return database . HighSeverity
2015-11-13 19:11:28 +00:00
case "critical" :
2017-01-19 18:42:37 +00:00
return database . CriticalSeverity
2017-01-15 15:52:13 +00:00
default :
2018-09-05 22:31:36 +00:00
log . Warningf ( "could not determine a vulnerability severity from: %s" , priority )
2017-01-19 18:42:37 +00:00
return database . UnknownSeverity
2015-11-13 19:11:28 +00:00
}
}