From f57387fb6b690b8064583388162a6996b29af4bd Mon Sep 17 00:00:00 2001 From: Flavio Castelli Date: Tue, 26 Sep 2017 00:32:16 +0200 Subject: [PATCH] Reintroduce image scanning for openSUSE and SLE Handle scanning of openSUSE and SUSE Linux Enterprise images. --- cmd/clair/main.go | 1 + ext/featurens/osrelease/osrelease.go | 2 +- ext/vulnsrc/suse/suse.go | 464 ++++++++++++++++++ ext/vulnsrc/suse/suse_test.go | 154 ++++++ .../suse/testdata/fetcher_opensuse_test.1.xml | 66 +++ .../suse/testdata/fetcher_sle_test.1.xml | 69 +++ 6 files changed, 755 insertions(+), 1 deletion(-) create mode 100644 ext/vulnsrc/suse/suse.go create mode 100644 ext/vulnsrc/suse/suse_test.go create mode 100644 ext/vulnsrc/suse/testdata/fetcher_opensuse_test.1.xml create mode 100644 ext/vulnsrc/suse/testdata/fetcher_sle_test.1.xml diff --git a/cmd/clair/main.go b/cmd/clair/main.go index efed24ba..9406dfd7 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -54,6 +54,7 @@ import ( _ "github.com/coreos/clair/ext/vulnsrc/debian" _ "github.com/coreos/clair/ext/vulnsrc/oracle" _ "github.com/coreos/clair/ext/vulnsrc/rhel" + _ "github.com/coreos/clair/ext/vulnsrc/suse" _ "github.com/coreos/clair/ext/vulnsrc/ubuntu" ) diff --git a/ext/featurens/osrelease/osrelease.go b/ext/featurens/osrelease/osrelease.go index 1139739d..ec8dc490 100644 --- a/ext/featurens/osrelease/osrelease.go +++ b/ext/featurens/osrelease/osrelease.go @@ -84,7 +84,7 @@ func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { switch OS { case "debian", "ubuntu": versionFormat = dpkg.ParserName - case "centos", "rhel", "fedora", "amzn", "ol", "oracle": + case "centos", "rhel", "fedora", "amzn", "ol", "oracle", "opensuse", "sles": versionFormat = rpm.ParserName default: return nil, nil diff --git a/ext/vulnsrc/suse/suse.go b/ext/vulnsrc/suse/suse.go new file mode 100644 index 00000000..e2cef03f --- /dev/null +++ b/ext/vulnsrc/suse/suse.go @@ -0,0 +1,464 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package suse implements a vulnerability source updater using the +// SUSE Linux and openSUSE OVAL Database. +package suse + +import ( + "bufio" + "encoding/xml" + "io" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + log "github.com/sirupsen/logrus" + + "fmt" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/versionfmt" + "github.com/coreos/clair/ext/versionfmt/rpm" + "github.com/coreos/clair/ext/vulnsrc" + "github.com/coreos/clair/pkg/commonerr" +) + +const ( + ovalURI = "http://ftp.suse.com/pub/projects/security/oval/" +) + +var ( + ignoredCriterions []string + suseOpenSUSEInstalledCommentRegexp = regexp.MustCompile(`(SUSE Linux Enterprise |openSUSE ).*is installed`) + suseInstalledCommentRegexp = regexp.MustCompile(`SUSE Linux Enterprise[A-Za-z\s]*? (\d+)[\w\s]*?(SP(\d+))? is installed`) +) + +type oval struct { + Timestamp string `xml:"generator>timestamp"` + Definitions []definition `xml:"definitions>definition"` +} + +type definition struct { + Title string `xml:"metadata>title"` + Description string `xml:"metadata>description"` + References []reference `xml:"metadata>reference"` + Criteria criteria `xml:"criteria"` +} + +type reference struct { + Source string `xml:"source,attr"` + URI string `xml:"ref_url,attr"` +} + +type criteria struct { + Operator string `xml:"operator,attr"` + Criterias []*criteria `xml:"criteria"` + Criterions []criterion `xml:"criterion"` +} + +type criterion struct { + Comment string `xml:"comment,attr"` +} + +type flavor int + +const ( + SUSE flavor = iota + OpenSUSE +) + +type updater struct { + Name string + NamespaceName string + FilePrefix string + UpdaterFlag string + FileRegexp *regexp.Regexp +} + +func newUpdater(f flavor) updater { + var up updater + + switch f { + case SUSE: + up.Name = "SUSE Linux" + up.NamespaceName = "sles" + up.FilePrefix = "suse.linux.enterprise." + up.UpdaterFlag = "SUSEUpdater" + up.FileRegexp = regexp.MustCompile(`suse.linux.enterprise.(\d+).xml`) + case OpenSUSE: + up.Name = "openSUSE" + up.NamespaceName = "opensuse" + up.FilePrefix = "opensuse.leap." + up.UpdaterFlag = "openSUSEUpdater" + up.FileRegexp = regexp.MustCompile(`opensuse.leap.(\d+\.*\d*).xml`) + default: + panic("Unrecognized flavor") + } + + return up +} + +func init() { + suseUpdater := newUpdater(SUSE) + openSUSEUpdater := newUpdater(OpenSUSE) + vulnsrc.RegisterUpdater("suse", &suseUpdater) + vulnsrc.RegisterUpdater("opensuse", &openSUSEUpdater) +} + +func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { + log.WithField("package", u.Name).Info("Start fetching vulnerabilities") + + // openSUSE and SUSE have one single xml file for all the products, there are no incremental + // xml files. We store into the database the value of the generation timestamp + // of the latest file we parsed. + flagValue, err := datastore.GetKeyValue(u.UpdaterFlag) + if err != nil { + return resp, err + } + log.WithField("flagvalue", flagValue) + + if flagValue == "" { + flagValue = "0" + } + + // this contains the modification time of the most recent + // file expressed as unix time (int64) + latestOval, _ := strconv.ParseInt(flagValue, 10, 64) + + // Fetch the update list. + r, err := http.Get(ovalURI) + if err != nil { + log.Fatal(err) + } + defer r.Body.Close() + + var ovalFiles []string + var generationTimes []int64 + + scanner := bufio.NewScanner(r.Body) + for scanner.Scan() { + line := scanner.Text() + r := u.FileRegexp.FindStringSubmatch(line) + if len(r) != 2 { + continue + } + + ovalFile := ovalURI + u.FilePrefix + r[1] + ".xml" + log.WithFields( + log.Fields{ + "ovalFile": ovalFile, + "updater": u.Name, + }).Debug("file to check") + + // do not fetch the entire file to get the value of the + // creation time. Rely on the "latest modified time" + // value of the file hosted on the remote server. + timestamp, err := getLatestModifiedTime(ovalFile) + if err != nil { + log.WithError(err).WithField("ovalFile", ovalFile).Warning("Ignoring OVAL file") + } + + if timestamp > latestOval { + ovalFiles = append(ovalFiles, ovalFile) + } + } + + for _, oval := range ovalFiles { + // Download the oval XML file. + r, err := http.Get(oval) + if err != nil { + log.WithError(err).Error("could not download", u.Name, "update list") + return resp, commonerr.ErrCouldNotDownload + } + + match := u.FileRegexp.FindStringSubmatch(oval) + if len(match) != 2 { + log.Error("Skipping ", oval, "because it's not possible to extract osVersion") + continue + } + osVersion := match[1] + + // Parse the XML. + vs, generationTime, err := parseOval(r.Body, u.NamespaceName, osVersion) + if err != nil { + return resp, err + } + generationTimes = append(generationTimes, generationTime) + + // Collect vulnerabilities. + for _, v := range vs { + resp.Vulnerabilities = append(resp.Vulnerabilities, v) + } + } + + // Set the flag if we found anything. + if len(generationTimes) > 0 { + resp.FlagName = u.UpdaterFlag + resp.FlagValue = strconv.FormatInt(latest(generationTimes), 10) + } else { + log.WithField("package", u.Name).Debug("no update") + } + + return resp, nil +} + +// Get the latest modification time of a remote file +// expressed as unix time +func getLatestModifiedTime(url string) (int64, error) { + resp, err := http.Head(url) + if err != nil { + return 0, err + } + + last_modified := resp.Header.Get("Last-Modified") + if len(last_modified) == 0 { + return 0, fmt.Errorf("last modified header missing") + } + + // "Thu, 30 Nov 2017 03:07:57 GMT + layout := "Mon, 2 Jan 2006 15:04:05 MST" + timestamp, err := time.Parse(layout, last_modified) + if err != nil { + return 0, err + } + + return timestamp.Unix(), nil +} + +func latest(values []int64) (ret int64) { + for _, element := range values { + if element > ret { + ret = element + } + } + return +} + +func (u *updater) Clean() {} + +func parseOval(ovalReader io.Reader, osFlavor, osVersion string) (vulnerabilities []database.Vulnerability, generationTime int64, err error) { + // Decode the XML. + var ov oval + err = xml.NewDecoder(ovalReader).Decode(&ov) + if err != nil { + log.WithError(err).Error("could not decode XML") + err = commonerr.ErrCouldNotParse + return + } + + // timestamp format 2017-10-23T04:07:14 + layout := "2006-1-2T15:04:05" + timestamp, err := time.Parse(layout, ov.Timestamp) + if err != nil { + return + } + generationTime = timestamp.Unix() + + // Iterate over the definitions and collect any vulnerabilities that affect + // at least one package. + for _, definition := range ov.Definitions { + pkgs := toFeatureVersions(definition.Criteria, osFlavor, osVersion) + if len(pkgs) > 0 { + vulnerability := database.Vulnerability{ + Name: name(definition), + Link: link(definition), + //TODO: handle that once openSUSE/SLE OVAL files have severity info + Severity: database.UnknownSeverity, + Description: description(definition), + } + for _, p := range pkgs { + vulnerability.FixedIn = append(vulnerability.FixedIn, p) + } + vulnerabilities = append(vulnerabilities, vulnerability) + } + } + + return +} + +func getCriterions(node criteria) [][]criterion { + // Filter useless criterions. + var criterions []criterion + for _, c := range node.Criterions { + ignored := false + + for _, ignoredItem := range ignoredCriterions { + if strings.Contains(c.Comment, ignoredItem) { + ignored = true + break + } + } + + if !ignored { + criterions = append(criterions, c) + } + } + + if node.Operator == "AND" { + return [][]criterion{criterions} + } else if node.Operator == "OR" { + var possibilities [][]criterion + for _, c := range criterions { + possibilities = append(possibilities, []criterion{c}) + } + return possibilities + } + + return [][]criterion{} +} + +func getPossibilities(node criteria) [][]criterion { + if len(node.Criterias) == 0 { + return getCriterions(node) + } + + var possibilitiesToCompose [][][]criterion + for _, criteria := range node.Criterias { + possibilitiesToCompose = append(possibilitiesToCompose, getPossibilities(*criteria)) + } + if len(node.Criterions) > 0 { + possibilitiesToCompose = append(possibilitiesToCompose, getCriterions(node)) + } + + var possibilities [][]criterion + if node.Operator == "AND" { + for _, possibility := range possibilitiesToCompose[0] { + possibilities = append(possibilities, possibility) + } + + for _, possibilityGroup := range possibilitiesToCompose[1:] { + var newPossibilities [][]criterion + + for _, possibility := range possibilities { + for _, possibilityInGroup := range possibilityGroup { + var p []criterion + p = append(p, possibility...) + p = append(p, possibilityInGroup...) + newPossibilities = append(newPossibilities, p) + } + } + + possibilities = newPossibilities + } + } else if node.Operator == "OR" { + for _, possibilityGroup := range possibilitiesToCompose { + for _, possibility := range possibilityGroup { + possibilities = append(possibilities, possibility) + } + } + } + + return possibilities +} + +func toFeatureVersions(criteria criteria, osFlavor, osVersion string) []database.FeatureVersion { + // There are duplicates in SUSE .xml files. + // This map is for deduplication. + featureVersionParameters := make(map[string]database.FeatureVersion) + + possibilities := getPossibilities(criteria) + for _, criterions := range possibilities { + var featureVersion database.FeatureVersion + + // Attempt to parse package data from trees of criterions. + for _, c := range criterions { + if match := suseInstalledCommentRegexp.FindStringSubmatch(c.Comment); match != nil { + if len(match) != 4 { + log.WithField("comment", c.Comment).Warning("could not extract sles name and version from comment") + } else { + osVersion = match[1] + if match[3] != "" { + osVersion = fmt.Sprintf("%s.%s", osVersion, match[3]) + } + } + } + + if suseOpenSUSEInstalledCommentRegexp.FindStringSubmatch(c.Comment) == nil && strings.HasSuffix(c.Comment, " is installed") { + name, version, err := splitPackageNameAndVersion(c.Comment[:len(c.Comment)-13]) + if err != nil { + log.WithError(err).WithField("comment", c.Comment).Warning("Could not extract package name and version from comment") + } else { + featureVersion.Feature.Name = name + version := version + err := versionfmt.Valid(rpm.ParserName, version) + if err != nil { + log.WithError(err).WithField("version", version).Warning("could not parse package version. skipping") + } else { + if version != versionfmt.MaxVersion { + featureVersion.Version = version + } + } + } + } + } + + featureVersion.Feature.Namespace.Name = fmt.Sprintf("%s:%s", osFlavor, osVersion) + featureVersion.Feature.Namespace.VersionFormat = rpm.ParserName + + if featureVersion.Feature.Namespace.Name != "" && featureVersion.Feature.Name != "" && featureVersion.Version != "" { + featureVersionParameters[featureVersion.Feature.Namespace.Name+":"+featureVersion.Feature.Name] = featureVersion + } else { + log.WithField("criterions", fmt.Sprintf("%v", criterions)).Warning("could not determine a valid package from criterions") + } + } + + // Convert the map to slice. + var featureVersionParametersArray []database.FeatureVersion + for _, fv := range featureVersionParameters { + featureVersionParametersArray = append(featureVersionParametersArray, fv) + } + + return featureVersionParametersArray +} + +func description(def definition) (desc string) { + // It is much more faster to proceed like this than using a Replacer. + desc = strings.Replace(def.Description, "\n\n\n", " ", -1) + desc = strings.Replace(desc, "\n\n", " ", -1) + desc = strings.Replace(desc, "\n", " ", -1) + return +} + +func name(def definition) string { + return strings.TrimSpace(def.Title) +} + +func link(def definition) (link string) { + for _, reference := range def.References { + if reference.Source == "CVE" { + link = reference.URI + break + } + } + + return +} + +func splitPackageNameAndVersion(fullname string) (name, version string, err error) { + re := regexp.MustCompile(`-\d+\.`) + + matches := re.FindStringSubmatchIndex(fullname) + + if matches == nil { + err = fmt.Errorf("Cannot extract package name and version from %s", fullname) + } else { + name = fullname[:matches[0]] + version = fullname[matches[0]+1:] + } + + return +} diff --git a/ext/vulnsrc/suse/suse_test.go b/ext/vulnsrc/suse/suse_test.go new file mode 100644 index 00000000..d10d0588 --- /dev/null +++ b/ext/vulnsrc/suse/suse_test.go @@ -0,0 +1,154 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package suse + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/versionfmt/rpm" + "github.com/stretchr/testify/assert" +) + +func TestOpenSUSEParser(t *testing.T) { + _, filename, _, _ := runtime.Caller(0) + path := filepath.Join(filepath.Dir(filename)) + + // Test parsing testdata/fetcher_opensuse_test.1.xml + testFile, _ := os.Open(path + "/testdata/fetcher_opensuse_test.1.xml") + defer testFile.Close() + + u := newUpdater(OpenSUSE) + osVersion := "42.3" + + vulnerabilities, generationTime, err := parseOval(testFile, u.NamespaceName, osVersion) + assert.Nil(t, err) + assert.Equal(t, int64(1467000286), generationTime) + + if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { + assert.Equal(t, "CVE-2012-2150", vulnerabilities[0].Name) + assert.Equal(t, "http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2012-2150", vulnerabilities[0].Link) + assert.Equal(t, `xfs_metadump in xfsprogs before 3.2.4 does not properly obfuscate file data, which allows remote attackers to obtain sensitive information by reading a generated image.`, vulnerabilities[0].Description) + + expectedFeatureVersions := []database.FeatureVersion{ + { + Feature: database.Feature{ + Namespace: database.Namespace{ + Name: fmt.Sprintf("%s:%s", u.NamespaceName, osVersion), + VersionFormat: rpm.ParserName, + }, + Name: "xfsprogs", + }, + Version: "3.2.1-5.1", + }, + { + Feature: database.Feature{ + Namespace: database.Namespace{ + Name: fmt.Sprintf("%s:%s", u.NamespaceName, osVersion), + VersionFormat: rpm.ParserName, + }, + Name: "xfsprogs-devel", + }, + Version: "3.2.1-5.1", + }, + } + + for _, expectedFeatureVersion := range expectedFeatureVersions { + assert.Contains(t, vulnerabilities[0].FixedIn, expectedFeatureVersion) + } + } + +} + +func TestSUSEParser(t *testing.T) { + _, filename, _, _ := runtime.Caller(0) + path := filepath.Join(filepath.Dir(filename)) + + // Test parsing testdata/fetcher_opensuse_test.1.xml + testFile, _ := os.Open(path + "/testdata/fetcher_sle_test.1.xml") + defer testFile.Close() + + u := newUpdater(SUSE) + osVersion := "12" + + vulnerabilities, generationTime, err := parseOval(testFile, u.NamespaceName, osVersion) + assert.Nil(t, err) + assert.Equal(t, int64(1467000286), generationTime) + + if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { + assert.Equal(t, "CVE-2012-2150", vulnerabilities[0].Name) + assert.Equal(t, "http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2012-2150", vulnerabilities[0].Link) + assert.Equal(t, `xfs_metadump in xfsprogs before 3.2.4 does not properly obfuscate file data, which allows remote attackers to obtain sensitive information by reading a generated image.`, vulnerabilities[0].Description) + + expectedFeatureVersions := []database.FeatureVersion{ + { + Feature: database.Feature{ + Namespace: database.Namespace{ + Name: fmt.Sprintf("%s:%s", u.NamespaceName, osVersion), + VersionFormat: rpm.ParserName, + }, + Name: "xfsprogs", + }, + Version: "3.2.1-3.5", + }, + { + Feature: database.Feature{ + Namespace: database.Namespace{ + Name: "sles:12.1", + VersionFormat: rpm.ParserName, + }, + Name: "xfsprogs", + }, + Version: "3.2.1-3.5", + }, + } + + for _, expectedFeatureVersion := range expectedFeatureVersions { + assert.Contains(t, vulnerabilities[0].FixedIn, expectedFeatureVersion) + } + } +} + +func TestPkgInstalledCommentRegexp(t *testing.T) { + testData := map[string][]string{ + "krb5-1.12.1-19.1 is installed": []string{"krb5", "1.12.1-19.1"}, + "krb5-32bit-1.12.1-19.1 is installed": []string{"krb5-32bit", "1.12.1-19.1"}, + "krb5-client-1.12.1-19.1 is installed": []string{"krb5-client", "1.12.1-19.1"}, + "krb5-plugin-kdb-ldap-1.12.1-19.1 is installed": []string{"krb5-plugin-kdb-ldap", "1.12.1-19.1"}, + "sysvinit-tools-2.88+-96.1 is installed": []string{"sysvinit-tools", "2.88+-96.1"}, + "ntp-4.2.8p10-63.3 is installed": []string{"ntp", "4.2.8p10-63.3"}, + "libid3tag0-0.15.1b-182.58 is installed": []string{"libid3tag0", "0.15.1b-182.58"}, + "libopenssl-devel-1.0.2j-55.1 is installed": []string{"libopenssl-devel", "1.0.2j-55.1"}, + "libMagickCore-6_Q16-1-6.8.8.1-5.8 is installed": []string{"libMagickCore-6_Q16-1", "6.8.8.1-5.8"}, + "libGraphicsMagick++-Q16-12-1.3.25-11.44.1 is installed": []string{"libGraphicsMagick++-Q16-12", "1.3.25-11.44.1"}, + "freerdp-2.0.0~git.1463131968.4e66df7-11.69 is installed": []string{"freerdp", "2.0.0~git.1463131968.4e66df7-11.69"}, + "libfreerdp2-2.0.0~git.1463131968.4e66df7-11.69 is installed": []string{"libfreerdp2", "2.0.0~git.1463131968.4e66df7-11.69"}, + "ruby2.1-rubygem-sle2docker-0.2.3-5.1 is installed": []string{"ruby2.1-rubygem-sle2docker", "0.2.3-5.1"}, + "xen-libs-4.4.1_06-2.2 is installed": []string{"xen-libs", "4.4.1_06-2.2"}, + "runc-0.1.1+gitr2816_02f8fa7 is installed": []string{"runc", "0.1.1+gitr2816_02f8fa7"}, + } + + for pkg, expectations := range testData { + name, version, err := splitPackageNameAndVersion(pkg[:len(pkg)-len(" is installed")]) + assert.Nil(t, err) + assert.Equal(t, expectations[0], name) + assert.Equal(t, expectations[1], version) + } + +} diff --git a/ext/vulnsrc/suse/testdata/fetcher_opensuse_test.1.xml b/ext/vulnsrc/suse/testdata/fetcher_opensuse_test.1.xml new file mode 100644 index 00000000..258f8254 --- /dev/null +++ b/ext/vulnsrc/suse/testdata/fetcher_opensuse_test.1.xml @@ -0,0 +1,66 @@ + + + + + Marcus Updateinfo to OVAL Converter + 5.5 + 2016-06-27T04:04:46 + + + + + CVE-2012-2150 + + openSUSE Leap 42.1 + + + xfs_metadump in xfsprogs before 3.2.4 does not properly obfuscate file data, which allows remote attackers to obtain sensitive information by reading a generated image. + + + + + + + + + + + + + + + + + + + + + + + + + + + xfsprogs-devel + + + openSUSE-release + + + xfsprogs + + + + + 0:3.2.1-5.1 + + + 42.1 + + + diff --git a/ext/vulnsrc/suse/testdata/fetcher_sle_test.1.xml b/ext/vulnsrc/suse/testdata/fetcher_sle_test.1.xml new file mode 100644 index 00000000..e718def9 --- /dev/null +++ b/ext/vulnsrc/suse/testdata/fetcher_sle_test.1.xml @@ -0,0 +1,69 @@ + + + + + Marcus Updateinfo to OVAL Converter + 5.5 + 2016-06-27T04:04:46 + + + + + CVE-2012-2150 + + SUSE Linux Enterprise Server 12 + + + xfs_metadump in xfsprogs before 3.2.4 does not properly obfuscate file data, which allows remote attackers to obtain sensitive information by reading a generated image. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + sles-release + + + xfsprogs + + + + + 0:3.2.1-3.5 + + + 12 + + + 12.1 + + +