From c6c8fce39a5c28645b9626bc3774bd6b6aadd427 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 19 Feb 2019 16:32:24 -0500 Subject: [PATCH 01/12] pgsql: Add feature_type to initial schema feature_type is for differentiating the binary packages and source packages. --- .../pgsql/migrations/00001_initial_schema.go | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/database/pgsql/migrations/00001_initial_schema.go b/database/pgsql/migrations/00001_initial_schema.go index c073e286..3c74bd14 100644 --- a/database/pgsql/migrations/00001_initial_schema.go +++ b/database/pgsql/migrations/00001_initial_schema.go @@ -19,7 +19,12 @@ var ( // the ancestry. entities = MigrationQuery{ Up: []string{ - // namespaces + `CREATE TABLE IF NOT EXISTS feature_type ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE);`, + + `INSERT INTO feature_type(name) VALUES ('source'), ('binary')`, + `CREATE TABLE IF NOT EXISTS namespace ( id SERIAL PRIMARY KEY, name TEXT NULL, @@ -27,13 +32,13 @@ var ( UNIQUE (name, version_format));`, `CREATE INDEX ON namespace(name);`, - // features `CREATE TABLE IF NOT EXISTS feature ( id SERIAL PRIMARY KEY, name TEXT NOT NULL, version TEXT NOT NULL, version_format TEXT NOT NULL, - UNIQUE (name, version, version_format));`, + type INT REFERENCES feature_type ON DELETE CASCADE, + UNIQUE (name, version, version_format, type));`, `CREATE INDEX ON feature(name);`, `CREATE TABLE IF NOT EXISTS namespaced_feature ( @@ -43,17 +48,15 @@ var ( UNIQUE (namespace_id, feature_id));`, }, Down: []string{ - `DROP TABLE IF EXISTS namespace, feature, namespaced_feature CASCADE;`, + `DROP TABLE IF EXISTS namespace, feature, namespaced_feature, feature_type CASCADE;`, }, } // detector is analysis extensions used by the worker. detector = MigrationQuery{ Up: []string{ - // Detector Type `CREATE TYPE detector_type AS ENUM ('namespace', 'feature');`, - // Detector `CREATE TABLE IF NOT EXISTS detector ( id SERIAL PRIMARY KEY, name TEXT NOT NULL, @@ -70,7 +73,6 @@ var ( // layer contains all metadata and scanned features and namespaces. layer = MigrationQuery{ Up: []string{ - // layers `CREATE TABLE IF NOT EXISTS layer( id SERIAL PRIMARY KEY, hash TEXT NOT NULL UNIQUE);`, @@ -107,7 +109,6 @@ var ( // layers. ancestry = MigrationQuery{ Up: []string{ - // ancestry `CREATE TABLE IF NOT EXISTS ancestry ( id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE);`, @@ -145,7 +146,6 @@ var ( Up: []string{ `CREATE TYPE severity AS ENUM ('Unknown', 'Negligible', 'Low', 'Medium', 'High', 'Critical', 'Defcon1');`, - // vulnerability `CREATE TABLE IF NOT EXISTS vulnerability ( id SERIAL PRIMARY KEY, namespace_id INT REFERENCES Namespace, @@ -159,13 +159,18 @@ var ( `CREATE INDEX ON vulnerability(namespace_id, name);`, `CREATE INDEX ON vulnerability(namespace_id);`, + // vulnerability_affected_feature is a de-normalized table to store + // the affected features in a independent place other than the + // feature table to reduce table lock issue, and makes it easier for + // decoupling updater and the Clair main logic. `CREATE TABLE IF NOT EXISTS vulnerability_affected_feature ( id SERIAL PRIMARY KEY, vulnerability_id INT REFERENCES vulnerability ON DELETE CASCADE, feature_name TEXT NOT NULL, + feature_type INT NOT NULL REFERENCES feature_type ON DELETE CASCADE, affected_version TEXT, fixedin TEXT);`, - `CREATE INDEX ON vulnerability_affected_feature(vulnerability_id, feature_name);`, + `CREATE INDEX ON vulnerability_affected_feature(vulnerability_id, feature_name, feature_type);`, `CREATE TABLE IF NOT EXISTS vulnerability_affected_namespaced_feature( id SERIAL PRIMARY KEY, @@ -176,8 +181,8 @@ var ( `CREATE INDEX ON vulnerability_affected_namespaced_feature(namespaced_feature_id);`, }, Down: []string{ - `DROP TYPE IF EXISTS severity;`, `DROP TABLE IF EXISTS vulnerability, vulnerability_affected_feature, vulnerability_affected_namespaced_feature CASCADE;`, + `DROP TYPE IF EXISTS severity;`, }, } From 00eed77b451b8913771feef7a40067dd246d7872 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 19 Feb 2019 16:33:27 -0500 Subject: [PATCH 02/12] database: Add feature_type database model --- database/affected_feature_type.go | 26 -------------- database/feature_type.go | 52 ++++++++++++++++++++++++++++ database/pgsql/feature_type.go | 53 +++++++++++++++++++++++++++++ database/pgsql/feature_type_test.go | 38 +++++++++++++++++++++ 4 files changed, 143 insertions(+), 26 deletions(-) delete mode 100644 database/affected_feature_type.go create mode 100644 database/feature_type.go create mode 100644 database/pgsql/feature_type.go create mode 100644 database/pgsql/feature_type_test.go diff --git a/database/affected_feature_type.go b/database/affected_feature_type.go deleted file mode 100644 index 950ddeae..00000000 --- a/database/affected_feature_type.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2018 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package database - -// AffectedFeatureType indicates the type of feature that a vulnerability -// affects. -type AffectedFeatureType string - -const ( - // AffectSourcePackage indicates the vulnerability affects a source package. - AffectSourcePackage AffectedFeatureType = "source" - // AffectBinaryPackage indicates the vulnerability affects a binary package. - AffectBinaryPackage AffectedFeatureType = "binary" -) diff --git a/database/feature_type.go b/database/feature_type.go new file mode 100644 index 00000000..dea59fb2 --- /dev/null +++ b/database/feature_type.go @@ -0,0 +1,52 @@ +// Copyright 2019 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "database/sql/driver" + "fmt" +) + +// FeatureType indicates the type of feature that a vulnerability +// affects. +type FeatureType string + +const ( + SourcePackage FeatureType = "source" + BinaryPackage FeatureType = "binary" +) + +var featureTypes = []FeatureType{ + SourcePackage, + BinaryPackage, +} + +// Scan implements the database/sql.Scanner interface. +func (t *FeatureType) Scan(value interface{}) error { + val := value.(string) + for _, ft := range featureTypes { + if string(ft) == val { + *t = ft + return nil + } + } + + panic(fmt.Sprintf("invalid feature type received from database: '%s'", val)) +} + +// Value implements the database/sql/driver.Valuer interface. +func (t *FeatureType) Value() (driver.Value, error) { + return string(*t), nil +} diff --git a/database/pgsql/feature_type.go b/database/pgsql/feature_type.go new file mode 100644 index 00000000..bccf0cd8 --- /dev/null +++ b/database/pgsql/feature_type.go @@ -0,0 +1,53 @@ +// Copyright 2019 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pgsql + +import "github.com/coreos/clair/database" + +const ( + selectAllFeatureTypes = `SELECT id, name FROM feature_type` +) + +type featureTypes struct { + byID map[int]database.FeatureType + byName map[database.FeatureType]int +} + +func newFeatureTypes() *featureTypes { + return &featureTypes{make(map[int]database.FeatureType), make(map[database.FeatureType]int)} +} + +func (tx *pgSession) getFeatureTypeMap() (*featureTypes, error) { + rows, err := tx.Query(selectAllFeatureTypes) + if err != nil { + return nil, err + } + + types := newFeatureTypes() + for rows.Next() { + var ( + id int + name database.FeatureType + ) + if err := rows.Scan(&id, &name); err != nil { + return nil, err + } + + types.byID[id] = name + types.byName[name] = id + } + + return types, nil +} diff --git a/database/pgsql/feature_type_test.go b/database/pgsql/feature_type_test.go new file mode 100644 index 00000000..f8cbf732 --- /dev/null +++ b/database/pgsql/feature_type_test.go @@ -0,0 +1,38 @@ +// Copyright 2019 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pgsql + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/coreos/clair/database" +) + +func TestGetFeatureTypeMap(t *testing.T) { + tx, cleanup := createTestPgSession(t, "TestGetFeatureTypeMap") + defer cleanup() + + types, err := tx.getFeatureTypeMap() + if err != nil { + require.Nil(t, err, err.Error()) + } + + require.Equal(t, database.SourcePackage, types.byID[1]) + require.Equal(t, database.BinaryPackage, types.byID[2]) + require.Equal(t, 1, types.byName[database.SourcePackage]) + require.Equal(t, 2, types.byName[database.BinaryPackage]) +} From 7dd989c0f21bc5c4cb390f575dca9973829ef9ce Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 19 Feb 2019 16:36:31 -0500 Subject: [PATCH 03/12] database: Rename affected Type to feature type --- database/models.go | 8 ++++---- ext/vulnsrc/alpine/alpine.go | 4 ++-- ext/vulnsrc/debian/debian.go | 4 ++-- ext/vulnsrc/debian/debian_test.go | 10 +++++----- ext/vulnsrc/oracle/oracle.go | 4 ++-- ext/vulnsrc/oracle/oracle_test.go | 6 +++--- ext/vulnsrc/rhel/rhel.go | 4 ++-- ext/vulnsrc/rhel/rhel_test.go | 10 +++++----- ext/vulnsrc/ubuntu/ubuntu.go | 4 ++-- ext/vulnsrc/ubuntu/ubuntu_test.go | 6 +++--- updater.go | 2 +- updater_test.go | 12 ++++++------ 12 files changed, 37 insertions(+), 37 deletions(-) diff --git a/database/models.go b/database/models.go index 5f63cbc1..41042260 100644 --- a/database/models.go +++ b/database/models.go @@ -199,10 +199,10 @@ type VulnerabilityWithFixedIn struct { // by a Vulnerability. Namespace and Feature Name is unique. Affected Feature is // bound to vulnerability. type AffectedFeature struct { - // AffectedType determines which type of package it affects. - AffectedType AffectedFeatureType - Namespace Namespace - FeatureName string + // FeatureType determines which type of package it affects. + FeatureType FeatureType + Namespace Namespace + FeatureName string // FixedInVersion is known next feature version that's not affected by the // vulnerability. Empty FixedInVersion means the unaffected version is // unknown. diff --git a/ext/vulnsrc/alpine/alpine.go b/ext/vulnsrc/alpine/alpine.go index 12a550b9..7e9c91aa 100644 --- a/ext/vulnsrc/alpine/alpine.go +++ b/ext/vulnsrc/alpine/alpine.go @@ -40,7 +40,7 @@ const ( nvdURLPrefix = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=" // affected type indicates if the affected feature hint is for binary or // source package. - affectedType = database.AffectBinaryPackage + affectedType = database.BinaryPackage ) func init() { @@ -177,7 +177,7 @@ func (file *secDB) Vulnerabilities() (vulns []database.VulnerabilityWithAffected vuln.Affected = []database.AffectedFeature{ { - AffectedType: affectedType, + FeatureType: affectedType, FeatureName: pkg.Pkg.Name, AffectedVersion: version, FixedInVersion: fixedInVersion, diff --git a/ext/vulnsrc/debian/debian.go b/ext/vulnsrc/debian/debian.go index d516e4a3..efe89497 100644 --- a/ext/vulnsrc/debian/debian.go +++ b/ext/vulnsrc/debian/debian.go @@ -38,7 +38,7 @@ const ( url = "https://security-tracker.debian.org/tracker/data/json" cveURLPrefix = "https://security-tracker.debian.org/tracker" updaterFlag = "debianUpdater" - affectedType = database.AffectSourcePackage + affectedType = database.SourcePackage ) type jsonData map[string]map[string]jsonVuln @@ -215,7 +215,7 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []database.VulnerabilityWi // Create and add the feature version. pkg := database.AffectedFeature{ - AffectedType: affectedType, + FeatureType: affectedType, FeatureName: pkgName, AffectedVersion: version, FixedInVersion: fixedInVersion, diff --git a/ext/vulnsrc/debian/debian_test.go b/ext/vulnsrc/debian/debian_test.go index 03750c0e..e620e5fb 100644 --- a/ext/vulnsrc/debian/debian_test.go +++ b/ext/vulnsrc/debian/debian_test.go @@ -41,7 +41,7 @@ func TestDebianParser(t *testing.T) { expectedFeatures := []database.AffectedFeature{ { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "debian:8", VersionFormat: dpkg.ParserName, @@ -50,7 +50,7 @@ func TestDebianParser(t *testing.T) { AffectedVersion: versionfmt.MaxVersion, }, { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "debian:unstable", VersionFormat: dpkg.ParserName, @@ -71,7 +71,7 @@ func TestDebianParser(t *testing.T) { expectedFeatures := []database.AffectedFeature{ { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "debian:8", VersionFormat: dpkg.ParserName, @@ -81,7 +81,7 @@ func TestDebianParser(t *testing.T) { AffectedVersion: "0.7.0", }, { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "debian:unstable", VersionFormat: dpkg.ParserName, @@ -91,7 +91,7 @@ func TestDebianParser(t *testing.T) { AffectedVersion: "0.7.0", }, { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "debian:8", VersionFormat: dpkg.ParserName, diff --git a/ext/vulnsrc/oracle/oracle.go b/ext/vulnsrc/oracle/oracle.go index 6fe35c95..93b4d97e 100644 --- a/ext/vulnsrc/oracle/oracle.go +++ b/ext/vulnsrc/oracle/oracle.go @@ -41,7 +41,7 @@ const ( ovalURI = "https://linux.oracle.com/oval/" elsaFilePrefix = "com.oracle.elsa-" updaterFlag = "oracleUpdater" - affectedType = database.AffectBinaryPackage + affectedType = database.BinaryPackage ) var ( @@ -365,7 +365,7 @@ func toFeatures(criteria criteria) []database.AffectedFeature { } else if strings.Contains(c.Comment, " is earlier than ") { const prefixLen = len(" is earlier than ") featureVersion.FeatureName = strings.TrimSpace(c.Comment[:strings.Index(c.Comment, " is earlier than ")]) - featureVersion.AffectedType = affectedType + featureVersion.FeatureType = affectedType version := c.Comment[strings.Index(c.Comment, " is earlier than ")+prefixLen:] err := versionfmt.Valid(rpm.ParserName, version) if err != nil { diff --git a/ext/vulnsrc/oracle/oracle_test.go b/ext/vulnsrc/oracle/oracle_test.go index a7071b94..f0c778ba 100644 --- a/ext/vulnsrc/oracle/oracle_test.go +++ b/ext/vulnsrc/oracle/oracle_test.go @@ -43,7 +43,7 @@ func TestOracleParserOneCve(t *testing.T) { expectedFeatures := []database.AffectedFeature{ { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "oracle:7", VersionFormat: rpm.ParserName, @@ -53,7 +53,7 @@ func TestOracleParserOneCve(t *testing.T) { AffectedVersion: "0:3.1.1-7.el7_1", }, { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "oracle:7", VersionFormat: rpm.ParserName, @@ -63,7 +63,7 @@ func TestOracleParserOneCve(t *testing.T) { AffectedVersion: "0:3.1.1-7.el7_1", }, { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "oracle:7", VersionFormat: rpm.ParserName, diff --git a/ext/vulnsrc/rhel/rhel.go b/ext/vulnsrc/rhel/rhel.go index e67e4bcf..a3a6abc5 100644 --- a/ext/vulnsrc/rhel/rhel.go +++ b/ext/vulnsrc/rhel/rhel.go @@ -43,7 +43,7 @@ const ( ovalURI = "https://www.redhat.com/security/data/oval/" rhsaFilePrefix = "com.redhat.rhsa-" updaterFlag = "rhelUpdater" - affectedType = database.AffectBinaryPackage + affectedType = database.BinaryPackage ) var ( @@ -333,7 +333,7 @@ func toFeatures(criteria criteria) []database.AffectedFeature { } else if strings.Contains(c.Comment, " is earlier than ") { const prefixLen = len(" is earlier than ") featureVersion.FeatureName = strings.TrimSpace(c.Comment[:strings.Index(c.Comment, " is earlier than ")]) - featureVersion.AffectedType = affectedType + featureVersion.FeatureType = affectedType version := c.Comment[strings.Index(c.Comment, " is earlier than ")+prefixLen:] err := versionfmt.Valid(rpm.ParserName, version) if err != nil { diff --git a/ext/vulnsrc/rhel/rhel_test.go b/ext/vulnsrc/rhel/rhel_test.go index 13c2e538..061656a8 100644 --- a/ext/vulnsrc/rhel/rhel_test.go +++ b/ext/vulnsrc/rhel/rhel_test.go @@ -46,7 +46,7 @@ func TestRHELParserMultipleCVE(t *testing.T) { database.MediumSeverity, database.MediumSeverity} expectedFeatures := []database.AffectedFeature{ { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "centos:6", VersionFormat: rpm.ParserName, @@ -56,7 +56,7 @@ func TestRHELParserMultipleCVE(t *testing.T) { AffectedVersion: "0:38.1.0-1.el6_6", }, { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "centos:7", VersionFormat: rpm.ParserName, @@ -96,7 +96,7 @@ func TestRHELParserOneCVE(t *testing.T) { expectedFeatures := []database.AffectedFeature{ { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "centos:7", VersionFormat: rpm.ParserName, @@ -106,7 +106,7 @@ func TestRHELParserOneCVE(t *testing.T) { FixedInVersion: "0:3.1.1-7.el7_1", }, { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "centos:7", VersionFormat: rpm.ParserName, @@ -116,7 +116,7 @@ func TestRHELParserOneCVE(t *testing.T) { FixedInVersion: "0:3.1.1-7.el7_1", }, { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "centos:7", VersionFormat: rpm.ParserName, diff --git a/ext/vulnsrc/ubuntu/ubuntu.go b/ext/vulnsrc/ubuntu/ubuntu.go index 30ecff93..1fef9e34 100644 --- a/ext/vulnsrc/ubuntu/ubuntu.go +++ b/ext/vulnsrc/ubuntu/ubuntu.go @@ -39,7 +39,7 @@ const ( trackerURI = "https://git.launchpad.net/ubuntu-cve-tracker" updaterFlag = "ubuntuUpdater" cveURL = "http://people.ubuntu.com/~ubuntu-security/cve/%s" - affectedType = database.AffectSourcePackage + affectedType = database.SourcePackage ) var ( @@ -335,7 +335,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability // Create and add the new package. featureVersion := database.AffectedFeature{ - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: releaseName, VersionFormat: dpkg.ParserName, diff --git a/ext/vulnsrc/ubuntu/ubuntu_test.go b/ext/vulnsrc/ubuntu/ubuntu_test.go index 1cb79061..fcbc36c3 100644 --- a/ext/vulnsrc/ubuntu/ubuntu_test.go +++ b/ext/vulnsrc/ubuntu/ubuntu_test.go @@ -46,7 +46,7 @@ func TestUbuntuParser(t *testing.T) { expectedFeatures := []database.AffectedFeature{ { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "ubuntu:14.04", VersionFormat: dpkg.ParserName, @@ -55,7 +55,7 @@ func TestUbuntuParser(t *testing.T) { AffectedVersion: versionfmt.MaxVersion, }, { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "ubuntu:15.04", VersionFormat: dpkg.ParserName, @@ -65,7 +65,7 @@ func TestUbuntuParser(t *testing.T) { AffectedVersion: "0.4-3", }, { - AffectedType: affectedType, + FeatureType: affectedType, Namespace: database.Namespace{ Name: "ubuntu:15.10", VersionFormat: dpkg.ParserName, diff --git a/updater.go b/updater.go index bd092a9b..94b1aadc 100644 --- a/updater.go +++ b/updater.go @@ -425,7 +425,7 @@ func doVulnerabilitiesNamespacing(vulnerabilities []database.VulnerabilityWithAf for _, fv := range namespacedFeatures { // validate vulnerabilities, throw out the invalid vulnerabilities - if fv.AffectedType == "" || fv.AffectedVersion == "" || fv.FeatureName == "" || fv.Namespace.Name == "" || fv.Namespace.VersionFormat == "" { + if fv.FeatureType == "" || fv.AffectedVersion == "" || fv.FeatureName == "" || fv.Namespace.Name == "" || fv.Namespace.VersionFormat == "" { log.WithFields(log.Fields{ "Name": fv.FeatureName, "Affected Version": fv.AffectedVersion, diff --git a/updater_test.go b/updater_test.go index 93ad2ecd..b4741d13 100644 --- a/updater_test.go +++ b/updater_test.go @@ -183,7 +183,7 @@ func newmockUpdaterDatastore() *mockUpdaterDatastore { func TestDoVulnerabilitiesNamespacing(t *testing.T) { fv1 := database.AffectedFeature{ - AffectedType: database.AffectSourcePackage, + FeatureType: database.SourcePackage, Namespace: database.Namespace{Name: "Namespace1"}, FeatureName: "Feature1", FixedInVersion: "0.1", @@ -191,7 +191,7 @@ func TestDoVulnerabilitiesNamespacing(t *testing.T) { } fv2 := database.AffectedFeature{ - AffectedType: database.AffectSourcePackage, + FeatureType: database.SourcePackage, Namespace: database.Namespace{Name: "Namespace2"}, FeatureName: "Feature1", FixedInVersion: "0.2", @@ -199,7 +199,7 @@ func TestDoVulnerabilitiesNamespacing(t *testing.T) { } fv3 := database.AffectedFeature{ - AffectedType: database.AffectSourcePackage, + FeatureType: database.SourcePackage, Namespace: database.Namespace{Name: "Namespace2"}, FeatureName: "Feature2", FixedInVersion: "0.3", @@ -237,9 +237,9 @@ func TestCreatVulnerabilityNotification(t *testing.T) { VersionFormat: vf1, } af1 := database.AffectedFeature{ - AffectedType: database.AffectSourcePackage, - Namespace: ns1, - FeatureName: "feature 1", + FeatureType: database.SourcePackage, + Namespace: ns1, + FeatureName: "feature 1", } v1 := database.VulnerabilityWithAffected{ From 0e0d8b38bba4c62552c98ad5b98242ddd2c3464b Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 19 Feb 2019 16:38:07 -0500 Subject: [PATCH 04/12] featurefmt: Extract source packages and binary packages The featurefmt now extracts both binary packages and source packages from the package manager infos. --- ext/featurefmt/apk/apk.go | 2 + ext/featurefmt/apk/apk_test.go | 22 +- ext/featurefmt/dpkg/dpkg.go | 121 +++++--- ext/featurefmt/dpkg/dpkg_test.go | 249 ++++++++++------ ext/featurefmt/rpm/rpm.go | 107 +++---- ext/featurefmt/rpm/rpm_test.go | 487 +++++++++++++++++++------------ 6 files changed, 608 insertions(+), 380 deletions(-) diff --git a/ext/featurefmt/apk/apk.go b/ext/featurefmt/apk/apk.go index f0d29d21..6389b825 100644 --- a/ext/featurefmt/apk/apk.go +++ b/ext/featurefmt/apk/apk.go @@ -55,6 +55,7 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) line := scanner.Text() if len(line) < 2 { if valid(&pkg) { + pkg.Type = database.BinaryPackage packages.Add(pkg) pkg = database.Feature{VersionFormat: dpkg.ParserName} } @@ -81,6 +82,7 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) // in case of no terminal line if valid(&pkg) { + pkg.Type = database.BinaryPackage packages.Add(pkg) } diff --git a/ext/featurefmt/apk/apk_test.go b/ext/featurefmt/apk/apk_test.go index 60461ed5..72ec5098 100644 --- a/ext/featurefmt/apk/apk_test.go +++ b/ext/featurefmt/apk/apk_test.go @@ -28,17 +28,17 @@ func TestAPKFeatureDetection(t *testing.T) { "valid case", map[string]string{"lib/apk/db/installed": "apk/testdata/valid"}, []database.Feature{ - {"musl", "1.1.14-r10", "", "", dpkg.ParserName}, - {"busybox", "1.24.2-r9", "", "", dpkg.ParserName}, - {"alpine-baselayout", "3.0.3-r0", "", "", dpkg.ParserName}, - {"alpine-keys", "1.1-r0", "", "", dpkg.ParserName}, - {"zlib", "1.2.8-r2", "", "", dpkg.ParserName}, - {"libcrypto1.0", "1.0.2h-r1", "", "", dpkg.ParserName}, - {"libssl1.0", "1.0.2h-r1", "", "", dpkg.ParserName}, - {"apk-tools", "2.6.7-r0", "", "", dpkg.ParserName}, - {"scanelf", "1.1.6-r0", "", "", dpkg.ParserName}, - {"musl-utils", "1.1.14-r10", "", "", dpkg.ParserName}, - {"libc-utils", "0.7-r0", "", "", dpkg.ParserName}, + {"apk-tools", "2.6.7-r0", "dpkg", "binary"}, + {"musl", "1.1.14-r10", "dpkg", "binary"}, + {"libssl1.0", "1.0.2h-r1", "dpkg", "binary"}, + {"libc-utils", "0.7-r0", "dpkg", "binary"}, + {"busybox", "1.24.2-r9", "dpkg", "binary"}, + {"scanelf", "1.1.6-r0", "dpkg", "binary"}, + {"alpine-keys", "1.1-r0", "dpkg", "binary"}, + {"libcrypto1.0", "1.0.2h-r1", "dpkg", "binary"}, + {"zlib", "1.2.8-r2", "dpkg", "binary"}, + {"musl-utils", "1.1.14-r10", "dpkg", "binary"}, + {"alpine-baselayout", "3.0.3-r0", "dpkg", "binary"}, }, }, } { diff --git a/ext/featurefmt/dpkg/dpkg.go b/ext/featurefmt/dpkg/dpkg.go index 4e0b60e2..e68f7be1 100644 --- a/ext/featurefmt/dpkg/dpkg.go +++ b/ext/featurefmt/dpkg/dpkg.go @@ -37,22 +37,12 @@ var ( type lister struct{} -func init() { - featurefmt.RegisterLister("dpkg", "1.0", &lister{}) -} - -func valid(pkg *database.Feature) bool { - return pkg.Name != "" && pkg.Version != "" +func (l lister) RequiredFilenames() []string { + return []string{"var/lib/dpkg/status"} } -func addSourcePackage(pkg *database.Feature) { - if pkg.SourceName == "" { - pkg.SourceName = pkg.Name - } - - if pkg.SourceVersion == "" { - pkg.SourceVersion = pkg.Version - } +func init() { + featurefmt.RegisterLister("dpkg", "1.0", &lister{}) } func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) { @@ -61,21 +51,45 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) return []database.Feature{}, nil } + packages := mapset.NewSet() + scanner := bufio.NewScanner(strings.NewReader(string(f))) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + binary, source := parseDpkgDB(scanner) + if binary != nil { + packages.Add(*binary) + } + + if source != nil { + packages.Add(*source) + } + } + + return database.ConvertFeatureSetToFeatures(packages), nil +} + +// parseDpkgDB consumes the status file scanner exactly one package info, until +// EOF or empty space, and generate the parsed packages from it. +func parseDpkgDB(scanner *bufio.Scanner) (binaryPackage *database.Feature, sourcePackage *database.Feature) { var ( - pkg = database.Feature{VersionFormat: dpkg.ParserName} - pkgs = mapset.NewSet() - err error + name string + version string + sourceName string + sourceVersion string ) - scanner := bufio.NewScanner(strings.NewReader(string(f))) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "Package: ") { - // Package line - // Defines the name of the package + for { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + break + } - pkg.Name = strings.TrimSpace(strings.TrimPrefix(line, "Package: ")) - pkg.Version = "" + if strings.HasPrefix(line, "Package: ") { + name = strings.TrimSpace(strings.TrimPrefix(line, "Package: ")) } else if strings.HasPrefix(line, "Source: ") { // Source line (Optional) // Gives the name of the source package @@ -87,14 +101,9 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) md[dpkgSrcCaptureRegexpNames[i]] = strings.TrimSpace(n) } - pkg.SourceName = md["name"] + sourceName = md["name"] if md["version"] != "" { - version := md["version"] - if err = versionfmt.Valid(dpkg.ParserName, version); err != nil { - log.WithError(err).WithField("version", string(line[1])).Warning("could not parse package version. skipping") - } else { - pkg.SourceVersion = version - } + sourceVersion = md["version"] } } else if strings.HasPrefix(line, "Version: ") { // Version line @@ -102,25 +111,43 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) // This version is less important than a version retrieved from a Source line // because the Debian vulnerabilities often skips the epoch from the Version field // which is not present in the Source version, and because +bX revisions don't matter - version := strings.TrimPrefix(line, "Version: ") - if err = versionfmt.Valid(dpkg.ParserName, version); err != nil { - log.WithError(err).WithField("version", string(line[1])).Warning("could not parse package version. skipping") - } else { - pkg.Version = version - } - } else if line == "" { - pkg = database.Feature{VersionFormat: dpkg.ParserName} + version = strings.TrimPrefix(line, "Version: ") } - if valid(&pkg) { - addSourcePackage(&pkg) - pkgs.Add(pkg) + if !scanner.Scan() { + break } } - return database.ConvertFeatureSetToFeatures(pkgs), nil -} + if name != "" && version != "" { + if err := versionfmt.Valid(dpkg.ParserName, version); err != nil { + log.WithError(err).WithFields(log.Fields{"name": name, "version": version}).Warning("skipped unparseable package") + } else { + binaryPackage = &database.Feature{name, version, dpkg.ParserName, database.BinaryPackage} + } + } -func (l lister) RequiredFilenames() []string { - return []string{"var/lib/dpkg/status"} + // Source version and names are computed from binary package names and versions + // in dpkg. + // Source package name: + // https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/pkg-format.c#n338 + // Source package version: + // https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/pkg-format.c#n355 + if sourceName == "" { + sourceName = name + } + + if sourceVersion == "" { + sourceVersion = version + } + + if sourceName != "" && sourceVersion != "" { + if err := versionfmt.Valid(dpkg.ParserName, version); err != nil { + log.WithError(err).WithFields(log.Fields{"name": name, "version": version}).Warning("skipped unparseable package") + } else { + sourcePackage = &database.Feature{sourceName, sourceVersion, dpkg.ParserName, database.SourcePackage} + } + } + + return } diff --git a/ext/featurefmt/dpkg/dpkg_test.go b/ext/featurefmt/dpkg/dpkg_test.go index ff1de7a0..6bfb9bc7 100644 --- a/ext/featurefmt/dpkg/dpkg_test.go +++ b/ext/featurefmt/dpkg/dpkg_test.go @@ -28,105 +28,168 @@ func TestListFeatures(t *testing.T) { "valid status file", map[string]string{"var/lib/dpkg/status": "dpkg/testdata/valid"}, []database.Feature{ - {"adduser", "3.116ubuntu1", "adduser", "3.116ubuntu1", dpkg.ParserName}, - {"apt", "1.6.3ubuntu0.1", "apt", "1.6.3ubuntu0.1", dpkg.ParserName}, - {"base-files", "10.1ubuntu2.2", "base-files", "10.1ubuntu2.2", dpkg.ParserName}, - {"base-passwd", "3.5.44", "base-passwd", "3.5.44", dpkg.ParserName}, - {"bash", "4.4.18-2ubuntu1", "bash", "4.4.18-2ubuntu1", dpkg.ParserName}, - {"bsdutils", "1:2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1", dpkg.ParserName}, - {"bzip2", "1.0.6-8.1", "bzip2", "1.0.6-8.1", dpkg.ParserName}, - {"coreutils", "8.28-1ubuntu1", "coreutils", "8.28-1ubuntu1", dpkg.ParserName}, - {"dash", "0.5.8-2.10", "dash", "0.5.8-2.10", dpkg.ParserName}, - {"debconf", "1.5.66", "debconf", "1.5.66", dpkg.ParserName}, - {"debianutils", "4.8.4", "debianutils", "4.8.4", dpkg.ParserName}, - {"diffutils", "1:3.6-1", "diffutils", "1:3.6-1", dpkg.ParserName}, - {"dpkg", "1.19.0.5ubuntu2", "dpkg", "1.19.0.5ubuntu2", dpkg.ParserName}, - {"e2fsprogs", "1.44.1-1", "e2fsprogs", "1.44.1-1", dpkg.ParserName}, - {"fdisk", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1", dpkg.ParserName}, - {"findutils", "4.6.0+git+20170828-2", "findutils", "4.6.0+git+20170828-2", dpkg.ParserName}, - {"gcc-8-base", "8-20180414-1ubuntu2", "gcc-8", "8-20180414-1ubuntu2", dpkg.ParserName}, - {"gpgv", "2.2.4-1ubuntu1.1", "gnupg2", "2.2.4-1ubuntu1.1", dpkg.ParserName}, - {"grep", "3.1-2", "grep", "3.1-2", dpkg.ParserName}, - {"gzip", "1.6-5ubuntu1", "gzip", "1.6-5ubuntu1", dpkg.ParserName}, - {"hostname", "3.20", "hostname", "3.20", dpkg.ParserName}, - {"init-system-helpers", "1.51", "init-system-helpers", "1.51", dpkg.ParserName}, - {"libacl1", "2.2.52-3build1", "acl", "2.2.52-3build1", dpkg.ParserName}, - {"libapt-pkg5.0", "1.6.3ubuntu0.1", "apt", "1.6.3ubuntu0.1", dpkg.ParserName}, - {"libattr1", "1:2.4.47-2build1", "attr", "1:2.4.47-2build1", dpkg.ParserName}, - {"libaudit-common", "1:2.8.2-1ubuntu1", "audit", "1:2.8.2-1ubuntu1", dpkg.ParserName}, - {"libaudit1", "1:2.8.2-1ubuntu1", "audit", "1:2.8.2-1ubuntu1", dpkg.ParserName}, - {"libblkid1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1", dpkg.ParserName}, - {"libbz2-1.0", "1.0.6-8.1", "bzip2", "1.0.6-8.1", dpkg.ParserName}, - {"libc-bin", "2.27-3ubuntu1", "glibc", "2.27-3ubuntu1", dpkg.ParserName}, - {"libc6", "2.27-3ubuntu1", "glibc", "2.27-3ubuntu1", dpkg.ParserName}, - {"libcap-ng0", "0.7.7-3.1", "libcap-ng", "0.7.7-3.1", dpkg.ParserName}, - {"libcom-err2", "1.44.1-1", "e2fsprogs", "1.44.1-1", dpkg.ParserName}, - {"libdb5.3", "5.3.28-13.1ubuntu1", "db5.3", "5.3.28-13.1ubuntu1", dpkg.ParserName}, - {"libdebconfclient0", "0.213ubuntu1", "cdebconf", "0.213ubuntu1", dpkg.ParserName}, - {"libext2fs2", "1.44.1-1", "e2fsprogs", "1.44.1-1", dpkg.ParserName}, - {"libfdisk1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1", dpkg.ParserName}, - {"libffi6", "3.2.1-8", "libffi", "3.2.1-8", dpkg.ParserName}, - {"libgcc1", "1:8-20180414-1ubuntu2", "gcc-8", "8-20180414-1ubuntu2", dpkg.ParserName}, - {"libgcrypt20", "1.8.1-4ubuntu1.1", "libgcrypt20", "1.8.1-4ubuntu1.1", dpkg.ParserName}, - {"libgmp10", "2:6.1.2+dfsg-2", "gmp", "2:6.1.2+dfsg-2", dpkg.ParserName}, - {"libgnutls30", "3.5.18-1ubuntu1", "gnutls28", "3.5.18-1ubuntu1", dpkg.ParserName}, - {"libgpg-error0", "1.27-6", "libgpg-error", "1.27-6", dpkg.ParserName}, - {"libhogweed4", "3.4-1", "nettle", "3.4-1", dpkg.ParserName}, - {"libidn2-0", "2.0.4-1.1build2", "libidn2", "2.0.4-1.1build2", dpkg.ParserName}, - {"liblz4-1", "0.0~r131-2ubuntu3", "lz4", "0.0~r131-2ubuntu3", dpkg.ParserName}, - {"liblzma5", "5.2.2-1.3", "xz-utils", "5.2.2-1.3", dpkg.ParserName}, - {"libmount1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1", dpkg.ParserName}, - {"libncurses5", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04", dpkg.ParserName}, - {"libncursesw5", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04", dpkg.ParserName}, - {"libnettle6", "3.4-1", "nettle", "3.4-1", dpkg.ParserName}, - {"libp11-kit0", "0.23.9-2", "p11-kit", "0.23.9-2", dpkg.ParserName}, - {"libpam-modules", "1.1.8-3.6ubuntu2", "pam", "1.1.8-3.6ubuntu2", dpkg.ParserName}, - {"libpam-modules-bin", "1.1.8-3.6ubuntu2", "pam", "1.1.8-3.6ubuntu2", dpkg.ParserName}, - {"libpam-runtime", "1.1.8-3.6ubuntu2", "pam", "1.1.8-3.6ubuntu2", dpkg.ParserName}, - {"libpam0g", "1.1.8-3.6ubuntu2", "pam", "1.1.8-3.6ubuntu2", dpkg.ParserName}, - {"libpcre3", "2:8.39-9", "pcre3", "2:8.39-9", dpkg.ParserName}, - {"libprocps6", "2:3.3.12-3ubuntu1.1", "procps", "2:3.3.12-3ubuntu1.1", dpkg.ParserName}, - {"libseccomp2", "2.3.1-2.1ubuntu4", "libseccomp", "2.3.1-2.1ubuntu4", dpkg.ParserName}, - {"libselinux1", "2.7-2build2", "libselinux", "2.7-2build2", dpkg.ParserName}, - {"libsemanage-common", "2.7-2build2", "libsemanage", "2.7-2build2", dpkg.ParserName}, - {"libsemanage1", "2.7-2build2", "libsemanage", "2.7-2build2", dpkg.ParserName}, - {"libsepol1", "2.7-1", "libsepol", "2.7-1", dpkg.ParserName}, - {"libsmartcols1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1", dpkg.ParserName}, - {"libss2", "1.44.1-1", "e2fsprogs", "1.44.1-1", dpkg.ParserName}, - {"libstdc++6", "8-20180414-1ubuntu2", "gcc-8", "8-20180414-1ubuntu2", dpkg.ParserName}, - {"libsystemd0", "237-3ubuntu10.3", "systemd", "237-3ubuntu10.3", dpkg.ParserName}, - {"libtasn1-6", "4.13-2", "libtasn1-6", "4.13-2", dpkg.ParserName}, - {"libtinfo5", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04", dpkg.ParserName}, - {"libudev1", "237-3ubuntu10.3", "systemd", "237-3ubuntu10.3", dpkg.ParserName}, - {"libunistring2", "0.9.9-0ubuntu1", "libunistring", "0.9.9-0ubuntu1", dpkg.ParserName}, - {"libuuid1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1", dpkg.ParserName}, - {"libzstd1", "1.3.3+dfsg-2ubuntu1", "libzstd", "1.3.3+dfsg-2ubuntu1", dpkg.ParserName}, - {"login", "1:4.5-1ubuntu1", "shadow", "1:4.5-1ubuntu1", dpkg.ParserName}, - {"lsb-base", "9.20170808ubuntu1", "lsb", "9.20170808ubuntu1", dpkg.ParserName}, - {"mawk", "1.3.3-17ubuntu3", "mawk", "1.3.3-17ubuntu3", dpkg.ParserName}, - {"mount", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1", dpkg.ParserName}, - {"ncurses-base", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04", dpkg.ParserName}, - {"ncurses-bin", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04", dpkg.ParserName}, - {"passwd", "1:4.5-1ubuntu1", "shadow", "1:4.5-1ubuntu1", dpkg.ParserName}, - {"perl-base", "5.26.1-6ubuntu0.2", "perl", "5.26.1-6ubuntu0.2", dpkg.ParserName}, - {"procps", "2:3.3.12-3ubuntu1.1", "procps", "2:3.3.12-3ubuntu1.1", dpkg.ParserName}, - {"sed", "4.4-2", "sed", "4.4-2", dpkg.ParserName}, - {"sensible-utils", "0.0.12", "sensible-utils", "0.0.12", dpkg.ParserName}, - {"sysvinit-utils", "2.88dsf-59.10ubuntu1", "sysvinit", "2.88dsf-59.10ubuntu1", dpkg.ParserName}, - {"tar", "1.29b-2", "tar", "1.29b-2", dpkg.ParserName}, - {"ubuntu-keyring", "2018.02.28", "ubuntu-keyring", "2018.02.28", dpkg.ParserName}, - {"util-linux", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1", dpkg.ParserName}, - {"zlib1g", "1:1.2.11.dfsg-0ubuntu2", "zlib", "1:1.2.11.dfsg-0ubuntu2", dpkg.ParserName}, + {"libapt-pkg5.0", "1.6.3ubuntu0.1", "dpkg", "binary"}, + {"perl-base", "5.26.1-6ubuntu0.2", "dpkg", "binary"}, + {"libmount1", "2.31.1-0.4ubuntu3.1", "dpkg", "binary"}, + {"perl", "5.26.1-6ubuntu0.2", "dpkg", "source"}, + {"libgnutls30", "3.5.18-1ubuntu1", "dpkg", "binary"}, + {"liblzma5", "5.2.2-1.3", "dpkg", "binary"}, + {"ncurses-bin", "6.1-1ubuntu1.18.04", "dpkg", "binary"}, + {"lsb", "9.20170808ubuntu1", "dpkg", "source"}, + {"sed", "4.4-2", "dpkg", "source"}, + {"libsystemd0", "237-3ubuntu10.3", "dpkg", "binary"}, + {"procps", "2:3.3.12-3ubuntu1.1", "dpkg", "source"}, + {"login", "1:4.5-1ubuntu1", "dpkg", "binary"}, + {"libunistring2", "0.9.9-0ubuntu1", "dpkg", "binary"}, + {"sed", "4.4-2", "dpkg", "binary"}, + {"libselinux", "2.7-2build2", "dpkg", "source"}, + {"libseccomp", "2.3.1-2.1ubuntu4", "dpkg", "source"}, + {"libss2", "1.44.1-1", "dpkg", "binary"}, + {"liblz4-1", "0.0~r131-2ubuntu3", "dpkg", "binary"}, + {"libsemanage1", "2.7-2build2", "dpkg", "binary"}, + {"libtasn1-6", "4.13-2", "dpkg", "source"}, + {"libzstd1", "1.3.3+dfsg-2ubuntu1", "dpkg", "binary"}, + {"fdisk", "2.31.1-0.4ubuntu3.1", "dpkg", "binary"}, + {"xz-utils", "5.2.2-1.3", "dpkg", "source"}, + {"lsb-base", "9.20170808ubuntu1", "dpkg", "binary"}, + {"libpam-modules-bin", "1.1.8-3.6ubuntu2", "dpkg", "binary"}, + {"dash", "0.5.8-2.10", "dpkg", "binary"}, + {"gnupg2", "2.2.4-1ubuntu1.1", "dpkg", "source"}, + {"libfdisk1", "2.31.1-0.4ubuntu3.1", "dpkg", "binary"}, + {"lz4", "0.0~r131-2ubuntu3", "dpkg", "source"}, + {"libpam0g", "1.1.8-3.6ubuntu2", "dpkg", "binary"}, + {"libc-bin", "2.27-3ubuntu1", "dpkg", "binary"}, + {"libcap-ng", "0.7.7-3.1", "dpkg", "source"}, + {"libcom-err2", "1.44.1-1", "dpkg", "binary"}, + {"libudev1", "237-3ubuntu10.3", "dpkg", "binary"}, + {"debconf", "1.5.66", "dpkg", "binary"}, + {"tar", "1.29b-2", "dpkg", "binary"}, + {"diffutils", "1:3.6-1", "dpkg", "source"}, + {"gcc-8", "8-20180414-1ubuntu2", "dpkg", "source"}, + {"e2fsprogs", "1.44.1-1", "dpkg", "source"}, + {"bzip2", "1.0.6-8.1", "dpkg", "source"}, + {"diffutils", "1:3.6-1", "dpkg", "binary"}, + {"grep", "3.1-2", "dpkg", "binary"}, + {"libgcc1", "1:8-20180414-1ubuntu2", "dpkg", "binary"}, + {"bash", "4.4.18-2ubuntu1", "dpkg", "source"}, + {"libtinfo5", "6.1-1ubuntu1.18.04", "dpkg", "binary"}, + {"procps", "2:3.3.12-3ubuntu1.1", "dpkg", "binary"}, + {"bzip2", "1.0.6-8.1", "dpkg", "binary"}, + {"init-system-helpers", "1.51", "dpkg", "binary"}, + {"libncursesw5", "6.1-1ubuntu1.18.04", "dpkg", "binary"}, + {"init-system-helpers", "1.51", "dpkg", "source"}, + {"libpam-modules", "1.1.8-3.6ubuntu2", "dpkg", "binary"}, + {"libext2fs2", "1.44.1-1", "dpkg", "binary"}, + {"libacl1", "2.2.52-3build1", "dpkg", "binary"}, + {"hostname", "3.20", "dpkg", "binary"}, + {"libgpg-error", "1.27-6", "dpkg", "source"}, + {"acl", "2.2.52-3build1", "dpkg", "source"}, + {"apt", "1.6.3ubuntu0.1", "dpkg", "binary"}, + {"base-files", "10.1ubuntu2.2", "dpkg", "source"}, + {"libgpg-error0", "1.27-6", "dpkg", "binary"}, + {"audit", "1:2.8.2-1ubuntu1", "dpkg", "source"}, + {"hostname", "3.20", "dpkg", "source"}, + {"gzip", "1.6-5ubuntu1", "dpkg", "binary"}, + {"libc6", "2.27-3ubuntu1", "dpkg", "binary"}, + {"libnettle6", "3.4-1", "dpkg", "binary"}, + {"sysvinit-utils", "2.88dsf-59.10ubuntu1", "dpkg", "binary"}, + {"debianutils", "4.8.4", "dpkg", "source"}, + {"libstdc++6", "8-20180414-1ubuntu2", "dpkg", "binary"}, + {"libsepol", "2.7-1", "dpkg", "source"}, + {"libpcre3", "2:8.39-9", "dpkg", "binary"}, + {"libuuid1", "2.31.1-0.4ubuntu3.1", "dpkg", "binary"}, + {"systemd", "237-3ubuntu10.3", "dpkg", "source"}, + {"tar", "1.29b-2", "dpkg", "source"}, + {"ubuntu-keyring", "2018.02.28", "dpkg", "source"}, + {"passwd", "1:4.5-1ubuntu1", "dpkg", "binary"}, + {"sysvinit", "2.88dsf-59.10ubuntu1", "dpkg", "source"}, + {"libidn2-0", "2.0.4-1.1build2", "dpkg", "binary"}, + {"libhogweed4", "3.4-1", "dpkg", "binary"}, + {"db5.3", "5.3.28-13.1ubuntu1", "dpkg", "source"}, + {"sensible-utils", "0.0.12", "dpkg", "source"}, + {"dpkg", "1.19.0.5ubuntu2", "dpkg", "source"}, + {"libp11-kit0", "0.23.9-2", "dpkg", "binary"}, + {"glibc", "2.27-3ubuntu1", "dpkg", "source"}, + {"mount", "2.31.1-0.4ubuntu3.1", "dpkg", "binary"}, + {"libsemanage-common", "2.7-2build2", "dpkg", "binary"}, + {"libblkid1", "2.31.1-0.4ubuntu3.1", "dpkg", "binary"}, + {"libdebconfclient0", "0.213ubuntu1", "dpkg", "binary"}, + {"libffi", "3.2.1-8", "dpkg", "source"}, + {"pam", "1.1.8-3.6ubuntu2", "dpkg", "source"}, + {"bsdutils", "1:2.31.1-0.4ubuntu3.1", "dpkg", "binary"}, + {"libtasn1-6", "4.13-2", "dpkg", "binary"}, + {"libaudit-common", "1:2.8.2-1ubuntu1", "dpkg", "binary"}, + {"gpgv", "2.2.4-1ubuntu1.1", "dpkg", "binary"}, + {"libzstd", "1.3.3+dfsg-2ubuntu1", "dpkg", "source"}, + {"base-passwd", "3.5.44", "dpkg", "source"}, + {"adduser", "3.116ubuntu1", "dpkg", "binary"}, + {"libattr1", "1:2.4.47-2build1", "dpkg", "binary"}, + {"libncurses5", "6.1-1ubuntu1.18.04", "dpkg", "binary"}, + {"coreutils", "8.28-1ubuntu1", "dpkg", "binary"}, + {"base-passwd", "3.5.44", "dpkg", "binary"}, + {"ubuntu-keyring", "2018.02.28", "dpkg", "binary"}, + {"adduser", "3.116ubuntu1", "dpkg", "source"}, + {"libsmartcols1", "2.31.1-0.4ubuntu3.1", "dpkg", "binary"}, + {"libunistring", "0.9.9-0ubuntu1", "dpkg", "source"}, + {"mawk", "1.3.3-17ubuntu3", "dpkg", "source"}, + {"coreutils", "8.28-1ubuntu1", "dpkg", "source"}, + {"attr", "1:2.4.47-2build1", "dpkg", "source"}, + {"gmp", "2:6.1.2+dfsg-2", "dpkg", "source"}, + {"libsemanage", "2.7-2build2", "dpkg", "source"}, + {"libselinux1", "2.7-2build2", "dpkg", "binary"}, + {"libseccomp2", "2.3.1-2.1ubuntu4", "dpkg", "binary"}, + {"zlib1g", "1:1.2.11.dfsg-0ubuntu2", "dpkg", "binary"}, + {"dash", "0.5.8-2.10", "dpkg", "source"}, + {"gnutls28", "3.5.18-1ubuntu1", "dpkg", "source"}, + {"libpam-runtime", "1.1.8-3.6ubuntu2", "dpkg", "binary"}, + {"libgcrypt20", "1.8.1-4ubuntu1.1", "dpkg", "source"}, + {"sensible-utils", "0.0.12", "dpkg", "binary"}, + {"p11-kit", "0.23.9-2", "dpkg", "source"}, + {"ncurses-base", "6.1-1ubuntu1.18.04", "dpkg", "binary"}, + {"e2fsprogs", "1.44.1-1", "dpkg", "binary"}, + {"libgcrypt20", "1.8.1-4ubuntu1.1", "dpkg", "binary"}, + {"libprocps6", "2:3.3.12-3ubuntu1.1", "dpkg", "binary"}, + {"debconf", "1.5.66", "dpkg", "source"}, + {"gcc-8-base", "8-20180414-1ubuntu2", "dpkg", "binary"}, + {"base-files", "10.1ubuntu2.2", "dpkg", "binary"}, + {"libbz2-1.0", "1.0.6-8.1", "dpkg", "binary"}, + {"grep", "3.1-2", "dpkg", "source"}, + {"bash", "4.4.18-2ubuntu1", "dpkg", "binary"}, + {"libgmp10", "2:6.1.2+dfsg-2", "dpkg", "binary"}, + {"shadow", "1:4.5-1ubuntu1", "dpkg", "source"}, + {"libidn2", "2.0.4-1.1build2", "dpkg", "source"}, + {"gzip", "1.6-5ubuntu1", "dpkg", "source"}, + {"util-linux", "2.31.1-0.4ubuntu3.1", "dpkg", "binary"}, + {"libaudit1", "1:2.8.2-1ubuntu1", "dpkg", "binary"}, + {"libsepol1", "2.7-1", "dpkg", "binary"}, + {"pcre3", "2:8.39-9", "dpkg", "source"}, + {"apt", "1.6.3ubuntu0.1", "dpkg", "source"}, + {"nettle", "3.4-1", "dpkg", "source"}, + {"util-linux", "2.31.1-0.4ubuntu3.1", "dpkg", "source"}, + {"libcap-ng0", "0.7.7-3.1", "dpkg", "binary"}, + {"debianutils", "4.8.4", "dpkg", "binary"}, + {"ncurses", "6.1-1ubuntu1.18.04", "dpkg", "source"}, + {"libffi6", "3.2.1-8", "dpkg", "binary"}, + {"cdebconf", "0.213ubuntu1", "dpkg", "source"}, + {"findutils", "4.6.0+git+20170828-2", "dpkg", "source"}, + {"libdb5.3", "5.3.28-13.1ubuntu1", "dpkg", "binary"}, + {"zlib", "1:1.2.11.dfsg-0ubuntu2", "dpkg", "source"}, + {"findutils", "4.6.0+git+20170828-2", "dpkg", "binary"}, + {"dpkg", "1.19.0.5ubuntu2", "dpkg", "binary"}, + {"mawk", "1.3.3-17ubuntu3", "dpkg", "binary"}, }, }, { "corrupted status file", map[string]string{"var/lib/dpkg/status": "dpkg/testdata/corrupted"}, []database.Feature{ - {"libpam-runtime", "1.1.8-3.1ubuntu3", "pam", "1.1.8-3.1ubuntu3", dpkg.ParserName}, - {"libpam-modules-bin", "1.1.8-3.1ubuntu3", "pam", "1.1.8-3.1ubuntu3", dpkg.ParserName}, - {"makedev", "2.3.1-93ubuntu1", "makedev", "2.3.1-93ubuntu1", dpkg.ParserName}, - {"libgcc1", "1:5.1.1-12ubuntu1", "gcc-5", "5.1.1-12ubuntu1", dpkg.ParserName}, + {"libpam-modules-bin", "1.1.8-3.1ubuntu3", "dpkg", "binary"}, + {"gcc-5", "5.1.1-12ubuntu1", "dpkg", "source"}, + {"makedev", "2.3.1-93ubuntu1", "dpkg", "binary"}, + {"libgcc1", "1:5.1.1-12ubuntu1", "dpkg", "binary"}, + {"pam", "1.1.8-3.1ubuntu3", "dpkg", "source"}, + {"makedev", "2.3.1-93ubuntu1", "dpkg", "source"}, + {"libpam-runtime", "1.1.8-3.1ubuntu3", "dpkg", "binary"}, }, }, } { diff --git a/ext/featurefmt/rpm/rpm.go b/ext/featurefmt/rpm/rpm.go index 5ad70191..388212ea 100644 --- a/ext/featurefmt/rpm/rpm.go +++ b/ext/featurefmt/rpm/rpm.go @@ -45,6 +45,10 @@ func init() { featurefmt.RegisterLister("rpm", "1.0", &lister{}) } +func (l lister) RequiredFilenames() []string { + return []string{"var/lib/rpm/Packages"} +} + func isIgnored(packageName string) bool { for _, pkg := range ignoredPackages { if pkg == packageName { @@ -55,12 +59,6 @@ func isIgnored(packageName string) bool { return false } -func valid(pkg *database.Feature) bool { - return pkg.Name != "" && pkg.Version != "" && - ((pkg.SourceName == "" && pkg.SourceVersion != "") || - (pkg.SourceName != "" && pkg.SourceVersion != "")) -} - func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) { f, hasFile := files["var/lib/rpm/Packages"] if !hasFile { @@ -84,7 +82,7 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) // Extract binary package names because RHSA refers to binary package names. out, err := exec.Command("rpm", "--dbpath", tmpDir, "-qa", "--qf", "%{NAME} %{EPOCH}:%{VERSION}-%{RELEASE} %{SOURCERPM}\n").CombinedOutput() if err != nil { - log.WithError(err).WithField("output", string(out)).Error("could not query RPM") + log.WithError(err).WithField("output", string(out)).Error("failed to query RPM") // Do not bubble up because we probably won't be able to fix it, // the database must be corrupted return []database.Feature{}, nil @@ -93,39 +91,51 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) packages := mapset.NewSet() scanner := bufio.NewScanner(strings.NewReader(string(out))) for scanner.Scan() { - line := strings.Split(scanner.Text(), " ") - if len(line) != 3 { - // We may see warnings on some RPM versions: - // "warning: Generating 12 missing index(es), please wait..." - continue + rpmPackage, srpmPackage := parseRPMOutput(scanner.Text()) + if rpmPackage != nil { + packages.Add(*rpmPackage) } - if isIgnored(line[0]) { - continue + if srpmPackage != nil { + packages.Add(*srpmPackage) } + } - pkg := database.Feature{Name: line[0], VersionFormat: rpm.ParserName} - pkg.Version = strings.Replace(line[1], "(none):", "", -1) - if err := versionfmt.Valid(rpm.ParserName, pkg.Version); err != nil { - log.WithError(err).WithField("version", line[1]).Warning("skipped unparseable package") - continue - } + return database.ConvertFeatureSetToFeatures(packages), nil +} - if err := parseSourceRPM(line[2], &pkg); err != nil { - log.WithError(err).WithField("sourcerpm", line[2]).Warning("skipped unparseable package") - continue - } +func parseRPMOutput(raw string) (rpmPackage *database.Feature, srpmPackage *database.Feature) { + line := strings.Split(raw, " ") + if len(line) != 3 { + // We may see warnings on some RPM versions: + // "warning: Generating 12 missing index(es), please wait..." + return + } - if valid(&pkg) { - packages.Add(pkg) - } + if isIgnored(line[0]) { + return } - return database.ConvertFeatureSetToFeatures(packages), nil -} + name, version, srpm := line[0], strings.Replace(line[1], "(none):", "", -1), line[2] + if err := versionfmt.Valid(rpm.ParserName, version); err != nil { + log.WithError(err).WithFields(log.Fields{"name": name, "version": version}).Warning("skipped unparseable package") + return + } -func (l lister) RequiredFilenames() []string { - return []string{"var/lib/rpm/Packages"} + rpmPackage = &database.Feature{name, version, rpm.ParserName, database.BinaryPackage} + srpmName, srpmVersion, srpmRelease, _, err := parseSourceRPM(srpm) + if err != nil { + log.WithError(err).WithFields(log.Fields{"name": name, "sourcerpm": srpm}).Warning("skipped unparseable package") + return + } + + srpmVersion = srpmVersion + "-" + srpmRelease + if err = versionfmt.Valid(rpm.ParserName, srpmVersion); err != nil { + return + } + + srpmPackage = &database.Feature{srpmName, srpmVersion, rpm.ParserName, database.SourcePackage} + return } type rpmParserState string @@ -140,11 +150,9 @@ const ( // parseSourceRPM parses the source rpm package representation string // http://ftp.rpm.org/max-rpm/ch-rpm-file-format.html -func parseSourceRPM(sourceRPM string, pkg *database.Feature) error { +func parseSourceRPM(sourceRPM string) (name string, version string, release string, architecture string, err error) { state := parseRPM previousCheckPoint := len(sourceRPM) - release := "" - version := "" for i := len(sourceRPM) - 1; i >= 0; i-- { switch state { case parseRPM: @@ -153,16 +161,18 @@ func parseSourceRPM(sourceRPM string, pkg *database.Feature) error { packageType := strutil.Substring(sourceRPM, i+1, len(sourceRPM)) previousCheckPoint = i if packageType != "rpm" { - return fmt.Errorf("unexpected package type, expect: 'rpm', got: '%s'", packageType) + err = fmt.Errorf("unexpected package type, expect: 'rpm', got: '%s'", packageType) + return } } case parseArchitecture: if string(sourceRPM[i]) == "." { state = parseRelease - architecture := strutil.Substring(sourceRPM, i+1, previousCheckPoint) + architecture = strutil.Substring(sourceRPM, i+1, previousCheckPoint) previousCheckPoint = i if architecture != "src" && architecture != "nosrc" { - return fmt.Errorf("unexpected package architecture, expect: 'src' or 'nosrc', got: '%s'", architecture) + err = fmt.Errorf("unexpected package architecture, expect: 'src' or 'nosrc', got: '%s'", architecture) + return } } case parseRelease: @@ -171,7 +181,8 @@ func parseSourceRPM(sourceRPM string, pkg *database.Feature) error { release = strutil.Substring(sourceRPM, i+1, previousCheckPoint) previousCheckPoint = i if release == "" { - return fmt.Errorf("unexpected package release, expect: not empty") + err = fmt.Errorf("unexpected package release, expect: not empty") + return } } case parseVersion: @@ -181,7 +192,8 @@ func parseSourceRPM(sourceRPM string, pkg *database.Feature) error { version = strutil.Substring(sourceRPM, i+1, previousCheckPoint) previousCheckPoint = i if version == "" { - return fmt.Errorf("unexpected package version, expect: not empty") + err = fmt.Errorf("unexpected package version, expect: not empty") + return } break } @@ -189,20 +201,15 @@ func parseSourceRPM(sourceRPM string, pkg *database.Feature) error { } if state != terminate { - return fmt.Errorf("unexpected termination while parsing '%s'", state) - } - - concatVersion := version + "-" + release - if err := versionfmt.Valid(rpm.ParserName, concatVersion); err != nil { - return err + err = fmt.Errorf("unexpected termination while parsing '%s'", state) + return } - name := strutil.Substring(sourceRPM, 0, previousCheckPoint) + name = strutil.Substring(sourceRPM, 0, previousCheckPoint) if name == "" { - return fmt.Errorf("unexpected package name, expect: not empty") + err = fmt.Errorf("unexpected package name, expect: not empty") + return } - pkg.SourceName = name - pkg.SourceVersion = concatVersion - return nil + return } diff --git a/ext/featurefmt/rpm/rpm_test.go b/ext/featurefmt/rpm/rpm_test.go index d727d203..ae5a70c5 100644 --- a/ext/featurefmt/rpm/rpm_test.go +++ b/ext/featurefmt/rpm/rpm_test.go @@ -25,179 +25,307 @@ import ( ) var expectedBigCaseInfo = []database.Feature{ - {"publicsuffix-list-dafsa", "20180514-1.fc28", "publicsuffix-list", "20180514-1.fc28", rpm.ParserName}, - {"libreport-filesystem", "2.9.5-1.fc28", "libreport", "2.9.5-1.fc28", rpm.ParserName}, - {"fedora-gpg-keys", "28-5", "fedora-repos", "28-5", rpm.ParserName}, - {"fedora-release", "28-2", "fedora-release", "28-2", rpm.ParserName}, - {"filesystem", "3.8-2.fc28", "filesystem", "3.8-2.fc28", rpm.ParserName}, - {"tzdata", "2018e-1.fc28", "tzdata", "2018e-1.fc28", rpm.ParserName}, - {"pcre2", "10.31-10.fc28", "pcre2", "10.31-10.fc28", rpm.ParserName}, - {"glibc-minimal-langpack", "2.27-32.fc28", "glibc", "2.27-32.fc28", rpm.ParserName}, - {"glibc-common", "2.27-32.fc28", "glibc", "2.27-32.fc28", rpm.ParserName}, - {"bash", "4.4.23-1.fc28", "bash", "4.4.23-1.fc28", rpm.ParserName}, - {"zlib", "1.2.11-8.fc28", "zlib", "1.2.11-8.fc28", rpm.ParserName}, - {"bzip2-libs", "1.0.6-26.fc28", "bzip2", "1.0.6-26.fc28", rpm.ParserName}, - {"libcap", "2.25-9.fc28", "libcap", "2.25-9.fc28", rpm.ParserName}, - {"libgpg-error", "1.31-1.fc28", "libgpg-error", "1.31-1.fc28", rpm.ParserName}, - {"libzstd", "1.3.5-1.fc28", "zstd", "1.3.5-1.fc28", rpm.ParserName}, - {"expat", "2.2.5-3.fc28", "expat", "2.2.5-3.fc28", rpm.ParserName}, - {"nss-util", "3.38.0-1.0.fc28", "nss-util", "3.38.0-1.0.fc28", rpm.ParserName}, - {"libcom_err", "1.44.2-0.fc28", "e2fsprogs", "1.44.2-0.fc28", rpm.ParserName}, - {"libffi", "3.1-16.fc28", "libffi", "3.1-16.fc28", rpm.ParserName}, - {"libgcrypt", "1.8.3-1.fc28", "libgcrypt", "1.8.3-1.fc28", rpm.ParserName}, - {"libxml2", "2.9.8-4.fc28", "libxml2", "2.9.8-4.fc28", rpm.ParserName}, - {"libacl", "2.2.53-1.fc28", "acl", "2.2.53-1.fc28", rpm.ParserName}, - {"sed", "4.5-1.fc28", "sed", "4.5-1.fc28", rpm.ParserName}, - {"libmount", "2.32.1-1.fc28", "util-linux", "2.32.1-1.fc28", rpm.ParserName}, - {"p11-kit", "0.23.12-1.fc28", "p11-kit", "0.23.12-1.fc28", rpm.ParserName}, - {"libidn2", "2.0.5-1.fc28", "libidn2", "2.0.5-1.fc28", rpm.ParserName}, - {"libcap-ng", "0.7.9-4.fc28", "libcap-ng", "0.7.9-4.fc28", rpm.ParserName}, - {"lz4-libs", "1.8.1.2-4.fc28", "lz4", "1.8.1.2-4.fc28", rpm.ParserName}, - {"libassuan", "2.5.1-3.fc28", "libassuan", "2.5.1-3.fc28", rpm.ParserName}, - {"keyutils-libs", "1.5.10-6.fc28", "keyutils", "1.5.10-6.fc28", rpm.ParserName}, - {"glib2", "2.56.1-4.fc28", "glib2", "2.56.1-4.fc28", rpm.ParserName}, - {"systemd-libs", "238-9.git0e0aa59.fc28", "systemd", "238-9.git0e0aa59.fc28", rpm.ParserName}, - {"dbus-libs", "1:1.12.10-1.fc28", "dbus", "1.12.10-1.fc28", rpm.ParserName}, - {"libtasn1", "4.13-2.fc28", "libtasn1", "4.13-2.fc28", rpm.ParserName}, - {"ca-certificates", "2018.2.24-1.0.fc28", "ca-certificates", "2018.2.24-1.0.fc28", rpm.ParserName}, - {"libarchive", "3.3.1-4.fc28", "libarchive", "3.3.1-4.fc28", rpm.ParserName}, - {"openssl", "1:1.1.0h-3.fc28", "openssl", "1.1.0h-3.fc28", rpm.ParserName}, - {"libusbx", "1.0.22-1.fc28", "libusbx", "1.0.22-1.fc28", rpm.ParserName}, - {"libsemanage", "2.8-2.fc28", "libsemanage", "2.8-2.fc28", rpm.ParserName}, - {"libutempter", "1.1.6-14.fc28", "libutempter", "1.1.6-14.fc28", rpm.ParserName}, - {"mpfr", "3.1.6-1.fc28", "mpfr", "3.1.6-1.fc28", rpm.ParserName}, - {"gnutls", "3.6.3-4.fc28", "gnutls", "3.6.3-4.fc28", rpm.ParserName}, - {"gzip", "1.9-3.fc28", "gzip", "1.9-3.fc28", rpm.ParserName}, - {"acl", "2.2.53-1.fc28", "acl", "2.2.53-1.fc28", rpm.ParserName}, - {"nss-softokn-freebl", "3.38.0-1.0.fc28", "nss-softokn", "3.38.0-1.0.fc28", rpm.ParserName}, - {"nss", "3.38.0-1.0.fc28", "nss", "3.38.0-1.0.fc28", rpm.ParserName}, - {"libmetalink", "0.1.3-6.fc28", "libmetalink", "0.1.3-6.fc28", rpm.ParserName}, - {"libdb-utils", "5.3.28-30.fc28", "libdb", "5.3.28-30.fc28", rpm.ParserName}, - {"file-libs", "5.33-7.fc28", "file", "5.33-7.fc28", rpm.ParserName}, - {"libsss_idmap", "1.16.3-2.fc28", "sssd", "1.16.3-2.fc28", rpm.ParserName}, - {"libsigsegv", "2.11-5.fc28", "libsigsegv", "2.11-5.fc28", rpm.ParserName}, - {"krb5-libs", "1.16.1-13.fc28", "krb5", "1.16.1-13.fc28", rpm.ParserName}, - {"libnsl2", "1.2.0-2.20180605git4a062cf.fc28", "libnsl2", "1.2.0-2.20180605git4a062cf.fc28", rpm.ParserName}, - {"python3-pip", "9.0.3-2.fc28", "python-pip", "9.0.3-2.fc28", rpm.ParserName}, - {"python3", "3.6.6-1.fc28", "python3", "3.6.6-1.fc28", rpm.ParserName}, - {"pam", "1.3.1-1.fc28", "pam", "1.3.1-1.fc28", rpm.ParserName}, - {"python3-gobject-base", "3.28.3-1.fc28", "pygobject3", "3.28.3-1.fc28", rpm.ParserName}, - {"python3-smartcols", "0.3.0-2.fc28", "python-smartcols", "0.3.0-2.fc28", rpm.ParserName}, - {"python3-iniparse", "0.4-30.fc28", "python-iniparse", "0.4-30.fc28", rpm.ParserName}, - {"openldap", "2.4.46-3.fc28", "openldap", "2.4.46-3.fc28", rpm.ParserName}, - {"libseccomp", "2.3.3-2.fc28", "libseccomp", "2.3.3-2.fc28", rpm.ParserName}, - {"npth", "1.5-4.fc28", "npth", "1.5-4.fc28", rpm.ParserName}, - {"gpgme", "1.10.0-4.fc28", "gpgme", "1.10.0-4.fc28", rpm.ParserName}, - {"json-c", "0.13.1-2.fc28", "json-c", "0.13.1-2.fc28", rpm.ParserName}, - {"libyaml", "0.1.7-5.fc28", "libyaml", "0.1.7-5.fc28", rpm.ParserName}, - {"libpkgconf", "1.4.2-1.fc28", "pkgconf", "1.4.2-1.fc28", rpm.ParserName}, - {"pkgconf-pkg-config", "1.4.2-1.fc28", "pkgconf", "1.4.2-1.fc28", rpm.ParserName}, - {"iptables-libs", "1.6.2-3.fc28", "iptables", "1.6.2-3.fc28", rpm.ParserName}, - {"device-mapper-libs", "1.02.146-5.fc28", "lvm2", "2.02.177-5.fc28", rpm.ParserName}, - {"systemd-pam", "238-9.git0e0aa59.fc28", "systemd", "238-9.git0e0aa59.fc28", rpm.ParserName}, - {"systemd", "238-9.git0e0aa59.fc28", "systemd", "238-9.git0e0aa59.fc28", rpm.ParserName}, - {"elfutils-default-yama-scope", "0.173-1.fc28", "elfutils", "0.173-1.fc28", rpm.ParserName}, - {"libcurl", "7.59.0-6.fc28", "curl", "7.59.0-6.fc28", rpm.ParserName}, - {"python3-librepo", "1.8.1-7.fc28", "librepo", "1.8.1-7.fc28", rpm.ParserName}, - {"rpm-plugin-selinux", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28", rpm.ParserName}, - {"rpm", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28", rpm.ParserName}, - {"libdnf", "0.11.1-3.fc28", "libdnf", "0.11.1-3.fc28", rpm.ParserName}, - {"rpm-build-libs", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28", rpm.ParserName}, - {"python3-rpm", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28", rpm.ParserName}, - {"dnf", "2.7.5-12.fc28", "dnf", "2.7.5-12.fc28", rpm.ParserName}, - {"deltarpm", "3.6-25.fc28", "deltarpm", "3.6-25.fc28", rpm.ParserName}, - {"sssd-client", "1.16.3-2.fc28", "sssd", "1.16.3-2.fc28", rpm.ParserName}, - {"cracklib-dicts", "2.9.6-13.fc28", "cracklib", "2.9.6-13.fc28", rpm.ParserName}, - {"tar", "2:1.30-3.fc28", "tar", "1.30-3.fc28", rpm.ParserName}, - {"diffutils", "3.6-4.fc28", "diffutils", "3.6-4.fc28", rpm.ParserName}, - {"langpacks-en", "1.0-12.fc28", "langpacks", "1.0-12.fc28", rpm.ParserName}, - {"libgcc", "8.1.1-5.fc28", "gcc", "8.1.1-5.fc28", rpm.ParserName}, - {"pkgconf-m4", "1.4.2-1.fc28", "pkgconf", "1.4.2-1.fc28", rpm.ParserName}, - {"dnf-conf", "2.7.5-12.fc28", "dnf", "2.7.5-12.fc28", rpm.ParserName}, - {"fedora-repos", "28-5", "fedora-repos", "28-5", rpm.ParserName}, - {"setup", "2.11.4-1.fc28", "setup", "2.11.4-1.fc28", rpm.ParserName}, - {"basesystem", "11-5.fc28", "basesystem", "11-5.fc28", rpm.ParserName}, - {"ncurses-base", "6.1-5.20180224.fc28", "ncurses", "6.1-5.20180224.fc28", rpm.ParserName}, - {"libselinux", "2.8-1.fc28", "libselinux", "2.8-1.fc28", rpm.ParserName}, - {"ncurses-libs", "6.1-5.20180224.fc28", "ncurses", "6.1-5.20180224.fc28", rpm.ParserName}, - {"glibc", "2.27-32.fc28", "glibc", "2.27-32.fc28", rpm.ParserName}, - {"libsepol", "2.8-1.fc28", "libsepol", "2.8-1.fc28", rpm.ParserName}, - {"xz-libs", "5.2.4-2.fc28", "xz", "5.2.4-2.fc28", rpm.ParserName}, - {"info", "6.5-4.fc28", "texinfo", "6.5-4.fc28", rpm.ParserName}, - {"libdb", "5.3.28-30.fc28", "libdb", "5.3.28-30.fc28", rpm.ParserName}, - {"elfutils-libelf", "0.173-1.fc28", "elfutils", "0.173-1.fc28", rpm.ParserName}, - {"popt", "1.16-14.fc28", "popt", "1.16-14.fc28", rpm.ParserName}, - {"nspr", "4.19.0-1.fc28", "nspr", "4.19.0-1.fc28", rpm.ParserName}, - {"libxcrypt", "4.1.2-1.fc28", "libxcrypt", "4.1.2-1.fc28", rpm.ParserName}, - {"lua-libs", "5.3.4-10.fc28", "lua", "5.3.4-10.fc28", rpm.ParserName}, - {"libuuid", "2.32.1-1.fc28", "util-linux", "2.32.1-1.fc28", rpm.ParserName}, - {"readline", "7.0-11.fc28", "readline", "7.0-11.fc28", rpm.ParserName}, - {"libattr", "2.4.48-3.fc28", "attr", "2.4.48-3.fc28", rpm.ParserName}, - {"coreutils-single", "8.29-7.fc28", "coreutils", "8.29-7.fc28", rpm.ParserName}, - {"libblkid", "2.32.1-1.fc28", "util-linux", "2.32.1-1.fc28", rpm.ParserName}, - {"gmp", "1:6.1.2-7.fc28", "gmp", "6.1.2-7.fc28", rpm.ParserName}, - {"libunistring", "0.9.10-1.fc28", "libunistring", "0.9.10-1.fc28", rpm.ParserName}, - {"sqlite-libs", "3.22.0-4.fc28", "sqlite", "3.22.0-4.fc28", rpm.ParserName}, - {"audit-libs", "2.8.4-2.fc28", "audit", "2.8.4-2.fc28", rpm.ParserName}, - {"chkconfig", "1.10-4.fc28", "chkconfig", "1.10-4.fc28", rpm.ParserName}, - {"libsmartcols", "2.32.1-1.fc28", "util-linux", "2.32.1-1.fc28", rpm.ParserName}, - {"pcre", "8.42-3.fc28", "pcre", "8.42-3.fc28", rpm.ParserName}, - {"grep", "3.1-5.fc28", "grep", "3.1-5.fc28", rpm.ParserName}, - {"crypto-policies", "20180425-5.git6ad4018.fc28", "crypto-policies", "20180425-5.git6ad4018.fc28", rpm.ParserName}, - {"gdbm-libs", "1:1.14.1-4.fc28", "gdbm", "1.14.1-4.fc28", rpm.ParserName}, - {"p11-kit-trust", "0.23.12-1.fc28", "p11-kit", "0.23.12-1.fc28", rpm.ParserName}, - {"openssl-libs", "1:1.1.0h-3.fc28", "openssl", "1.1.0h-3.fc28", rpm.ParserName}, - {"ima-evm-utils", "1.1-2.fc28", "ima-evm-utils", "1.1-2.fc28", rpm.ParserName}, - {"gdbm", "1:1.14.1-4.fc28", "gdbm", "1.14.1-4.fc28", rpm.ParserName}, - {"gobject-introspection", "1.56.1-1.fc28", "gobject-introspection", "1.56.1-1.fc28", rpm.ParserName}, - {"shadow-utils", "2:4.6-1.fc28", "shadow-utils", "4.6-1.fc28", rpm.ParserName}, - {"libpsl", "0.20.2-2.fc28", "libpsl", "0.20.2-2.fc28", rpm.ParserName}, - {"nettle", "3.4-2.fc28", "nettle", "3.4-2.fc28", rpm.ParserName}, - {"libfdisk", "2.32.1-1.fc28", "util-linux", "2.32.1-1.fc28", rpm.ParserName}, - {"cracklib", "2.9.6-13.fc28", "cracklib", "2.9.6-13.fc28", rpm.ParserName}, - {"libcomps", "0.1.8-11.fc28", "libcomps", "0.1.8-11.fc28", rpm.ParserName}, - {"nss-softokn", "3.38.0-1.0.fc28", "nss-softokn", "3.38.0-1.0.fc28", rpm.ParserName}, - {"nss-sysinit", "3.38.0-1.0.fc28", "nss", "3.38.0-1.0.fc28", rpm.ParserName}, - {"libksba", "1.3.5-7.fc28", "libksba", "1.3.5-7.fc28", rpm.ParserName}, - {"kmod-libs", "25-2.fc28", "kmod", "25-2.fc28", rpm.ParserName}, - {"libsss_nss_idmap", "1.16.3-2.fc28", "sssd", "1.16.3-2.fc28", rpm.ParserName}, - {"libverto", "0.3.0-5.fc28", "libverto", "0.3.0-5.fc28", rpm.ParserName}, - {"gawk", "4.2.1-1.fc28", "gawk", "4.2.1-1.fc28", rpm.ParserName}, - {"libtirpc", "1.0.3-3.rc2.fc28", "libtirpc", "1.0.3-3.rc2.fc28", rpm.ParserName}, - {"python3-libs", "3.6.6-1.fc28", "python3", "3.6.6-1.fc28", rpm.ParserName}, - {"python3-setuptools", "39.2.0-6.fc28", "python-setuptools", "39.2.0-6.fc28", rpm.ParserName}, - {"libpwquality", "1.4.0-7.fc28", "libpwquality", "1.4.0-7.fc28", rpm.ParserName}, - {"util-linux", "2.32.1-1.fc28", "util-linux", "2.32.1-1.fc28", rpm.ParserName}, - {"python3-libcomps", "0.1.8-11.fc28", "libcomps", "0.1.8-11.fc28", rpm.ParserName}, - {"python3-six", "1.11.0-3.fc28", "python-six", "1.11.0-3.fc28", rpm.ParserName}, - {"cyrus-sasl-lib", "2.1.27-0.2rc7.fc28", "cyrus-sasl", "2.1.27-0.2rc7.fc28", rpm.ParserName}, - {"libssh", "0.8.2-1.fc28", "libssh", "0.8.2-1.fc28", rpm.ParserName}, - {"qrencode-libs", "3.4.4-5.fc28", "qrencode", "3.4.4-5.fc28", rpm.ParserName}, - {"gnupg2", "2.2.8-1.fc28", "gnupg2", "2.2.8-1.fc28", rpm.ParserName}, - {"python3-gpg", "1.10.0-4.fc28", "gpgme", "1.10.0-4.fc28", rpm.ParserName}, - {"libargon2", "20161029-5.fc28", "argon2", "20161029-5.fc28", rpm.ParserName}, - {"libmodulemd", "1.6.2-2.fc28", "libmodulemd", "1.6.2-2.fc28", rpm.ParserName}, - {"pkgconf", "1.4.2-1.fc28", "pkgconf", "1.4.2-1.fc28", rpm.ParserName}, - {"libpcap", "14:1.9.0-1.fc28", "libpcap", "1.9.0-1.fc28", rpm.ParserName}, - {"device-mapper", "1.02.146-5.fc28", "lvm2", "2.02.177-5.fc28", rpm.ParserName}, - {"cryptsetup-libs", "2.0.4-1.fc28", "cryptsetup", "2.0.4-1.fc28", rpm.ParserName}, - {"elfutils-libs", "0.173-1.fc28", "elfutils", "0.173-1.fc28", rpm.ParserName}, - {"dbus", "1:1.12.10-1.fc28", "dbus", "1.12.10-1.fc28", rpm.ParserName}, - {"libnghttp2", "1.32.1-1.fc28", "nghttp2", "1.32.1-1.fc28", rpm.ParserName}, - {"librepo", "1.8.1-7.fc28", "librepo", "1.8.1-7.fc28", rpm.ParserName}, - {"curl", "7.59.0-6.fc28", "curl", "7.59.0-6.fc28", rpm.ParserName}, - {"rpm-libs", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28", rpm.ParserName}, - {"libsolv", "0.6.35-1.fc28", "libsolv", "0.6.35-1.fc28", rpm.ParserName}, - {"python3-hawkey", "0.11.1-3.fc28", "libdnf", "0.11.1-3.fc28", rpm.ParserName}, - {"rpm-sign-libs", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28", rpm.ParserName}, - {"python3-dnf", "2.7.5-12.fc28", "dnf", "2.7.5-12.fc28", rpm.ParserName}, - {"dnf-yum", "2.7.5-12.fc28", "dnf", "2.7.5-12.fc28", rpm.ParserName}, - {"rpm-plugin-systemd-inhibit", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28", rpm.ParserName}, - {"nss-tools", "3.38.0-1.0.fc28", "nss", "3.38.0-1.0.fc28", rpm.ParserName}, - {"openssl-pkcs11", "0.4.8-1.fc28", "openssl-pkcs11", "0.4.8-1.fc28", rpm.ParserName}, - {"vim-minimal", "2:8.1.328-1.fc28", "vim", "8.1.328-1.fc28", rpm.ParserName}, - {"glibc-langpack-en", "2.27-32.fc28", "glibc", "2.27-32.fc28", rpm.ParserName}, - {"rootfiles", "8.1-22.fc28", "rootfiles", "8.1-22.fc28", rpm.ParserName}, + {"libmount", "2.32.1-1.fc28", "rpm", "binary"}, + {"libffi", "3.1-16.fc28", "rpm", "binary"}, + {"libunistring", "0.9.10-1.fc28", "rpm", "binary"}, + {"fedora-repos", "28-5", "rpm", "binary"}, + {"libarchive", "3.3.1-4.fc28", "rpm", "source"}, + {"langpacks", "1.0-12.fc28", "rpm", "source"}, + {"readline", "7.0-11.fc28", "rpm", "source"}, + {"gzip", "1.9-3.fc28", "rpm", "source"}, + {"libverto", "0.3.0-5.fc28", "rpm", "source"}, + {"ncurses-base", "6.1-5.20180224.fc28", "rpm", "binary"}, + {"libfdisk", "2.32.1-1.fc28", "rpm", "binary"}, + {"libselinux", "2.8-1.fc28", "rpm", "source"}, + {"nss-util", "3.38.0-1.0.fc28", "rpm", "source"}, + {"mpfr", "3.1.6-1.fc28", "rpm", "source"}, + {"libunistring", "0.9.10-1.fc28", "rpm", "source"}, + {"libpcap", "14:1.9.0-1.fc28", "rpm", "binary"}, + {"libarchive", "3.3.1-4.fc28", "rpm", "binary"}, + {"gmp", "1:6.1.2-7.fc28", "rpm", "binary"}, + {"crypto-policies", "20180425-5.git6ad4018.fc28", "rpm", "source"}, + {"gzip", "1.9-3.fc28", "rpm", "binary"}, + {"fedora-release", "28-2", "rpm", "source"}, + {"zlib", "1.2.11-8.fc28", "rpm", "binary"}, + {"crypto-policies", "20180425-5.git6ad4018.fc28", "rpm", "binary"}, + {"lz4", "1.8.1.2-4.fc28", "rpm", "source"}, + {"keyutils", "1.5.10-6.fc28", "rpm", "source"}, + {"gpgme", "1.10.0-4.fc28", "rpm", "binary"}, + {"libgpg-error", "1.31-1.fc28", "rpm", "binary"}, + {"gnutls", "3.6.3-4.fc28", "rpm", "source"}, + {"coreutils", "8.29-7.fc28", "rpm", "source"}, + {"libsepol", "2.8-1.fc28", "rpm", "source"}, + {"libssh", "0.8.2-1.fc28", "rpm", "binary"}, + {"libpwquality", "1.4.0-7.fc28", "rpm", "binary"}, + {"dnf-conf", "2.7.5-12.fc28", "rpm", "binary"}, + {"basesystem", "11-5.fc28", "rpm", "source"}, + {"setup", "2.11.4-1.fc28", "rpm", "binary"}, + {"libmetalink", "0.1.3-6.fc28", "rpm", "source"}, + {"texinfo", "6.5-4.fc28", "rpm", "source"}, + {"expat", "2.2.5-3.fc28", "rpm", "source"}, + {"ncurses", "6.1-5.20180224.fc28", "rpm", "source"}, + {"libpwquality", "1.4.0-7.fc28", "rpm", "source"}, + {"pcre", "8.42-3.fc28", "rpm", "binary"}, + {"sssd", "1.16.3-2.fc28", "rpm", "source"}, + {"basesystem", "11-5.fc28", "rpm", "binary"}, + {"systemd-pam", "238-9.git0e0aa59.fc28", "rpm", "binary"}, + {"python3-six", "1.11.0-3.fc28", "rpm", "binary"}, + {"libcurl", "7.59.0-6.fc28", "rpm", "binary"}, + {"qrencode", "3.4.4-5.fc28", "rpm", "source"}, + {"xz", "5.2.4-2.fc28", "rpm", "source"}, + {"libpkgconf", "1.4.2-1.fc28", "rpm", "binary"}, + {"libzstd", "1.3.5-1.fc28", "rpm", "binary"}, + {"bash", "4.4.23-1.fc28", "rpm", "binary"}, + {"cyrus-sasl", "2.1.27-0.2rc7.fc28", "rpm", "source"}, + {"ncurses-libs", "6.1-5.20180224.fc28", "rpm", "binary"}, + {"xz-libs", "5.2.4-2.fc28", "rpm", "binary"}, + {"dbus", "1.12.10-1.fc28", "rpm", "source"}, + {"grep", "3.1-5.fc28", "rpm", "binary"}, + {"libusbx", "1.0.22-1.fc28", "rpm", "binary"}, + {"audit", "2.8.4-2.fc28", "rpm", "source"}, + {"sed", "4.5-1.fc28", "rpm", "binary"}, + {"sqlite", "3.22.0-4.fc28", "rpm", "source"}, + {"openldap", "2.4.46-3.fc28", "rpm", "binary"}, + {"gawk", "4.2.1-1.fc28", "rpm", "binary"}, + {"gpgme", "1.10.0-4.fc28", "rpm", "source"}, + {"lvm2", "2.02.177-5.fc28", "rpm", "source"}, + {"nspr", "4.19.0-1.fc28", "rpm", "source"}, + {"libsolv", "0.6.35-1.fc28", "rpm", "source"}, + {"info", "6.5-4.fc28", "rpm", "binary"}, + {"openssl-libs", "1:1.1.0h-3.fc28", "rpm", "binary"}, + {"libxcrypt", "4.1.2-1.fc28", "rpm", "binary"}, + {"libselinux", "2.8-1.fc28", "rpm", "binary"}, + {"libgcc", "8.1.1-5.fc28", "rpm", "binary"}, + {"cracklib", "2.9.6-13.fc28", "rpm", "binary"}, + {"python3-libs", "3.6.6-1.fc28", "rpm", "binary"}, + {"glibc-langpack-en", "2.27-32.fc28", "rpm", "binary"}, + {"json-c", "0.13.1-2.fc28", "rpm", "binary"}, + {"gnupg2", "2.2.8-1.fc28", "rpm", "source"}, + {"openssl", "1:1.1.0h-3.fc28", "rpm", "binary"}, + {"glibc-common", "2.27-32.fc28", "rpm", "binary"}, + {"p11-kit-trust", "0.23.12-1.fc28", "rpm", "binary"}, + {"zstd", "1.3.5-1.fc28", "rpm", "source"}, + {"libxml2", "2.9.8-4.fc28", "rpm", "source"}, + {"dbus", "1:1.12.10-1.fc28", "rpm", "binary"}, + {"ca-certificates", "2018.2.24-1.0.fc28", "rpm", "binary"}, + {"libcomps", "0.1.8-11.fc28", "rpm", "binary"}, + {"nss", "3.38.0-1.0.fc28", "rpm", "binary"}, + {"libcom_err", "1.44.2-0.fc28", "rpm", "binary"}, + {"keyutils-libs", "1.5.10-6.fc28", "rpm", "binary"}, + {"libseccomp", "2.3.3-2.fc28", "rpm", "binary"}, + {"elfutils-libs", "0.173-1.fc28", "rpm", "binary"}, + {"libuuid", "2.32.1-1.fc28", "rpm", "binary"}, + {"pkgconf", "1.4.2-1.fc28", "rpm", "source"}, + {"grep", "3.1-5.fc28", "rpm", "source"}, + {"libpcap", "1.9.0-1.fc28", "rpm", "source"}, + {"deltarpm", "3.6-25.fc28", "rpm", "binary"}, + {"krb5-libs", "1.16.1-13.fc28", "rpm", "binary"}, + {"glibc", "2.27-32.fc28", "rpm", "binary"}, + {"libseccomp", "2.3.3-2.fc28", "rpm", "source"}, + {"libsemanage", "2.8-2.fc28", "rpm", "binary"}, + {"openssl-pkcs11", "0.4.8-1.fc28", "rpm", "binary"}, + {"libxml2", "2.9.8-4.fc28", "rpm", "binary"}, + {"e2fsprogs", "1.44.2-0.fc28", "rpm", "source"}, + {"file-libs", "5.33-7.fc28", "rpm", "binary"}, + {"elfutils-default-yama-scope", "0.173-1.fc28", "rpm", "binary"}, + {"glibc", "2.27-32.fc28", "rpm", "source"}, + {"publicsuffix-list-dafsa", "20180514-1.fc28", "rpm", "binary"}, + {"popt", "1.16-14.fc28", "rpm", "binary"}, + {"libnsl2", "1.2.0-2.20180605git4a062cf.fc28", "rpm", "binary"}, + {"lua-libs", "5.3.4-10.fc28", "rpm", "binary"}, + {"libsemanage", "2.8-2.fc28", "rpm", "source"}, + {"glibc-minimal-langpack", "2.27-32.fc28", "rpm", "binary"}, + {"attr", "2.4.48-3.fc28", "rpm", "source"}, + {"gdbm", "1.14.1-4.fc28", "rpm", "source"}, + {"pkgconf", "1.4.2-1.fc28", "rpm", "binary"}, + {"acl", "2.2.53-1.fc28", "rpm", "source"}, + {"gnutls", "3.6.3-4.fc28", "rpm", "binary"}, + {"fedora-repos", "28-5", "rpm", "source"}, + {"python3-pip", "9.0.3-2.fc28", "rpm", "binary"}, + {"libnsl2", "1.2.0-2.20180605git4a062cf.fc28", "rpm", "source"}, + {"rpm", "4.14.1-9.fc28", "rpm", "binary"}, + {"libutempter", "1.1.6-14.fc28", "rpm", "source"}, + {"libdnf", "0.11.1-3.fc28", "rpm", "source"}, + {"vim-minimal", "2:8.1.328-1.fc28", "rpm", "binary"}, + {"tzdata", "2018e-1.fc28", "rpm", "binary"}, + {"nettle", "3.4-2.fc28", "rpm", "binary"}, + {"python-pip", "9.0.3-2.fc28", "rpm", "source"}, + {"python-six", "1.11.0-3.fc28", "rpm", "source"}, + {"diffutils", "3.6-4.fc28", "rpm", "binary"}, + {"rpm-plugin-selinux", "4.14.1-9.fc28", "rpm", "binary"}, + {"shadow-utils", "2:4.6-1.fc28", "rpm", "binary"}, + {"pkgconf-pkg-config", "1.4.2-1.fc28", "rpm", "binary"}, + {"cracklib-dicts", "2.9.6-13.fc28", "rpm", "binary"}, + {"libblkid", "2.32.1-1.fc28", "rpm", "binary"}, + {"python-setuptools", "39.2.0-6.fc28", "rpm", "source"}, + {"libsss_idmap", "1.16.3-2.fc28", "rpm", "binary"}, + {"libksba", "1.3.5-7.fc28", "rpm", "source"}, + {"sssd-client", "1.16.3-2.fc28", "rpm", "binary"}, + {"curl", "7.59.0-6.fc28", "rpm", "binary"}, + {"pam", "1.3.1-1.fc28", "rpm", "binary"}, + {"libsigsegv", "2.11-5.fc28", "rpm", "binary"}, + {"langpacks-en", "1.0-12.fc28", "rpm", "binary"}, + {"nss-softokn-freebl", "3.38.0-1.0.fc28", "rpm", "binary"}, + {"glib2", "2.56.1-4.fc28", "rpm", "binary"}, + {"python3-gobject-base", "3.28.3-1.fc28", "rpm", "binary"}, + {"libffi", "3.1-16.fc28", "rpm", "source"}, + {"libmodulemd", "1.6.2-2.fc28", "rpm", "source"}, + {"openssl", "1.1.0h-3.fc28", "rpm", "source"}, + {"libyaml", "0.1.7-5.fc28", "rpm", "source"}, + {"pam", "1.3.1-1.fc28", "rpm", "source"}, + {"iptables", "1.6.2-3.fc28", "rpm", "source"}, + {"util-linux", "2.32.1-1.fc28", "rpm", "source"}, + {"libsmartcols", "2.32.1-1.fc28", "rpm", "binary"}, + {"dnf", "2.7.5-12.fc28", "rpm", "binary"}, + {"glib2", "2.56.1-4.fc28", "rpm", "source"}, + {"lua", "5.3.4-10.fc28", "rpm", "source"}, + {"nss-softokn", "3.38.0-1.0.fc28", "rpm", "source"}, + {"python3-dnf", "2.7.5-12.fc28", "rpm", "binary"}, + {"filesystem", "3.8-2.fc28", "rpm", "binary"}, + {"libsss_nss_idmap", "1.16.3-2.fc28", "rpm", "binary"}, + {"pcre2", "10.31-10.fc28", "rpm", "source"}, + {"libyaml", "0.1.7-5.fc28", "rpm", "binary"}, + {"python3-rpm", "4.14.1-9.fc28", "rpm", "binary"}, + {"zlib", "1.2.11-8.fc28", "rpm", "source"}, + {"libutempter", "1.1.6-14.fc28", "rpm", "binary"}, + {"pcre2", "10.31-10.fc28", "rpm", "binary"}, + {"libtirpc", "1.0.3-3.rc2.fc28", "rpm", "source"}, + {"pkgconf-m4", "1.4.2-1.fc28", "rpm", "binary"}, + {"libreport", "2.9.5-1.fc28", "rpm", "source"}, + {"vim", "8.1.328-1.fc28", "rpm", "source"}, + {"file", "5.33-7.fc28", "rpm", "source"}, + {"shadow-utils", "4.6-1.fc28", "rpm", "source"}, + {"sqlite-libs", "3.22.0-4.fc28", "rpm", "binary"}, + {"setup", "2.11.4-1.fc28", "rpm", "source"}, + {"gcc", "8.1.1-5.fc28", "rpm", "source"}, + {"mpfr", "3.1.6-1.fc28", "rpm", "binary"}, + {"device-mapper", "1.02.146-5.fc28", "rpm", "binary"}, + {"p11-kit", "0.23.12-1.fc28", "rpm", "source"}, + {"fedora-release", "28-2", "rpm", "binary"}, + {"libnghttp2", "1.32.1-1.fc28", "rpm", "binary"}, + {"libcap-ng", "0.7.9-4.fc28", "rpm", "source"}, + {"iptables-libs", "1.6.2-3.fc28", "rpm", "binary"}, + {"audit-libs", "2.8.4-2.fc28", "rpm", "binary"}, + {"libsigsegv", "2.11-5.fc28", "rpm", "source"}, + {"rootfiles", "8.1-22.fc28", "rpm", "source"}, + {"kmod-libs", "25-2.fc28", "rpm", "binary"}, + {"lz4-libs", "1.8.1.2-4.fc28", "rpm", "binary"}, + {"libassuan", "2.5.1-3.fc28", "rpm", "source"}, + {"p11-kit", "0.23.12-1.fc28", "rpm", "binary"}, + {"nss-sysinit", "3.38.0-1.0.fc28", "rpm", "binary"}, + {"libcap-ng", "0.7.9-4.fc28", "rpm", "binary"}, + {"bash", "4.4.23-1.fc28", "rpm", "source"}, + {"pygobject3", "3.28.3-1.fc28", "rpm", "source"}, + {"dnf-yum", "2.7.5-12.fc28", "rpm", "binary"}, + {"nss-softokn", "3.38.0-1.0.fc28", "rpm", "binary"}, + {"expat", "2.2.5-3.fc28", "rpm", "binary"}, + {"libassuan", "2.5.1-3.fc28", "rpm", "binary"}, + {"libdb", "5.3.28-30.fc28", "rpm", "binary"}, + {"tar", "2:1.30-3.fc28", "rpm", "binary"}, + {"sed", "4.5-1.fc28", "rpm", "source"}, + {"libmetalink", "0.1.3-6.fc28", "rpm", "binary"}, + {"python-smartcols", "0.3.0-2.fc28", "rpm", "source"}, + {"systemd", "238-9.git0e0aa59.fc28", "rpm", "source"}, + {"python-iniparse", "0.4-30.fc28", "rpm", "source"}, + {"libsepol", "2.8-1.fc28", "rpm", "binary"}, + {"libattr", "2.4.48-3.fc28", "rpm", "binary"}, + {"python3-smartcols", "0.3.0-2.fc28", "rpm", "binary"}, + {"libdb", "5.3.28-30.fc28", "rpm", "source"}, + {"libmodulemd", "1.6.2-2.fc28", "rpm", "binary"}, + {"python3-hawkey", "0.11.1-3.fc28", "rpm", "binary"}, + {"dbus-libs", "1:1.12.10-1.fc28", "rpm", "binary"}, + {"chkconfig", "1.10-4.fc28", "rpm", "source"}, + {"libargon2", "20161029-5.fc28", "rpm", "binary"}, + {"openssl-pkcs11", "0.4.8-1.fc28", "rpm", "source"}, + {"libusbx", "1.0.22-1.fc28", "rpm", "source"}, + {"python3-setuptools", "39.2.0-6.fc28", "rpm", "binary"}, + {"chkconfig", "1.10-4.fc28", "rpm", "binary"}, + {"openldap", "2.4.46-3.fc28", "rpm", "source"}, + {"bzip2", "1.0.6-26.fc28", "rpm", "source"}, + {"npth", "1.5-4.fc28", "rpm", "source"}, + {"libtirpc", "1.0.3-3.rc2.fc28", "rpm", "binary"}, + {"util-linux", "2.32.1-1.fc28", "rpm", "binary"}, + {"nss", "3.38.0-1.0.fc28", "rpm", "source"}, + {"elfutils", "0.173-1.fc28", "rpm", "source"}, + {"libcomps", "0.1.8-11.fc28", "rpm", "source"}, + {"libxcrypt", "4.1.2-1.fc28", "rpm", "source"}, + {"gnupg2", "2.2.8-1.fc28", "rpm", "binary"}, + {"libdnf", "0.11.1-3.fc28", "rpm", "binary"}, + {"cracklib", "2.9.6-13.fc28", "rpm", "source"}, + {"libidn2", "2.0.5-1.fc28", "rpm", "source"}, + {"bzip2-libs", "1.0.6-26.fc28", "rpm", "binary"}, + {"json-c", "0.13.1-2.fc28", "rpm", "source"}, + {"gdbm", "1:1.14.1-4.fc28", "rpm", "binary"}, + {"pcre", "8.42-3.fc28", "rpm", "source"}, + {"systemd", "238-9.git0e0aa59.fc28", "rpm", "binary"}, + {"cryptsetup-libs", "2.0.4-1.fc28", "rpm", "binary"}, + {"dnf", "2.7.5-12.fc28", "rpm", "source"}, + {"ca-certificates", "2018.2.24-1.0.fc28", "rpm", "source"}, + {"libidn2", "2.0.5-1.fc28", "rpm", "binary"}, + {"libpsl", "0.20.2-2.fc28", "rpm", "binary"}, + {"gdbm-libs", "1:1.14.1-4.fc28", "rpm", "binary"}, + {"kmod", "25-2.fc28", "rpm", "source"}, + {"libreport-filesystem", "2.9.5-1.fc28", "rpm", "binary"}, + {"ima-evm-utils", "1.1-2.fc28", "rpm", "source"}, + {"nghttp2", "1.32.1-1.fc28", "rpm", "source"}, + {"cyrus-sasl-lib", "2.1.27-0.2rc7.fc28", "rpm", "binary"}, + {"libsolv", "0.6.35-1.fc28", "rpm", "binary"}, + {"cryptsetup", "2.0.4-1.fc28", "rpm", "source"}, + {"filesystem", "3.8-2.fc28", "rpm", "source"}, + {"libcap", "2.25-9.fc28", "rpm", "source"}, + {"libpsl", "0.20.2-2.fc28", "rpm", "source"}, + {"deltarpm", "3.6-25.fc28", "rpm", "source"}, + {"fedora-gpg-keys", "28-5", "rpm", "binary"}, + {"ima-evm-utils", "1.1-2.fc28", "rpm", "binary"}, + {"nss-tools", "3.38.0-1.0.fc28", "rpm", "binary"}, + {"libtasn1", "4.13-2.fc28", "rpm", "source"}, + {"elfutils-libelf", "0.173-1.fc28", "rpm", "binary"}, + {"device-mapper-libs", "1.02.146-5.fc28", "rpm", "binary"}, + {"gobject-introspection", "1.56.1-1.fc28", "rpm", "source"}, + {"publicsuffix-list", "20180514-1.fc28", "rpm", "source"}, + {"libcap", "2.25-9.fc28", "rpm", "binary"}, + {"librepo", "1.8.1-7.fc28", "rpm", "binary"}, + {"rpm-sign-libs", "4.14.1-9.fc28", "rpm", "binary"}, + {"coreutils-single", "8.29-7.fc28", "rpm", "binary"}, + {"libacl", "2.2.53-1.fc28", "rpm", "binary"}, + {"popt", "1.16-14.fc28", "rpm", "source"}, + {"libtasn1", "4.13-2.fc28", "rpm", "binary"}, + {"gawk", "4.2.1-1.fc28", "rpm", "source"}, + {"diffutils", "3.6-4.fc28", "rpm", "source"}, + {"libgpg-error", "1.31-1.fc28", "rpm", "source"}, + {"libdb-utils", "5.3.28-30.fc28", "rpm", "binary"}, + {"python3-iniparse", "0.4-30.fc28", "rpm", "binary"}, + {"acl", "2.2.53-1.fc28", "rpm", "binary"}, + {"libssh", "0.8.2-1.fc28", "rpm", "source"}, + {"python3-librepo", "1.8.1-7.fc28", "rpm", "binary"}, + {"gobject-introspection", "1.56.1-1.fc28", "rpm", "binary"}, + {"rpm", "4.14.1-9.fc28", "rpm", "source"}, + {"libgcrypt", "1.8.3-1.fc28", "rpm", "source"}, + {"curl", "7.59.0-6.fc28", "rpm", "source"}, + {"tzdata", "2018e-1.fc28", "rpm", "source"}, + {"krb5", "1.16.1-13.fc28", "rpm", "source"}, + {"librepo", "1.8.1-7.fc28", "rpm", "source"}, + {"python3-gpg", "1.10.0-4.fc28", "rpm", "binary"}, + {"nettle", "3.4-2.fc28", "rpm", "source"}, + {"libgcrypt", "1.8.3-1.fc28", "rpm", "binary"}, + {"python3", "3.6.6-1.fc28", "rpm", "binary"}, + {"python3-libcomps", "0.1.8-11.fc28", "rpm", "binary"}, + {"rpm-libs", "4.14.1-9.fc28", "rpm", "binary"}, + {"nspr", "4.19.0-1.fc28", "rpm", "binary"}, + {"argon2", "20161029-5.fc28", "rpm", "source"}, + {"tar", "1.30-3.fc28", "rpm", "source"}, + {"qrencode-libs", "3.4.4-5.fc28", "rpm", "binary"}, + {"gmp", "6.1.2-7.fc28", "rpm", "source"}, + {"libverto", "0.3.0-5.fc28", "rpm", "binary"}, + {"python3", "3.6.6-1.fc28", "rpm", "source"}, + {"libksba", "1.3.5-7.fc28", "rpm", "binary"}, + {"readline", "7.0-11.fc28", "rpm", "binary"}, + {"rpm-build-libs", "4.14.1-9.fc28", "rpm", "binary"}, + {"npth", "1.5-4.fc28", "rpm", "binary"}, + {"rootfiles", "8.1-22.fc28", "rpm", "binary"}, + {"rpm-plugin-systemd-inhibit", "4.14.1-9.fc28", "rpm", "binary"}, + {"systemd-libs", "238-9.git0e0aa59.fc28", "rpm", "binary"}, + {"nss-util", "3.38.0-1.0.fc28", "rpm", "binary"}, } func TestRpmFeatureDetection(t *testing.T) { @@ -206,8 +334,10 @@ func TestRpmFeatureDetection(t *testing.T) { "valid small case", map[string]string{"var/lib/rpm/Packages": "rpm/testdata/valid"}, []database.Feature{ - {"centos-release", "7-1.1503.el7.centos.2.8", "centos-release", "7-1.1503.el7.centos.2.8", rpm.ParserName}, - {"filesystem", "3.2-18.el7", "filesystem", "3.2-18.el7", rpm.ParserName}, + {"centos-release", "7-1.1503.el7.centos.2.8", "rpm", "binary"}, + {"filesystem", "3.2-18.el7", "rpm", "binary"}, + {"centos-release", "7-1.1503.el7.centos.2.8", "rpm", "source"}, + {"filesystem", "3.2-18.el7", "rpm", "source"}, }, }, { @@ -248,15 +378,14 @@ func TestParseSourceRPM(t *testing.T) { // actual expected: name="lua", version="5.3.4", release="10.fc-28" {"lua-5.3.4-10.fc-28.src.rpm", "lua-5.3.4", "10.fc-28", ""}, } { - pkg := database.Feature{} - err := parseSourceRPM(test.sourceRPM, &pkg) + name, version, release, _, err := parseSourceRPM(test.sourceRPM) if test.expectedErr != "" { require.EqualError(t, err, test.expectedErr) continue } require.Nil(t, err) - require.Equal(t, test.expectedName, pkg.SourceName) - require.Equal(t, test.expectedVersion, pkg.SourceVersion) + require.Equal(t, test.expectedName, name) + require.Equal(t, test.expectedVersion, version+"-"+release) } } From f61675355e7a296989e778f37257e6e416e6f208 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 19 Feb 2019 16:39:37 -0500 Subject: [PATCH 05/12] database: Update feature model Remove source name/version fields Add Type field to indicate if it's binary package or source package --- database/models.go | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/database/models.go b/database/models.go index 41042260..f1c7a3c0 100644 --- a/database/models.go +++ b/database/models.go @@ -155,18 +155,33 @@ type Namespace struct { VersionFormat string } +func NewNamespace(name string, versionFormat string) *Namespace { + return &Namespace{name, versionFormat} +} + // Feature represents a package detected in a layer but the namespace is not // determined. // -// e.g. Name: Libssl1.0, Version: 1.0, Name: Openssl, Version: 1.0, VersionFormat: dpkg. +// e.g. Name: Libssl1.0, Version: 1.0, VersionFormat: dpkg, Type: binary // dpkg is the version format of the installer package manager, which in this // case could be dpkg or apk. type Feature struct { Name string Version string - SourceName string - SourceVersion string VersionFormat string + Type FeatureType +} + +func NewFeature(name string, version string, versionFormat string, featureType FeatureType) *Feature { + return &Feature{name, version, versionFormat, featureType} +} + +func NewBinaryPackage(name string, version string, versionFormat string) *Feature { + return &Feature{name, version, versionFormat, BinaryPackage} +} + +func NewSourcePackage(name string, version string, versionFormat string) *Feature { + return &Feature{name, version, versionFormat, SourcePackage} } // NamespacedFeature is a feature with determined namespace and can be affected @@ -179,6 +194,11 @@ type NamespacedFeature struct { Namespace Namespace } +func NewNamespacedFeature(namespace *Namespace, feature *Feature) *NamespacedFeature { + // TODO: namespaced feature should use pointer values + return &NamespacedFeature{*feature, *namespace} +} + // AffectedNamespacedFeature is a namespaced feature affected by the // vulnerabilities with fixed-in versions for this feature. type AffectedNamespacedFeature struct { From 073c685c5b085813a9ffbec20fa3c49332f7ec66 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 19 Feb 2019 16:40:22 -0500 Subject: [PATCH 06/12] pgsql: Add proper tests for database migration --- database/pgsql/migrations_test.go | 60 +++++++++++ database/pgsql/testutil.go | 169 +++++++++++++++++++++++++++++- 2 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 database/pgsql/migrations_test.go diff --git a/database/pgsql/migrations_test.go b/database/pgsql/migrations_test.go new file mode 100644 index 00000000..e3b2eb30 --- /dev/null +++ b/database/pgsql/migrations_test.go @@ -0,0 +1,60 @@ +// Copyright 2019 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pgsql + +import ( + "testing" + + _ "github.com/lib/pq" + "github.com/remind101/migrate" + "github.com/stretchr/testify/require" + + "github.com/coreos/clair/database/pgsql/migrations" +) + +var userTableCount = `SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname='public'` + +func TestMigration(t *testing.T) { + db, cleanup := createAndConnectTestDB(t, "TestMigration") + defer cleanup() + + err := migrate.NewPostgresMigrator(db).Exec(migrate.Up, migrations.Migrations...) + if err != nil { + require.Nil(t, err, err.Error()) + } + + err = migrate.NewPostgresMigrator(db).Exec(migrate.Down, migrations.Migrations...) + if err != nil { + require.Nil(t, err, err.Error()) + } + + rows, err := db.Query(userTableCount) + if err != nil { + panic(err) + } + + var ( + tables []string + table string + ) + for rows.Next() { + if err = rows.Scan(&table); err != nil { + panic(err) + } + tables = append(tables, table) + } + + require.True(t, len(tables) == 1 && tables[0] == "schema_migrations", "Only `schema_migrations` should be left") +} diff --git a/database/pgsql/testutil.go b/database/pgsql/testutil.go index 45765b35..abfb743a 100644 --- a/database/pgsql/testutil.go +++ b/database/pgsql/testutil.go @@ -15,21 +15,34 @@ package pgsql import ( + "database/sql" + "fmt" + "io/ioutil" + "math/rand" + "os" + "path/filepath" + "runtime" + "strings" "testing" + "time" + "github.com/remind101/migrate" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/coreos/clair/database" + "github.com/coreos/clair/database/pgsql/migrations" "github.com/coreos/clair/pkg/pagination" ) // int keys must be the consistent with the database ID. var ( realFeatures = map[int]database.Feature{ - 1: {"ourchat", "0.5", "ourchat", "0.5", "dpkg"}, - 2: {"openssl", "1.0", "openssl", "1.0", "dpkg"}, - 3: {"openssl", "2.0", "openssl", "2.0", "dpkg"}, - 4: {"fake", "2.0", "fake", "2.0", "rpm"}, + 1: {"ourchat", "0.5", "dpkg", "source"}, + 2: {"openssl", "1.0", "dpkg", "source"}, + 3: {"openssl", "2.0", "dpkg", "source"}, + 4: {"fake", "2.0", "rpm", "source"}, + 5: {"mount", "2.31.1-0.4ubuntu3.1", "dpkg", "binary"}, } realNamespaces = map[int]database.Namespace{ @@ -146,6 +159,7 @@ var ( Name: "ourchat", Version: "0.6", VersionFormat: "dpkg", + Type: "source", }, } @@ -260,3 +274,150 @@ func mustMarshalToken(key pagination.Key, v interface{}) pagination.Token { return token } + +var userDBCount = `SELECT count(datname) FROM pg_database WHERE datistemplate = FALSE AND datname != 'postgres';` + +func createAndConnectTestDB(t *testing.T, testName string) (*sql.DB, func()) { + uri := "postgres@127.0.0.1:5432" + connectionTemplate := "postgresql://%s?sslmode=disable" + if envURI := os.Getenv("CLAIR_TEST_PGSQL"); envURI != "" { + uri = envURI + } + + db, err := sql.Open("postgres", fmt.Sprintf(connectionTemplate, uri)) + if err != nil { + panic(err) + } + + testName = strings.ToLower(testName) + dbName := fmt.Sprintf("test_%s_%s", testName, time.Now().UTC().Format("2006_01_02_15_04_05")) + t.Logf("creating temporary database name = %s", dbName) + _, err = db.Exec("CREATE DATABASE " + dbName) + if err != nil { + panic(err) + } + + testDB, err := sql.Open("postgres", fmt.Sprintf(connectionTemplate, uri+"/"+dbName)) + if err != nil { + panic(err) + } + + return testDB, func() { + t.Logf("cleaning up temporary database %s", dbName) + defer db.Close() + if err := testDB.Close(); err != nil { + panic(err) + } + + if _, err := db.Exec(`DROP DATABASE ` + dbName); err != nil { + panic(err) + } + + // ensure the database is cleaned up + var count int + if err := db.QueryRow(userDBCount).Scan(&count); err != nil { + panic(err) + } + } +} + +func createTestPgSQL(t *testing.T, testName string) (*pgSQL, func()) { + connection, cleanup := createAndConnectTestDB(t, testName) + err := migrate.NewPostgresMigrator(connection).Exec(migrate.Up, migrations.Migrations...) + if err != nil { + require.Nil(t, err, err.Error()) + } + + return &pgSQL{connection, nil, Config{PaginationKey: pagination.Must(pagination.NewKey()).String()}}, cleanup +} + +func createTestPgSQLWithFixtures(t *testing.T, testName string) (*pgSQL, func()) { + connection, cleanup := createTestPgSQL(t, testName) + session, err := connection.Begin() + if err != nil { + panic(err) + } + + defer session.Rollback() + + loadFixtures(session.(*pgSession)) + if err = session.Commit(); err != nil { + panic(err) + } + + return connection, cleanup +} + +func createTestPgSession(t *testing.T, testName string) (*pgSession, func()) { + connection, cleanup := createTestPgSQL(t, testName) + session, err := connection.Begin() + if err != nil { + panic(err) + } + + return session.(*pgSession), func() { + session.Rollback() + cleanup() + } +} + +func createTestPgSessionWithFixtures(t *testing.T, testName string) (*pgSession, func()) { + tx, cleanup := createTestPgSession(t, testName) + defer func() { + // ensure to cleanup when loadFixtures failed + if r := recover(); r != nil { + cleanup() + } + }() + + loadFixtures(tx) + return tx, cleanup +} + +func loadFixtures(tx *pgSession) { + _, filename, _, _ := runtime.Caller(0) + fixturePath := filepath.Join(filepath.Dir(filename)) + "/testdata/data.sql" + d, err := ioutil.ReadFile(fixturePath) + if err != nil { + panic(err) + } + + _, err = tx.Exec(string(d)) + if err != nil { + panic(err) + } +} + +func assertVulnerabilityWithAffectedEqual(t *testing.T, expected database.VulnerabilityWithAffected, actual database.VulnerabilityWithAffected) bool { + return assert.Equal(t, expected.Vulnerability, actual.Vulnerability) && assertAffectedFeaturesEqual(t, expected.Affected, actual.Affected) +} + +func assertAffectedFeaturesEqual(t *testing.T, expected []database.AffectedFeature, actual []database.AffectedFeature) bool { + if assert.Len(t, actual, len(expected)) { + has := map[database.AffectedFeature]bool{} + for _, i := range expected { + has[i] = false + } + for _, i := range actual { + if visited, ok := has[i]; !ok { + return false + } else if visited { + return false + } + has[i] = true + } + return true + } + return false +} + +func genRandomNamespaces(t *testing.T, count int) []database.Namespace { + r := make([]database.Namespace, count) + for i := 0; i < count; i++ { + r[i] = database.Namespace{ + Name: fmt.Sprint(rand.Int()), + VersionFormat: "dpkg", + } + } + return r +} From 5fa1ac89b9946f2e32ac666080b4f78ad1f9bbfa Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 19 Feb 2019 16:41:06 -0500 Subject: [PATCH 07/12] database: Add StorageError type --- database/database.go | 9 ++++----- database/error.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 database/error.go diff --git a/database/database.go b/database/database.go index f760061f..0c479575 100644 --- a/database/database.go +++ b/database/database.go @@ -17,7 +17,6 @@ package database import ( - "errors" "fmt" "time" @@ -27,20 +26,20 @@ import ( var ( // ErrBackendException is an error that occurs when the database backend // does not work properly (ie. unreachable). - ErrBackendException = errors.New("database: an error occurred when querying the backend") + ErrBackendException = NewStorageError("an error occurred when querying the backend") // ErrInconsistent is an error that occurs when a database consistency check // fails (i.e. when an entity which is supposed to be unique is detected // twice) - ErrInconsistent = errors.New("database: inconsistent database") + ErrInconsistent = NewStorageError("inconsistent database") // ErrInvalidParameters is an error that occurs when the parameters are not valid. - ErrInvalidParameters = errors.New("database: parameters are not valid") + ErrInvalidParameters = NewStorageError("parameters are not valid") // ErrMissingEntities is an error that occurs when an associated immutable // entity doesn't exist in the database. This error can indicate a wrong // implementation or corrupted database. - ErrMissingEntities = errors.New("database: associated immutable entities are missing in the database") + ErrMissingEntities = NewStorageError("associated immutable entities are missing in the database") ) // RegistrableComponentConfig is a configuration block that can be used to diff --git a/database/error.go b/database/error.go new file mode 100644 index 00000000..bfbe3af6 --- /dev/null +++ b/database/error.go @@ -0,0 +1,35 @@ +// Copyright 2019 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +// StorageError is database error +type StorageError struct { + reason string + original error +} + +func (e *StorageError) Error() string { + return e.reason +} + +// NewStorageErrorWithInternalError creates a new database error +func NewStorageErrorWithInternalError(reason string, originalError error) *StorageError { + return &StorageError{reason, originalError} +} + +// NewStorageError creates a new database error +func NewStorageError(reason string) *StorageError { + return &StorageError{reason, nil} +} From 79af05e67d6e6f09bd1913dbfe405ebdbd9a9c59 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 19 Feb 2019 16:42:00 -0500 Subject: [PATCH 08/12] pgsql: Fix postgres queries for feature_type --- database/pgsql/ancestry.go | 6 +- database/pgsql/complex_test.go | 75 ++++------- database/pgsql/feature.go | 94 +++++++------- database/pgsql/feature_test.go | 185 +++++++++++---------------- database/pgsql/layer.go | 7 +- database/pgsql/layer_test.go | 8 +- database/pgsql/notification_test.go | 6 +- database/pgsql/pgsql_test.go | 1 + database/pgsql/queries.go | 20 +-- database/pgsql/testdata/data.sql | 17 +-- database/pgsql/vulnerability.go | 22 ++-- database/pgsql/vulnerability_test.go | 28 +--- 12 files changed, 197 insertions(+), 272 deletions(-) diff --git a/database/pgsql/ancestry.go b/database/pgsql/ancestry.go index fa0c0ad5..90ddd811 100644 --- a/database/pgsql/ancestry.go +++ b/database/pgsql/ancestry.go @@ -23,10 +23,11 @@ const ( findAncestryFeatures = ` SELECT namespace.name, namespace.version_format, feature.name, - feature.version, feature.version_format, ancestry_layer.ancestry_index, + feature.version, feature.version_format, feature_type.name, ancestry_layer.ancestry_index, ancestry_feature.feature_detector_id, ancestry_feature.namespace_detector_id - FROM namespace, feature, namespaced_feature, ancestry_layer, ancestry_feature + FROM namespace, feature, feature_type, namespaced_feature, ancestry_layer, ancestry_feature WHERE ancestry_layer.ancestry_id = $1 + AND feature_type.id = feature.type AND ancestry_feature.ancestry_layer_id = ancestry_layer.id AND ancestry_feature.namespaced_feature_id = namespaced_feature.id AND namespaced_feature.feature_id = feature.id @@ -256,6 +257,7 @@ func (tx *pgSession) findAncestryFeatures(ancestryID int64, detectors detectorMa &feature.Feature.Name, &feature.Feature.Version, &feature.Feature.VersionFormat, + &feature.Feature.Type, &index, &featureDetectorID, &namespaceDetectorID, diff --git a/database/pgsql/complex_test.go b/database/pgsql/complex_test.go index de8b0f20..e4a10928 100644 --- a/database/pgsql/complex_test.go +++ b/database/pgsql/complex_test.go @@ -25,6 +25,7 @@ import ( "github.com/pborman/uuid" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" @@ -65,19 +66,13 @@ func testGenRandomVulnerabilityAndNamespacedFeature(t *testing.T, store database for i := 0; i < numFeatures; i++ { version := rand.Intn(numFeatures) - features[i] = database.Feature{ - Name: featureName, - VersionFormat: featureVersionFormat, - Version: strconv.Itoa(version), - } - + features[i] = *database.NewSourcePackage(featureName, strconv.Itoa(version), featureVersionFormat) nsFeatures[i] = database.NamespacedFeature{ Namespace: namespace, Feature: features[i], } } - // insert features if !assert.Nil(t, tx.PersistFeatures(features)) { t.FailNow() } @@ -98,6 +93,7 @@ func testGenRandomVulnerabilityAndNamespacedFeature(t *testing.T, store database { Namespace: namespace, FeatureName: featureName, + FeatureType: database.SourcePackage, AffectedVersion: strconv.Itoa(version), FixedInVersion: strconv.Itoa(version), }, @@ -117,7 +113,6 @@ func TestConcurrency(t *testing.T) { t.FailNow() } defer store.Close() - start := time.Now() var wg sync.WaitGroup wg.Add(100) @@ -137,65 +132,39 @@ func TestConcurrency(t *testing.T) { fmt.Println("total", time.Since(start)) } -func genRandomNamespaces(t *testing.T, count int) []database.Namespace { - r := make([]database.Namespace, count) - for i := 0; i < count; i++ { - r[i] = database.Namespace{ - Name: uuid.New(), - VersionFormat: "dpkg", - } - } - return r -} - func TestCaching(t *testing.T) { store, err := openDatabaseForTest("Caching", false) if !assert.Nil(t, err) { t.FailNow() } defer store.Close() - nsFeatures, vulnerabilities := testGenRandomVulnerabilityAndNamespacedFeature(t, store) + tx, err := store.Begin() + if !assert.Nil(t, err) { + t.FailNow() + } - fmt.Printf("%d features, %d vulnerabilities are generated", len(nsFeatures), len(vulnerabilities)) - - var wg sync.WaitGroup - wg.Add(2) - go func() { - defer wg.Done() - tx, err := store.Begin() - if !assert.Nil(t, err) { - t.FailNow() - } - - assert.Nil(t, tx.PersistNamespacedFeatures(nsFeatures)) - fmt.Println("finished to insert namespaced features") - - tx.Commit() - }() - - go func() { - defer wg.Done() - tx, err := store.Begin() - if !assert.Nil(t, err) { - t.FailNow() - } - - assert.Nil(t, tx.InsertVulnerabilities(vulnerabilities)) - fmt.Println("finished to insert vulnerabilities") - tx.Commit() + require.Nil(t, tx.PersistNamespacedFeatures(nsFeatures)) + if err := tx.Commit(); err != nil { + panic(err) + } - }() + tx, err = store.Begin() + if !assert.Nil(t, err) { + t.FailNow() + } - wg.Wait() + require.Nil(t, tx.InsertVulnerabilities(vulnerabilities)) + if err := tx.Commit(); err != nil { + panic(err) + } - tx, err := store.Begin() + tx, err = store.Begin() if !assert.Nil(t, err) { t.FailNow() } defer tx.Rollback() - // Verify consistency now. affected, err := tx.FindAffectedNamespacedFeatures(nsFeatures) if !assert.Nil(t, err) { t.FailNow() @@ -220,7 +189,7 @@ func TestCaching(t *testing.T) { actualAffectedNames = append(actualAffectedNames, s.Name) } - assert.Len(t, strutil.Difference(expectedAffectedNames, actualAffectedNames), 0) - assert.Len(t, strutil.Difference(actualAffectedNames, expectedAffectedNames), 0) + require.Len(t, strutil.Difference(expectedAffectedNames, actualAffectedNames), 0, "\nvulns: %#v\nfeature:%#v\nexpected:%#v\nactual:%#v", vulnerabilities, ansf.NamespacedFeature, expectedAffectedNames, actualAffectedNames) + require.Len(t, strutil.Difference(actualAffectedNames, expectedAffectedNames), 0) } } diff --git a/database/pgsql/feature.go b/database/pgsql/feature.go index f716130f..3df7ac6a 100644 --- a/database/pgsql/feature.go +++ b/database/pgsql/feature.go @@ -46,6 +46,7 @@ const ( AND nf.feature_id = f.id AND nf.namespace_id = v.namespace_id AND vaf.feature_name = f.name + AND vaf.feature_type = f.type AND vaf.vulnerability_id = v.id AND v.deleted_at IS NULL` @@ -68,6 +69,11 @@ func (tx *pgSession) PersistFeatures(features []database.Feature) error { return nil } + types, err := tx.getFeatureTypeMap() + if err != nil { + return err + } + // Sorting is needed before inserting into database to prevent deadlock. sort.Slice(features, func(i, j int) bool { return features[i].Name < features[j].Name || @@ -78,13 +84,13 @@ func (tx *pgSession) PersistFeatures(features []database.Feature) error { // TODO(Sida): A better interface for bulk insertion is needed. keys := make([]interface{}, 0, len(features)*3) for _, f := range features { - keys = append(keys, f.Name, f.Version, f.VersionFormat) + keys = append(keys, f.Name, f.Version, f.VersionFormat, types.byName[f.Type]) if f.Name == "" || f.Version == "" || f.VersionFormat == "" { return commonerr.NewBadRequestError("Empty feature name, version or version format is not allowed") } } - _, err := tx.Exec(queryPersistFeature(len(features)), keys...) + _, err = tx.Exec(queryPersistFeature(len(features)), keys...) return handleError("queryPersistFeature", err) } @@ -240,55 +246,31 @@ func (tx *pgSession) PersistNamespacedFeatures(features []database.NamespacedFea return nil } -// FindAffectedNamespacedFeatures looks up cache table and retrieves all -// vulnerabilities associated with the features. +// FindAffectedNamespacedFeatures retrieves vulnerabilities associated with the +// feature. func (tx *pgSession) FindAffectedNamespacedFeatures(features []database.NamespacedFeature) ([]database.NullableAffectedNamespacedFeature, error) { if len(features) == 0 { return nil, nil } - returnFeatures := make([]database.NullableAffectedNamespacedFeature, len(features)) - - // featureMap is used to keep track of duplicated features. - featureMap := map[database.NamespacedFeature][]*database.NullableAffectedNamespacedFeature{} - // initialize return value and generate unique feature request queries. - for i, f := range features { - returnFeatures[i] = database.NullableAffectedNamespacedFeature{ - AffectedNamespacedFeature: database.AffectedNamespacedFeature{ - NamespacedFeature: f, - }, - } - - featureMap[f] = append(featureMap[f], &returnFeatures[i]) - } - - // query unique namespaced features - distinctFeatures := []database.NamespacedFeature{} - for f := range featureMap { - distinctFeatures = append(distinctFeatures, f) - } - - nsFeatureIDs, err := tx.findNamespacedFeatureIDs(distinctFeatures) + vulnerableFeatures := make([]database.NullableAffectedNamespacedFeature, len(features)) + featureIDs, err := tx.findNamespacedFeatureIDs(features) if err != nil { return nil, err } - toQuery := []int64{} - featureIDMap := map[int64][]*database.NullableAffectedNamespacedFeature{} - for i, id := range nsFeatureIDs { + for i, id := range featureIDs { if id.Valid { - toQuery = append(toQuery, id.Int64) - for _, f := range featureMap[distinctFeatures[i]] { - f.Valid = id.Valid - featureIDMap[id.Int64] = append(featureIDMap[id.Int64], f) - } + vulnerableFeatures[i].Valid = true + vulnerableFeatures[i].NamespacedFeature = features[i] } } - rows, err := tx.Query(searchNamespacedFeaturesVulnerabilities, pq.Array(toQuery)) + rows, err := tx.Query(searchNamespacedFeaturesVulnerabilities, pq.Array(featureIDs)) if err != nil { return nil, handleError("searchNamespacedFeaturesVulnerabilities", err) } + defer rows.Close() for rows.Next() { @@ -296,6 +278,7 @@ func (tx *pgSession) FindAffectedNamespacedFeatures(features []database.Namespac featureID int64 vuln database.VulnerabilityWithFixedIn ) + err := rows.Scan(&featureID, &vuln.Name, &vuln.Description, @@ -306,16 +289,19 @@ func (tx *pgSession) FindAffectedNamespacedFeatures(features []database.Namespac &vuln.Namespace.Name, &vuln.Namespace.VersionFormat, ) + if err != nil { return nil, handleError("searchNamespacedFeaturesVulnerabilities", err) } - for _, f := range featureIDMap[featureID] { - f.AffectedBy = append(f.AffectedBy, vuln) + for i, id := range featureIDs { + if id.Valid && id.Int64 == featureID { + vulnerableFeatures[i].AffectedNamespacedFeature.AffectedBy = append(vulnerableFeatures[i].AffectedNamespacedFeature.AffectedBy, vuln) + } } } - return returnFeatures, nil + return vulnerableFeatures, nil } func (tx *pgSession) findNamespacedFeatureIDs(nfs []database.NamespacedFeature) ([]sql.NullInt64, error) { @@ -323,11 +309,10 @@ func (tx *pgSession) findNamespacedFeatureIDs(nfs []database.NamespacedFeature) return nil, nil } - nfsMap := map[database.NamespacedFeature]sql.NullInt64{} - keys := make([]interface{}, 0, len(nfs)*4) + nfsMap := map[database.NamespacedFeature]int64{} + keys := make([]interface{}, 0, len(nfs)*5) for _, nf := range nfs { - keys = append(keys, nf.Name, nf.Version, nf.VersionFormat, nf.Namespace.Name) - nfsMap[nf] = sql.NullInt64{} + keys = append(keys, nf.Name, nf.Version, nf.VersionFormat, nf.Type, nf.Namespace.Name) } rows, err := tx.Query(querySearchNamespacedFeature(len(nfs)), keys...) @@ -337,12 +322,12 @@ func (tx *pgSession) findNamespacedFeatureIDs(nfs []database.NamespacedFeature) defer rows.Close() var ( - id sql.NullInt64 + id int64 nf database.NamespacedFeature ) for rows.Next() { - err := rows.Scan(&id, &nf.Name, &nf.Version, &nf.VersionFormat, &nf.Namespace.Name) + err := rows.Scan(&id, &nf.Name, &nf.Version, &nf.VersionFormat, &nf.Type, &nf.Namespace.Name) nf.Namespace.VersionFormat = nf.VersionFormat if err != nil { return nil, handleError("searchNamespacedFeature", err) @@ -352,7 +337,11 @@ func (tx *pgSession) findNamespacedFeatureIDs(nfs []database.NamespacedFeature) ids := make([]sql.NullInt64, len(nfs)) for i, nf := range nfs { - ids[i] = nfsMap[nf] + if id, ok := nfsMap[nf]; ok { + ids[i] = sql.NullInt64{id, true} + } else { + ids[i] = sql.NullInt64{} + } } return ids, nil @@ -363,11 +352,17 @@ func (tx *pgSession) findFeatureIDs(fs []database.Feature) ([]sql.NullInt64, err return nil, nil } + types, err := tx.getFeatureTypeMap() + if err != nil { + return nil, err + } + fMap := map[database.Feature]sql.NullInt64{} - keys := make([]interface{}, 0, len(fs)*3) + keys := make([]interface{}, 0, len(fs)*4) for _, f := range fs { - keys = append(keys, f.Name, f.Version, f.VersionFormat) + typeID := types.byName[f.Type] + keys = append(keys, f.Name, f.Version, f.VersionFormat, typeID) fMap[f] = sql.NullInt64{} } @@ -382,10 +377,13 @@ func (tx *pgSession) findFeatureIDs(fs []database.Feature) ([]sql.NullInt64, err f database.Feature ) for rows.Next() { - err := rows.Scan(&id, &f.Name, &f.Version, &f.VersionFormat) + var typeID int + err := rows.Scan(&id, &f.Name, &f.Version, &f.VersionFormat, &typeID) if err != nil { return nil, handleError("querySearchFeatureID", err) } + + f.Type = types.byID[typeID] fMap[f] = id } diff --git a/database/pgsql/feature_test.go b/database/pgsql/feature_test.go index 2823e1e8..574bfeab 100644 --- a/database/pgsql/feature_test.go +++ b/database/pgsql/feature_test.go @@ -18,134 +18,53 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/coreos/clair/database" - - // register dpkg feature lister for testing - _ "github.com/coreos/clair/ext/featurefmt/dpkg" ) func TestPersistFeatures(t *testing.T) { - datastore, tx := openSessionForTest(t, "PersistFeatures", false) - defer closeTest(t, datastore, tx) + tx, cleanup := createTestPgSession(t, "TestPersistFeatures") + defer cleanup() - f1 := database.Feature{} - f2 := database.Feature{Name: "n", Version: "v", VersionFormat: "vf"} + invalid := database.Feature{} + valid := *database.NewBinaryPackage("mount", "2.31.1-0.4ubuntu3.1", "dpkg") - // empty - assert.Nil(t, tx.PersistFeatures([]database.Feature{})) // invalid - assert.NotNil(t, tx.PersistFeatures([]database.Feature{f1})) - // duplicated - assert.Nil(t, tx.PersistFeatures([]database.Feature{f2, f2})) + require.NotNil(t, tx.PersistFeatures([]database.Feature{invalid})) // existing - assert.Nil(t, tx.PersistFeatures([]database.Feature{f2})) + require.Nil(t, tx.PersistFeatures([]database.Feature{valid})) + require.Nil(t, tx.PersistFeatures([]database.Feature{valid})) - fs := listFeatures(t, tx) - assert.Len(t, fs, 1) - assert.Equal(t, f2, fs[0]) + features := selectAllFeatures(t, tx) + assert.Equal(t, []database.Feature{valid}, features) } func TestPersistNamespacedFeatures(t *testing.T) { - datastore, tx := openSessionForTest(t, "PersistNamespacedFeatures", true) - defer closeTest(t, datastore, tx) + tx, cleanup := createTestPgSessionWithFixtures(t, "TestPersistNamespacedFeatures") + defer cleanup() // existing features - f1 := database.Feature{ - Name: "ourchat", - Version: "0.5", - VersionFormat: "dpkg", - } - + f1 := database.NewSourcePackage("ourchat", "0.5", "dpkg") // non-existing features - f2 := database.Feature{ - Name: "fake!", - } - - f3 := database.Feature{ - Name: "openssl", - Version: "2.0", - VersionFormat: "dpkg", - } - + f2 := database.NewSourcePackage("fake!", "", "") // exising namespace - n1 := database.Namespace{ - Name: "debian:7", - VersionFormat: "dpkg", - } - - n3 := database.Namespace{ - Name: "debian:8", - VersionFormat: "dpkg", - } - + n1 := database.NewNamespace("debian:7", "dpkg") // non-existing namespace - n2 := database.Namespace{ - Name: "debian:non", - VersionFormat: "dpkg", - } - + n2 := database.NewNamespace("debian:non", "dpkg") // existing namespaced feature - nf1 := database.NamespacedFeature{ - Namespace: n1, - Feature: f1, - } - + nf1 := database.NewNamespacedFeature(n1, f1) // invalid namespaced feature - nf2 := database.NamespacedFeature{ - Namespace: n2, - Feature: f2, - } - - // new namespaced feature affected by vulnerability - nf3 := database.NamespacedFeature{ - Namespace: n3, - Feature: f3, - } - + nf2 := database.NewNamespacedFeature(n2, f2) // namespaced features with namespaces or features not in the database will // generate error. assert.Nil(t, tx.PersistNamespacedFeatures([]database.NamespacedFeature{})) - - assert.NotNil(t, tx.PersistNamespacedFeatures([]database.NamespacedFeature{nf1, nf2})) + assert.NotNil(t, tx.PersistNamespacedFeatures([]database.NamespacedFeature{*nf1, *nf2})) // valid case: insert nf3 - assert.Nil(t, tx.PersistNamespacedFeatures([]database.NamespacedFeature{nf1, nf3})) + assert.Nil(t, tx.PersistNamespacedFeatures([]database.NamespacedFeature{*nf1})) all := listNamespacedFeatures(t, tx) - assert.Contains(t, all, nf1) - assert.Contains(t, all, nf3) -} - -func TestVulnerableFeature(t *testing.T) { - datastore, tx := openSessionForTest(t, "VulnerableFeature", true) - defer closeTest(t, datastore, tx) - - f1 := database.Feature{ - Name: "openssl", - Version: "1.3", - VersionFormat: "dpkg", - } - - n1 := database.Namespace{ - Name: "debian:7", - VersionFormat: "dpkg", - } - - nf1 := database.NamespacedFeature{ - Namespace: n1, - Feature: f1, - } - assert.Nil(t, tx.PersistFeatures([]database.Feature{f1})) - assert.Nil(t, tx.PersistNamespacedFeatures([]database.NamespacedFeature{nf1})) - assert.Nil(t, tx.CacheAffectedNamespacedFeatures([]database.NamespacedFeature{nf1})) - // ensure the namespaced feature is affected correctly - anf, err := tx.FindAffectedNamespacedFeatures([]database.NamespacedFeature{nf1}) - if assert.Nil(t, err) && - assert.Len(t, anf, 1) && - assert.True(t, anf[0].Valid) && - assert.Len(t, anf[0].AffectedBy, 1) { - assert.Equal(t, "CVE-OPENSSL-1-DEB7", anf[0].AffectedBy[0].Name) - } + assert.Contains(t, all, *nf1) } func TestFindAffectedNamespacedFeatures(t *testing.T) { @@ -156,6 +75,7 @@ func TestFindAffectedNamespacedFeatures(t *testing.T) { Name: "openssl", Version: "1.0", VersionFormat: "dpkg", + Type: database.SourcePackage, }, Namespace: database.Namespace{ Name: "debian:7", @@ -173,30 +93,41 @@ func TestFindAffectedNamespacedFeatures(t *testing.T) { } func listNamespacedFeatures(t *testing.T, tx *pgSession) []database.NamespacedFeature { - rows, err := tx.Query(`SELECT f.name, f.version, f.version_format, n.name, n.version_format + types, err := tx.getFeatureTypeMap() + if err != nil { + panic(err) + } + + rows, err := tx.Query(`SELECT f.name, f.version, f.version_format, f.type, n.name, n.version_format FROM feature AS f, namespace AS n, namespaced_feature AS nf WHERE nf.feature_id = f.id AND nf.namespace_id = n.id`) if err != nil { - t.Error(err) - t.FailNow() + panic(err) } nf := []database.NamespacedFeature{} for rows.Next() { f := database.NamespacedFeature{} - err := rows.Scan(&f.Name, &f.Version, &f.VersionFormat, &f.Namespace.Name, &f.Namespace.VersionFormat) + var typeID int + err := rows.Scan(&f.Name, &f.Version, &f.VersionFormat, &typeID, &f.Namespace.Name, &f.Namespace.VersionFormat) if err != nil { - t.Error(err) - t.FailNow() + panic(err) } + + f.Type = types.byID[typeID] nf = append(nf, f) } return nf } -func listFeatures(t *testing.T, tx *pgSession) []database.Feature { - rows, err := tx.Query("SELECT name, version, version_format FROM feature") +func selectAllFeatures(t *testing.T, tx *pgSession) []database.Feature { + types, err := tx.getFeatureTypeMap() + if err != nil { + panic(err) + } + + rows, err := tx.Query("SELECT name, version, version_format, type FROM feature") if err != nil { t.FailNow() } @@ -204,7 +135,9 @@ func listFeatures(t *testing.T, tx *pgSession) []database.Feature { fs := []database.Feature{} for rows.Next() { f := database.Feature{} - err := rows.Scan(&f.Name, &f.Version, &f.VersionFormat) + var typeID int + err := rows.Scan(&f.Name, &f.Version, &f.VersionFormat, &typeID) + f.Type = types.byID[typeID] if err != nil { t.FailNow() } @@ -233,3 +166,33 @@ func assertNamespacedFeatureEqual(t *testing.T, expected []database.NamespacedFe } return false } + +func TestFindNamespacedFeatureIDs(t *testing.T) { + tx, cleanup := createTestPgSessionWithFixtures(t, "TestFindNamespacedFeatureIDs") + defer cleanup() + + features := []database.NamespacedFeature{} + expectedIDs := []int{} + for id, feature := range realNamespacedFeatures { + features = append(features, feature) + expectedIDs = append(expectedIDs, id) + } + + features = append(features, realNamespacedFeatures[1]) // test duplicated + expectedIDs = append(expectedIDs, 1) + + namespace := realNamespaces[1] + features = append(features, *database.NewNamespacedFeature(&namespace, database.NewBinaryPackage("not-found", "1.0", "dpkg"))) // test not found feature + + ids, err := tx.findNamespacedFeatureIDs(features) + require.Nil(t, err) + require.Len(t, ids, len(expectedIDs)+1) + for i, id := range ids { + if i == len(ids)-1 { + require.False(t, id.Valid) + } else { + require.True(t, id.Valid) + require.Equal(t, expectedIDs[i], int(id.Int64)) + } + } +} diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go index 782e541c..a071eb4b 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer.go @@ -37,9 +37,10 @@ const ( SELECT id FROM layer WHERE hash = $1` findLayerFeatures = ` - SELECT f.name, f.version, f.version_format, lf.detector_id - FROM layer_feature AS lf, feature AS f + SELECT f.name, f.version, f.version_format, t.name, lf.detector_id + FROM layer_feature AS lf, feature AS f, feature_type AS t WHERE lf.feature_id = f.id + AND t.id = f.type AND lf.layer_id = $1` findLayerNamespaces = ` @@ -307,7 +308,7 @@ func (tx *pgSession) findLayerFeatures(layerID int64, detectors detectorMap) ([] detectorID int64 feature database.LayerFeature ) - if err := rows.Scan(&feature.Name, &feature.Version, &feature.VersionFormat, &detectorID); err != nil { + if err := rows.Scan(&feature.Name, &feature.Version, &feature.VersionFormat, &feature.Type, &detectorID); err != nil { return nil, handleError("findLayerFeatures", err) } diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer_test.go index 478b2171..5211eb11 100644 --- a/database/pgsql/layer_test.go +++ b/database/pgsql/layer_test.go @@ -43,12 +43,12 @@ var persistLayerTests = []struct { features: []database.LayerFeature{ {realFeatures[1], realDetectors[1]}, }, - err: "database: parameters are not valid", + err: "parameters are not valid", }, { title: "layer with non-existing feature", name: "random-forest", - err: "database: associated immutable entities are missing in the database", + err: "associated immutable entities are missing in the database", by: []database.Detector{realDetectors[2]}, features: []database.LayerFeature{ {fakeFeatures[1], realDetectors[2]}, @@ -57,7 +57,7 @@ var persistLayerTests = []struct { { title: "layer with non-existing namespace", name: "random-forest2", - err: "database: associated immutable entities are missing in the database", + err: "associated immutable entities are missing in the database", by: []database.Detector{realDetectors[1]}, namespaces: []database.LayerNamespace{ {fakeNamespaces[1], realDetectors[1]}, @@ -66,7 +66,7 @@ var persistLayerTests = []struct { { title: "layer with non-existing detector", name: "random-forest3", - err: "database: associated immutable entities are missing in the database", + err: "associated immutable entities are missing in the database", by: []database.Detector{fakeDetector[1]}, }, { diff --git a/database/pgsql/notification_test.go b/database/pgsql/notification_test.go index 0a23abca..da3b3248 100644 --- a/database/pgsql/notification_test.go +++ b/database/pgsql/notification_test.go @@ -211,8 +211,8 @@ func TestInsertVulnerabilityNotifications(t *testing.T) { } func TestFindNewNotification(t *testing.T) { - datastore, tx := openSessionForTest(t, "FindNewNotification", true) - defer closeTest(t, datastore, tx) + tx, cleanup := createTestPgSessionWithFixtures(t, "TestFindNewNotification") + defer cleanup() noti, ok, err := tx.FindNewNotification(time.Now()) if assert.Nil(t, err) && assert.True(t, ok) { @@ -229,7 +229,7 @@ func TestFindNewNotification(t *testing.T) { assert.Nil(t, err) assert.False(t, ok) // can find the notified after a period of time - noti, ok, err = tx.FindNewNotification(time.Now().Add(time.Duration(1000))) + noti, ok, err = tx.FindNewNotification(time.Now().Add(time.Duration(10 * time.Second))) if assert.Nil(t, err) && assert.True(t, ok) { assert.Equal(t, "test", noti.Name) assert.NotEqual(t, time.Time{}, noti.Notified) diff --git a/database/pgsql/pgsql_test.go b/database/pgsql/pgsql_test.go index 863445a5..03eda8c7 100644 --- a/database/pgsql/pgsql_test.go +++ b/database/pgsql/pgsql_test.go @@ -109,6 +109,7 @@ func dropTemplateDatabase(url string, name string) { } } + func TestMain(m *testing.M) { fURL, fName := genTemplateDatabase("fixture", true) nfURL, nfName := genTemplateDatabase("nonfixture", false) diff --git a/database/pgsql/queries.go b/database/pgsql/queries.go index 2d4b7e99..5cd5c3c9 100644 --- a/database/pgsql/queries.go +++ b/database/pgsql/queries.go @@ -52,21 +52,22 @@ func querySearchNotDeletedVulnerabilityID(count int) string { func querySearchFeatureID(featureCount int) string { return fmt.Sprintf(` - SELECT id, name, version, version_format - FROM Feature WHERE (name, version, version_format) IN (%s)`, - queryString(3, featureCount), + SELECT id, name, version, version_format, type + FROM Feature WHERE (name, version, version_format, type) IN (%s)`, + queryString(4, featureCount), ) } func querySearchNamespacedFeature(nsfCount int) string { return fmt.Sprintf(` - SELECT nf.id, f.name, f.version, f.version_format, n.name - FROM namespaced_feature AS nf, feature AS f, namespace AS n + SELECT nf.id, f.name, f.version, f.version_format, t.name, n.name + FROM namespaced_feature AS nf, feature AS f, namespace AS n, feature_type AS t WHERE nf.feature_id = f.id AND nf.namespace_id = n.id AND n.version_format = f.version_format - AND (f.name, f.version, f.version_format, n.name) IN (%s)`, - queryString(4, nsfCount), + AND f.type = t.id + AND (f.name, f.version, f.version_format, t.name, n.name) IN (%s)`, + queryString(5, nsfCount), ) } @@ -110,10 +111,11 @@ func queryInsertNotifications(count int) string { func queryPersistFeature(count int) string { return queryPersist(count, "feature", - "feature_name_version_version_format_key", + "feature_name_version_version_format_type_key", "name", "version", - "version_format") + "version_format", + "type") } func queryPersistLayerFeature(count int) string { diff --git a/database/pgsql/testdata/data.sql b/database/pgsql/testdata/data.sql index e7484209..4c90ae0d 100644 --- a/database/pgsql/testdata/data.sql +++ b/database/pgsql/testdata/data.sql @@ -4,11 +4,12 @@ INSERT INTO namespace (id, name, version_format) VALUES (2, 'debian:8', 'dpkg'), (3, 'fake:1.0', 'rpm'); -INSERT INTO feature (id, name, version, version_format) VALUES - (1, 'ourchat', '0.5', 'dpkg'), - (2, 'openssl', '1.0', 'dpkg'), - (3, 'openssl', '2.0', 'dpkg'), - (4, 'fake', '2.0', 'rpm'); +INSERT INTO feature (id, name, version, version_format, type) VALUES + (1, 'ourchat', '0.5', 'dpkg', 1), + (2, 'openssl', '1.0', 'dpkg', 1), + (3, 'openssl', '2.0', 'dpkg', 1), + (4, 'fake', '2.0', 'rpm', 1), + (5, 'mount', '2.31.1-0.4ubuntu3.1', 'dpkg', 2); INSERT INTO namespaced_feature(id, feature_id, namespace_id) VALUES (1, 1, 1), -- ourchat 0.5, debian:7 @@ -112,9 +113,9 @@ INSERT INTO vulnerability (id, namespace_id, name, description, link, severity) INSERT INTO vulnerability (id, namespace_id, name, description, link, severity, deleted_at) VALUES (3, 1, 'CVE-DELETED', '', '', 'Unknown', '2017-08-08 17:49:31.668483'); -INSERT INTO vulnerability_affected_feature(id, vulnerability_id, feature_name, affected_version, fixedin) VALUES -(1, 1, 'openssl', '2.0', '2.0'), -(2, 1, 'libssl', '1.9-abc', '1.9-abc'); +INSERT INTO vulnerability_affected_feature(id, vulnerability_id, feature_name, affected_version, fixedin, feature_type) VALUES +(1, 1, 'openssl', '2.0', '2.0', 1), +(2, 1, 'libssl', '1.9-abc', '1.9-abc', 1); INSERT INTO vulnerability_affected_namespaced_feature(id, vulnerability_id, namespaced_feature_id, added_by) VALUES (1, 1, 2, 1); diff --git a/database/pgsql/vulnerability.go b/database/pgsql/vulnerability.go index 93518a87..e96d6d47 100644 --- a/database/pgsql/vulnerability.go +++ b/database/pgsql/vulnerability.go @@ -39,15 +39,15 @@ const ( ` insertVulnerabilityAffected = ` - INSERT INTO vulnerability_affected_feature(vulnerability_id, feature_name, affected_version, fixedin) - VALUES ($1, $2, $3, $4) + INSERT INTO vulnerability_affected_feature(vulnerability_id, feature_name, affected_version, feature_type, fixedin) + VALUES ($1, $2, $3, $4, $5) RETURNING ID ` searchVulnerabilityAffected = ` - SELECT vulnerability_id, feature_name, affected_version, fixedin - FROM vulnerability_affected_feature - WHERE vulnerability_id = ANY($1) + SELECT vulnerability_id, feature_name, affected_version, t.name, fixedin + FROM vulnerability_affected_feature AS vaf, feature_type AS t + WHERE t.id = vaf.feature_type AND vulnerability_id = ANY($1) ` searchVulnerabilityByID = ` @@ -58,7 +58,7 @@ const ( searchVulnerabilityPotentialAffected = ` WITH req AS ( - SELECT vaf.id AS vaf_id, n.id AS n_id, vaf.feature_name AS name, v.id AS vulnerability_id + SELECT vaf.id AS vaf_id, n.id AS n_id, vaf.feature_name AS name, vaf.feature_type AS type, v.id AS vulnerability_id FROM vulnerability_affected_feature AS vaf, vulnerability AS v, namespace AS n @@ -69,6 +69,7 @@ const ( SELECT req.vulnerability_id, nf.id, f.version, req.vaf_id AS added_by FROM feature AS f, namespaced_feature AS nf, req WHERE f.name = req.name + AND f.type = req.type AND nf.namespace_id = req.n_id AND nf.feature_id = f.id` @@ -180,7 +181,7 @@ func (tx *pgSession) FindVulnerabilities(vulnerabilities []database.Vulnerabilit f database.AffectedFeature ) - err := rows.Scan(&id, &f.FeatureName, &f.AffectedVersion, &f.FixedInVersion) + err := rows.Scan(&id, &f.FeatureName, &f.AffectedVersion, &f.FeatureType, &f.FixedInVersion) if err != nil { return nil, handleError("searchVulnerabilityAffected", err) } @@ -220,6 +221,11 @@ func (tx *pgSession) insertVulnerabilityAffected(vulnerabilityIDs []int64, vulne affectedID int64 ) + types, err := tx.getFeatureTypeMap() + if err != nil { + return nil, err + } + //TODO(Sida): Change to bulk insert. stmt, err := tx.Prepare(insertVulnerabilityAffected) if err != nil { @@ -231,7 +237,7 @@ func (tx *pgSession) insertVulnerabilityAffected(vulnerabilityIDs []int64, vulne // affected feature row ID -> affected feature affectedFeatures := map[int64]database.AffectedFeature{} for _, f := range vuln.Affected { - err := stmt.QueryRow(vulnerabilityIDs[i], f.FeatureName, f.AffectedVersion, f.FixedInVersion).Scan(&affectedID) + err := stmt.QueryRow(vulnerabilityIDs[i], f.FeatureName, f.AffectedVersion, types.byName[f.FeatureType], f.FixedInVersion).Scan(&affectedID) if err != nil { return nil, handleError("insertVulnerabilityAffected", err) } diff --git a/database/pgsql/vulnerability_test.go b/database/pgsql/vulnerability_test.go index bfa465b2..759bfe2f 100644 --- a/database/pgsql/vulnerability_test.go +++ b/database/pgsql/vulnerability_test.go @@ -106,6 +106,7 @@ func TestCachingVulnerable(t *testing.T) { Name: "openssl", Version: "1.0", VersionFormat: dpkg.ParserName, + Type: database.SourcePackage, }, Namespace: ns, } @@ -120,6 +121,7 @@ func TestCachingVulnerable(t *testing.T) { { Namespace: ns, FeatureName: "openssl", + FeatureType: database.SourcePackage, AffectedVersion: "2.0", FixedInVersion: "2.1", }, @@ -136,6 +138,7 @@ func TestCachingVulnerable(t *testing.T) { { Namespace: ns, FeatureName: "openssl", + FeatureType: database.SourcePackage, AffectedVersion: "2.1", FixedInVersion: "2.2", }, @@ -209,12 +212,14 @@ func TestFindVulnerabilities(t *testing.T) { Affected: []database.AffectedFeature{ { FeatureName: "openssl", + FeatureType: database.SourcePackage, AffectedVersion: "2.0", FixedInVersion: "2.0", Namespace: ns, }, { FeatureName: "libssl", + FeatureType: database.SourcePackage, AffectedVersion: "1.9-abc", FixedInVersion: "1.9-abc", Namespace: ns, @@ -318,26 +323,3 @@ func TestFindVulnerabilityIDs(t *testing.T) { } } } - -func assertVulnerabilityWithAffectedEqual(t *testing.T, expected database.VulnerabilityWithAffected, actual database.VulnerabilityWithAffected) bool { - return assert.Equal(t, expected.Vulnerability, actual.Vulnerability) && assertAffectedFeaturesEqual(t, expected.Affected, actual.Affected) -} - -func assertAffectedFeaturesEqual(t *testing.T, expected []database.AffectedFeature, actual []database.AffectedFeature) bool { - if assert.Len(t, actual, len(expected)) { - has := map[database.AffectedFeature]bool{} - for _, i := range expected { - has[i] = false - } - for _, i := range actual { - if visited, ok := has[i]; !ok { - return false - } else if visited { - return false - } - has[i] = true - } - return true - } - return false -} From 23ccd9b53ba0a8bcf800fecdbd72d5cbefd2ea60 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 19 Feb 2019 16:42:14 -0500 Subject: [PATCH 09/12] worker: Fix tests for feature_type --- worker_test.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/worker_test.go b/worker_test.go index aef37985..2ad86272 100644 --- a/worker_test.go +++ b/worker_test.go @@ -284,11 +284,15 @@ func TestProcessAncestryWithDistUpgrade(t *testing.T) { {Name: "db", Version: "5.1.29-5"}, {Name: "ustr", Version: "1.0.4-3"}, {Name: "xz-utils", Version: "5.1.1alpha+20120614-2"}, + {Name: "libdb5.1", Version: "5.1.29-5"}, } nonUpgradedMap := map[database.Feature]struct{}{} for _, f := range nonUpgradedFeatures { f.VersionFormat = "dpkg" + f.Type = database.SourcePackage + nonUpgradedMap[f] = struct{}{} + f.Type = database.BinaryPackage nonUpgradedMap[f] = struct{}{} } @@ -318,12 +322,12 @@ func TestProcessAncestryWithDistUpgrade(t *testing.T) { features = append(features, l.Features...) } - assert.Len(t, features, 74) + assert.Len(t, features, 161) for _, f := range features { if _, ok := nonUpgradedMap[f.Feature]; ok { - assert.Equal(t, "debian:7", f.Namespace.Name) + assert.Equal(t, "debian:7", f.Namespace.Name, "%#v", f) } else { - assert.Equal(t, "debian:8", f.Namespace.Name) + assert.Equal(t, "debian:8", f.Namespace.Name, "#%v", f) } } } @@ -352,8 +356,8 @@ func TestProcessLayers(t *testing.T) { assert.Len(t, LayerWithContents[1].Namespaces, 1) assert.Len(t, LayerWithContents[2].Namespaces, 1) assert.Len(t, LayerWithContents[0].Features, 0) - assert.Len(t, LayerWithContents[1].Features, 52) - assert.Len(t, LayerWithContents[2].Features, 74) + assert.Len(t, LayerWithContents[1].Features, 132) + assert.Len(t, LayerWithContents[2].Features, 191) // Ensure each layer has expected namespaces and features detected if blank, ok := datastore.layers["blank"]; ok { @@ -371,7 +375,7 @@ func TestProcessLayers(t *testing.T) { {database.Namespace{"debian:7", dpkg.ParserName}, database.NewNamespaceDetector("os-release", "1.0")}, }, wheezy.Namespaces) - assert.Len(t, wheezy.Features, 52) + assert.Len(t, wheezy.Features, 132) } else { assert.Fail(t, "wheezy is not stored") return @@ -382,7 +386,7 @@ func TestProcessLayers(t *testing.T) { assert.Equal(t, []database.LayerNamespace{ {database.Namespace{"debian:8", dpkg.ParserName}, database.NewNamespaceDetector("os-release", "1.0")}, }, jessie.Namespaces) - assert.Len(t, jessie.Features, 74) + assert.Len(t, jessie.Features, 191) } else { assert.Fail(t, "jessie is not stored") return From 870e8123769a3dd717bfdcd21473a8e691806653 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 19 Feb 2019 16:42:49 -0500 Subject: [PATCH 10/12] Travis: Drop support for postgres 9.4 postgres 9.4 doesn't support ON CONFLICT, which is required in our implementation. --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7aa6b31c..fc2d8eb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,11 +31,6 @@ notifications: matrix: include: - - addons: - apt: - packages: - - rpm - postgresql: 9.4 - addons: apt: packages: From 32b11e54eb287ed0d686ba72fe413b773b748a38 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 19 Feb 2019 17:03:44 -0500 Subject: [PATCH 11/12] api/v3: Add feature type to API feature API now returns every feature found by the detectors and also indicates the type of feature. --- api/v3/clairpb/clair.pb.go | 175 ++++++++++++++++-------------- api/v3/clairpb/clair.proto | 3 + api/v3/clairpb/clair.swagger.json | 4 + api/v3/clairpb/convert.go | 5 + 4 files changed, 105 insertions(+), 82 deletions(-) diff --git a/api/v3/clairpb/clair.pb.go b/api/v3/clairpb/clair.pb.go index 1306dbee..6985801a 100644 --- a/api/v3/clairpb/clair.pb.go +++ b/api/v3/clairpb/clair.pb.go @@ -232,6 +232,9 @@ type Feature struct { Detector *Detector `protobuf:"bytes,5,opt,name=detector" json:"detector,omitempty"` // The list of vulnerabilities that affect the feature. Vulnerabilities []*Vulnerability `protobuf:"bytes,6,rep,name=vulnerabilities" json:"vulnerabilities,omitempty"` + // The feature type indicates if the feature represents a source package or + // binary package. + FeatureType string `protobuf:"bytes,7,opt,name=feature_type,json=featureType" json:"feature_type,omitempty"` } func (m *Feature) Reset() { *m = Feature{} } @@ -281,6 +284,13 @@ func (m *Feature) GetVulnerabilities() []*Vulnerability { return nil } +func (m *Feature) GetFeatureType() string { + if m != nil { + return m.FeatureType + } + return "" +} + type Layer struct { // The sha256 tarsum for the layer. Hash string `protobuf:"bytes,1,opt,name=hash" json:"hash,omitempty"` @@ -1091,89 +1101,90 @@ var _StatusService_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("api/v3/clairpb/clair.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1336 bytes of a gzipped FileDescriptorProto + // 1350 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4b, 0x6f, 0x1b, 0x55, 0x14, 0x66, 0x9c, 0x3a, 0xb6, 0x8f, 0xed, 0xc4, 0xbd, 0x49, 0x13, 0x67, 0xd2, 0x47, 0x32, 0x50, 0x51, 0x0a, 0xb2, 0x85, 0x5b, 0xa4, 0xb6, 0x2c, 0x90, 0x9b, 0x38, 0x21, 0x52, 0x1b, 0xa2, 0x49, - 0x1a, 0x09, 0x10, 0x32, 0x37, 0x9e, 0xe3, 0x64, 0x94, 0xf1, 0xcc, 0x30, 0x73, 0x9d, 0xd4, 0xaa, - 0xca, 0x82, 0x1d, 0x3b, 0x04, 0x0b, 0x56, 0xfc, 0x00, 0x36, 0x88, 0xff, 0xc0, 0x9e, 0x05, 0x6c, - 0x61, 0xc7, 0x82, 0x3f, 0xc0, 0x1e, 0xdd, 0xc7, 0x4c, 0x66, 0x92, 0x49, 0xe2, 0x76, 0xe5, 0x7b, - 0xde, 0x8f, 0xfb, 0xdd, 0x73, 0xc6, 0xa0, 0x53, 0xdf, 0x6e, 0x1e, 0xdd, 0x6b, 0xf6, 0x1c, 0x6a, - 0x07, 0xfe, 0x9e, 0xfc, 0x6d, 0xf8, 0x81, 0xc7, 0x3c, 0x52, 0xe9, 0x79, 0x01, 0x7a, 0x61, 0x43, - 0xf0, 0xf4, 0x5b, 0xfb, 0x9e, 0xb7, 0xef, 0x60, 0x53, 0xc8, 0xf6, 0x86, 0xfd, 0x26, 0xb3, 0x07, - 0x18, 0x32, 0x3a, 0xf0, 0xa5, 0xba, 0x7e, 0x5d, 0x29, 0x70, 0x8f, 0xd4, 0x75, 0x3d, 0x46, 0x99, - 0xed, 0xb9, 0xa1, 0x94, 0x1a, 0x3f, 0xe6, 0xa0, 0xba, 0x3b, 0x74, 0x5c, 0x0c, 0xe8, 0x9e, 0xed, - 0xd8, 0x6c, 0x44, 0x08, 0x5c, 0x71, 0xe9, 0x00, 0xeb, 0xda, 0x92, 0x76, 0xa7, 0x64, 0x8a, 0x33, - 0xb9, 0x0d, 0x53, 0xfc, 0x37, 0xf4, 0x69, 0x0f, 0xbb, 0x42, 0x9a, 0x13, 0xd2, 0x6a, 0xcc, 0xdd, - 0xe4, 0x6a, 0x4b, 0x50, 0xb6, 0x30, 0xec, 0x05, 0xb6, 0xcf, 0x43, 0xd4, 0x27, 0x84, 0x4e, 0x92, - 0xc5, 0x9d, 0x3b, 0xb6, 0x7b, 0x58, 0xbf, 0x22, 0x9d, 0xf3, 0x33, 0xd1, 0xa1, 0x18, 0xe2, 0x11, - 0x06, 0x36, 0x1b, 0xd5, 0xf3, 0x82, 0x1f, 0xd3, 0x5c, 0x36, 0x40, 0x46, 0x2d, 0xca, 0x68, 0x7d, - 0x52, 0xca, 0x22, 0x9a, 0x2c, 0x40, 0xb1, 0x6f, 0x3f, 0x47, 0xab, 0xbb, 0x37, 0xaa, 0x17, 0x84, - 0xac, 0x20, 0xe8, 0xc7, 0x23, 0xf2, 0x18, 0xae, 0xd2, 0x7e, 0x1f, 0x7b, 0x0c, 0xad, 0xee, 0x11, - 0x06, 0x21, 0x2f, 0xb8, 0x5e, 0x5c, 0x9a, 0xb8, 0x53, 0x6e, 0x5d, 0x6b, 0x24, 0xdb, 0xd7, 0x58, - 0x43, 0xca, 0x86, 0x01, 0x9a, 0xb5, 0x48, 0x7f, 0x57, 0xa9, 0x1b, 0xbf, 0x6b, 0x50, 0x5c, 0x45, - 0x86, 0x3d, 0xe6, 0x05, 0x99, 0x4d, 0xa9, 0x43, 0x41, 0xf9, 0x56, 0xdd, 0x88, 0x48, 0xd2, 0x82, - 0xbc, 0xc5, 0x46, 0x3e, 0x8a, 0x0e, 0x4c, 0xb5, 0xae, 0xa7, 0x43, 0x46, 0x4e, 0x1b, 0xab, 0x3b, - 0x23, 0x1f, 0x4d, 0xa9, 0x6a, 0x7c, 0x09, 0x79, 0x41, 0x93, 0x45, 0x98, 0x5f, 0xed, 0xec, 0x74, - 0x56, 0x76, 0x3e, 0x31, 0xbb, 0xab, 0xdd, 0x9d, 0x4f, 0xb7, 0x3a, 0xdd, 0x8d, 0xcd, 0xdd, 0xf6, - 0x93, 0x8d, 0xd5, 0xda, 0x1b, 0xe4, 0x06, 0x2c, 0x9c, 0x16, 0x6e, 0xb6, 0x9f, 0x76, 0xb6, 0xb7, - 0xda, 0x2b, 0x9d, 0x9a, 0x96, 0x65, 0xbb, 0xd6, 0x69, 0xef, 0x3c, 0x33, 0x3b, 0xb5, 0x9c, 0xb1, - 0x0d, 0xa5, 0xcd, 0xe8, 0xba, 0x32, 0x0b, 0x6a, 0x41, 0xd1, 0x52, 0xb9, 0x89, 0x8a, 0xca, 0xad, - 0xb9, 0xec, 0xcc, 0xcd, 0x58, 0xcf, 0xf8, 0x2e, 0x07, 0x05, 0xd5, 0xc3, 0x4c, 0x9f, 0x1f, 0x40, - 0x29, 0xc6, 0x88, 0x72, 0x3a, 0x9f, 0x76, 0x1a, 0xe7, 0x64, 0x9e, 0x68, 0x26, 0x7b, 0x3b, 0x91, - 0xee, 0xed, 0x6d, 0x98, 0x52, 0xc7, 0x6e, 0xdf, 0x0b, 0x06, 0x94, 0x29, 0x2c, 0x55, 0x15, 0x77, - 0x4d, 0x30, 0x53, 0xb5, 0xe4, 0xc7, 0xab, 0x85, 0x74, 0x60, 0xfa, 0x28, 0xf1, 0x14, 0x6c, 0x0c, - 0xeb, 0x93, 0x02, 0x33, 0x8b, 0x69, 0xd3, 0xd4, 0x7b, 0x31, 0x4f, 0xdb, 0x18, 0x8b, 0x90, 0x7f, - 0x42, 0x47, 0x28, 0x40, 0x73, 0x40, 0xc3, 0x83, 0xa8, 0x1f, 0xfc, 0x6c, 0x7c, 0xab, 0x41, 0x79, - 0x85, 0x7b, 0xd9, 0x66, 0x94, 0x0d, 0x43, 0x72, 0x1f, 0x4a, 0x51, 0xfc, 0xb0, 0xae, 0x89, 0x68, - 0xe7, 0x25, 0x7a, 0xa2, 0x48, 0x56, 0xa1, 0xe6, 0xd0, 0x90, 0x75, 0x87, 0xbe, 0x45, 0x19, 0x76, - 0xf9, 0x93, 0x57, 0xcd, 0xd5, 0x1b, 0xf2, 0xb9, 0x37, 0xa2, 0x79, 0xd0, 0xd8, 0x89, 0xe6, 0x81, - 0x39, 0xc5, 0x6d, 0x9e, 0x09, 0x13, 0xce, 0x34, 0x1e, 0x02, 0x59, 0x47, 0xd6, 0x76, 0x7b, 0x18, - 0xb2, 0x60, 0x64, 0xe2, 0x57, 0x43, 0x0c, 0x19, 0x79, 0x13, 0xaa, 0x54, 0xb1, 0xba, 0x89, 0xeb, - 0xac, 0x44, 0x4c, 0x7e, 0x5f, 0xc6, 0xaf, 0x13, 0x30, 0x93, 0xb2, 0x0d, 0x7d, 0xcf, 0x0d, 0x91, - 0xac, 0x41, 0x31, 0xd2, 0x13, 0x76, 0xe5, 0xd6, 0xdd, 0x74, 0x35, 0x19, 0x46, 0x8d, 0x98, 0x11, - 0xdb, 0x92, 0xf7, 0x61, 0x32, 0x14, 0x0d, 0x52, 0x65, 0x2d, 0xa4, 0xbd, 0x24, 0x3a, 0x68, 0x2a, - 0x45, 0xfd, 0x6b, 0xa8, 0x46, 0x8e, 0x64, 0xfb, 0xdf, 0x81, 0xbc, 0xc3, 0x0f, 0x2a, 0x91, 0x99, - 0xb4, 0x0b, 0xa1, 0x63, 0x4a, 0x0d, 0x3e, 0x2f, 0x64, 0x73, 0xd1, 0xea, 0xf6, 0x25, 0x9a, 0x79, - 0xe4, 0x8b, 0xe6, 0x45, 0xa4, 0xaf, 0x18, 0xa1, 0xfe, 0x93, 0x06, 0xc5, 0x28, 0x81, 0xcc, 0xa7, - 0x90, 0xba, 0xea, 0xdc, 0xb8, 0x57, 0xbd, 0x0e, 0x93, 0x22, 0xc7, 0xb0, 0x3e, 0x21, 0x4c, 0x9a, - 0xe3, 0xf7, 0x53, 0x96, 0xa8, 0xcc, 0x8d, 0xbf, 0x73, 0x30, 0xb3, 0xe5, 0x85, 0xaf, 0x75, 0xdf, - 0x64, 0x0e, 0x26, 0xd5, 0x6b, 0x93, 0xa3, 0x4e, 0x51, 0x64, 0xe5, 0x54, 0x76, 0xef, 0xa6, 0xb3, - 0xcb, 0x88, 0x27, 0x78, 0xa9, 0xcc, 0xf4, 0xdf, 0x34, 0x28, 0xc5, 0xdc, 0xac, 0x57, 0xc3, 0x79, - 0x3e, 0x65, 0x07, 0x2a, 0xb8, 0x38, 0x13, 0x13, 0x0a, 0x07, 0x48, 0xad, 0x93, 0xd8, 0x0f, 0x5e, - 0x21, 0x76, 0xe3, 0x63, 0x69, 0xda, 0x71, 0xb9, 0x34, 0x72, 0xa4, 0x3f, 0x82, 0x4a, 0x52, 0x40, - 0x6a, 0x30, 0x71, 0x88, 0x23, 0x95, 0x0a, 0x3f, 0x92, 0x59, 0xc8, 0x1f, 0x51, 0x67, 0x18, 0x2d, - 0x40, 0x49, 0x3c, 0xca, 0x3d, 0xd0, 0x8c, 0x0d, 0x98, 0x4d, 0x87, 0x54, 0x4f, 0xe2, 0x04, 0xca, - 0xda, 0x98, 0x50, 0x36, 0x7e, 0xd1, 0x60, 0x6e, 0x1d, 0xd9, 0xa6, 0xc7, 0xec, 0xbe, 0xdd, 0x13, - 0xfb, 0x3a, 0xba, 0xad, 0xfb, 0x30, 0xe7, 0x39, 0x56, 0x37, 0x39, 0x73, 0x46, 0x5d, 0x9f, 0xee, - 0x47, 0xd7, 0x36, 0xeb, 0x39, 0x56, 0x6a, 0x3e, 0x6d, 0xd1, 0x7d, 0x0e, 0xbd, 0x39, 0x17, 0x8f, - 0xb3, 0xac, 0x64, 0x19, 0xb3, 0x2e, 0x1e, 0x9f, 0xb5, 0x9a, 0x85, 0xbc, 0x63, 0x0f, 0x6c, 0x26, - 0x46, 0x70, 0xde, 0x94, 0x44, 0x0c, 0xed, 0x2b, 0x27, 0xd0, 0x36, 0xfe, 0xca, 0xc1, 0xfc, 0x99, - 0x84, 0x55, 0xfd, 0xbb, 0x50, 0x71, 0x13, 0x7c, 0xd5, 0x85, 0xd6, 0x19, 0x18, 0x67, 0x19, 0x37, - 0x52, 0xcc, 0x94, 0x1f, 0xfd, 0x5f, 0x0d, 0x2a, 0x49, 0xf1, 0x79, 0x3b, 0xba, 0x17, 0x20, 0x65, - 0x68, 0x45, 0x3b, 0x5a, 0x91, 0xfc, 0xcb, 0x42, 0xba, 0x43, 0x4b, 0xad, 0x98, 0x98, 0xe6, 0x56, - 0x16, 0x3a, 0xc8, 0xad, 0x64, 0x95, 0x11, 0x49, 0x1e, 0xc2, 0x84, 0xe7, 0x58, 0x6a, 0xa3, 0xbc, - 0x7d, 0x0a, 0x70, 0x74, 0x1f, 0xe3, 0xde, 0x3b, 0xa8, 0x80, 0x60, 0x63, 0x68, 0x72, 0x1b, 0x6e, - 0xea, 0xe2, 0xb1, 0xf8, 0x8a, 0x79, 0x15, 0x53, 0x17, 0x8f, 0x8d, 0x3f, 0x72, 0xb0, 0x70, 0xae, - 0x0a, 0x59, 0x86, 0x4a, 0x6f, 0x18, 0x04, 0xe8, 0xb2, 0x24, 0x10, 0xca, 0x8a, 0x27, 0x6e, 0x72, - 0x11, 0x4a, 0x2e, 0x3e, 0x67, 0xc9, 0x2b, 0x2f, 0x72, 0xc6, 0x05, 0xd7, 0xdc, 0x86, 0x6a, 0x0a, - 0x2e, 0xa2, 0x13, 0x97, 0xac, 0xc2, 0xb4, 0x05, 0xf9, 0x1c, 0x80, 0xc6, 0x69, 0xd6, 0xf3, 0xe2, - 0x91, 0x7e, 0x38, 0x66, 0xe1, 0x8d, 0x0d, 0xd7, 0xc2, 0xe7, 0x68, 0xb5, 0x13, 0x53, 0xc8, 0x4c, - 0xb8, 0xd3, 0x3f, 0x82, 0x99, 0x0c, 0x15, 0x5e, 0x8c, 0xcd, 0xd9, 0xa2, 0x0b, 0x79, 0x53, 0x12, - 0x31, 0x34, 0x72, 0x09, 0xcc, 0xde, 0x83, 0x1b, 0x4f, 0x69, 0x70, 0x98, 0x84, 0x50, 0x3b, 0x34, - 0x91, 0x5a, 0xd1, 0x53, 0xcb, 0xc0, 0x93, 0xb1, 0x04, 0x37, 0xcf, 0x33, 0x92, 0x88, 0x35, 0x08, - 0xd4, 0xd6, 0x91, 0xa9, 0x07, 0x2d, 0x3d, 0x19, 0x6b, 0x70, 0x35, 0xc1, 0x7b, 0xed, 0xb9, 0xd0, - 0xfa, 0x4f, 0x83, 0xe9, 0xa8, 0xda, 0x6d, 0x0c, 0x8e, 0xec, 0x1e, 0x92, 0x21, 0x94, 0x13, 0x3b, - 0x80, 0x2c, 0x5d, 0xb0, 0x1e, 0x44, 0x32, 0xfa, 0xf2, 0xa5, 0x0b, 0xc4, 0x58, 0xfe, 0xe6, 0xcf, - 0x7f, 0x7e, 0xc8, 0x2d, 0x92, 0x85, 0x66, 0xb4, 0x04, 0x9a, 0x2f, 0x52, 0x3b, 0xe2, 0x25, 0x39, - 0x84, 0x4a, 0x72, 0xda, 0x91, 0xe5, 0x4b, 0x87, 0xaf, 0x6e, 0x5c, 0xa4, 0xa2, 0x22, 0xcf, 0x8a, - 0xc8, 0x53, 0x46, 0x29, 0x8e, 0xfc, 0x48, 0xbb, 0xdb, 0xfa, 0x39, 0x07, 0x33, 0xc9, 0x96, 0x47, - 0xb5, 0xbf, 0x84, 0xe9, 0x53, 0x83, 0x83, 0xbc, 0x75, 0xc9, 0x5c, 0x91, 0xa9, 0xdc, 0x1e, 0x6b, - 0xfa, 0x18, 0x37, 0x44, 0x36, 0xf3, 0xe4, 0x5a, 0x33, 0x39, 0x79, 0xc2, 0xe6, 0x0b, 0xd9, 0x83, - 0xef, 0x35, 0x98, 0xcb, 0x46, 0x03, 0x39, 0xb5, 0x07, 0x2f, 0x04, 0x9a, 0xfe, 0xde, 0x78, 0xca, - 0xe9, 0xa4, 0xee, 0x66, 0x27, 0xd5, 0x72, 0xa1, 0x2a, 0x51, 0x13, 0x35, 0xe9, 0x0b, 0x28, 0xc5, - 0xe0, 0x23, 0x37, 0xcf, 0x14, 0x9e, 0x42, 0xaa, 0x7e, 0xeb, 0x5c, 0xb9, 0x8a, 0x3e, 0x2d, 0xa2, - 0x97, 0x48, 0xa1, 0x29, 0x31, 0xf9, 0xf8, 0x26, 0xcc, 0xf4, 0xbc, 0x41, 0xda, 0xcc, 0xdf, 0xfb, - 0xac, 0xa0, 0xfe, 0xb9, 0xee, 0x4d, 0x8a, 0x0f, 0xd1, 0x7b, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, - 0xcb, 0x5c, 0xce, 0x34, 0xd2, 0x0e, 0x00, 0x00, + 0x1a, 0x09, 0x10, 0x1a, 0x6e, 0x3c, 0xc7, 0xc9, 0x28, 0xe3, 0x19, 0x33, 0x73, 0x9d, 0xd4, 0xaa, + 0xca, 0x82, 0x1d, 0x5b, 0x58, 0xb0, 0xe2, 0x07, 0xb0, 0x41, 0x48, 0xfc, 0x04, 0xf6, 0x2c, 0x60, + 0x0b, 0x3b, 0x16, 0xfc, 0x01, 0xf6, 0xe8, 0x3e, 0x66, 0x32, 0x93, 0x4c, 0x12, 0xb7, 0x2b, 0xdf, + 0x7b, 0xde, 0x8f, 0xef, 0x9e, 0x33, 0x06, 0x9d, 0x0e, 0x9c, 0xe6, 0xd1, 0xbd, 0x66, 0xd7, 0xa5, + 0x4e, 0x30, 0xd8, 0x93, 0xbf, 0x8d, 0x41, 0xe0, 0x33, 0x9f, 0x54, 0xba, 0x7e, 0x80, 0x7e, 0xd8, + 0x10, 0x34, 0xfd, 0xd6, 0xbe, 0xef, 0xef, 0xbb, 0xd8, 0x14, 0xbc, 0xbd, 0x61, 0xaf, 0xc9, 0x9c, + 0x3e, 0x86, 0x8c, 0xf6, 0x07, 0x52, 0x5c, 0xbf, 0xae, 0x04, 0xb8, 0x45, 0xea, 0x79, 0x3e, 0xa3, + 0xcc, 0xf1, 0xbd, 0x50, 0x72, 0x8d, 0x1f, 0x72, 0x50, 0xdd, 0x1d, 0xba, 0x1e, 0x06, 0x74, 0xcf, + 0x71, 0x1d, 0x36, 0x22, 0x04, 0xae, 0x78, 0xb4, 0x8f, 0x75, 0x6d, 0x49, 0xbb, 0x53, 0x32, 0xc5, + 0x99, 0xdc, 0x86, 0x29, 0xfe, 0x1b, 0x0e, 0x68, 0x17, 0x2d, 0xc1, 0xcd, 0x09, 0x6e, 0x35, 0xa6, + 0x6e, 0x72, 0xb1, 0x25, 0x28, 0xdb, 0x18, 0x76, 0x03, 0x67, 0xc0, 0x5d, 0xd4, 0x27, 0x84, 0x4c, + 0x92, 0xc4, 0x8d, 0xbb, 0x8e, 0x77, 0x58, 0xbf, 0x22, 0x8d, 0xf3, 0x33, 0xd1, 0xa1, 0x18, 0xe2, + 0x11, 0x06, 0x0e, 0x1b, 0xd5, 0xf3, 0x82, 0x1e, 0xdf, 0x39, 0xaf, 0x8f, 0x8c, 0xda, 0x94, 0xd1, + 0xfa, 0xa4, 0xe4, 0x45, 0x77, 0xb2, 0x00, 0xc5, 0x9e, 0xf3, 0x1c, 0x6d, 0x6b, 0x6f, 0x54, 0x2f, + 0x08, 0x5e, 0x41, 0xdc, 0x1f, 0x8f, 0xc8, 0x63, 0xb8, 0x4a, 0x7b, 0x3d, 0xec, 0x32, 0xb4, 0xad, + 0x23, 0x0c, 0x42, 0x9e, 0x70, 0xbd, 0xb8, 0x34, 0x71, 0xa7, 0xdc, 0xba, 0xd6, 0x48, 0x96, 0xaf, + 0xb1, 0x86, 0x94, 0x0d, 0x03, 0x34, 0x6b, 0x91, 0xfc, 0xae, 0x12, 0x37, 0x7e, 0xd7, 0xa0, 0xb8, + 0x8a, 0x0c, 0xbb, 0xcc, 0x0f, 0x32, 0x8b, 0x52, 0x87, 0x82, 0xb2, 0xad, 0xaa, 0x11, 0x5d, 0x49, + 0x0b, 0xf2, 0x36, 0x1b, 0x0d, 0x50, 0x54, 0x60, 0xaa, 0x75, 0x3d, 0xed, 0x32, 0x32, 0xda, 0x58, + 0xdd, 0x19, 0x0d, 0xd0, 0x94, 0xa2, 0xc6, 0x97, 0x90, 0x17, 0x77, 0xb2, 0x08, 0xf3, 0xab, 0x9d, + 0x9d, 0xce, 0xca, 0xce, 0x27, 0xa6, 0xb5, 0x6a, 0xed, 0x7c, 0xba, 0xd5, 0xb1, 0x36, 0x36, 0x77, + 0xdb, 0x4f, 0x36, 0x56, 0x6b, 0x6f, 0x90, 0x1b, 0xb0, 0x70, 0x9a, 0xb9, 0xd9, 0x7e, 0xda, 0xd9, + 0xde, 0x6a, 0xaf, 0x74, 0x6a, 0x5a, 0x96, 0xee, 0x5a, 0xa7, 0xbd, 0xf3, 0xcc, 0xec, 0xd4, 0x72, + 0xc6, 0x36, 0x94, 0x36, 0xa3, 0x76, 0x65, 0x26, 0xd4, 0x82, 0xa2, 0xad, 0x62, 0x13, 0x19, 0x95, + 0x5b, 0x73, 0xd9, 0x91, 0x9b, 0xb1, 0x9c, 0xf1, 0x6b, 0x0e, 0x0a, 0xaa, 0x86, 0x99, 0x36, 0x3f, + 0x80, 0x52, 0x8c, 0x11, 0x65, 0x74, 0x3e, 0x6d, 0x34, 0x8e, 0xc9, 0x3c, 0x91, 0x4c, 0xd6, 0x76, + 0x22, 0x5d, 0xdb, 0xdb, 0x30, 0xa5, 0x8e, 0x56, 0xcf, 0x0f, 0xfa, 0x94, 0x29, 0x2c, 0x55, 0x15, + 0x75, 0x4d, 0x10, 0x53, 0xb9, 0xe4, 0xc7, 0xcb, 0x85, 0x74, 0x60, 0xfa, 0x28, 0xf1, 0x14, 0x1c, + 0x0c, 0xeb, 0x93, 0x02, 0x33, 0x8b, 0x69, 0xd5, 0xd4, 0x7b, 0x31, 0x4f, 0xeb, 0x90, 0x65, 0xa8, + 0xf4, 0x64, 0x45, 0x2c, 0x01, 0x02, 0x89, 0xcd, 0xb2, 0xa2, 0xf1, 0x1e, 0x1b, 0x8b, 0x90, 0x7f, + 0x42, 0x47, 0x28, 0x70, 0x75, 0x40, 0xc3, 0x83, 0xa8, 0x64, 0xfc, 0x6c, 0x7c, 0xab, 0x41, 0x79, + 0x85, 0x3b, 0xda, 0x66, 0x94, 0x0d, 0x43, 0x72, 0x1f, 0x4a, 0x51, 0x88, 0x61, 0x5d, 0x13, 0x01, + 0x9d, 0x97, 0xcb, 0x89, 0x20, 0x59, 0x85, 0x9a, 0x4b, 0x43, 0x66, 0x0d, 0x07, 0x36, 0x65, 0x68, + 0xf1, 0xa9, 0xa0, 0xea, 0xaf, 0x37, 0xe4, 0x44, 0x68, 0x44, 0x23, 0xa3, 0xb1, 0x13, 0x8d, 0x0c, + 0x73, 0x8a, 0xeb, 0x3c, 0x13, 0x2a, 0x9c, 0x68, 0x3c, 0x04, 0xb2, 0x8e, 0xac, 0xed, 0x75, 0x31, + 0x64, 0xc1, 0xc8, 0xc4, 0xaf, 0x86, 0x18, 0x32, 0xf2, 0x26, 0x54, 0xa9, 0x22, 0x59, 0x89, 0x8e, + 0x57, 0x22, 0x22, 0x6f, 0xa9, 0xf1, 0xcb, 0x04, 0xcc, 0xa4, 0x74, 0xc3, 0x81, 0xef, 0x85, 0x48, + 0xd6, 0xa0, 0x18, 0xc9, 0x09, 0xbd, 0x72, 0xeb, 0x6e, 0x3a, 0x9b, 0x0c, 0xa5, 0x46, 0x4c, 0x88, + 0x75, 0xc9, 0xfb, 0x30, 0x19, 0x8a, 0x02, 0xa9, 0xb4, 0x16, 0xd2, 0x56, 0x12, 0x15, 0x34, 0x95, + 0xa0, 0xfe, 0x35, 0x54, 0x23, 0x43, 0xb2, 0xfc, 0xef, 0x40, 0xde, 0xe5, 0x07, 0x15, 0xc8, 0x4c, + 0xda, 0x84, 0x90, 0x31, 0xa5, 0x04, 0x1f, 0x29, 0xb2, 0xb8, 0x68, 0x5b, 0xaa, 0x95, 0xdc, 0xf3, + 0x45, 0x23, 0x25, 0x92, 0x57, 0x84, 0x50, 0xff, 0x51, 0x83, 0x62, 0x14, 0x40, 0xe6, 0x6b, 0x49, + 0xb5, 0x3a, 0x37, 0x6e, 0xab, 0xd7, 0x61, 0x52, 0xc4, 0x18, 0xd6, 0x27, 0x84, 0x4a, 0x73, 0xfc, + 0x7a, 0xca, 0x14, 0x95, 0xba, 0xf1, 0x77, 0x0e, 0x66, 0xb6, 0xfc, 0xf0, 0xb5, 0xfa, 0x4d, 0xe6, + 0x60, 0x52, 0x3d, 0x48, 0x39, 0x0d, 0xd5, 0x8d, 0xac, 0x9c, 0x8a, 0xee, 0xdd, 0x74, 0x74, 0x19, + 0xfe, 0x04, 0x2d, 0x15, 0x99, 0xfe, 0x9b, 0x06, 0xa5, 0x98, 0x9a, 0xf5, 0x6a, 0x38, 0x6d, 0x40, + 0xd9, 0x81, 0x72, 0x2e, 0xce, 0xc4, 0x84, 0xc2, 0x01, 0x52, 0xfb, 0xc4, 0xf7, 0x83, 0x57, 0xf0, + 0xdd, 0xf8, 0x58, 0xaa, 0x76, 0x3c, 0xce, 0x8d, 0x0c, 0xe9, 0x8f, 0xa0, 0x92, 0x64, 0x90, 0x1a, + 0x4c, 0x1c, 0xe2, 0x48, 0x85, 0xc2, 0x8f, 0x64, 0x16, 0xf2, 0x47, 0xd4, 0x1d, 0x46, 0x3b, 0x52, + 0x5e, 0x1e, 0xe5, 0x1e, 0x68, 0xc6, 0x06, 0xcc, 0xa6, 0x5d, 0xaa, 0x27, 0x71, 0x02, 0x65, 0x6d, + 0x4c, 0x28, 0x1b, 0x3f, 0x6b, 0x30, 0xb7, 0x8e, 0x6c, 0xd3, 0x67, 0x4e, 0xcf, 0xe9, 0x8a, 0x95, + 0x1e, 0x75, 0xeb, 0x3e, 0xcc, 0xf9, 0xae, 0x6d, 0x25, 0xc7, 0xd2, 0xc8, 0x1a, 0xd0, 0xfd, 0xa8, + 0x6d, 0xb3, 0xbe, 0x6b, 0xa7, 0x46, 0xd8, 0x16, 0xdd, 0xe7, 0xd0, 0x9b, 0xf3, 0xf0, 0x38, 0x4b, + 0x4b, 0xa6, 0x31, 0xeb, 0xe1, 0xf1, 0x59, 0xad, 0x59, 0xc8, 0xbb, 0x4e, 0xdf, 0x61, 0x62, 0x4a, + 0xe7, 0x4d, 0x79, 0x89, 0xa1, 0x7d, 0xe5, 0x04, 0xda, 0xc6, 0x5f, 0x39, 0x98, 0x3f, 0x13, 0xb0, + 0xca, 0x7f, 0x17, 0x2a, 0x5e, 0x82, 0xae, 0xaa, 0xd0, 0x3a, 0x03, 0xe3, 0x2c, 0xe5, 0x46, 0x8a, + 0x98, 0xb2, 0xa3, 0xff, 0xab, 0x41, 0x25, 0xc9, 0x3e, 0x6f, 0x8d, 0x77, 0x03, 0xa4, 0x0c, 0xed, + 0x68, 0x8d, 0xab, 0x2b, 0xff, 0xf8, 0x90, 0xe6, 0xd0, 0x56, 0x5b, 0x28, 0xbe, 0x73, 0x2d, 0x1b, + 0x5d, 0xe4, 0x5a, 0x32, 0xcb, 0xe8, 0x4a, 0x1e, 0xc2, 0x84, 0xef, 0xda, 0x6a, 0xe9, 0xbc, 0x7d, + 0x0a, 0x70, 0x74, 0x1f, 0xe3, 0xda, 0xbb, 0xa8, 0x80, 0xe0, 0x60, 0x68, 0x72, 0x1d, 0xae, 0xea, + 0xe1, 0xb1, 0xf8, 0xd0, 0x79, 0x15, 0x55, 0x0f, 0x8f, 0x8d, 0x3f, 0x72, 0xb0, 0x70, 0xae, 0x08, + 0x5f, 0x49, 0xdd, 0x61, 0x10, 0xa0, 0xc7, 0x92, 0x40, 0x28, 0x2b, 0x9a, 0xe8, 0xe4, 0x22, 0x94, + 0x3c, 0x7c, 0xce, 0x92, 0x2d, 0x2f, 0x72, 0xc2, 0x05, 0x6d, 0x6e, 0x43, 0x35, 0x05, 0x17, 0x51, + 0x89, 0x4b, 0xb6, 0x65, 0x5a, 0x83, 0x7c, 0x0e, 0x40, 0xe3, 0x30, 0xeb, 0x79, 0xf1, 0x48, 0x3f, + 0x1c, 0x33, 0xf1, 0xc6, 0x86, 0x67, 0xe3, 0x73, 0xb4, 0xdb, 0x89, 0x29, 0x64, 0x26, 0xcc, 0xe9, + 0x1f, 0xc1, 0x4c, 0x86, 0x08, 0x4f, 0xc6, 0xe1, 0x64, 0x51, 0x85, 0xbc, 0x29, 0x2f, 0x31, 0x34, + 0x72, 0x09, 0xcc, 0xde, 0x83, 0x1b, 0x4f, 0x69, 0x70, 0x98, 0x84, 0x50, 0x3b, 0x34, 0x91, 0xda, + 0xd1, 0x53, 0xcb, 0xc0, 0x93, 0xb1, 0x04, 0x37, 0xcf, 0x53, 0x92, 0x88, 0x35, 0x08, 0xd4, 0xd6, + 0x91, 0xa9, 0x07, 0x2d, 0x2d, 0x19, 0x6b, 0x70, 0x35, 0x41, 0x7b, 0xed, 0xb9, 0xd0, 0xfa, 0x4f, + 0x83, 0xe9, 0x28, 0xdb, 0x6d, 0x0c, 0x8e, 0x9c, 0x2e, 0x92, 0x21, 0x94, 0x13, 0x3b, 0x80, 0x2c, + 0x5d, 0xb0, 0x1e, 0x44, 0x30, 0xfa, 0xf2, 0xa5, 0x0b, 0xc4, 0x58, 0xfe, 0xe6, 0xcf, 0x7f, 0xbe, + 0xcf, 0x2d, 0x92, 0x85, 0x66, 0xb4, 0x04, 0x9a, 0x2f, 0x52, 0x3b, 0xe2, 0x25, 0x39, 0x84, 0x4a, + 0x72, 0xda, 0x91, 0xe5, 0x4b, 0x87, 0xaf, 0x6e, 0x5c, 0x24, 0xa2, 0x3c, 0xcf, 0x0a, 0xcf, 0x53, + 0x46, 0x29, 0xf6, 0xfc, 0x48, 0xbb, 0xdb, 0xfa, 0x29, 0x07, 0x33, 0xc9, 0x92, 0x47, 0xb9, 0xbf, + 0x84, 0xe9, 0x53, 0x83, 0x83, 0xbc, 0x75, 0xc9, 0x5c, 0x91, 0xa1, 0xdc, 0x1e, 0x6b, 0xfa, 0x18, + 0x37, 0x44, 0x34, 0xf3, 0xe4, 0x5a, 0x33, 0x39, 0x79, 0xc2, 0xe6, 0x0b, 0x59, 0x83, 0xef, 0x34, + 0x98, 0xcb, 0x46, 0x03, 0x39, 0xb5, 0x07, 0x2f, 0x04, 0x9a, 0xfe, 0xde, 0x78, 0xc2, 0xe9, 0xa0, + 0xee, 0x66, 0x07, 0xd5, 0xf2, 0xa0, 0x2a, 0x51, 0x13, 0x15, 0xe9, 0x0b, 0x28, 0xc5, 0xe0, 0x23, + 0x37, 0xcf, 0x24, 0x9e, 0x42, 0xaa, 0x7e, 0xeb, 0x5c, 0xbe, 0xf2, 0x3e, 0x2d, 0xbc, 0x97, 0x48, + 0xa1, 0x29, 0x31, 0xf9, 0xf8, 0x26, 0xcc, 0x74, 0xfd, 0x7e, 0x5a, 0x6d, 0xb0, 0xf7, 0x59, 0x41, + 0xfd, 0xb9, 0xdd, 0x9b, 0x14, 0x1f, 0xa2, 0xf7, 0xfe, 0x0f, 0x00, 0x00, 0xff, 0xff, 0x1f, 0x9e, + 0x9e, 0x7b, 0xf5, 0x0e, 0x00, 0x00, } diff --git a/api/v3/clairpb/clair.proto b/api/v3/clairpb/clair.proto index 015ab3ac..339b0c7c 100644 --- a/api/v3/clairpb/clair.proto +++ b/api/v3/clairpb/clair.proto @@ -80,6 +80,9 @@ message Feature { Detector detector = 5; // The list of vulnerabilities that affect the feature. repeated Vulnerability vulnerabilities = 6; + // The feature type indicates if the feature represents a source package or + // binary package. + string feature_type = 7; } message Layer { diff --git a/api/v3/clairpb/clair.swagger.json b/api/v3/clairpb/clair.swagger.json index 19396b43..f8216d04 100644 --- a/api/v3/clairpb/clair.swagger.json +++ b/api/v3/clairpb/clair.swagger.json @@ -330,6 +330,10 @@ "$ref": "#/definitions/clairVulnerability" }, "description": "The list of vulnerabilities that affect the feature." + }, + "feature_type": { + "type": "string", + "description": "The feature type indicates if the feature represents a source package or\nbinary package." } } }, diff --git a/api/v3/clairpb/convert.go b/api/v3/clairpb/convert.go index fe54b2f9..1334d1e9 100644 --- a/api/v3/clairpb/convert.go +++ b/api/v3/clairpb/convert.go @@ -99,6 +99,7 @@ func NotificationFromDatabaseModel(dbNotification database.VulnerabilityNotifica return ¬i, nil } +// VulnerabilityFromDatabaseModel converts database Vulnerability to api Vulnerability. func VulnerabilityFromDatabaseModel(dbVuln database.Vulnerability) (*Vulnerability, error) { metaString := "" if dbVuln.Metadata != nil { @@ -119,6 +120,7 @@ func VulnerabilityFromDatabaseModel(dbVuln database.Vulnerability) (*Vulnerabili }, nil } +// VulnerabilityWithFixedInFromDatabaseModel converts database VulnerabilityWithFixedIn to api Vulnerability. func VulnerabilityWithFixedInFromDatabaseModel(dbVuln database.VulnerabilityWithFixedIn) (*Vulnerability, error) { vuln, err := VulnerabilityFromDatabaseModel(dbVuln.Vulnerability) if err != nil { @@ -145,9 +147,11 @@ func NamespacedFeatureFromDatabaseModel(feature database.AncestryFeature) *Featu VersionFormat: feature.Namespace.VersionFormat, Version: version, Detector: DetectorFromDatabaseModel(feature.FeatureBy), + FeatureType: string(feature.Type), } } +// DetectorFromDatabaseModel converts database detector to api detector. func DetectorFromDatabaseModel(detector database.Detector) *Detector { return &Detector{ Name: detector.Name, @@ -156,6 +160,7 @@ func DetectorFromDatabaseModel(detector database.Detector) *Detector { } } +// DetectorsFromDatabaseModel converts database detectors to api detectors. func DetectorsFromDatabaseModel(dbDetectors []database.Detector) []*Detector { detectors := make([]*Detector, 0, len(dbDetectors)) for _, d := range dbDetectors { From 5a94499fdb52ecb07978af0132d91bf2c3e6e088 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Tue, 19 Feb 2019 17:57:38 -0500 Subject: [PATCH 12/12] update according to comments --- database/pgsql/complex_test.go | 33 +++++++++------------------------ database/pgsql/feature.go | 1 - 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/database/pgsql/complex_test.go b/database/pgsql/complex_test.go index e4a10928..af35d1f9 100644 --- a/database/pgsql/complex_test.go +++ b/database/pgsql/complex_test.go @@ -134,46 +134,31 @@ func TestConcurrency(t *testing.T) { func TestCaching(t *testing.T) { store, err := openDatabaseForTest("Caching", false) - if !assert.Nil(t, err) { - t.FailNow() - } + require.Nil(t, err) defer store.Close() + nsFeatures, vulnerabilities := testGenRandomVulnerabilityAndNamespacedFeature(t, store) tx, err := store.Begin() - if !assert.Nil(t, err) { - t.FailNow() - } + require.Nil(t, err) require.Nil(t, tx.PersistNamespacedFeatures(nsFeatures)) - if err := tx.Commit(); err != nil { - panic(err) - } + require.Nil(t, tx.Commit()) tx, err = store.Begin() - if !assert.Nil(t, err) { - t.FailNow() - } + require.Nil(t, tx.Commit()) require.Nil(t, tx.InsertVulnerabilities(vulnerabilities)) - if err := tx.Commit(); err != nil { - panic(err) - } + require.Nil(t, tx.Commit()) tx, err = store.Begin() - if !assert.Nil(t, err) { - t.FailNow() - } + require.Nil(t, err) defer tx.Rollback() affected, err := tx.FindAffectedNamespacedFeatures(nsFeatures) - if !assert.Nil(t, err) { - t.FailNow() - } + require.Nil(t, err) for _, ansf := range affected { - if !assert.True(t, ansf.Valid) { - t.FailNow() - } + require.True(t, ansf.Valid) expectedAffectedNames := []string{} for _, vuln := range vulnerabilities { diff --git a/database/pgsql/feature.go b/database/pgsql/feature.go index 3df7ac6a..e81cd2a2 100644 --- a/database/pgsql/feature.go +++ b/database/pgsql/feature.go @@ -270,7 +270,6 @@ func (tx *pgSession) FindAffectedNamespacedFeatures(features []database.Namespac if err != nil { return nil, handleError("searchNamespacedFeaturesVulnerabilities", err) } - defer rows.Close() for rows.Next() {