From 0a997145eda373fafce1b495f7be80da55cbeb62 Mon Sep 17 00:00:00 2001 From: liang chenye Date: Tue, 3 May 2016 15:25:31 +0800 Subject: [PATCH] support multiple namespaces in one layer; add database migration Signed-off-by: liang chenye --- database/database.go | 10 +- database/mock.go | 26 ++-- database/models.go | 21 +++- database/pgsql/complex_test.go | 7 +- database/pgsql/feature.go | 6 +- database/pgsql/feature_test.go | 28 +++-- database/pgsql/layer.go | 117 ++++++++++++++---- database/pgsql/layer_test.go | 106 +++++++++++----- .../migrations/20151222113213_Initial.sql | 5 + .../20160524162211_multiple_namespace.sql | 66 ++++++++++ database/pgsql/namespace.go | 10 +- database/pgsql/namespace_test.go | 17 +-- database/pgsql/notification_test.go | 16 ++- database/pgsql/queries.go | 59 ++++++--- database/pgsql/testdata/data.sql | 25 ++-- database/pgsql/vulnerability.go | 48 +++---- database/pgsql/vulnerability_test.go | 42 ++++--- updater/fetchers/debian/debian.go | 3 +- updater/fetchers/debian/debian_test.go | 12 +- updater/fetchers/rhel/rhel.go | 7 +- updater/fetchers/rhel/rhel_test.go | 10 +- updater/fetchers/ubuntu/ubuntu.go | 2 +- updater/fetchers/ubuntu/ubuntu_test.go | 6 +- worker/detectors/feature/dpkg/dpkg.go | 13 ++ worker/detectors/feature/rpm/rpm.go | 13 ++ worker/detectors/features.go | 22 +++- worker/detectors/namespace.go | 10 +- .../namespace/aptsources/aptsources.go | 3 +- .../namespace/aptsources/aptsources_test.go | 3 +- .../namespace/lsbrelease/lsbrelease.go | 3 +- .../namespace/lsbrelease/lsbrelease_test.go | 5 +- .../namespace/osrelease/osrelease.go | 3 +- .../namespace/osrelease/osrelease_test.go | 7 +- .../namespace/redhatrelease/redhatrelease.go | 3 +- .../redhatrelease/redhatrelease_test.go | 5 +- worker/detectors/namespace/test.go | 2 +- worker/worker.go | 116 ++++++----------- worker/worker_test.go | 33 +++-- 38 files changed, 586 insertions(+), 304 deletions(-) create mode 100644 database/pgsql/migrations/20160524162211_multiple_namespace.sql diff --git a/database/database.go b/database/database.go index 4ca13e42..ee2e59bc 100644 --- a/database/database.go +++ b/database/database.go @@ -93,7 +93,7 @@ type Datastore interface { // The Limit and page parameters are used to paginate the return list. // The first given page should be 0. The function will then return the next available page. // If there is no more page, -1 has to be returned. - ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error) + ListVulnerabilities(namespace Namespace, limit int, page int) ([]Vulnerability, int, error) // InsertVulnerabilities stores the given Vulnerabilities in the database, updating them if // necessary. A vulnerability is uniquely identified by its Namespace and its Name. @@ -110,22 +110,22 @@ type Datastore interface { InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error // FindVulnerability retrieves a Vulnerability from the database, including the FixedIn list. - FindVulnerability(namespaceName, name string) (Vulnerability, error) + FindVulnerability(namespace Namespace, name string) (Vulnerability, error) // DeleteVulnerability removes a Vulnerability from the database. // It has to create a Notification that will contain the old Vulnerability. - DeleteVulnerability(namespaceName, name string) error + DeleteVulnerability(namespace Namespace, name string) error // InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions of existing ones to // the specified Vulnerability in the database. // It has has to create a Notification that will contain the old and the updated Vulnerability. - InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error + InsertVulnerabilityFixes(vulnerabilityNamespace Namespace, vulnerabilityName string, fixes []FeatureVersion) error // DeleteVulnerabilityFix removes a FixedIn Feature from the specified Vulnerability in the // database. It can be used to store the fact that a Vulnerability no longer affects the given // Feature in any Version. // It has has to create a Notification that will contain the old and the updated Vulnerability. - DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error + DeleteVulnerabilityFix(vulnerabilityNamespace Namespace, vulnerabilityName, featureName string) error // # Notification // GetAvailableNotification returns the Name, Created, Notified and Deleted fields of a diff --git a/database/mock.go b/database/mock.go index 9a0963c8..6d3c1076 100644 --- a/database/mock.go +++ b/database/mock.go @@ -23,12 +23,12 @@ type MockDatastore struct { FctInsertLayer func(Layer) error FctFindLayer func(name string, withFeatures, withVulnerabilities bool) (Layer, error) FctDeleteLayer func(name string) error - FctListVulnerabilities func(namespaceName string, limit int, page int) ([]Vulnerability, int, error) + FctListVulnerabilities func(namespace Namespace, limit int, page int) ([]Vulnerability, int, error) FctInsertVulnerabilities func(vulnerabilities []Vulnerability, createNotification bool) error - FctFindVulnerability func(namespaceName, name string) (Vulnerability, error) - FctDeleteVulnerability func(namespaceName, name string) error - FctInsertVulnerabilityFixes func(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error - FctDeleteVulnerabilityFix func(vulnerabilityNamespace, vulnerabilityName, featureName string) error + FctFindVulnerability func(namespace Namespace, name string) (Vulnerability, error) + FctDeleteVulnerability func(namespace Namespace, name string) error + FctInsertVulnerabilityFixes func(vulnerabilityNamespace Namespace, vulnerabilityName string, fixes []FeatureVersion) error + FctDeleteVulnerabilityFix func(vulnerabilityNamespace Namespace, vulnerabilityName, featureName string) error FctGetAvailableNotification func(renotifyInterval time.Duration) (VulnerabilityNotification, error) FctGetNotification func(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error) FctSetNotificationNotified func(name string) error @@ -70,9 +70,9 @@ func (mds *MockDatastore) DeleteLayer(name string) error { panic("required mock function not implemented") } -func (mds *MockDatastore) ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error) { +func (mds *MockDatastore) ListVulnerabilities(namespace Namespace, limit int, page int) ([]Vulnerability, int, error) { if mds.FctListVulnerabilities != nil { - return mds.FctListVulnerabilities(namespaceName, limit, page) + return mds.FctListVulnerabilities(namespace, limit, page) } panic("required mock function not implemented") } @@ -84,28 +84,28 @@ func (mds *MockDatastore) InsertVulnerabilities(vulnerabilities []Vulnerability, panic("required mock function not implemented") } -func (mds *MockDatastore) FindVulnerability(namespaceName, name string) (Vulnerability, error) { +func (mds *MockDatastore) FindVulnerability(namespace Namespace, name string) (Vulnerability, error) { if mds.FctFindVulnerability != nil { - return mds.FctFindVulnerability(namespaceName, name) + return mds.FctFindVulnerability(namespace, name) } panic("required mock function not implemented") } -func (mds *MockDatastore) DeleteVulnerability(namespaceName, name string) error { +func (mds *MockDatastore) DeleteVulnerability(namespace Namespace, name string) error { if mds.FctDeleteVulnerability != nil { - return mds.FctDeleteVulnerability(namespaceName, name) + return mds.FctDeleteVulnerability(namespace, name) } panic("required mock function not implemented") } -func (mds *MockDatastore) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error { +func (mds *MockDatastore) InsertVulnerabilityFixes(vulnerabilityNamespace Namespace, vulnerabilityName string, fixes []FeatureVersion) error { if mds.FctInsertVulnerabilityFixes != nil { return mds.FctInsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName, fixes) } panic("required mock function not implemented") } -func (mds *MockDatastore) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error { +func (mds *MockDatastore) DeleteVulnerabilityFix(vulnerabilityNamespace Namespace, vulnerabilityName, featureName string) error { if mds.FctDeleteVulnerabilityFix != nil { return mds.FctDeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName) } diff --git a/database/models.go b/database/models.go index a44291b8..dd795ac5 100644 --- a/database/models.go +++ b/database/models.go @@ -33,14 +33,31 @@ type Layer struct { Name string EngineVersion int Parent *Layer - Namespace *Namespace + Namespaces []Namespace Features []FeatureVersion } type Namespace struct { Model - Name string + Name string + Version types.Version +} + +func (ns *Namespace) IsEmpty() bool { + if ns.Name == "" && ns.Version.String() == "" { + return true + } + + return false +} + +func (ns *Namespace) Equal(namespace Namespace) bool { + if ns.Name == namespace.Name && ns.Version.Compare(namespace.Version) == 0 { + return true + } + + return false } type Feature struct { diff --git a/database/pgsql/complex_test.go b/database/pgsql/complex_test.go index 46ba504a..b74b1034 100644 --- a/database/pgsql/complex_test.go +++ b/database/pgsql/complex_test.go @@ -46,8 +46,11 @@ func TestRaceAffects(t *testing.T) { // Insert the Feature on which we'll work. feature := database.Feature{ - Namespace: database.Namespace{Name: "TestRaceAffectsFeatureNamespace1"}, - Name: "TestRaceAffecturesFeature1", + Namespace: database.Namespace{ + Name: "TestRaceAffectsFeatureNamespace", + Version: types.NewVersionUnsafe("1.0"), + }, + Name: "TestRaceAffecturesFeature1", } _, err = datastore.insertFeature(feature) if err != nil { diff --git a/database/pgsql/feature.go b/database/pgsql/feature.go index a2f2abe8..ffd1c643 100644 --- a/database/pgsql/feature.go +++ b/database/pgsql/feature.go @@ -31,7 +31,7 @@ func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) { // Do cache lookup. if pgSQL.cache != nil { promCacheQueriesTotal.WithLabelValues("feature").Inc() - id, found := pgSQL.cache.Get("feature:" + feature.Namespace.Name + ":" + feature.Name) + id, found := pgSQL.cache.Get("feature:" + feature.Namespace.Name + ":" + feature.Namespace.Version.String() + ":" + feature.Name) if found { promCacheHitsTotal.WithLabelValues("feature").Inc() return id.(int), nil @@ -55,7 +55,7 @@ func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) { } if pgSQL.cache != nil { - pgSQL.cache.Add("feature:"+feature.Namespace.Name+":"+feature.Name, id) + pgSQL.cache.Add("feature:"+feature.Namespace.Name+":"+feature.Namespace.Version.String()+":"+feature.Name, id) } return id, nil @@ -67,7 +67,7 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion) } // Do cache lookup. - cacheIndex := "featureversion:" + featureVersion.Feature.Namespace.Name + ":" + featureVersion.Feature.Name + ":" + featureVersion.Version.String() + cacheIndex := "featureversion:" + featureVersion.Feature.Namespace.Name + ":" + featureVersion.Feature.Namespace.Version.String() + ":" + featureVersion.Feature.Name + ":" + featureVersion.Version.String() if pgSQL.cache != nil { promCacheQueriesTotal.WithLabelValues("featureversion").Inc() id, found := pgSQL.cache.Get(cacheIndex) diff --git a/database/pgsql/feature_test.go b/database/pgsql/feature_test.go index a857c30f..4ad0fffa 100644 --- a/database/pgsql/feature_test.go +++ b/database/pgsql/feature_test.go @@ -45,8 +45,11 @@ func TestInsertFeature(t *testing.T) { // Insert Feature and ensure we can find it. feature := database.Feature{ - Namespace: database.Namespace{Name: "TestInsertFeatureNamespace1"}, - Name: "TestInsertFeature1", + Namespace: database.Namespace{ + Name: "TestInsertFeatureNamespace", + Version: types.NewVersionUnsafe("1.0"), + }, + Name: "TestInsertFeature1", } id1, err := datastore.insertFeature(feature) assert.Nil(t, err) @@ -69,15 +72,21 @@ func TestInsertFeature(t *testing.T) { }, { Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertFeatureNamespace2"}, - Name: "TestInsertFeature2", + Namespace: database.Namespace{ + Name: "TestInsertFeatureNamespace", + Version: types.NewVersionUnsafe("2.0"), + }, + Name: "TestInsertFeature2", }, Version: types.NewVersionUnsafe(""), }, { Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertFeatureNamespace2"}, - Name: "TestInsertFeature2", + Namespace: database.Namespace{ + Name: "TestInsertFeatureNamespace", + Version: types.NewVersionUnsafe("2.0"), + }, + Name: "TestInsertFeature2", }, Version: types.NewVersionUnsafe("bad version"), }, @@ -90,8 +99,11 @@ func TestInsertFeature(t *testing.T) { // Insert FeatureVersion and ensure we can find it. featureVersion := database.FeatureVersion{ Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertFeatureNamespace1"}, - Name: "TestInsertFeature1", + Namespace: database.Namespace{ + Name: "TestInsertFeatureNamespace", + Version: types.NewVersionUnsafe("1.0"), + }, + Name: "TestInsertFeature1", }, Version: types.NewVersionUnsafe("2:3.0-imba"), } diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go index 66a41000..c2386c58 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer.go @@ -37,11 +37,10 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo var layer database.Layer var parentID zero.Int var parentName zero.String - var namespaceID zero.Int - var namespaceName sql.NullString t := time.Now() - err := pgSQL.QueryRow(searchLayer, name).Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentID, &parentName, &namespaceID, &namespaceName) + err := pgSQL.QueryRow(searchLayer, name). + Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentID, &parentName) observeQueryTime("FindLayer", "searchLayer", t) if err != nil { @@ -53,12 +52,50 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo Model: database.Model{ID: int(parentID.Int64)}, Name: parentName.String, } - } - if !namespaceID.IsZero() { - layer.Namespace = &database.Namespace{ - Model: database.Model{ID: int(namespaceID.Int64)}, - Name: namespaceName.String, + + // Find its parent's namespaces + t = time.Now() + rows, err := pgSQL.Query(searchLayerNamespace, parentID) + observeQueryTime("FindLayer", "searchParentLayerNamespace", t) + if err != nil { + return layer, handleError("searchLayerNamespace", err) } + defer rows.Close() + + for rows.Next() { + var pn database.Namespace + + err = rows.Scan(&pn.ID, &pn.Name, &pn.Version) + if err != nil { + return layer, handleError("searchLayerNamespace.Scan()", err) + } + layer.Parent.Namespaces = append(layer.Parent.Namespaces, pn) + } + if err = rows.Err(); err != nil { + return layer, handleError("searchLayerNamespace.Rows()", err) + } + } + + // Find its namespaces + t = time.Now() + rows, err := pgSQL.Query(searchLayerNamespace, layer.ID) + observeQueryTime("FindLayer", "searchLayerNamespace", t) + if err != nil { + return layer, handleError("searchLayerNamespace", err) + } + defer rows.Close() + + for rows.Next() { + var namespace database.Namespace + + err = rows.Scan(&namespace.ID, &namespace.Name, &namespace.Version) + if err != nil { + return layer, handleError("searchLayerNamespace.Scan()", err) + } + layer.Namespaces = append(layer.Namespaces, namespace) + } + if err = rows.Err(); err != nil { + return layer, handleError("searchLayerNamespace.Rows()", err) } // Find its features @@ -128,8 +165,9 @@ func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion var featureVersion database.FeatureVersion err = rows.Scan(&featureVersion.ID, &modification, &featureVersion.Feature.Namespace.ID, - &featureVersion.Feature.Namespace.Name, &featureVersion.Feature.ID, - &featureVersion.Feature.Name, &featureVersion.ID, &featureVersion.Version, + &featureVersion.Feature.Namespace.Name, &featureVersion.Feature.Namespace.Version, + &featureVersion.Feature.ID, &featureVersion.Feature.Name, + &featureVersion.ID, &featureVersion.Version, &featureVersion.AddedBy.ID, &featureVersion.AddedBy.Name) if err != nil { return featureVersions, handleError("searchLayerFeatureVersion.Scan()", err) @@ -184,7 +222,8 @@ func loadAffectedBy(tx *sql.Tx, featureVersions []database.FeatureVersion) error var vulnerability database.Vulnerability err := rows.Scan(&featureversionID, &vulnerability.ID, &vulnerability.Name, &vulnerability.Description, &vulnerability.Link, &vulnerability.Severity, - &vulnerability.Metadata, &vulnerability.Namespace.Name, &vulnerability.FixedBy) + &vulnerability.Metadata, &vulnerability.Namespace.Name, &vulnerability.Namespace.Version, + &vulnerability.FixedBy) if err != nil { return handleError("searchFeatureVersionVulnerability.Scan()", err) } @@ -234,6 +273,23 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { // We do `defer observeQueryTime` here because we don't want to observe existing layers. defer observeQueryTime("InsertLayer", "all", tf) + // Insert Namespaces + mapNamespaceIDs := make(map[string]int) + for i, _ := range layer.Namespaces { + id, err := pgSQL.insertNamespace(layer.Namespaces[i]) + if err != nil { + return err + } + if layer.Namespaces[i].ID == 0 { + layer.Namespaces[i].ID = id + } + + // Layer's namespaces has high priority than its parent. + // Once a layer has a 'same' namespace with its parent, + // it will only keep its namespace. + mapNamespaceIDs[layer.Namespaces[i].Name] = id + } + // Get parent ID. var parentID zero.Int if layer.Parent != nil { @@ -243,20 +299,13 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { } parentID = zero.IntFrom(int64(layer.Parent.ID)) - } - // Find or insert namespace if provided. - var namespaceID zero.Int - if layer.Namespace != nil { - n, err := pgSQL.insertNamespace(*layer.Namespace) - if err != nil { - return err - } - namespaceID = zero.IntFrom(int64(n)) - } else if layer.Namespace == nil && layer.Parent != nil { - // Import the Namespace from the parent if it has one and this layer doesn't specify one. - if layer.Parent.Namespace != nil { - namespaceID = zero.IntFrom(int64(layer.Parent.Namespace.ID)) + for _, pn := range layer.Parent.Namespaces { + if _, ok := mapNamespaceIDs[pn.Name]; !ok { + // Layer will inherit its parent's namespace + mapNamespaceIDs[pn.Name] = pn.ID + layer.Namespaces = append(layer.Namespaces, pn) + } } } @@ -269,7 +318,7 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { if layer.ID == 0 { // Insert a new layer. - err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID, namespaceID). + err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID). Scan(&layer.ID) if err != nil { tx.Rollback() @@ -283,12 +332,19 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { } } else { // Update an existing layer. - _, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion, namespaceID) + _, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion) if err != nil { tx.Rollback() return handleError("updateLayer", err) } + // Remove all existing LayerNamespace. + _, err = tx.Exec(removeLayerNamespace, layer.ID) + if err != nil { + tx.Rollback() + return handleError("removeLayerNamespace", err) + } + // Remove all existing Layer_diff_FeatureVersion. _, err = tx.Exec(removeLayerDiffFeatureVersion, layer.ID) if err != nil { @@ -297,6 +353,15 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { } } + // Insert Layer_Namespace + for _, id := range mapNamespaceIDs { + _, err = tx.Exec(soiLayerNamespace, layer.ID, id) + if err != nil { + tx.Rollback() + return err + } + } + // Update Layer_diff_FeatureVersion now. err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer) if err != nil { diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer_test.go index c45cbbed..b1d005df 100644 --- a/database/pgsql/layer_test.go +++ b/database/pgsql/layer_test.go @@ -33,13 +33,17 @@ func TestFindLayer(t *testing.T) { } defer datastore.Close() + testDebian7 := database.Namespace{ + Name: "debian", + Version: types.NewVersionUnsafe("7"), + } // Layer-0: no parent, no namespace, no feature, no vulnerability layer, err := datastore.FindLayer("layer-0", false, false) if assert.Nil(t, err) && assert.NotNil(t, layer) { assert.Equal(t, "layer-0", layer.Name) - assert.Nil(t, layer.Namespace) assert.Nil(t, layer.Parent) assert.Equal(t, 1, layer.EngineVersion) + assert.Len(t, layer.Namespaces, 0) assert.Len(t, layer.Features, 0) } @@ -52,18 +56,19 @@ func TestFindLayer(t *testing.T) { layer, err = datastore.FindLayer("layer-1", false, false) if assert.Nil(t, err) && assert.NotNil(t, layer) { assert.Equal(t, layer.Name, "layer-1") - assert.Equal(t, "debian:7", layer.Namespace.Name) if assert.NotNil(t, layer.Parent) { assert.Equal(t, "layer-0", layer.Parent.Name) } assert.Equal(t, 1, layer.EngineVersion) + assert.Len(t, layer.Namespaces, 1) + assert.True(t, testDebian7.Equal(layer.Namespaces[0])) assert.Len(t, layer.Features, 0) } layer, err = datastore.FindLayer("layer-1", true, false) if assert.Nil(t, err) && assert.NotNil(t, layer) && assert.Len(t, layer.Features, 2) { for _, featureVersion := range layer.Features { - assert.Equal(t, "debian:7", featureVersion.Feature.Namespace.Name) + assert.True(t, testDebian7.Equal(featureVersion.Feature.Namespace)) switch featureVersion.Feature.Name { case "wechat": @@ -79,7 +84,7 @@ func TestFindLayer(t *testing.T) { layer, err = datastore.FindLayer("layer-1", true, true) if assert.Nil(t, err) && assert.NotNil(t, layer) && assert.Len(t, layer.Features, 2) { for _, featureVersion := range layer.Features { - assert.Equal(t, "debian:7", featureVersion.Feature.Namespace.Name) + assert.True(t, testDebian7.Equal(featureVersion.Feature.Namespace)) switch featureVersion.Feature.Name { case "wechat": @@ -88,7 +93,7 @@ func TestFindLayer(t *testing.T) { assert.Equal(t, types.NewVersionUnsafe("1.0"), featureVersion.Version) if assert.Len(t, featureVersion.AffectedBy, 1) { - assert.Equal(t, "debian:7", featureVersion.AffectedBy[0].Namespace.Name) + assert.True(t, testDebian7.Equal(featureVersion.AffectedBy[0].Namespace)) assert.Equal(t, "CVE-OPENSSL-1-DEB7", featureVersion.AffectedBy[0].Name) assert.Equal(t, types.High, featureVersion.AffectedBy[0].Severity) assert.Equal(t, "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", featureVersion.AffectedBy[0].Description) @@ -137,44 +142,56 @@ func testInsertLayerInvalid(t *testing.T, datastore database.Datastore) { } func testInsertLayerTree(t *testing.T, datastore database.Datastore) { + testInsertLayerNamespace1 := database.Namespace{ + Name: "TestInsertLayerNamespace", + Version: types.NewVersionUnsafe("1"), + } + testInsertLayerNamespace2 := database.Namespace{ + Name: "TestInsertLayerNamespace", + Version: types.NewVersionUnsafe("2"), + } + testInsertLayerNamespace3 := database.Namespace{ + Name: "TestInsertLayerNamespace", + Version: types.NewVersionUnsafe("3"), + } f1 := database.FeatureVersion{ Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"}, + Namespace: testInsertLayerNamespace2, Name: "TestInsertLayerFeature1", }, Version: types.NewVersionUnsafe("1.0"), } f2 := database.FeatureVersion{ Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"}, + Namespace: testInsertLayerNamespace2, Name: "TestInsertLayerFeature2", }, Version: types.NewVersionUnsafe("0.34"), } f3 := database.FeatureVersion{ Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"}, + Namespace: testInsertLayerNamespace2, Name: "TestInsertLayerFeature3", }, Version: types.NewVersionUnsafe("0.56"), } f4 := database.FeatureVersion{ Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"}, + Namespace: testInsertLayerNamespace3, Name: "TestInsertLayerFeature2", }, Version: types.NewVersionUnsafe("0.34"), } f5 := database.FeatureVersion{ Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"}, + Namespace: testInsertLayerNamespace3, Name: "TestInsertLayerFeature3", }, Version: types.NewVersionUnsafe("0.56"), } f6 := database.FeatureVersion{ Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"}, + Namespace: testInsertLayerNamespace3, Name: "TestInsertLayerFeature4", }, Version: types.NewVersionUnsafe("0.666"), @@ -185,16 +202,16 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { Name: "TestInsertLayer1", }, { - Name: "TestInsertLayer2", - Parent: &database.Layer{Name: "TestInsertLayer1"}, - Namespace: &database.Namespace{Name: "TestInsertLayerNamespace1"}, + Name: "TestInsertLayer2", + Parent: &database.Layer{Name: "TestInsertLayer1"}, + Namespaces: []database.Namespace{testInsertLayerNamespace1}, }, // This layer changes the namespace and adds Features. { - Name: "TestInsertLayer3", - Parent: &database.Layer{Name: "TestInsertLayer2"}, - Namespace: &database.Namespace{Name: "TestInsertLayerNamespace2"}, - Features: []database.FeatureVersion{f1, f2, f3}, + Name: "TestInsertLayer3", + Parent: &database.Layer{Name: "TestInsertLayer2"}, + Namespaces: []database.Namespace{testInsertLayerNamespace2}, + Features: []database.FeatureVersion{f1, f2, f3}, }, // This layer covers the case where the last layer doesn't provide any new Feature. { @@ -206,9 +223,9 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { // It also modifies the Namespace ("upgrade") but keeps some Features not upgraded, their // Namespaces should then remain unchanged. { - Name: "TestInsertLayer4b", - Parent: &database.Layer{Name: "TestInsertLayer3"}, - Namespace: &database.Namespace{Name: "TestInsertLayerNamespace3"}, + Name: "TestInsertLayer4b", + Parent: &database.Layer{Name: "TestInsertLayer3"}, + Namespaces: []database.Namespace{testInsertLayerNamespace3}, Features: []database.FeatureVersion{ // Deletes TestInsertLayerFeature1. // Keep TestInsertLayerFeature2 (old Namespace should be kept): @@ -238,8 +255,8 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { } l4a := retrievedLayers["TestInsertLayer4a"] - if assert.NotNil(t, l4a.Namespace) { - assert.Equal(t, "TestInsertLayerNamespace2", l4a.Namespace.Name) + if assert.Len(t, l4a.Namespaces, 1) { + assert.True(t, testInsertLayerNamespace2.Equal(l4a.Namespaces[0])) } assert.Len(t, l4a.Features, 3) for _, featureVersion := range l4a.Features { @@ -249,8 +266,8 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { } l4b := retrievedLayers["TestInsertLayer4b"] - if assert.NotNil(t, l4b.Namespace) { - assert.Equal(t, "TestInsertLayerNamespace3", l4b.Namespace.Name) + if assert.Len(t, l4b.Namespaces, 1) { + assert.True(t, testInsertLayerNamespace3.Equal(l4b.Namespaces[0])) } assert.Len(t, l4b.Features, 3) for _, featureVersion := range l4b.Features { @@ -261,9 +278,17 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { } func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { + testInsertLayerNamespace3 := database.Namespace{ + Name: "TestInsertLayerNamespace", + Version: types.NewVersionUnsafe("3"), + } + testInsertLayerNamespaceUpdated1 := database.Namespace{ + Name: "TestInsertLayerNamespaceUpdated", + Version: types.NewVersionUnsafe("1"), + } f7 := database.FeatureVersion{ Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"}, + Namespace: testInsertLayerNamespace3, Name: "TestInsertLayerFeature7", }, Version: types.NewVersionUnsafe("0.01"), @@ -271,10 +296,10 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { l3, _ := datastore.FindLayer("TestInsertLayer3", true, false) l3u := database.Layer{ - Name: l3.Name, - Parent: l3.Parent, - Namespace: &database.Namespace{Name: "TestInsertLayerNamespaceUpdated1"}, - Features: []database.FeatureVersion{f7}, + Name: l3.Name, + Parent: l3.Parent, + Namespaces: []database.Namespace{testInsertLayerNamespaceUpdated1}, + Features: []database.FeatureVersion{f7}, } l4u := database.Layer{ @@ -290,7 +315,9 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { l3uf, err := datastore.FindLayer(l3u.Name, true, false) if assert.Nil(t, err) { - assert.Equal(t, l3.Namespace.Name, l3uf.Namespace.Name) + if assert.Len(t, l3.Namespaces, 1) && assert.Len(t, l3uf.Namespaces, 1) { + assert.True(t, l3.Namespaces[0].Equal(l3uf.Namespaces[0])) + } assert.Equal(t, l3.EngineVersion, l3uf.EngineVersion) assert.Len(t, l3uf.Features, len(l3.Features)) } @@ -303,7 +330,9 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { l3uf, err = datastore.FindLayer(l3u.Name, true, false) if assert.Nil(t, err) { - assert.Equal(t, l3u.Namespace.Name, l3uf.Namespace.Name) + if assert.Len(t, l3u.Namespaces, 1) && assert.Len(t, l3uf.Namespaces, 2) { + assert.True(t, containNS(l3uf.Namespaces, l3u.Namespaces[0]), "Updated layer should have %#v", l3u.Namespaces[0]) + } assert.Equal(t, l3u.EngineVersion, l3uf.EngineVersion) if assert.Len(t, l3uf.Features, 1) { assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l3uf.Features[0]) @@ -319,7 +348,9 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { l4uf, err := datastore.FindLayer(l3u.Name, true, false) if assert.Nil(t, err) { - assert.Equal(t, l3u.Namespace.Name, l4uf.Namespace.Name) + if assert.Len(t, l3u.Namespaces, 1) && assert.Len(t, l4uf.Namespaces, 2) { + assert.True(t, containNS(l4uf.Namespaces, l3u.Namespaces[0]), "Updated layer should have %#v", l3u.Namespaces[0]) + } assert.Equal(t, l4u.EngineVersion, l4uf.EngineVersion) if assert.Len(t, l4uf.Features, 1) { assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l4uf.Features[0]) @@ -349,3 +380,12 @@ func cmpFV(a, b database.FeatureVersion) bool { a.Feature.Namespace.Name == b.Feature.Namespace.Name && a.Version.String() == b.Version.String() } + +func containNS(namespaces []database.Namespace, namespace database.Namespace) bool { + for _, n := range namespaces { + if n.Equal(namespace) { + return true + } + } + return false +} diff --git a/database/pgsql/migrations/20151222113213_Initial.sql b/database/pgsql/migrations/20151222113213_Initial.sql index 7bb024d4..4b2d3d12 100644 --- a/database/pgsql/migrations/20151222113213_Initial.sql +++ b/database/pgsql/migrations/20151222113213_Initial.sql @@ -172,3 +172,8 @@ DROP TABLE IF EXISTS Namespace, KeyValue, Lock CASCADE; + +DROP TYPE IF EXISTS modification, + severity + CASCADE; + diff --git a/database/pgsql/migrations/20160524162211_multiple_namespace.sql b/database/pgsql/migrations/20160524162211_multiple_namespace.sql new file mode 100644 index 00000000..ae1f32d2 --- /dev/null +++ b/database/pgsql/migrations/20160524162211_multiple_namespace.sql @@ -0,0 +1,66 @@ +-- Copyright 2015 clair authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +goose Up + +-- ----------------------------------------------------- +-- Namespace table and data +-- ----------------------------------------------------- +ALTER TABLE Namespace ADD version VARCHAR(128) NULL; +UPDATE Namespace SET version = split_part(Namespace.Name, ':', 2), name = split_part(Namespace.Name,':', 1); + +-- ----------------------------------------------------- +-- LayerNamespace table and data +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS LayerNamespace ( + id SERIAL PRIMARY KEY, + layer_id INT NOT NULL REFERENCES Layer ON DELETE CASCADE, + namespace_id INT NOT NULL REFERENCES Namespace ON DELETE CASCADE, + UNIQUE (layer_id, namespace_id)); +CREATE INDEX ON LayerNamespace (layer_id); +CREATE INDEX ON LayerNamespace (layer_id, namespace_id); + +INSERT INTO LayerNamespace(layer_id, namespace_id) + SELECT id, namespace_id + from Layer; + +-- ----------------------------------------------------- +-- Layer table +-- ----------------------------------------------------- +ALTER TABLE Layer DROP COLUMN namespace_id; + +-- +goose Down + +-- ----------------------------------------------------- +-- Layer table and data +-- ----------------------------------------------------- +ALTER TABLE Layer ADD namespace_id INT NULL REFERENCES Namespace; + CREATE INDEX ON Layer (namespace_id); + +UPDATE Layer l SET namespace_id = +(SELECT namespace_id from LayerNamespace ln +WHERE l.id = ln.layer_id LIMIT 1); + +-- ----------------------------------------------------- +-- LayerNamespace table (and data) +-- ----------------------------------------------------- +DROP TABLE IF EXISTS LayerNamespace + CASCADE; + +-- ----------------------------------------------------- +-- LayerNamespace data and table +-- ----------------------------------------------------- +UPDATE Namespace n SET name = concat(n.name, ':', n.version); + +ALTER TABLE Namespace DROP COLUMN version; diff --git a/database/pgsql/namespace.go b/database/pgsql/namespace.go index 3c85c784..d48c4910 100644 --- a/database/pgsql/namespace.go +++ b/database/pgsql/namespace.go @@ -22,13 +22,13 @@ import ( ) func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) { - if namespace.Name == "" { + if namespace.IsEmpty() { return 0, cerrors.NewBadRequestError("could not find/insert invalid Namespace") } if pgSQL.cache != nil { promCacheQueriesTotal.WithLabelValues("namespace").Inc() - if id, found := pgSQL.cache.Get("namespace:" + namespace.Name); found { + if id, found := pgSQL.cache.Get("namespace:" + namespace.Name + ":" + namespace.Version.String()); found { promCacheHitsTotal.WithLabelValues("namespace").Inc() return id.(int), nil } @@ -38,13 +38,13 @@ func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) { defer observeQueryTime("insertNamespace", "all", time.Now()) var id int - err := pgSQL.QueryRow(soiNamespace, namespace.Name).Scan(&id) + err := pgSQL.QueryRow(soiNamespace, namespace.Name, &namespace.Version).Scan(&id) if err != nil { return 0, handleError("soiNamespace", err) } if pgSQL.cache != nil { - pgSQL.cache.Add("namespace:"+namespace.Name, id) + pgSQL.cache.Add("namespace:"+namespace.Name+":"+namespace.Version.String(), id) } return id, nil @@ -60,7 +60,7 @@ func (pgSQL *pgSQL) ListNamespaces() (namespaces []database.Namespace, err error for rows.Next() { var namespace database.Namespace - err = rows.Scan(&namespace.ID, &namespace.Name) + err = rows.Scan(&namespace.ID, &namespace.Name, &namespace.Version) if err != nil { return namespaces, handleError("listNamespace.Scan()", err) } diff --git a/database/pgsql/namespace_test.go b/database/pgsql/namespace_test.go index b9bf96fe..f6189e8b 100644 --- a/database/pgsql/namespace_test.go +++ b/database/pgsql/namespace_test.go @@ -18,9 +18,9 @@ import ( "fmt" "testing" - "github.com/stretchr/testify/assert" - "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" + "github.com/stretchr/testify/assert" ) func TestInsertNamespace(t *testing.T) { @@ -37,9 +37,9 @@ func TestInsertNamespace(t *testing.T) { assert.Zero(t, id0) // Insert Namespace and ensure we can find it. - id1, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace1"}) + id1, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace", Version: types.NewVersionUnsafe("1")}) assert.Nil(t, err) - id2, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace1"}) + id2, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace", Version: types.NewVersionUnsafe("1")}) assert.Nil(t, err) assert.Equal(t, id1, id2) } @@ -55,12 +55,13 @@ func TestListNamespace(t *testing.T) { namespaces, err := datastore.ListNamespaces() assert.Nil(t, err) if assert.Len(t, namespaces, 2) { + testDebian7 := database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("7")} + testDebian8 := database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")} for _, namespace := range namespaces { - switch namespace.Name { - case "debian:7", "debian:8": + if namespace.Equal(testDebian7) || namespace.Equal(testDebian8) { continue - default: - assert.Error(t, fmt.Errorf("ListNamespaces should not have returned '%s'", namespace.Name)) + } else { + assert.Error(t, fmt.Errorf("ListNamespaces should not have returned '%s:%s'", namespace.Name, namespace.Version.String())) } } } diff --git a/database/pgsql/notification_test.go b/database/pgsql/notification_test.go index 3f90f349..cd45fb3b 100644 --- a/database/pgsql/notification_test.go +++ b/database/pgsql/notification_test.go @@ -39,13 +39,19 @@ func TestNotification(t *testing.T) { // Create some data. f1 := database.Feature{ - Name: "TestNotificationFeature1", - Namespace: database.Namespace{Name: "TestNotificationNamespace1"}, + Name: "TestNotificationFeature1", + Namespace: database.Namespace{ + Name: "TestNotificationNamespace", + Version: types.NewVersionUnsafe("1.0"), + }, } f2 := database.Feature{ - Name: "TestNotificationFeature2", - Namespace: database.Namespace{Name: "TestNotificationNamespace1"}, + Name: "TestNotificationFeature2", + Namespace: database.Namespace{ + Name: "TestNotificationNamespace", + Version: types.NewVersionUnsafe("1.0"), + }, } l1 := database.Layer{ @@ -201,7 +207,7 @@ func TestNotification(t *testing.T) { } // Delete a vulnerability and verify the notification. - if assert.Nil(t, datastore.DeleteVulnerability(v1b.Namespace.Name, v1b.Name)) { + if assert.Nil(t, datastore.DeleteVulnerability(v1b.Namespace, v1b.Name)) { notification, err = datastore.GetAvailableNotification(time.Second) assert.Nil(t, err) assert.NotEmpty(t, notification.Name) diff --git a/database/pgsql/queries.go b/database/pgsql/queries.go index 9cff94b8..6f291a2b 100644 --- a/database/pgsql/queries.go +++ b/database/pgsql/queries.go @@ -29,17 +29,17 @@ const ( // namespace.go soiNamespace = ` WITH new_namespace AS ( - INSERT INTO Namespace(name) - SELECT CAST($1 AS VARCHAR) - WHERE NOT EXISTS (SELECT name FROM Namespace WHERE name = $1) + INSERT INTO Namespace(name, version) + SELECT CAST($1 AS VARCHAR), CAST($2 AS VARCHAR) + WHERE NOT EXISTS (SELECT name FROM Namespace WHERE name = $1 AND version = $2) RETURNING id ) - SELECT id FROM Namespace WHERE name = $1 + SELECT id FROM Namespace WHERE name = $1 AND version = $2 UNION SELECT id FROM new_namespace` - searchNamespace = `SELECT id FROM Namespace WHERE name = $1` - listNamespace = `SELECT id, name FROM Namespace` + searchNamespace = `SELECT id FROM Namespace WHERE name = $1 AND version = $2` + listNamespace = `SELECT id, name, version FROM Namespace` // feature.go soiFeature = ` @@ -77,10 +77,9 @@ const ( // layer.go searchLayer = ` - SELECT l.id, l.name, l.engineversion, p.id, p.name, n.id, n.name + SELECT l.id, l.name, l.engineversion, p.id, p.name FROM Layer l LEFT JOIN Layer p ON l.parent_id = p.id - LEFT JOIN Namespace n ON l.namespace_id = n.id WHERE l.name = $1;` searchLayerFeatureVersion = ` @@ -93,7 +92,7 @@ const ( FROM Layer l, layer_tree lt WHERE l.id = lt.parent_id ) - SELECT ldf.featureversion_id, ldf.modification, fn.id, fn.name, f.id, f.name, fv.id, fv.version, ltree.id, ltree.name + SELECT ldf.featureversion_id, ldf.modification, fn.id, fn.name, fn.version, f.id, f.name, fv.id, fv.version, ltree.id, ltree.name FROM Layer_diff_FeatureVersion ldf JOIN ( SELECT row_number() over (ORDER BY depth DESC), id, name FROM layer_tree @@ -103,7 +102,7 @@ const ( searchFeatureVersionVulnerability = ` SELECT vafv.featureversion_id, v.id, v.name, v.description, v.link, v.severity, v.metadata, - vn.name, vfif.version + vn.name, vn.version, vfif.version FROM Vulnerability_Affects_FeatureVersion vafv, Vulnerability v, Namespace vn, Vulnerability_FixedIn_Feature vfif WHERE vafv.featureversion_id = ANY($1::integer[]) @@ -112,12 +111,17 @@ const ( AND v.namespace_id = vn.id AND v.deleted_at IS NULL` - insertLayer = ` - INSERT INTO Layer(name, engineversion, parent_id, namespace_id, created_at) - VALUES($1, $2, $3, $4, CURRENT_TIMESTAMP) - RETURNING id` + searchLayerNamespace = ` + SELECT n.id, n.name, n.version + FROM LayerNamespace ln, Namespace n + WHERE ln.layer_id = $1 AND ln.namespace_id = n.id` - updateLayer = `UPDATE LAYER SET engineversion = $2, namespace_id = $3 WHERE id = $1` + insertLayer = ` + INSERT INTO Layer(name, engineversion, parent_id, created_at) + VALUES($1, $2, $3, CURRENT_TIMESTAMP) + RETURNING id` + + updateLayer = `UPDATE LAYER SET engineversion = $2 WHERE id = $1` removeLayerDiffFeatureVersion = ` DELETE FROM Layer_diff_FeatureVersion @@ -131,6 +135,21 @@ const ( removeLayer = `DELETE FROM Layer WHERE name = $1` + removeLayerNamespace = ` + DELETE FROM LayerNamespace + WHERE layer_id = $1` + + soiLayerNamespace = ` + WITH new_layernamespace AS ( + INSERT INTO LayerNamespace(layer_id, namespace_id) + SELECT CAST($1 AS INTEGER), CAST($2 AS INTEGER) + WHERE NOT EXISTS (SELECT id FROM LayerNamespace WHERE layer_id = $1 AND namespace_id = $2) + RETURNING id + ) + SELECT id FROM LayerNamespace WHERE layer_id = $1 AND namespace_id = $2 + UNION + SELECT id FROM new_layernamespace` + // lock.go insertLock = `INSERT INTO Lock(name, owner, until) VALUES($1, $2, $3)` searchLock = `SELECT owner, until FROM Lock WHERE name = $1` @@ -140,12 +159,12 @@ const ( // vulnerability.go searchVulnerabilityBase = ` - SELECT v.id, v.name, n.id, n.name, v.description, v.link, v.severity, v.metadata + SELECT v.id, v.name, n.id, n.name, n.version, v.description, v.link, v.severity, v.metadata FROM Vulnerability v JOIN Namespace n ON v.namespace_id = n.id` searchVulnerabilityForUpdate = ` FOR UPDATE OF v` - searchVulnerabilityByNamespaceAndName = ` WHERE n.name = $1 AND v.name = $2 AND v.deleted_at IS NULL` + searchVulnerabilityByNamespaceAndName = ` WHERE n.name = $1 AND n.version = $2 AND v.name = $3 AND v.deleted_at IS NULL` searchVulnerabilityByID = ` WHERE v.id = $1` - searchVulnerabilityByNamespace = ` WHERE n.name = $1 AND v.deleted_at IS NULL + searchVulnerabilityByNamespaceID = ` WHERE n.id = $1 AND v.deleted_at IS NULL AND v.id >= $2 ORDER BY v.id LIMIT $3` @@ -170,8 +189,8 @@ const ( removeVulnerability = ` UPDATE Vulnerability SET deleted_at = CURRENT_TIMESTAMP - WHERE namespace_id = (SELECT id FROM Namespace WHERE name = $1) - AND name = $2 + WHERE namespace_id = (SELECT id FROM Namespace WHERE name = $1 AND version = $2) + AND name = $3 AND deleted_at IS NULL RETURNING id` diff --git a/database/pgsql/testdata/data.sql b/database/pgsql/testdata/data.sql index 7a48ef64..210d3f3e 100644 --- a/database/pgsql/testdata/data.sql +++ b/database/pgsql/testdata/data.sql @@ -12,9 +12,9 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -INSERT INTO namespace (id, name) VALUES - (1, 'debian:7'), - (2, 'debian:8'); +INSERT INTO namespace (id, name, version) VALUES + (1, 'debian', '7'), + (2, 'debian', '8'); INSERT INTO feature (id, namespace_id, name) VALUES (1, 1, 'wechat'), @@ -28,12 +28,18 @@ INSERT INTO featureversion (id, feature_id, version) VALUES (3, 2, '2.0'), (4, 3, '1.0'); -INSERT INTO layer (id, name, engineversion, parent_id, namespace_id) VALUES - (1, 'layer-0', 1, NULL, NULL), - (2, 'layer-1', 1, 1, 1), - (3, 'layer-2', 1, 2, 1), - (4, 'layer-3a', 1, 3, 1), - (5, 'layer-3b', 1, 3, 2); +INSERT INTO layer (id, name, engineversion, parent_id) VALUES + (1, 'layer-0', 1, NULL), + (2, 'layer-1', 1, 1), + (3, 'layer-2', 1, 2), + (4, 'layer-3a', 1, 3), + (5, 'layer-3b', 1, 3); + +INSERT INTO layernamespace (id, layer_id, namespace_id) VALUES + (1, 2, 1), + (2, 3, 1), + (3, 4, 1), + (4, 5, 2); INSERT INTO layer_diff_featureversion (id, layer_id, featureversion_id, modification) VALUES (1, 2, 1, 'add'), @@ -58,6 +64,7 @@ SELECT pg_catalog.setval(pg_get_serial_sequence('namespace', 'id'), (SELECT MAX( SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('featureversion', 'id'), (SELECT MAX(id) FROM featureversion)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('layer', 'id'), (SELECT MAX(id) FROM layer)+1); +SELECT pg_catalog.setval(pg_get_serial_sequence('layernamespace', 'id'), (SELECT MAX(id) FROM layernamespace)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('layer_diff_featureversion', 'id'), (SELECT MAX(id) FROM layer_diff_featureversion)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability', 'id'), (SELECT MAX(id) FROM vulnerability)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_fixedin_feature', 'id'), (SELECT MAX(id) FROM vulnerability_fixedin_feature)+1); diff --git a/database/pgsql/vulnerability.go b/database/pgsql/vulnerability.go index 74ee9828..5a1eb32b 100644 --- a/database/pgsql/vulnerability.go +++ b/database/pgsql/vulnerability.go @@ -28,12 +28,12 @@ import ( "github.com/guregu/null/zero" ) -func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID int) ([]database.Vulnerability, int, error) { +func (pgSQL *pgSQL) ListVulnerabilities(namespace database.Namespace, limit int, startID int) ([]database.Vulnerability, int, error) { defer observeQueryTime("listVulnerabilities", "all", time.Now()) // Query Namespace. var id int - err := pgSQL.QueryRow(searchNamespace, namespaceName).Scan(&id) + err := pgSQL.QueryRow(searchNamespace, namespace.Name, &namespace.Version).Scan(&id) if err != nil { return nil, -1, handleError("searchNamespace", err) } else if id == 0 { @@ -41,10 +41,10 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID } // Query. - query := searchVulnerabilityBase + searchVulnerabilityByNamespace - rows, err := pgSQL.Query(query, namespaceName, startID, limit+1) + query := searchVulnerabilityBase + searchVulnerabilityByNamespaceID + rows, err := pgSQL.Query(query, id, startID, limit+1) if err != nil { - return nil, -1, handleError("searchVulnerabilityByNamespace", err) + return nil, -1, handleError("searchVulnerabilityByNamespaceID", err) } defer rows.Close() @@ -60,6 +60,7 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID &vulnerability.Name, &vulnerability.Namespace.ID, &vulnerability.Namespace.Name, + &vulnerability.Namespace.Version, &vulnerability.Description, &vulnerability.Link, &vulnerability.Severity, @@ -83,11 +84,11 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID return vulns, nextID, nil } -func (pgSQL *pgSQL) FindVulnerability(namespaceName, name string) (database.Vulnerability, error) { - return findVulnerability(pgSQL, namespaceName, name, false) +func (pgSQL *pgSQL) FindVulnerability(namespace database.Namespace, name string) (database.Vulnerability, error) { + return findVulnerability(pgSQL, namespace, name, false) } -func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bool) (database.Vulnerability, error) { +func findVulnerability(queryer Queryer, namespace database.Namespace, name string, forUpdate bool) (database.Vulnerability, error) { defer observeQueryTime("findVulnerability", "all", time.Now()) queryName := "searchVulnerabilityBase+searchVulnerabilityByNamespaceAndName" @@ -97,7 +98,7 @@ func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bo query = query + searchVulnerabilityForUpdate } - return scanVulnerability(queryer, queryName, queryer.QueryRow(query, namespaceName, name)) + return scanVulnerability(queryer, queryName, queryer.QueryRow(query, namespace.Name, &namespace.Version, name)) } func (pgSQL *pgSQL) findVulnerabilityByIDWithDeleted(id int) (database.Vulnerability, error) { @@ -117,6 +118,7 @@ func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql. &vulnerability.Name, &vulnerability.Namespace.ID, &vulnerability.Namespace.Name, + &vulnerability.Namespace.Version, &vulnerability.Description, &vulnerability.Link, &vulnerability.Severity, @@ -193,7 +195,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on tf := time.Now() // Verify parameters - if vulnerability.Name == "" || vulnerability.Namespace.Name == "" { + if vulnerability.Name == "" || vulnerability.Namespace.IsEmpty() { return cerrors.NewBadRequestError("insertVulnerability needs at least the Name and the Namespace") } if !onlyFixedIn && !vulnerability.Severity.IsValid() { @@ -204,11 +206,12 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on for i := 0; i < len(vulnerability.FixedIn); i++ { fifv := &vulnerability.FixedIn[i] - if fifv.Feature.Namespace.Name == "" { + if fifv.Feature.Namespace.IsEmpty() { // As there is no Namespace on that FixedIn FeatureVersion, set it to the Vulnerability's // Namespace. fifv.Feature.Namespace.Name = vulnerability.Namespace.Name - } else if fifv.Feature.Namespace.Name != vulnerability.Namespace.Name { + fifv.Feature.Namespace.Version = vulnerability.Namespace.Version + } else if !fifv.Feature.Namespace.Equal(vulnerability.Namespace) { msg := "could not insert an invalid vulnerability that contains FixedIn FeatureVersion that are not in the same namespace as the Vulnerability" log.Warning(msg) return cerrors.NewBadRequestError(msg) @@ -226,7 +229,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on } // Find existing vulnerability and its Vulnerability_FixedIn_Features (for update). - existingVulnerability, err := findVulnerability(tx, vulnerability.Namespace.Name, vulnerability.Name, true) + existingVulnerability, err := findVulnerability(tx, vulnerability.Namespace, vulnerability.Name, true) if err != nil && err != cerrors.ErrNotFound { tx.Rollback() return err @@ -264,7 +267,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on } // Mark the old vulnerability as non latest. - _, err = tx.Exec(removeVulnerability, vulnerability.Namespace.Name, vulnerability.Name) + _, err = tx.Exec(removeVulnerability, vulnerability.Namespace.Name, &vulnerability.Namespace.Version, vulnerability.Name) if err != nil { tx.Rollback() return handleError("removeVulnerability", err) @@ -497,13 +500,14 @@ func linkVulnerabilityToFeatureVersions(tx *sql.Tx, fixedInID, vulnerabilityID, return nil } -func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []database.FeatureVersion) error { +func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace database.Namespace, vulnerabilityName string, fixes []database.FeatureVersion) error { defer observeQueryTime("InsertVulnerabilityFixes", "all", time.Now()) v := database.Vulnerability{ Name: vulnerabilityName, Namespace: database.Namespace{ - Name: vulnerabilityNamespace, + Name: vulnerabilityNamespace.Name, + Version: vulnerabilityNamespace.Version, }, FixedIn: fixes, } @@ -511,20 +515,22 @@ func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabili return pgSQL.insertVulnerability(v, true, true) } -func (pgSQL *pgSQL) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error { +func (pgSQL *pgSQL) DeleteVulnerabilityFix(vulnerabilityNamespace database.Namespace, vulnerabilityName, featureName string) error { defer observeQueryTime("DeleteVulnerabilityFix", "all", time.Now()) v := database.Vulnerability{ Name: vulnerabilityName, Namespace: database.Namespace{ - Name: vulnerabilityNamespace, + Name: vulnerabilityNamespace.Name, + Version: vulnerabilityNamespace.Version, }, FixedIn: []database.FeatureVersion{ { Feature: database.Feature{ Name: featureName, Namespace: database.Namespace{ - Name: vulnerabilityNamespace, + Name: vulnerabilityNamespace.Name, + Version: vulnerabilityNamespace.Version, }, }, Version: types.MinVersion, @@ -535,7 +541,7 @@ func (pgSQL *pgSQL) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerability return pgSQL.insertVulnerability(v, true, true) } -func (pgSQL *pgSQL) DeleteVulnerability(namespaceName, name string) error { +func (pgSQL *pgSQL) DeleteVulnerability(namespace database.Namespace, name string) error { defer observeQueryTime("DeleteVulnerability", "all", time.Now()) // Begin transaction. @@ -546,7 +552,7 @@ func (pgSQL *pgSQL) DeleteVulnerability(namespaceName, name string) error { } var vulnerabilityID int - err = tx.QueryRow(removeVulnerability, namespaceName, name).Scan(&vulnerabilityID) + err = tx.QueryRow(removeVulnerability, namespace.Name, &namespace.Version, name).Scan(&vulnerabilityID) if err != nil { tx.Rollback() return handleError("removeVulnerability", err) diff --git a/database/pgsql/vulnerability_test.go b/database/pgsql/vulnerability_test.go index d20c2e35..ae92ea7f 100644 --- a/database/pgsql/vulnerability_test.go +++ b/database/pgsql/vulnerability_test.go @@ -33,8 +33,12 @@ func TestFindVulnerability(t *testing.T) { } defer datastore.Close() + testExistNamespace := database.Namespace{ + Name: "debian", + Version: types.NewVersionUnsafe("7"), + } // Find a vulnerability that does not exist. - _, err = datastore.FindVulnerability("", "") + _, err = datastore.FindVulnerability(database.Namespace{}, "") assert.Equal(t, cerrors.ErrNotFound, err) // Find a normal vulnerability. @@ -43,7 +47,7 @@ func TestFindVulnerability(t *testing.T) { Description: "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", Link: "http://google.com/#q=CVE-OPENSSL-1-DEB7", Severity: types.High, - Namespace: database.Namespace{Name: "debian:7"}, + Namespace: testExistNamespace, FixedIn: []database.FeatureVersion{ { Feature: database.Feature{Name: "openssl"}, @@ -56,7 +60,7 @@ func TestFindVulnerability(t *testing.T) { }, } - v1f, err := datastore.FindVulnerability("debian:7", "CVE-OPENSSL-1-DEB7") + v1f, err := datastore.FindVulnerability(testExistNamespace, "CVE-OPENSSL-1-DEB7") if assert.Nil(t, err) { equalsVuln(t, &v1, &v1f) } @@ -65,11 +69,11 @@ func TestFindVulnerability(t *testing.T) { v2 := database.Vulnerability{ Name: "CVE-NOPE", Description: "A vulnerability affecting nothing", - Namespace: database.Namespace{Name: "debian:7"}, + Namespace: testExistNamespace, Severity: types.Unknown, } - v2f, err := datastore.FindVulnerability("debian:7", "CVE-NOPE") + v2f, err := datastore.FindVulnerability(testExistNamespace, "CVE-NOPE") if assert.Nil(t, err) { equalsVuln(t, &v2, &v2f) } @@ -83,16 +87,24 @@ func TestDeleteVulnerability(t *testing.T) { } defer datastore.Close() + testExistNamespace := database.Namespace{ + Name: "debian", + Version: types.NewVersionUnsafe("7"), + } + testNonExistNamespace := database.Namespace{ + Name: "TestDeleteVulnerabilityNamespace", + Version: types.NewVersionUnsafe("1.0"), + } // Delete non-existing Vulnerability. - err = datastore.DeleteVulnerability("TestDeleteVulnerabilityNamespace1", "CVE-OPENSSL-1-DEB7") + err = datastore.DeleteVulnerability(testNonExistNamespace, "CVE-OPENSSL-1-DEB7") assert.Equal(t, cerrors.ErrNotFound, err) - err = datastore.DeleteVulnerability("debian:7", "TestDeleteVulnerabilityVulnerability1") + err = datastore.DeleteVulnerability(testExistNamespace, "TestDeleteVulnerabilityVulnerability1") assert.Equal(t, cerrors.ErrNotFound, err) // Delete Vulnerability. - err = datastore.DeleteVulnerability("debian:7", "CVE-OPENSSL-1-DEB7") + err = datastore.DeleteVulnerability(testExistNamespace, "CVE-OPENSSL-1-DEB7") if assert.Nil(t, err) { - _, err := datastore.FindVulnerability("debian:7", "CVE-OPENSSL-1-DEB7") + _, err := datastore.FindVulnerability(testExistNamespace, "CVE-OPENSSL-1-DEB7") assert.Equal(t, cerrors.ErrNotFound, err) } } @@ -106,8 +118,8 @@ func TestInsertVulnerability(t *testing.T) { defer datastore.Close() // Create some data. - n1 := database.Namespace{Name: "TestInsertVulnerabilityNamespace1"} - n2 := database.Namespace{Name: "TestInsertVulnerabilityNamespace2"} + n1 := database.Namespace{Name: "TestInsertVulnerabilityNamespace", Version: types.NewVersionUnsafe("1.0")} + n2 := database.Namespace{Name: "TestInsertVulnerabilityNamespace", Version: types.NewVersionUnsafe("2.0")} f1 := database.FeatureVersion{ Feature: database.Feature{ @@ -216,7 +228,7 @@ func TestInsertVulnerability(t *testing.T) { } err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}, true) if assert.Nil(t, err) { - v1f, err := datastore.FindVulnerability(n1.Name, v1.Name) + v1f, err := datastore.FindVulnerability(n1, v1.Name) if assert.Nil(t, err) { equalsVuln(t, &v1, &v1f) } @@ -232,7 +244,7 @@ func TestInsertVulnerability(t *testing.T) { err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}, true) if assert.Nil(t, err) { - v1f, err := datastore.FindVulnerability(n1.Name, v1.Name) + v1f, err := datastore.FindVulnerability(n1, v1.Name) if assert.Nil(t, err) { // We already had f1 before the update. // Add it to the struct for comparison. @@ -252,7 +264,7 @@ func TestInsertVulnerability(t *testing.T) { func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) { assert.Equal(t, expected.Name, actual.Name) - assert.Equal(t, expected.Namespace.Name, actual.Namespace.Name) + assert.True(t, expected.Namespace.Equal(actual.Namespace)) assert.Equal(t, expected.Description, actual.Description) assert.Equal(t, expected.Link, actual.Link) assert.Equal(t, expected.Severity, actual.Severity) @@ -265,7 +277,7 @@ func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) { if expectedFeatureVersion.Feature.Name == actualFeatureVersion.Feature.Name { found = true - assert.Equal(t, expected.Namespace.Name, actualFeatureVersion.Feature.Namespace.Name) + assert.True(t, expected.Namespace.Equal(actualFeatureVersion.Feature.Namespace)) assert.Equal(t, expectedFeatureVersion.Version, actualFeatureVersion.Version) } } diff --git a/updater/fetchers/debian/debian.go b/updater/fetchers/debian/debian.go index e21bd3f2..590819e1 100644 --- a/updater/fetchers/debian/debian.go +++ b/updater/fetchers/debian/debian.go @@ -192,7 +192,8 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability, Feature: database.Feature{ Name: pkgName, Namespace: database.Namespace{ - Name: "debian:" + database.DebianReleasesMapping[releaseName], + Name: "debian", + Version: types.NewVersionUnsafe(database.DebianReleasesMapping[releaseName]), }, }, Version: version, diff --git a/updater/fetchers/debian/debian_test.go b/updater/fetchers/debian/debian_test.go index e092909d..4bfd83b0 100644 --- a/updater/fetchers/debian/debian_test.go +++ b/updater/fetchers/debian/debian_test.go @@ -41,14 +41,14 @@ func TestDebianParser(t *testing.T) { expectedFeatureVersions := []database.FeatureVersion{ { Feature: database.Feature{ - Namespace: database.Namespace{Name: "debian:8"}, + Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")}, Name: "aptdaemon", }, Version: types.MaxVersion, }, { Feature: database.Feature{ - Namespace: database.Namespace{Name: "debian:unstable"}, + Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("unstable")}, Name: "aptdaemon", }, @@ -67,21 +67,21 @@ func TestDebianParser(t *testing.T) { expectedFeatureVersions := []database.FeatureVersion{ { Feature: database.Feature{ - Namespace: database.Namespace{Name: "debian:8"}, + Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")}, Name: "aptdaemon", }, Version: types.NewVersionUnsafe("0.7.0"), }, { Feature: database.Feature{ - Namespace: database.Namespace{Name: "debian:unstable"}, + Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("unstable")}, Name: "aptdaemon", }, Version: types.NewVersionUnsafe("0.7.0"), }, { Feature: database.Feature{ - Namespace: database.Namespace{Name: "debian:8"}, + Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")}, Name: "asterisk", }, Version: types.NewVersionUnsafe("0.5.56"), @@ -99,7 +99,7 @@ func TestDebianParser(t *testing.T) { expectedFeatureVersions := []database.FeatureVersion{ { Feature: database.Feature{ - Namespace: database.Namespace{Name: "debian:8"}, + Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")}, Name: "asterisk", }, Version: types.MinVersion, diff --git a/updater/fetchers/rhel/rhel.go b/updater/fetchers/rhel/rhel.go index de4072d1..099a5c42 100644 --- a/updater/fetchers/rhel/rhel.go +++ b/updater/fetchers/rhel/rhel.go @@ -291,13 +291,14 @@ func toFeatureVersions(criteria criteria) []database.FeatureVersion { } if osVersion > firstConsideredRHEL { - featureVersion.Feature.Namespace.Name = "centos" + ":" + strconv.Itoa(osVersion) + featureVersion.Feature.Namespace.Name = "centos" + featureVersion.Feature.Namespace.Version = types.NewVersionUnsafe(strconv.Itoa(osVersion)) } else { continue } - if featureVersion.Feature.Namespace.Name != "" && featureVersion.Feature.Name != "" && featureVersion.Version.String() != "" { - featureVersionParameters[featureVersion.Feature.Namespace.Name+":"+featureVersion.Feature.Name] = featureVersion + if !featureVersion.Feature.Namespace.IsEmpty() && featureVersion.Feature.Name != "" && featureVersion.Version.String() != "" { + featureVersionParameters[featureVersion.Feature.Namespace.Name+":"+featureVersion.Feature.Namespace.Version.String()+":"+featureVersion.Feature.Name] = featureVersion } else { log.Warningf("could not determine a valid package from criterions: %v", criterions) } diff --git a/updater/fetchers/rhel/rhel_test.go b/updater/fetchers/rhel/rhel_test.go index 0e9ddbd0..2b1d7b96 100644 --- a/updater/fetchers/rhel/rhel_test.go +++ b/updater/fetchers/rhel/rhel_test.go @@ -41,21 +41,21 @@ func TestRHELParser(t *testing.T) { expectedFeatureVersions := []database.FeatureVersion{ { Feature: database.Feature{ - Namespace: database.Namespace{Name: "centos:7"}, + Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")}, Name: "xerces-c", }, Version: types.NewVersionUnsafe("3.1.1-7.el7_1"), }, { Feature: database.Feature{ - Namespace: database.Namespace{Name: "centos:7"}, + Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")}, Name: "xerces-c-devel", }, Version: types.NewVersionUnsafe("3.1.1-7.el7_1"), }, { Feature: database.Feature{ - Namespace: database.Namespace{Name: "centos:7"}, + Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")}, Name: "xerces-c-doc", }, Version: types.NewVersionUnsafe("3.1.1-7.el7_1"), @@ -79,14 +79,14 @@ func TestRHELParser(t *testing.T) { expectedFeatureVersions := []database.FeatureVersion{ { Feature: database.Feature{ - Namespace: database.Namespace{Name: "centos:6"}, + Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("6")}, Name: "firefox", }, Version: types.NewVersionUnsafe("38.1.0-1.el6_6"), }, { Feature: database.Feature{ - Namespace: database.Namespace{Name: "centos:7"}, + Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")}, Name: "firefox", }, Version: types.NewVersionUnsafe("38.1.0-1.el7_1"), diff --git a/updater/fetchers/ubuntu/ubuntu.go b/updater/fetchers/ubuntu/ubuntu.go index 2bcc2b09..fd3dc6f1 100644 --- a/updater/fetchers/ubuntu/ubuntu.go +++ b/updater/fetchers/ubuntu/ubuntu.go @@ -376,7 +376,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability // Create and add the new package. featureVersion := database.FeatureVersion{ Feature: database.Feature{ - Namespace: database.Namespace{Name: "ubuntu:" + database.UbuntuReleasesMapping[md["release"]]}, + Namespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe(database.UbuntuReleasesMapping[md["release"]])}, Name: md["package"], }, Version: version, diff --git a/updater/fetchers/ubuntu/ubuntu_test.go b/updater/fetchers/ubuntu/ubuntu_test.go index d76d457e..c7b343ad 100644 --- a/updater/fetchers/ubuntu/ubuntu_test.go +++ b/updater/fetchers/ubuntu/ubuntu_test.go @@ -45,21 +45,21 @@ func TestUbuntuParser(t *testing.T) { expectedFeatureVersions := []database.FeatureVersion{ { Feature: database.Feature{ - Namespace: database.Namespace{Name: "ubuntu:14.04"}, + Namespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("14.04")}, Name: "libmspack", }, Version: types.MaxVersion, }, { Feature: database.Feature{ - Namespace: database.Namespace{Name: "ubuntu:15.04"}, + Namespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("15.04")}, Name: "libmspack", }, Version: types.NewVersionUnsafe("0.4-3"), }, { Feature: database.Feature{ - Namespace: database.Namespace{Name: "ubuntu:15.10"}, + Namespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("15.10")}, Name: "libmspack-anotherpkg", }, Version: types.NewVersionUnsafe("0.1"), diff --git a/worker/detectors/feature/dpkg/dpkg.go b/worker/detectors/feature/dpkg/dpkg.go index 6154d2ce..2f8e38de 100644 --- a/worker/detectors/feature/dpkg/dpkg.go +++ b/worker/detectors/feature/dpkg/dpkg.go @@ -113,3 +113,16 @@ func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database func (detector *DpkgFeaturesDetector) GetRequiredFiles() []string { return []string{"var/lib/dpkg/status"} } + +//Supported checks if the input Namespace is supported by the underling detector +func (detector *DpkgFeaturesDetector) Supported(namespace database.Namespace) bool { + supports := []string{"debian", "ubuntu"} + + for _, support := range supports { + if strings.HasPrefix(namespace.Name, support) { + return true + } + } + + return false +} diff --git a/worker/detectors/feature/rpm/rpm.go b/worker/detectors/feature/rpm/rpm.go index b5012639..7cac500b 100644 --- a/worker/detectors/feature/rpm/rpm.go +++ b/worker/detectors/feature/rpm/rpm.go @@ -118,3 +118,16 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database. func (detector *RpmFeaturesDetector) GetRequiredFiles() []string { return []string{"var/lib/rpm/Packages"} } + +//Supported checks if the input Namespace is supported by the underling detector +func (detector *RpmFeaturesDetector) Supported(namespace database.Namespace) bool { + supports := []string{"centos", "red hat", "fedora"} + + for _, support := range supports { + if strings.HasPrefix(namespace.Name, support) { + return true + } + } + + return false +} diff --git a/worker/detectors/features.go b/worker/detectors/features.go index da8d8c06..164d0de2 100644 --- a/worker/detectors/features.go +++ b/worker/detectors/features.go @@ -28,6 +28,8 @@ type FeaturesDetector interface { // GetRequiredFiles returns the list of files required for Detect, without // leading /. GetRequiredFiles() []string + //Supported checks if the input Namespace is supported by the underling detector + Supported(namespace database.Namespace) bool } var ( @@ -54,15 +56,25 @@ func RegisterFeaturesDetector(name string, f FeaturesDetector) { } // DetectFeatures detects a list of FeatureVersion using every registered FeaturesDetector. -func DetectFeatures(data map[string][]byte) ([]database.FeatureVersion, error) { +func DetectFeatures(data map[string][]byte, namespaces []database.Namespace) ([]database.FeatureVersion, error) { var packages []database.FeatureVersion for _, detector := range featuresDetectors { - pkgs, err := detector.Detect(data) - if err != nil { - return []database.FeatureVersion{}, err + for _, namespace := range namespaces { + if detector.Supported(namespace) { + pkgs, err := detector.Detect(data) + if err != nil { + return []database.FeatureVersion{}, err + } + // Ensure that every feature has a Namespace associated + for i := 0; i < len(pkgs); i++ { + pkgs[i].Feature.Namespace.Name = namespace.Name + pkgs[i].Feature.Namespace.Version = namespace.Version + } + packages = append(packages, pkgs...) + break + } } - packages = append(packages, pkgs...) } return packages, nil diff --git a/worker/detectors/namespace.go b/worker/detectors/namespace.go index 7d00cdfc..39fcc2a7 100644 --- a/worker/detectors/namespace.go +++ b/worker/detectors/namespace.go @@ -60,18 +60,18 @@ func RegisterNamespaceDetector(name string, f NamespaceDetector) { namespaceDetectors[name] = f } -// DetectNamespace finds the OS of the layer by using every registered NamespaceDetector. -func DetectNamespace(data map[string][]byte) *database.Namespace { +// DetectNamespaces finds the namespaces of the layer by using every registered NamespaceDetector. +func DetectNamespaces(data map[string][]byte) (namespaces []database.Namespace) { for _, detector := range namespaceDetectors { if namespace := detector.Detect(data); namespace != nil { - return namespace + namespaces = append(namespaces, *namespace) } } - return nil + return } -// GetRequiredFilesNamespace returns the list of files required for DetectNamespace for every +// GetRequiredFilesNamespace returns the list of files required for DetectNamespaces for every // registered NamespaceDetector, without leading /. func GetRequiredFilesNamespace() (files []string) { for _, detector := range namespaceDetectors { diff --git a/worker/detectors/namespace/aptsources/aptsources.go b/worker/detectors/namespace/aptsources/aptsources.go index 69b6e30f..2d9ebbb1 100644 --- a/worker/detectors/namespace/aptsources/aptsources.go +++ b/worker/detectors/namespace/aptsources/aptsources.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" "github.com/coreos/clair/worker/detectors" ) @@ -75,7 +76,7 @@ func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *dat } if OS != "" && version != "" { - return &database.Namespace{Name: OS + ":" + version} + return &database.Namespace{Name: OS, Version: types.NewVersionUnsafe(version)} } return nil } diff --git a/worker/detectors/namespace/aptsources/aptsources_test.go b/worker/detectors/namespace/aptsources/aptsources_test.go index d502dc48..8d83aa3f 100644 --- a/worker/detectors/namespace/aptsources/aptsources_test.go +++ b/worker/detectors/namespace/aptsources/aptsources_test.go @@ -18,12 +18,13 @@ import ( "testing" "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" "github.com/coreos/clair/worker/detectors/namespace" ) var aptSourcesOSTests = []namespace.NamespaceTest{ { - ExpectedNamespace: database.Namespace{Name: "debian:unstable"}, + ExpectedNamespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("unstable")}, Data: map[string][]byte{ "etc/os-release": []byte( `PRETTY_NAME="Debian GNU/Linux stretch/sid" diff --git a/worker/detectors/namespace/lsbrelease/lsbrelease.go b/worker/detectors/namespace/lsbrelease/lsbrelease.go index eab19984..b4ac9602 100644 --- a/worker/detectors/namespace/lsbrelease/lsbrelease.go +++ b/worker/detectors/namespace/lsbrelease/lsbrelease.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" "github.com/coreos/clair/worker/detectors" ) @@ -70,7 +71,7 @@ func (detector *LsbReleaseNamespaceDetector) Detect(data map[string][]byte) *dat } if OS != "" && version != "" { - return &database.Namespace{Name: OS + ":" + version} + return &database.Namespace{Name: OS, Version: types.NewVersionUnsafe(version)} } return nil } diff --git a/worker/detectors/namespace/lsbrelease/lsbrelease_test.go b/worker/detectors/namespace/lsbrelease/lsbrelease_test.go index 9aa3b64f..29471f68 100644 --- a/worker/detectors/namespace/lsbrelease/lsbrelease_test.go +++ b/worker/detectors/namespace/lsbrelease/lsbrelease_test.go @@ -18,12 +18,13 @@ import ( "testing" "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" "github.com/coreos/clair/worker/detectors/namespace" ) var lsbReleaseOSTests = []namespace.NamespaceTest{ { - ExpectedNamespace: database.Namespace{Name: "ubuntu:12.04"}, + ExpectedNamespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("12.04")}, Data: map[string][]byte{ "etc/lsb-release": []byte( `DISTRIB_ID=Ubuntu @@ -33,7 +34,7 @@ DISTRIB_DESCRIPTION="Ubuntu 12.04 LTS"`), }, }, { // We don't care about the minor version of Debian - ExpectedNamespace: database.Namespace{Name: "debian:7"}, + ExpectedNamespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("7")}, Data: map[string][]byte{ "etc/lsb-release": []byte( `DISTRIB_ID=Debian diff --git a/worker/detectors/namespace/osrelease/osrelease.go b/worker/detectors/namespace/osrelease/osrelease.go index 118fb9fd..03c9e2d9 100644 --- a/worker/detectors/namespace/osrelease/osrelease.go +++ b/worker/detectors/namespace/osrelease/osrelease.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" "github.com/coreos/clair/worker/detectors" ) @@ -65,7 +66,7 @@ func (detector *OsReleaseNamespaceDetector) Detect(data map[string][]byte) *data } if OS != "" && version != "" { - return &database.Namespace{Name: OS + ":" + version} + return &database.Namespace{Name: OS, Version: types.NewVersionUnsafe(version)} } return nil } diff --git a/worker/detectors/namespace/osrelease/osrelease_test.go b/worker/detectors/namespace/osrelease/osrelease_test.go index 4b08cf33..f9370e0c 100644 --- a/worker/detectors/namespace/osrelease/osrelease_test.go +++ b/worker/detectors/namespace/osrelease/osrelease_test.go @@ -18,12 +18,13 @@ import ( "testing" "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" "github.com/coreos/clair/worker/detectors/namespace" ) var osReleaseOSTests = []namespace.NamespaceTest{ { - ExpectedNamespace: database.Namespace{Name: "debian:8"}, + ExpectedNamespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")}, Data: map[string][]byte{ "etc/os-release": []byte( `PRETTY_NAME="Debian GNU/Linux 8 (jessie)" @@ -37,7 +38,7 @@ BUG_REPORT_URL="https://bugs.debian.org/"`), }, }, { - ExpectedNamespace: database.Namespace{Name: "ubuntu:15.10"}, + ExpectedNamespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("15.10")}, Data: map[string][]byte{ "etc/os-release": []byte( `NAME="Ubuntu" @@ -52,7 +53,7 @@ BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"`), }, }, { // Doesn't have quotes around VERSION_ID - ExpectedNamespace: database.Namespace{Name: "fedora:20"}, + ExpectedNamespace: database.Namespace{Name: "fedora", Version: types.NewVersionUnsafe("20")}, Data: map[string][]byte{ "etc/os-release": []byte( `NAME=Fedora diff --git a/worker/detectors/namespace/redhatrelease/redhatrelease.go b/worker/detectors/namespace/redhatrelease/redhatrelease.go index a6569b07..6eb8e9f3 100644 --- a/worker/detectors/namespace/redhatrelease/redhatrelease.go +++ b/worker/detectors/namespace/redhatrelease/redhatrelease.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" "github.com/coreos/clair/worker/detectors" ) @@ -46,7 +47,7 @@ func (detector *RedhatReleaseNamespaceDetector) Detect(data map[string][]byte) * r := redhatReleaseRegexp.FindStringSubmatch(string(f)) if len(r) == 4 { - return &database.Namespace{Name: strings.ToLower(r[1]) + ":" + r[3]} + return &database.Namespace{Name: strings.ToLower(r[1]), Version: types.NewVersionUnsafe(r[3])} } } diff --git a/worker/detectors/namespace/redhatrelease/redhatrelease_test.go b/worker/detectors/namespace/redhatrelease/redhatrelease_test.go index 25c786ac..da0d3eab 100644 --- a/worker/detectors/namespace/redhatrelease/redhatrelease_test.go +++ b/worker/detectors/namespace/redhatrelease/redhatrelease_test.go @@ -18,18 +18,19 @@ import ( "testing" "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" "github.com/coreos/clair/worker/detectors/namespace" ) var redhatReleaseTests = []namespace.NamespaceTest{ { - ExpectedNamespace: database.Namespace{Name: "centos:6"}, + ExpectedNamespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("6")}, Data: map[string][]byte{ "etc/centos-release": []byte(`CentOS release 6.6 (Final)`), }, }, { - ExpectedNamespace: database.Namespace{Name: "centos:7"}, + ExpectedNamespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")}, Data: map[string][]byte{ "etc/system-release": []byte(`CentOS Linux release 7.1.1503 (Core)`), }, diff --git a/worker/detectors/namespace/test.go b/worker/detectors/namespace/test.go index 679d23db..ee88d93e 100644 --- a/worker/detectors/namespace/test.go +++ b/worker/detectors/namespace/test.go @@ -29,6 +29,6 @@ type NamespaceTest struct { func TestNamespaceDetector(t *testing.T, detector detectors.NamespaceDetector, tests []NamespaceTest) { for _, test := range tests { - assert.Equal(t, test.ExpectedNamespace, *detector.Detect(test.Data)) + assert.True(t, test.ExpectedNamespace.Equal(*detector.Detect(test.Data))) } } diff --git a/worker/worker.go b/worker/worker.go index 23ee2d39..616c9a18 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -46,7 +46,7 @@ var ( ErrParentUnknown = cerrors.NewBadRequestError("worker: parent layer is unknown, it must be processed first") ) -// Process detects the Namespace of a layer, the features it adds/removes, and +// Process detects the Namespaces of a layer, the features it adds/removes, and // then stores everything in the database. // TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an // older engine version and that processes them. @@ -104,7 +104,7 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s } // Analyze the content. - layer.Namespace, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent) + layer.Namespaces, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent) if err != nil { return err } @@ -112,99 +112,65 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s return datastore.InsertLayer(layer) } -// detectContent downloads a layer's archive and extracts its Namespace and Features. -func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) { - data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize) +// detectContent downloads a layer's archive and extracts its Namespaces and Features. +func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespaces []database.Namespace, features []database.FeatureVersion, err error) { + data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), + detectors.GetRequiredFilesNamespace()...), maxFileSize) if err != nil { log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err) return } // Detect namespace. - namespace = detectNamespace(name, data, parent) + namespaces, err = detectNamespaces(name, data, parent) + if err != nil { + return + } + if len(namespaces) > 0 { + log.Debugf("layer %s: %d Namespaces detected: ", name, len(namespaces)) + for _, namespace := range namespaces { + log.Debugf("\t%s", namespace.Name) + } + } else { + log.Debugf("layer %s: Package System is unknown.", name) + } // Detect features. - featureVersions, err = detectFeatureVersions(name, data, namespace, parent) + features, err = detectors.DetectFeatures(data, namespaces) if err != nil { return } - if len(featureVersions) > 0 { - log.Debugf("layer %s: detected %d features", name, len(featureVersions)) + if len(features) > 0 { + log.Debugf("layer %s: detected %d features", name, len(features)) } return } -func detectNamespace(name string, data map[string][]byte, parent *database.Layer) (namespace *database.Namespace) { - // Use registered detectors to get the Namespace. - namespace = detectors.DetectNamespace(data) - if namespace != nil { - log.Debugf("layer %s: detected namespace %q", name, namespace.Name) - return - } +func detectNamespaces(name string, data map[string][]byte, parent *database.Layer) (namespaces []database.Namespace, err error) { + namespaces = detectors.DetectNamespaces(data) - // Use the parent's Namespace. + // Inherit the non-detected namespace from its parent if parent != nil { - namespace = parent.Namespace - if namespace != nil { - log.Debugf("layer %s: detected namespace %q (from parent)", name, namespace.Name) - return + mapNamespaces := make(map[string]database.Namespace) + + // Layer's namespaces has high priority than its parent. + for _, n := range namespaces { + mapNamespaces[n.Name] = n + } + + for _, pn := range parent.Namespaces { + if _, ok := mapNamespaces[pn.Name]; !ok { + mapNamespaces[pn.Name] = pn + log.Debugf("layer %s: detected namespace %q (from parent)", name, pn.Name) + } + } + + namespaces = []database.Namespace{} + for _, namespace := range mapNamespaces { + namespaces = append(namespaces, namespace) } } return } - -func detectFeatureVersions(name string, data map[string][]byte, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) { - // TODO(Quentin-M): We need to pass the parent image to DetectFeatures because it's possible that - // some detectors would need it in order to produce the entire feature list (if they can only - // detect a diff). Also, we should probably pass the detected namespace so detectors could - // make their own decision. - features, err = detectors.DetectFeatures(data) - if err != nil { - return - } - - // If there are no FeatureVersions, use parent's FeatureVersions if possible. - // TODO(Quentin-M): We eventually want to give the choice to each detectors to use none/some of - // their parent's FeatureVersions. It would be useful for detectors that can't find their entire - // result using one Layer. - if len(features) == 0 && parent != nil { - features = parent.Features - return - } - - // Build a map of the namespaces for each FeatureVersion in our parent layer. - parentFeatureNamespaces := make(map[string]database.Namespace) - if parent != nil { - for _, parentFeature := range parent.Features { - parentFeatureNamespaces[parentFeature.Feature.Name+":"+parentFeature.Version.String()] = parentFeature.Feature.Namespace - } - } - - // Ensure that each FeatureVersion has an associated Namespace. - for i, feature := range features { - if feature.Feature.Namespace.Name != "" { - // There is a Namespace associated. - continue - } - - if parentFeatureNamespace, ok := parentFeatureNamespaces[feature.Feature.Name+":"+feature.Version.String()]; ok { - // The FeatureVersion is present in the parent layer; associate with their Namespace. - features[i].Feature.Namespace = parentFeatureNamespace - continue - } - - if namespace != nil { - // The Namespace has been detected in this layer; associate it. - features[i].Feature.Namespace = *namespace - continue - } - - log.Warningf("layer %s: Layer's namespace is unknown but non-namespaced features have been detected", name) - err = ErrUnsupported - return - } - - return -} diff --git a/worker/worker_test.go b/worker/worker_test.go index 31829e44..459757f3 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -61,7 +61,7 @@ func TestProcessWithDistUpgrade(t *testing.T) { } // Create the list of FeatureVersions that should not been upgraded from one layer to another. - nonUpgradedFeatureVersions := []database.FeatureVersion{ + upgradedFeatureVersions := []database.FeatureVersion{ {Feature: database.Feature{Name: "libtext-wrapi18n-perl"}, Version: types.NewVersionUnsafe("0.06-7")}, {Feature: database.Feature{Name: "libtext-charwidth-perl"}, Version: types.NewVersionUnsafe("0.04-7")}, {Feature: database.Feature{Name: "libtext-iconv-perl"}, Version: types.NewVersionUnsafe("1.7-5")}, @@ -82,31 +82,40 @@ func TestProcessWithDistUpgrade(t *testing.T) { assert.Nil(t, Process(datastore, "Docker", "wheezy", "blank", testDataPath+"wheezy.tar.gz", nil)) assert.Nil(t, Process(datastore, "Docker", "jessie", "wheezy", testDataPath+"jessie.tar.gz", nil)) + testDebian7 := database.Namespace{ + Name: "debian", + Version: types.NewVersionUnsafe("7"), + } // Ensure that the 'wheezy' layer has the expected namespace and features. wheezy, ok := datastore.layers["wheezy"] if assert.True(t, ok, "layer 'wheezy' not processed") { - assert.Equal(t, "debian:7", wheezy.Namespace.Name) + assert.True(t, testDebian7.Equal(wheezy.Namespaces[0])) assert.Len(t, wheezy.Features, 52) - - for _, nufv := range nonUpgradedFeatureVersions { - nufv.Feature.Namespace.Name = "debian:7" + for _, nufv := range upgradedFeatureVersions { + nufv.Feature.Namespace = testDebian7 assert.Contains(t, wheezy.Features, nufv) } } + testDebian8 := database.Namespace{ + Name: "debian", + Version: types.NewVersionUnsafe("8"), + } // Ensure that the 'wheezy' layer has the expected namespace and non-upgraded features. jessie, ok := datastore.layers["jessie"] + if assert.True(t, ok, "layer 'jessie' not processed") { - assert.Equal(t, "debian:8", jessie.Namespace.Name) + assert.True(t, testDebian8.Equal(jessie.Namespaces[0])) assert.Len(t, jessie.Features, 74) - for _, nufv := range nonUpgradedFeatureVersions { - nufv.Feature.Namespace.Name = "debian:7" - assert.Contains(t, jessie.Features, nufv) - } - for _, nufv := range nonUpgradedFeatureVersions { - nufv.Feature.Namespace.Name = "debian:8" + for _, nufv := range upgradedFeatureVersions { + nufv.Feature.Namespace = testDebian7 assert.NotContains(t, jessie.Features, nufv) } + + for _, nufv := range upgradedFeatureVersions { + nufv.Feature.Namespace = testDebian8 + assert.Contains(t, jessie.Features, nufv) + } } }