support multiple namespaces in one layer; add database migration

Signed-off-by: liang chenye <liangchenye@huawei.com>
This commit is contained in:
liang chenye 2016-05-03 15:25:31 +08:00
parent 951efed1ff
commit 0a997145ed
38 changed files with 586 additions and 304 deletions

View File

@ -93,7 +93,7 @@ type Datastore interface {
// The Limit and page parameters are used to paginate the return list. // The Limit and page parameters are used to paginate the return list.
// The first given page should be 0. The function will then return the next available page. // The first given page should be 0. The function will then return the next available page.
// If there is no more page, -1 has to be returned. // If there is no more page, -1 has to be returned.
ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error) ListVulnerabilities(namespace Namespace, limit int, page int) ([]Vulnerability, int, error)
// InsertVulnerabilities stores the given Vulnerabilities in the database, updating them if // InsertVulnerabilities stores the given Vulnerabilities in the database, updating them if
// necessary. A vulnerability is uniquely identified by its Namespace and its Name. // necessary. A vulnerability is uniquely identified by its Namespace and its Name.
@ -110,22 +110,22 @@ type Datastore interface {
InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error
// FindVulnerability retrieves a Vulnerability from the database, including the FixedIn list. // FindVulnerability retrieves a Vulnerability from the database, including the FixedIn list.
FindVulnerability(namespaceName, name string) (Vulnerability, error) FindVulnerability(namespace Namespace, name string) (Vulnerability, error)
// DeleteVulnerability removes a Vulnerability from the database. // DeleteVulnerability removes a Vulnerability from the database.
// It has to create a Notification that will contain the old Vulnerability. // It has to create a Notification that will contain the old Vulnerability.
DeleteVulnerability(namespaceName, name string) error DeleteVulnerability(namespace Namespace, name string) error
// InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions of existing ones to // InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions of existing ones to
// the specified Vulnerability in the database. // the specified Vulnerability in the database.
// It has has to create a Notification that will contain the old and the updated Vulnerability. // It has has to create a Notification that will contain the old and the updated Vulnerability.
InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error InsertVulnerabilityFixes(vulnerabilityNamespace Namespace, vulnerabilityName string, fixes []FeatureVersion) error
// DeleteVulnerabilityFix removes a FixedIn Feature from the specified Vulnerability in the // DeleteVulnerabilityFix removes a FixedIn Feature from the specified Vulnerability in the
// database. It can be used to store the fact that a Vulnerability no longer affects the given // database. It can be used to store the fact that a Vulnerability no longer affects the given
// Feature in any Version. // Feature in any Version.
// It has has to create a Notification that will contain the old and the updated Vulnerability. // It has has to create a Notification that will contain the old and the updated Vulnerability.
DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error DeleteVulnerabilityFix(vulnerabilityNamespace Namespace, vulnerabilityName, featureName string) error
// # Notification // # Notification
// GetAvailableNotification returns the Name, Created, Notified and Deleted fields of a // GetAvailableNotification returns the Name, Created, Notified and Deleted fields of a

View File

@ -23,12 +23,12 @@ type MockDatastore struct {
FctInsertLayer func(Layer) error FctInsertLayer func(Layer) error
FctFindLayer func(name string, withFeatures, withVulnerabilities bool) (Layer, error) FctFindLayer func(name string, withFeatures, withVulnerabilities bool) (Layer, error)
FctDeleteLayer func(name string) error FctDeleteLayer func(name string) error
FctListVulnerabilities func(namespaceName string, limit int, page int) ([]Vulnerability, int, error) FctListVulnerabilities func(namespace Namespace, limit int, page int) ([]Vulnerability, int, error)
FctInsertVulnerabilities func(vulnerabilities []Vulnerability, createNotification bool) error FctInsertVulnerabilities func(vulnerabilities []Vulnerability, createNotification bool) error
FctFindVulnerability func(namespaceName, name string) (Vulnerability, error) FctFindVulnerability func(namespace Namespace, name string) (Vulnerability, error)
FctDeleteVulnerability func(namespaceName, name string) error FctDeleteVulnerability func(namespace Namespace, name string) error
FctInsertVulnerabilityFixes func(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error FctInsertVulnerabilityFixes func(vulnerabilityNamespace Namespace, vulnerabilityName string, fixes []FeatureVersion) error
FctDeleteVulnerabilityFix func(vulnerabilityNamespace, vulnerabilityName, featureName string) error FctDeleteVulnerabilityFix func(vulnerabilityNamespace Namespace, vulnerabilityName, featureName string) error
FctGetAvailableNotification func(renotifyInterval time.Duration) (VulnerabilityNotification, error) FctGetAvailableNotification func(renotifyInterval time.Duration) (VulnerabilityNotification, error)
FctGetNotification func(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error) FctGetNotification func(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error)
FctSetNotificationNotified func(name string) error FctSetNotificationNotified func(name string) error
@ -70,9 +70,9 @@ func (mds *MockDatastore) DeleteLayer(name string) error {
panic("required mock function not implemented") panic("required mock function not implemented")
} }
func (mds *MockDatastore) ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error) { func (mds *MockDatastore) ListVulnerabilities(namespace Namespace, limit int, page int) ([]Vulnerability, int, error) {
if mds.FctListVulnerabilities != nil { if mds.FctListVulnerabilities != nil {
return mds.FctListVulnerabilities(namespaceName, limit, page) return mds.FctListVulnerabilities(namespace, limit, page)
} }
panic("required mock function not implemented") panic("required mock function not implemented")
} }
@ -84,28 +84,28 @@ func (mds *MockDatastore) InsertVulnerabilities(vulnerabilities []Vulnerability,
panic("required mock function not implemented") panic("required mock function not implemented")
} }
func (mds *MockDatastore) FindVulnerability(namespaceName, name string) (Vulnerability, error) { func (mds *MockDatastore) FindVulnerability(namespace Namespace, name string) (Vulnerability, error) {
if mds.FctFindVulnerability != nil { if mds.FctFindVulnerability != nil {
return mds.FctFindVulnerability(namespaceName, name) return mds.FctFindVulnerability(namespace, name)
} }
panic("required mock function not implemented") panic("required mock function not implemented")
} }
func (mds *MockDatastore) DeleteVulnerability(namespaceName, name string) error { func (mds *MockDatastore) DeleteVulnerability(namespace Namespace, name string) error {
if mds.FctDeleteVulnerability != nil { if mds.FctDeleteVulnerability != nil {
return mds.FctDeleteVulnerability(namespaceName, name) return mds.FctDeleteVulnerability(namespace, name)
} }
panic("required mock function not implemented") panic("required mock function not implemented")
} }
func (mds *MockDatastore) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error { func (mds *MockDatastore) InsertVulnerabilityFixes(vulnerabilityNamespace Namespace, vulnerabilityName string, fixes []FeatureVersion) error {
if mds.FctInsertVulnerabilityFixes != nil { if mds.FctInsertVulnerabilityFixes != nil {
return mds.FctInsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName, fixes) return mds.FctInsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName, fixes)
} }
panic("required mock function not implemented") panic("required mock function not implemented")
} }
func (mds *MockDatastore) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error { func (mds *MockDatastore) DeleteVulnerabilityFix(vulnerabilityNamespace Namespace, vulnerabilityName, featureName string) error {
if mds.FctDeleteVulnerabilityFix != nil { if mds.FctDeleteVulnerabilityFix != nil {
return mds.FctDeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName) return mds.FctDeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName)
} }

View File

@ -33,14 +33,31 @@ type Layer struct {
Name string Name string
EngineVersion int EngineVersion int
Parent *Layer Parent *Layer
Namespace *Namespace Namespaces []Namespace
Features []FeatureVersion Features []FeatureVersion
} }
type Namespace struct { type Namespace struct {
Model Model
Name string Name string
Version types.Version
}
func (ns *Namespace) IsEmpty() bool {
if ns.Name == "" && ns.Version.String() == "" {
return true
}
return false
}
func (ns *Namespace) Equal(namespace Namespace) bool {
if ns.Name == namespace.Name && ns.Version.Compare(namespace.Version) == 0 {
return true
}
return false
} }
type Feature struct { type Feature struct {

View File

@ -46,8 +46,11 @@ func TestRaceAffects(t *testing.T) {
// Insert the Feature on which we'll work. // Insert the Feature on which we'll work.
feature := database.Feature{ feature := database.Feature{
Namespace: database.Namespace{Name: "TestRaceAffectsFeatureNamespace1"}, Namespace: database.Namespace{
Name: "TestRaceAffecturesFeature1", Name: "TestRaceAffectsFeatureNamespace",
Version: types.NewVersionUnsafe("1.0"),
},
Name: "TestRaceAffecturesFeature1",
} }
_, err = datastore.insertFeature(feature) _, err = datastore.insertFeature(feature)
if err != nil { if err != nil {

View File

@ -31,7 +31,7 @@ func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) {
// Do cache lookup. // Do cache lookup.
if pgSQL.cache != nil { if pgSQL.cache != nil {
promCacheQueriesTotal.WithLabelValues("feature").Inc() promCacheQueriesTotal.WithLabelValues("feature").Inc()
id, found := pgSQL.cache.Get("feature:" + feature.Namespace.Name + ":" + feature.Name) id, found := pgSQL.cache.Get("feature:" + feature.Namespace.Name + ":" + feature.Namespace.Version.String() + ":" + feature.Name)
if found { if found {
promCacheHitsTotal.WithLabelValues("feature").Inc() promCacheHitsTotal.WithLabelValues("feature").Inc()
return id.(int), nil return id.(int), nil
@ -55,7 +55,7 @@ func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) {
} }
if pgSQL.cache != nil { if pgSQL.cache != nil {
pgSQL.cache.Add("feature:"+feature.Namespace.Name+":"+feature.Name, id) pgSQL.cache.Add("feature:"+feature.Namespace.Name+":"+feature.Namespace.Version.String()+":"+feature.Name, id)
} }
return id, nil return id, nil
@ -67,7 +67,7 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion)
} }
// Do cache lookup. // Do cache lookup.
cacheIndex := "featureversion:" + featureVersion.Feature.Namespace.Name + ":" + featureVersion.Feature.Name + ":" + featureVersion.Version.String() cacheIndex := "featureversion:" + featureVersion.Feature.Namespace.Name + ":" + featureVersion.Feature.Namespace.Version.String() + ":" + featureVersion.Feature.Name + ":" + featureVersion.Version.String()
if pgSQL.cache != nil { if pgSQL.cache != nil {
promCacheQueriesTotal.WithLabelValues("featureversion").Inc() promCacheQueriesTotal.WithLabelValues("featureversion").Inc()
id, found := pgSQL.cache.Get(cacheIndex) id, found := pgSQL.cache.Get(cacheIndex)

View File

@ -45,8 +45,11 @@ func TestInsertFeature(t *testing.T) {
// Insert Feature and ensure we can find it. // Insert Feature and ensure we can find it.
feature := database.Feature{ feature := database.Feature{
Namespace: database.Namespace{Name: "TestInsertFeatureNamespace1"}, Namespace: database.Namespace{
Name: "TestInsertFeature1", Name: "TestInsertFeatureNamespace",
Version: types.NewVersionUnsafe("1.0"),
},
Name: "TestInsertFeature1",
} }
id1, err := datastore.insertFeature(feature) id1, err := datastore.insertFeature(feature)
assert.Nil(t, err) assert.Nil(t, err)
@ -69,15 +72,21 @@ func TestInsertFeature(t *testing.T) {
}, },
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertFeatureNamespace2"}, Namespace: database.Namespace{
Name: "TestInsertFeature2", Name: "TestInsertFeatureNamespace",
Version: types.NewVersionUnsafe("2.0"),
},
Name: "TestInsertFeature2",
}, },
Version: types.NewVersionUnsafe(""), Version: types.NewVersionUnsafe(""),
}, },
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertFeatureNamespace2"}, Namespace: database.Namespace{
Name: "TestInsertFeature2", Name: "TestInsertFeatureNamespace",
Version: types.NewVersionUnsafe("2.0"),
},
Name: "TestInsertFeature2",
}, },
Version: types.NewVersionUnsafe("bad version"), Version: types.NewVersionUnsafe("bad version"),
}, },
@ -90,8 +99,11 @@ func TestInsertFeature(t *testing.T) {
// Insert FeatureVersion and ensure we can find it. // Insert FeatureVersion and ensure we can find it.
featureVersion := database.FeatureVersion{ featureVersion := database.FeatureVersion{
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertFeatureNamespace1"}, Namespace: database.Namespace{
Name: "TestInsertFeature1", Name: "TestInsertFeatureNamespace",
Version: types.NewVersionUnsafe("1.0"),
},
Name: "TestInsertFeature1",
}, },
Version: types.NewVersionUnsafe("2:3.0-imba"), Version: types.NewVersionUnsafe("2:3.0-imba"),
} }

View File

@ -37,11 +37,10 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
var layer database.Layer var layer database.Layer
var parentID zero.Int var parentID zero.Int
var parentName zero.String var parentName zero.String
var namespaceID zero.Int
var namespaceName sql.NullString
t := time.Now() t := time.Now()
err := pgSQL.QueryRow(searchLayer, name).Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentID, &parentName, &namespaceID, &namespaceName) err := pgSQL.QueryRow(searchLayer, name).
Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentID, &parentName)
observeQueryTime("FindLayer", "searchLayer", t) observeQueryTime("FindLayer", "searchLayer", t)
if err != nil { if err != nil {
@ -53,12 +52,50 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
Model: database.Model{ID: int(parentID.Int64)}, Model: database.Model{ID: int(parentID.Int64)},
Name: parentName.String, Name: parentName.String,
} }
}
if !namespaceID.IsZero() { // Find its parent's namespaces
layer.Namespace = &database.Namespace{ t = time.Now()
Model: database.Model{ID: int(namespaceID.Int64)}, rows, err := pgSQL.Query(searchLayerNamespace, parentID)
Name: namespaceName.String, observeQueryTime("FindLayer", "searchParentLayerNamespace", t)
if err != nil {
return layer, handleError("searchLayerNamespace", err)
} }
defer rows.Close()
for rows.Next() {
var pn database.Namespace
err = rows.Scan(&pn.ID, &pn.Name, &pn.Version)
if err != nil {
return layer, handleError("searchLayerNamespace.Scan()", err)
}
layer.Parent.Namespaces = append(layer.Parent.Namespaces, pn)
}
if err = rows.Err(); err != nil {
return layer, handleError("searchLayerNamespace.Rows()", err)
}
}
// Find its namespaces
t = time.Now()
rows, err := pgSQL.Query(searchLayerNamespace, layer.ID)
observeQueryTime("FindLayer", "searchLayerNamespace", t)
if err != nil {
return layer, handleError("searchLayerNamespace", err)
}
defer rows.Close()
for rows.Next() {
var namespace database.Namespace
err = rows.Scan(&namespace.ID, &namespace.Name, &namespace.Version)
if err != nil {
return layer, handleError("searchLayerNamespace.Scan()", err)
}
layer.Namespaces = append(layer.Namespaces, namespace)
}
if err = rows.Err(); err != nil {
return layer, handleError("searchLayerNamespace.Rows()", err)
} }
// Find its features // Find its features
@ -128,8 +165,9 @@ func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion
var featureVersion database.FeatureVersion var featureVersion database.FeatureVersion
err = rows.Scan(&featureVersion.ID, &modification, &featureVersion.Feature.Namespace.ID, err = rows.Scan(&featureVersion.ID, &modification, &featureVersion.Feature.Namespace.ID,
&featureVersion.Feature.Namespace.Name, &featureVersion.Feature.ID, &featureVersion.Feature.Namespace.Name, &featureVersion.Feature.Namespace.Version,
&featureVersion.Feature.Name, &featureVersion.ID, &featureVersion.Version, &featureVersion.Feature.ID, &featureVersion.Feature.Name,
&featureVersion.ID, &featureVersion.Version,
&featureVersion.AddedBy.ID, &featureVersion.AddedBy.Name) &featureVersion.AddedBy.ID, &featureVersion.AddedBy.Name)
if err != nil { if err != nil {
return featureVersions, handleError("searchLayerFeatureVersion.Scan()", err) return featureVersions, handleError("searchLayerFeatureVersion.Scan()", err)
@ -184,7 +222,8 @@ func loadAffectedBy(tx *sql.Tx, featureVersions []database.FeatureVersion) error
var vulnerability database.Vulnerability var vulnerability database.Vulnerability
err := rows.Scan(&featureversionID, &vulnerability.ID, &vulnerability.Name, err := rows.Scan(&featureversionID, &vulnerability.ID, &vulnerability.Name,
&vulnerability.Description, &vulnerability.Link, &vulnerability.Severity, &vulnerability.Description, &vulnerability.Link, &vulnerability.Severity,
&vulnerability.Metadata, &vulnerability.Namespace.Name, &vulnerability.FixedBy) &vulnerability.Metadata, &vulnerability.Namespace.Name, &vulnerability.Namespace.Version,
&vulnerability.FixedBy)
if err != nil { if err != nil {
return handleError("searchFeatureVersionVulnerability.Scan()", err) return handleError("searchFeatureVersionVulnerability.Scan()", err)
} }
@ -234,6 +273,23 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
// We do `defer observeQueryTime` here because we don't want to observe existing layers. // We do `defer observeQueryTime` here because we don't want to observe existing layers.
defer observeQueryTime("InsertLayer", "all", tf) defer observeQueryTime("InsertLayer", "all", tf)
// Insert Namespaces
mapNamespaceIDs := make(map[string]int)
for i, _ := range layer.Namespaces {
id, err := pgSQL.insertNamespace(layer.Namespaces[i])
if err != nil {
return err
}
if layer.Namespaces[i].ID == 0 {
layer.Namespaces[i].ID = id
}
// Layer's namespaces has high priority than its parent.
// Once a layer has a 'same' namespace with its parent,
// it will only keep its namespace.
mapNamespaceIDs[layer.Namespaces[i].Name] = id
}
// Get parent ID. // Get parent ID.
var parentID zero.Int var parentID zero.Int
if layer.Parent != nil { if layer.Parent != nil {
@ -243,20 +299,13 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
} }
parentID = zero.IntFrom(int64(layer.Parent.ID)) parentID = zero.IntFrom(int64(layer.Parent.ID))
}
// Find or insert namespace if provided. for _, pn := range layer.Parent.Namespaces {
var namespaceID zero.Int if _, ok := mapNamespaceIDs[pn.Name]; !ok {
if layer.Namespace != nil { // Layer will inherit its parent's namespace
n, err := pgSQL.insertNamespace(*layer.Namespace) mapNamespaceIDs[pn.Name] = pn.ID
if err != nil { layer.Namespaces = append(layer.Namespaces, pn)
return err }
}
namespaceID = zero.IntFrom(int64(n))
} else if layer.Namespace == nil && layer.Parent != nil {
// Import the Namespace from the parent if it has one and this layer doesn't specify one.
if layer.Parent.Namespace != nil {
namespaceID = zero.IntFrom(int64(layer.Parent.Namespace.ID))
} }
} }
@ -269,7 +318,7 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
if layer.ID == 0 { if layer.ID == 0 {
// Insert a new layer. // Insert a new layer.
err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID, namespaceID). err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID).
Scan(&layer.ID) Scan(&layer.ID)
if err != nil { if err != nil {
tx.Rollback() tx.Rollback()
@ -283,12 +332,19 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
} }
} else { } else {
// Update an existing layer. // Update an existing layer.
_, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion, namespaceID) _, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion)
if err != nil { if err != nil {
tx.Rollback() tx.Rollback()
return handleError("updateLayer", err) return handleError("updateLayer", err)
} }
// Remove all existing LayerNamespace.
_, err = tx.Exec(removeLayerNamespace, layer.ID)
if err != nil {
tx.Rollback()
return handleError("removeLayerNamespace", err)
}
// Remove all existing Layer_diff_FeatureVersion. // Remove all existing Layer_diff_FeatureVersion.
_, err = tx.Exec(removeLayerDiffFeatureVersion, layer.ID) _, err = tx.Exec(removeLayerDiffFeatureVersion, layer.ID)
if err != nil { if err != nil {
@ -297,6 +353,15 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
} }
} }
// Insert Layer_Namespace
for _, id := range mapNamespaceIDs {
_, err = tx.Exec(soiLayerNamespace, layer.ID, id)
if err != nil {
tx.Rollback()
return err
}
}
// Update Layer_diff_FeatureVersion now. // Update Layer_diff_FeatureVersion now.
err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer) err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer)
if err != nil { if err != nil {

View File

@ -33,13 +33,17 @@ func TestFindLayer(t *testing.T) {
} }
defer datastore.Close() defer datastore.Close()
testDebian7 := database.Namespace{
Name: "debian",
Version: types.NewVersionUnsafe("7"),
}
// Layer-0: no parent, no namespace, no feature, no vulnerability // Layer-0: no parent, no namespace, no feature, no vulnerability
layer, err := datastore.FindLayer("layer-0", false, false) layer, err := datastore.FindLayer("layer-0", false, false)
if assert.Nil(t, err) && assert.NotNil(t, layer) { if assert.Nil(t, err) && assert.NotNil(t, layer) {
assert.Equal(t, "layer-0", layer.Name) assert.Equal(t, "layer-0", layer.Name)
assert.Nil(t, layer.Namespace)
assert.Nil(t, layer.Parent) assert.Nil(t, layer.Parent)
assert.Equal(t, 1, layer.EngineVersion) assert.Equal(t, 1, layer.EngineVersion)
assert.Len(t, layer.Namespaces, 0)
assert.Len(t, layer.Features, 0) assert.Len(t, layer.Features, 0)
} }
@ -52,18 +56,19 @@ func TestFindLayer(t *testing.T) {
layer, err = datastore.FindLayer("layer-1", false, false) layer, err = datastore.FindLayer("layer-1", false, false)
if assert.Nil(t, err) && assert.NotNil(t, layer) { if assert.Nil(t, err) && assert.NotNil(t, layer) {
assert.Equal(t, layer.Name, "layer-1") assert.Equal(t, layer.Name, "layer-1")
assert.Equal(t, "debian:7", layer.Namespace.Name)
if assert.NotNil(t, layer.Parent) { if assert.NotNil(t, layer.Parent) {
assert.Equal(t, "layer-0", layer.Parent.Name) assert.Equal(t, "layer-0", layer.Parent.Name)
} }
assert.Equal(t, 1, layer.EngineVersion) assert.Equal(t, 1, layer.EngineVersion)
assert.Len(t, layer.Namespaces, 1)
assert.True(t, testDebian7.Equal(layer.Namespaces[0]))
assert.Len(t, layer.Features, 0) assert.Len(t, layer.Features, 0)
} }
layer, err = datastore.FindLayer("layer-1", true, false) layer, err = datastore.FindLayer("layer-1", true, false)
if assert.Nil(t, err) && assert.NotNil(t, layer) && assert.Len(t, layer.Features, 2) { if assert.Nil(t, err) && assert.NotNil(t, layer) && assert.Len(t, layer.Features, 2) {
for _, featureVersion := range layer.Features { for _, featureVersion := range layer.Features {
assert.Equal(t, "debian:7", featureVersion.Feature.Namespace.Name) assert.True(t, testDebian7.Equal(featureVersion.Feature.Namespace))
switch featureVersion.Feature.Name { switch featureVersion.Feature.Name {
case "wechat": case "wechat":
@ -79,7 +84,7 @@ func TestFindLayer(t *testing.T) {
layer, err = datastore.FindLayer("layer-1", true, true) layer, err = datastore.FindLayer("layer-1", true, true)
if assert.Nil(t, err) && assert.NotNil(t, layer) && assert.Len(t, layer.Features, 2) { if assert.Nil(t, err) && assert.NotNil(t, layer) && assert.Len(t, layer.Features, 2) {
for _, featureVersion := range layer.Features { for _, featureVersion := range layer.Features {
assert.Equal(t, "debian:7", featureVersion.Feature.Namespace.Name) assert.True(t, testDebian7.Equal(featureVersion.Feature.Namespace))
switch featureVersion.Feature.Name { switch featureVersion.Feature.Name {
case "wechat": case "wechat":
@ -88,7 +93,7 @@ func TestFindLayer(t *testing.T) {
assert.Equal(t, types.NewVersionUnsafe("1.0"), featureVersion.Version) assert.Equal(t, types.NewVersionUnsafe("1.0"), featureVersion.Version)
if assert.Len(t, featureVersion.AffectedBy, 1) { if assert.Len(t, featureVersion.AffectedBy, 1) {
assert.Equal(t, "debian:7", featureVersion.AffectedBy[0].Namespace.Name) assert.True(t, testDebian7.Equal(featureVersion.AffectedBy[0].Namespace))
assert.Equal(t, "CVE-OPENSSL-1-DEB7", featureVersion.AffectedBy[0].Name) assert.Equal(t, "CVE-OPENSSL-1-DEB7", featureVersion.AffectedBy[0].Name)
assert.Equal(t, types.High, featureVersion.AffectedBy[0].Severity) assert.Equal(t, types.High, featureVersion.AffectedBy[0].Severity)
assert.Equal(t, "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", featureVersion.AffectedBy[0].Description) assert.Equal(t, "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", featureVersion.AffectedBy[0].Description)
@ -137,44 +142,56 @@ func testInsertLayerInvalid(t *testing.T, datastore database.Datastore) {
} }
func testInsertLayerTree(t *testing.T, datastore database.Datastore) { func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
testInsertLayerNamespace1 := database.Namespace{
Name: "TestInsertLayerNamespace",
Version: types.NewVersionUnsafe("1"),
}
testInsertLayerNamespace2 := database.Namespace{
Name: "TestInsertLayerNamespace",
Version: types.NewVersionUnsafe("2"),
}
testInsertLayerNamespace3 := database.Namespace{
Name: "TestInsertLayerNamespace",
Version: types.NewVersionUnsafe("3"),
}
f1 := database.FeatureVersion{ f1 := database.FeatureVersion{
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"}, Namespace: testInsertLayerNamespace2,
Name: "TestInsertLayerFeature1", Name: "TestInsertLayerFeature1",
}, },
Version: types.NewVersionUnsafe("1.0"), Version: types.NewVersionUnsafe("1.0"),
} }
f2 := database.FeatureVersion{ f2 := database.FeatureVersion{
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"}, Namespace: testInsertLayerNamespace2,
Name: "TestInsertLayerFeature2", Name: "TestInsertLayerFeature2",
}, },
Version: types.NewVersionUnsafe("0.34"), Version: types.NewVersionUnsafe("0.34"),
} }
f3 := database.FeatureVersion{ f3 := database.FeatureVersion{
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"}, Namespace: testInsertLayerNamespace2,
Name: "TestInsertLayerFeature3", Name: "TestInsertLayerFeature3",
}, },
Version: types.NewVersionUnsafe("0.56"), Version: types.NewVersionUnsafe("0.56"),
} }
f4 := database.FeatureVersion{ f4 := database.FeatureVersion{
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"}, Namespace: testInsertLayerNamespace3,
Name: "TestInsertLayerFeature2", Name: "TestInsertLayerFeature2",
}, },
Version: types.NewVersionUnsafe("0.34"), Version: types.NewVersionUnsafe("0.34"),
} }
f5 := database.FeatureVersion{ f5 := database.FeatureVersion{
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"}, Namespace: testInsertLayerNamespace3,
Name: "TestInsertLayerFeature3", Name: "TestInsertLayerFeature3",
}, },
Version: types.NewVersionUnsafe("0.56"), Version: types.NewVersionUnsafe("0.56"),
} }
f6 := database.FeatureVersion{ f6 := database.FeatureVersion{
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"}, Namespace: testInsertLayerNamespace3,
Name: "TestInsertLayerFeature4", Name: "TestInsertLayerFeature4",
}, },
Version: types.NewVersionUnsafe("0.666"), Version: types.NewVersionUnsafe("0.666"),
@ -185,16 +202,16 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
Name: "TestInsertLayer1", Name: "TestInsertLayer1",
}, },
{ {
Name: "TestInsertLayer2", Name: "TestInsertLayer2",
Parent: &database.Layer{Name: "TestInsertLayer1"}, Parent: &database.Layer{Name: "TestInsertLayer1"},
Namespace: &database.Namespace{Name: "TestInsertLayerNamespace1"}, Namespaces: []database.Namespace{testInsertLayerNamespace1},
}, },
// This layer changes the namespace and adds Features. // This layer changes the namespace and adds Features.
{ {
Name: "TestInsertLayer3", Name: "TestInsertLayer3",
Parent: &database.Layer{Name: "TestInsertLayer2"}, Parent: &database.Layer{Name: "TestInsertLayer2"},
Namespace: &database.Namespace{Name: "TestInsertLayerNamespace2"}, Namespaces: []database.Namespace{testInsertLayerNamespace2},
Features: []database.FeatureVersion{f1, f2, f3}, Features: []database.FeatureVersion{f1, f2, f3},
}, },
// This layer covers the case where the last layer doesn't provide any new Feature. // This layer covers the case where the last layer doesn't provide any new Feature.
{ {
@ -206,9 +223,9 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
// It also modifies the Namespace ("upgrade") but keeps some Features not upgraded, their // It also modifies the Namespace ("upgrade") but keeps some Features not upgraded, their
// Namespaces should then remain unchanged. // Namespaces should then remain unchanged.
{ {
Name: "TestInsertLayer4b", Name: "TestInsertLayer4b",
Parent: &database.Layer{Name: "TestInsertLayer3"}, Parent: &database.Layer{Name: "TestInsertLayer3"},
Namespace: &database.Namespace{Name: "TestInsertLayerNamespace3"}, Namespaces: []database.Namespace{testInsertLayerNamespace3},
Features: []database.FeatureVersion{ Features: []database.FeatureVersion{
// Deletes TestInsertLayerFeature1. // Deletes TestInsertLayerFeature1.
// Keep TestInsertLayerFeature2 (old Namespace should be kept): // Keep TestInsertLayerFeature2 (old Namespace should be kept):
@ -238,8 +255,8 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
} }
l4a := retrievedLayers["TestInsertLayer4a"] l4a := retrievedLayers["TestInsertLayer4a"]
if assert.NotNil(t, l4a.Namespace) { if assert.Len(t, l4a.Namespaces, 1) {
assert.Equal(t, "TestInsertLayerNamespace2", l4a.Namespace.Name) assert.True(t, testInsertLayerNamespace2.Equal(l4a.Namespaces[0]))
} }
assert.Len(t, l4a.Features, 3) assert.Len(t, l4a.Features, 3)
for _, featureVersion := range l4a.Features { for _, featureVersion := range l4a.Features {
@ -249,8 +266,8 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
} }
l4b := retrievedLayers["TestInsertLayer4b"] l4b := retrievedLayers["TestInsertLayer4b"]
if assert.NotNil(t, l4b.Namespace) { if assert.Len(t, l4b.Namespaces, 1) {
assert.Equal(t, "TestInsertLayerNamespace3", l4b.Namespace.Name) assert.True(t, testInsertLayerNamespace3.Equal(l4b.Namespaces[0]))
} }
assert.Len(t, l4b.Features, 3) assert.Len(t, l4b.Features, 3)
for _, featureVersion := range l4b.Features { for _, featureVersion := range l4b.Features {
@ -261,9 +278,17 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
} }
func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
testInsertLayerNamespace3 := database.Namespace{
Name: "TestInsertLayerNamespace",
Version: types.NewVersionUnsafe("3"),
}
testInsertLayerNamespaceUpdated1 := database.Namespace{
Name: "TestInsertLayerNamespaceUpdated",
Version: types.NewVersionUnsafe("1"),
}
f7 := database.FeatureVersion{ f7 := database.FeatureVersion{
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"}, Namespace: testInsertLayerNamespace3,
Name: "TestInsertLayerFeature7", Name: "TestInsertLayerFeature7",
}, },
Version: types.NewVersionUnsafe("0.01"), Version: types.NewVersionUnsafe("0.01"),
@ -271,10 +296,10 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
l3, _ := datastore.FindLayer("TestInsertLayer3", true, false) l3, _ := datastore.FindLayer("TestInsertLayer3", true, false)
l3u := database.Layer{ l3u := database.Layer{
Name: l3.Name, Name: l3.Name,
Parent: l3.Parent, Parent: l3.Parent,
Namespace: &database.Namespace{Name: "TestInsertLayerNamespaceUpdated1"}, Namespaces: []database.Namespace{testInsertLayerNamespaceUpdated1},
Features: []database.FeatureVersion{f7}, Features: []database.FeatureVersion{f7},
} }
l4u := database.Layer{ l4u := database.Layer{
@ -290,7 +315,9 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
l3uf, err := datastore.FindLayer(l3u.Name, true, false) l3uf, err := datastore.FindLayer(l3u.Name, true, false)
if assert.Nil(t, err) { if assert.Nil(t, err) {
assert.Equal(t, l3.Namespace.Name, l3uf.Namespace.Name) if assert.Len(t, l3.Namespaces, 1) && assert.Len(t, l3uf.Namespaces, 1) {
assert.True(t, l3.Namespaces[0].Equal(l3uf.Namespaces[0]))
}
assert.Equal(t, l3.EngineVersion, l3uf.EngineVersion) assert.Equal(t, l3.EngineVersion, l3uf.EngineVersion)
assert.Len(t, l3uf.Features, len(l3.Features)) assert.Len(t, l3uf.Features, len(l3.Features))
} }
@ -303,7 +330,9 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
l3uf, err = datastore.FindLayer(l3u.Name, true, false) l3uf, err = datastore.FindLayer(l3u.Name, true, false)
if assert.Nil(t, err) { if assert.Nil(t, err) {
assert.Equal(t, l3u.Namespace.Name, l3uf.Namespace.Name) if assert.Len(t, l3u.Namespaces, 1) && assert.Len(t, l3uf.Namespaces, 2) {
assert.True(t, containNS(l3uf.Namespaces, l3u.Namespaces[0]), "Updated layer should have %#v", l3u.Namespaces[0])
}
assert.Equal(t, l3u.EngineVersion, l3uf.EngineVersion) assert.Equal(t, l3u.EngineVersion, l3uf.EngineVersion)
if assert.Len(t, l3uf.Features, 1) { if assert.Len(t, l3uf.Features, 1) {
assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l3uf.Features[0]) assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l3uf.Features[0])
@ -319,7 +348,9 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
l4uf, err := datastore.FindLayer(l3u.Name, true, false) l4uf, err := datastore.FindLayer(l3u.Name, true, false)
if assert.Nil(t, err) { if assert.Nil(t, err) {
assert.Equal(t, l3u.Namespace.Name, l4uf.Namespace.Name) if assert.Len(t, l3u.Namespaces, 1) && assert.Len(t, l4uf.Namespaces, 2) {
assert.True(t, containNS(l4uf.Namespaces, l3u.Namespaces[0]), "Updated layer should have %#v", l3u.Namespaces[0])
}
assert.Equal(t, l4u.EngineVersion, l4uf.EngineVersion) assert.Equal(t, l4u.EngineVersion, l4uf.EngineVersion)
if assert.Len(t, l4uf.Features, 1) { if assert.Len(t, l4uf.Features, 1) {
assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l4uf.Features[0]) assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l4uf.Features[0])
@ -349,3 +380,12 @@ func cmpFV(a, b database.FeatureVersion) bool {
a.Feature.Namespace.Name == b.Feature.Namespace.Name && a.Feature.Namespace.Name == b.Feature.Namespace.Name &&
a.Version.String() == b.Version.String() a.Version.String() == b.Version.String()
} }
func containNS(namespaces []database.Namespace, namespace database.Namespace) bool {
for _, n := range namespaces {
if n.Equal(namespace) {
return true
}
}
return false
}

View File

@ -172,3 +172,8 @@ DROP TABLE IF EXISTS Namespace,
KeyValue, KeyValue,
Lock Lock
CASCADE; CASCADE;
DROP TYPE IF EXISTS modification,
severity
CASCADE;

View File

@ -0,0 +1,66 @@
-- Copyright 2015 clair authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-- +goose Up
-- -----------------------------------------------------
-- Namespace table and data
-- -----------------------------------------------------
ALTER TABLE Namespace ADD version VARCHAR(128) NULL;
UPDATE Namespace SET version = split_part(Namespace.Name, ':', 2), name = split_part(Namespace.Name,':', 1);
-- -----------------------------------------------------
-- LayerNamespace table and data
-- -----------------------------------------------------
CREATE TABLE IF NOT EXISTS LayerNamespace (
id SERIAL PRIMARY KEY,
layer_id INT NOT NULL REFERENCES Layer ON DELETE CASCADE,
namespace_id INT NOT NULL REFERENCES Namespace ON DELETE CASCADE,
UNIQUE (layer_id, namespace_id));
CREATE INDEX ON LayerNamespace (layer_id);
CREATE INDEX ON LayerNamespace (layer_id, namespace_id);
INSERT INTO LayerNamespace(layer_id, namespace_id)
SELECT id, namespace_id
from Layer;
-- -----------------------------------------------------
-- Layer table
-- -----------------------------------------------------
ALTER TABLE Layer DROP COLUMN namespace_id;
-- +goose Down
-- -----------------------------------------------------
-- Layer table and data
-- -----------------------------------------------------
ALTER TABLE Layer ADD namespace_id INT NULL REFERENCES Namespace;
CREATE INDEX ON Layer (namespace_id);
UPDATE Layer l SET namespace_id =
(SELECT namespace_id from LayerNamespace ln
WHERE l.id = ln.layer_id LIMIT 1);
-- -----------------------------------------------------
-- LayerNamespace table (and data)
-- -----------------------------------------------------
DROP TABLE IF EXISTS LayerNamespace
CASCADE;
-- -----------------------------------------------------
-- LayerNamespace data and table
-- -----------------------------------------------------
UPDATE Namespace n SET name = concat(n.name, ':', n.version);
ALTER TABLE Namespace DROP COLUMN version;

View File

@ -22,13 +22,13 @@ import (
) )
func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) { func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) {
if namespace.Name == "" { if namespace.IsEmpty() {
return 0, cerrors.NewBadRequestError("could not find/insert invalid Namespace") return 0, cerrors.NewBadRequestError("could not find/insert invalid Namespace")
} }
if pgSQL.cache != nil { if pgSQL.cache != nil {
promCacheQueriesTotal.WithLabelValues("namespace").Inc() promCacheQueriesTotal.WithLabelValues("namespace").Inc()
if id, found := pgSQL.cache.Get("namespace:" + namespace.Name); found { if id, found := pgSQL.cache.Get("namespace:" + namespace.Name + ":" + namespace.Version.String()); found {
promCacheHitsTotal.WithLabelValues("namespace").Inc() promCacheHitsTotal.WithLabelValues("namespace").Inc()
return id.(int), nil return id.(int), nil
} }
@ -38,13 +38,13 @@ func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) {
defer observeQueryTime("insertNamespace", "all", time.Now()) defer observeQueryTime("insertNamespace", "all", time.Now())
var id int var id int
err := pgSQL.QueryRow(soiNamespace, namespace.Name).Scan(&id) err := pgSQL.QueryRow(soiNamespace, namespace.Name, &namespace.Version).Scan(&id)
if err != nil { if err != nil {
return 0, handleError("soiNamespace", err) return 0, handleError("soiNamespace", err)
} }
if pgSQL.cache != nil { if pgSQL.cache != nil {
pgSQL.cache.Add("namespace:"+namespace.Name, id) pgSQL.cache.Add("namespace:"+namespace.Name+":"+namespace.Version.String(), id)
} }
return id, nil return id, nil
@ -60,7 +60,7 @@ func (pgSQL *pgSQL) ListNamespaces() (namespaces []database.Namespace, err error
for rows.Next() { for rows.Next() {
var namespace database.Namespace var namespace database.Namespace
err = rows.Scan(&namespace.ID, &namespace.Name) err = rows.Scan(&namespace.ID, &namespace.Name, &namespace.Version)
if err != nil { if err != nil {
return namespaces, handleError("listNamespace.Scan()", err) return namespaces, handleError("listNamespace.Scan()", err)
} }

View File

@ -18,9 +18,9 @@ import (
"fmt" "fmt"
"testing" "testing"
"github.com/stretchr/testify/assert"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/stretchr/testify/assert"
) )
func TestInsertNamespace(t *testing.T) { func TestInsertNamespace(t *testing.T) {
@ -37,9 +37,9 @@ func TestInsertNamespace(t *testing.T) {
assert.Zero(t, id0) assert.Zero(t, id0)
// Insert Namespace and ensure we can find it. // Insert Namespace and ensure we can find it.
id1, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace1"}) id1, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace", Version: types.NewVersionUnsafe("1")})
assert.Nil(t, err) assert.Nil(t, err)
id2, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace1"}) id2, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace", Version: types.NewVersionUnsafe("1")})
assert.Nil(t, err) assert.Nil(t, err)
assert.Equal(t, id1, id2) assert.Equal(t, id1, id2)
} }
@ -55,12 +55,13 @@ func TestListNamespace(t *testing.T) {
namespaces, err := datastore.ListNamespaces() namespaces, err := datastore.ListNamespaces()
assert.Nil(t, err) assert.Nil(t, err)
if assert.Len(t, namespaces, 2) { if assert.Len(t, namespaces, 2) {
testDebian7 := database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("7")}
testDebian8 := database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")}
for _, namespace := range namespaces { for _, namespace := range namespaces {
switch namespace.Name { if namespace.Equal(testDebian7) || namespace.Equal(testDebian8) {
case "debian:7", "debian:8":
continue continue
default: } else {
assert.Error(t, fmt.Errorf("ListNamespaces should not have returned '%s'", namespace.Name)) assert.Error(t, fmt.Errorf("ListNamespaces should not have returned '%s:%s'", namespace.Name, namespace.Version.String()))
} }
} }
} }

View File

@ -39,13 +39,19 @@ func TestNotification(t *testing.T) {
// Create some data. // Create some data.
f1 := database.Feature{ f1 := database.Feature{
Name: "TestNotificationFeature1", Name: "TestNotificationFeature1",
Namespace: database.Namespace{Name: "TestNotificationNamespace1"}, Namespace: database.Namespace{
Name: "TestNotificationNamespace",
Version: types.NewVersionUnsafe("1.0"),
},
} }
f2 := database.Feature{ f2 := database.Feature{
Name: "TestNotificationFeature2", Name: "TestNotificationFeature2",
Namespace: database.Namespace{Name: "TestNotificationNamespace1"}, Namespace: database.Namespace{
Name: "TestNotificationNamespace",
Version: types.NewVersionUnsafe("1.0"),
},
} }
l1 := database.Layer{ l1 := database.Layer{
@ -201,7 +207,7 @@ func TestNotification(t *testing.T) {
} }
// Delete a vulnerability and verify the notification. // Delete a vulnerability and verify the notification.
if assert.Nil(t, datastore.DeleteVulnerability(v1b.Namespace.Name, v1b.Name)) { if assert.Nil(t, datastore.DeleteVulnerability(v1b.Namespace, v1b.Name)) {
notification, err = datastore.GetAvailableNotification(time.Second) notification, err = datastore.GetAvailableNotification(time.Second)
assert.Nil(t, err) assert.Nil(t, err)
assert.NotEmpty(t, notification.Name) assert.NotEmpty(t, notification.Name)

View File

@ -29,17 +29,17 @@ const (
// namespace.go // namespace.go
soiNamespace = ` soiNamespace = `
WITH new_namespace AS ( WITH new_namespace AS (
INSERT INTO Namespace(name) INSERT INTO Namespace(name, version)
SELECT CAST($1 AS VARCHAR) SELECT CAST($1 AS VARCHAR), CAST($2 AS VARCHAR)
WHERE NOT EXISTS (SELECT name FROM Namespace WHERE name = $1) WHERE NOT EXISTS (SELECT name FROM Namespace WHERE name = $1 AND version = $2)
RETURNING id RETURNING id
) )
SELECT id FROM Namespace WHERE name = $1 SELECT id FROM Namespace WHERE name = $1 AND version = $2
UNION UNION
SELECT id FROM new_namespace` SELECT id FROM new_namespace`
searchNamespace = `SELECT id FROM Namespace WHERE name = $1` searchNamespace = `SELECT id FROM Namespace WHERE name = $1 AND version = $2`
listNamespace = `SELECT id, name FROM Namespace` listNamespace = `SELECT id, name, version FROM Namespace`
// feature.go // feature.go
soiFeature = ` soiFeature = `
@ -77,10 +77,9 @@ const (
// layer.go // layer.go
searchLayer = ` searchLayer = `
SELECT l.id, l.name, l.engineversion, p.id, p.name, n.id, n.name SELECT l.id, l.name, l.engineversion, p.id, p.name
FROM Layer l FROM Layer l
LEFT JOIN Layer p ON l.parent_id = p.id LEFT JOIN Layer p ON l.parent_id = p.id
LEFT JOIN Namespace n ON l.namespace_id = n.id
WHERE l.name = $1;` WHERE l.name = $1;`
searchLayerFeatureVersion = ` searchLayerFeatureVersion = `
@ -93,7 +92,7 @@ const (
FROM Layer l, layer_tree lt FROM Layer l, layer_tree lt
WHERE l.id = lt.parent_id WHERE l.id = lt.parent_id
) )
SELECT ldf.featureversion_id, ldf.modification, fn.id, fn.name, f.id, f.name, fv.id, fv.version, ltree.id, ltree.name SELECT ldf.featureversion_id, ldf.modification, fn.id, fn.name, fn.version, f.id, f.name, fv.id, fv.version, ltree.id, ltree.name
FROM Layer_diff_FeatureVersion ldf FROM Layer_diff_FeatureVersion ldf
JOIN ( JOIN (
SELECT row_number() over (ORDER BY depth DESC), id, name FROM layer_tree SELECT row_number() over (ORDER BY depth DESC), id, name FROM layer_tree
@ -103,7 +102,7 @@ const (
searchFeatureVersionVulnerability = ` searchFeatureVersionVulnerability = `
SELECT vafv.featureversion_id, v.id, v.name, v.description, v.link, v.severity, v.metadata, SELECT vafv.featureversion_id, v.id, v.name, v.description, v.link, v.severity, v.metadata,
vn.name, vfif.version vn.name, vn.version, vfif.version
FROM Vulnerability_Affects_FeatureVersion vafv, Vulnerability v, FROM Vulnerability_Affects_FeatureVersion vafv, Vulnerability v,
Namespace vn, Vulnerability_FixedIn_Feature vfif Namespace vn, Vulnerability_FixedIn_Feature vfif
WHERE vafv.featureversion_id = ANY($1::integer[]) WHERE vafv.featureversion_id = ANY($1::integer[])
@ -112,12 +111,17 @@ const (
AND v.namespace_id = vn.id AND v.namespace_id = vn.id
AND v.deleted_at IS NULL` AND v.deleted_at IS NULL`
insertLayer = ` searchLayerNamespace = `
INSERT INTO Layer(name, engineversion, parent_id, namespace_id, created_at) SELECT n.id, n.name, n.version
VALUES($1, $2, $3, $4, CURRENT_TIMESTAMP) FROM LayerNamespace ln, Namespace n
RETURNING id` WHERE ln.layer_id = $1 AND ln.namespace_id = n.id`
updateLayer = `UPDATE LAYER SET engineversion = $2, namespace_id = $3 WHERE id = $1` insertLayer = `
INSERT INTO Layer(name, engineversion, parent_id, created_at)
VALUES($1, $2, $3, CURRENT_TIMESTAMP)
RETURNING id`
updateLayer = `UPDATE LAYER SET engineversion = $2 WHERE id = $1`
removeLayerDiffFeatureVersion = ` removeLayerDiffFeatureVersion = `
DELETE FROM Layer_diff_FeatureVersion DELETE FROM Layer_diff_FeatureVersion
@ -131,6 +135,21 @@ const (
removeLayer = `DELETE FROM Layer WHERE name = $1` removeLayer = `DELETE FROM Layer WHERE name = $1`
removeLayerNamespace = `
DELETE FROM LayerNamespace
WHERE layer_id = $1`
soiLayerNamespace = `
WITH new_layernamespace AS (
INSERT INTO LayerNamespace(layer_id, namespace_id)
SELECT CAST($1 AS INTEGER), CAST($2 AS INTEGER)
WHERE NOT EXISTS (SELECT id FROM LayerNamespace WHERE layer_id = $1 AND namespace_id = $2)
RETURNING id
)
SELECT id FROM LayerNamespace WHERE layer_id = $1 AND namespace_id = $2
UNION
SELECT id FROM new_layernamespace`
// lock.go // lock.go
insertLock = `INSERT INTO Lock(name, owner, until) VALUES($1, $2, $3)` insertLock = `INSERT INTO Lock(name, owner, until) VALUES($1, $2, $3)`
searchLock = `SELECT owner, until FROM Lock WHERE name = $1` searchLock = `SELECT owner, until FROM Lock WHERE name = $1`
@ -140,12 +159,12 @@ const (
// vulnerability.go // vulnerability.go
searchVulnerabilityBase = ` searchVulnerabilityBase = `
SELECT v.id, v.name, n.id, n.name, v.description, v.link, v.severity, v.metadata SELECT v.id, v.name, n.id, n.name, n.version, v.description, v.link, v.severity, v.metadata
FROM Vulnerability v JOIN Namespace n ON v.namespace_id = n.id` FROM Vulnerability v JOIN Namespace n ON v.namespace_id = n.id`
searchVulnerabilityForUpdate = ` FOR UPDATE OF v` searchVulnerabilityForUpdate = ` FOR UPDATE OF v`
searchVulnerabilityByNamespaceAndName = ` WHERE n.name = $1 AND v.name = $2 AND v.deleted_at IS NULL` searchVulnerabilityByNamespaceAndName = ` WHERE n.name = $1 AND n.version = $2 AND v.name = $3 AND v.deleted_at IS NULL`
searchVulnerabilityByID = ` WHERE v.id = $1` searchVulnerabilityByID = ` WHERE v.id = $1`
searchVulnerabilityByNamespace = ` WHERE n.name = $1 AND v.deleted_at IS NULL searchVulnerabilityByNamespaceID = ` WHERE n.id = $1 AND v.deleted_at IS NULL
AND v.id >= $2 AND v.id >= $2
ORDER BY v.id ORDER BY v.id
LIMIT $3` LIMIT $3`
@ -170,8 +189,8 @@ const (
removeVulnerability = ` removeVulnerability = `
UPDATE Vulnerability UPDATE Vulnerability
SET deleted_at = CURRENT_TIMESTAMP SET deleted_at = CURRENT_TIMESTAMP
WHERE namespace_id = (SELECT id FROM Namespace WHERE name = $1) WHERE namespace_id = (SELECT id FROM Namespace WHERE name = $1 AND version = $2)
AND name = $2 AND name = $3
AND deleted_at IS NULL AND deleted_at IS NULL
RETURNING id` RETURNING id`

View File

@ -12,9 +12,9 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
INSERT INTO namespace (id, name) VALUES INSERT INTO namespace (id, name, version) VALUES
(1, 'debian:7'), (1, 'debian', '7'),
(2, 'debian:8'); (2, 'debian', '8');
INSERT INTO feature (id, namespace_id, name) VALUES INSERT INTO feature (id, namespace_id, name) VALUES
(1, 1, 'wechat'), (1, 1, 'wechat'),
@ -28,12 +28,18 @@ INSERT INTO featureversion (id, feature_id, version) VALUES
(3, 2, '2.0'), (3, 2, '2.0'),
(4, 3, '1.0'); (4, 3, '1.0');
INSERT INTO layer (id, name, engineversion, parent_id, namespace_id) VALUES INSERT INTO layer (id, name, engineversion, parent_id) VALUES
(1, 'layer-0', 1, NULL, NULL), (1, 'layer-0', 1, NULL),
(2, 'layer-1', 1, 1, 1), (2, 'layer-1', 1, 1),
(3, 'layer-2', 1, 2, 1), (3, 'layer-2', 1, 2),
(4, 'layer-3a', 1, 3, 1), (4, 'layer-3a', 1, 3),
(5, 'layer-3b', 1, 3, 2); (5, 'layer-3b', 1, 3);
INSERT INTO layernamespace (id, layer_id, namespace_id) VALUES
(1, 2, 1),
(2, 3, 1),
(3, 4, 1),
(4, 5, 2);
INSERT INTO layer_diff_featureversion (id, layer_id, featureversion_id, modification) VALUES INSERT INTO layer_diff_featureversion (id, layer_id, featureversion_id, modification) VALUES
(1, 2, 1, 'add'), (1, 2, 1, 'add'),
@ -58,6 +64,7 @@ SELECT pg_catalog.setval(pg_get_serial_sequence('namespace', 'id'), (SELECT MAX(
SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1);
SELECT pg_catalog.setval(pg_get_serial_sequence('featureversion', 'id'), (SELECT MAX(id) FROM featureversion)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('featureversion', 'id'), (SELECT MAX(id) FROM featureversion)+1);
SELECT pg_catalog.setval(pg_get_serial_sequence('layer', 'id'), (SELECT MAX(id) FROM layer)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('layer', 'id'), (SELECT MAX(id) FROM layer)+1);
SELECT pg_catalog.setval(pg_get_serial_sequence('layernamespace', 'id'), (SELECT MAX(id) FROM layernamespace)+1);
SELECT pg_catalog.setval(pg_get_serial_sequence('layer_diff_featureversion', 'id'), (SELECT MAX(id) FROM layer_diff_featureversion)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('layer_diff_featureversion', 'id'), (SELECT MAX(id) FROM layer_diff_featureversion)+1);
SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability', 'id'), (SELECT MAX(id) FROM vulnerability)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability', 'id'), (SELECT MAX(id) FROM vulnerability)+1);
SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_fixedin_feature', 'id'), (SELECT MAX(id) FROM vulnerability_fixedin_feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_fixedin_feature', 'id'), (SELECT MAX(id) FROM vulnerability_fixedin_feature)+1);

View File

@ -28,12 +28,12 @@ import (
"github.com/guregu/null/zero" "github.com/guregu/null/zero"
) )
func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID int) ([]database.Vulnerability, int, error) { func (pgSQL *pgSQL) ListVulnerabilities(namespace database.Namespace, limit int, startID int) ([]database.Vulnerability, int, error) {
defer observeQueryTime("listVulnerabilities", "all", time.Now()) defer observeQueryTime("listVulnerabilities", "all", time.Now())
// Query Namespace. // Query Namespace.
var id int var id int
err := pgSQL.QueryRow(searchNamespace, namespaceName).Scan(&id) err := pgSQL.QueryRow(searchNamespace, namespace.Name, &namespace.Version).Scan(&id)
if err != nil { if err != nil {
return nil, -1, handleError("searchNamespace", err) return nil, -1, handleError("searchNamespace", err)
} else if id == 0 { } else if id == 0 {
@ -41,10 +41,10 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID
} }
// Query. // Query.
query := searchVulnerabilityBase + searchVulnerabilityByNamespace query := searchVulnerabilityBase + searchVulnerabilityByNamespaceID
rows, err := pgSQL.Query(query, namespaceName, startID, limit+1) rows, err := pgSQL.Query(query, id, startID, limit+1)
if err != nil { if err != nil {
return nil, -1, handleError("searchVulnerabilityByNamespace", err) return nil, -1, handleError("searchVulnerabilityByNamespaceID", err)
} }
defer rows.Close() defer rows.Close()
@ -60,6 +60,7 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID
&vulnerability.Name, &vulnerability.Name,
&vulnerability.Namespace.ID, &vulnerability.Namespace.ID,
&vulnerability.Namespace.Name, &vulnerability.Namespace.Name,
&vulnerability.Namespace.Version,
&vulnerability.Description, &vulnerability.Description,
&vulnerability.Link, &vulnerability.Link,
&vulnerability.Severity, &vulnerability.Severity,
@ -83,11 +84,11 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID
return vulns, nextID, nil return vulns, nextID, nil
} }
func (pgSQL *pgSQL) FindVulnerability(namespaceName, name string) (database.Vulnerability, error) { func (pgSQL *pgSQL) FindVulnerability(namespace database.Namespace, name string) (database.Vulnerability, error) {
return findVulnerability(pgSQL, namespaceName, name, false) return findVulnerability(pgSQL, namespace, name, false)
} }
func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bool) (database.Vulnerability, error) { func findVulnerability(queryer Queryer, namespace database.Namespace, name string, forUpdate bool) (database.Vulnerability, error) {
defer observeQueryTime("findVulnerability", "all", time.Now()) defer observeQueryTime("findVulnerability", "all", time.Now())
queryName := "searchVulnerabilityBase+searchVulnerabilityByNamespaceAndName" queryName := "searchVulnerabilityBase+searchVulnerabilityByNamespaceAndName"
@ -97,7 +98,7 @@ func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bo
query = query + searchVulnerabilityForUpdate query = query + searchVulnerabilityForUpdate
} }
return scanVulnerability(queryer, queryName, queryer.QueryRow(query, namespaceName, name)) return scanVulnerability(queryer, queryName, queryer.QueryRow(query, namespace.Name, &namespace.Version, name))
} }
func (pgSQL *pgSQL) findVulnerabilityByIDWithDeleted(id int) (database.Vulnerability, error) { func (pgSQL *pgSQL) findVulnerabilityByIDWithDeleted(id int) (database.Vulnerability, error) {
@ -117,6 +118,7 @@ func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql.
&vulnerability.Name, &vulnerability.Name,
&vulnerability.Namespace.ID, &vulnerability.Namespace.ID,
&vulnerability.Namespace.Name, &vulnerability.Namespace.Name,
&vulnerability.Namespace.Version,
&vulnerability.Description, &vulnerability.Description,
&vulnerability.Link, &vulnerability.Link,
&vulnerability.Severity, &vulnerability.Severity,
@ -193,7 +195,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on
tf := time.Now() tf := time.Now()
// Verify parameters // Verify parameters
if vulnerability.Name == "" || vulnerability.Namespace.Name == "" { if vulnerability.Name == "" || vulnerability.Namespace.IsEmpty() {
return cerrors.NewBadRequestError("insertVulnerability needs at least the Name and the Namespace") return cerrors.NewBadRequestError("insertVulnerability needs at least the Name and the Namespace")
} }
if !onlyFixedIn && !vulnerability.Severity.IsValid() { if !onlyFixedIn && !vulnerability.Severity.IsValid() {
@ -204,11 +206,12 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on
for i := 0; i < len(vulnerability.FixedIn); i++ { for i := 0; i < len(vulnerability.FixedIn); i++ {
fifv := &vulnerability.FixedIn[i] fifv := &vulnerability.FixedIn[i]
if fifv.Feature.Namespace.Name == "" { if fifv.Feature.Namespace.IsEmpty() {
// As there is no Namespace on that FixedIn FeatureVersion, set it to the Vulnerability's // As there is no Namespace on that FixedIn FeatureVersion, set it to the Vulnerability's
// Namespace. // Namespace.
fifv.Feature.Namespace.Name = vulnerability.Namespace.Name fifv.Feature.Namespace.Name = vulnerability.Namespace.Name
} else if fifv.Feature.Namespace.Name != vulnerability.Namespace.Name { fifv.Feature.Namespace.Version = vulnerability.Namespace.Version
} else if !fifv.Feature.Namespace.Equal(vulnerability.Namespace) {
msg := "could not insert an invalid vulnerability that contains FixedIn FeatureVersion that are not in the same namespace as the Vulnerability" msg := "could not insert an invalid vulnerability that contains FixedIn FeatureVersion that are not in the same namespace as the Vulnerability"
log.Warning(msg) log.Warning(msg)
return cerrors.NewBadRequestError(msg) return cerrors.NewBadRequestError(msg)
@ -226,7 +229,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on
} }
// Find existing vulnerability and its Vulnerability_FixedIn_Features (for update). // Find existing vulnerability and its Vulnerability_FixedIn_Features (for update).
existingVulnerability, err := findVulnerability(tx, vulnerability.Namespace.Name, vulnerability.Name, true) existingVulnerability, err := findVulnerability(tx, vulnerability.Namespace, vulnerability.Name, true)
if err != nil && err != cerrors.ErrNotFound { if err != nil && err != cerrors.ErrNotFound {
tx.Rollback() tx.Rollback()
return err return err
@ -264,7 +267,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on
} }
// Mark the old vulnerability as non latest. // Mark the old vulnerability as non latest.
_, err = tx.Exec(removeVulnerability, vulnerability.Namespace.Name, vulnerability.Name) _, err = tx.Exec(removeVulnerability, vulnerability.Namespace.Name, &vulnerability.Namespace.Version, vulnerability.Name)
if err != nil { if err != nil {
tx.Rollback() tx.Rollback()
return handleError("removeVulnerability", err) return handleError("removeVulnerability", err)
@ -497,13 +500,14 @@ func linkVulnerabilityToFeatureVersions(tx *sql.Tx, fixedInID, vulnerabilityID,
return nil return nil
} }
func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []database.FeatureVersion) error { func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace database.Namespace, vulnerabilityName string, fixes []database.FeatureVersion) error {
defer observeQueryTime("InsertVulnerabilityFixes", "all", time.Now()) defer observeQueryTime("InsertVulnerabilityFixes", "all", time.Now())
v := database.Vulnerability{ v := database.Vulnerability{
Name: vulnerabilityName, Name: vulnerabilityName,
Namespace: database.Namespace{ Namespace: database.Namespace{
Name: vulnerabilityNamespace, Name: vulnerabilityNamespace.Name,
Version: vulnerabilityNamespace.Version,
}, },
FixedIn: fixes, FixedIn: fixes,
} }
@ -511,20 +515,22 @@ func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabili
return pgSQL.insertVulnerability(v, true, true) return pgSQL.insertVulnerability(v, true, true)
} }
func (pgSQL *pgSQL) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error { func (pgSQL *pgSQL) DeleteVulnerabilityFix(vulnerabilityNamespace database.Namespace, vulnerabilityName, featureName string) error {
defer observeQueryTime("DeleteVulnerabilityFix", "all", time.Now()) defer observeQueryTime("DeleteVulnerabilityFix", "all", time.Now())
v := database.Vulnerability{ v := database.Vulnerability{
Name: vulnerabilityName, Name: vulnerabilityName,
Namespace: database.Namespace{ Namespace: database.Namespace{
Name: vulnerabilityNamespace, Name: vulnerabilityNamespace.Name,
Version: vulnerabilityNamespace.Version,
}, },
FixedIn: []database.FeatureVersion{ FixedIn: []database.FeatureVersion{
{ {
Feature: database.Feature{ Feature: database.Feature{
Name: featureName, Name: featureName,
Namespace: database.Namespace{ Namespace: database.Namespace{
Name: vulnerabilityNamespace, Name: vulnerabilityNamespace.Name,
Version: vulnerabilityNamespace.Version,
}, },
}, },
Version: types.MinVersion, Version: types.MinVersion,
@ -535,7 +541,7 @@ func (pgSQL *pgSQL) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerability
return pgSQL.insertVulnerability(v, true, true) return pgSQL.insertVulnerability(v, true, true)
} }
func (pgSQL *pgSQL) DeleteVulnerability(namespaceName, name string) error { func (pgSQL *pgSQL) DeleteVulnerability(namespace database.Namespace, name string) error {
defer observeQueryTime("DeleteVulnerability", "all", time.Now()) defer observeQueryTime("DeleteVulnerability", "all", time.Now())
// Begin transaction. // Begin transaction.
@ -546,7 +552,7 @@ func (pgSQL *pgSQL) DeleteVulnerability(namespaceName, name string) error {
} }
var vulnerabilityID int var vulnerabilityID int
err = tx.QueryRow(removeVulnerability, namespaceName, name).Scan(&vulnerabilityID) err = tx.QueryRow(removeVulnerability, namespace.Name, &namespace.Version, name).Scan(&vulnerabilityID)
if err != nil { if err != nil {
tx.Rollback() tx.Rollback()
return handleError("removeVulnerability", err) return handleError("removeVulnerability", err)

View File

@ -33,8 +33,12 @@ func TestFindVulnerability(t *testing.T) {
} }
defer datastore.Close() defer datastore.Close()
testExistNamespace := database.Namespace{
Name: "debian",
Version: types.NewVersionUnsafe("7"),
}
// Find a vulnerability that does not exist. // Find a vulnerability that does not exist.
_, err = datastore.FindVulnerability("", "") _, err = datastore.FindVulnerability(database.Namespace{}, "")
assert.Equal(t, cerrors.ErrNotFound, err) assert.Equal(t, cerrors.ErrNotFound, err)
// Find a normal vulnerability. // Find a normal vulnerability.
@ -43,7 +47,7 @@ func TestFindVulnerability(t *testing.T) {
Description: "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", Description: "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0",
Link: "http://google.com/#q=CVE-OPENSSL-1-DEB7", Link: "http://google.com/#q=CVE-OPENSSL-1-DEB7",
Severity: types.High, Severity: types.High,
Namespace: database.Namespace{Name: "debian:7"}, Namespace: testExistNamespace,
FixedIn: []database.FeatureVersion{ FixedIn: []database.FeatureVersion{
{ {
Feature: database.Feature{Name: "openssl"}, Feature: database.Feature{Name: "openssl"},
@ -56,7 +60,7 @@ func TestFindVulnerability(t *testing.T) {
}, },
} }
v1f, err := datastore.FindVulnerability("debian:7", "CVE-OPENSSL-1-DEB7") v1f, err := datastore.FindVulnerability(testExistNamespace, "CVE-OPENSSL-1-DEB7")
if assert.Nil(t, err) { if assert.Nil(t, err) {
equalsVuln(t, &v1, &v1f) equalsVuln(t, &v1, &v1f)
} }
@ -65,11 +69,11 @@ func TestFindVulnerability(t *testing.T) {
v2 := database.Vulnerability{ v2 := database.Vulnerability{
Name: "CVE-NOPE", Name: "CVE-NOPE",
Description: "A vulnerability affecting nothing", Description: "A vulnerability affecting nothing",
Namespace: database.Namespace{Name: "debian:7"}, Namespace: testExistNamespace,
Severity: types.Unknown, Severity: types.Unknown,
} }
v2f, err := datastore.FindVulnerability("debian:7", "CVE-NOPE") v2f, err := datastore.FindVulnerability(testExistNamespace, "CVE-NOPE")
if assert.Nil(t, err) { if assert.Nil(t, err) {
equalsVuln(t, &v2, &v2f) equalsVuln(t, &v2, &v2f)
} }
@ -83,16 +87,24 @@ func TestDeleteVulnerability(t *testing.T) {
} }
defer datastore.Close() defer datastore.Close()
testExistNamespace := database.Namespace{
Name: "debian",
Version: types.NewVersionUnsafe("7"),
}
testNonExistNamespace := database.Namespace{
Name: "TestDeleteVulnerabilityNamespace",
Version: types.NewVersionUnsafe("1.0"),
}
// Delete non-existing Vulnerability. // Delete non-existing Vulnerability.
err = datastore.DeleteVulnerability("TestDeleteVulnerabilityNamespace1", "CVE-OPENSSL-1-DEB7") err = datastore.DeleteVulnerability(testNonExistNamespace, "CVE-OPENSSL-1-DEB7")
assert.Equal(t, cerrors.ErrNotFound, err) assert.Equal(t, cerrors.ErrNotFound, err)
err = datastore.DeleteVulnerability("debian:7", "TestDeleteVulnerabilityVulnerability1") err = datastore.DeleteVulnerability(testExistNamespace, "TestDeleteVulnerabilityVulnerability1")
assert.Equal(t, cerrors.ErrNotFound, err) assert.Equal(t, cerrors.ErrNotFound, err)
// Delete Vulnerability. // Delete Vulnerability.
err = datastore.DeleteVulnerability("debian:7", "CVE-OPENSSL-1-DEB7") err = datastore.DeleteVulnerability(testExistNamespace, "CVE-OPENSSL-1-DEB7")
if assert.Nil(t, err) { if assert.Nil(t, err) {
_, err := datastore.FindVulnerability("debian:7", "CVE-OPENSSL-1-DEB7") _, err := datastore.FindVulnerability(testExistNamespace, "CVE-OPENSSL-1-DEB7")
assert.Equal(t, cerrors.ErrNotFound, err) assert.Equal(t, cerrors.ErrNotFound, err)
} }
} }
@ -106,8 +118,8 @@ func TestInsertVulnerability(t *testing.T) {
defer datastore.Close() defer datastore.Close()
// Create some data. // Create some data.
n1 := database.Namespace{Name: "TestInsertVulnerabilityNamespace1"} n1 := database.Namespace{Name: "TestInsertVulnerabilityNamespace", Version: types.NewVersionUnsafe("1.0")}
n2 := database.Namespace{Name: "TestInsertVulnerabilityNamespace2"} n2 := database.Namespace{Name: "TestInsertVulnerabilityNamespace", Version: types.NewVersionUnsafe("2.0")}
f1 := database.FeatureVersion{ f1 := database.FeatureVersion{
Feature: database.Feature{ Feature: database.Feature{
@ -216,7 +228,7 @@ func TestInsertVulnerability(t *testing.T) {
} }
err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}, true) err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}, true)
if assert.Nil(t, err) { if assert.Nil(t, err) {
v1f, err := datastore.FindVulnerability(n1.Name, v1.Name) v1f, err := datastore.FindVulnerability(n1, v1.Name)
if assert.Nil(t, err) { if assert.Nil(t, err) {
equalsVuln(t, &v1, &v1f) equalsVuln(t, &v1, &v1f)
} }
@ -232,7 +244,7 @@ func TestInsertVulnerability(t *testing.T) {
err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}, true) err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}, true)
if assert.Nil(t, err) { if assert.Nil(t, err) {
v1f, err := datastore.FindVulnerability(n1.Name, v1.Name) v1f, err := datastore.FindVulnerability(n1, v1.Name)
if assert.Nil(t, err) { if assert.Nil(t, err) {
// We already had f1 before the update. // We already had f1 before the update.
// Add it to the struct for comparison. // Add it to the struct for comparison.
@ -252,7 +264,7 @@ func TestInsertVulnerability(t *testing.T) {
func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) { func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) {
assert.Equal(t, expected.Name, actual.Name) assert.Equal(t, expected.Name, actual.Name)
assert.Equal(t, expected.Namespace.Name, actual.Namespace.Name) assert.True(t, expected.Namespace.Equal(actual.Namespace))
assert.Equal(t, expected.Description, actual.Description) assert.Equal(t, expected.Description, actual.Description)
assert.Equal(t, expected.Link, actual.Link) assert.Equal(t, expected.Link, actual.Link)
assert.Equal(t, expected.Severity, actual.Severity) assert.Equal(t, expected.Severity, actual.Severity)
@ -265,7 +277,7 @@ func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) {
if expectedFeatureVersion.Feature.Name == actualFeatureVersion.Feature.Name { if expectedFeatureVersion.Feature.Name == actualFeatureVersion.Feature.Name {
found = true found = true
assert.Equal(t, expected.Namespace.Name, actualFeatureVersion.Feature.Namespace.Name) assert.True(t, expected.Namespace.Equal(actualFeatureVersion.Feature.Namespace))
assert.Equal(t, expectedFeatureVersion.Version, actualFeatureVersion.Version) assert.Equal(t, expectedFeatureVersion.Version, actualFeatureVersion.Version)
} }
} }

View File

@ -192,7 +192,8 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability,
Feature: database.Feature{ Feature: database.Feature{
Name: pkgName, Name: pkgName,
Namespace: database.Namespace{ Namespace: database.Namespace{
Name: "debian:" + database.DebianReleasesMapping[releaseName], Name: "debian",
Version: types.NewVersionUnsafe(database.DebianReleasesMapping[releaseName]),
}, },
}, },
Version: version, Version: version,

View File

@ -41,14 +41,14 @@ func TestDebianParser(t *testing.T) {
expectedFeatureVersions := []database.FeatureVersion{ expectedFeatureVersions := []database.FeatureVersion{
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:8"}, Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")},
Name: "aptdaemon", Name: "aptdaemon",
}, },
Version: types.MaxVersion, Version: types.MaxVersion,
}, },
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:unstable"}, Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("unstable")},
Name: "aptdaemon", Name: "aptdaemon",
}, },
@ -67,21 +67,21 @@ func TestDebianParser(t *testing.T) {
expectedFeatureVersions := []database.FeatureVersion{ expectedFeatureVersions := []database.FeatureVersion{
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:8"}, Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")},
Name: "aptdaemon", Name: "aptdaemon",
}, },
Version: types.NewVersionUnsafe("0.7.0"), Version: types.NewVersionUnsafe("0.7.0"),
}, },
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:unstable"}, Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("unstable")},
Name: "aptdaemon", Name: "aptdaemon",
}, },
Version: types.NewVersionUnsafe("0.7.0"), Version: types.NewVersionUnsafe("0.7.0"),
}, },
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:8"}, Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")},
Name: "asterisk", Name: "asterisk",
}, },
Version: types.NewVersionUnsafe("0.5.56"), Version: types.NewVersionUnsafe("0.5.56"),
@ -99,7 +99,7 @@ func TestDebianParser(t *testing.T) {
expectedFeatureVersions := []database.FeatureVersion{ expectedFeatureVersions := []database.FeatureVersion{
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:8"}, Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")},
Name: "asterisk", Name: "asterisk",
}, },
Version: types.MinVersion, Version: types.MinVersion,

View File

@ -291,13 +291,14 @@ func toFeatureVersions(criteria criteria) []database.FeatureVersion {
} }
if osVersion > firstConsideredRHEL { if osVersion > firstConsideredRHEL {
featureVersion.Feature.Namespace.Name = "centos" + ":" + strconv.Itoa(osVersion) featureVersion.Feature.Namespace.Name = "centos"
featureVersion.Feature.Namespace.Version = types.NewVersionUnsafe(strconv.Itoa(osVersion))
} else { } else {
continue continue
} }
if featureVersion.Feature.Namespace.Name != "" && featureVersion.Feature.Name != "" && featureVersion.Version.String() != "" { if !featureVersion.Feature.Namespace.IsEmpty() && featureVersion.Feature.Name != "" && featureVersion.Version.String() != "" {
featureVersionParameters[featureVersion.Feature.Namespace.Name+":"+featureVersion.Feature.Name] = featureVersion featureVersionParameters[featureVersion.Feature.Namespace.Name+":"+featureVersion.Feature.Namespace.Version.String()+":"+featureVersion.Feature.Name] = featureVersion
} else { } else {
log.Warningf("could not determine a valid package from criterions: %v", criterions) log.Warningf("could not determine a valid package from criterions: %v", criterions)
} }

View File

@ -41,21 +41,21 @@ func TestRHELParser(t *testing.T) {
expectedFeatureVersions := []database.FeatureVersion{ expectedFeatureVersions := []database.FeatureVersion{
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:7"}, Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")},
Name: "xerces-c", Name: "xerces-c",
}, },
Version: types.NewVersionUnsafe("3.1.1-7.el7_1"), Version: types.NewVersionUnsafe("3.1.1-7.el7_1"),
}, },
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:7"}, Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")},
Name: "xerces-c-devel", Name: "xerces-c-devel",
}, },
Version: types.NewVersionUnsafe("3.1.1-7.el7_1"), Version: types.NewVersionUnsafe("3.1.1-7.el7_1"),
}, },
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:7"}, Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")},
Name: "xerces-c-doc", Name: "xerces-c-doc",
}, },
Version: types.NewVersionUnsafe("3.1.1-7.el7_1"), Version: types.NewVersionUnsafe("3.1.1-7.el7_1"),
@ -79,14 +79,14 @@ func TestRHELParser(t *testing.T) {
expectedFeatureVersions := []database.FeatureVersion{ expectedFeatureVersions := []database.FeatureVersion{
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:6"}, Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("6")},
Name: "firefox", Name: "firefox",
}, },
Version: types.NewVersionUnsafe("38.1.0-1.el6_6"), Version: types.NewVersionUnsafe("38.1.0-1.el6_6"),
}, },
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:7"}, Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")},
Name: "firefox", Name: "firefox",
}, },
Version: types.NewVersionUnsafe("38.1.0-1.el7_1"), Version: types.NewVersionUnsafe("38.1.0-1.el7_1"),

View File

@ -376,7 +376,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability
// Create and add the new package. // Create and add the new package.
featureVersion := database.FeatureVersion{ featureVersion := database.FeatureVersion{
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:" + database.UbuntuReleasesMapping[md["release"]]}, Namespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe(database.UbuntuReleasesMapping[md["release"]])},
Name: md["package"], Name: md["package"],
}, },
Version: version, Version: version,

View File

@ -45,21 +45,21 @@ func TestUbuntuParser(t *testing.T) {
expectedFeatureVersions := []database.FeatureVersion{ expectedFeatureVersions := []database.FeatureVersion{
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:14.04"}, Namespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("14.04")},
Name: "libmspack", Name: "libmspack",
}, },
Version: types.MaxVersion, Version: types.MaxVersion,
}, },
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:15.04"}, Namespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("15.04")},
Name: "libmspack", Name: "libmspack",
}, },
Version: types.NewVersionUnsafe("0.4-3"), Version: types.NewVersionUnsafe("0.4-3"),
}, },
{ {
Feature: database.Feature{ Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:15.10"}, Namespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("15.10")},
Name: "libmspack-anotherpkg", Name: "libmspack-anotherpkg",
}, },
Version: types.NewVersionUnsafe("0.1"), Version: types.NewVersionUnsafe("0.1"),

View File

@ -113,3 +113,16 @@ func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database
func (detector *DpkgFeaturesDetector) GetRequiredFiles() []string { func (detector *DpkgFeaturesDetector) GetRequiredFiles() []string {
return []string{"var/lib/dpkg/status"} return []string{"var/lib/dpkg/status"}
} }
//Supported checks if the input Namespace is supported by the underling detector
func (detector *DpkgFeaturesDetector) Supported(namespace database.Namespace) bool {
supports := []string{"debian", "ubuntu"}
for _, support := range supports {
if strings.HasPrefix(namespace.Name, support) {
return true
}
}
return false
}

View File

@ -118,3 +118,16 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.
func (detector *RpmFeaturesDetector) GetRequiredFiles() []string { func (detector *RpmFeaturesDetector) GetRequiredFiles() []string {
return []string{"var/lib/rpm/Packages"} return []string{"var/lib/rpm/Packages"}
} }
//Supported checks if the input Namespace is supported by the underling detector
func (detector *RpmFeaturesDetector) Supported(namespace database.Namespace) bool {
supports := []string{"centos", "red hat", "fedora"}
for _, support := range supports {
if strings.HasPrefix(namespace.Name, support) {
return true
}
}
return false
}

View File

@ -28,6 +28,8 @@ type FeaturesDetector interface {
// GetRequiredFiles returns the list of files required for Detect, without // GetRequiredFiles returns the list of files required for Detect, without
// leading /. // leading /.
GetRequiredFiles() []string GetRequiredFiles() []string
//Supported checks if the input Namespace is supported by the underling detector
Supported(namespace database.Namespace) bool
} }
var ( var (
@ -54,15 +56,25 @@ func RegisterFeaturesDetector(name string, f FeaturesDetector) {
} }
// DetectFeatures detects a list of FeatureVersion using every registered FeaturesDetector. // DetectFeatures detects a list of FeatureVersion using every registered FeaturesDetector.
func DetectFeatures(data map[string][]byte) ([]database.FeatureVersion, error) { func DetectFeatures(data map[string][]byte, namespaces []database.Namespace) ([]database.FeatureVersion, error) {
var packages []database.FeatureVersion var packages []database.FeatureVersion
for _, detector := range featuresDetectors { for _, detector := range featuresDetectors {
pkgs, err := detector.Detect(data) for _, namespace := range namespaces {
if err != nil { if detector.Supported(namespace) {
return []database.FeatureVersion{}, err pkgs, err := detector.Detect(data)
if err != nil {
return []database.FeatureVersion{}, err
}
// Ensure that every feature has a Namespace associated
for i := 0; i < len(pkgs); i++ {
pkgs[i].Feature.Namespace.Name = namespace.Name
pkgs[i].Feature.Namespace.Version = namespace.Version
}
packages = append(packages, pkgs...)
break
}
} }
packages = append(packages, pkgs...)
} }
return packages, nil return packages, nil

View File

@ -60,18 +60,18 @@ func RegisterNamespaceDetector(name string, f NamespaceDetector) {
namespaceDetectors[name] = f namespaceDetectors[name] = f
} }
// DetectNamespace finds the OS of the layer by using every registered NamespaceDetector. // DetectNamespaces finds the namespaces of the layer by using every registered NamespaceDetector.
func DetectNamespace(data map[string][]byte) *database.Namespace { func DetectNamespaces(data map[string][]byte) (namespaces []database.Namespace) {
for _, detector := range namespaceDetectors { for _, detector := range namespaceDetectors {
if namespace := detector.Detect(data); namespace != nil { if namespace := detector.Detect(data); namespace != nil {
return namespace namespaces = append(namespaces, *namespace)
} }
} }
return nil return
} }
// GetRequiredFilesNamespace returns the list of files required for DetectNamespace for every // GetRequiredFilesNamespace returns the list of files required for DetectNamespaces for every
// registered NamespaceDetector, without leading /. // registered NamespaceDetector, without leading /.
func GetRequiredFilesNamespace() (files []string) { func GetRequiredFilesNamespace() (files []string) {
for _, detector := range namespaceDetectors { for _, detector := range namespaceDetectors {

View File

@ -19,6 +19,7 @@ import (
"strings" "strings"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors" "github.com/coreos/clair/worker/detectors"
) )
@ -75,7 +76,7 @@ func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *dat
} }
if OS != "" && version != "" { if OS != "" && version != "" {
return &database.Namespace{Name: OS + ":" + version} return &database.Namespace{Name: OS, Version: types.NewVersionUnsafe(version)}
} }
return nil return nil
} }

View File

@ -18,12 +18,13 @@ import (
"testing" "testing"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors/namespace" "github.com/coreos/clair/worker/detectors/namespace"
) )
var aptSourcesOSTests = []namespace.NamespaceTest{ var aptSourcesOSTests = []namespace.NamespaceTest{
{ {
ExpectedNamespace: database.Namespace{Name: "debian:unstable"}, ExpectedNamespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("unstable")},
Data: map[string][]byte{ Data: map[string][]byte{
"etc/os-release": []byte( "etc/os-release": []byte(
`PRETTY_NAME="Debian GNU/Linux stretch/sid" `PRETTY_NAME="Debian GNU/Linux stretch/sid"

View File

@ -20,6 +20,7 @@ import (
"strings" "strings"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors" "github.com/coreos/clair/worker/detectors"
) )
@ -70,7 +71,7 @@ func (detector *LsbReleaseNamespaceDetector) Detect(data map[string][]byte) *dat
} }
if OS != "" && version != "" { if OS != "" && version != "" {
return &database.Namespace{Name: OS + ":" + version} return &database.Namespace{Name: OS, Version: types.NewVersionUnsafe(version)}
} }
return nil return nil
} }

View File

@ -18,12 +18,13 @@ import (
"testing" "testing"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors/namespace" "github.com/coreos/clair/worker/detectors/namespace"
) )
var lsbReleaseOSTests = []namespace.NamespaceTest{ var lsbReleaseOSTests = []namespace.NamespaceTest{
{ {
ExpectedNamespace: database.Namespace{Name: "ubuntu:12.04"}, ExpectedNamespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("12.04")},
Data: map[string][]byte{ Data: map[string][]byte{
"etc/lsb-release": []byte( "etc/lsb-release": []byte(
`DISTRIB_ID=Ubuntu `DISTRIB_ID=Ubuntu
@ -33,7 +34,7 @@ DISTRIB_DESCRIPTION="Ubuntu 12.04 LTS"`),
}, },
}, },
{ // We don't care about the minor version of Debian { // We don't care about the minor version of Debian
ExpectedNamespace: database.Namespace{Name: "debian:7"}, ExpectedNamespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("7")},
Data: map[string][]byte{ Data: map[string][]byte{
"etc/lsb-release": []byte( "etc/lsb-release": []byte(
`DISTRIB_ID=Debian `DISTRIB_ID=Debian

View File

@ -20,6 +20,7 @@ import (
"strings" "strings"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors" "github.com/coreos/clair/worker/detectors"
) )
@ -65,7 +66,7 @@ func (detector *OsReleaseNamespaceDetector) Detect(data map[string][]byte) *data
} }
if OS != "" && version != "" { if OS != "" && version != "" {
return &database.Namespace{Name: OS + ":" + version} return &database.Namespace{Name: OS, Version: types.NewVersionUnsafe(version)}
} }
return nil return nil
} }

View File

@ -18,12 +18,13 @@ import (
"testing" "testing"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors/namespace" "github.com/coreos/clair/worker/detectors/namespace"
) )
var osReleaseOSTests = []namespace.NamespaceTest{ var osReleaseOSTests = []namespace.NamespaceTest{
{ {
ExpectedNamespace: database.Namespace{Name: "debian:8"}, ExpectedNamespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")},
Data: map[string][]byte{ Data: map[string][]byte{
"etc/os-release": []byte( "etc/os-release": []byte(
`PRETTY_NAME="Debian GNU/Linux 8 (jessie)" `PRETTY_NAME="Debian GNU/Linux 8 (jessie)"
@ -37,7 +38,7 @@ BUG_REPORT_URL="https://bugs.debian.org/"`),
}, },
}, },
{ {
ExpectedNamespace: database.Namespace{Name: "ubuntu:15.10"}, ExpectedNamespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("15.10")},
Data: map[string][]byte{ Data: map[string][]byte{
"etc/os-release": []byte( "etc/os-release": []byte(
`NAME="Ubuntu" `NAME="Ubuntu"
@ -52,7 +53,7 @@ BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"`),
}, },
}, },
{ // Doesn't have quotes around VERSION_ID { // Doesn't have quotes around VERSION_ID
ExpectedNamespace: database.Namespace{Name: "fedora:20"}, ExpectedNamespace: database.Namespace{Name: "fedora", Version: types.NewVersionUnsafe("20")},
Data: map[string][]byte{ Data: map[string][]byte{
"etc/os-release": []byte( "etc/os-release": []byte(
`NAME=Fedora `NAME=Fedora

View File

@ -19,6 +19,7 @@ import (
"strings" "strings"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors" "github.com/coreos/clair/worker/detectors"
) )
@ -46,7 +47,7 @@ func (detector *RedhatReleaseNamespaceDetector) Detect(data map[string][]byte) *
r := redhatReleaseRegexp.FindStringSubmatch(string(f)) r := redhatReleaseRegexp.FindStringSubmatch(string(f))
if len(r) == 4 { if len(r) == 4 {
return &database.Namespace{Name: strings.ToLower(r[1]) + ":" + r[3]} return &database.Namespace{Name: strings.ToLower(r[1]), Version: types.NewVersionUnsafe(r[3])}
} }
} }

View File

@ -18,18 +18,19 @@ import (
"testing" "testing"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors/namespace" "github.com/coreos/clair/worker/detectors/namespace"
) )
var redhatReleaseTests = []namespace.NamespaceTest{ var redhatReleaseTests = []namespace.NamespaceTest{
{ {
ExpectedNamespace: database.Namespace{Name: "centos:6"}, ExpectedNamespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("6")},
Data: map[string][]byte{ Data: map[string][]byte{
"etc/centos-release": []byte(`CentOS release 6.6 (Final)`), "etc/centos-release": []byte(`CentOS release 6.6 (Final)`),
}, },
}, },
{ {
ExpectedNamespace: database.Namespace{Name: "centos:7"}, ExpectedNamespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")},
Data: map[string][]byte{ Data: map[string][]byte{
"etc/system-release": []byte(`CentOS Linux release 7.1.1503 (Core)`), "etc/system-release": []byte(`CentOS Linux release 7.1.1503 (Core)`),
}, },

View File

@ -29,6 +29,6 @@ type NamespaceTest struct {
func TestNamespaceDetector(t *testing.T, detector detectors.NamespaceDetector, tests []NamespaceTest) { func TestNamespaceDetector(t *testing.T, detector detectors.NamespaceDetector, tests []NamespaceTest) {
for _, test := range tests { for _, test := range tests {
assert.Equal(t, test.ExpectedNamespace, *detector.Detect(test.Data)) assert.True(t, test.ExpectedNamespace.Equal(*detector.Detect(test.Data)))
} }
} }

View File

@ -46,7 +46,7 @@ var (
ErrParentUnknown = cerrors.NewBadRequestError("worker: parent layer is unknown, it must be processed first") ErrParentUnknown = cerrors.NewBadRequestError("worker: parent layer is unknown, it must be processed first")
) )
// Process detects the Namespace of a layer, the features it adds/removes, and // Process detects the Namespaces of a layer, the features it adds/removes, and
// then stores everything in the database. // then stores everything in the database.
// TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an // TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an
// older engine version and that processes them. // older engine version and that processes them.
@ -104,7 +104,7 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s
} }
// Analyze the content. // Analyze the content.
layer.Namespace, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent) layer.Namespaces, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent)
if err != nil { if err != nil {
return err return err
} }
@ -112,99 +112,65 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s
return datastore.InsertLayer(layer) return datastore.InsertLayer(layer)
} }
// detectContent downloads a layer's archive and extracts its Namespace and Features. // detectContent downloads a layer's archive and extracts its Namespaces and Features.
func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) { func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespaces []database.Namespace, features []database.FeatureVersion, err error) {
data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize) data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(),
detectors.GetRequiredFilesNamespace()...), maxFileSize)
if err != nil { if err != nil {
log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err) log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
return return
} }
// Detect namespace. // Detect namespace.
namespace = detectNamespace(name, data, parent) namespaces, err = detectNamespaces(name, data, parent)
if err != nil {
return
}
if len(namespaces) > 0 {
log.Debugf("layer %s: %d Namespaces detected: ", name, len(namespaces))
for _, namespace := range namespaces {
log.Debugf("\t%s", namespace.Name)
}
} else {
log.Debugf("layer %s: Package System is unknown.", name)
}
// Detect features. // Detect features.
featureVersions, err = detectFeatureVersions(name, data, namespace, parent) features, err = detectors.DetectFeatures(data, namespaces)
if err != nil { if err != nil {
return return
} }
if len(featureVersions) > 0 { if len(features) > 0 {
log.Debugf("layer %s: detected %d features", name, len(featureVersions)) log.Debugf("layer %s: detected %d features", name, len(features))
} }
return return
} }
func detectNamespace(name string, data map[string][]byte, parent *database.Layer) (namespace *database.Namespace) { func detectNamespaces(name string, data map[string][]byte, parent *database.Layer) (namespaces []database.Namespace, err error) {
// Use registered detectors to get the Namespace. namespaces = detectors.DetectNamespaces(data)
namespace = detectors.DetectNamespace(data)
if namespace != nil {
log.Debugf("layer %s: detected namespace %q", name, namespace.Name)
return
}
// Use the parent's Namespace. // Inherit the non-detected namespace from its parent
if parent != nil { if parent != nil {
namespace = parent.Namespace mapNamespaces := make(map[string]database.Namespace)
if namespace != nil {
log.Debugf("layer %s: detected namespace %q (from parent)", name, namespace.Name) // Layer's namespaces has high priority than its parent.
return for _, n := range namespaces {
mapNamespaces[n.Name] = n
}
for _, pn := range parent.Namespaces {
if _, ok := mapNamespaces[pn.Name]; !ok {
mapNamespaces[pn.Name] = pn
log.Debugf("layer %s: detected namespace %q (from parent)", name, pn.Name)
}
}
namespaces = []database.Namespace{}
for _, namespace := range mapNamespaces {
namespaces = append(namespaces, namespace)
} }
} }
return return
} }
func detectFeatureVersions(name string, data map[string][]byte, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) {
// TODO(Quentin-M): We need to pass the parent image to DetectFeatures because it's possible that
// some detectors would need it in order to produce the entire feature list (if they can only
// detect a diff). Also, we should probably pass the detected namespace so detectors could
// make their own decision.
features, err = detectors.DetectFeatures(data)
if err != nil {
return
}
// If there are no FeatureVersions, use parent's FeatureVersions if possible.
// TODO(Quentin-M): We eventually want to give the choice to each detectors to use none/some of
// their parent's FeatureVersions. It would be useful for detectors that can't find their entire
// result using one Layer.
if len(features) == 0 && parent != nil {
features = parent.Features
return
}
// Build a map of the namespaces for each FeatureVersion in our parent layer.
parentFeatureNamespaces := make(map[string]database.Namespace)
if parent != nil {
for _, parentFeature := range parent.Features {
parentFeatureNamespaces[parentFeature.Feature.Name+":"+parentFeature.Version.String()] = parentFeature.Feature.Namespace
}
}
// Ensure that each FeatureVersion has an associated Namespace.
for i, feature := range features {
if feature.Feature.Namespace.Name != "" {
// There is a Namespace associated.
continue
}
if parentFeatureNamespace, ok := parentFeatureNamespaces[feature.Feature.Name+":"+feature.Version.String()]; ok {
// The FeatureVersion is present in the parent layer; associate with their Namespace.
features[i].Feature.Namespace = parentFeatureNamespace
continue
}
if namespace != nil {
// The Namespace has been detected in this layer; associate it.
features[i].Feature.Namespace = *namespace
continue
}
log.Warningf("layer %s: Layer's namespace is unknown but non-namespaced features have been detected", name)
err = ErrUnsupported
return
}
return
}

View File

@ -61,7 +61,7 @@ func TestProcessWithDistUpgrade(t *testing.T) {
} }
// Create the list of FeatureVersions that should not been upgraded from one layer to another. // Create the list of FeatureVersions that should not been upgraded from one layer to another.
nonUpgradedFeatureVersions := []database.FeatureVersion{ upgradedFeatureVersions := []database.FeatureVersion{
{Feature: database.Feature{Name: "libtext-wrapi18n-perl"}, Version: types.NewVersionUnsafe("0.06-7")}, {Feature: database.Feature{Name: "libtext-wrapi18n-perl"}, Version: types.NewVersionUnsafe("0.06-7")},
{Feature: database.Feature{Name: "libtext-charwidth-perl"}, Version: types.NewVersionUnsafe("0.04-7")}, {Feature: database.Feature{Name: "libtext-charwidth-perl"}, Version: types.NewVersionUnsafe("0.04-7")},
{Feature: database.Feature{Name: "libtext-iconv-perl"}, Version: types.NewVersionUnsafe("1.7-5")}, {Feature: database.Feature{Name: "libtext-iconv-perl"}, Version: types.NewVersionUnsafe("1.7-5")},
@ -82,31 +82,40 @@ func TestProcessWithDistUpgrade(t *testing.T) {
assert.Nil(t, Process(datastore, "Docker", "wheezy", "blank", testDataPath+"wheezy.tar.gz", nil)) assert.Nil(t, Process(datastore, "Docker", "wheezy", "blank", testDataPath+"wheezy.tar.gz", nil))
assert.Nil(t, Process(datastore, "Docker", "jessie", "wheezy", testDataPath+"jessie.tar.gz", nil)) assert.Nil(t, Process(datastore, "Docker", "jessie", "wheezy", testDataPath+"jessie.tar.gz", nil))
testDebian7 := database.Namespace{
Name: "debian",
Version: types.NewVersionUnsafe("7"),
}
// Ensure that the 'wheezy' layer has the expected namespace and features. // Ensure that the 'wheezy' layer has the expected namespace and features.
wheezy, ok := datastore.layers["wheezy"] wheezy, ok := datastore.layers["wheezy"]
if assert.True(t, ok, "layer 'wheezy' not processed") { if assert.True(t, ok, "layer 'wheezy' not processed") {
assert.Equal(t, "debian:7", wheezy.Namespace.Name) assert.True(t, testDebian7.Equal(wheezy.Namespaces[0]))
assert.Len(t, wheezy.Features, 52) assert.Len(t, wheezy.Features, 52)
for _, nufv := range upgradedFeatureVersions {
for _, nufv := range nonUpgradedFeatureVersions { nufv.Feature.Namespace = testDebian7
nufv.Feature.Namespace.Name = "debian:7"
assert.Contains(t, wheezy.Features, nufv) assert.Contains(t, wheezy.Features, nufv)
} }
} }
testDebian8 := database.Namespace{
Name: "debian",
Version: types.NewVersionUnsafe("8"),
}
// Ensure that the 'wheezy' layer has the expected namespace and non-upgraded features. // Ensure that the 'wheezy' layer has the expected namespace and non-upgraded features.
jessie, ok := datastore.layers["jessie"] jessie, ok := datastore.layers["jessie"]
if assert.True(t, ok, "layer 'jessie' not processed") { if assert.True(t, ok, "layer 'jessie' not processed") {
assert.Equal(t, "debian:8", jessie.Namespace.Name) assert.True(t, testDebian8.Equal(jessie.Namespaces[0]))
assert.Len(t, jessie.Features, 74) assert.Len(t, jessie.Features, 74)
for _, nufv := range nonUpgradedFeatureVersions { for _, nufv := range upgradedFeatureVersions {
nufv.Feature.Namespace.Name = "debian:7" nufv.Feature.Namespace = testDebian7
assert.Contains(t, jessie.Features, nufv)
}
for _, nufv := range nonUpgradedFeatureVersions {
nufv.Feature.Namespace.Name = "debian:8"
assert.NotContains(t, jessie.Features, nufv) assert.NotContains(t, jessie.Features, nufv)
} }
for _, nufv := range upgradedFeatureVersions {
nufv.Feature.Namespace = testDebian8
assert.Contains(t, jessie.Features, nufv)
}
} }
} }