arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

support multiple namespaces in one layer; add database migration

Browse Source 
Signed-off-by: liang chenye <liangchenye@huawei.com>
This commit is contained in:
liang chenye 2016-05-03 15:25:31 +08:00
parent 951efed1ff
commit 0a997145ed
38 changed files with 586 additions and 304 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
database/database.go
View File

@ -93,7 +93,7 @@ type Datastore interface {
// The Limit and page parameters are used to paginate the return list.
// The first given page should be 0. The function will then return the next available page.
// If there is no more page, -1 has to be returned.
ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error)
ListVulnerabilities(namespace Namespace, limit int, page int) ([]Vulnerability, int, error)
// InsertVulnerabilities stores the given Vulnerabilities in the database, updating them if
// necessary. A vulnerability is uniquely identified by its Namespace and its Name.
@ -110,22 +110,22 @@ type Datastore interface {
InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error
// FindVulnerability retrieves a Vulnerability from the database, including the FixedIn list.
FindVulnerability(namespaceName, name string) (Vulnerability, error)
FindVulnerability(namespace Namespace, name string) (Vulnerability, error)
// DeleteVulnerability removes a Vulnerability from the database.
// It has to create a Notification that will contain the old Vulnerability.
DeleteVulnerability(namespaceName, name string) error
DeleteVulnerability(namespace Namespace, name string) error
// InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions of existing ones to
// the specified Vulnerability in the database.
// It has has to create a Notification that will contain the old and the updated Vulnerability.
InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error
InsertVulnerabilityFixes(vulnerabilityNamespace Namespace, vulnerabilityName string, fixes []FeatureVersion) error
// DeleteVulnerabilityFix removes a FixedIn Feature from the specified Vulnerability in the
// database. It can be used to store the fact that a Vulnerability no longer affects the given
// Feature in any Version.
// It has has to create a Notification that will contain the old and the updated Vulnerability.
DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error
DeleteVulnerabilityFix(vulnerabilityNamespace Namespace, vulnerabilityName, featureName string) error
// # Notification
// GetAvailableNotification returns the Name, Created, Notified and Deleted fields of a

26
database/mock.go
View File

@ -23,12 +23,12 @@ type MockDatastore struct {
FctInsertLayer func(Layer) error
FctFindLayer func(name string, withFeatures, withVulnerabilities bool) (Layer, error)
FctDeleteLayer func(name string) error
FctListVulnerabilities func(namespaceName string, limit int, page int) ([]Vulnerability, int, error)
FctListVulnerabilities func(namespace Namespace, limit int, page int) ([]Vulnerability, int, error)
FctInsertVulnerabilities func(vulnerabilities []Vulnerability, createNotification bool) error
FctFindVulnerability func(namespaceName, name string) (Vulnerability, error)
FctDeleteVulnerability func(namespaceName, name string) error
FctInsertVulnerabilityFixes func(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error
FctDeleteVulnerabilityFix func(vulnerabilityNamespace, vulnerabilityName, featureName string) error
FctFindVulnerability func(namespace Namespace, name string) (Vulnerability, error)
FctDeleteVulnerability func(namespace Namespace, name string) error
FctInsertVulnerabilityFixes func(vulnerabilityNamespace Namespace, vulnerabilityName string, fixes []FeatureVersion) error
FctDeleteVulnerabilityFix func(vulnerabilityNamespace Namespace, vulnerabilityName, featureName string) error
FctGetAvailableNotification func(renotifyInterval time.Duration) (VulnerabilityNotification, error)
FctGetNotification func(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error)
FctSetNotificationNotified func(name string) error
@ -70,9 +70,9 @@ func (mds *MockDatastore) DeleteLayer(name string) error {
panic("required mock function not implemented")
}
func (mds *MockDatastore) ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error) {
func (mds *MockDatastore) ListVulnerabilities(namespace Namespace, limit int, page int) ([]Vulnerability, int, error) {
if mds.FctListVulnerabilities != nil {
return mds.FctListVulnerabilities(namespaceName, limit, page)
return mds.FctListVulnerabilities(namespace, limit, page)
}
panic("required mock function not implemented")
}
@ -84,28 +84,28 @@ func (mds *MockDatastore) InsertVulnerabilities(vulnerabilities []Vulnerability,
panic("required mock function not implemented")
}
func (mds *MockDatastore) FindVulnerability(namespaceName, name string) (Vulnerability, error) {
func (mds *MockDatastore) FindVulnerability(namespace Namespace, name string) (Vulnerability, error) {
if mds.FctFindVulnerability != nil {
return mds.FctFindVulnerability(namespaceName, name)
return mds.FctFindVulnerability(namespace, name)
}
panic("required mock function not implemented")
}
func (mds *MockDatastore) DeleteVulnerability(namespaceName, name string) error {
func (mds *MockDatastore) DeleteVulnerability(namespace Namespace, name string) error {
if mds.FctDeleteVulnerability != nil {
return mds.FctDeleteVulnerability(namespaceName, name)
return mds.FctDeleteVulnerability(namespace, name)
}
panic("required mock function not implemented")
}
func (mds *MockDatastore) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error {
func (mds *MockDatastore) InsertVulnerabilityFixes(vulnerabilityNamespace Namespace, vulnerabilityName string, fixes []FeatureVersion) error {
if mds.FctInsertVulnerabilityFixes != nil {
return mds.FctInsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName, fixes)
}
panic("required mock function not implemented")
}
func (mds *MockDatastore) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error {
func (mds *MockDatastore) DeleteVulnerabilityFix(vulnerabilityNamespace Namespace, vulnerabilityName, featureName string) error {
if mds.FctDeleteVulnerabilityFix != nil {
return mds.FctDeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName)
}

21
database/models.go
View File

@ -33,14 +33,31 @@ type Layer struct {
Name string
EngineVersion int
Parent *Layer
Namespace *Namespace
Namespaces []Namespace
Features []FeatureVersion
}
type Namespace struct {
Model
Name string
Name string
Version types.Version
}
func (ns *Namespace) IsEmpty() bool {
if ns.Name == "" && ns.Version.String() == "" {
return true
}
return false
}
func (ns *Namespace) Equal(namespace Namespace) bool {
if ns.Name == namespace.Name && ns.Version.Compare(namespace.Version) == 0 {
return true
}
return false
}
type Feature struct {

7
database/pgsql/complex_test.go
View File

@ -46,8 +46,11 @@ func TestRaceAffects(t *testing.T) {
// Insert the Feature on which we'll work.
feature := database.Feature{
Namespace: database.Namespace{Name: "TestRaceAffectsFeatureNamespace1"},
Name: "TestRaceAffecturesFeature1",
Namespace: database.Namespace{
Name: "TestRaceAffectsFeatureNamespace",
Version: types.NewVersionUnsafe("1.0"),
},
Name: "TestRaceAffecturesFeature1",
}
_, err = datastore.insertFeature(feature)
if err != nil {

6
database/pgsql/feature.go
View File

@ -31,7 +31,7 @@ func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) {
// Do cache lookup.
if pgSQL.cache != nil {
promCacheQueriesTotal.WithLabelValues("feature").Inc()
id, found := pgSQL.cache.Get("feature:" + feature.Namespace.Name + ":" + feature.Name)
id, found := pgSQL.cache.Get("feature:" + feature.Namespace.Name + ":" + feature.Namespace.Version.String() + ":" + feature.Name)
if found {
promCacheHitsTotal.WithLabelValues("feature").Inc()
return id.(int), nil
@ -55,7 +55,7 @@ func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) {
}
if pgSQL.cache != nil {
pgSQL.cache.Add("feature:"+feature.Namespace.Name+":"+feature.Name, id)
pgSQL.cache.Add("feature:"+feature.Namespace.Name+":"+feature.Namespace.Version.String()+":"+feature.Name, id)
}
return id, nil
@ -67,7 +67,7 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion)
}
// Do cache lookup.
cacheIndex := "featureversion:" + featureVersion.Feature.Namespace.Name + ":" + featureVersion.Feature.Name + ":" + featureVersion.Version.String()
cacheIndex := "featureversion:" + featureVersion.Feature.Namespace.Name + ":" + featureVersion.Feature.Namespace.Version.String() + ":" + featureVersion.Feature.Name + ":" + featureVersion.Version.String()
if pgSQL.cache != nil {
promCacheQueriesTotal.WithLabelValues("featureversion").Inc()
id, found := pgSQL.cache.Get(cacheIndex)

28
database/pgsql/feature_test.go
View File

@ -45,8 +45,11 @@ func TestInsertFeature(t *testing.T) {
// Insert Feature and ensure we can find it.
feature := database.Feature{
Namespace: database.Namespace{Name: "TestInsertFeatureNamespace1"},
Name: "TestInsertFeature1",
Namespace: database.Namespace{
Name: "TestInsertFeatureNamespace",
Version: types.NewVersionUnsafe("1.0"),
},
Name: "TestInsertFeature1",
}
id1, err := datastore.insertFeature(feature)
assert.Nil(t, err)
@ -69,15 +72,21 @@ func TestInsertFeature(t *testing.T) {
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertFeatureNamespace2"},
Name: "TestInsertFeature2",
Namespace: database.Namespace{
Name: "TestInsertFeatureNamespace",
Version: types.NewVersionUnsafe("2.0"),
},
Name: "TestInsertFeature2",
},
Version: types.NewVersionUnsafe(""),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertFeatureNamespace2"},
Name: "TestInsertFeature2",
Namespace: database.Namespace{
Name: "TestInsertFeatureNamespace",
Version: types.NewVersionUnsafe("2.0"),
},
Name: "TestInsertFeature2",
},
Version: types.NewVersionUnsafe("bad version"),
},
@ -90,8 +99,11 @@ func TestInsertFeature(t *testing.T) {
// Insert FeatureVersion and ensure we can find it.
featureVersion := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertFeatureNamespace1"},
Name: "TestInsertFeature1",
Namespace: database.Namespace{
Name: "TestInsertFeatureNamespace",
Version: types.NewVersionUnsafe("1.0"),
},
Name: "TestInsertFeature1",
},
Version: types.NewVersionUnsafe("2:3.0-imba"),
}

117
database/pgsql/layer.go
View File

@ -37,11 +37,10 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
var layer database.Layer
var parentID zero.Int
var parentName zero.String
var namespaceID zero.Int
var namespaceName sql.NullString
t := time.Now()
err := pgSQL.QueryRow(searchLayer, name).Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentID, &parentName, &namespaceID, &namespaceName)
err := pgSQL.QueryRow(searchLayer, name).
Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentID, &parentName)
observeQueryTime("FindLayer", "searchLayer", t)
if err != nil {
@ -53,12 +52,50 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
Model: database.Model{ID: int(parentID.Int64)},
Name: parentName.String,
}
}
if !namespaceID.IsZero() {
layer.Namespace = &database.Namespace{
Model: database.Model{ID: int(namespaceID.Int64)},
Name: namespaceName.String,
// Find its parent's namespaces
t = time.Now()
rows, err := pgSQL.Query(searchLayerNamespace, parentID)
observeQueryTime("FindLayer", "searchParentLayerNamespace", t)
if err != nil {
return layer, handleError("searchLayerNamespace", err)
}
defer rows.Close()
for rows.Next() {
var pn database.Namespace
err = rows.Scan(&pn.ID, &pn.Name, &pn.Version)
if err != nil {
return layer, handleError("searchLayerNamespace.Scan()", err)
}
layer.Parent.Namespaces = append(layer.Parent.Namespaces, pn)
}
if err = rows.Err(); err != nil {
return layer, handleError("searchLayerNamespace.Rows()", err)
}
}
// Find its namespaces
t = time.Now()
rows, err := pgSQL.Query(searchLayerNamespace, layer.ID)
observeQueryTime("FindLayer", "searchLayerNamespace", t)
if err != nil {
return layer, handleError("searchLayerNamespace", err)
}
defer rows.Close()
for rows.Next() {
var namespace database.Namespace
err = rows.Scan(&namespace.ID, &namespace.Name, &namespace.Version)
if err != nil {
return layer, handleError("searchLayerNamespace.Scan()", err)
}
layer.Namespaces = append(layer.Namespaces, namespace)
}
if err = rows.Err(); err != nil {
return layer, handleError("searchLayerNamespace.Rows()", err)
}
// Find its features
@ -128,8 +165,9 @@ func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion
var featureVersion database.FeatureVersion
err = rows.Scan(&featureVersion.ID, &modification, &featureVersion.Feature.Namespace.ID,
&featureVersion.Feature.Namespace.Name, &featureVersion.Feature.ID,
&featureVersion.Feature.Name, &featureVersion.ID, &featureVersion.Version,
&featureVersion.Feature.Namespace.Name, &featureVersion.Feature.Namespace.Version,
&featureVersion.Feature.ID, &featureVersion.Feature.Name,
&featureVersion.ID, &featureVersion.Version,
&featureVersion.AddedBy.ID, &featureVersion.AddedBy.Name)
if err != nil {
return featureVersions, handleError("searchLayerFeatureVersion.Scan()", err)
@ -184,7 +222,8 @@ func loadAffectedBy(tx *sql.Tx, featureVersions []database.FeatureVersion) error
var vulnerability database.Vulnerability
err := rows.Scan(&featureversionID, &vulnerability.ID, &vulnerability.Name,
&vulnerability.Description, &vulnerability.Link, &vulnerability.Severity,
&vulnerability.Metadata, &vulnerability.Namespace.Name, &vulnerability.FixedBy)
&vulnerability.Metadata, &vulnerability.Namespace.Name, &vulnerability.Namespace.Version,
&vulnerability.FixedBy)
if err != nil {
return handleError("searchFeatureVersionVulnerability.Scan()", err)
}
@ -234,6 +273,23 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
// We do `defer observeQueryTime` here because we don't want to observe existing layers.
defer observeQueryTime("InsertLayer", "all", tf)
// Insert Namespaces
mapNamespaceIDs := make(map[string]int)
for i, _ := range layer.Namespaces {
id, err := pgSQL.insertNamespace(layer.Namespaces[i])
if err != nil {
return err
}
if layer.Namespaces[i].ID == 0 {
layer.Namespaces[i].ID = id
}
// Layer's namespaces has high priority than its parent.
// Once a layer has a 'same' namespace with its parent,
// it will only keep its namespace.
mapNamespaceIDs[layer.Namespaces[i].Name] = id
}
// Get parent ID.
var parentID zero.Int
if layer.Parent != nil {
@ -243,20 +299,13 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
}
parentID = zero.IntFrom(int64(layer.Parent.ID))
}
// Find or insert namespace if provided.
var namespaceID zero.Int
if layer.Namespace != nil {
n, err := pgSQL.insertNamespace(*layer.Namespace)
if err != nil {
return err
}
namespaceID = zero.IntFrom(int64(n))
} else if layer.Namespace == nil && layer.Parent != nil {
// Import the Namespace from the parent if it has one and this layer doesn't specify one.
if layer.Parent.Namespace != nil {
namespaceID = zero.IntFrom(int64(layer.Parent.Namespace.ID))
for _, pn := range layer.Parent.Namespaces {
if _, ok := mapNamespaceIDs[pn.Name]; !ok {
// Layer will inherit its parent's namespace
mapNamespaceIDs[pn.Name] = pn.ID
layer.Namespaces = append(layer.Namespaces, pn)
}
}
}
@ -269,7 +318,7 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
if layer.ID == 0 {
// Insert a new layer.
err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID, namespaceID).
err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID).
Scan(&layer.ID)
if err != nil {
tx.Rollback()
@ -283,12 +332,19 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
}
} else {
// Update an existing layer.
_, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion, namespaceID)
_, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion)
if err != nil {
tx.Rollback()
return handleError("updateLayer", err)
}
// Remove all existing LayerNamespace.
_, err = tx.Exec(removeLayerNamespace, layer.ID)
if err != nil {
tx.Rollback()
return handleError("removeLayerNamespace", err)
}
// Remove all existing Layer_diff_FeatureVersion.
_, err = tx.Exec(removeLayerDiffFeatureVersion, layer.ID)
if err != nil {
@ -297,6 +353,15 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
}
}
// Insert Layer_Namespace
for _, id := range mapNamespaceIDs {
_, err = tx.Exec(soiLayerNamespace, layer.ID, id)
if err != nil {
tx.Rollback()
return err
}
}
// Update Layer_diff_FeatureVersion now.
err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer)
if err != nil {

106
database/pgsql/layer_test.go
View File

@ -33,13 +33,17 @@ func TestFindLayer(t *testing.T) {
}
defer datastore.Close()
testDebian7 := database.Namespace{
Name: "debian",
Version: types.NewVersionUnsafe("7"),
}
// Layer-0: no parent, no namespace, no feature, no vulnerability
layer, err := datastore.FindLayer("layer-0", false, false)
if assert.Nil(t, err) && assert.NotNil(t, layer) {
assert.Equal(t, "layer-0", layer.Name)
assert.Nil(t, layer.Namespace)
assert.Nil(t, layer.Parent)
assert.Equal(t, 1, layer.EngineVersion)
assert.Len(t, layer.Namespaces, 0)
assert.Len(t, layer.Features, 0)
}
@ -52,18 +56,19 @@ func TestFindLayer(t *testing.T) {
layer, err = datastore.FindLayer("layer-1", false, false)
if assert.Nil(t, err) && assert.NotNil(t, layer) {
assert.Equal(t, layer.Name, "layer-1")
assert.Equal(t, "debian:7", layer.Namespace.Name)
if assert.NotNil(t, layer.Parent) {
assert.Equal(t, "layer-0", layer.Parent.Name)
}
assert.Equal(t, 1, layer.EngineVersion)
assert.Len(t, layer.Namespaces, 1)
assert.True(t, testDebian7.Equal(layer.Namespaces[0]))
assert.Len(t, layer.Features, 0)
}
layer, err = datastore.FindLayer("layer-1", true, false)
if assert.Nil(t, err) && assert.NotNil(t, layer) && assert.Len(t, layer.Features, 2) {
for _, featureVersion := range layer.Features {
assert.Equal(t, "debian:7", featureVersion.Feature.Namespace.Name)
assert.True(t, testDebian7.Equal(featureVersion.Feature.Namespace))
switch featureVersion.Feature.Name {
case "wechat":
@ -79,7 +84,7 @@ func TestFindLayer(t *testing.T) {
layer, err = datastore.FindLayer("layer-1", true, true)
if assert.Nil(t, err) && assert.NotNil(t, layer) && assert.Len(t, layer.Features, 2) {
for _, featureVersion := range layer.Features {
assert.Equal(t, "debian:7", featureVersion.Feature.Namespace.Name)
assert.True(t, testDebian7.Equal(featureVersion.Feature.Namespace))
switch featureVersion.Feature.Name {
case "wechat":
@ -88,7 +93,7 @@ func TestFindLayer(t *testing.T) {
assert.Equal(t, types.NewVersionUnsafe("1.0"), featureVersion.Version)
if assert.Len(t, featureVersion.AffectedBy, 1) {
assert.Equal(t, "debian:7", featureVersion.AffectedBy[0].Namespace.Name)
assert.True(t, testDebian7.Equal(featureVersion.AffectedBy[0].Namespace))
assert.Equal(t, "CVE-OPENSSL-1-DEB7", featureVersion.AffectedBy[0].Name)
assert.Equal(t, types.High, featureVersion.AffectedBy[0].Severity)
assert.Equal(t, "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", featureVersion.AffectedBy[0].Description)
@ -137,44 +142,56 @@ func testInsertLayerInvalid(t *testing.T, datastore database.Datastore) {
}
func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
testInsertLayerNamespace1 := database.Namespace{
Name: "TestInsertLayerNamespace",
Version: types.NewVersionUnsafe("1"),
}
testInsertLayerNamespace2 := database.Namespace{
Name: "TestInsertLayerNamespace",
Version: types.NewVersionUnsafe("2"),
}
testInsertLayerNamespace3 := database.Namespace{
Name: "TestInsertLayerNamespace",
Version: types.NewVersionUnsafe("3"),
}
f1 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"},
Namespace: testInsertLayerNamespace2,
Name: "TestInsertLayerFeature1",
},
Version: types.NewVersionUnsafe("1.0"),
}
f2 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"},
Namespace: testInsertLayerNamespace2,
Name: "TestInsertLayerFeature2",
},
Version: types.NewVersionUnsafe("0.34"),
}
f3 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"},
Namespace: testInsertLayerNamespace2,
Name: "TestInsertLayerFeature3",
},
Version: types.NewVersionUnsafe("0.56"),
}
f4 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"},
Namespace: testInsertLayerNamespace3,
Name: "TestInsertLayerFeature2",
},
Version: types.NewVersionUnsafe("0.34"),
}
f5 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"},
Namespace: testInsertLayerNamespace3,
Name: "TestInsertLayerFeature3",
},
Version: types.NewVersionUnsafe("0.56"),
}
f6 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"},
Namespace: testInsertLayerNamespace3,
Name: "TestInsertLayerFeature4",
},
Version: types.NewVersionUnsafe("0.666"),
@ -185,16 +202,16 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
Name: "TestInsertLayer1",
},
{
Name: "TestInsertLayer2",
Parent: &database.Layer{Name: "TestInsertLayer1"},
Namespace: &database.Namespace{Name: "TestInsertLayerNamespace1"},
Name: "TestInsertLayer2",
Parent: &database.Layer{Name: "TestInsertLayer1"},
Namespaces: []database.Namespace{testInsertLayerNamespace1},
},
// This layer changes the namespace and adds Features.
{
Name: "TestInsertLayer3",
Parent: &database.Layer{Name: "TestInsertLayer2"},
Namespace: &database.Namespace{Name: "TestInsertLayerNamespace2"},
Features: []database.FeatureVersion{f1, f2, f3},
Name: "TestInsertLayer3",
Parent: &database.Layer{Name: "TestInsertLayer2"},
Namespaces: []database.Namespace{testInsertLayerNamespace2},
Features: []database.FeatureVersion{f1, f2, f3},
},
// This layer covers the case where the last layer doesn't provide any new Feature.
{
@ -206,9 +223,9 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
// It also modifies the Namespace ("upgrade") but keeps some Features not upgraded, their
// Namespaces should then remain unchanged.
{
Name: "TestInsertLayer4b",
Parent: &database.Layer{Name: "TestInsertLayer3"},
Namespace: &database.Namespace{Name: "TestInsertLayerNamespace3"},
Name: "TestInsertLayer4b",
Parent: &database.Layer{Name: "TestInsertLayer3"},
Namespaces: []database.Namespace{testInsertLayerNamespace3},
Features: []database.FeatureVersion{
// Deletes TestInsertLayerFeature1.
// Keep TestInsertLayerFeature2 (old Namespace should be kept):
@ -238,8 +255,8 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
}
l4a := retrievedLayers["TestInsertLayer4a"]
if assert.NotNil(t, l4a.Namespace) {
assert.Equal(t, "TestInsertLayerNamespace2", l4a.Namespace.Name)
if assert.Len(t, l4a.Namespaces, 1) {
assert.True(t, testInsertLayerNamespace2.Equal(l4a.Namespaces[0]))
}
assert.Len(t, l4a.Features, 3)
for _, featureVersion := range l4a.Features {
@ -249,8 +266,8 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
}
l4b := retrievedLayers["TestInsertLayer4b"]
if assert.NotNil(t, l4b.Namespace) {
assert.Equal(t, "TestInsertLayerNamespace3", l4b.Namespace.Name)
if assert.Len(t, l4b.Namespaces, 1) {
assert.True(t, testInsertLayerNamespace3.Equal(l4b.Namespaces[0]))
}
assert.Len(t, l4b.Features, 3)
for _, featureVersion := range l4b.Features {
@ -261,9 +278,17 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
}
func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
testInsertLayerNamespace3 := database.Namespace{
Name: "TestInsertLayerNamespace",
Version: types.NewVersionUnsafe("3"),
}
testInsertLayerNamespaceUpdated1 := database.Namespace{
Name: "TestInsertLayerNamespaceUpdated",
Version: types.NewVersionUnsafe("1"),
}
f7 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"},
Namespace: testInsertLayerNamespace3,
Name: "TestInsertLayerFeature7",
},
Version: types.NewVersionUnsafe("0.01"),
@ -271,10 +296,10 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
l3, _ := datastore.FindLayer("TestInsertLayer3", true, false)
l3u := database.Layer{
Name: l3.Name,
Parent: l3.Parent,
Namespace: &database.Namespace{Name: "TestInsertLayerNamespaceUpdated1"},
Features: []database.FeatureVersion{f7},
Name: l3.Name,
Parent: l3.Parent,
Namespaces: []database.Namespace{testInsertLayerNamespaceUpdated1},
Features: []database.FeatureVersion{f7},
}
l4u := database.Layer{
@ -290,7 +315,9 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
l3uf, err := datastore.FindLayer(l3u.Name, true, false)
if assert.Nil(t, err) {
assert.Equal(t, l3.Namespace.Name, l3uf.Namespace.Name)
if assert.Len(t, l3.Namespaces, 1) && assert.Len(t, l3uf.Namespaces, 1) {
assert.True(t, l3.Namespaces[0].Equal(l3uf.Namespaces[0]))
}
assert.Equal(t, l3.EngineVersion, l3uf.EngineVersion)
assert.Len(t, l3uf.Features, len(l3.Features))
}
@ -303,7 +330,9 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
l3uf, err = datastore.FindLayer(l3u.Name, true, false)
if assert.Nil(t, err) {
assert.Equal(t, l3u.Namespace.Name, l3uf.Namespace.Name)
if assert.Len(t, l3u.Namespaces, 1) && assert.Len(t, l3uf.Namespaces, 2) {
assert.True(t, containNS(l3uf.Namespaces, l3u.Namespaces[0]), "Updated layer should have %#v", l3u.Namespaces[0])
}
assert.Equal(t, l3u.EngineVersion, l3uf.EngineVersion)
if assert.Len(t, l3uf.Features, 1) {
assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l3uf.Features[0])
@ -319,7 +348,9 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
l4uf, err := datastore.FindLayer(l3u.Name, true, false)
if assert.Nil(t, err) {
assert.Equal(t, l3u.Namespace.Name, l4uf.Namespace.Name)
if assert.Len(t, l3u.Namespaces, 1) && assert.Len(t, l4uf.Namespaces, 2) {
assert.True(t, containNS(l4uf.Namespaces, l3u.Namespaces[0]), "Updated layer should have %#v", l3u.Namespaces[0])
}
assert.Equal(t, l4u.EngineVersion, l4uf.EngineVersion)
if assert.Len(t, l4uf.Features, 1) {
assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l4uf.Features[0])
@ -349,3 +380,12 @@ func cmpFV(a, b database.FeatureVersion) bool {
a.Feature.Namespace.Name == b.Feature.Namespace.Name &&
a.Version.String() == b.Version.String()
}
func containNS(namespaces []database.Namespace, namespace database.Namespace) bool {
for _, n := range namespaces {
if n.Equal(namespace) {
return true
}
}
return false
}

5
database/pgsql/migrations/20151222113213_Initial.sql
View File

@ -172,3 +172,8 @@ DROP TABLE IF EXISTS Namespace,
KeyValue,
Lock
CASCADE;
DROP TYPE IF EXISTS modification,
severity
CASCADE;

66
database/pgsql/migrations/20160524162211_multiple_namespace.sql Normal file
View File

@ -0,0 +1,66 @@
-- Copyright 2015 clair authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-- +goose Up
-- -----------------------------------------------------
-- Namespace table and data
-- -----------------------------------------------------
ALTER TABLE Namespace ADD version VARCHAR(128) NULL;
UPDATE Namespace SET version = split_part(Namespace.Name, ':', 2), name = split_part(Namespace.Name,':', 1);
-- -----------------------------------------------------
-- LayerNamespace table and data
-- -----------------------------------------------------
CREATE TABLE IF NOT EXISTS LayerNamespace (
id SERIAL PRIMARY KEY,
layer_id INT NOT NULL REFERENCES Layer ON DELETE CASCADE,
namespace_id INT NOT NULL REFERENCES Namespace ON DELETE CASCADE,
UNIQUE (layer_id, namespace_id));
CREATE INDEX ON LayerNamespace (layer_id);
CREATE INDEX ON LayerNamespace (layer_id, namespace_id);
INSERT INTO LayerNamespace(layer_id, namespace_id)
SELECT id, namespace_id
from Layer;
-- -----------------------------------------------------
-- Layer table
-- -----------------------------------------------------
ALTER TABLE Layer DROP COLUMN namespace_id;
-- +goose Down
-- -----------------------------------------------------
-- Layer table and data
-- -----------------------------------------------------
ALTER TABLE Layer ADD namespace_id INT NULL REFERENCES Namespace;
CREATE INDEX ON Layer (namespace_id);
UPDATE Layer l SET namespace_id =
(SELECT namespace_id from LayerNamespace ln
WHERE l.id = ln.layer_id LIMIT 1);
-- -----------------------------------------------------
-- LayerNamespace table (and data)
-- -----------------------------------------------------
DROP TABLE IF EXISTS LayerNamespace
CASCADE;
-- -----------------------------------------------------
-- LayerNamespace data and table
-- -----------------------------------------------------
UPDATE Namespace n SET name = concat(n.name, ':', n.version);
ALTER TABLE Namespace DROP COLUMN version;

10
database/pgsql/namespace.go
View File

@ -22,13 +22,13 @@ import (
)
func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) {
if namespace.Name == "" {
if namespace.IsEmpty() {
return 0, cerrors.NewBadRequestError("could not find/insert invalid Namespace")
}
if pgSQL.cache != nil {
promCacheQueriesTotal.WithLabelValues("namespace").Inc()
if id, found := pgSQL.cache.Get("namespace:" + namespace.Name); found {
if id, found := pgSQL.cache.Get("namespace:" + namespace.Name + ":" + namespace.Version.String()); found {
promCacheHitsTotal.WithLabelValues("namespace").Inc()
return id.(int), nil
}
@ -38,13 +38,13 @@ func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) {
defer observeQueryTime("insertNamespace", "all", time.Now())
var id int
err := pgSQL.QueryRow(soiNamespace, namespace.Name).Scan(&id)
err := pgSQL.QueryRow(soiNamespace, namespace.Name, &namespace.Version).Scan(&id)
if err != nil {
return 0, handleError("soiNamespace", err)
}
if pgSQL.cache != nil {
pgSQL.cache.Add("namespace:"+namespace.Name, id)
pgSQL.cache.Add("namespace:"+namespace.Name+":"+namespace.Version.String(), id)
}
return id, nil
@ -60,7 +60,7 @@ func (pgSQL *pgSQL) ListNamespaces() (namespaces []database.Namespace, err error
for rows.Next() {
var namespace database.Namespace
err = rows.Scan(&namespace.ID, &namespace.Name)
err = rows.Scan(&namespace.ID, &namespace.Name, &namespace.Version)
if err != nil {
return namespaces, handleError("listNamespace.Scan()", err)
}

17
database/pgsql/namespace_test.go
View File

@ -18,9 +18,9 @@ import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/stretchr/testify/assert"
)
func TestInsertNamespace(t *testing.T) {
@ -37,9 +37,9 @@ func TestInsertNamespace(t *testing.T) {
assert.Zero(t, id0)
// Insert Namespace and ensure we can find it.
id1, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace1"})
id1, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace", Version: types.NewVersionUnsafe("1")})
assert.Nil(t, err)
id2, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace1"})
id2, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace", Version: types.NewVersionUnsafe("1")})
assert.Nil(t, err)
assert.Equal(t, id1, id2)
}
@ -55,12 +55,13 @@ func TestListNamespace(t *testing.T) {
namespaces, err := datastore.ListNamespaces()
assert.Nil(t, err)
if assert.Len(t, namespaces, 2) {
testDebian7 := database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("7")}
testDebian8 := database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")}
for _, namespace := range namespaces {
switch namespace.Name {
case "debian:7", "debian:8":
if namespace.Equal(testDebian7) || namespace.Equal(testDebian8) {
continue
default:
assert.Error(t, fmt.Errorf("ListNamespaces should not have returned '%s'", namespace.Name))
} else {
assert.Error(t, fmt.Errorf("ListNamespaces should not have returned '%s:%s'", namespace.Name, namespace.Version.String()))
}
}
}

16
database/pgsql/notification_test.go
View File

@ -39,13 +39,19 @@ func TestNotification(t *testing.T) {
// Create some data.
f1 := database.Feature{
Name: "TestNotificationFeature1",
Namespace: database.Namespace{Name: "TestNotificationNamespace1"},
Name: "TestNotificationFeature1",
Namespace: database.Namespace{
Name: "TestNotificationNamespace",
Version: types.NewVersionUnsafe("1.0"),
},
}
f2 := database.Feature{
Name: "TestNotificationFeature2",
Namespace: database.Namespace{Name: "TestNotificationNamespace1"},
Name: "TestNotificationFeature2",
Namespace: database.Namespace{
Name: "TestNotificationNamespace",
Version: types.NewVersionUnsafe("1.0"),
},
}
l1 := database.Layer{
@ -201,7 +207,7 @@ func TestNotification(t *testing.T) {
}
// Delete a vulnerability and verify the notification.
if assert.Nil(t, datastore.DeleteVulnerability(v1b.Namespace.Name, v1b.Name)) {
if assert.Nil(t, datastore.DeleteVulnerability(v1b.Namespace, v1b.Name)) {
notification, err = datastore.GetAvailableNotification(time.Second)
assert.Nil(t, err)
assert.NotEmpty(t, notification.Name)

59
database/pgsql/queries.go
View File

@ -29,17 +29,17 @@ const (
// namespace.go
soiNamespace = `
WITH new_namespace AS (
INSERT INTO Namespace(name)
SELECT CAST($1 AS VARCHAR)
WHERE NOT EXISTS (SELECT name FROM Namespace WHERE name = $1)
INSERT INTO Namespace(name, version)
SELECT CAST($1 AS VARCHAR), CAST($2 AS VARCHAR)
WHERE NOT EXISTS (SELECT name FROM Namespace WHERE name = $1 AND version = $2)
RETURNING id
)
SELECT id FROM Namespace WHERE name = $1
SELECT id FROM Namespace WHERE name = $1 AND version = $2
UNION
SELECT id FROM new_namespace`
searchNamespace = `SELECT id FROM Namespace WHERE name = $1`
listNamespace = `SELECT id, name FROM Namespace`
searchNamespace = `SELECT id FROM Namespace WHERE name = $1 AND version = $2`
listNamespace = `SELECT id, name, version FROM Namespace`
// feature.go
soiFeature = `
@ -77,10 +77,9 @@ const (
// layer.go
searchLayer = `
SELECT l.id, l.name, l.engineversion, p.id, p.name, n.id, n.name
SELECT l.id, l.name, l.engineversion, p.id, p.name
FROM Layer l
LEFT JOIN Layer p ON l.parent_id = p.id
LEFT JOIN Namespace n ON l.namespace_id = n.id
WHERE l.name = $1;`
searchLayerFeatureVersion = `
@ -93,7 +92,7 @@ const (
FROM Layer l, layer_tree lt
WHERE l.id = lt.parent_id
)
SELECT ldf.featureversion_id, ldf.modification, fn.id, fn.name, f.id, f.name, fv.id, fv.version, ltree.id, ltree.name
SELECT ldf.featureversion_id, ldf.modification, fn.id, fn.name, fn.version, f.id, f.name, fv.id, fv.version, ltree.id, ltree.name
FROM Layer_diff_FeatureVersion ldf
JOIN (
SELECT row_number() over (ORDER BY depth DESC), id, name FROM layer_tree
@ -103,7 +102,7 @@ const (
searchFeatureVersionVulnerability = `
SELECT vafv.featureversion_id, v.id, v.name, v.description, v.link, v.severity, v.metadata,
vn.name, vfif.version
vn.name, vn.version, vfif.version
FROM Vulnerability_Affects_FeatureVersion vafv, Vulnerability v,
Namespace vn, Vulnerability_FixedIn_Feature vfif
WHERE vafv.featureversion_id = ANY($1::integer[])
@ -112,12 +111,17 @@ const (
AND v.namespace_id = vn.id
AND v.deleted_at IS NULL`
insertLayer = `
INSERT INTO Layer(name, engineversion, parent_id, namespace_id, created_at)
VALUES($1, $2, $3, $4, CURRENT_TIMESTAMP)
RETURNING id`
searchLayerNamespace = `
SELECT n.id, n.name, n.version
FROM LayerNamespace ln, Namespace n
WHERE ln.layer_id = $1 AND ln.namespace_id = n.id`
updateLayer = `UPDATE LAYER SET engineversion = $2, namespace_id = $3 WHERE id = $1`
insertLayer = `
INSERT INTO Layer(name, engineversion, parent_id, created_at)
VALUES($1, $2, $3, CURRENT_TIMESTAMP)
RETURNING id`
updateLayer = `UPDATE LAYER SET engineversion = $2 WHERE id = $1`
removeLayerDiffFeatureVersion = `
DELETE FROM Layer_diff_FeatureVersion
@ -131,6 +135,21 @@ const (
removeLayer = `DELETE FROM Layer WHERE name = $1`
removeLayerNamespace = `
DELETE FROM LayerNamespace
WHERE layer_id = $1`
soiLayerNamespace = `
WITH new_layernamespace AS (
INSERT INTO LayerNamespace(layer_id, namespace_id)
SELECT CAST($1 AS INTEGER), CAST($2 AS INTEGER)
WHERE NOT EXISTS (SELECT id FROM LayerNamespace WHERE layer_id = $1 AND namespace_id = $2)
RETURNING id
)
SELECT id FROM LayerNamespace WHERE layer_id = $1 AND namespace_id = $2
UNION
SELECT id FROM new_layernamespace`
// lock.go
insertLock = `INSERT INTO Lock(name, owner, until) VALUES($1, $2, $3)`
searchLock = `SELECT owner, until FROM Lock WHERE name = $1`
@ -140,12 +159,12 @@ const (
// vulnerability.go
searchVulnerabilityBase = `
SELECT v.id, v.name, n.id, n.name, v.description, v.link, v.severity, v.metadata
SELECT v.id, v.name, n.id, n.name, n.version, v.description, v.link, v.severity, v.metadata
FROM Vulnerability v JOIN Namespace n ON v.namespace_id = n.id`
searchVulnerabilityForUpdate = ` FOR UPDATE OF v`
searchVulnerabilityByNamespaceAndName = ` WHERE n.name = $1 AND v.name = $2 AND v.deleted_at IS NULL`
searchVulnerabilityByNamespaceAndName = ` WHERE n.name = $1 AND n.version = $2 AND v.name = $3 AND v.deleted_at IS NULL`
searchVulnerabilityByID = ` WHERE v.id = $1`
searchVulnerabilityByNamespace = ` WHERE n.name = $1 AND v.deleted_at IS NULL
searchVulnerabilityByNamespaceID = ` WHERE n.id = $1 AND v.deleted_at IS NULL
AND v.id >= $2
ORDER BY v.id
LIMIT $3`
@ -170,8 +189,8 @@ const (
removeVulnerability = `
UPDATE Vulnerability
SET deleted_at = CURRENT_TIMESTAMP
WHERE namespace_id = (SELECT id FROM Namespace WHERE name = $1)
AND name = $2
WHERE namespace_id = (SELECT id FROM Namespace WHERE name = $1 AND version = $2)
AND name = $3
AND deleted_at IS NULL
RETURNING id`

25
database/pgsql/testdata/data.sql vendored
View File

@ -12,9 +12,9 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
INSERT INTO namespace (id, name) VALUES
(1, 'debian:7'),
(2, 'debian:8');
INSERT INTO namespace (id, name, version) VALUES
(1, 'debian', '7'),
(2, 'debian', '8');
INSERT INTO feature (id, namespace_id, name) VALUES
(1, 1, 'wechat'),
@ -28,12 +28,18 @@ INSERT INTO featureversion (id, feature_id, version) VALUES
(3, 2, '2.0'),
(4, 3, '1.0');
INSERT INTO layer (id, name, engineversion, parent_id, namespace_id) VALUES
(1, 'layer-0', 1, NULL, NULL),
(2, 'layer-1', 1, 1, 1),
(3, 'layer-2', 1, 2, 1),
(4, 'layer-3a', 1, 3, 1),
(5, 'layer-3b', 1, 3, 2);
INSERT INTO layer (id, name, engineversion, parent_id) VALUES
(1, 'layer-0', 1, NULL),
(2, 'layer-1', 1, 1),
(3, 'layer-2', 1, 2),
(4, 'layer-3a', 1, 3),
(5, 'layer-3b', 1, 3);
INSERT INTO layernamespace (id, layer_id, namespace_id) VALUES
(1, 2, 1),
(2, 3, 1),
(3, 4, 1),
(4, 5, 2);
INSERT INTO layer_diff_featureversion (id, layer_id, featureversion_id, modification) VALUES
(1, 2, 1, 'add'),
@ -58,6 +64,7 @@ SELECT pg_catalog.setval(pg_get_serial_sequence('namespace', 'id'), (SELECT MAX(
SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1);
SELECT pg_catalog.setval(pg_get_serial_sequence('featureversion', 'id'), (SELECT MAX(id) FROM featureversion)+1);
SELECT pg_catalog.setval(pg_get_serial_sequence('layer', 'id'), (SELECT MAX(id) FROM layer)+1);
SELECT pg_catalog.setval(pg_get_serial_sequence('layernamespace', 'id'), (SELECT MAX(id) FROM layernamespace)+1);
SELECT pg_catalog.setval(pg_get_serial_sequence('layer_diff_featureversion', 'id'), (SELECT MAX(id) FROM layer_diff_featureversion)+1);
SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability', 'id'), (SELECT MAX(id) FROM vulnerability)+1);
SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_fixedin_feature', 'id'), (SELECT MAX(id) FROM vulnerability_fixedin_feature)+1);

48
database/pgsql/vulnerability.go
View File

@ -28,12 +28,12 @@ import (
"github.com/guregu/null/zero"
)
func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID int) ([]database.Vulnerability, int, error) {
func (pgSQL *pgSQL) ListVulnerabilities(namespace database.Namespace, limit int, startID int) ([]database.Vulnerability, int, error) {
defer observeQueryTime("listVulnerabilities", "all", time.Now())
// Query Namespace.
var id int
err := pgSQL.QueryRow(searchNamespace, namespaceName).Scan(&id)
err := pgSQL.QueryRow(searchNamespace, namespace.Name, &namespace.Version).Scan(&id)
if err != nil {
return nil, -1, handleError("searchNamespace", err)
} else if id == 0 {
@ -41,10 +41,10 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID
}
// Query.
query := searchVulnerabilityBase + searchVulnerabilityByNamespace
rows, err := pgSQL.Query(query, namespaceName, startID, limit+1)
query := searchVulnerabilityBase + searchVulnerabilityByNamespaceID
rows, err := pgSQL.Query(query, id, startID, limit+1)
if err != nil {
return nil, -1, handleError("searchVulnerabilityByNamespace", err)
return nil, -1, handleError("searchVulnerabilityByNamespaceID", err)
}
defer rows.Close()
@ -60,6 +60,7 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID
&vulnerability.Name,
&vulnerability.Namespace.ID,
&vulnerability.Namespace.Name,
&vulnerability.Namespace.Version,
&vulnerability.Description,
&vulnerability.Link,
&vulnerability.Severity,
@ -83,11 +84,11 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID
return vulns, nextID, nil
}
func (pgSQL *pgSQL) FindVulnerability(namespaceName, name string) (database.Vulnerability, error) {
return findVulnerability(pgSQL, namespaceName, name, false)
func (pgSQL *pgSQL) FindVulnerability(namespace database.Namespace, name string) (database.Vulnerability, error) {
return findVulnerability(pgSQL, namespace, name, false)
}
func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bool) (database.Vulnerability, error) {
func findVulnerability(queryer Queryer, namespace database.Namespace, name string, forUpdate bool) (database.Vulnerability, error) {
defer observeQueryTime("findVulnerability", "all", time.Now())
queryName := "searchVulnerabilityBase+searchVulnerabilityByNamespaceAndName"
@ -97,7 +98,7 @@ func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bo
query = query + searchVulnerabilityForUpdate
}
return scanVulnerability(queryer, queryName, queryer.QueryRow(query, namespaceName, name))
return scanVulnerability(queryer, queryName, queryer.QueryRow(query, namespace.Name, &namespace.Version, name))
}
func (pgSQL *pgSQL) findVulnerabilityByIDWithDeleted(id int) (database.Vulnerability, error) {
@ -117,6 +118,7 @@ func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql.
&vulnerability.Name,
&vulnerability.Namespace.ID,
&vulnerability.Namespace.Name,
&vulnerability.Namespace.Version,
&vulnerability.Description,
&vulnerability.Link,
&vulnerability.Severity,
@ -193,7 +195,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on
tf := time.Now()
// Verify parameters
if vulnerability.Name == "" || vulnerability.Namespace.Name == "" {
if vulnerability.Name == "" || vulnerability.Namespace.IsEmpty() {
return cerrors.NewBadRequestError("insertVulnerability needs at least the Name and the Namespace")
}
if !onlyFixedIn && !vulnerability.Severity.IsValid() {
@ -204,11 +206,12 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on
for i := 0; i < len(vulnerability.FixedIn); i++ {
fifv := &vulnerability.FixedIn[i]
if fifv.Feature.Namespace.Name == "" {
if fifv.Feature.Namespace.IsEmpty() {
// As there is no Namespace on that FixedIn FeatureVersion, set it to the Vulnerability's
// Namespace.
fifv.Feature.Namespace.Name = vulnerability.Namespace.Name
} else if fifv.Feature.Namespace.Name != vulnerability.Namespace.Name {
fifv.Feature.Namespace.Version = vulnerability.Namespace.Version
} else if !fifv.Feature.Namespace.Equal(vulnerability.Namespace) {
msg := "could not insert an invalid vulnerability that contains FixedIn FeatureVersion that are not in the same namespace as the Vulnerability"
log.Warning(msg)
return cerrors.NewBadRequestError(msg)
@ -226,7 +229,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on
}
// Find existing vulnerability and its Vulnerability_FixedIn_Features (for update).
existingVulnerability, err := findVulnerability(tx, vulnerability.Namespace.Name, vulnerability.Name, true)
existingVulnerability, err := findVulnerability(tx, vulnerability.Namespace, vulnerability.Name, true)
if err != nil && err != cerrors.ErrNotFound {
tx.Rollback()
return err
@ -264,7 +267,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on
}
// Mark the old vulnerability as non latest.
_, err = tx.Exec(removeVulnerability, vulnerability.Namespace.Name, vulnerability.Name)
_, err = tx.Exec(removeVulnerability, vulnerability.Namespace.Name, &vulnerability.Namespace.Version, vulnerability.Name)
if err != nil {
tx.Rollback()
return handleError("removeVulnerability", err)
@ -497,13 +500,14 @@ func linkVulnerabilityToFeatureVersions(tx *sql.Tx, fixedInID, vulnerabilityID,
return nil
}
func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []database.FeatureVersion) error {
func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace database.Namespace, vulnerabilityName string, fixes []database.FeatureVersion) error {
defer observeQueryTime("InsertVulnerabilityFixes", "all", time.Now())
v := database.Vulnerability{
Name: vulnerabilityName,
Namespace: database.Namespace{
Name: vulnerabilityNamespace,
Name: vulnerabilityNamespace.Name,
Version: vulnerabilityNamespace.Version,
},
FixedIn: fixes,
}
@ -511,20 +515,22 @@ func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabili
return pgSQL.insertVulnerability(v, true, true)
}
func (pgSQL *pgSQL) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error {
func (pgSQL *pgSQL) DeleteVulnerabilityFix(vulnerabilityNamespace database.Namespace, vulnerabilityName, featureName string) error {
defer observeQueryTime("DeleteVulnerabilityFix", "all", time.Now())
v := database.Vulnerability{
Name: vulnerabilityName,
Namespace: database.Namespace{
Name: vulnerabilityNamespace,
Name: vulnerabilityNamespace.Name,
Version: vulnerabilityNamespace.Version,
},
FixedIn: []database.FeatureVersion{
{
Feature: database.Feature{
Name: featureName,
Namespace: database.Namespace{
Name: vulnerabilityNamespace,
Name: vulnerabilityNamespace.Name,
Version: vulnerabilityNamespace.Version,
},
},
Version: types.MinVersion,
@ -535,7 +541,7 @@ func (pgSQL *pgSQL) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerability
return pgSQL.insertVulnerability(v, true, true)
}
func (pgSQL *pgSQL) DeleteVulnerability(namespaceName, name string) error {
func (pgSQL *pgSQL) DeleteVulnerability(namespace database.Namespace, name string) error {
defer observeQueryTime("DeleteVulnerability", "all", time.Now())
// Begin transaction.
@ -546,7 +552,7 @@ func (pgSQL *pgSQL) DeleteVulnerability(namespaceName, name string) error {
}
var vulnerabilityID int
err = tx.QueryRow(removeVulnerability, namespaceName, name).Scan(&vulnerabilityID)
err = tx.QueryRow(removeVulnerability, namespace.Name, &namespace.Version, name).Scan(&vulnerabilityID)
if err != nil {
tx.Rollback()
return handleError("removeVulnerability", err)

42
database/pgsql/vulnerability_test.go
View File

@ -33,8 +33,12 @@ func TestFindVulnerability(t *testing.T) {
}
defer datastore.Close()
testExistNamespace := database.Namespace{
Name: "debian",
Version: types.NewVersionUnsafe("7"),
}
// Find a vulnerability that does not exist.
_, err = datastore.FindVulnerability("", "")
_, err = datastore.FindVulnerability(database.Namespace{}, "")
assert.Equal(t, cerrors.ErrNotFound, err)
// Find a normal vulnerability.
@ -43,7 +47,7 @@ func TestFindVulnerability(t *testing.T) {
Description: "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0",
Link: "http://google.com/#q=CVE-OPENSSL-1-DEB7",
Severity: types.High,
Namespace: database.Namespace{Name: "debian:7"},
Namespace: testExistNamespace,
FixedIn: []database.FeatureVersion{
{
Feature: database.Feature{Name: "openssl"},
@ -56,7 +60,7 @@ func TestFindVulnerability(t *testing.T) {
},
}
v1f, err := datastore.FindVulnerability("debian:7", "CVE-OPENSSL-1-DEB7")
v1f, err := datastore.FindVulnerability(testExistNamespace, "CVE-OPENSSL-1-DEB7")
if assert.Nil(t, err) {
equalsVuln(t, &v1, &v1f)
}
@ -65,11 +69,11 @@ func TestFindVulnerability(t *testing.T) {
v2 := database.Vulnerability{
Name: "CVE-NOPE",
Description: "A vulnerability affecting nothing",
Namespace: database.Namespace{Name: "debian:7"},
Namespace: testExistNamespace,
Severity: types.Unknown,
}
v2f, err := datastore.FindVulnerability("debian:7", "CVE-NOPE")
v2f, err := datastore.FindVulnerability(testExistNamespace, "CVE-NOPE")
if assert.Nil(t, err) {
equalsVuln(t, &v2, &v2f)
}
@ -83,16 +87,24 @@ func TestDeleteVulnerability(t *testing.T) {
}
defer datastore.Close()
testExistNamespace := database.Namespace{
Name: "debian",
Version: types.NewVersionUnsafe("7"),
}
testNonExistNamespace := database.Namespace{
Name: "TestDeleteVulnerabilityNamespace",
Version: types.NewVersionUnsafe("1.0"),
}
// Delete non-existing Vulnerability.
err = datastore.DeleteVulnerability("TestDeleteVulnerabilityNamespace1", "CVE-OPENSSL-1-DEB7")
err = datastore.DeleteVulnerability(testNonExistNamespace, "CVE-OPENSSL-1-DEB7")
assert.Equal(t, cerrors.ErrNotFound, err)
err = datastore.DeleteVulnerability("debian:7", "TestDeleteVulnerabilityVulnerability1")
err = datastore.DeleteVulnerability(testExistNamespace, "TestDeleteVulnerabilityVulnerability1")
assert.Equal(t, cerrors.ErrNotFound, err)
// Delete Vulnerability.
err = datastore.DeleteVulnerability("debian:7", "CVE-OPENSSL-1-DEB7")
err = datastore.DeleteVulnerability(testExistNamespace, "CVE-OPENSSL-1-DEB7")
if assert.Nil(t, err) {
_, err := datastore.FindVulnerability("debian:7", "CVE-OPENSSL-1-DEB7")
_, err := datastore.FindVulnerability(testExistNamespace, "CVE-OPENSSL-1-DEB7")
assert.Equal(t, cerrors.ErrNotFound, err)
}
}
@ -106,8 +118,8 @@ func TestInsertVulnerability(t *testing.T) {
defer datastore.Close()
// Create some data.
n1 := database.Namespace{Name: "TestInsertVulnerabilityNamespace1"}
n2 := database.Namespace{Name: "TestInsertVulnerabilityNamespace2"}
n1 := database.Namespace{Name: "TestInsertVulnerabilityNamespace", Version: types.NewVersionUnsafe("1.0")}
n2 := database.Namespace{Name: "TestInsertVulnerabilityNamespace", Version: types.NewVersionUnsafe("2.0")}
f1 := database.FeatureVersion{
Feature: database.Feature{
@ -216,7 +228,7 @@ func TestInsertVulnerability(t *testing.T) {
}
err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}, true)
if assert.Nil(t, err) {
v1f, err := datastore.FindVulnerability(n1.Name, v1.Name)
v1f, err := datastore.FindVulnerability(n1, v1.Name)
if assert.Nil(t, err) {
equalsVuln(t, &v1, &v1f)
}
@ -232,7 +244,7 @@ func TestInsertVulnerability(t *testing.T) {
err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}, true)
if assert.Nil(t, err) {
v1f, err := datastore.FindVulnerability(n1.Name, v1.Name)
v1f, err := datastore.FindVulnerability(n1, v1.Name)
if assert.Nil(t, err) {
// We already had f1 before the update.
// Add it to the struct for comparison.
@ -252,7 +264,7 @@ func TestInsertVulnerability(t *testing.T) {
func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) {
assert.Equal(t, expected.Name, actual.Name)
assert.Equal(t, expected.Namespace.Name, actual.Namespace.Name)
assert.True(t, expected.Namespace.Equal(actual.Namespace))
assert.Equal(t, expected.Description, actual.Description)
assert.Equal(t, expected.Link, actual.Link)
assert.Equal(t, expected.Severity, actual.Severity)
@ -265,7 +277,7 @@ func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) {
if expectedFeatureVersion.Feature.Name == actualFeatureVersion.Feature.Name {
found = true
assert.Equal(t, expected.Namespace.Name, actualFeatureVersion.Feature.Namespace.Name)
assert.True(t, expected.Namespace.Equal(actualFeatureVersion.Feature.Namespace))
assert.Equal(t, expectedFeatureVersion.Version, actualFeatureVersion.Version)
}
}

3
updater/fetchers/debian/debian.go
View File

@ -192,7 +192,8 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability,
Feature: database.Feature{
Name: pkgName,
Namespace: database.Namespace{
Name: "debian:" + database.DebianReleasesMapping[releaseName],
Name: "debian",
Version: types.NewVersionUnsafe(database.DebianReleasesMapping[releaseName]),
},
},
Version: version,

12
updater/fetchers/debian/debian_test.go
View File

@ -41,14 +41,14 @@ func TestDebianParser(t *testing.T) {
expectedFeatureVersions := []database.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:8"},
Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")},
Name: "aptdaemon",
},
Version: types.MaxVersion,
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:unstable"},
Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("unstable")},
Name: "aptdaemon",
},
@ -67,21 +67,21 @@ func TestDebianParser(t *testing.T) {
expectedFeatureVersions := []database.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:8"},
Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")},
Name: "aptdaemon",
},
Version: types.NewVersionUnsafe("0.7.0"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:unstable"},
Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("unstable")},
Name: "aptdaemon",
},
Version: types.NewVersionUnsafe("0.7.0"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:8"},
Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")},
Name: "asterisk",
},
Version: types.NewVersionUnsafe("0.5.56"),
@ -99,7 +99,7 @@ func TestDebianParser(t *testing.T) {
expectedFeatureVersions := []database.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:8"},
Namespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")},
Name: "asterisk",
},
Version: types.MinVersion,

7
updater/fetchers/rhel/rhel.go
View File

@ -291,13 +291,14 @@ func toFeatureVersions(criteria criteria) []database.FeatureVersion {
}
if osVersion > firstConsideredRHEL {
featureVersion.Feature.Namespace.Name = "centos" + ":" + strconv.Itoa(osVersion)
featureVersion.Feature.Namespace.Name = "centos"
featureVersion.Feature.Namespace.Version = types.NewVersionUnsafe(strconv.Itoa(osVersion))
} else {
continue
}
if featureVersion.Feature.Namespace.Name != "" && featureVersion.Feature.Name != "" && featureVersion.Version.String() != "" {
featureVersionParameters[featureVersion.Feature.Namespace.Name+":"+featureVersion.Feature.Name] = featureVersion
if !featureVersion.Feature.Namespace.IsEmpty() && featureVersion.Feature.Name != "" && featureVersion.Version.String() != "" {
featureVersionParameters[featureVersion.Feature.Namespace.Name+":"+featureVersion.Feature.Namespace.Version.String()+":"+featureVersion.Feature.Name] = featureVersion
} else {
log.Warningf("could not determine a valid package from criterions: %v", criterions)
}

10
updater/fetchers/rhel/rhel_test.go
View File

@ -41,21 +41,21 @@ func TestRHELParser(t *testing.T) {
expectedFeatureVersions := []database.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:7"},
Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")},
Name: "xerces-c",
},
Version: types.NewVersionUnsafe("3.1.1-7.el7_1"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:7"},
Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")},
Name: "xerces-c-devel",
},
Version: types.NewVersionUnsafe("3.1.1-7.el7_1"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:7"},
Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")},
Name: "xerces-c-doc",
},
Version: types.NewVersionUnsafe("3.1.1-7.el7_1"),
@ -79,14 +79,14 @@ func TestRHELParser(t *testing.T) {
expectedFeatureVersions := []database.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:6"},
Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("6")},
Name: "firefox",
},
Version: types.NewVersionUnsafe("38.1.0-1.el6_6"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:7"},
Namespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")},
Name: "firefox",
},
Version: types.NewVersionUnsafe("38.1.0-1.el7_1"),

2
updater/fetchers/ubuntu/ubuntu.go
View File

@ -376,7 +376,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability
// Create and add the new package.
featureVersion := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:" + database.UbuntuReleasesMapping[md["release"]]},
Namespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe(database.UbuntuReleasesMapping[md["release"]])},
Name: md["package"],
},
Version: version,

6
updater/fetchers/ubuntu/ubuntu_test.go
View File

@ -45,21 +45,21 @@ func TestUbuntuParser(t *testing.T) {
expectedFeatureVersions := []database.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:14.04"},
Namespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("14.04")},
Name: "libmspack",
},
Version: types.MaxVersion,
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:15.04"},
Namespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("15.04")},
Name: "libmspack",
},
Version: types.NewVersionUnsafe("0.4-3"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:15.10"},
Namespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("15.10")},
Name: "libmspack-anotherpkg",
},
Version: types.NewVersionUnsafe("0.1"),

13
worker/detectors/feature/dpkg/dpkg.go
View File

@ -113,3 +113,16 @@ func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database
func (detector *DpkgFeaturesDetector) GetRequiredFiles() []string {
return []string{"var/lib/dpkg/status"}
}
//Supported checks if the input Namespace is supported by the underling detector
func (detector *DpkgFeaturesDetector) Supported(namespace database.Namespace) bool {
supports := []string{"debian", "ubuntu"}
for _, support := range supports {
if strings.HasPrefix(namespace.Name, support) {
return true
}
}
return false
}

13
worker/detectors/feature/rpm/rpm.go
View File

@ -118,3 +118,16 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.
func (detector *RpmFeaturesDetector) GetRequiredFiles() []string {
return []string{"var/lib/rpm/Packages"}
}
//Supported checks if the input Namespace is supported by the underling detector
func (detector *RpmFeaturesDetector) Supported(namespace database.Namespace) bool {
supports := []string{"centos", "red hat", "fedora"}
for _, support := range supports {
if strings.HasPrefix(namespace.Name, support) {
return true
}
}
return false
}

22
worker/detectors/features.go
View File

@ -28,6 +28,8 @@ type FeaturesDetector interface {
// GetRequiredFiles returns the list of files required for Detect, without
// leading /.
GetRequiredFiles() []string
//Supported checks if the input Namespace is supported by the underling detector
Supported(namespace database.Namespace) bool
}
var (
@ -54,15 +56,25 @@ func RegisterFeaturesDetector(name string, f FeaturesDetector) {
}
// DetectFeatures detects a list of FeatureVersion using every registered FeaturesDetector.
func DetectFeatures(data map[string][]byte) ([]database.FeatureVersion, error) {
func DetectFeatures(data map[string][]byte, namespaces []database.Namespace) ([]database.FeatureVersion, error) {
var packages []database.FeatureVersion
for _, detector := range featuresDetectors {
pkgs, err := detector.Detect(data)
if err != nil {
return []database.FeatureVersion{}, err
for _, namespace := range namespaces {
if detector.Supported(namespace) {
pkgs, err := detector.Detect(data)
if err != nil {
return []database.FeatureVersion{}, err
}
// Ensure that every feature has a Namespace associated
for i := 0; i < len(pkgs); i++ {
pkgs[i].Feature.Namespace.Name = namespace.Name
pkgs[i].Feature.Namespace.Version = namespace.Version
}
packages = append(packages, pkgs...)
break
}
}
packages = append(packages, pkgs...)
}
return packages, nil

10
worker/detectors/namespace.go
View File

@ -60,18 +60,18 @@ func RegisterNamespaceDetector(name string, f NamespaceDetector) {
namespaceDetectors[name] = f
}
// DetectNamespace finds the OS of the layer by using every registered NamespaceDetector.
func DetectNamespace(data map[string][]byte) *database.Namespace {
// DetectNamespaces finds the namespaces of the layer by using every registered NamespaceDetector.
func DetectNamespaces(data map[string][]byte) (namespaces []database.Namespace) {
for _, detector := range namespaceDetectors {
if namespace := detector.Detect(data); namespace != nil {
return namespace
namespaces = append(namespaces, *namespace)
}
}
return nil
return
}
// GetRequiredFilesNamespace returns the list of files required for DetectNamespace for every
// GetRequiredFilesNamespace returns the list of files required for DetectNamespaces for every
// registered NamespaceDetector, without leading /.
func GetRequiredFilesNamespace() (files []string) {
for _, detector := range namespaceDetectors {

3
worker/detectors/namespace/aptsources/aptsources.go
View File

@ -19,6 +19,7 @@ import (
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors"
)
@ -75,7 +76,7 @@ func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *dat
}
if OS != "" && version != "" {
return &database.Namespace{Name: OS + ":" + version}
return &database.Namespace{Name: OS, Version: types.NewVersionUnsafe(version)}
}
return nil
}

3
worker/detectors/namespace/aptsources/aptsources_test.go
View File

@ -18,12 +18,13 @@ import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors/namespace"
)
var aptSourcesOSTests = []namespace.NamespaceTest{
{
ExpectedNamespace: database.Namespace{Name: "debian:unstable"},
ExpectedNamespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("unstable")},
Data: map[string][]byte{
"etc/os-release": []byte(
`PRETTY_NAME="Debian GNU/Linux stretch/sid"

3
worker/detectors/namespace/lsbrelease/lsbrelease.go
View File

@ -20,6 +20,7 @@ import (
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors"
)
@ -70,7 +71,7 @@ func (detector *LsbReleaseNamespaceDetector) Detect(data map[string][]byte) *dat
}
if OS != "" && version != "" {
return &database.Namespace{Name: OS + ":" + version}
return &database.Namespace{Name: OS, Version: types.NewVersionUnsafe(version)}
}
return nil
}

5
worker/detectors/namespace/lsbrelease/lsbrelease_test.go
View File

@ -18,12 +18,13 @@ import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors/namespace"
)
var lsbReleaseOSTests = []namespace.NamespaceTest{
{
ExpectedNamespace: database.Namespace{Name: "ubuntu:12.04"},
ExpectedNamespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("12.04")},
Data: map[string][]byte{
"etc/lsb-release": []byte(
`DISTRIB_ID=Ubuntu
@ -33,7 +34,7 @@ DISTRIB_DESCRIPTION="Ubuntu 12.04 LTS"`),
},
},
{ // We don't care about the minor version of Debian
ExpectedNamespace: database.Namespace{Name: "debian:7"},
ExpectedNamespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("7")},
Data: map[string][]byte{
"etc/lsb-release": []byte(
`DISTRIB_ID=Debian

3
worker/detectors/namespace/osrelease/osrelease.go
View File

@ -20,6 +20,7 @@ import (
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors"
)
@ -65,7 +66,7 @@ func (detector *OsReleaseNamespaceDetector) Detect(data map[string][]byte) *data
}
if OS != "" && version != "" {
return &database.Namespace{Name: OS + ":" + version}
return &database.Namespace{Name: OS, Version: types.NewVersionUnsafe(version)}
}
return nil
}

7
worker/detectors/namespace/osrelease/osrelease_test.go
View File

@ -18,12 +18,13 @@ import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors/namespace"
)
var osReleaseOSTests = []namespace.NamespaceTest{
{
ExpectedNamespace: database.Namespace{Name: "debian:8"},
ExpectedNamespace: database.Namespace{Name: "debian", Version: types.NewVersionUnsafe("8")},
Data: map[string][]byte{
"etc/os-release": []byte(
`PRETTY_NAME="Debian GNU/Linux 8 (jessie)"
@ -37,7 +38,7 @@ BUG_REPORT_URL="https://bugs.debian.org/"`),
},
},
{
ExpectedNamespace: database.Namespace{Name: "ubuntu:15.10"},
ExpectedNamespace: database.Namespace{Name: "ubuntu", Version: types.NewVersionUnsafe("15.10")},
Data: map[string][]byte{
"etc/os-release": []byte(
`NAME="Ubuntu"
@ -52,7 +53,7 @@ BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"`),
},
},
{ // Doesn't have quotes around VERSION_ID
ExpectedNamespace: database.Namespace{Name: "fedora:20"},
ExpectedNamespace: database.Namespace{Name: "fedora", Version: types.NewVersionUnsafe("20")},
Data: map[string][]byte{
"etc/os-release": []byte(
`NAME=Fedora

3
worker/detectors/namespace/redhatrelease/redhatrelease.go
View File

@ -19,6 +19,7 @@ import (
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors"
)
@ -46,7 +47,7 @@ func (detector *RedhatReleaseNamespaceDetector) Detect(data map[string][]byte) *
r := redhatReleaseRegexp.FindStringSubmatch(string(f))
if len(r) == 4 {
return &database.Namespace{Name: strings.ToLower(r[1]) + ":" + r[3]}
return &database.Namespace{Name: strings.ToLower(r[1]), Version: types.NewVersionUnsafe(r[3])}
}
}

5
worker/detectors/namespace/redhatrelease/redhatrelease_test.go
View File

@ -18,18 +18,19 @@ import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors/namespace"
)
var redhatReleaseTests = []namespace.NamespaceTest{
{
ExpectedNamespace: database.Namespace{Name: "centos:6"},
ExpectedNamespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("6")},
Data: map[string][]byte{
"etc/centos-release": []byte(`CentOS release 6.6 (Final)`),
},
},
{
ExpectedNamespace: database.Namespace{Name: "centos:7"},
ExpectedNamespace: database.Namespace{Name: "centos", Version: types.NewVersionUnsafe("7")},
Data: map[string][]byte{
"etc/system-release": []byte(`CentOS Linux release 7.1.1503 (Core)`),
},

2
worker/detectors/namespace/test.go
View File

@ -29,6 +29,6 @@ type NamespaceTest struct {
func TestNamespaceDetector(t *testing.T, detector detectors.NamespaceDetector, tests []NamespaceTest) {
for _, test := range tests {
assert.Equal(t, test.ExpectedNamespace, *detector.Detect(test.Data))
assert.True(t, test.ExpectedNamespace.Equal(*detector.Detect(test.Data)))
}
}

116
worker/worker.go
View File

@ -46,7 +46,7 @@ var (
ErrParentUnknown = cerrors.NewBadRequestError("worker: parent layer is unknown, it must be processed first")
)
// Process detects the Namespace of a layer, the features it adds/removes, and
// Process detects the Namespaces of a layer, the features it adds/removes, and
// then stores everything in the database.
// TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an
// older engine version and that processes them.
@ -104,7 +104,7 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s
}
// Analyze the content.
layer.Namespace, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent)
layer.Namespaces, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent)
if err != nil {
return err
}
@ -112,99 +112,65 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s
return datastore.InsertLayer(layer)
}
// detectContent downloads a layer's archive and extracts its Namespace and Features.
func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) {
data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize)
// detectContent downloads a layer's archive and extracts its Namespaces and Features.
func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespaces []database.Namespace, features []database.FeatureVersion, err error) {
data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(),
detectors.GetRequiredFilesNamespace()...), maxFileSize)
if err != nil {
log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
return
}
// Detect namespace.
namespace = detectNamespace(name, data, parent)
namespaces, err = detectNamespaces(name, data, parent)
if err != nil {
return
}
if len(namespaces) > 0 {
log.Debugf("layer %s: %d Namespaces detected: ", name, len(namespaces))
for _, namespace := range namespaces {
log.Debugf("\t%s", namespace.Name)
}
} else {
log.Debugf("layer %s: Package System is unknown.", name)
}
// Detect features.
featureVersions, err = detectFeatureVersions(name, data, namespace, parent)
features, err = detectors.DetectFeatures(data, namespaces)
if err != nil {
return
}
if len(featureVersions) > 0 {
log.Debugf("layer %s: detected %d features", name, len(featureVersions))
if len(features) > 0 {
log.Debugf("layer %s: detected %d features", name, len(features))
}
return
}
func detectNamespace(name string, data map[string][]byte, parent *database.Layer) (namespace *database.Namespace) {
// Use registered detectors to get the Namespace.
namespace = detectors.DetectNamespace(data)
if namespace != nil {
log.Debugf("layer %s: detected namespace %q", name, namespace.Name)
return
}
func detectNamespaces(name string, data map[string][]byte, parent *database.Layer) (namespaces []database.Namespace, err error) {
namespaces = detectors.DetectNamespaces(data)
// Use the parent's Namespace.
// Inherit the non-detected namespace from its parent
if parent != nil {
namespace = parent.Namespace
if namespace != nil {
log.Debugf("layer %s: detected namespace %q (from parent)", name, namespace.Name)
return
mapNamespaces := make(map[string]database.Namespace)
// Layer's namespaces has high priority than its parent.
for _, n := range namespaces {
mapNamespaces[n.Name] = n
}
for _, pn := range parent.Namespaces {
if _, ok := mapNamespaces[pn.Name]; !ok {
mapNamespaces[pn.Name] = pn
log.Debugf("layer %s: detected namespace %q (from parent)", name, pn.Name)
}
}
namespaces = []database.Namespace{}
for _, namespace := range mapNamespaces {
namespaces = append(namespaces, namespace)
}
}
return
}
func detectFeatureVersions(name string, data map[string][]byte, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) {
// TODO(Quentin-M): We need to pass the parent image to DetectFeatures because it's possible that
// some detectors would need it in order to produce the entire feature list (if they can only
// detect a diff). Also, we should probably pass the detected namespace so detectors could
// make their own decision.
features, err = detectors.DetectFeatures(data)
if err != nil {
return
}
// If there are no FeatureVersions, use parent's FeatureVersions if possible.
// TODO(Quentin-M): We eventually want to give the choice to each detectors to use none/some of
// their parent's FeatureVersions. It would be useful for detectors that can't find their entire
// result using one Layer.
if len(features) == 0 && parent != nil {
features = parent.Features
return
}
// Build a map of the namespaces for each FeatureVersion in our parent layer.
parentFeatureNamespaces := make(map[string]database.Namespace)
if parent != nil {
for _, parentFeature := range parent.Features {
parentFeatureNamespaces[parentFeature.Feature.Name+":"+parentFeature.Version.String()] = parentFeature.Feature.Namespace
}
}
// Ensure that each FeatureVersion has an associated Namespace.
for i, feature := range features {
if feature.Feature.Namespace.Name != "" {
// There is a Namespace associated.
continue
}
if parentFeatureNamespace, ok := parentFeatureNamespaces[feature.Feature.Name+":"+feature.Version.String()]; ok {
// The FeatureVersion is present in the parent layer; associate with their Namespace.
features[i].Feature.Namespace = parentFeatureNamespace
continue
}
if namespace != nil {
// The Namespace has been detected in this layer; associate it.
features[i].Feature.Namespace = *namespace
continue
}
log.Warningf("layer %s: Layer's namespace is unknown but non-namespaced features have been detected", name)
err = ErrUnsupported
return
}
return
}

33
worker/worker_test.go
View File

@ -61,7 +61,7 @@ func TestProcessWithDistUpgrade(t *testing.T) {
}
// Create the list of FeatureVersions that should not been upgraded from one layer to another.
nonUpgradedFeatureVersions := []database.FeatureVersion{
upgradedFeatureVersions := []database.FeatureVersion{
{Feature: database.Feature{Name: "libtext-wrapi18n-perl"}, Version: types.NewVersionUnsafe("0.06-7")},
{Feature: database.Feature{Name: "libtext-charwidth-perl"}, Version: types.NewVersionUnsafe("0.04-7")},
{Feature: database.Feature{Name: "libtext-iconv-perl"}, Version: types.NewVersionUnsafe("1.7-5")},
@ -82,31 +82,40 @@ func TestProcessWithDistUpgrade(t *testing.T) {
assert.Nil(t, Process(datastore, "Docker", "wheezy", "blank", testDataPath+"wheezy.tar.gz", nil))
assert.Nil(t, Process(datastore, "Docker", "jessie", "wheezy", testDataPath+"jessie.tar.gz", nil))
testDebian7 := database.Namespace{
Name: "debian",
Version: types.NewVersionUnsafe("7"),
}
// Ensure that the 'wheezy' layer has the expected namespace and features.
wheezy, ok := datastore.layers["wheezy"]
if assert.True(t, ok, "layer 'wheezy' not processed") {
assert.Equal(t, "debian:7", wheezy.Namespace.Name)
assert.True(t, testDebian7.Equal(wheezy.Namespaces[0]))
assert.Len(t, wheezy.Features, 52)
for _, nufv := range nonUpgradedFeatureVersions {
nufv.Feature.Namespace.Name = "debian:7"
for _, nufv := range upgradedFeatureVersions {
nufv.Feature.Namespace = testDebian7
assert.Contains(t, wheezy.Features, nufv)
}
}
testDebian8 := database.Namespace{
Name: "debian",
Version: types.NewVersionUnsafe("8"),
}
// Ensure that the 'wheezy' layer has the expected namespace and non-upgraded features.
jessie, ok := datastore.layers["jessie"]
if assert.True(t, ok, "layer 'jessie' not processed") {
assert.Equal(t, "debian:8", jessie.Namespace.Name)
assert.True(t, testDebian8.Equal(jessie.Namespaces[0]))
assert.Len(t, jessie.Features, 74)
for _, nufv := range nonUpgradedFeatureVersions {
nufv.Feature.Namespace.Name = "debian:7"
assert.Contains(t, jessie.Features, nufv)
}
for _, nufv := range nonUpgradedFeatureVersions {
nufv.Feature.Namespace.Name = "debian:8"
for _, nufv := range upgradedFeatureVersions {
nufv.Feature.Namespace = testDebian7
assert.NotContains(t, jessie.Features, nufv)
}
for _, nufv := range upgradedFeatureVersions {
nufv.Feature.Namespace = testDebian8
assert.Contains(t, jessie.Features, nufv)
}
}
}