added support for detect multiple namespaces in a layer
created table layer_namespace to store the many to many unique mapping of layers and namespaces changed v1 api to provide a list of namespaces for each layer changed namespace detector to use all registered detectors to detect namespaces updated tests for multiple namespaces Fixes #150
This commit is contained in:
parent
aa6a81c60c
commit
bffa6499b7
@ -33,7 +33,7 @@ type Error struct {
|
|||||||
|
|
||||||
type Layer struct {
|
type Layer struct {
|
||||||
Name string `json:"Name,omitempty"`
|
Name string `json:"Name,omitempty"`
|
||||||
NamespaceName string `json:"NamespaceName,omitempty"`
|
NamespaceNames []string `json:"NamespaceNames,omitempty"`
|
||||||
Path string `json:"Path,omitempty"`
|
Path string `json:"Path,omitempty"`
|
||||||
Headers map[string]string `json:"Headers,omitempty"`
|
Headers map[string]string `json:"Headers,omitempty"`
|
||||||
ParentName string `json:"ParentName,omitempty"`
|
ParentName string `json:"ParentName,omitempty"`
|
||||||
@ -52,8 +52,8 @@ func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabil
|
|||||||
layer.ParentName = dbLayer.Parent.Name
|
layer.ParentName = dbLayer.Parent.Name
|
||||||
}
|
}
|
||||||
|
|
||||||
if dbLayer.Namespace != nil {
|
for _, ns := range dbLayer.Namespaces {
|
||||||
layer.NamespaceName = dbLayer.Namespace.Name
|
layer.NamespaceNames = append(layer.NamespaceNames, ns.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
if withFeatures || withVulnerabilities && dbLayer.Features != nil {
|
if withFeatures || withVulnerabilities && dbLayer.Features != nil {
|
||||||
|
@ -31,7 +31,7 @@ type Layer struct {
|
|||||||
Name string
|
Name string
|
||||||
EngineVersion int
|
EngineVersion int
|
||||||
Parent *Layer
|
Parent *Layer
|
||||||
Namespace *Namespace
|
Namespaces []Namespace
|
||||||
Features []FeatureVersion
|
Features []FeatureVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -52,9 +52,6 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
|
|||||||
&layer.EngineVersion,
|
&layer.EngineVersion,
|
||||||
&parentID,
|
&parentID,
|
||||||
&parentName,
|
&parentName,
|
||||||
&nsID,
|
|
||||||
&nsName,
|
|
||||||
&nsVersionFormat,
|
|
||||||
)
|
)
|
||||||
observeQueryTime("FindLayer", "searchLayer", t)
|
observeQueryTime("FindLayer", "searchLayer", t)
|
||||||
|
|
||||||
@ -68,11 +65,23 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
|
|||||||
Name: parentName.String,
|
Name: parentName.String,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
rows, err := pgSQL.Query(searchLayerNamespace, layer.ID)
|
||||||
|
defer rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
return layer, handleError("searchLayerNamespace", err)
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
err = rows.Scan(&nsID, &nsName, &nsVersionFormat)
|
||||||
|
if err != nil {
|
||||||
|
return layer, handleError("searchLayerNamespace", err)
|
||||||
|
}
|
||||||
if !nsID.IsZero() {
|
if !nsID.IsZero() {
|
||||||
layer.Namespace = &database.Namespace{
|
layer.Namespaces = append(layer.Namespaces, database.Namespace{
|
||||||
Model: database.Model{ID: int(nsID.Int64)},
|
Model: database.Model{ID: int(nsID.Int64)},
|
||||||
Name: nsName.String,
|
Name: nsName.String,
|
||||||
VersionFormat: nsVersionFormat.String,
|
VersionFormat: nsVersionFormat.String,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -277,18 +286,22 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
|
|||||||
parentID = zero.IntFrom(int64(layer.Parent.ID))
|
parentID = zero.IntFrom(int64(layer.Parent.ID))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find or insert namespace if provided.
|
// namespaceIDs will contain inherited and new namespaces
|
||||||
var namespaceID zero.Int
|
namespaceIDs := make(map[int]struct{})
|
||||||
if layer.Namespace != nil {
|
|
||||||
n, err := pgSQL.insertNamespace(*layer.Namespace)
|
// try to insert the new namespaces
|
||||||
|
for _, ns := range layer.Namespaces {
|
||||||
|
n, err := pgSQL.insertNamespace(ns)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return handleError("pgSQL.insertNamespace", err)
|
||||||
}
|
}
|
||||||
namespaceID = zero.IntFrom(int64(n))
|
namespaceIDs[n] = struct{}{}
|
||||||
} else if layer.Namespace == nil && layer.Parent != nil {
|
}
|
||||||
// Import the Namespace from the parent if it has one and this layer doesn't specify one.
|
|
||||||
if layer.Parent.Namespace != nil {
|
// inherit namespaces from parent layer
|
||||||
namespaceID = zero.IntFrom(int64(layer.Parent.Namespace.ID))
|
if layer.Parent != nil {
|
||||||
|
for _, ns := range layer.Parent.Namespaces {
|
||||||
|
namespaceIDs[ns.ID] = struct{}{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -301,7 +314,7 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
|
|||||||
|
|
||||||
if layer.ID == 0 {
|
if layer.ID == 0 {
|
||||||
// Insert a new layer.
|
// Insert a new layer.
|
||||||
err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID, namespaceID).
|
err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID).
|
||||||
Scan(&layer.ID)
|
Scan(&layer.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
tx.Rollback()
|
tx.Rollback()
|
||||||
@ -315,12 +328,18 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Update an existing layer.
|
// Update an existing layer.
|
||||||
_, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion, namespaceID)
|
_, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
tx.Rollback()
|
tx.Rollback()
|
||||||
return handleError("updateLayer", err)
|
return handleError("updateLayer", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// replace the old namespace in the database
|
||||||
|
_, err := tx.Exec(removeLayerNamespace, layer.ID)
|
||||||
|
if err != nil {
|
||||||
|
tx.Rollback()
|
||||||
|
return handleError("removeLayerNamespace", err)
|
||||||
|
}
|
||||||
// Remove all existing Layer_diff_FeatureVersion.
|
// Remove all existing Layer_diff_FeatureVersion.
|
||||||
_, err = tx.Exec(removeLayerDiffFeatureVersion, layer.ID)
|
_, err = tx.Exec(removeLayerDiffFeatureVersion, layer.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -329,6 +348,30 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// insert the layer's namespaces
|
||||||
|
stmt, err := tx.Prepare(insertLayerNamespace)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
tx.Rollback()
|
||||||
|
return handleError("failed to prepare statement", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
err = stmt.Close()
|
||||||
|
if err != nil {
|
||||||
|
tx.Rollback()
|
||||||
|
log.WithError(err).Error("failed to close prepared statement")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
for nsid := range namespaceIDs {
|
||||||
|
_, err := stmt.Exec(layer.ID, nsid)
|
||||||
|
if err != nil {
|
||||||
|
tx.Rollback()
|
||||||
|
return handleError("insertLayerNamespace", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Update Layer_diff_FeatureVersion now.
|
// Update Layer_diff_FeatureVersion now.
|
||||||
err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer)
|
err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -37,7 +37,7 @@ func TestFindLayer(t *testing.T) {
|
|||||||
layer, err := datastore.FindLayer("layer-0", false, false)
|
layer, err := datastore.FindLayer("layer-0", false, false)
|
||||||
if assert.Nil(t, err) && assert.NotNil(t, layer) {
|
if assert.Nil(t, err) && assert.NotNil(t, layer) {
|
||||||
assert.Equal(t, "layer-0", layer.Name)
|
assert.Equal(t, "layer-0", layer.Name)
|
||||||
assert.Nil(t, layer.Namespace)
|
assert.Len(t, layer.Namespaces, 0)
|
||||||
assert.Nil(t, layer.Parent)
|
assert.Nil(t, layer.Parent)
|
||||||
assert.Equal(t, 1, layer.EngineVersion)
|
assert.Equal(t, 1, layer.EngineVersion)
|
||||||
assert.Len(t, layer.Features, 0)
|
assert.Len(t, layer.Features, 0)
|
||||||
@ -52,7 +52,7 @@ func TestFindLayer(t *testing.T) {
|
|||||||
layer, err = datastore.FindLayer("layer-1", false, false)
|
layer, err = datastore.FindLayer("layer-1", false, false)
|
||||||
if assert.Nil(t, err) && assert.NotNil(t, layer) {
|
if assert.Nil(t, err) && assert.NotNil(t, layer) {
|
||||||
assert.Equal(t, layer.Name, "layer-1")
|
assert.Equal(t, layer.Name, "layer-1")
|
||||||
assert.Equal(t, "debian:7", layer.Namespace.Name)
|
assertExpectedNamespaceName(t, &layer, []string{"debian:7"})
|
||||||
if assert.NotNil(t, layer.Parent) {
|
if assert.NotNil(t, layer.Parent) {
|
||||||
assert.Equal(t, "layer-0", layer.Parent.Name)
|
assert.Equal(t, "layer-0", layer.Parent.Name)
|
||||||
}
|
}
|
||||||
@ -100,6 +100,27 @@ func TestFindLayer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Testing Multiple namespaces layer-3b has debian:7 and debian:8 namespaces
|
||||||
|
layer, err = datastore.FindLayer("layer-3b", true, true)
|
||||||
|
|
||||||
|
if assert.Nil(t, err) && assert.NotNil(t, layer) && assert.Len(t, layer.Features, 2) {
|
||||||
|
assert.Equal(t, "layer-3b", layer.Name)
|
||||||
|
// validate the namespace
|
||||||
|
assertExpectedNamespaceName(t, &layer, []string{"debian:7", "debian:8"})
|
||||||
|
for _, featureVersion := range layer.Features {
|
||||||
|
switch featureVersion.Feature.Namespace.Name {
|
||||||
|
case "debian:7":
|
||||||
|
assert.Equal(t, "wechat", featureVersion.Feature.Name)
|
||||||
|
assert.Equal(t, "0.5", featureVersion.Version)
|
||||||
|
case "debian:8":
|
||||||
|
assert.Equal(t, "openssl", featureVersion.Feature.Name)
|
||||||
|
assert.Equal(t, "1.0", featureVersion.Version)
|
||||||
|
default:
|
||||||
|
t.Errorf("unexpected package %s for layer-3b", featureVersion.Feature.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInsertLayer(t *testing.T) {
|
func TestInsertLayer(t *testing.T) {
|
||||||
@ -205,19 +226,19 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
|
|||||||
{
|
{
|
||||||
Name: "TestInsertLayer2",
|
Name: "TestInsertLayer2",
|
||||||
Parent: &database.Layer{Name: "TestInsertLayer1"},
|
Parent: &database.Layer{Name: "TestInsertLayer1"},
|
||||||
Namespace: &database.Namespace{
|
Namespaces: []database.Namespace{database.Namespace{
|
||||||
Name: "TestInsertLayerNamespace1",
|
Name: "TestInsertLayerNamespace1",
|
||||||
VersionFormat: dpkg.ParserName,
|
VersionFormat: dpkg.ParserName,
|
||||||
},
|
}},
|
||||||
},
|
},
|
||||||
// This layer changes the namespace and adds Features.
|
// This layer changes the namespace and adds Features.
|
||||||
{
|
{
|
||||||
Name: "TestInsertLayer3",
|
Name: "TestInsertLayer3",
|
||||||
Parent: &database.Layer{Name: "TestInsertLayer2"},
|
Parent: &database.Layer{Name: "TestInsertLayer2"},
|
||||||
Namespace: &database.Namespace{
|
Namespaces: []database.Namespace{database.Namespace{
|
||||||
Name: "TestInsertLayerNamespace2",
|
Name: "TestInsertLayerNamespace2",
|
||||||
VersionFormat: dpkg.ParserName,
|
VersionFormat: dpkg.ParserName,
|
||||||
},
|
}},
|
||||||
Features: []database.FeatureVersion{f1, f2, f3},
|
Features: []database.FeatureVersion{f1, f2, f3},
|
||||||
},
|
},
|
||||||
// This layer covers the case where the last layer doesn't provide any new Feature.
|
// This layer covers the case where the last layer doesn't provide any new Feature.
|
||||||
@ -232,10 +253,10 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
|
|||||||
{
|
{
|
||||||
Name: "TestInsertLayer4b",
|
Name: "TestInsertLayer4b",
|
||||||
Parent: &database.Layer{Name: "TestInsertLayer3"},
|
Parent: &database.Layer{Name: "TestInsertLayer3"},
|
||||||
Namespace: &database.Namespace{
|
Namespaces: []database.Namespace{database.Namespace{
|
||||||
Name: "TestInsertLayerNamespace3",
|
Name: "TestInsertLayerNamespace3",
|
||||||
VersionFormat: dpkg.ParserName,
|
VersionFormat: dpkg.ParserName,
|
||||||
},
|
}},
|
||||||
Features: []database.FeatureVersion{
|
Features: []database.FeatureVersion{
|
||||||
// Deletes TestInsertLayerFeature1.
|
// Deletes TestInsertLayerFeature1.
|
||||||
// Keep TestInsertLayerFeature2 (old Namespace should be kept):
|
// Keep TestInsertLayerFeature2 (old Namespace should be kept):
|
||||||
@ -264,10 +285,9 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
|
|||||||
assert.Nil(t, err)
|
assert.Nil(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// layer inherits all namespaces from its ancestries
|
||||||
l4a := retrievedLayers["TestInsertLayer4a"]
|
l4a := retrievedLayers["TestInsertLayer4a"]
|
||||||
if assert.NotNil(t, l4a.Namespace) {
|
assertExpectedNamespaceName(t, &l4a, []string{"TestInsertLayerNamespace2", "TestInsertLayerNamespace1"})
|
||||||
assert.Equal(t, "TestInsertLayerNamespace2", l4a.Namespace.Name)
|
|
||||||
}
|
|
||||||
assert.Len(t, l4a.Features, 3)
|
assert.Len(t, l4a.Features, 3)
|
||||||
for _, featureVersion := range l4a.Features {
|
for _, featureVersion := range l4a.Features {
|
||||||
if cmpFV(featureVersion, f1) && cmpFV(featureVersion, f2) && cmpFV(featureVersion, f3) {
|
if cmpFV(featureVersion, f1) && cmpFV(featureVersion, f2) && cmpFV(featureVersion, f3) {
|
||||||
@ -276,9 +296,7 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
l4b := retrievedLayers["TestInsertLayer4b"]
|
l4b := retrievedLayers["TestInsertLayer4b"]
|
||||||
if assert.NotNil(t, l4b.Namespace) {
|
assertExpectedNamespaceName(t, &l4b, []string{"TestInsertLayerNamespace1", "TestInsertLayerNamespace2", "TestInsertLayerNamespace3"})
|
||||||
assert.Equal(t, "TestInsertLayerNamespace3", l4b.Namespace.Name)
|
|
||||||
}
|
|
||||||
assert.Len(t, l4b.Features, 3)
|
assert.Len(t, l4b.Features, 3)
|
||||||
for _, featureVersion := range l4b.Features {
|
for _, featureVersion := range l4b.Features {
|
||||||
if cmpFV(featureVersion, f2) && cmpFV(featureVersion, f5) && cmpFV(featureVersion, f6) {
|
if cmpFV(featureVersion, f2) && cmpFV(featureVersion, f5) && cmpFV(featureVersion, f6) {
|
||||||
@ -303,10 +321,10 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
|
|||||||
l3u := database.Layer{
|
l3u := database.Layer{
|
||||||
Name: l3.Name,
|
Name: l3.Name,
|
||||||
Parent: l3.Parent,
|
Parent: l3.Parent,
|
||||||
Namespace: &database.Namespace{
|
Namespaces: []database.Namespace{database.Namespace{
|
||||||
Name: "TestInsertLayerNamespaceUpdated1",
|
Name: "TestInsertLayerNamespaceUpdated1",
|
||||||
VersionFormat: dpkg.ParserName,
|
VersionFormat: dpkg.ParserName,
|
||||||
},
|
}},
|
||||||
Features: []database.FeatureVersion{f7},
|
Features: []database.FeatureVersion{f7},
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -323,7 +341,7 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
|
|||||||
|
|
||||||
l3uf, err := datastore.FindLayer(l3u.Name, true, false)
|
l3uf, err := datastore.FindLayer(l3u.Name, true, false)
|
||||||
if assert.Nil(t, err) {
|
if assert.Nil(t, err) {
|
||||||
assert.Equal(t, l3.Namespace.Name, l3uf.Namespace.Name)
|
assertSameNamespaceName(t, &l3, &l3uf)
|
||||||
assert.Equal(t, l3.EngineVersion, l3uf.EngineVersion)
|
assert.Equal(t, l3.EngineVersion, l3uf.EngineVersion)
|
||||||
assert.Len(t, l3uf.Features, len(l3.Features))
|
assert.Len(t, l3uf.Features, len(l3.Features))
|
||||||
}
|
}
|
||||||
@ -336,7 +354,7 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
|
|||||||
|
|
||||||
l3uf, err = datastore.FindLayer(l3u.Name, true, false)
|
l3uf, err = datastore.FindLayer(l3u.Name, true, false)
|
||||||
if assert.Nil(t, err) {
|
if assert.Nil(t, err) {
|
||||||
assert.Equal(t, l3u.Namespace.Name, l3uf.Namespace.Name)
|
assertSameNamespaceName(t, &l3u, &l3uf)
|
||||||
assert.Equal(t, l3u.EngineVersion, l3uf.EngineVersion)
|
assert.Equal(t, l3u.EngineVersion, l3uf.EngineVersion)
|
||||||
if assert.Len(t, l3uf.Features, 1) {
|
if assert.Len(t, l3uf.Features, 1) {
|
||||||
assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l3uf.Features[0])
|
assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l3uf.Features[0])
|
||||||
@ -352,7 +370,7 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
|
|||||||
|
|
||||||
l4uf, err := datastore.FindLayer(l3u.Name, true, false)
|
l4uf, err := datastore.FindLayer(l3u.Name, true, false)
|
||||||
if assert.Nil(t, err) {
|
if assert.Nil(t, err) {
|
||||||
assert.Equal(t, l3u.Namespace.Name, l4uf.Namespace.Name)
|
assertSameNamespaceName(t, &l3u, &l4uf)
|
||||||
assert.Equal(t, l4u.EngineVersion, l4uf.EngineVersion)
|
assert.Equal(t, l4u.EngineVersion, l4uf.EngineVersion)
|
||||||
if assert.Len(t, l4uf.Features, 1) {
|
if assert.Len(t, l4uf.Features, 1) {
|
||||||
assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l4uf.Features[0])
|
assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l4uf.Features[0])
|
||||||
@ -360,21 +378,56 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func assertSameNamespaceName(t *testing.T, layer1 *database.Layer, layer2 *database.Layer) {
|
||||||
|
assert.Len(t, compareStringLists(extractNamespaceName(layer1), extractNamespaceName(layer2)), 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertExpectedNamespaceName(t *testing.T, layer *database.Layer, expectedNames []string) {
|
||||||
|
assert.Len(t, compareStringLists(extractNamespaceName(layer), expectedNames), 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractNamespaceName(layer *database.Layer) []string {
|
||||||
|
slist := make([]string, 0, len(layer.Namespaces))
|
||||||
|
for _, ns := range layer.Namespaces {
|
||||||
|
slist = append(slist, ns.Name)
|
||||||
|
}
|
||||||
|
return slist
|
||||||
|
}
|
||||||
|
|
||||||
func testInsertLayerDelete(t *testing.T, datastore database.Datastore) {
|
func testInsertLayerDelete(t *testing.T, datastore database.Datastore) {
|
||||||
err := datastore.DeleteLayer("TestInsertLayerX")
|
err := datastore.DeleteLayer("TestInsertLayerX")
|
||||||
assert.Equal(t, commonerr.ErrNotFound, err)
|
assert.Equal(t, commonerr.ErrNotFound, err)
|
||||||
|
|
||||||
|
// ensure layer_namespace table is cleaned up once a layer is removed
|
||||||
|
layer3, err := datastore.FindLayer("TestInsertLayer3", false, false)
|
||||||
|
layer4a, err := datastore.FindLayer("TestInsertLayer4a", false, false)
|
||||||
|
layer4b, err := datastore.FindLayer("TestInsertLayer4b", false, false)
|
||||||
|
|
||||||
err = datastore.DeleteLayer("TestInsertLayer3")
|
err = datastore.DeleteLayer("TestInsertLayer3")
|
||||||
assert.Nil(t, err)
|
assert.Nil(t, err)
|
||||||
|
|
||||||
_, err = datastore.FindLayer("TestInsertLayer3", false, false)
|
_, err = datastore.FindLayer("TestInsertLayer3", false, false)
|
||||||
assert.Equal(t, commonerr.ErrNotFound, err)
|
assert.Equal(t, commonerr.ErrNotFound, err)
|
||||||
|
assertNotInLayerNamespace(t, layer3.ID, datastore)
|
||||||
_, err = datastore.FindLayer("TestInsertLayer4a", false, false)
|
_, err = datastore.FindLayer("TestInsertLayer4a", false, false)
|
||||||
assert.Equal(t, commonerr.ErrNotFound, err)
|
assert.Equal(t, commonerr.ErrNotFound, err)
|
||||||
|
assertNotInLayerNamespace(t, layer4a.ID, datastore)
|
||||||
_, err = datastore.FindLayer("TestInsertLayer4b", true, false)
|
_, err = datastore.FindLayer("TestInsertLayer4b", true, false)
|
||||||
assert.Equal(t, commonerr.ErrNotFound, err)
|
assert.Equal(t, commonerr.ErrNotFound, err)
|
||||||
|
assertNotInLayerNamespace(t, layer4b.ID, datastore)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertNotInLayerNamespace(t *testing.T, layerID int, datastore database.Datastore) {
|
||||||
|
pg, ok := datastore.(*pgSQL)
|
||||||
|
if !assert.True(t, ok) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
tx, err := pg.Begin()
|
||||||
|
if !assert.Nil(t, err) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rows, err := tx.Query(searchLayerNamespace, layerID)
|
||||||
|
assert.False(t, rows.Next())
|
||||||
}
|
}
|
||||||
|
|
||||||
func cmpFV(a, b database.FeatureVersion) bool {
|
func cmpFV(a, b database.FeatureVersion) bool {
|
||||||
|
44
database/pgsql/migrations/00008_add_multiplens.go
Normal file
44
database/pgsql/migrations/00008_add_multiplens.go
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
// Copyright 2016 clair authors
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package migrations
|
||||||
|
|
||||||
|
import "github.com/remind101/migrate"
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
RegisterMigration(migrate.Migration{
|
||||||
|
ID: 8,
|
||||||
|
Up: migrate.Queries([]string{
|
||||||
|
// set on deletion, remove the corresponding rows in database
|
||||||
|
`CREATE TABLE IF NOT EXISTS Layer_Namespace(
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
layer_id INT REFERENCES Layer(id) ON DELETE CASCADE,
|
||||||
|
namespace_id INT REFERENCES Namespace(id) ON DELETE CASCADE,
|
||||||
|
unique(layer_id, namespace_id)
|
||||||
|
);`,
|
||||||
|
`CREATE INDEX ON Layer_Namespace (namespace_id);`,
|
||||||
|
`CREATE INDEX ON Layer_Namespace (layer_id);`,
|
||||||
|
// move the namespace_id to the table
|
||||||
|
`INSERT INTO Layer_Namespace (layer_id, namespace_id) SELECT id, namespace_id FROM Layer;`,
|
||||||
|
// alter the Layer table to remove the column
|
||||||
|
`ALTER TABLE IF EXISTS Layer DROP namespace_id;`,
|
||||||
|
}),
|
||||||
|
Down: migrate.Queries([]string{
|
||||||
|
`ALTER TABLE IF EXISTS Layer ADD namespace_id INT NULL REFERENCES Namespace;`,
|
||||||
|
`CREATE INDEX ON Layer (namespace_id);`,
|
||||||
|
`UPDATE IF EXISTS Layer SET namespace_id = (SELECT lns.namespace_id FROM Layer_Namespace lns WHERE Layer.id = lns.layer_id LIMIT 1);`,
|
||||||
|
`DROP TABLE IF EXISTS Layer_Namespace;`,
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
}
|
@ -77,12 +77,17 @@ const (
|
|||||||
|
|
||||||
// layer.go
|
// layer.go
|
||||||
searchLayer = `
|
searchLayer = `
|
||||||
SELECT l.id, l.name, l.engineversion, p.id, p.name, n.id, n.name, n.version_format
|
SELECT l.id, l.name, l.engineversion, p.id, p.name
|
||||||
FROM Layer l
|
FROM Layer l
|
||||||
LEFT JOIN Layer p ON l.parent_id = p.id
|
LEFT JOIN Layer p ON l.parent_id = p.id
|
||||||
LEFT JOIN Namespace n ON l.namespace_id = n.id
|
|
||||||
WHERE l.name = $1;`
|
WHERE l.name = $1;`
|
||||||
|
|
||||||
|
searchLayerNamespace = `
|
||||||
|
SELECT n.id, n.name, n.version_format
|
||||||
|
FROM Namespace n
|
||||||
|
JOIN Layer_Namespace lns ON lns.namespace_id = n.id
|
||||||
|
WHERE lns.layer_id = $1`
|
||||||
|
|
||||||
searchLayerFeatureVersion = `
|
searchLayerFeatureVersion = `
|
||||||
WITH RECURSIVE layer_tree(id, name, parent_id, depth, path, cycle) AS(
|
WITH RECURSIVE layer_tree(id, name, parent_id, depth, path, cycle) AS(
|
||||||
SELECT l.id, l.name, l.parent_id, 1, ARRAY[l.id], false
|
SELECT l.id, l.name, l.parent_id, 1, ARRAY[l.id], false
|
||||||
@ -113,11 +118,14 @@ const (
|
|||||||
AND v.deleted_at IS NULL`
|
AND v.deleted_at IS NULL`
|
||||||
|
|
||||||
insertLayer = `
|
insertLayer = `
|
||||||
INSERT INTO Layer(name, engineversion, parent_id, namespace_id, created_at)
|
INSERT INTO Layer(name, engineversion, parent_id, created_at)
|
||||||
VALUES($1, $2, $3, $4, CURRENT_TIMESTAMP)
|
VALUES($1, $2, $3, CURRENT_TIMESTAMP)
|
||||||
RETURNING id`
|
RETURNING id`
|
||||||
|
|
||||||
updateLayer = `UPDATE LAYER SET engineversion = $2, namespace_id = $3 WHERE id = $1`
|
insertLayerNamespace = `INSERT INTO Layer_Namespace(layer_id, namespace_id) VALUES($1, $2)`
|
||||||
|
removeLayerNamespace = `DELETE FROM Layer_Namespace WHERE layer_id = $1`
|
||||||
|
|
||||||
|
updateLayer = `UPDATE LAYER SET engineversion = $2 WHERE id = $1`
|
||||||
|
|
||||||
removeLayerDiffFeatureVersion = `
|
removeLayerDiffFeatureVersion = `
|
||||||
DELETE FROM Layer_diff_FeatureVersion
|
DELETE FROM Layer_diff_FeatureVersion
|
||||||
|
20
database/pgsql/testdata/data.sql
vendored
20
database/pgsql/testdata/data.sql
vendored
@ -28,12 +28,19 @@ INSERT INTO featureversion (id, feature_id, version) VALUES
|
|||||||
(3, 2, '2.0'),
|
(3, 2, '2.0'),
|
||||||
(4, 3, '1.0');
|
(4, 3, '1.0');
|
||||||
|
|
||||||
INSERT INTO layer (id, name, engineversion, parent_id, namespace_id) VALUES
|
INSERT INTO layer (id, name, engineversion, parent_id) VALUES
|
||||||
(1, 'layer-0', 1, NULL, NULL),
|
(1, 'layer-0', 1, NULL),
|
||||||
(2, 'layer-1', 1, 1, 1),
|
(2, 'layer-1', 1, 1),
|
||||||
(3, 'layer-2', 1, 2, 1),
|
(3, 'layer-2', 1, 2),
|
||||||
(4, 'layer-3a', 1, 3, 1),
|
(4, 'layer-3a', 1, 3),
|
||||||
(5, 'layer-3b', 1, 3, 2);
|
(5, 'layer-3b', 1, 3);
|
||||||
|
|
||||||
|
INSERT INTO layer_namespace (id, layer_id, namespace_id) VALUES
|
||||||
|
(1, 2, 1),
|
||||||
|
(2, 3, 1),
|
||||||
|
(3, 4, 1),
|
||||||
|
(4, 5, 2),
|
||||||
|
(5, 5, 1);
|
||||||
|
|
||||||
INSERT INTO layer_diff_featureversion (id, layer_id, featureversion_id, modification) VALUES
|
INSERT INTO layer_diff_featureversion (id, layer_id, featureversion_id, modification) VALUES
|
||||||
(1, 2, 1, 'add'),
|
(1, 2, 1, 'add'),
|
||||||
@ -58,6 +65,7 @@ SELECT pg_catalog.setval(pg_get_serial_sequence('namespace', 'id'), (SELECT MAX(
|
|||||||
SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1);
|
SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1);
|
||||||
SELECT pg_catalog.setval(pg_get_serial_sequence('featureversion', 'id'), (SELECT MAX(id) FROM featureversion)+1);
|
SELECT pg_catalog.setval(pg_get_serial_sequence('featureversion', 'id'), (SELECT MAX(id) FROM featureversion)+1);
|
||||||
SELECT pg_catalog.setval(pg_get_serial_sequence('layer', 'id'), (SELECT MAX(id) FROM layer)+1);
|
SELECT pg_catalog.setval(pg_get_serial_sequence('layer', 'id'), (SELECT MAX(id) FROM layer)+1);
|
||||||
|
SELECT pg_catalog.setval(pg_get_serial_sequence('layer_namespace', 'id'), (SELECT MAX(id) FROM layer_namespace)+1);
|
||||||
SELECT pg_catalog.setval(pg_get_serial_sequence('layer_diff_featureversion', 'id'), (SELECT MAX(id) FROM layer_diff_featureversion)+1);
|
SELECT pg_catalog.setval(pg_get_serial_sequence('layer_diff_featureversion', 'id'), (SELECT MAX(id) FROM layer_diff_featureversion)+1);
|
||||||
SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability', 'id'), (SELECT MAX(id) FROM vulnerability)+1);
|
SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability', 'id'), (SELECT MAX(id) FROM vulnerability)+1);
|
||||||
SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_fixedin_feature', 'id'), (SELECT MAX(id) FROM vulnerability_fixedin_feature)+1);
|
SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_fixedin_feature', 'id'), (SELECT MAX(id) FROM vulnerability_fixedin_feature)+1);
|
||||||
|
@ -68,26 +68,25 @@ func RegisterDetector(name string, d Detector) {
|
|||||||
detectors[name] = d
|
detectors[name] = d
|
||||||
}
|
}
|
||||||
|
|
||||||
// Detect iterators through all registered Detectors and returns the first
|
// Detect iterators through all registered Detectors and returns all non-nil detected namespaces
|
||||||
// non-nil detected namespace.
|
func Detect(files tarutil.FilesMap) ([]database.Namespace, error) {
|
||||||
func Detect(files tarutil.FilesMap) (*database.Namespace, error) {
|
|
||||||
detectorsM.RLock()
|
detectorsM.RLock()
|
||||||
defer detectorsM.RUnlock()
|
defer detectorsM.RUnlock()
|
||||||
|
var namespaces []database.Namespace
|
||||||
for name, detector := range detectors {
|
for name, detector := range detectors {
|
||||||
namespace, err := detector.Detect(files)
|
namespace, err := detector.Detect(files)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.WithError(err).WithField("name", name).Warning("failed while attempting to detect namespace")
|
log.WithError(err).WithField("name", name).Warning("failed while attempting to detect namespace")
|
||||||
return nil, err
|
return []database.Namespace{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if namespace != nil {
|
if namespace != nil {
|
||||||
log.WithFields(log.Fields{"name": name, "namespace": namespace.Name}).Debug("detected namespace")
|
log.WithFields(log.Fields{"name": name, "namespace": namespace.Name}).Debug("detected namespace")
|
||||||
return namespace, nil
|
namespaces = append(namespaces, *namespace)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, nil
|
return namespaces, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RequiredFilenames returns the total list of files required for all
|
// RequiredFilenames returns the total list of files required for all
|
||||||
|
30
worker.go
30
worker.go
@ -105,7 +105,7 @@ func ProcessLayer(datastore database.Datastore, imageFormat, name, parentName, p
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Analyze the content.
|
// Analyze the content.
|
||||||
layer.Namespace, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent)
|
layer.Namespaces, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -115,7 +115,7 @@ func ProcessLayer(datastore database.Datastore, imageFormat, name, parentName, p
|
|||||||
|
|
||||||
// detectContent downloads a layer's archive and extracts its Namespace and
|
// detectContent downloads a layer's archive and extracts its Namespace and
|
||||||
// Features.
|
// Features.
|
||||||
func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) {
|
func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespaces []database.Namespace, featureVersions []database.FeatureVersion, err error) {
|
||||||
totalRequiredFiles := append(featurefmt.RequiredFilenames(), featurens.RequiredFilenames()...)
|
totalRequiredFiles := append(featurefmt.RequiredFilenames(), featurens.RequiredFilenames()...)
|
||||||
files, err := imagefmt.Extract(imageFormat, path, headers, totalRequiredFiles)
|
files, err := imagefmt.Extract(imageFormat, path, headers, totalRequiredFiles)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -123,16 +123,21 @@ func detectContent(imageFormat, name, path string, headers map[string]string, pa
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace, err = detectNamespace(name, files, parent)
|
namespaces, err = detectNamespaces(name, files, parent)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Detect features.
|
// Detect features.
|
||||||
featureVersions, err = detectFeatureVersions(name, files, namespace, parent)
|
var fv []database.FeatureVersion
|
||||||
|
// detect feature versions in all namespaces
|
||||||
|
for _, namespace := range namespaces {
|
||||||
|
fv, err = detectFeatureVersions(name, files, &namespace, parent)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
featureVersions = append(featureVersions, fv...)
|
||||||
|
}
|
||||||
if len(featureVersions) > 0 {
|
if len(featureVersions) > 0 {
|
||||||
log.WithFields(log.Fields{logLayerName: name, "feature count": len(featureVersions)}).Debug("detected features")
|
log.WithFields(log.Fields{logLayerName: name, "feature count": len(featureVersions)}).Debug("detected features")
|
||||||
}
|
}
|
||||||
@ -140,23 +145,24 @@ func detectContent(imageFormat, name, path string, headers map[string]string, pa
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func detectNamespace(name string, files tarutil.FilesMap, parent *database.Layer) (namespace *database.Namespace, err error) {
|
func detectNamespaces(name string, files tarutil.FilesMap, parent *database.Layer) (namespaces []database.Namespace, err error) {
|
||||||
namespace, err = featurens.Detect(files)
|
namespaces, err = featurens.Detect(files)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if namespace != nil {
|
if len(namespaces) > 0 {
|
||||||
log.WithFields(log.Fields{logLayerName: name, "detected namespace": namespace.Name}).Debug("detected namespace")
|
for _, ns := range namespaces {
|
||||||
|
log.WithFields(log.Fields{logLayerName: name, "detected namespace": ns.Name}).Debug("detected namespace")
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback to the parent's namespace.
|
// Fallback to the parent's namespace.
|
||||||
if parent != nil {
|
if parent != nil {
|
||||||
namespace = parent.Namespace
|
for _, ns := range parent.Namespaces {
|
||||||
if namespace != nil {
|
log.WithFields(log.Fields{logLayerName: name, "detected namespace": ns.Name}).Debug("detected namespace (from parent)")
|
||||||
log.WithFields(log.Fields{logLayerName: name, "detected namespace": namespace.Name}).Debug("detected namespace (from parent)")
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
|
@ -85,7 +85,10 @@ func TestProcessWithDistUpgrade(t *testing.T) {
|
|||||||
// Ensure that the 'wheezy' layer has the expected namespace and features.
|
// Ensure that the 'wheezy' layer has the expected namespace and features.
|
||||||
wheezy, ok := datastore.layers["wheezy"]
|
wheezy, ok := datastore.layers["wheezy"]
|
||||||
if assert.True(t, ok, "layer 'wheezy' not processed") {
|
if assert.True(t, ok, "layer 'wheezy' not processed") {
|
||||||
assert.Equal(t, "debian:7", wheezy.Namespace.Name)
|
if !assert.Len(t, wheezy.Namespaces, 1) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
assert.Equal(t, "debian:7", wheezy.Namespaces[0].Name)
|
||||||
assert.Len(t, wheezy.Features, 52)
|
assert.Len(t, wheezy.Features, 52)
|
||||||
|
|
||||||
for _, nufv := range nonUpgradedFeatureVersions {
|
for _, nufv := range nonUpgradedFeatureVersions {
|
||||||
@ -98,7 +101,10 @@ func TestProcessWithDistUpgrade(t *testing.T) {
|
|||||||
// Ensure that the 'wheezy' layer has the expected namespace and non-upgraded features.
|
// Ensure that the 'wheezy' layer has the expected namespace and non-upgraded features.
|
||||||
jessie, ok := datastore.layers["jessie"]
|
jessie, ok := datastore.layers["jessie"]
|
||||||
if assert.True(t, ok, "layer 'jessie' not processed") {
|
if assert.True(t, ok, "layer 'jessie' not processed") {
|
||||||
assert.Equal(t, "debian:8", jessie.Namespace.Name)
|
if !assert.Len(t, jessie.Namespaces, 1) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
assert.Equal(t, "debian:8", jessie.Namespaces[0].Name)
|
||||||
assert.Len(t, jessie.Features, 74)
|
assert.Len(t, jessie.Features, 74)
|
||||||
|
|
||||||
for _, nufv := range nonUpgradedFeatureVersions {
|
for _, nufv := range nonUpgradedFeatureVersions {
|
||||||
|
Loading…
Reference in New Issue
Block a user