From bffa6499b76509a978d31b8290de30b97e72bfbe Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Fri, 12 May 2017 16:59:17 -0400 Subject: [PATCH] added support for detect multiple namespaces in a layer created table layer_namespace to store the many to many unique mapping of layers and namespaces changed v1 api to provide a list of namespaces for each layer changed namespace detector to use all registered detectors to detect namespaces updated tests for multiple namespaces Fixes #150 --- api/v1/models.go | 6 +- database/models.go | 2 +- database/pgsql/layer.go | 83 ++++++++++++---- database/pgsql/layer_test.go | 95 +++++++++++++++---- .../pgsql/migrations/00008_add_multiplens.go | 44 +++++++++ database/pgsql/queries.go | 24 +++-- database/pgsql/testdata/data.sql | 20 ++-- ext/featurens/driver.go | 13 ++- worker.go | 34 ++++--- worker_test.go | 10 +- 10 files changed, 249 insertions(+), 82 deletions(-) create mode 100644 database/pgsql/migrations/00008_add_multiplens.go diff --git a/api/v1/models.go b/api/v1/models.go index 866d71a5..b050b428 100644 --- a/api/v1/models.go +++ b/api/v1/models.go @@ -33,7 +33,7 @@ type Error struct { type Layer struct { Name string `json:"Name,omitempty"` - NamespaceName string `json:"NamespaceName,omitempty"` + NamespaceNames []string `json:"NamespaceNames,omitempty"` Path string `json:"Path,omitempty"` Headers map[string]string `json:"Headers,omitempty"` ParentName string `json:"ParentName,omitempty"` @@ -52,8 +52,8 @@ func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabil layer.ParentName = dbLayer.Parent.Name } - if dbLayer.Namespace != nil { - layer.NamespaceName = dbLayer.Namespace.Name + for _, ns := range dbLayer.Namespaces { + layer.NamespaceNames = append(layer.NamespaceNames, ns.Name) } if withFeatures || withVulnerabilities && dbLayer.Features != nil { diff --git a/database/models.go b/database/models.go index 2d645e6c..a793241f 100644 --- a/database/models.go +++ b/database/models.go @@ -31,7 +31,7 @@ type Layer struct { Name string EngineVersion int Parent *Layer - Namespace *Namespace + Namespaces []Namespace Features []FeatureVersion } diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go index 095e6307..64e9a475 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer.go @@ -52,9 +52,6 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo &layer.EngineVersion, &parentID, &parentName, - &nsID, - &nsName, - &nsVersionFormat, ) observeQueryTime("FindLayer", "searchLayer", t) @@ -68,11 +65,23 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo Name: parentName.String, } } - if !nsID.IsZero() { - layer.Namespace = &database.Namespace{ - Model: database.Model{ID: int(nsID.Int64)}, - Name: nsName.String, - VersionFormat: nsVersionFormat.String, + + rows, err := pgSQL.Query(searchLayerNamespace, layer.ID) + defer rows.Close() + if err != nil { + return layer, handleError("searchLayerNamespace", err) + } + for rows.Next() { + err = rows.Scan(&nsID, &nsName, &nsVersionFormat) + if err != nil { + return layer, handleError("searchLayerNamespace", err) + } + if !nsID.IsZero() { + layer.Namespaces = append(layer.Namespaces, database.Namespace{ + Model: database.Model{ID: int(nsID.Int64)}, + Name: nsName.String, + VersionFormat: nsVersionFormat.String, + }) } } @@ -277,18 +286,22 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { parentID = zero.IntFrom(int64(layer.Parent.ID)) } - // Find or insert namespace if provided. - var namespaceID zero.Int - if layer.Namespace != nil { - n, err := pgSQL.insertNamespace(*layer.Namespace) + // namespaceIDs will contain inherited and new namespaces + namespaceIDs := make(map[int]struct{}) + + // try to insert the new namespaces + for _, ns := range layer.Namespaces { + n, err := pgSQL.insertNamespace(ns) if err != nil { - return err + return handleError("pgSQL.insertNamespace", err) } - namespaceID = zero.IntFrom(int64(n)) - } else if layer.Namespace == nil && layer.Parent != nil { - // Import the Namespace from the parent if it has one and this layer doesn't specify one. - if layer.Parent.Namespace != nil { - namespaceID = zero.IntFrom(int64(layer.Parent.Namespace.ID)) + namespaceIDs[n] = struct{}{} + } + + // inherit namespaces from parent layer + if layer.Parent != nil { + for _, ns := range layer.Parent.Namespaces { + namespaceIDs[ns.ID] = struct{}{} } } @@ -301,7 +314,7 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { if layer.ID == 0 { // Insert a new layer. - err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID, namespaceID). + err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID). Scan(&layer.ID) if err != nil { tx.Rollback() @@ -315,12 +328,18 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { } } else { // Update an existing layer. - _, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion, namespaceID) + _, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion) if err != nil { tx.Rollback() return handleError("updateLayer", err) } + // replace the old namespace in the database + _, err := tx.Exec(removeLayerNamespace, layer.ID) + if err != nil { + tx.Rollback() + return handleError("removeLayerNamespace", err) + } // Remove all existing Layer_diff_FeatureVersion. _, err = tx.Exec(removeLayerDiffFeatureVersion, layer.ID) if err != nil { @@ -329,6 +348,30 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { } } + // insert the layer's namespaces + stmt, err := tx.Prepare(insertLayerNamespace) + + if err != nil { + tx.Rollback() + return handleError("failed to prepare statement", err) + } + + defer func() { + err = stmt.Close() + if err != nil { + tx.Rollback() + log.WithError(err).Error("failed to close prepared statement") + } + }() + + for nsid := range namespaceIDs { + _, err := stmt.Exec(layer.ID, nsid) + if err != nil { + tx.Rollback() + return handleError("insertLayerNamespace", err) + } + } + // Update Layer_diff_FeatureVersion now. err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer) if err != nil { diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer_test.go index 72ede4d8..6f35bbde 100644 --- a/database/pgsql/layer_test.go +++ b/database/pgsql/layer_test.go @@ -37,7 +37,7 @@ func TestFindLayer(t *testing.T) { layer, err := datastore.FindLayer("layer-0", false, false) if assert.Nil(t, err) && assert.NotNil(t, layer) { assert.Equal(t, "layer-0", layer.Name) - assert.Nil(t, layer.Namespace) + assert.Len(t, layer.Namespaces, 0) assert.Nil(t, layer.Parent) assert.Equal(t, 1, layer.EngineVersion) assert.Len(t, layer.Features, 0) @@ -52,7 +52,7 @@ func TestFindLayer(t *testing.T) { layer, err = datastore.FindLayer("layer-1", false, false) if assert.Nil(t, err) && assert.NotNil(t, layer) { assert.Equal(t, layer.Name, "layer-1") - assert.Equal(t, "debian:7", layer.Namespace.Name) + assertExpectedNamespaceName(t, &layer, []string{"debian:7"}) if assert.NotNil(t, layer.Parent) { assert.Equal(t, "layer-0", layer.Parent.Name) } @@ -100,6 +100,27 @@ func TestFindLayer(t *testing.T) { } } } + + // Testing Multiple namespaces layer-3b has debian:7 and debian:8 namespaces + layer, err = datastore.FindLayer("layer-3b", true, true) + + if assert.Nil(t, err) && assert.NotNil(t, layer) && assert.Len(t, layer.Features, 2) { + assert.Equal(t, "layer-3b", layer.Name) + // validate the namespace + assertExpectedNamespaceName(t, &layer, []string{"debian:7", "debian:8"}) + for _, featureVersion := range layer.Features { + switch featureVersion.Feature.Namespace.Name { + case "debian:7": + assert.Equal(t, "wechat", featureVersion.Feature.Name) + assert.Equal(t, "0.5", featureVersion.Version) + case "debian:8": + assert.Equal(t, "openssl", featureVersion.Feature.Name) + assert.Equal(t, "1.0", featureVersion.Version) + default: + t.Errorf("unexpected package %s for layer-3b", featureVersion.Feature.Name) + } + } + } } func TestInsertLayer(t *testing.T) { @@ -205,19 +226,19 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { { Name: "TestInsertLayer2", Parent: &database.Layer{Name: "TestInsertLayer1"}, - Namespace: &database.Namespace{ + Namespaces: []database.Namespace{database.Namespace{ Name: "TestInsertLayerNamespace1", VersionFormat: dpkg.ParserName, - }, + }}, }, // This layer changes the namespace and adds Features. { Name: "TestInsertLayer3", Parent: &database.Layer{Name: "TestInsertLayer2"}, - Namespace: &database.Namespace{ + Namespaces: []database.Namespace{database.Namespace{ Name: "TestInsertLayerNamespace2", VersionFormat: dpkg.ParserName, - }, + }}, Features: []database.FeatureVersion{f1, f2, f3}, }, // This layer covers the case where the last layer doesn't provide any new Feature. @@ -232,10 +253,10 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { { Name: "TestInsertLayer4b", Parent: &database.Layer{Name: "TestInsertLayer3"}, - Namespace: &database.Namespace{ + Namespaces: []database.Namespace{database.Namespace{ Name: "TestInsertLayerNamespace3", VersionFormat: dpkg.ParserName, - }, + }}, Features: []database.FeatureVersion{ // Deletes TestInsertLayerFeature1. // Keep TestInsertLayerFeature2 (old Namespace should be kept): @@ -264,10 +285,9 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { assert.Nil(t, err) } + // layer inherits all namespaces from its ancestries l4a := retrievedLayers["TestInsertLayer4a"] - if assert.NotNil(t, l4a.Namespace) { - assert.Equal(t, "TestInsertLayerNamespace2", l4a.Namespace.Name) - } + assertExpectedNamespaceName(t, &l4a, []string{"TestInsertLayerNamespace2", "TestInsertLayerNamespace1"}) assert.Len(t, l4a.Features, 3) for _, featureVersion := range l4a.Features { if cmpFV(featureVersion, f1) && cmpFV(featureVersion, f2) && cmpFV(featureVersion, f3) { @@ -276,9 +296,7 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { } l4b := retrievedLayers["TestInsertLayer4b"] - if assert.NotNil(t, l4b.Namespace) { - assert.Equal(t, "TestInsertLayerNamespace3", l4b.Namespace.Name) - } + assertExpectedNamespaceName(t, &l4b, []string{"TestInsertLayerNamespace1", "TestInsertLayerNamespace2", "TestInsertLayerNamespace3"}) assert.Len(t, l4b.Features, 3) for _, featureVersion := range l4b.Features { if cmpFV(featureVersion, f2) && cmpFV(featureVersion, f5) && cmpFV(featureVersion, f6) { @@ -303,10 +321,10 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { l3u := database.Layer{ Name: l3.Name, Parent: l3.Parent, - Namespace: &database.Namespace{ + Namespaces: []database.Namespace{database.Namespace{ Name: "TestInsertLayerNamespaceUpdated1", VersionFormat: dpkg.ParserName, - }, + }}, Features: []database.FeatureVersion{f7}, } @@ -323,7 +341,7 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { l3uf, err := datastore.FindLayer(l3u.Name, true, false) if assert.Nil(t, err) { - assert.Equal(t, l3.Namespace.Name, l3uf.Namespace.Name) + assertSameNamespaceName(t, &l3, &l3uf) assert.Equal(t, l3.EngineVersion, l3uf.EngineVersion) assert.Len(t, l3uf.Features, len(l3.Features)) } @@ -336,7 +354,7 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { l3uf, err = datastore.FindLayer(l3u.Name, true, false) if assert.Nil(t, err) { - assert.Equal(t, l3u.Namespace.Name, l3uf.Namespace.Name) + assertSameNamespaceName(t, &l3u, &l3uf) assert.Equal(t, l3u.EngineVersion, l3uf.EngineVersion) if assert.Len(t, l3uf.Features, 1) { assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l3uf.Features[0]) @@ -352,7 +370,7 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { l4uf, err := datastore.FindLayer(l3u.Name, true, false) if assert.Nil(t, err) { - assert.Equal(t, l3u.Namespace.Name, l4uf.Namespace.Name) + assertSameNamespaceName(t, &l3u, &l4uf) assert.Equal(t, l4u.EngineVersion, l4uf.EngineVersion) if assert.Len(t, l4uf.Features, 1) { assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l4uf.Features[0]) @@ -360,21 +378,56 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { } } +func assertSameNamespaceName(t *testing.T, layer1 *database.Layer, layer2 *database.Layer) { + assert.Len(t, compareStringLists(extractNamespaceName(layer1), extractNamespaceName(layer2)), 0) +} + +func assertExpectedNamespaceName(t *testing.T, layer *database.Layer, expectedNames []string) { + assert.Len(t, compareStringLists(extractNamespaceName(layer), expectedNames), 0) +} + +func extractNamespaceName(layer *database.Layer) []string { + slist := make([]string, 0, len(layer.Namespaces)) + for _, ns := range layer.Namespaces { + slist = append(slist, ns.Name) + } + return slist +} + func testInsertLayerDelete(t *testing.T, datastore database.Datastore) { err := datastore.DeleteLayer("TestInsertLayerX") assert.Equal(t, commonerr.ErrNotFound, err) + // ensure layer_namespace table is cleaned up once a layer is removed + layer3, err := datastore.FindLayer("TestInsertLayer3", false, false) + layer4a, err := datastore.FindLayer("TestInsertLayer4a", false, false) + layer4b, err := datastore.FindLayer("TestInsertLayer4b", false, false) + err = datastore.DeleteLayer("TestInsertLayer3") assert.Nil(t, err) _, err = datastore.FindLayer("TestInsertLayer3", false, false) assert.Equal(t, commonerr.ErrNotFound, err) - + assertNotInLayerNamespace(t, layer3.ID, datastore) _, err = datastore.FindLayer("TestInsertLayer4a", false, false) assert.Equal(t, commonerr.ErrNotFound, err) - + assertNotInLayerNamespace(t, layer4a.ID, datastore) _, err = datastore.FindLayer("TestInsertLayer4b", true, false) assert.Equal(t, commonerr.ErrNotFound, err) + assertNotInLayerNamespace(t, layer4b.ID, datastore) +} + +func assertNotInLayerNamespace(t *testing.T, layerID int, datastore database.Datastore) { + pg, ok := datastore.(*pgSQL) + if !assert.True(t, ok) { + return + } + tx, err := pg.Begin() + if !assert.Nil(t, err) { + return + } + rows, err := tx.Query(searchLayerNamespace, layerID) + assert.False(t, rows.Next()) } func cmpFV(a, b database.FeatureVersion) bool { diff --git a/database/pgsql/migrations/00008_add_multiplens.go b/database/pgsql/migrations/00008_add_multiplens.go new file mode 100644 index 00000000..ecfb4762 --- /dev/null +++ b/database/pgsql/migrations/00008_add_multiplens.go @@ -0,0 +1,44 @@ +// Copyright 2016 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrations + +import "github.com/remind101/migrate" + +func init() { + RegisterMigration(migrate.Migration{ + ID: 8, + Up: migrate.Queries([]string{ + // set on deletion, remove the corresponding rows in database + `CREATE TABLE IF NOT EXISTS Layer_Namespace( + id SERIAL PRIMARY KEY, + layer_id INT REFERENCES Layer(id) ON DELETE CASCADE, + namespace_id INT REFERENCES Namespace(id) ON DELETE CASCADE, + unique(layer_id, namespace_id) + );`, + `CREATE INDEX ON Layer_Namespace (namespace_id);`, + `CREATE INDEX ON Layer_Namespace (layer_id);`, + // move the namespace_id to the table + `INSERT INTO Layer_Namespace (layer_id, namespace_id) SELECT id, namespace_id FROM Layer;`, + // alter the Layer table to remove the column + `ALTER TABLE IF EXISTS Layer DROP namespace_id;`, + }), + Down: migrate.Queries([]string{ + `ALTER TABLE IF EXISTS Layer ADD namespace_id INT NULL REFERENCES Namespace;`, + `CREATE INDEX ON Layer (namespace_id);`, + `UPDATE IF EXISTS Layer SET namespace_id = (SELECT lns.namespace_id FROM Layer_Namespace lns WHERE Layer.id = lns.layer_id LIMIT 1);`, + `DROP TABLE IF EXISTS Layer_Namespace;`, + }), + }) +} diff --git a/database/pgsql/queries.go b/database/pgsql/queries.go index 9529c29c..3fedf8d0 100644 --- a/database/pgsql/queries.go +++ b/database/pgsql/queries.go @@ -77,12 +77,17 @@ const ( // layer.go searchLayer = ` - SELECT l.id, l.name, l.engineversion, p.id, p.name, n.id, n.name, n.version_format - FROM Layer l - LEFT JOIN Layer p ON l.parent_id = p.id - LEFT JOIN Namespace n ON l.namespace_id = n.id + SELECT l.id, l.name, l.engineversion, p.id, p.name + FROM Layer l + LEFT JOIN Layer p ON l.parent_id = p.id WHERE l.name = $1;` + searchLayerNamespace = ` + SELECT n.id, n.name, n.version_format + FROM Namespace n + JOIN Layer_Namespace lns ON lns.namespace_id = n.id + WHERE lns.layer_id = $1` + searchLayerFeatureVersion = ` WITH RECURSIVE layer_tree(id, name, parent_id, depth, path, cycle) AS( SELECT l.id, l.name, l.parent_id, 1, ARRAY[l.id], false @@ -113,11 +118,14 @@ const ( AND v.deleted_at IS NULL` insertLayer = ` - INSERT INTO Layer(name, engineversion, parent_id, namespace_id, created_at) - VALUES($1, $2, $3, $4, CURRENT_TIMESTAMP) - RETURNING id` + INSERT INTO Layer(name, engineversion, parent_id, created_at) + VALUES($1, $2, $3, CURRENT_TIMESTAMP) + RETURNING id` + + insertLayerNamespace = `INSERT INTO Layer_Namespace(layer_id, namespace_id) VALUES($1, $2)` + removeLayerNamespace = `DELETE FROM Layer_Namespace WHERE layer_id = $1` - updateLayer = `UPDATE LAYER SET engineversion = $2, namespace_id = $3 WHERE id = $1` + updateLayer = `UPDATE LAYER SET engineversion = $2 WHERE id = $1` removeLayerDiffFeatureVersion = ` DELETE FROM Layer_diff_FeatureVersion diff --git a/database/pgsql/testdata/data.sql b/database/pgsql/testdata/data.sql index 49a92985..fffe346c 100644 --- a/database/pgsql/testdata/data.sql +++ b/database/pgsql/testdata/data.sql @@ -28,12 +28,19 @@ INSERT INTO featureversion (id, feature_id, version) VALUES (3, 2, '2.0'), (4, 3, '1.0'); -INSERT INTO layer (id, name, engineversion, parent_id, namespace_id) VALUES - (1, 'layer-0', 1, NULL, NULL), - (2, 'layer-1', 1, 1, 1), - (3, 'layer-2', 1, 2, 1), - (4, 'layer-3a', 1, 3, 1), - (5, 'layer-3b', 1, 3, 2); +INSERT INTO layer (id, name, engineversion, parent_id) VALUES + (1, 'layer-0', 1, NULL), + (2, 'layer-1', 1, 1), + (3, 'layer-2', 1, 2), + (4, 'layer-3a', 1, 3), + (5, 'layer-3b', 1, 3); + +INSERT INTO layer_namespace (id, layer_id, namespace_id) VALUES +(1, 2, 1), +(2, 3, 1), +(3, 4, 1), +(4, 5, 2), +(5, 5, 1); INSERT INTO layer_diff_featureversion (id, layer_id, featureversion_id, modification) VALUES (1, 2, 1, 'add'), @@ -58,6 +65,7 @@ SELECT pg_catalog.setval(pg_get_serial_sequence('namespace', 'id'), (SELECT MAX( SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('featureversion', 'id'), (SELECT MAX(id) FROM featureversion)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('layer', 'id'), (SELECT MAX(id) FROM layer)+1); +SELECT pg_catalog.setval(pg_get_serial_sequence('layer_namespace', 'id'), (SELECT MAX(id) FROM layer_namespace)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('layer_diff_featureversion', 'id'), (SELECT MAX(id) FROM layer_diff_featureversion)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability', 'id'), (SELECT MAX(id) FROM vulnerability)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_fixedin_feature', 'id'), (SELECT MAX(id) FROM vulnerability_fixedin_feature)+1); diff --git a/ext/featurens/driver.go b/ext/featurens/driver.go index c0d221bd..e6cc18b0 100644 --- a/ext/featurens/driver.go +++ b/ext/featurens/driver.go @@ -68,26 +68,25 @@ func RegisterDetector(name string, d Detector) { detectors[name] = d } -// Detect iterators through all registered Detectors and returns the first -// non-nil detected namespace. -func Detect(files tarutil.FilesMap) (*database.Namespace, error) { +// Detect iterators through all registered Detectors and returns all non-nil detected namespaces +func Detect(files tarutil.FilesMap) ([]database.Namespace, error) { detectorsM.RLock() defer detectorsM.RUnlock() - + var namespaces []database.Namespace for name, detector := range detectors { namespace, err := detector.Detect(files) if err != nil { log.WithError(err).WithField("name", name).Warning("failed while attempting to detect namespace") - return nil, err + return []database.Namespace{}, err } if namespace != nil { log.WithFields(log.Fields{"name": name, "namespace": namespace.Name}).Debug("detected namespace") - return namespace, nil + namespaces = append(namespaces, *namespace) } } - return nil, nil + return namespaces, nil } // RequiredFilenames returns the total list of files required for all diff --git a/worker.go b/worker.go index 76e2605c..862f46d8 100644 --- a/worker.go +++ b/worker.go @@ -105,7 +105,7 @@ func ProcessLayer(datastore database.Datastore, imageFormat, name, parentName, p } // Analyze the content. - layer.Namespace, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent) + layer.Namespaces, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent) if err != nil { return err } @@ -115,7 +115,7 @@ func ProcessLayer(datastore database.Datastore, imageFormat, name, parentName, p // detectContent downloads a layer's archive and extracts its Namespace and // Features. -func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) { +func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespaces []database.Namespace, featureVersions []database.FeatureVersion, err error) { totalRequiredFiles := append(featurefmt.RequiredFilenames(), featurens.RequiredFilenames()...) files, err := imagefmt.Extract(imageFormat, path, headers, totalRequiredFiles) if err != nil { @@ -123,15 +123,20 @@ func detectContent(imageFormat, name, path string, headers map[string]string, pa return } - namespace, err = detectNamespace(name, files, parent) + namespaces, err = detectNamespaces(name, files, parent) if err != nil { return } // Detect features. - featureVersions, err = detectFeatureVersions(name, files, namespace, parent) - if err != nil { - return + var fv []database.FeatureVersion + // detect feature versions in all namespaces + for _, namespace := range namespaces { + fv, err = detectFeatureVersions(name, files, &namespace, parent) + if err != nil { + return + } + featureVersions = append(featureVersions, fv...) } if len(featureVersions) > 0 { log.WithFields(log.Fields{logLayerName: name, "feature count": len(featureVersions)}).Debug("detected features") @@ -140,23 +145,24 @@ func detectContent(imageFormat, name, path string, headers map[string]string, pa return } -func detectNamespace(name string, files tarutil.FilesMap, parent *database.Layer) (namespace *database.Namespace, err error) { - namespace, err = featurens.Detect(files) +func detectNamespaces(name string, files tarutil.FilesMap, parent *database.Layer) (namespaces []database.Namespace, err error) { + namespaces, err = featurens.Detect(files) if err != nil { return } - if namespace != nil { - log.WithFields(log.Fields{logLayerName: name, "detected namespace": namespace.Name}).Debug("detected namespace") + if len(namespaces) > 0 { + for _, ns := range namespaces { + log.WithFields(log.Fields{logLayerName: name, "detected namespace": ns.Name}).Debug("detected namespace") + } return } // Fallback to the parent's namespace. if parent != nil { - namespace = parent.Namespace - if namespace != nil { - log.WithFields(log.Fields{logLayerName: name, "detected namespace": namespace.Name}).Debug("detected namespace (from parent)") - return + for _, ns := range parent.Namespaces { + log.WithFields(log.Fields{logLayerName: name, "detected namespace": ns.Name}).Debug("detected namespace (from parent)") } + return } return diff --git a/worker_test.go b/worker_test.go index b0f14dbc..37c42f4c 100644 --- a/worker_test.go +++ b/worker_test.go @@ -85,7 +85,10 @@ func TestProcessWithDistUpgrade(t *testing.T) { // Ensure that the 'wheezy' layer has the expected namespace and features. wheezy, ok := datastore.layers["wheezy"] if assert.True(t, ok, "layer 'wheezy' not processed") { - assert.Equal(t, "debian:7", wheezy.Namespace.Name) + if !assert.Len(t, wheezy.Namespaces, 1) { + return + } + assert.Equal(t, "debian:7", wheezy.Namespaces[0].Name) assert.Len(t, wheezy.Features, 52) for _, nufv := range nonUpgradedFeatureVersions { @@ -98,7 +101,10 @@ func TestProcessWithDistUpgrade(t *testing.T) { // Ensure that the 'wheezy' layer has the expected namespace and non-upgraded features. jessie, ok := datastore.layers["jessie"] if assert.True(t, ok, "layer 'jessie' not processed") { - assert.Equal(t, "debian:8", jessie.Namespace.Name) + if !assert.Len(t, jessie.Namespaces, 1) { + return + } + assert.Equal(t, "debian:8", jessie.Namespaces[0].Name) assert.Len(t, jessie.Features, 74) for _, nufv := range nonUpgradedFeatureVersions {