From 9227a668eb625dfeafd4d582925fb1115265861c Mon Sep 17 00:00:00 2001 From: liang chenye Date: Thu, 21 Apr 2016 17:01:18 +0800 Subject: [PATCH] support multiple namespaces in one layer Signed-off-by: liang chenye --- api/v1/models.go | 6 +- database/models.go | 2 +- database/pgsql/layer.go | 58 ++++++++++-------- database/pgsql/layer_test.go | 53 +++++++++------- .../migrations/20151222113213_Initial.sql | 16 ++++- database/pgsql/namespace.go | 27 ++++++++ database/pgsql/queries.go | 29 ++++++--- database/pgsql/testdata/data.sql | 19 ++++-- worker/detectors/feature/dpkg/dpkg.go | 13 ++++ worker/detectors/feature/rpm/rpm.go | 13 ++++ worker/detectors/features.go | 21 +++++-- worker/detectors/namespace.go | 10 +-- worker/worker.go | 61 +++++++++---------- worker/worker_test.go | 4 +- 14 files changed, 221 insertions(+), 111 deletions(-) diff --git a/api/v1/models.go b/api/v1/models.go index f5498eb5..e800e7a7 100644 --- a/api/v1/models.go +++ b/api/v1/models.go @@ -35,7 +35,7 @@ type Error struct { type Layer struct { Name string `json:"Name,omitempty"` - NamespaceName string `json:"NamespaceName,omitempty"` + NamespacesName []string `json:"NamespacesName,omitempty"` Path string `json:"Path,omitempty"` ParentName string `json:"ParentName,omitempty"` Format string `json:"Format,omitempty"` @@ -53,8 +53,8 @@ func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabil layer.ParentName = dbLayer.Parent.Name } - if dbLayer.Namespace != nil { - layer.NamespaceName = dbLayer.Namespace.Name + for _, namespace := range dbLayer.Namespaces { + layer.NamespacesName = append(layer.NamespacesName, namespace.Name) } if withFeatures || withVulnerabilities && dbLayer.Features != nil { diff --git a/database/models.go b/database/models.go index a44291b8..b8386930 100644 --- a/database/models.go +++ b/database/models.go @@ -33,7 +33,7 @@ type Layer struct { Name string EngineVersion int Parent *Layer - Namespace *Namespace + Namespaces []Namespace Features []FeatureVersion } diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go index 71e5726a..1dad7c77 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer.go @@ -37,13 +37,10 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo var layer database.Layer var parentID zero.Int var parentName zero.String - var namespaceID zero.Int - var namespaceName sql.NullString t := time.Now() err := pgSQL.QueryRow(searchLayer, name). - Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentID, &parentName, &namespaceID, - &namespaceName) + Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentID, &parentName) observeQueryTime("FindLayer", "searchLayer", t) if err != nil { @@ -56,11 +53,25 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo Name: parentName.String, } } - if !namespaceID.IsZero() { - layer.Namespace = &database.Namespace{ - Model: database.Model{ID: int(namespaceID.Int64)}, - Name: namespaceName.String, + + // Find its namespaces + rows, err := pgSQL.Query(searchLayerNamespace, layer.ID) + if err != nil { + return layer, handleError("searchLayerNamespace", err) + } + defer rows.Close() + + for rows.Next() { + var namespace database.Namespace + + err = rows.Scan(&namespace.ID, &namespace.Name) + if err != nil { + return layer, handleError("searchLayerNamespace.Scan()", err) } + layer.Namespaces = append(layer.Namespaces, namespace) + } + if err = rows.Err(); err != nil { + return layer, handleError("searchLayerNamespace.Rows()", err) } // Find its features @@ -247,21 +258,6 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { parentID = zero.IntFrom(int64(layer.Parent.ID)) } - // Find or insert namespace if provided. - var namespaceID zero.Int - if layer.Namespace != nil { - n, err := pgSQL.insertNamespace(*layer.Namespace) - if err != nil { - return err - } - namespaceID = zero.IntFrom(int64(n)) - } else if layer.Namespace == nil && layer.Parent != nil { - // Import the Namespace from the parent if it has one and this layer doesn't specify one. - if layer.Parent.Namespace != nil { - namespaceID = zero.IntFrom(int64(layer.Parent.Namespace.ID)) - } - } - // Begin transaction. tx, err := pgSQL.Begin() if err != nil { @@ -271,7 +267,7 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { if layer.ID == 0 { // Insert a new layer. - err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID, namespaceID). + err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID). Scan(&layer.ID) if err != nil { tx.Rollback() @@ -285,7 +281,7 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { } } else { // Update an existing layer. - _, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion, namespaceID) + _, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion) if err != nil { tx.Rollback() return handleError("updateLayer", err) @@ -299,6 +295,18 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { } } + // Insert Namespace and Layer_Namespace + for _, namespace := range layer.Namespaces { + n, err := pgSQL.insertNamespace(namespace) + if err != nil { + return err + } + _, err = pgSQL.insertLayerNamespace(layer.ID, n) + if err != nil { + return err + } + } + // Update Layer_diff_FeatureVersion now. err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer) if err != nil { diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer_test.go index dec0384c..5f1731aa 100644 --- a/database/pgsql/layer_test.go +++ b/database/pgsql/layer_test.go @@ -36,9 +36,9 @@ func TestFindLayer(t *testing.T) { layer, err := datastore.FindLayer("layer-0", false, false) if assert.Nil(t, err) && assert.NotNil(t, layer) { assert.Equal(t, "layer-0", layer.Name) - assert.Nil(t, layer.Namespace) assert.Nil(t, layer.Parent) assert.Equal(t, 1, layer.EngineVersion) + assert.Len(t, layer.Namespaces, 0) assert.Len(t, layer.Features, 0) } @@ -51,11 +51,12 @@ func TestFindLayer(t *testing.T) { layer, err = datastore.FindLayer("layer-1", false, false) if assert.Nil(t, err) && assert.NotNil(t, layer) { assert.Equal(t, layer.Name, "layer-1") - assert.Equal(t, "debian:7", layer.Namespace.Name) if assert.NotNil(t, layer.Parent) { assert.Equal(t, "layer-0", layer.Parent.Name) } assert.Equal(t, 1, layer.EngineVersion) + assert.Len(t, layer.Namespaces, 1) + assert.Equal(t, "debian:7", layer.Namespaces[0].Name) assert.Len(t, layer.Features, 0) } @@ -184,16 +185,16 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { Name: "TestInsertLayer1", }, { - Name: "TestInsertLayer2", - Parent: &database.Layer{Name: "TestInsertLayer1"}, - Namespace: &database.Namespace{Name: "TestInsertLayerNamespace1"}, + Name: "TestInsertLayer2", + Parent: &database.Layer{Name: "TestInsertLayer1"}, + Namespaces: []database.Namespace{database.Namespace{Name: "TestInsertLayerNamespace1"}}, }, // This layer changes the namespace and adds Features. { - Name: "TestInsertLayer3", - Parent: &database.Layer{Name: "TestInsertLayer2"}, - Namespace: &database.Namespace{Name: "TestInsertLayerNamespace2"}, - Features: []database.FeatureVersion{f1, f2, f3}, + Name: "TestInsertLayer3", + Parent: &database.Layer{Name: "TestInsertLayer2"}, + Namespaces: []database.Namespace{database.Namespace{Name: "TestInsertLayerNamespace2"}}, + Features: []database.FeatureVersion{f1, f2, f3}, }, // This layer covers the case where the last layer doesn't provide any new Feature. { @@ -205,9 +206,9 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { // It also modifies the Namespace ("upgrade") but keeps some Features not upgraded, their // Namespaces should then remain unchanged. { - Name: "TestInsertLayer4b", - Parent: &database.Layer{Name: "TestInsertLayer3"}, - Namespace: &database.Namespace{Name: "TestInsertLayerNamespace3"}, + Name: "TestInsertLayer4b", + Parent: &database.Layer{Name: "TestInsertLayer3"}, + Namespaces: []database.Namespace{database.Namespace{Name: "TestInsertLayerNamespace3"}}, Features: []database.FeatureVersion{ // Deletes TestInsertLayerFeature1. // Keep TestInsertLayerFeature2 (old Namespace should be kept): @@ -237,8 +238,8 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { } l4a := retrievedLayers["TestInsertLayer4a"] - if assert.NotNil(t, l4a.Namespace) { - assert.Equal(t, "TestInsertLayerNamespace2", l4a.Namespace.Name) + if assert.Len(t, l4a.Namespaces, 1) { + assert.Equal(t, "TestInsertLayerNamespace2", l4a.Namespaces[0].Name) } assert.Len(t, l4a.Features, 3) for _, featureVersion := range l4a.Features { @@ -248,8 +249,8 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { } l4b := retrievedLayers["TestInsertLayer4b"] - if assert.NotNil(t, l4b.Namespace) { - assert.Equal(t, "TestInsertLayerNamespace3", l4b.Namespace.Name) + if assert.Len(t, l4b.Namespaces, 1) { + assert.Equal(t, "TestInsertLayerNamespace3", l4b.Namespaces[0].Name) } assert.Len(t, l4b.Features, 3) for _, featureVersion := range l4b.Features { @@ -270,10 +271,10 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { l3, _ := datastore.FindLayer("TestInsertLayer3", true, false) l3u := database.Layer{ - Name: l3.Name, - Parent: l3.Parent, - Namespace: &database.Namespace{Name: "TestInsertLayerNamespaceUpdated1"}, - Features: []database.FeatureVersion{f7}, + Name: l3.Name, + Parent: l3.Parent, + Namespaces: []database.Namespace{database.Namespace{Name: "TestInsertLayerNamespaceUpdated1"}}, + Features: []database.FeatureVersion{f7}, } l4u := database.Layer{ @@ -289,7 +290,9 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { l3uf, err := datastore.FindLayer(l3u.Name, true, false) if assert.Nil(t, err) { - assert.Equal(t, l3.Namespace.Name, l3uf.Namespace.Name) + if assert.Len(t, l3.Namespaces, 1) && assert.Len(t, l3uf.Namespaces, 1) { + assert.Equal(t, l3.Namespaces[0].Name, l3uf.Namespaces[0].Name) + } assert.Equal(t, l3.EngineVersion, l3uf.EngineVersion) assert.Len(t, l3uf.Features, len(l3.Features)) } @@ -302,7 +305,9 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { l3uf, err = datastore.FindLayer(l3u.Name, true, false) if assert.Nil(t, err) { - assert.Equal(t, l3u.Namespace.Name, l3uf.Namespace.Name) + if assert.Len(t, l3u.Namespaces, 1) && assert.Len(t, l3uf.Namespaces, 1) { + assert.Equal(t, l3u.Namespaces[0].Name, l3uf.Namespaces[0].Name) + } assert.Equal(t, l3u.EngineVersion, l3uf.EngineVersion) if assert.Len(t, l3uf.Features, 1) { assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l3uf.Features[0]) @@ -318,7 +323,9 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { l4uf, err := datastore.FindLayer(l3u.Name, true, false) if assert.Nil(t, err) { - assert.Equal(t, l3u.Namespace.Name, l4uf.Namespace.Name) + if assert.Len(t, l3u.Namespaces, 1) && assert.Len(t, l4uf.Namespaces, 1) { + assert.Equal(t, l3u.Namespaces[0].Name, l4uf.Namespaces[0].Name) + } assert.Equal(t, l4u.EngineVersion, l4uf.EngineVersion) if assert.Len(t, l4uf.Features, 1) { assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l4uf.Features[0]) diff --git a/database/pgsql/migrations/20151222113213_Initial.sql b/database/pgsql/migrations/20151222113213_Initial.sql index 7bb024d4..a0ecf3ea 100644 --- a/database/pgsql/migrations/20151222113213_Initial.sql +++ b/database/pgsql/migrations/20151222113213_Initial.sql @@ -30,11 +30,23 @@ CREATE TABLE IF NOT EXISTS Layer ( name VARCHAR(128) NOT NULL UNIQUE, engineversion SMALLINT NOT NULL, parent_id INT NULL REFERENCES Layer ON DELETE CASCADE, - namespace_id INT NULL REFERENCES Namespace, created_at TIMESTAMP WITH TIME ZONE); CREATE INDEX ON Layer (parent_id); -CREATE INDEX ON Layer (namespace_id); + + +-- ----------------------------------------------------- +-- Table Layer_Namespace +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS LayerNamespace ( + id SERIAL PRIMARY KEY, + layer_id INT NOT NULL REFERENCES Layer ON DELETE CASCADE, + namespace_id INT NOT NULL REFERENCES Namespace ON DELETE CASCADE, + + UNIQUE (layer_id, namespace_id)); + +CREATE INDEX ON LayerNamespace (layer_id); +CREATE INDEX ON LayerNamespace (layer_id, namespace_id); -- ----------------------------------------------------- diff --git a/database/pgsql/namespace.go b/database/pgsql/namespace.go index 3c85c784..dc0458bd 100644 --- a/database/pgsql/namespace.go +++ b/database/pgsql/namespace.go @@ -15,6 +15,7 @@ package pgsql import ( + "strconv" "time" "github.com/coreos/clair/database" @@ -50,6 +51,32 @@ func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) { return id, nil } +func (pgSQL *pgSQL) insertLayerNamespace(layerID int, namespaceID int) (int, error) { + if pgSQL.cache != nil { + promCacheQueriesTotal.WithLabelValues("layerNamespace").Inc() + if id, found := pgSQL.cache.Get("layer:" + strconv.Itoa(layerID) + "namespace:" + strconv.Itoa(namespaceID)); found { + promCacheHitsTotal.WithLabelValues("layerNamespace").Inc() + return id.(int), nil + } + } + + // We do `defer observeQueryTime` here because we don't want to observe cached layerNamespaces. + defer observeQueryTime("soiLayerNamespace", "all", time.Now()) + + var id int + err := pgSQL.QueryRow(soiLayerNamespace, layerID, namespaceID).Scan(&id) + if err != nil { + return 0, nil + return 0, handleError("soiLayerNamespace", err) + } + + if pgSQL.cache != nil { + pgSQL.cache.Add("layer:"+strconv.Itoa(layerID)+"namespace:"+strconv.Itoa(namespaceID), id) + } + + return id, nil +} + func (pgSQL *pgSQL) ListNamespaces() (namespaces []database.Namespace, err error) { rows, err := pgSQL.Query(listNamespace) if err != nil { diff --git a/database/pgsql/queries.go b/database/pgsql/queries.go index aba20d70..1a0696a0 100644 --- a/database/pgsql/queries.go +++ b/database/pgsql/queries.go @@ -77,10 +77,9 @@ const ( // layer.go searchLayer = ` - SELECT l.id, l.name, l.engineversion, p.id, p.name, n.id, n.name + SELECT l.id, l.name, l.engineversion, p.id, p.name FROM Layer l LEFT JOIN Layer p ON l.parent_id = p.id - LEFT JOIN Namespace n ON l.namespace_id = n.id WHERE l.name = $1;` searchLayerFeatureVersion = ` @@ -112,12 +111,17 @@ const ( AND v.namespace_id = vn.id AND v.deleted_at IS NULL` - insertLayer = ` - INSERT INTO Layer(name, engineversion, parent_id, namespace_id, created_at) - VALUES($1, $2, $3, $4, CURRENT_TIMESTAMP) - RETURNING id` + searchLayerNamespace = ` + SELECT n.id, n.name + From LayerNamespace ln, Namespace n + WHERE ln.layer_id = $1 AND ln.namespace_id = n.id` - updateLayer = `UPDATE LAYER SET engineversion = $2, namespace_id = $3 WHERE id = $1` + insertLayer = ` + INSERT INTO Layer(name, engineversion, parent_id, created_at) + VALUES($1, $2, $3, CURRENT_TIMESTAMP) + RETURNING id` + + updateLayer = `UPDATE LAYER SET engineversion = $2 WHERE id = $1` removeLayerDiffFeatureVersion = ` DELETE FROM Layer_diff_FeatureVersion @@ -131,6 +135,17 @@ const ( removeLayer = `DELETE FROM Layer WHERE name = $1` + soiLayerNamespace = ` + WITH new_layernamespace AS ( + INSERT INTO LayerNamespace(layer_id, namespace_id) + SELECT CAST($1 AS INTEGER), CAST($2 AS INTEGER) + WHERE NOT EXISTS (SELECT id FROM LayerNamespace WHERE layer_id = $1 AND namespace_id = $2) + RETURNING id + ) + SELECT id FROM LayerNamespace WHERE layer_id = $1 AND namespace_id = $2 + UNION + SELECT id FROM new_layernamespace` + // lock.go insertLock = `INSERT INTO Lock(name, owner, until) VALUES($1, $2, $3)` searchLock = `SELECT owner, until FROM Lock WHERE name = $1` diff --git a/database/pgsql/testdata/data.sql b/database/pgsql/testdata/data.sql index 7a48ef64..8cf9b9cf 100644 --- a/database/pgsql/testdata/data.sql +++ b/database/pgsql/testdata/data.sql @@ -28,12 +28,18 @@ INSERT INTO featureversion (id, feature_id, version) VALUES (3, 2, '2.0'), (4, 3, '1.0'); -INSERT INTO layer (id, name, engineversion, parent_id, namespace_id) VALUES - (1, 'layer-0', 1, NULL, NULL), - (2, 'layer-1', 1, 1, 1), - (3, 'layer-2', 1, 2, 1), - (4, 'layer-3a', 1, 3, 1), - (5, 'layer-3b', 1, 3, 2); +INSERT INTO layer (id, name, engineversion, parent_id) VALUES + (1, 'layer-0', 1, NULL), + (2, 'layer-1', 1, 1), + (3, 'layer-2', 1, 2), + (4, 'layer-3a', 1, 3), + (5, 'layer-3b', 1, 3); + +INSERT INTO layernamespace (id, layer_id, namespace_id) VALUES + (1, 2, 1), + (2, 3, 1), + (3, 4, 1), + (4, 5, 2); INSERT INTO layer_diff_featureversion (id, layer_id, featureversion_id, modification) VALUES (1, 2, 1, 'add'), @@ -58,6 +64,7 @@ SELECT pg_catalog.setval(pg_get_serial_sequence('namespace', 'id'), (SELECT MAX( SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('featureversion', 'id'), (SELECT MAX(id) FROM featureversion)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('layer', 'id'), (SELECT MAX(id) FROM layer)+1); +SELECT pg_catalog.setval(pg_get_serial_sequence('layernamespace', 'id'), (SELECT MAX(id) FROM layernamespace)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('layer_diff_featureversion', 'id'), (SELECT MAX(id) FROM layer_diff_featureversion)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability', 'id'), (SELECT MAX(id) FROM vulnerability)+1); SELECT pg_catalog.setval(pg_get_serial_sequence('vulnerability_fixedin_feature', 'id'), (SELECT MAX(id) FROM vulnerability_fixedin_feature)+1); diff --git a/worker/detectors/feature/dpkg/dpkg.go b/worker/detectors/feature/dpkg/dpkg.go index 6154d2ce..f3976366 100644 --- a/worker/detectors/feature/dpkg/dpkg.go +++ b/worker/detectors/feature/dpkg/dpkg.go @@ -113,3 +113,16 @@ func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database func (detector *DpkgFeaturesDetector) GetRequiredFiles() []string { return []string{"var/lib/dpkg/status"} } + +//Supported check if the input Namespace is supported by the underling detector +func (detector *DpkgFeaturesDetector) Supported(namespace database.Namespace) bool { + supports := []string{"centos", "red hat", "fedora"} + + for _, support := range supports { + if strings.HasPrefix(namespace.Name, support) { + return true + } + } + + return false +} diff --git a/worker/detectors/feature/rpm/rpm.go b/worker/detectors/feature/rpm/rpm.go index b5012639..a60f204c 100644 --- a/worker/detectors/feature/rpm/rpm.go +++ b/worker/detectors/feature/rpm/rpm.go @@ -118,3 +118,16 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database. func (detector *RpmFeaturesDetector) GetRequiredFiles() []string { return []string{"var/lib/rpm/Packages"} } + +//Supported check if the input Namespace is supported by the underling detector +func (detector *RpmFeaturesDetector) Supported(namespace database.Namespace) bool { + supports := []string{"ubuntu", "debian"} + + for _, support := range supports { + if strings.HasPrefix(namespace.Name, support) { + return true + } + } + + return false +} diff --git a/worker/detectors/features.go b/worker/detectors/features.go index da8d8c06..2e812b0b 100644 --- a/worker/detectors/features.go +++ b/worker/detectors/features.go @@ -28,6 +28,8 @@ type FeaturesDetector interface { // GetRequiredFiles returns the list of files required for Detect, without // leading /. GetRequiredFiles() []string + //Supported check if the input Namespace is supported by the underling detector + Supported(namespace database.Namespace) bool } var ( @@ -54,15 +56,24 @@ func RegisterFeaturesDetector(name string, f FeaturesDetector) { } // DetectFeatures detects a list of FeatureVersion using every registered FeaturesDetector. -func DetectFeatures(data map[string][]byte) ([]database.FeatureVersion, error) { +func DetectFeatures(data map[string][]byte, namespaces []database.Namespace) ([]database.FeatureVersion, error) { var packages []database.FeatureVersion for _, detector := range featuresDetectors { - pkgs, err := detector.Detect(data) - if err != nil { - return []database.FeatureVersion{}, err + for _, namespace := range namespaces { + if detector.Supported(namespace) { + pkgs, err := detector.Detect(data) + if err != nil { + return []database.FeatureVersion{}, err + } + // Ensure that every feature has a Namespace associated + for i := 0; i < len(pkgs); i++ { + pkgs[i].Feature.Namespace.Name = namespace.Name + } + packages = append(packages, pkgs...) + break + } } - packages = append(packages, pkgs...) } return packages, nil diff --git a/worker/detectors/namespace.go b/worker/detectors/namespace.go index 7d00cdfc..39fcc2a7 100644 --- a/worker/detectors/namespace.go +++ b/worker/detectors/namespace.go @@ -60,18 +60,18 @@ func RegisterNamespaceDetector(name string, f NamespaceDetector) { namespaceDetectors[name] = f } -// DetectNamespace finds the OS of the layer by using every registered NamespaceDetector. -func DetectNamespace(data map[string][]byte) *database.Namespace { +// DetectNamespaces finds the namespaces of the layer by using every registered NamespaceDetector. +func DetectNamespaces(data map[string][]byte) (namespaces []database.Namespace) { for _, detector := range namespaceDetectors { if namespace := detector.Detect(data); namespace != nil { - return namespace + namespaces = append(namespaces, *namespace) } } - return nil + return } -// GetRequiredFilesNamespace returns the list of files required for DetectNamespace for every +// GetRequiredFilesNamespace returns the list of files required for DetectNamespaces for every // registered NamespaceDetector, without leading /. func GetRequiredFilesNamespace() (files []string) { for _, detector := range namespaceDetectors { diff --git a/worker/worker.go b/worker/worker.go index 74824679..29a99fe2 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -46,7 +46,7 @@ var ( ErrParentUnknown = cerrors.NewBadRequestError("worker: parent layer is unknown, it must be processed first") ) -// Process detects the Namespace of a layer, the features it adds/removes, and +// Process detects the Namespaces of a layer, the features it adds/removes, and // then stores everything in the database. // TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an // older engine version and that processes them. @@ -104,7 +104,7 @@ func Process(datastore database.Datastore, name, parentName, path, imageFormat s } // Analyze the content. - layer.Namespace, layer.Features, err = detectContent(name, path, imageFormat, layer.Parent) + layer.Namespaces, layer.Features, err = detectContent(name, path, imageFormat, layer.Parent) if err != nil { return err } @@ -112,8 +112,8 @@ func Process(datastore database.Datastore, name, parentName, path, imageFormat s return datastore.InsertLayer(layer) } -// detectContent downloads a layer's archive and extracts its Namespace and Features. -func detectContent(name, path, imageFormat string, parent *database.Layer) (namespace *database.Namespace, features []database.FeatureVersion, err error) { +// detectContent downloads a layer's archive and extracts its Namespaces and Features. +func detectContent(name, path, imageFormat string, parent *database.Layer) (namespaces []database.Namespace, features []database.FeatureVersion, err error) { data, err := detectors.DetectData(path, imageFormat, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize) if err != nil { @@ -122,18 +122,21 @@ func detectContent(name, path, imageFormat string, parent *database.Layer) (name } // Detect namespace. - namespace, err = detectNamespace(data, parent) + namespaces, err = detectNamespaces(data, parent) if err != nil { return } - if namespace != nil { - log.Debugf("layer %s: Namespace is %s.", name, namespace.Name) + if len(namespaces) > 0 { + log.Debugf("layer %s: %d Namespaces detected: ", name, len(namespaces)) + for _, namespace := range namespaces { + log.Debugf("\t%s", namespace.Name) + } } else { - log.Debugf("layer %s: OS is unknown.", name) + log.Debugf("layer %s: Package System is unknown.", name) } // Detect features. - features, err = detectFeatures(name, data, namespace) + features, err = detectFeatures(name, data, namespaces) if err != nil { return } @@ -150,43 +153,37 @@ func detectContent(name, path, imageFormat string, parent *database.Layer) (name return } -func detectNamespace(data map[string][]byte, parent *database.Layer) (namespace *database.Namespace, err error) { - namespace = detectors.DetectNamespace(data) +func detectNamespaces(data map[string][]byte, parent *database.Layer) (namespaces []database.Namespace, err error) { + namespaces = detectors.DetectNamespaces(data) - // Attempt to detect the OS from the parent layer. - if namespace == nil && parent != nil { - namespace = parent.Namespace - if err != nil { - return + // Inherit the non-detected namespace from its parent + if parent != nil { + mapNamespaces := make(map[string]database.Namespace) + for _, n := range namespaces { + mapNamespaces[n.Name] = n + } + for _, pn := range parent.Namespaces { + mapNamespaces[pn.Name] = pn + } + + namespaces = []database.Namespace{} + for _, namespace := range mapNamespaces { + namespaces = append(namespaces, namespace) } } return } -func detectFeatures(name string, data map[string][]byte, namespace *database.Namespace) (features []database.FeatureVersion, err error) { +func detectFeatures(name string, data map[string][]byte, namespaces []database.Namespace) (features []database.FeatureVersion, err error) { // TODO(Quentin-M): We need to pass the parent image DetectFeatures because it's possible that // some detectors would need it in order to produce the entire feature list (if they can only // detect a diff). Also, we should probably pass the detected namespace so detectors could // make their own decision. - features, err = detectors.DetectFeatures(data) + features, err = detectors.DetectFeatures(data, namespaces) if err != nil { return } - // Ensure that every feature has a Namespace associated, otherwise associate the detected - // namespace. If there is no detected namespace, we'll throw an error. - for i := 0; i < len(features); i++ { - if features[i].Feature.Namespace.Name == "" { - if namespace != nil { - features[i].Feature.Namespace = *namespace - } else { - log.Warningf("layer %s: Layer's namespace is unknown but non-namespaced features have been detected", name) - err = ErrUnsupported - return - } - } - } - return } diff --git a/worker/worker_test.go b/worker/worker_test.go index b36a867d..d5465aeb 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -53,12 +53,12 @@ func TestProcessWithDistUpgrade(t *testing.T) { wheezy, err := datastore.FindLayer("wheezy", true, false) if assert.Nil(t, err) { - assert.Equal(t, "debian:7", wheezy.Namespace.Name) + assert.Equal(t, "debian:7", wheezy.Namespaces[0].Name) assert.Len(t, wheezy.Features, 52) jessie, err := datastore.FindLayer("jessie", true, false) if assert.Nil(t, err) { - assert.Equal(t, "debian:8", jessie.Namespace.Name) + assert.Equal(t, "debian:8", jessie.Namespaces[0].Name) assert.Len(t, jessie.Features, 74) // These FeatureVersions haven't been upgraded.