Merge e3ce454c13 into f8a1359a60

Documentation/api_v1.md

@@ -66,6 +66,7 @@ In order to analyze a container image, this route has to be called for each laye
 This request blocks for the entire duration of the downloading and indexing of the layer and displays the provided Layer with an updated `IndexByVersion` property.
 The Name field must be unique globally. Consequently, using the Blob digest describing the Layer content is not sufficient as Clair won't be able to differentiate two empty filesystem diffs that belong to two different image trees.
 The Authorization field is an optional value whose contents will fill the Authorization HTTP Header when requesting the layer via HTTP.
+The NamespaceName field is an optional value used when the namespace can't be determined by Clair.
 
 #### Example Request
 
@@ -82,7 +83,8 @@ POST http://localhost:6060/v1/layers HTTP/1.1
       "Authorization": "Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWV9.EkN-DOsnsuRjRO6BxXemmJDm3HbxrbRzXglbN2S4sOkopdU4IsDxTI8jO19W_A4K8ZPJijNLis4EZsHeY559a4DFOd50_OqgHGuERTqYZyuhtF39yxJPAjUESwxk2J5k_4zM3O-vtd1Ghyo4IbqKKSy6J9mTniYJPenn5-HIirE"
     },
     "ParentName": "140f9bdfeb9784cf8730e9dab5dd12fbd704151cf555ac8cae650451794e5ac2",
-    "Format": "Docker"
+    "Format": "Docker",
+    "NamespaceName": "debian:7"
   }
 }
 ```

api/v1/routes.go

@@ -109,7 +109,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx
 		return postLayerRoute, http.StatusBadRequest
 	}
 
-	err = clair.ProcessLayer(ctx.Store, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers)
+	err = clair.ProcessLayer(ctx.Store, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers, request.Layer.NamespaceName)
 	if err != nil {
 		if err == tarutil.ErrCouldNotExtract ||
 			err == tarutil.ErrExtractedFileTooBig ||

database/database.go

@@ -76,6 +76,9 @@ type Datastore interface {
 	// ListNamespaces returns the entire list of known Namespaces.
 	ListNamespaces() ([]Namespace, error)
 
+	//GetNamespace returns the namespace with name namespaceName
+	GetNamespace(namespaceName string) (*Namespace, error)
+
 	// InsertLayer stores a Layer in the database.
 	//
 	// A Layer is uniquely identified by its Name.

database/mock.go

@@ -20,6 +20,7 @@ import "time"
 // The default behavior of each method is to simply panic.
 type MockDatastore struct {
 	FctListNamespaces           func() ([]Namespace, error)
+	FctGetNamespace             func(namespaceName string) (*Namespace, error)
 	FctInsertLayer              func(Layer) error
 	FctFindLayer                func(name string, withFeatures, withVulnerabilities bool) (Layer, error)
 	FctDeleteLayer              func(name string) error
@@ -49,6 +50,13 @@ func (mds *MockDatastore) ListNamespaces() ([]Namespace, error) {
 	panic("required mock function not implemented")
 }
 
+func (mds *MockDatastore) GetNamespace(namespaceName string) (*Namespace, error) {
+	if mds.FctGetNamespace != nil {
+		return mds.FctGetNamespace(namespaceName)
+	}
+	panic("required mock function not implemented")
+}
+
 func (mds *MockDatastore) InsertLayer(layer Layer) error {
 	if mds.FctInsertLayer != nil {
 		return mds.FctInsertLayer(layer)

database/pgsql/namespace.go

@@ -73,3 +73,12 @@ func (pgSQL *pgSQL) ListNamespaces() (namespaces []database.Namespace, err error
 
 	return namespaces, err
 }
+
+func (pgSQL *pgSQL) GetNamespace(namespaceName string) (*database.Namespace, error) {
+	var ns database.Namespace
+	err := pgSQL.QueryRow(getNamespace, namespaceName).Scan(&ns.ID, &ns.Name, &ns.VersionFormat)
+	if err != nil {
+		return nil, handleError("getNamespace", err)
+	}
+	return &ns, nil
+}

database/pgsql/namespace_test.go

@@ -72,3 +72,22 @@ func TestListNamespace(t *testing.T) {
 		}
 	}
 }
+
+func TestGetNamespace(t *testing.T) {
+	datastore, err := openDatabaseForTest("GetNamespace", true)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer datastore.Close()
+
+	// Invalid Namespace.
+	ns0, err := datastore.GetNamespace("Not present")
+	assert.NotNil(t, err)
+	assert.Nil(t, ns0)
+
+	// Valid Namespace.
+	ns1, err := datastore.GetNamespace("debian:7")
+	assert.Nil(t, err)
+	assert.Equal(t, ns1.Name, "debian:7")
+}

database/pgsql/queries.go

@@ -40,6 +40,7 @@ const (
 
 	searchNamespace = `SELECT id FROM Namespace WHERE name = $1`
 	listNamespace   = `SELECT id, name, version_format FROM Namespace`
+	getNamespace    = `SELECT id, name, version_format FROM Namespace WHERE name = $1`
 
 	// feature.go
 	soiFeature = `

worker.go

@@ -56,7 +56,7 @@ func cleanURL(str string) string {
 //
 // TODO(Quentin-M): We could have a goroutine that looks for layers that have
 // been analyzed with an older engine version and that processes them.
-func ProcessLayer(datastore database.Datastore, imageFormat, name, parentName, path string, headers map[string]string) error {
+func ProcessLayer(datastore database.Datastore, imageFormat, name, parentName, path string, headers map[string]string, namespaceName string) error {
 	// Verify parameters.
 	if name == "" {
 		return commonerr.NewBadRequestError("could not process a layer which does not have a name")
@@ -104,8 +104,19 @@ func ProcessLayer(datastore database.Datastore, imageFormat, name, parentName, p
 		log.WithFields(log.Fields{logLayerName: name, "past engine version": layer.EngineVersion, "current engine version": Version}).Debug("layer content has already been processed in the past with older engine. analyzing again")
 	}
 
+	// Check to see if the user's provided Namespace exists
+	var namespace *database.Namespace
+	if namespaceName != "" {
+		namespace, err = datastore.GetNamespace(namespaceName)
+		if err != nil {
+			if err == commonerr.ErrNotFound {
+				return commonerr.NewBadRequestError("could not find provided namespace")
+			}
+			return err
+		}
+	}
 	// Analyze the content.
-	layer.Namespace, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent)
+	layer.Namespace, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent, namespace)
 	if err != nil {
 		return err
 	}
@@ -115,7 +126,7 @@ func ProcessLayer(datastore database.Datastore, imageFormat, name, parentName, p
 
 // detectContent downloads a layer's archive and extracts its Namespace and
 // Features.
-func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) {
+func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer, ns *database.Namespace) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) {
 	totalRequiredFiles := append(featurefmt.RequiredFilenames(), featurens.RequiredFilenames()...)
 	files, err := imagefmt.Extract(imageFormat, path, headers, totalRequiredFiles)
 	if err != nil {
@@ -128,6 +139,11 @@ func detectContent(imageFormat, name, path string, headers map[string]string, pa
 		return
 	}
 
+	if namespace == nil && ns != nil {
+		log.WithFields(log.Fields{logLayerName: name, "namespace provided by user": ns.Name}).Debug("detected namespace")
+		namespace = ns
+	}
+
 	// Detect features.
 	featureVersions, err = detectFeatureVersions(name, files, namespace, parent)
 	if err != nil {

worker_test.go

@@ -34,12 +34,14 @@ import (
 
 type mockDatastore struct {
 	database.MockDatastore
-	layers map[string]database.Layer
+	layers     map[string]database.Layer
+	namespaces map[string]database.Namespace
 }
 
 func newMockDatastore() *mockDatastore {
 	return &mockDatastore{
-		layers: make(map[string]database.Layer),
+		layers:     make(map[string]database.Layer),
+		namespaces: make(map[string]database.Namespace),
 	}
 }
 
@@ -59,6 +61,15 @@ func TestProcessWithDistUpgrade(t *testing.T) {
 		}
 		return database.Layer{}, commonerr.ErrNotFound
 	}
+	datastore.FctGetNamespace = func(namespaceName string) (*database.Namespace, error) {
+		if namespace, exists := datastore.namespaces[namespaceName]; exists {
+			return &namespace, nil
+		}
+		return nil, commonerr.ErrNotFound
+	}
+
+	// Add Namespace
+	datastore.namespaces["debian:7"] = database.Namespace{Name: "debian:7"}
 
 	// Create the list of FeatureVersions that should not been upgraded from one layer to another.
 	nonUpgradedFeatureVersions := []database.FeatureVersion{
@@ -78,9 +89,17 @@ func TestProcessWithDistUpgrade(t *testing.T) {
 	// wheezy.tar: FROM debian:wheezy
 	// jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update &&
 	//             apt-get -y dist-upgrade
-	assert.Nil(t, ProcessLayer(datastore, "Docker", "blank", "", testDataPath+"blank.tar.gz", nil))
-	assert.Nil(t, ProcessLayer(datastore, "Docker", "wheezy", "blank", testDataPath+"wheezy.tar.gz", nil))
-	assert.Nil(t, ProcessLayer(datastore, "Docker", "jessie", "wheezy", testDataPath+"jessie.tar.gz", nil))
+	assert.Nil(t, ProcessLayer(datastore, "Docker", "blank", "", testDataPath+"blank.tar.gz", nil, ""))
+	assert.Nil(t, ProcessLayer(datastore, "Docker", "blankWithNamespace", "", testDataPath+"blank.tar.gz", nil, "debian:7"))
+	assert.Nil(t, ProcessLayer(datastore, "Docker", "wheezy", "blank", testDataPath+"wheezy.tar.gz", nil, ""))
+	assert.Nil(t, ProcessLayer(datastore, "Docker", "jessie", "wheezy", testDataPath+"jessie.tar.gz", nil, ""))
+
+	// Ensure that the 'blankWithNamespace' layer has the provided namespace
+	blankWithNamespace, ok := datastore.layers["blankWithNamespace"]
+	if assert.True(t, ok, "layer 'blankWithNamespace' not processed") {
+		assert.Equal(t, "debian:7", blankWithNamespace.Namespace.Name)
+		assert.Len(t, blankWithNamespace.Features, 0)
+	}
 
 	// Ensure that the 'wheezy' layer has the expected namespace and features.
 	wheezy, ok := datastore.layers["wheezy"]