api/worker: introduce optional authorization

This allows clients to specify the contents of the HTTP Authorization
header so that Clair can access protected resources.

api/v1/README.md

@@ -59,6 +59,7 @@ Server: clair
 
 The POST route for the Layers resource performs the indexing of a Layer from the provided path and displays the provided Layer with an updated `IndexByVersion` property.
 This request blocks for the entire duration of the downloading and indexing of the layer.
+The Authorization field is an optional value whose contents will fill the Authorization HTTP Header when requesting the layer via HTTP.
 
 ###### Example Request
 
@@ -68,7 +69,8 @@ POST http://localhost:6060/v1/layers HTTP/1.1
 {
   "Layer": {
     "Name": "523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6",
-    "Path": "/mnt/layers/523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6/layer.tar",
+    "Path": "https://mystorage.com/layers/523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6/layer.tar",
+    "Authorization": "Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWV9.EkN-DOsnsuRjRO6BxXemmJDm3HbxrbRzXglbN2S4sOkopdU4IsDxTI8jO19W_A4K8ZPJijNLis4EZsHeY559a4DFOd50_OqgHGuERTqYZyuhtF39yxJPAjUESwxk2J5k_4zM3O-vtd1Ghyo4IbqKKSy6J9mTniYJPenn5-HIirE",
     "ParentName": "140f9bdfeb9784cf8730e9dab5dd12fbd704151cf555ac8cae650451794e5ac2",
     "Format": "Docker"
   }
@@ -85,7 +87,8 @@ Server: clair
 {
   "Layer": {
     "Name": "523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6",
-    "Path": "/mnt/layers/523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6/layer.tar",
+    "Path": "https://mystorage.com/layers/523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6/layer.tar",
+    "Authorization": "Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWV9.EkN-DOsnsuRjRO6BxXemmJDm3HbxrbRzXglbN2S4sOkopdU4IsDxTI8jO19W_A4K8ZPJijNLis4EZsHeY559a4DFOd50_OqgHGuERTqYZyuhtF39yxJPAjUESwxk2J5k_4zM3O-vtd1Ghyo4IbqKKSy6J9mTniYJPenn5-HIirE",
     "ParentName": "140f9bdfeb9784cf8730e9dab5dd12fbd704151cf555ac8cae650451794e5ac2",
     "Format": "Docker",
     "IndexedByVersion": 1

api/v1/models.go

@@ -37,6 +37,7 @@ type Layer struct {
 	Name             string    `json:"Name,omitempty"`
 	NamespaceName    string    `json:"NamespaceName,omitempty"`
 	Path             string    `json:"Path,omitempty"`
+	Authorization    string    `json:"Authorization,omitempty"`
 	ParentName       string    `json:"ParentName,omitempty"`
 	Format           string    `json:"Format,omitempty"`
 	IndexedByVersion int       `json:"IndexedByVersion,omitempty"`

api/v1/routes.go

@@ -109,7 +109,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx
 		return postLayerRoute, http.StatusBadRequest
 	}
 
-	err = worker.Process(ctx.Store, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Format)
+	err = worker.Process(ctx.Store, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Authorization, request.Layer.Format)
 	if err != nil {
 		if err == utils.ErrCouldNotExtract ||
 			err == utils.ErrExtractedFileTooBig ||
@@ -131,6 +131,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx
 		Name:             request.Layer.Name,
 		ParentName:       request.Layer.ParentName,
 		Path:             request.Layer.Path,
+		Authorization:    request.Layer.Authorization,
 		Format:           request.Layer.Format,
 		IndexedByVersion: worker.Version,
 	}})

worker/detectors/data.go

@@ -70,10 +70,17 @@ func RegisterDataDetector(name string, f DataDetector) {
 }
 
 // DetectData finds the Data of the layer by using every registered DataDetector
-func DetectData(path string, format string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) {
+func DetectData(path, authorization, format string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) {
 	var layerReader io.ReadCloser
 	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
-		r, err := http.Get(path)
+		request, err := http.NewRequest("GET", path, nil)
+		if err != nil {
+			return nil, ErrCouldNotFindLayer
+		}
+		if authorization != "" {
+			request.Header.Set("Authorization", authorization)
+		}
+		r, err := http.DefaultClient.Do(request)
 		if err != nil {
 			log.Warningf("could not download layer: %s", err)
 			return nil, ErrCouldNotFindLayer

worker/worker.go

@@ -50,7 +50,7 @@ var (
 // then stores everything in the database.
 // TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an
 // older engine version and that processes them.
-func Process(datastore database.Datastore, name, parentName, path, imageFormat string) error {
+func Process(datastore database.Datastore, name, parentName, path, authorization, imageFormat string) error {
 	// Verify parameters.
 	if name == "" {
 		return cerrors.NewBadRequestError("could not process a layer which does not have a name")
@@ -104,7 +104,7 @@ func Process(datastore database.Datastore, name, parentName, path, imageFormat s
 	}
 
 	// Analyze the content.
-	layer.Namespace, layer.Features, err = detectContent(name, path, imageFormat, layer.Parent)
+	layer.Namespace, layer.Features, err = detectContent(name, path, authorization, imageFormat, layer.Parent)
 	if err != nil {
 		return err
 	}
@@ -113,8 +113,8 @@ func Process(datastore database.Datastore, name, parentName, path, imageFormat s
 }
 
 // detectContent downloads a layer's archive and extracts its Namespace and Features.
-func detectContent(name, path, imageFormat string, parent *database.Layer) (namespace *database.Namespace, features []database.FeatureVersion, err error) {
+func detectContent(name, path, authorization, imageFormat string, parent *database.Layer) (namespace *database.Namespace, features []database.FeatureVersion, err error) {
-	data, err := detectors.DetectData(path, imageFormat, append(detectors.GetRequiredFilesFeatures(),
+	data, err := detectors.DetectData(path, authorization, imageFormat, append(detectors.GetRequiredFilesFeatures(),
 		detectors.GetRequiredFilesNamespace()...), maxFileSize)
 	if err != nil {
 		log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)

worker/worker_test.go

@@ -47,9 +47,9 @@ func TestProcessWithDistUpgrade(t *testing.T) {
 	// wheezy.tar: FROM debian:wheezy
 	// jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update &&
 	//             apt-get -y dist-upgrade
-	assert.Nil(t, Process(datastore, "blank", "", path+"blank.tar.gz", "Docker"))
+	assert.Nil(t, Process(datastore, "blank", "", path+"blank.tar.gz", "", "Docker"))
-	assert.Nil(t, Process(datastore, "wheezy", "blank", path+"wheezy.tar.gz", "Docker"))
+	assert.Nil(t, Process(datastore, "wheezy", "blank", path+"wheezy.tar.gz", "", "Docker"))
-	assert.Nil(t, Process(datastore, "jessie", "wheezy", path+"jessie.tar.gz", "Docker"))
+	assert.Nil(t, Process(datastore, "jessie", "wheezy", path+"jessie.tar.gz", "", "Docker"))
 
 	wheezy, err := datastore.FindLayer("wheezy", true, false)
 	if assert.Nil(t, err) {