api/worker: s/Authorization/Headers (#167)

This allows clients to specify any HTTP headers that need to be used in
order to allow Clair to download a layer, rather than just the
Authorization header.

api/v1/models.go

@@ -34,14 +34,14 @@ type Error struct {
 }
 
 type Layer struct {
-	Name             string    `json:"Name,omitempty"`
+	Name             string            `json:"Name,omitempty"`
-	NamespaceName    string    `json:"NamespaceName,omitempty"`
+	NamespaceName    string            `json:"NamespaceName,omitempty"`
-	Path             string    `json:"Path,omitempty"`
+	Path             string            `json:"Path,omitempty"`
-	Authorization    string    `json:"Authorization,omitempty"`
+	Headers          map[string]string `json:"Headers,omitempty"`
-	ParentName       string    `json:"ParentName,omitempty"`
+	ParentName       string            `json:"ParentName,omitempty"`
-	Format           string    `json:"Format,omitempty"`
+	Format           string            `json:"Format,omitempty"`
-	IndexedByVersion int       `json:"IndexedByVersion,omitempty"`
+	IndexedByVersion int               `json:"IndexedByVersion,omitempty"`
-	Features         []Feature `json:"Features,omitempty"`
+	Features         []Feature         `json:"Features,omitempty"`
 }
 
 func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabilities bool) Layer {

api/v1/routes.go

@@ -109,7 +109,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx
 		return postLayerRoute, http.StatusBadRequest
 	}
 
-	err = worker.Process(ctx.Store, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Authorization, request.Layer.Format)
+	err = worker.Process(ctx.Store, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers)
 	if err != nil {
 		if err == utils.ErrCouldNotExtract ||
 			err == utils.ErrExtractedFileTooBig ||
@@ -131,7 +131,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx
 		Name:             request.Layer.Name,
 		ParentName:       request.Layer.ParentName,
 		Path:             request.Layer.Path,
-		Authorization:    request.Layer.Authorization,
+		Headers:          request.Layer.Headers,
 		Format:           request.Layer.Format,
 		IndexedByVersion: worker.Version,
 	}})

worker/detectors/data.go

@@ -70,25 +70,35 @@ func RegisterDataDetector(name string, f DataDetector) {
 }
 
 // DetectData finds the Data of the layer by using every registered DataDetector
-func DetectData(path, authorization, format string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) {
+func DetectData(format, path string, headers map[string]string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) {
 	var layerReader io.ReadCloser
 	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
+		// Create a new HTTP request object.
 		request, err := http.NewRequest("GET", path, nil)
 		if err != nil {
 			return nil, ErrCouldNotFindLayer
 		}
-		if authorization != "" {
+
-			request.Header.Set("Authorization", authorization)
+		// Set any provided HTTP Headers.
+		if headers != nil {
+			for k, v := range headers {
+				request.Header.Set(k, v)
+			}
 		}
+
+		// Send the request and handle the response.
 		r, err := http.DefaultClient.Do(request)
 		if err != nil {
 			log.Warningf("could not download layer: %s", err)
 			return nil, ErrCouldNotFindLayer
 		}
+
+		// Fail if we don't receive a 2xx HTTP status code.
 		if math.Floor(float64(r.StatusCode/100)) != 2 {
 			log.Warningf("could not download layer: got status code %d, expected 2XX", r.StatusCode)
 			return nil, ErrCouldNotFindLayer
 		}
+
 		layerReader = r.Body
 	} else {
 		layerReader, err = os.Open(path)

worker/worker.go

@@ -50,7 +50,7 @@ var (
 // then stores everything in the database.
 // TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an
 // older engine version and that processes them.
-func Process(datastore database.Datastore, name, parentName, path, authorization, imageFormat string) error {
+func Process(datastore database.Datastore, imageFormat, name, parentName, path string, headers map[string]string) error {
 	// Verify parameters.
 	if name == "" {
 		return cerrors.NewBadRequestError("could not process a layer which does not have a name")
@@ -104,7 +104,7 @@ func Process(datastore database.Datastore, name, parentName, path, authorization
 	}
 
 	// Analyze the content.
-	layer.Namespace, layer.Features, err = detectContent(name, path, authorization, imageFormat, layer.Parent)
+	layer.Namespace, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent)
 	if err != nil {
 		return err
 	}
@@ -113,9 +113,8 @@ func Process(datastore database.Datastore, name, parentName, path, authorization
 }
 
 // detectContent downloads a layer's archive and extracts its Namespace and Features.
-func detectContent(name, path, authorization, imageFormat string, parent *database.Layer) (namespace *database.Namespace, features []database.FeatureVersion, err error) {
+func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, features []database.FeatureVersion, err error) {
-	data, err := detectors.DetectData(path, authorization, imageFormat, append(detectors.GetRequiredFilesFeatures(),
+	data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize)
-		detectors.GetRequiredFilesNamespace()...), maxFileSize)
 	if err != nil {
 		log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
 		return

worker/worker_test.go

@@ -47,9 +47,9 @@ func TestProcessWithDistUpgrade(t *testing.T) {
 	// wheezy.tar: FROM debian:wheezy
 	// jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update &&
 	//             apt-get -y dist-upgrade
-	assert.Nil(t, Process(datastore, "blank", "", path+"blank.tar.gz", "", "Docker"))
+	assert.Nil(t, Process(datastore, "Docker", "blank", "", path+"blank.tar.gz", nil))
-	assert.Nil(t, Process(datastore, "wheezy", "blank", path+"wheezy.tar.gz", "", "Docker"))
+	assert.Nil(t, Process(datastore, "Docker", "wheezy", "blank", path+"wheezy.tar.gz", nil))
-	assert.Nil(t, Process(datastore, "jessie", "wheezy", path+"jessie.tar.gz", "", "Docker"))
+	assert.Nil(t, Process(datastore, "Docker", "jessie", "wheezy", path+"jessie.tar.gz", nil))
 
 	wheezy, err := datastore.FindLayer("wheezy", true, false)
 	if assert.Nil(t, err) {