diff --git a/api/v1/models.go b/api/v1/models.go index 43fa9b85..0d6a6383 100644 --- a/api/v1/models.go +++ b/api/v1/models.go @@ -34,14 +34,14 @@ type Error struct { } type Layer struct { - Name string `json:"Name,omitempty"` - NamespaceName string `json:"NamespaceName,omitempty"` - Path string `json:"Path,omitempty"` - Authorization string `json:"Authorization,omitempty"` - ParentName string `json:"ParentName,omitempty"` - Format string `json:"Format,omitempty"` - IndexedByVersion int `json:"IndexedByVersion,omitempty"` - Features []Feature `json:"Features,omitempty"` + Name string `json:"Name,omitempty"` + NamespaceName string `json:"NamespaceName,omitempty"` + Path string `json:"Path,omitempty"` + Headers map[string]string `json:"Headers,omitempty"` + ParentName string `json:"ParentName,omitempty"` + Format string `json:"Format,omitempty"` + IndexedByVersion int `json:"IndexedByVersion,omitempty"` + Features []Feature `json:"Features,omitempty"` } func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabilities bool) Layer { diff --git a/api/v1/routes.go b/api/v1/routes.go index 37562965..482840bf 100644 --- a/api/v1/routes.go +++ b/api/v1/routes.go @@ -109,7 +109,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx return postLayerRoute, http.StatusBadRequest } - err = worker.Process(ctx.Store, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Authorization, request.Layer.Format) + err = worker.Process(ctx.Store, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers) if err != nil { if err == utils.ErrCouldNotExtract || err == utils.ErrExtractedFileTooBig || @@ -131,7 +131,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx Name: request.Layer.Name, ParentName: request.Layer.ParentName, Path: request.Layer.Path, - Authorization: request.Layer.Authorization, + Headers: request.Layer.Headers, Format: request.Layer.Format, IndexedByVersion: worker.Version, }}) diff --git a/worker/detectors/data.go b/worker/detectors/data.go index afd5942d..376a2548 100644 --- a/worker/detectors/data.go +++ b/worker/detectors/data.go @@ -70,25 +70,35 @@ func RegisterDataDetector(name string, f DataDetector) { } // DetectData finds the Data of the layer by using every registered DataDetector -func DetectData(path, authorization, format string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) { +func DetectData(format, path string, headers map[string]string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) { var layerReader io.ReadCloser if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + // Create a new HTTP request object. request, err := http.NewRequest("GET", path, nil) if err != nil { return nil, ErrCouldNotFindLayer } - if authorization != "" { - request.Header.Set("Authorization", authorization) + + // Set any provided HTTP Headers. + if headers != nil { + for k, v := range headers { + request.Header.Set(k, v) + } } + + // Send the request and handle the response. r, err := http.DefaultClient.Do(request) if err != nil { log.Warningf("could not download layer: %s", err) return nil, ErrCouldNotFindLayer } + + // Fail if we don't receive a 2xx HTTP status code. if math.Floor(float64(r.StatusCode/100)) != 2 { log.Warningf("could not download layer: got status code %d, expected 2XX", r.StatusCode) return nil, ErrCouldNotFindLayer } + layerReader = r.Body } else { layerReader, err = os.Open(path) diff --git a/worker/worker.go b/worker/worker.go index 914d279a..de33c5cd 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -50,7 +50,7 @@ var ( // then stores everything in the database. // TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an // older engine version and that processes them. -func Process(datastore database.Datastore, name, parentName, path, authorization, imageFormat string) error { +func Process(datastore database.Datastore, imageFormat, name, parentName, path string, headers map[string]string) error { // Verify parameters. if name == "" { return cerrors.NewBadRequestError("could not process a layer which does not have a name") @@ -104,7 +104,7 @@ func Process(datastore database.Datastore, name, parentName, path, authorization } // Analyze the content. - layer.Namespace, layer.Features, err = detectContent(name, path, authorization, imageFormat, layer.Parent) + layer.Namespace, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent) if err != nil { return err } @@ -113,9 +113,8 @@ func Process(datastore database.Datastore, name, parentName, path, authorization } // detectContent downloads a layer's archive and extracts its Namespace and Features. -func detectContent(name, path, authorization, imageFormat string, parent *database.Layer) (namespace *database.Namespace, features []database.FeatureVersion, err error) { - data, err := detectors.DetectData(path, authorization, imageFormat, append(detectors.GetRequiredFilesFeatures(), - detectors.GetRequiredFilesNamespace()...), maxFileSize) +func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, features []database.FeatureVersion, err error) { + data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize) if err != nil { log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err) return diff --git a/worker/worker_test.go b/worker/worker_test.go index 9709ba8b..e499e841 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -47,9 +47,9 @@ func TestProcessWithDistUpgrade(t *testing.T) { // wheezy.tar: FROM debian:wheezy // jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update && // apt-get -y dist-upgrade - assert.Nil(t, Process(datastore, "blank", "", path+"blank.tar.gz", "", "Docker")) - assert.Nil(t, Process(datastore, "wheezy", "blank", path+"wheezy.tar.gz", "", "Docker")) - assert.Nil(t, Process(datastore, "jessie", "wheezy", path+"jessie.tar.gz", "", "Docker")) + assert.Nil(t, Process(datastore, "Docker", "blank", "", path+"blank.tar.gz", nil)) + assert.Nil(t, Process(datastore, "Docker", "wheezy", "blank", path+"wheezy.tar.gz", nil)) + assert.Nil(t, Process(datastore, "Docker", "jessie", "wheezy", path+"jessie.tar.gz", nil)) wheezy, err := datastore.FindLayer("wheezy", true, false) if assert.Nil(t, err) {