From 45d9d1d6a85a44c8c20cab25a5f88bb7d4a8b626 Mon Sep 17 00:00:00 2001 From: jgsqware Date: Wed, 8 Jun 2016 18:52:15 +0200 Subject: [PATCH] remove old pull/push method --- cmd/clairctl/cmd/analyze.go | 22 ++--- cmd/clairctl/cmd/pull.go | 60 +++---------- cmd/clairctl/cmd/push.go | 36 ++------ cmd/clairctl/cmd/report.go | 4 +- cmd/clairctl/config/config.go | 2 + cmd/clairctl/docker/image.go | 2 +- cmd/clairctl/docker/pull.go | 83 ------------------ cmd/clairctl/docker/push.go | 87 ------------------- cmd/clairctl/dockercli/dockercli.go | 75 ++++++++++++---- cmd/clairctl/dockerdist/dockerdist.go | 16 ++++ cmd/clairctl/dockerdist/push.go | 8 +- .../{docker => dockerdist}/push_test.go | 2 +- 12 files changed, 116 insertions(+), 281 deletions(-) delete mode 100644 cmd/clairctl/docker/pull.go delete mode 100644 cmd/clairctl/docker/push.go rename cmd/clairctl/{docker => dockerdist}/push_test.go (97%) diff --git a/cmd/clairctl/cmd/analyze.go b/cmd/clairctl/cmd/analyze.go index 39345bcf..b67c780b 100644 --- a/cmd/clairctl/cmd/analyze.go +++ b/cmd/clairctl/cmd/analyze.go @@ -45,17 +45,17 @@ func analyze(imageName string) clair.ImageAnalysis { var err error var image docker.Image - if !docker.IsLocal { - image, err = docker.Pull(imageName) + if !config.IsLocal { + // image, err = docker.Pull(imageName) - if err != nil { - if err == config.ErrLoginNotFound { - fmt.Println(err) - } else { - fmt.Println(errInternalError) - } - logrus.Fatalf("pulling image %q: %v", imageName, err) - } + // if err != nil { + // if err == config.ErrLoginNotFound { + // fmt.Println(err) + // } else { + // fmt.Println(errInternalError) + // } + // logrus.Fatalf("pulling image %q: %v", imageName, err) + // } } else { image, err = docker.Parse(imageName) @@ -75,5 +75,5 @@ func analyze(imageName string) clair.ImageAnalysis { func init() { RootCmd.AddCommand(analyzeCmd) - analyzeCmd.Flags().BoolVarP(&docker.IsLocal, "local", "l", false, "Use local images") + analyzeCmd.Flags().BoolVarP(&config.IsLocal, "local", "l", false, "Use local images") } diff --git a/cmd/clairctl/cmd/pull.go b/cmd/clairctl/cmd/pull.go index 2b1ff5fe..fb47ff1b 100644 --- a/cmd/clairctl/cmd/pull.go +++ b/cmd/clairctl/cmd/pull.go @@ -6,14 +6,12 @@ import ( "os" "github.com/Sirupsen/logrus" - "github.com/coreos/clair/cmd/clairctl/docker" + "github.com/coreos/clair/cmd/clairctl/config" + "github.com/coreos/clair/cmd/clairctl/dockercli" "github.com/coreos/clair/cmd/clairctl/dockerdist" - "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest/schema1" "github.com/docker/docker/reference" "github.com/spf13/cobra" - - dockercli "github.com/fsouza/go-dockerclient" ) const pullTplt = ` @@ -36,33 +34,22 @@ var pullCmd = &cobra.Command{ imageName := args[0] var manifest schema1.SignedManifest - var named reference.Named + var image reference.Named + var err error - if !docker.IsLocal { - n, m, err := dockerdist.DownloadManifest(imageName, true) + if !config.IsLocal { + image, manifest, err = dockerdist.DownloadV1Manifest(imageName, true) if err != nil { fmt.Println(errInternalError) - logrus.Fatalf("parsing image %q: %v", imageName, err) - } - // Ensure that the manifest type is supported. - switch m.(type) { - case *schema1.SignedManifest: - manifest = m.(schema1.SignedManifest) - named = n - break - - default: - fmt.Println(errInternalError) - logrus.Fatalf("only v1 manifests are currently supported") + logrus.Fatalf("retrieving manifest for %q: %v", imageName, err) } } else { - var err error - named, manifest, err = localManifest(imageName) + image, manifest, err = dockercli.GetLocalManifest(imageName, false) if err != nil { fmt.Println(errInternalError) - logrus.Fatalf("parsing image %q: %v", imageName, err) + logrus.Fatalf("retrieving local manifest for %q: %v", imageName, err) } } @@ -71,10 +58,10 @@ var pullCmd = &cobra.Command{ Named reference.Named }{ V1Manifest: manifest, - Named: named, + Named: image, } - err := template.Must(template.New("pull").Parse(pullTplt)).Execute(os.Stdout, data) + err = template.Must(template.New("pull").Parse(pullTplt)).Execute(os.Stdout, data) if err != nil { fmt.Println(errInternalError) logrus.Fatalf("rendering image: %v", err) @@ -82,30 +69,7 @@ var pullCmd = &cobra.Command{ }, } -func localManifest(imageName string) (reference.Named, schema1.SignedManifest, error) { - manifest := schema1.SignedManifest{} - // Parse the image name as a docker image reference. - named, err := reference.ParseNamed(imageName) - if err != nil { - return nil, manifest, err - } - - //TODO: use socket by default, but check for DOCKER_HOST env variable - endpoint := "unix:///var/run/docker.sock" - client, _ := dockercli.NewClient(endpoint) - histories, _ := client.ImageHistory(imageName) - for _, history := range histories { - var d digest.Digest - d, err := digest.ParseDigest(history.ID) - if err != nil { - return nil, manifest, err - } - manifest.FSLayers = append(manifest.FSLayers, schema1.FSLayer{BlobSum: d}) - } - return named, manifest, nil -} - func init() { RootCmd.AddCommand(pullCmd) - pullCmd.Flags().BoolVarP(&docker.IsLocal, "local", "l", false, "Use local images") + pullCmd.Flags().BoolVarP(&config.IsLocal, "local", "l", false, "Use local images") } diff --git a/cmd/clairctl/cmd/push.go b/cmd/clairctl/cmd/push.go index 6c032c1d..170d8c12 100644 --- a/cmd/clairctl/cmd/push.go +++ b/cmd/clairctl/cmd/push.go @@ -6,7 +6,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/coreos/clair/cmd/clairctl/config" - "github.com/coreos/clair/cmd/clairctl/docker" "github.com/coreos/clair/cmd/clairctl/dockercli" "github.com/coreos/clair/cmd/clairctl/dockerdist" "github.com/coreos/clair/cmd/clairctl/server" @@ -28,44 +27,27 @@ var pushCmd = &cobra.Command{ startLocalServer() imageName := args[0] + var manifest schema1.SignedManifest var image reference.Named - var manifest *schema1.SignedManifest + var err error - if !docker.IsLocal { - n, m, err := dockerdist.DownloadManifest(imageName, true) + if !config.IsLocal { + image, manifest, err = dockerdist.DownloadV1Manifest(imageName, true) if err != nil { fmt.Println(errInternalError) - logrus.Fatalf("parsing local image %q: %v", imageName, err) - } - // Ensure that the manifest type is supported. - switch m.(type) { - case *schema1.SignedManifest: - manifest = m.(*schema1.SignedManifest) - image = n - break - - default: - fmt.Println(errInternalError) - logrus.Fatalf("only v1 manifests are currently supported") + logrus.Fatalf("retrieving manifest for %q: %v", imageName, err) } } else { - n, err := reference.ParseNamed(imageName) + image, manifest, err = dockercli.GetLocalManifest(imageName, true) if err != nil { fmt.Println(errInternalError) - logrus.Fatalf("pushing image %q: %v", imageName, err) + logrus.Fatalf("retrieving local manifest for %q: %v", imageName, err) } - m, err := dockercli.Save(n) - if err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("saving image %q: %v", imageName, err) - } - manifest = m - image = n } - if err := dockerdist.Push(image, *manifest); err != nil { + if err := dockerdist.Push(image, manifest); err != nil { if err != nil { fmt.Println(errInternalError) logrus.Fatalf("pushing image %q: %v", imageName, err) @@ -90,5 +72,5 @@ func startLocalServer() { func init() { RootCmd.AddCommand(pushCmd) - pushCmd.Flags().BoolVarP(&docker.IsLocal, "local", "l", false, "Use local images") + pushCmd.Flags().BoolVarP(&config.IsLocal, "local", "l", false, "Use local images") } diff --git a/cmd/clairctl/cmd/report.go b/cmd/clairctl/cmd/report.go index 5b7841b6..10518a87 100644 --- a/cmd/clairctl/cmd/report.go +++ b/cmd/clairctl/cmd/report.go @@ -7,7 +7,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/coreos/clair/cmd/clairctl/clair" - "github.com/coreos/clair/cmd/clairctl/docker" + "github.com/coreos/clair/cmd/clairctl/config" "github.com/coreos/clair/cmd/clairctl/xstrings" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -81,7 +81,7 @@ func saveReport(name string, content string) error { func init() { RootCmd.AddCommand(reportCmd) - reportCmd.Flags().BoolVarP(&docker.IsLocal, "local", "l", false, "Use local images") + reportCmd.Flags().BoolVarP(&config.IsLocal, "local", "l", false, "Use local images") reportCmd.Flags().StringP("format", "f", "html", "Format for Report [html,json]") viper.BindPFlag("clair.report.format", reportCmd.Flags().Lookup("format")) } diff --git a/cmd/clairctl/config/config.go b/cmd/clairctl/config/config.go index 69d1a613..4a1e48e8 100644 --- a/cmd/clairctl/config/config.go +++ b/cmd/clairctl/config/config.go @@ -23,6 +23,8 @@ import ( var ErrLoginNotFound = errors.New("user is not log in") +var IsLocal = false + type reportConfig struct { Path, Format string } diff --git a/cmd/clairctl/docker/image.go b/cmd/clairctl/docker/image.go index 2d21cb10..1acfccd6 100644 --- a/cmd/clairctl/docker/image.go +++ b/cmd/clairctl/docker/image.go @@ -30,7 +30,7 @@ const dockerImageRegex = "^(?:([^/]+)/)?(?:([^/]+)/)?([^@:/]+)(?:[@:](.+))?" const DockerHub = "registry-1.docker.io" const hubURI = "https://" + DockerHub + "/v2" -var IsLocal = false +var isLocal = false func TmpLocal() string { return viper.GetString("clairctl.tempFolder") diff --git a/cmd/clairctl/docker/pull.go b/cmd/clairctl/docker/pull.go deleted file mode 100644 index 85e88ea4..00000000 --- a/cmd/clairctl/docker/pull.go +++ /dev/null @@ -1,83 +0,0 @@ -package docker - -import ( - "encoding/json" - "fmt" - "net/http" - - "github.com/Sirupsen/logrus" - "github.com/coreos/clair/cmd/clairctl/config" - "github.com/coreos/clair/cmd/clairctl/docker/httpclient" -) - -//Pull Image from Registry or Hub depending on image name -func Pull(imageName string) (Image, error) { - image, err := Parse(imageName) - if err != nil { - return Image{}, err - } - - logrus.Info("pulling image: ", image) - - mURI := fmt.Sprintf("%v/%v/manifests/%v", image.Registry, image.Name, image.Tag) - client := httpclient.Get() - request, err := http.NewRequest("GET", mURI, nil) - response, err := client.Do(request) - if err != nil { - return Image{}, fmt.Errorf("retrieving manifest: %v", err) - } - - if response.StatusCode == http.StatusUnauthorized { - logrus.Info("Pull is Unauthorized") - err := AuthenticateResponse(response, request) - - if err != nil { - return Image{}, fmt.Errorf("authenticating: %v", err) - } - response, err = client.Do(request) - if err != nil { - return Image{}, fmt.Errorf("retrieving manifest: %v", err) - } - } - - if response.StatusCode != 200 { - switch response.StatusCode { - case http.StatusUnauthorized: - return Image{}, ErrUnauthorized - case http.StatusNotFound: - return Image{}, config.ErrLoginNotFound - default: - return Image{}, fmt.Errorf("receiving http error: %d", response.StatusCode) - } - } - if err := image.parseManifest(response); err != nil { - return Image{}, fmt.Errorf("parsing manifest: %v", err) - } - - return image, nil -} - -func (image *Image) parseManifest(response *http.Response) error { - - err := json.NewDecoder(response.Body).Decode(&image) - - if err != nil { - return fmt.Errorf("reading manifest body: %v", err) - } - - image.uniqueLayers() - return nil -} - -func (image *Image) uniqueLayers() { - encountered := map[Layer]bool{} - result := []Layer{} - - for index := range image.FsLayers { - if encountered[image.FsLayers[index]] != true { - encountered[image.FsLayers[index]] = true - result = append(result, image.FsLayers[index]) - } - } - image.FsLayers = result -} diff --git a/cmd/clairctl/docker/push.go b/cmd/clairctl/docker/push.go deleted file mode 100644 index 4951e1c0..00000000 --- a/cmd/clairctl/docker/push.go +++ /dev/null @@ -1,87 +0,0 @@ -package docker - -import ( - "fmt" - "strings" - - "github.com/Sirupsen/logrus" - "github.com/coreos/clair/api/v1" - "github.com/coreos/clair/cmd/clairctl/clair" - "github.com/coreos/clair/cmd/clairctl/config" - "github.com/coreos/clair/cmd/clairctl/xstrings" -) - -var registryMapping map[string]string - -//Push image to Clair for analysis -func Push(image Image) error { - layerCount := len(image.FsLayers) - - parentID := "" - - if layerCount == 0 { - logrus.Warningln("there is no layer to push") - } - localIP, err := config.LocalServerIP() - if err != nil { - return err - } - hURL := fmt.Sprintf("http://%v/v2", localIP) - if IsLocal { - hURL += "/local" - logrus.Infof("using %v as local url", hURL) - } - - for index, layer := range image.FsLayers { - lUID := xstrings.Substr(layer.BlobSum, 0, 12) - logrus.Infof("Pushing Layer %d/%d [%v]", index+1, layerCount, lUID) - - insertRegistryMapping(layer.BlobSum, image.Registry) - payload := v1.LayerEnvelope{Layer: &v1.Layer{ - Name: layer.BlobSum, - Path: image.BlobsURI(layer.BlobSum), - ParentName: parentID, - Format: "Docker", - }} - - //FIXME Update to TLS - if IsLocal { - payload.Layer.Name = layer.History - payload.Layer.Path += "/layer.tar" - } - payload.Layer.Path = strings.Replace(payload.Layer.Path, image.Registry, hURL, 1) - if err := clair.Push(payload); err != nil { - logrus.Infof("adding layer %d/%d [%v]: %v", index+1, layerCount, lUID, err) - if err != clair.ErrUnanalizedLayer { - return err - } - parentID = "" - } else { - parentID = payload.Layer.Name - } - } - if IsLocal { - if err := CleanLocal(); err != nil { - return err - } - } - return nil -} - -func insertRegistryMapping(layerDigest string, registryURI string) { - logrus.Debugf("Saving %s[%s]", layerDigest, registryURI) - registryMapping[layerDigest] = registryURI -} - -//GetRegistryMapping return the registryURI corresponding to the layerID passed as parameter -func GetRegistryMapping(layerDigest string) (string, error) { - registryURI, present := registryMapping[layerDigest] - if !present { - return "", fmt.Errorf("%v mapping not found", layerDigest) - } - return registryURI, nil -} - -func init() { - registryMapping = map[string]string{} -} diff --git a/cmd/clairctl/dockercli/dockercli.go b/cmd/clairctl/dockercli/dockercli.go index 31ae255e..0cd6375a 100644 --- a/cmd/clairctl/dockercli/dockercli.go +++ b/cmd/clairctl/dockercli/dockercli.go @@ -20,22 +20,39 @@ import ( dockerclient "github.com/fsouza/go-dockerclient" ) -//Save local images to tmp folder -func Save(image reference.Named) (*schema1.SignedManifest, error) { +//GetLocalManifest retrieve manifest for local image +func GetLocalManifest(imageName string, withExport bool) (reference.Named, schema1.SignedManifest, error) { - imageName := image.Name() + image, err := reference.ParseNamed(imageName) + if err != nil { + return nil, schema1.SignedManifest{}, err + } + var manifest schema1.SignedManifest + if withExport { + manifest, err = save(image.Name()) + } else { + manifest, err = historyFromCommand(image.Name()) + } + + if err != nil { + return nil, schema1.SignedManifest{}, err + } + return image, manifest, err +} + +func save(imageName string) (schema1.SignedManifest, error) { path := config.TmpLocal() + "/" + strings.Split(imageName, ":")[0] + "/blobs" if _, err := os.Stat(path); os.IsExist(err) { err := os.RemoveAll(path) if err != nil { - return nil, err + return schema1.SignedManifest{}, err } } err := os.MkdirAll(path, 0755) if err != nil { - return nil, err + return schema1.SignedManifest{}, err } logrus.Debugln("docker image to save: ", imageName) @@ -44,7 +61,7 @@ func Save(image reference.Named) (*schema1.SignedManifest, error) { // open output file fo, err := os.Create(path + "/output.tar") if err != nil { - return nil, err + return schema1.SignedManifest{}, err } // close fo on exit and check for its returned error defer func() { @@ -55,28 +72,30 @@ func Save(image reference.Named) (*schema1.SignedManifest, error) { // make a write buffer w := bufio.NewWriter(fo) - endpoint := "unix:///var/run/docker.sock" - client, _ := dockerclient.NewClient(endpoint) + client, err := dockerclient.NewClientFromEnv() + if err != nil { + return schema1.SignedManifest{}, err + } err = client.ExportImage(dockerclient.ExportImageOptions{Name: imageName, OutputStream: w}) if err != nil { - return nil, err + return schema1.SignedManifest{}, err } err = openAndUntar(path+"/output.tar", path) if err != nil { - return nil, err + return schema1.SignedManifest{}, err } err = os.Remove(path + "/output.tar") if err != nil { - return nil, err + return schema1.SignedManifest{}, err } return historyFromManifest(path) } -func historyFromManifest(path string) (*schema1.SignedManifest, error) { +func historyFromManifest(path string) (schema1.SignedManifest, error) { mf, err := os.Open(path + "/manifest.json") if err != nil { - return nil, err + return schema1.SignedManifest{}, err } defer mf.Close() @@ -89,9 +108,9 @@ func historyFromManifest(path string) (*schema1.SignedManifest, error) { var manifest []manifestItem if err = json.NewDecoder(mf).Decode(&manifest); err != nil { - return nil, err + return schema1.SignedManifest{}, err } else if len(manifest) != 1 { - return nil, err + return schema1.SignedManifest{}, err } var layers []string for _, layer := range manifest[0].Layers { @@ -103,12 +122,34 @@ func historyFromManifest(path string) (*schema1.SignedManifest, error) { var d digest.Digest d, err := digest.ParseDigest("sha256:" + strings.TrimSuffix(layer, "/layer.tar")) if err != nil { - return nil, err + return schema1.SignedManifest{}, err } m.FSLayers = append(m.FSLayers, schema1.FSLayer{BlobSum: d}) } - return &m, nil + return m, nil +} + +func historyFromCommand(imageName string) (schema1.SignedManifest, error) { + client, err := dockerclient.NewClientFromEnv() + if err != nil { + return schema1.SignedManifest{}, err + } + histories, err := client.ImageHistory(imageName) + if err != nil { + return schema1.SignedManifest{}, err + } + + manifest := schema1.SignedManifest{} + for _, history := range histories { + var d digest.Digest + d, err := digest.ParseDigest(history.ID) + if err != nil { + return schema1.SignedManifest{}, err + } + manifest.FSLayers = append(manifest.FSLayers, schema1.FSLayer{BlobSum: d}) + } + return manifest, nil } func openAndUntar(name, dst string) error { diff --git a/cmd/clairctl/dockerdist/dockerdist.go b/cmd/clairctl/dockerdist/dockerdist.go index f675ff24..a86cf693 100644 --- a/cmd/clairctl/dockerdist/dockerdist.go +++ b/cmd/clairctl/dockerdist/dockerdist.go @@ -183,3 +183,19 @@ func DownloadManifest(image string, insecure bool) (reference.Named, distlib.Man return named, manifest, nil } + +// DownloadV1Manifest the manifest for the given image in v1 schema format, using the given credentials. +func DownloadV1Manifest(imageName string, insecure bool) (reference.Named, schema1.SignedManifest, error) { + image, manifest, err := DownloadManifest(imageName, insecure) + + if err != nil { + return nil, schema1.SignedManifest{}, err + } + // Ensure that the manifest type is supported. + switch manifest.(type) { + case *schema1.SignedManifest: + return image, *manifest.(*schema1.SignedManifest), nil + default: + return nil, schema1.SignedManifest{}, errors.New("only v1 manifests are currently supported") + } +} diff --git a/cmd/clairctl/dockerdist/push.go b/cmd/clairctl/dockerdist/push.go index 8a962be3..81a6dfb0 100644 --- a/cmd/clairctl/dockerdist/push.go +++ b/cmd/clairctl/dockerdist/push.go @@ -30,14 +30,14 @@ func Push(image reference.Named, manifest schema1.SignedManifest) error { return err } hURL := fmt.Sprintf("http://%v/v2", localIP) - if docker.IsLocal { + if config.IsLocal { hURL = strings.Replace(hURL, "/v2", "/local", -1) logrus.Infof("using %v as local url", hURL) } for index, layer := range manifest.FSLayers { blobsum := layer.BlobSum.String() - if docker.IsLocal { + if config.IsLocal { blobsum = strings.TrimPrefix(blobsum, "sha256:") } @@ -53,7 +53,7 @@ func Push(image reference.Named, manifest schema1.SignedManifest) error { }} //FIXME Update to TLS - if docker.IsLocal { + if config.IsLocal { payload.Layer.Path += "/layer.tar" } @@ -68,7 +68,7 @@ func Push(image reference.Named, manifest schema1.SignedManifest) error { parentID = payload.Layer.Name } } - if docker.IsLocal { + if config.IsLocal { if err := docker.CleanLocal(); err != nil { return err } diff --git a/cmd/clairctl/docker/push_test.go b/cmd/clairctl/dockerdist/push_test.go similarity index 97% rename from cmd/clairctl/docker/push_test.go rename to cmd/clairctl/dockerdist/push_test.go index bc734d1c..3e4cd4ef 100644 --- a/cmd/clairctl/docker/push_test.go +++ b/cmd/clairctl/dockerdist/push_test.go @@ -1,4 +1,4 @@ -package docker +package dockerdist import "testing"