From 82cdeb237109d6d6c865218362687970e113112b Mon Sep 17 00:00:00 2001 From: jgsqware Date: Tue, 7 Jun 2016 23:30:36 +0200 Subject: [PATCH] start push --- cmd/clairctl/cmd/pull-docker.go | 74 ++++++---- cmd/clairctl/cmd/push-docker.go | 177 +++++++++++++++++++++-- cmd/clairctl/config/config.go | 4 + glide.yaml | 2 + vendor/github.com/fsouza/go-dockerclient | 1 + vendor/github.com/mholt/archiver | 1 + 6 files changed, 217 insertions(+), 42 deletions(-) create mode 160000 vendor/github.com/fsouza/go-dockerclient create mode 160000 vendor/github.com/mholt/archiver diff --git a/cmd/clairctl/cmd/pull-docker.go b/cmd/clairctl/cmd/pull-docker.go index d89590bd..726f09cc 100644 --- a/cmd/clairctl/cmd/pull-docker.go +++ b/cmd/clairctl/cmd/pull-docker.go @@ -8,11 +8,12 @@ import ( "github.com/Sirupsen/logrus" "github.com/coreos/clair/cmd/clairctl/docker" "github.com/coreos/clair/cmd/clairctl/dockerdist" + "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest/schema1" "github.com/docker/docker/reference" - "github.com/docker/engine-api/client" - "github.com/docker/engine-api/types" "github.com/spf13/cobra" + + dockercli "github.com/fsouza/go-dockerclient" ) const pullDockerTplt = ` @@ -34,60 +35,75 @@ var pullDockerCmd = &cobra.Command{ } imageName := args[0] + var manifest schema1.SignedManifest + var named reference.Named if !docker.IsLocal { - n, manifest, err := dockerdist.DownloadManifest(imageName, true) + n, m, err := dockerdist.DownloadManifest(imageName, true) if err != nil { fmt.Println(errInternalError) logrus.Fatalf("parsing image %q: %v", imageName, err) } // Ensure that the manifest type is supported. - switch manifest.(type) { + switch m.(type) { case *schema1.SignedManifest: + manifest = m.(schema1.SignedManifest) + named = n break default: fmt.Println(errInternalError) logrus.Fatalf("only v1 manifests are currently supported") } - data := struct { - V1Manifest *schema1.SignedManifest - Named reference.Named - }{ - V1Manifest: manifest.(*schema1.SignedManifest), - Named: n, - } - err = template.Must(template.New("pull").Parse(pullDockerTplt)).Execute(os.Stdout, data) + } else { + var err error + named, manifest, err = LocalManifest(imageName) if err != nil { fmt.Println(errInternalError) - logrus.Fatalf("rendering image: %v", err) + logrus.Fatalf("parsing image %q: %v", imageName, err) } - } else { - localManifest(imageName) + } + + data := struct { + V1Manifest schema1.SignedManifest + Named reference.Named + }{ + V1Manifest: manifest, + Named: named, + } + + err := template.Must(template.New("pull").Parse(pullDockerTplt)).Execute(os.Stdout, data) + if err != nil { + fmt.Println(errInternalError) + logrus.Fatalf("rendering image: %v", err) } }, } -func localManifest(imageName string) { - defaultHeaders := map[string]string{"User-Agent": "engine-api-cli-1.0"} - cli, err := client.NewClient("unix:///var/run/docker.sock", "v1.22", nil, defaultHeaders) +func LocalManifest(imageName string) (reference.Named, schema1.SignedManifest, error) { + manifest := schema1.SignedManifest{} + // Parse the image name as a docker image reference. + named, err := reference.ParseNamed(imageName) if err != nil { - panic(err) - } - t := types.ImageListOptions{MatchName: imageName} - cli.ImageList(t) - // histories, err := cli.ImageHistory(context.Background(), imageName) - - if err != nil { - panic(err) + return nil, manifest, err } - // for _, history := range histories { - // fmt.Println(history) - // } + endpoint := "unix:///var/run/docker.sock" + client, _ := dockercli.NewClient(endpoint) + histories, _ := client.ImageHistory(imageName) + for _, history := range histories { + var d digest.Digest + d, err := digest.ParseDigest(history.ID) + if err != nil { + return nil, manifest, err + } + manifest.FSLayers = append(manifest.FSLayers, schema1.FSLayer{BlobSum: d}) + } + return named, manifest, nil } + func init() { RootCmd.AddCommand(pullDockerCmd) pullDockerCmd.Flags().BoolVarP(&docker.IsLocal, "local", "l", false, "Use local images") diff --git a/cmd/clairctl/cmd/push-docker.go b/cmd/clairctl/cmd/push-docker.go index 769c5dce..6620b26f 100644 --- a/cmd/clairctl/cmd/push-docker.go +++ b/cmd/clairctl/cmd/push-docker.go @@ -1,14 +1,27 @@ package cmd import ( + "bufio" + "compress/bzip2" + "compress/gzip" + "encoding/json" "fmt" + "io" "os" + "strings" + "syscall" "github.com/Sirupsen/logrus" + "github.com/artyom/untar" + "github.com/coreos/clair/cmd/clairctl/config" "github.com/coreos/clair/cmd/clairctl/docker" "github.com/coreos/clair/cmd/clairctl/dockerdist" + "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest/schema1" + "github.com/docker/docker/reference" "github.com/spf13/cobra" + + dockercli "github.com/fsouza/go-dockerclient" ) var pushDockerCmd = &cobra.Command{ @@ -24,7 +37,6 @@ var pushDockerCmd = &cobra.Command{ startLocalServer() imageName := args[0] - var image docker.Image if !docker.IsLocal { image, manifest, err := dockerdist.DownloadManifest(imageName, true) @@ -52,31 +64,170 @@ var pushDockerCmd = &cobra.Command{ } } else { - var err error - image, err = docker.Parse(imageName) + named, err := reference.ParseNamed(imageName) if err != nil { fmt.Println(errInternalError) - logrus.Fatalf("parsing local image %q: %v", imageName, err) + logrus.Fatalf("pushing image %q: %v", imageName, err) } - err = docker.Prepare(&image) - logrus.Debugf("prepared image layers: %d", len(image.FsLayers)) + p, err := save(named) if err != nil { fmt.Println(errInternalError) - logrus.Fatalf("preparing local image %q from history: %v", imageName, err) + logrus.Fatalf("saving image %q: %v", imageName, err) } - logrus.Info("Pushing Image [OLD WAY] should be deprecated") - if err := docker.Push(image); err != nil { - if err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("pushing image %q: %v", imageName, err) - } + m, err := historyFromManifest(p) + if err != nil { + fmt.Println(errInternalError) + logrus.Fatalf("reading manifest %q: %v", imageName, err) } + + for _, layer := range m.FSLayers { + fmt.Println("ID: ", layer.BlobSum.String()) + } + // var err error + // image, err = docker.Parse(imageName) + // if err != nil { + // fmt.Println(errInternalError) + // logrus.Fatalf("parsing local image %q: %v", imageName, err) + // } + // err = docker.Prepare(&image) + // logrus.Debugf("prepared image layers: %d", len(image.FsLayers)) + // if err != nil { + // fmt.Println(errInternalError) + // logrus.Fatalf("preparing local image %q from history: %v", imageName, err) + // } + // logrus.Info("Pushing Image [OLD WAY] should be deprecated") + // if err := docker.Push(image); err != nil { + // if err != nil { + // fmt.Println(errInternalError) + // logrus.Fatalf("pushing image %q: %v", imageName, err) + // } + // } } fmt.Printf("%v has been pushed to Clair\n", imageName) }, } +func historyFromManifest(path string) (*schema1.SignedManifest, error) { + mf, err := os.Open(path + "/manifest.json") + if err != nil { + return nil, err + } + defer mf.Close() + + // https://github.com/docker/docker/blob/master/image/tarexport/tarexport.go#L17 + type manifestItem struct { + Config string + RepoTags []string + Layers []string + } + + var manifest []manifestItem + if err = json.NewDecoder(mf).Decode(&manifest); err != nil { + return nil, err + } else if len(manifest) != 1 { + return nil, err + } + var layers []string + for _, layer := range manifest[0].Layers { + layers = append(layers, strings.TrimSuffix(layer, "/layer.tar")) + } + var m schema1.SignedManifest + + for _, layer := range manifest[0].Layers { + var d digest.Digest + fmt.Println(strings.TrimSuffix(layer, "/layer.tar")) + d, err := digest.ParseDigest("sha256:" + strings.TrimSuffix(layer, "/layer.tar")) + if err != nil { + return nil, err + } + m.FSLayers = append(m.FSLayers, schema1.FSLayer{BlobSum: d}) + } + + return &m, nil +} + +func save(image reference.Named) (string, error) { + + imageName := image.Name() + path := config.TmpLocal() + "/" + strings.Split(imageName, ":")[0] + "/blobs" + + if _, err := os.Stat(path); os.IsExist(err) { + err := os.RemoveAll(path) + if err != nil { + return "", err + } + } + + err := os.MkdirAll(path, 0755) + if err != nil { + return "", err + } + + logrus.Debugln("docker image to save: ", imageName) + logrus.Debugln("saving in: ", path) + + // open output file + fo, err := os.Create(path + "/output.tar") + if err != nil { + return "", err + } + // close fo on exit and check for its returned error + defer func() { + if err := fo.Close(); err != nil { + panic(err) + } + }() + // make a write buffer + w := bufio.NewWriter(fo) + + endpoint := "unix:///var/run/docker.sock" + client, _ := dockercli.NewClient(endpoint) + err = client.ExportImage(dockercli.ExportImageOptions{Name: imageName, OutputStream: w}) + if err != nil { + return "", err + } + err = openAndUntar(path+"/output.tar", path) + if err != nil { + return "", err + } + + err = os.Remove(path + "/output.tar") + if err != nil { + return "", err + } + return path, err +} + +func openAndUntar(name, dst string) error { + var rd io.Reader + f, err := os.Open(name) + if err != nil { + return err + } + defer f.Close() + rd = f + if strings.HasSuffix(name, ".gz") || strings.HasSuffix(name, ".tgz") { + gr, err := gzip.NewReader(f) + if err != nil { + return err + } + defer gr.Close() + rd = gr + } else if strings.HasSuffix(name, ".bz2") { + rd = bzip2.NewReader(f) + } + if err := os.MkdirAll(dst, os.ModeDir|os.ModePerm); err != nil { + return err + } + // resetting umask is essential to have exact permissions on unpacked + // files; it's not not put inside untar function as it changes + // process-wide umask + mask := syscall.Umask(0) + defer syscall.Umask(mask) + return untar.Untar(rd, dst) +} + func init() { RootCmd.AddCommand(pushDockerCmd) pushDockerCmd.Flags().BoolVarP(&docker.IsLocal, "local", "l", false, "Use local images") diff --git a/cmd/clairctl/config/config.go b/cmd/clairctl/config/config.go index 75bf3d29..69d1a613 100644 --- a/cmd/clairctl/config/config.go +++ b/cmd/clairctl/config/config.go @@ -103,6 +103,10 @@ func Init(cfgFile string, logLevel string) { clair.Config() } +func TmpLocal() string { + return viper.GetString("clairctl.tempFolder") +} + func values() config { return config{ Clair: clairConfig{ diff --git a/glide.yaml b/glide.yaml index 2ad06ded..f8025bf4 100644 --- a/glide.yaml +++ b/glide.yaml @@ -101,3 +101,5 @@ import: version: ^1.11.2 subpackages: - api/server/httputils +- package: github.com/mholt/archiver + version: ^1.0.0 diff --git a/vendor/github.com/fsouza/go-dockerclient b/vendor/github.com/fsouza/go-dockerclient new file mode 160000 index 00000000..9df1f25d --- /dev/null +++ b/vendor/github.com/fsouza/go-dockerclient @@ -0,0 +1 @@ +Subproject commit 9df1f25d542e79d7909ef321b5c13c5d34ea7f1d diff --git a/vendor/github.com/mholt/archiver b/vendor/github.com/mholt/archiver new file mode 160000 index 00000000..53744088 --- /dev/null +++ b/vendor/github.com/mholt/archiver @@ -0,0 +1 @@ +Subproject commit 5374408853604727233976068707eaad9a40291b