arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

replace pull and push with docker lib

Browse Source
This commit is contained in:
jgsqware 2016-06-08 09:02:40 +02:00
parent 82cdeb2371
commit 7fc769e152
9 changed files with 279 additions and 421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
cmd/clairctl/cmd/pull-docker.go
View File

@ -1,110 +0,0 @@
package cmd
import (
"fmt"
"html/template"
"os"
"github.com/Sirupsen/logrus"
"github.com/coreos/clair/cmd/clairctl/docker"
"github.com/coreos/clair/cmd/clairctl/dockerdist"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/reference"
"github.com/spf13/cobra"
dockercli "github.com/fsouza/go-dockerclient"
)
const pullDockerTplt = `
Image: {{.Named.FullName}}
{{.V1Manifest.FSLayers | len}} layers found
{{range .V1Manifest.FSLayers}} ➜ {{.BlobSum}}
{{end}}
`
var pullDockerCmd = &cobra.Command{
Use: "pull-docker IMAGE",
Short: "Pull Docker image to Clair",
Long: `Upload a Docker image to Clair for further analysis`,
Run: func(cmd *cobra.Command, args []string) {
if len(args) != 1 {
fmt.Printf("clairctl: \"pull\" requires a minimum of 1 argument\n")
os.Exit(1)
}
imageName := args[0]
var manifest schema1.SignedManifest
var named reference.Named
if !docker.IsLocal {
n, m, err := dockerdist.DownloadManifest(imageName, true)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("parsing image %q: %v", imageName, err)
}
// Ensure that the manifest type is supported.
switch m.(type) {
case *schema1.SignedManifest:
manifest = m.(schema1.SignedManifest)
named = n
break
default:
fmt.Println(errInternalError)
logrus.Fatalf("only v1 manifests are currently supported")
}
} else {
var err error
named, manifest, err = LocalManifest(imageName)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("parsing image %q: %v", imageName, err)
}
}
data := struct {
V1Manifest schema1.SignedManifest
Named reference.Named
}{
V1Manifest: manifest,
Named: named,
}
err := template.Must(template.New("pull").Parse(pullDockerTplt)).Execute(os.Stdout, data)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("rendering image: %v", err)
}
},
}
func LocalManifest(imageName string) (reference.Named, schema1.SignedManifest, error) {
manifest := schema1.SignedManifest{}
// Parse the image name as a docker image reference.
named, err := reference.ParseNamed(imageName)
if err != nil {
return nil, manifest, err
}
endpoint := "unix:///var/run/docker.sock"
client, _ := dockercli.NewClient(endpoint)
histories, _ := client.ImageHistory(imageName)
for _, history := range histories {
var d digest.Digest
d, err := digest.ParseDigest(history.ID)
if err != nil {
return nil, manifest, err
}
manifest.FSLayers = append(manifest.FSLayers, schema1.FSLayer{BlobSum: d})
}
return named, manifest, nil
}
func init() {
RootCmd.AddCommand(pullDockerCmd)
pullDockerCmd.Flags().BoolVarP(&docker.IsLocal, "local", "l", false, "Use local images")
}

101
cmd/clairctl/cmd/pull.go
View File

@ -1,55 +1,80 @@
// Copyright © 2016 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"fmt"
"html/template"
"os"
"text/template"
"github.com/Sirupsen/logrus"
"github.com/coreos/clair/cmd/clairctl/docker"
"github.com/coreos/clair/cmd/clairctl/dockerdist"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/reference"
"github.com/spf13/cobra"
dockercli "github.com/fsouza/go-dockerclient"
)
const pullTplt = `
Image: {{.String}}
{{.FsLayers | len}} layers found
{{range .FsLayers}} ➜ {{.BlobSum}}
Image: {{.Named.FullName}}
{{.V1Manifest.FSLayers | len}} layers found
{{range .V1Manifest.FSLayers}} ➜ {{.BlobSum}}
{{end}}
`
// pingCmd represents the ping command
var pullCmd = &cobra.Command{
Use: "pull IMAGE",
Short: "Pull Docker image information",
Long: `Pull image information from Docker Hub or Registry`,
Short: "Pull Docker image to Clair",
Long: `Upload a Docker image to Clair for further analysis`,
Run: func(cmd *cobra.Command, args []string) {
//TODO how to use args with viper
if len(args) != 1 {
fmt.Printf("clairctl: \"pull\" requires a minimum of 1 argument\n")
os.Exit(1)
}
im := args[0]
image, err := docker.Pull(im)
imageName := args[0]
var manifest schema1.SignedManifest
var named reference.Named
if !docker.IsLocal {
n, m, err := dockerdist.DownloadManifest(imageName, true)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("pulling image %v: %v", args[0], err)
logrus.Fatalf("parsing image %q: %v", imageName, err)
}
// Ensure that the manifest type is supported.
switch m.(type) {
case *schema1.SignedManifest:
manifest = m.(schema1.SignedManifest)
named = n
break
default:
fmt.Println(errInternalError)
logrus.Fatalf("only v1 manifests are currently supported")
}
err = template.Must(template.New("pull").Parse(pullTplt)).Execute(os.Stdout, image)
} else {
var err error
named, manifest, err = localManifest(imageName)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("parsing image %q: %v", imageName, err)
}
}
data := struct {
V1Manifest schema1.SignedManifest
Named reference.Named
}{
V1Manifest: manifest,
Named: named,
}
err := template.Must(template.New("pull").Parse(pullTplt)).Execute(os.Stdout, data)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("rendering image: %v", err)
@ -57,6 +82,30 @@ var pullCmd = &cobra.Command{
},
}
func localManifest(imageName string) (reference.Named, schema1.SignedManifest, error) {
manifest := schema1.SignedManifest{}
// Parse the image name as a docker image reference.
named, err := reference.ParseNamed(imageName)
if err != nil {
return nil, manifest, err
}
//TODO: use socket by default, but check for DOCKER_HOST env variable
endpoint := "unix:///var/run/docker.sock"
client, _ := dockercli.NewClient(endpoint)
histories, _ := client.ImageHistory(imageName)
for _, history := range histories {
var d digest.Digest
d, err := digest.ParseDigest(history.ID)
if err != nil {
return nil, manifest, err
}
manifest.FSLayers = append(manifest.FSLayers, schema1.FSLayer{BlobSum: d})
}
return named, manifest, nil
}
func init() {
RootCmd.AddCommand(pullCmd)
pullCmd.Flags().BoolVarP(&docker.IsLocal, "local", "l", false, "Use local images")
}

234
cmd/clairctl/cmd/push-docker.go
View File

@ -1,234 +0,0 @@
package cmd
import (
"bufio"
"compress/bzip2"
"compress/gzip"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"syscall"
"github.com/Sirupsen/logrus"
"github.com/artyom/untar"
"github.com/coreos/clair/cmd/clairctl/config"
"github.com/coreos/clair/cmd/clairctl/docker"
"github.com/coreos/clair/cmd/clairctl/dockerdist"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/reference"
"github.com/spf13/cobra"
dockercli "github.com/fsouza/go-dockerclient"
)
var pushDockerCmd = &cobra.Command{
Use: "push-docker IMAGE",
Short: "Push Docker image to Clair",
Long: `Upload a Docker image to Clair for further analysis`,
Run: func(cmd *cobra.Command, args []string) {
if len(args) != 1 {
fmt.Printf("clairctl: \"push\" requires a minimum of 1 argument\n")
os.Exit(1)
}
startLocalServer()
imageName := args[0]
if !docker.IsLocal {
image, manifest, err := dockerdist.DownloadManifest(imageName, true)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("parsing local image %q: %v", imageName, err)
}
// Ensure that the manifest type is supported.
switch manifest.(type) {
case *schema1.SignedManifest:
break
default:
fmt.Println(errInternalError)
logrus.Fatalf("only v1 manifests are currently supported")
}
v1manifest := manifest.(*schema1.SignedManifest)
if err := dockerdist.Push(image, *v1manifest); err != nil {
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("pushing image %q: %v", imageName, err)
}
}
} else {
named, err := reference.ParseNamed(imageName)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("pushing image %q: %v", imageName, err)
}
p, err := save(named)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("saving image %q: %v", imageName, err)
}
m, err := historyFromManifest(p)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("reading manifest %q: %v", imageName, err)
}
for _, layer := range m.FSLayers {
fmt.Println("ID: ", layer.BlobSum.String())
}
// var err error
// image, err = docker.Parse(imageName)
// if err != nil {
// fmt.Println(errInternalError)
// logrus.Fatalf("parsing local image %q: %v", imageName, err)
// }
// err = docker.Prepare(&image)
// logrus.Debugf("prepared image layers: %d", len(image.FsLayers))
// if err != nil {
// fmt.Println(errInternalError)
// logrus.Fatalf("preparing local image %q from history: %v", imageName, err)
// }
// logrus.Info("Pushing Image [OLD WAY] should be deprecated")
// if err := docker.Push(image); err != nil {
// if err != nil {
// fmt.Println(errInternalError)
// logrus.Fatalf("pushing image %q: %v", imageName, err)
// }
// }
}
fmt.Printf("%v has been pushed to Clair\n", imageName)
},
}
func historyFromManifest(path string) (*schema1.SignedManifest, error) {
mf, err := os.Open(path + "/manifest.json")
if err != nil {
return nil, err
}
defer mf.Close()
// https://github.com/docker/docker/blob/master/image/tarexport/tarexport.go#L17
type manifestItem struct {
Config string
RepoTags []string
Layers []string
}
var manifest []manifestItem
if err = json.NewDecoder(mf).Decode(&manifest); err != nil {
return nil, err
} else if len(manifest) != 1 {
return nil, err
}
var layers []string
for _, layer := range manifest[0].Layers {
layers = append(layers, strings.TrimSuffix(layer, "/layer.tar"))
}
var m schema1.SignedManifest
for _, layer := range manifest[0].Layers {
var d digest.Digest
fmt.Println(strings.TrimSuffix(layer, "/layer.tar"))
d, err := digest.ParseDigest("sha256:" + strings.TrimSuffix(layer, "/layer.tar"))
if err != nil {
return nil, err
}
m.FSLayers = append(m.FSLayers, schema1.FSLayer{BlobSum: d})
}
return &m, nil
}
func save(image reference.Named) (string, error) {
imageName := image.Name()
path := config.TmpLocal() + "/" + strings.Split(imageName, ":")[0] + "/blobs"
if _, err := os.Stat(path); os.IsExist(err) {
err := os.RemoveAll(path)
if err != nil {
return "", err
}
}
err := os.MkdirAll(path, 0755)
if err != nil {
return "", err
}
logrus.Debugln("docker image to save: ", imageName)
logrus.Debugln("saving in: ", path)
// open output file
fo, err := os.Create(path + "/output.tar")
if err != nil {
return "", err
}
// close fo on exit and check for its returned error
defer func() {
if err := fo.Close(); err != nil {
panic(err)
}
}()
// make a write buffer
w := bufio.NewWriter(fo)
endpoint := "unix:///var/run/docker.sock"
client, _ := dockercli.NewClient(endpoint)
err = client.ExportImage(dockercli.ExportImageOptions{Name: imageName, OutputStream: w})
if err != nil {
return "", err
}
err = openAndUntar(path+"/output.tar", path)
if err != nil {
return "", err
}
err = os.Remove(path + "/output.tar")
if err != nil {
return "", err
}
return path, err
}
func openAndUntar(name, dst string) error {
var rd io.Reader
f, err := os.Open(name)
if err != nil {
return err
}
defer f.Close()
rd = f
if strings.HasSuffix(name, ".gz") || strings.HasSuffix(name, ".tgz") {
gr, err := gzip.NewReader(f)
if err != nil {
return err
}
defer gr.Close()
rd = gr
} else if strings.HasSuffix(name, ".bz2") {
rd = bzip2.NewReader(f)
}
if err := os.MkdirAll(dst, os.ModeDir|os.ModePerm); err != nil {
return err
}
// resetting umask is essential to have exact permissions on unpacked
// files; it's not not put inside untar function as it changes
// process-wide umask
mask := syscall.Umask(0)
defer syscall.Umask(mask)
return untar.Untar(rd, dst)
}
func init() {
RootCmd.AddCommand(pushDockerCmd)
pushDockerCmd.Flags().BoolVarP(&docker.IsLocal, "local", "l", false, "Use local images")
}

68
cmd/clairctl/cmd/push.go
View File

@ -7,7 +7,11 @@ import (
"github.com/Sirupsen/logrus"
"github.com/coreos/clair/cmd/clairctl/config"
"github.com/coreos/clair/cmd/clairctl/docker"
"github.com/coreos/clair/cmd/clairctl/dockercli"
"github.com/coreos/clair/cmd/clairctl/dockerdist"
"github.com/coreos/clair/cmd/clairctl/server"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/reference"
"github.com/spf13/cobra"
)
@ -16,7 +20,6 @@ var pushCmd = &cobra.Command{
Short: "Push Docker image to Clair",
Long: `Upload a Docker image to Clair for further analysis`,
Run: func(cmd *cobra.Command, args []string) {
if len(args) != 1 {
fmt.Printf("clairctl: \"push\" requires a minimum of 1 argument\n")
os.Exit(1)
@ -25,53 +28,53 @@ var pushCmd = &cobra.Command{
startLocalServer()
imageName := args[0]
var image reference.Named
var manifest *schema1.SignedManifest
var image docker.Image
if !docker.IsLocal {
var err error
image, err = docker.Pull(imageName)
if err != nil {
if err == config.ErrLoginNotFound {
fmt.Println(err)
} else {
fmt.Println(errInternalError)
}
logrus.Fatalf("pulling image %q: %v", imageName, err)
}
} else {
var err error
image, err = docker.Parse(imageName)
n, m, err := dockerdist.DownloadManifest(imageName, true)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("parsing local image %q: %v", imageName, err)
}
err = docker.Prepare(&image)
logrus.Debugf("prepared image layers: %d", len(image.FsLayers))
if err != nil {
// Ensure that the manifest type is supported.
switch m.(type) {
case *schema1.SignedManifest:
manifest = m.(*schema1.SignedManifest)
image = n
break
default:
fmt.Println(errInternalError)
logrus.Fatalf("preparing local image %q from history: %v", imageName, err)
}
logrus.Fatalf("only v1 manifests are currently supported")
}
logrus.Info("Pushing Image")
if err := docker.Push(image); err != nil {
} else {
n, err := reference.ParseNamed(imageName)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("pushing image %q: %v", imageName, err)
}
m, err := dockercli.Save(n)
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("saving image %q: %v", imageName, err)
}
manifest = m
image = n
}
if err := dockerdist.Push(image, *manifest); err != nil {
if err != nil {
fmt.Println(errInternalError)
logrus.Fatalf("pushing image %q: %v", imageName, err)
}
}
fmt.Printf("%v has been pushed to Clair\n", imageName)
},
}
func init() {
RootCmd.AddCommand(pushCmd)
pushCmd.Flags().BoolVarP(&docker.IsLocal, "local", "l", false, "Use local images")
}
//StartLocalServer start the clairctl local server needed for reverse proxy and file server
func startLocalServer() {
sURL, err := config.LocalServerIP()
if err != nil {
@ -84,3 +87,8 @@ func startLocalServer() {
logrus.Fatalf("starting local server: %v", err)
}
}
func init() {
RootCmd.AddCommand(pushCmd)
pushCmd.Flags().BoolVarP(&docker.IsLocal, "local", "l", false, "Use local images")
}

2
cmd/clairctl/docker/local.go
View File

@ -57,7 +57,7 @@ func FromHistory(im *Image) error {
return nil
}
func cleanLocal() error {
func CleanLocal() error {
logrus.Debugln("cleaning temporary local repository")
err := os.RemoveAll(TmpLocal())

2
cmd/clairctl/docker/push.go
View File

@ -61,7 +61,7 @@ func Push(image Image) error {
}
}
if IsLocal {
if err := cleanLocal(); err != nil {
if err := CleanLocal(); err != nil {
return err
}
}

141
cmd/clairctl/dockercli/dockercli.go Normal file
View File

@ -0,0 +1,141 @@
package dockercli
import (
"bufio"
"compress/bzip2"
"compress/gzip"
"encoding/json"
"io"
"os"
"strings"
"syscall"
"github.com/Sirupsen/logrus"
"github.com/artyom/untar"
"github.com/coreos/clair/cmd/clairctl/config"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/reference"
dockerclient "github.com/fsouza/go-dockerclient"
)
//Save local images to tmp folder
func Save(image reference.Named) (*schema1.SignedManifest, error) {
imageName := image.Name()
path := config.TmpLocal() + "/" + strings.Split(imageName, ":")[0] + "/blobs"
if _, err := os.Stat(path); os.IsExist(err) {
err := os.RemoveAll(path)
if err != nil {
return nil, err
}
}
err := os.MkdirAll(path, 0755)
if err != nil {
return nil, err
}
logrus.Debugln("docker image to save: ", imageName)
logrus.Debugln("saving in: ", path)
// open output file
fo, err := os.Create(path + "/output.tar")
if err != nil {
return nil, err
}
// close fo on exit and check for its returned error
defer func() {
if err := fo.Close(); err != nil {
panic(err)
}
}()
// make a write buffer
w := bufio.NewWriter(fo)
endpoint := "unix:///var/run/docker.sock"
client, _ := dockerclient.NewClient(endpoint)
err = client.ExportImage(dockerclient.ExportImageOptions{Name: imageName, OutputStream: w})
if err != nil {
return nil, err
}
err = openAndUntar(path+"/output.tar", path)
if err != nil {
return nil, err
}
err = os.Remove(path + "/output.tar")
if err != nil {
return nil, err
}
return historyFromManifest(path)
}
func historyFromManifest(path string) (*schema1.SignedManifest, error) {
mf, err := os.Open(path + "/manifest.json")
if err != nil {
return nil, err
}
defer mf.Close()
// https://github.com/docker/docker/blob/master/image/tarexport/tarexport.go#L17
type manifestItem struct {
Config string
RepoTags []string
Layers []string
}
var manifest []manifestItem
if err = json.NewDecoder(mf).Decode(&manifest); err != nil {
return nil, err
} else if len(manifest) != 1 {
return nil, err
}
var layers []string
for _, layer := range manifest[0].Layers {
layers = append(layers, strings.TrimSuffix(layer, "/layer.tar"))
}
var m schema1.SignedManifest
for _, layer := range manifest[0].Layers {
var d digest.Digest
d, err := digest.ParseDigest("sha256:" + strings.TrimSuffix(layer, "/layer.tar"))
if err != nil {
return nil, err
}
m.FSLayers = append(m.FSLayers, schema1.FSLayer{BlobSum: d})
}
return &m, nil
}
func openAndUntar(name, dst string) error {
var rd io.Reader
f, err := os.Open(name)
if err != nil {
return err
}
defer f.Close()
rd = f
if strings.HasSuffix(name, ".gz") || strings.HasSuffix(name, ".tgz") {
gr, err := gzip.NewReader(f)
if err != nil {
return err
}
defer gr.Close()
rd = gr
} else if strings.HasSuffix(name, ".bz2") {
rd = bzip2.NewReader(f)
}
if err := os.MkdirAll(dst, os.ModeDir|os.ModePerm); err != nil {
return err
}
// resetting umask is essential to have exact permissions on unpacked
// files; it's not not put inside untar function as it changes
// process-wide umask
mask := syscall.Umask(0)
defer syscall.Umask(mask)
return untar.Untar(rd, dst)
}

34
cmd/clairctl/dockerdist/push.go
View File

@ -31,28 +31,32 @@ func Push(image reference.Named, manifest schema1.SignedManifest) error {
}
hURL := fmt.Sprintf("http://%v/v2", localIP)
if docker.IsLocal {
hURL += "/local"
hURL = strings.Replace(hURL, "/v2", "/local", -1)
logrus.Infof("using %v as local url", hURL)
}
for index, layer := range manifest.FSLayers {
lUID := xstrings.Substr(layer.BlobSum.String(), 0, 12)
blobsum := layer.BlobSum.String()
if docker.IsLocal {
blobsum = strings.TrimPrefix(blobsum, "sha256:")
}
lUID := xstrings.Substr(blobsum, 0, 12)
logrus.Infof("Pushing Layer %d/%d [%v]", index+1, layerCount, lUID)
insertRegistryMapping(layer.BlobSum.String(), image.Hostname())
insertRegistryMapping(blobsum, image.Hostname())
payload := v1.LayerEnvelope{Layer: &v1.Layer{
Name: layer.BlobSum.String(),
Path: blobsURI(image.Hostname(), image.RemoteName(), layer.BlobSum.String()),
Name: blobsum,
Path: blobsURI(image.Hostname(), image.RemoteName(), blobsum),
ParentName: parentID,
Format: "Docker",
}}
//FIXME Update to TLS
//FIXME use local with new push
// if IsLocal {
// payload.Layer.Name = layer.History
// payload.Layer.Path += "/layer.tar"
// }
if docker.IsLocal {
payload.Layer.Path += "/layer.tar"
}
payload.Layer.Path = strings.Replace(payload.Layer.Path, image.Hostname(), hURL, 1)
if err := clair.Push(payload); err != nil {
logrus.Infof("adding layer %d/%d [%v]: %v", index+1, layerCount, lUID, err)
@ -64,11 +68,11 @@ func Push(image reference.Named, manifest schema1.SignedManifest) error {
parentID = payload.Layer.Name
}
}
// if IsLocal {
// if err := cleanLocal(); err != nil {
// return err
// }
// }
if docker.IsLocal {
if err := docker.CleanLocal(); err != nil {
return err
}
}
return nil
}

8
cmd/clairctl/server/server.go
View File

@ -72,10 +72,10 @@ func newSingleHostReverseProxy() *httputil.ReverseProxy {
logrus.Errorf("cannot parse url: %v", u)
}
var host string
if docker.IsLocal {
host, _ = docker.GetRegistryMapping(validID.FindStringSubmatch(u)[1])
} else {
host, _ = dockerdist.GetRegistryMapping(validID.FindStringSubmatch(u)[1])
host, err := dockerdist.GetRegistryMapping(validID.FindStringSubmatch(u)[1])
if err != nil {
logrus.Errorf("response error: %v", err)
return
}
out, _ := url.Parse(host)
request.URL.Scheme = out.Scheme