diff --git a/cmd/clairctl/clair/analyze.go b/cmd/clairctl/clair/analyze.go index 2ac20d2c..5186c45f 100644 --- a/cmd/clairctl/clair/analyze.go +++ b/cmd/clairctl/clair/analyze.go @@ -4,12 +4,46 @@ import ( "encoding/json" "fmt" "net/http" + "strings" + "github.com/Sirupsen/logrus" "github.com/coreos/clair/api/v1" + "github.com/coreos/clair/cmd/clairctl/config" + "github.com/coreos/clair/cmd/clairctl/xstrings" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/docker/reference" ) //Analyze get Analysis os specified layer -func Analyze(id string) (v1.LayerEnvelope, error) { + +//Analyze return Clair Image analysis +func Analyze(image reference.Named, manifest schema1.SignedManifest) ImageAnalysis { + c := len(manifest.FSLayers) + res := []v1.LayerEnvelope{} + + for i := range manifest.FSLayers { + blobsum := manifest.FSLayers[c-i-1].BlobSum.String() + if config.IsLocal { + blobsum = strings.TrimPrefix(blobsum, "sha256:") + } + lShort := xstrings.Substr(blobsum, 0, 12) + + if a, err := analyzeLayer(blobsum); err != nil { + logrus.Infof("analysing layer [%v] %d/%d: %v", lShort, i+1, c, err) + } else { + logrus.Infof("analysing layer [%v] %d/%d", lShort, i+1, c) + res = append(res, a) + } + } + return ImageAnalysis{ + Registry: xstrings.TrimPrefixSuffix(image.Hostname(), "http://", "/v2"), + ImageName: manifest.Name, + Tag: manifest.Tag, + Layers: res, + } +} + +func analyzeLayer(id string) (v1.LayerEnvelope, error) { lURI := fmt.Sprintf("%v/layers/%v?vulnerabilities", uri, id) response, err := http.Get(lURI) diff --git a/cmd/clairctl/clair/clair.go b/cmd/clairctl/clair/clair.go index bc93dba5..acc246fe 100644 --- a/cmd/clairctl/clair/clair.go +++ b/cmd/clairctl/clair/clair.go @@ -27,6 +27,14 @@ func (imageAnalysis ImageAnalysis) LastLayer() *v1.Layer { return imageAnalysis.Layers[len(imageAnalysis.Layers)-1].Layer } +func (imageAnalysis ImageAnalysis) CountVulnerabilities(l v1.Layer) int { + count := 0 + for _, feature := range l.Features { + count += len(feature.Vulnerabilities) + } + return count +} + func fmtURI(u string, port int) string { if port != 0 { diff --git a/cmd/clairctl/clair/push.go b/cmd/clairctl/clair/push.go index 883a4bb2..5c0c48e2 100644 --- a/cmd/clairctl/clair/push.go +++ b/cmd/clairctl/clair/push.go @@ -6,15 +6,83 @@ import ( "errors" "fmt" "net/http" + "os" + "strings" + "github.com/Sirupsen/logrus" "github.com/coreos/clair/api/v1" + "github.com/coreos/clair/cmd/clairctl/config" + "github.com/coreos/clair/cmd/clairctl/xstrings" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/docker/reference" ) // ErrUnanalizedLayer is returned when the layer was not correctly analyzed var ErrUnanalizedLayer = errors.New("layer cannot be analyzed") //Push send a layer to Clair for analysis -func Push(layer v1.LayerEnvelope) error { +var registryMapping map[string]string + +//Push image to Clair for analysis +func Push(image reference.Named, manifest schema1.SignedManifest) error { + layerCount := len(manifest.FSLayers) + + parentID := "" + + if layerCount == 0 { + logrus.Warningln("there is no layer to push") + } + localIP, err := config.LocalServerIP() + if err != nil { + return err + } + hURL := fmt.Sprintf("http://%v/v2", localIP) + if config.IsLocal { + hURL = strings.Replace(hURL, "/v2", "/local", -1) + logrus.Infof("using %v as local url", hURL) + } + + for index, layer := range manifest.FSLayers { + blobsum := layer.BlobSum.String() + if config.IsLocal { + blobsum = strings.TrimPrefix(blobsum, "sha256:") + } + + lUID := xstrings.Substr(blobsum, 0, 12) + logrus.Infof("Pushing Layer %d/%d [%v]", index+1, layerCount, lUID) + + insertRegistryMapping(blobsum, image.Hostname()) + payload := v1.LayerEnvelope{Layer: &v1.Layer{ + Name: blobsum, + Path: blobsURI(image.Hostname(), image.RemoteName(), blobsum), + ParentName: parentID, + Format: "Docker", + }} + + //FIXME Update to TLS + if config.IsLocal { + payload.Layer.Path += "/layer.tar" + } + payload.Layer.Path = strings.Replace(payload.Layer.Path, image.Hostname(), hURL, 1) + if err := pushLayer(payload); err != nil { + logrus.Infof("adding layer %d/%d [%v]: %v", index+1, layerCount, lUID, err) + if err != ErrUnanalizedLayer { + return err + } + parentID = "" + } else { + parentID = payload.Layer.Name + } + } + if config.IsLocal { + if err := cleanLocal(); err != nil { + return err + } + } + return nil +} + +func pushLayer(layer v1.LayerEnvelope) error { lJSON, err := json.Marshal(layer) if err != nil { return fmt.Errorf("marshalling layer: %v", err) @@ -42,3 +110,42 @@ func Push(layer v1.LayerEnvelope) error { return nil } + +func blobsURI(registry string, name string, digest string) string { + return strings.Join([]string{registry, name, "blobs", digest}, "/") +} + +func insertRegistryMapping(layerDigest string, registryURI string) { + if strings.Contains(registryURI, "docker") { + registryURI = "https://" + registryURI + "/v2" + + } else { + registryURI = "http://" + registryURI + "/v2" + } + logrus.Debugf("Saving %s[%s]", layerDigest, registryURI) + registryMapping[layerDigest] = registryURI +} + +//GetRegistryMapping return the registryURI corresponding to the layerID passed as parameter +func GetRegistryMapping(layerDigest string) (string, error) { + registryURI, present := registryMapping[layerDigest] + if !present { + return "", fmt.Errorf("%v mapping not found", layerDigest) + } + return registryURI, nil +} + +func cleanLocal() error { + logrus.Debugln("cleaning temporary local repository") + err := os.RemoveAll(config.TmpLocal()) + + if err != nil { + return fmt.Errorf("cleaning temporary local repository: %v", err) + } + + return nil +} + +func init() { + registryMapping = map[string]string{} +} diff --git a/cmd/clairctl/dockerdist/push_test.go b/cmd/clairctl/clair/push_test.go similarity index 97% rename from cmd/clairctl/dockerdist/push_test.go rename to cmd/clairctl/clair/push_test.go index 3e4cd4ef..5c7b2b73 100644 --- a/cmd/clairctl/dockerdist/push_test.go +++ b/cmd/clairctl/clair/push_test.go @@ -1,4 +1,4 @@ -package dockerdist +package clair import "testing" diff --git a/cmd/clairctl/clairctl b/cmd/clairctl/clairctl new file mode 100755 index 00000000..35c16111 Binary files /dev/null and b/cmd/clairctl/clairctl differ diff --git a/cmd/clairctl/cmd/analyze.go b/cmd/clairctl/cmd/analyze.go index b67c780b..da12e6ed 100644 --- a/cmd/clairctl/cmd/analyze.go +++ b/cmd/clairctl/cmd/analyze.go @@ -31,9 +31,23 @@ var analyzeCmd = &cobra.Command{ os.Exit(1) } - ia := analyze(args[0]) + config.ImageName = args[0] + image, manifest, err := docker.RetrieveManifest(config.ImageName, true) + if err != nil { + fmt.Println(errInternalError) + logrus.Fatalf("retrieving manifest for %q: %v", config.ImageName, err) + } - err := template.Must(template.New("analysis").Parse(analyzeTplt)).Execute(os.Stdout, ia) + startLocalServer() + if err := clair.Push(image, manifest); err != nil { + if err != nil { + fmt.Println(errInternalError) + logrus.Fatalf("pushing image %q: %v", image.String(), err) + } + } + + analysis := clair.Analyze(image, manifest) + err = template.Must(template.New("analysis").Parse(analyzeTplt)).Execute(os.Stdout, analysis) if err != nil { fmt.Println(errInternalError) logrus.Fatalf("rendering analysis: %v", err) @@ -41,38 +55,6 @@ var analyzeCmd = &cobra.Command{ }, } -func analyze(imageName string) clair.ImageAnalysis { - var err error - var image docker.Image - - if !config.IsLocal { - // image, err = docker.Pull(imageName) - - // if err != nil { - // if err == config.ErrLoginNotFound { - // fmt.Println(err) - // } else { - // fmt.Println(errInternalError) - // } - // logrus.Fatalf("pulling image %q: %v", imageName, err) - // } - - } else { - image, err = docker.Parse(imageName) - if err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("parsing local image %q: %v", imageName, err) - } - docker.FromHistory(&image) - if err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("getting local image %q from history: %v", imageName, err) - } - } - - return docker.Analyze(image) -} - func init() { RootCmd.AddCommand(analyzeCmd) analyzeCmd.Flags().BoolVarP(&config.IsLocal, "local", "l", false, "Use local images") diff --git a/cmd/clairctl/cmd/login.go b/cmd/clairctl/cmd/login.go deleted file mode 100644 index 708fa13b..00000000 --- a/cmd/clairctl/cmd/login.go +++ /dev/null @@ -1,123 +0,0 @@ -package cmd - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - - "golang.org/x/crypto/bcrypt" - "golang.org/x/crypto/ssh/terminal" - - "github.com/Sirupsen/logrus" - "github.com/coreos/clair/cmd/clairctl/config" - "github.com/coreos/clair/cmd/clairctl/docker" - "github.com/coreos/clair/cmd/clairctl/xstrings" - "github.com/spf13/cobra" -) - -type user struct { - Username string - Password string -} - -type userMapping map[string]user - -var loginCmd = &cobra.Command{ - Use: "login", - Short: "Log in to a Docker registry", - Long: `Log in to a Docker registry`, - Run: func(cmd *cobra.Command, args []string) { - - if len(args) > 1 { - fmt.Println("Only one argument is allowed") - os.Exit(1) - } - - var users userMapping - - if err := readConfigFile(&users, config.ClairctlConfig()); err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("reading clairctl file: %v", err) - } - - var reg string = docker.DockerHub - - if len(args) == 1 { - reg = args[0] - } - - var usr user - if err := askForUser(&usr); err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("encrypting password: %v", err) - } - - users[reg] = usr - - if err := writeConfigFile(users, config.ClairctlConfig()); err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("indenting login: %v", err) - } - - logged, err := docker.Login(reg) - - if err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("log in: %v", err) - } - - if !logged { - fmt.Println("Unauthorized: Wrong login/password, please try again") - os.Exit(1) - } - - fmt.Println("Login Successful") - }, -} - -func readConfigFile(users *userMapping, configFile string) error { - if _, err := os.Stat(configFile); err == nil { - f, err := ioutil.ReadFile(configFile) - if err != nil { - return err - } - - if err := json.Unmarshal(f, &users); err != nil { - return err - } - } else { - *users = userMapping{} - } - return nil -} - -func askForUser(usr *user) error { - fmt.Print("Username: ") - fmt.Scan(&usr.Username) - fmt.Print("Password: ") - pwd, err := terminal.ReadPassword(1) - fmt.Println(" ") - encryptedPwd, err := bcrypt.GenerateFromPassword(pwd, 5) - if err != nil { - return err - } - usr.Password = string(encryptedPwd) - return nil -} - -func writeConfigFile(users userMapping, configFile string) error { - s, err := xstrings.ToIndentJSON(users) - if err != nil { - return err - } - err = ioutil.WriteFile(configFile, s, os.ModePerm) - if err != nil { - return err - } - return nil -} - -func init() { - RootCmd.AddCommand(loginCmd) -} diff --git a/cmd/clairctl/cmd/login_test.go b/cmd/clairctl/cmd/login_test.go deleted file mode 100644 index 6ecc9715..00000000 --- a/cmd/clairctl/cmd/login_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package cmd - -import ( - "os" - "testing" - - "github.com/coreos/clair/cmd/clairctl/test" -) - -var loginData = []struct { - in string - out int -}{ - {"", 0}, - {`{ - "docker.io": { - "Username": "johndoe", - "Password": "$2a$05$Qe4TTO8HMmOht" - } -} -`, 1}, -} - -func TestReadConfigFile(t *testing.T) { - for _, ld := range loginData { - - tmpfile := test.CreateTmpConfigFile(ld.in) - defer os.Remove(tmpfile) // clean up - - var users userMapping - if err := readConfigFile(&users, tmpfile); err != nil { - t.Errorf("readConfigFile(&users,%q) failed => %v", tmpfile, err) - } - - if l := len(users); l != ld.out { - t.Errorf("readConfigFile(&users,%q) => %v users, want %v", tmpfile, l, ld.out) - } - } -} - -func TestWriteConfigFile(t *testing.T) { - users := userMapping{} - users["docker.io"] = user{Username: "johndoe", Password: "$2a$05$Qe4TTO8HMmOht"} - tmpfile := test.CreateTmpConfigFile("") - defer os.Remove(tmpfile) // clean up - - if err := writeConfigFile(users, tmpfile); err != nil { - t.Errorf("writeConfigFile(users,%q) failed => %v", tmpfile, err) - } - - users = userMapping{} - if err := readConfigFile(&users, tmpfile); err != nil { - t.Errorf("after writing: readConfigFile(&users,%q) failed => %v", tmpfile, err) - } - - if l := len(users); l != 1 { - t.Errorf("after writing: readConfigFile(&users,%q) => %v users, want %v", tmpfile, l, 1) - } -} diff --git a/cmd/clairctl/cmd/logout.go b/cmd/clairctl/cmd/logout.go deleted file mode 100644 index cb43a1e1..00000000 --- a/cmd/clairctl/cmd/logout.go +++ /dev/null @@ -1,54 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - - "github.com/Sirupsen/logrus" - "github.com/coreos/clair/cmd/clairctl/config" - "github.com/coreos/clair/cmd/clairctl/docker" - "github.com/spf13/cobra" -) - -var logoutCmd = &cobra.Command{ - Use: "logout", - Short: "Log out from a Docker registry", - Long: `Log out from a Docker registry`, - Run: func(cmd *cobra.Command, args []string) { - - if len(args) > 1 { - fmt.Println("Only one argument is allowed") - os.Exit(1) - } - var reg string = docker.DockerHub - - if len(args) == 1 { - reg = args[0] - } - - if _, err := os.Stat(config.ClairctlConfig()); err == nil { - var users userMapping - - if err := readConfigFile(&users, config.ClairctlConfig()); err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("reading clairctl file: %v", err) - } - if _, present := users[reg]; present { - delete(users, reg) - - if err := writeConfigFile(users, config.ClairctlConfig()); err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("indenting login: %v", err) - } - - fmt.Println("Log out successful") - return - } - } - fmt.Println("You are not logged in") - }, -} - -func init() { - RootCmd.AddCommand(logoutCmd) -} diff --git a/cmd/clairctl/cmd/pull.go b/cmd/clairctl/cmd/pull.go index fb47ff1b..35948d6b 100644 --- a/cmd/clairctl/cmd/pull.go +++ b/cmd/clairctl/cmd/pull.go @@ -7,8 +7,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/coreos/clair/cmd/clairctl/config" - "github.com/coreos/clair/cmd/clairctl/dockercli" - "github.com/coreos/clair/cmd/clairctl/dockerdist" + "github.com/coreos/clair/cmd/clairctl/docker" "github.com/docker/distribution/manifest/schema1" "github.com/docker/docker/reference" "github.com/spf13/cobra" @@ -32,25 +31,11 @@ var pullCmd = &cobra.Command{ os.Exit(1) } - imageName := args[0] - var manifest schema1.SignedManifest - var image reference.Named - var err error - - if !config.IsLocal { - image, manifest, err = dockerdist.DownloadV1Manifest(imageName, true) - - if err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("retrieving manifest for %q: %v", imageName, err) - } - - } else { - image, manifest, err = dockercli.GetLocalManifest(imageName, false) - if err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("retrieving local manifest for %q: %v", imageName, err) - } + config.ImageName = args[0] + image, manifest, err := docker.RetrieveManifest(config.ImageName, true) + if err != nil { + fmt.Println(errInternalError) + logrus.Fatalf("retrieving manifest for %q: %v", config.ImageName, err) } data := struct { diff --git a/cmd/clairctl/cmd/push.go b/cmd/clairctl/cmd/push.go index 170d8c12..5c9684ed 100644 --- a/cmd/clairctl/cmd/push.go +++ b/cmd/clairctl/cmd/push.go @@ -5,12 +5,10 @@ import ( "os" "github.com/Sirupsen/logrus" + "github.com/coreos/clair/cmd/clairctl/clair" "github.com/coreos/clair/cmd/clairctl/config" - "github.com/coreos/clair/cmd/clairctl/dockercli" - "github.com/coreos/clair/cmd/clairctl/dockerdist" + "github.com/coreos/clair/cmd/clairctl/docker" "github.com/coreos/clair/cmd/clairctl/server" - "github.com/docker/distribution/manifest/schema1" - "github.com/docker/docker/reference" "github.com/spf13/cobra" ) @@ -25,35 +23,20 @@ var pushCmd = &cobra.Command{ } startLocalServer() - - imageName := args[0] - var manifest schema1.SignedManifest - var image reference.Named - var err error - - if !config.IsLocal { - image, manifest, err = dockerdist.DownloadV1Manifest(imageName, true) - - if err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("retrieving manifest for %q: %v", imageName, err) - } - - } else { - image, manifest, err = dockercli.GetLocalManifest(imageName, true) - if err != nil { - fmt.Println(errInternalError) - logrus.Fatalf("retrieving local manifest for %q: %v", imageName, err) - } + config.ImageName = args[0] + image, manifest, err := docker.RetrieveManifest(config.ImageName, true) + if err != nil { + fmt.Println(errInternalError) + logrus.Fatalf("retrieving manifest for %q: %v", config.ImageName, err) } - if err := dockerdist.Push(image, manifest); err != nil { + if err := clair.Push(image, manifest); err != nil { if err != nil { fmt.Println(errInternalError) - logrus.Fatalf("pushing image %q: %v", imageName, err) + logrus.Fatalf("pushing image %q: %v", image.String(), err) } } - fmt.Printf("%v has been pushed to Clair\n", imageName) + fmt.Printf("%v has been pushed to Clair\n", image.String()) }, } diff --git a/cmd/clairctl/cmd/report.go b/cmd/clairctl/cmd/report.go index 10518a87..ebb9168f 100644 --- a/cmd/clairctl/cmd/report.go +++ b/cmd/clairctl/cmd/report.go @@ -8,6 +8,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/coreos/clair/cmd/clairctl/clair" "github.com/coreos/clair/cmd/clairctl/config" + "github.com/coreos/clair/cmd/clairctl/docker" "github.com/coreos/clair/cmd/clairctl/xstrings" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -23,7 +24,14 @@ var reportCmd = &cobra.Command{ os.Exit(1) } - analyzes := analyze(args[0]) + config.ImageName = args[0] + image, manifest, err := docker.RetrieveManifest(config.ImageName, true) + if err != nil { + fmt.Println(errInternalError) + logrus.Fatalf("retrieving manifest for %q: %v", config.ImageName, err) + } + + analyzes := clair.Analyze(image, manifest) imageName := strings.Replace(analyzes.ImageName, "/", "-", -1) + "-" + analyzes.Tag switch clair.Report.Format { case "html": diff --git a/cmd/clairctl/cmd/root.go b/cmd/clairctl/cmd/root.go index 62f11cea..95150bd8 100644 --- a/cmd/clairctl/cmd/root.go +++ b/cmd/clairctl/cmd/root.go @@ -19,6 +19,7 @@ import ( "fmt" "os" + "github.com/coreos/clair/cmd/clairctl/clair" "github.com/coreos/clair/cmd/clairctl/config" "github.com/spf13/cobra" ) @@ -56,4 +57,5 @@ func init() { func initConfig() { config.Init(cfgFile, logLevel) + clair.Config() } diff --git a/cmd/clairctl/config/config.go b/cmd/clairctl/config/config.go index 4a1e48e8..5ae215cf 100644 --- a/cmd/clairctl/config/config.go +++ b/cmd/clairctl/config/config.go @@ -16,7 +16,6 @@ import ( "gopkg.in/yaml.v2" "github.com/Sirupsen/logrus" - "github.com/coreos/clair/cmd/clairctl/clair" "github.com/coreos/clair/cmd/clairctl/xstrings" "github.com/spf13/viper" ) @@ -25,6 +24,8 @@ var ErrLoginNotFound = errors.New("user is not log in") var IsLocal = false +var ImageName string + type reportConfig struct { Path, Format string } @@ -102,7 +103,7 @@ func Init(cfgFile string, logLevel string) { if viper.Get("clairctl.tempFolder") == nil { viper.Set("clairctl.tempFolder", "/tmp/clairctl") } - clair.Config() + } func TmpLocal() string { diff --git a/cmd/clairctl/docker/analyse.go b/cmd/clairctl/docker/analyse.go deleted file mode 100644 index 9740faf2..00000000 --- a/cmd/clairctl/docker/analyse.go +++ /dev/null @@ -1,32 +0,0 @@ -package docker - -import ( - "github.com/Sirupsen/logrus" - "github.com/coreos/clair/api/v1" - "github.com/coreos/clair/cmd/clairctl/clair" - "github.com/coreos/clair/cmd/clairctl/xstrings" -) - -//Analyze return Clair Image analysis -func Analyze(image Image) clair.ImageAnalysis { - c := len(image.FsLayers) - res := []v1.LayerEnvelope{} - - for i := range image.FsLayers { - l := image.FsLayers[c-i-1].BlobSum - lShort := xstrings.Substr(l, 0, 12) - - if a, err := clair.Analyze(l); err != nil { - logrus.Infof("analysing layer [%v] %d/%d: %v", lShort, i+1, c, err) - } else { - logrus.Infof("analysing layer [%v] %d/%d", lShort, i+1, c) - res = append(res, a) - } - } - return clair.ImageAnalysis{ - Registry: xstrings.TrimPrefixSuffix(image.Registry, "http://", "/v2"), - ImageName: image.Name, - Tag: image.Tag, - Layers: res, - } -} diff --git a/cmd/clairctl/docker/docker.go b/cmd/clairctl/docker/docker.go new file mode 100644 index 00000000..31498b89 --- /dev/null +++ b/cmd/clairctl/docker/docker.go @@ -0,0 +1,19 @@ +package docker + +import ( + "github.com/coreos/clair/cmd/clairctl/config" + "github.com/coreos/clair/cmd/clairctl/docker/dockercli" + "github.com/coreos/clair/cmd/clairctl/docker/dockerdist" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/docker/reference" +) + +//RetrieveManifest get manifest from local or remote docker registry +func RetrieveManifest(imageName string, withExport bool) (image reference.Named, manifest schema1.SignedManifest, err error) { + if !config.IsLocal { + image, manifest, err = dockerdist.DownloadV1Manifest(imageName, true) + } else { + image, manifest, err = dockercli.GetLocalManifest(imageName, withExport) + } + return +} diff --git a/cmd/clairctl/dockercli/dockercli.go b/cmd/clairctl/docker/dockercli/dockercli.go similarity index 99% rename from cmd/clairctl/dockercli/dockercli.go rename to cmd/clairctl/docker/dockercli/dockercli.go index 0cd6375a..2b028234 100644 --- a/cmd/clairctl/dockercli/dockercli.go +++ b/cmd/clairctl/docker/dockercli/dockercli.go @@ -22,7 +22,7 @@ import ( //GetLocalManifest retrieve manifest for local image func GetLocalManifest(imageName string, withExport bool) (reference.Named, schema1.SignedManifest, error) { - + image, err := reference.ParseNamed(imageName) if err != nil { return nil, schema1.SignedManifest{}, err diff --git a/cmd/clairctl/docker/auth.go b/cmd/clairctl/docker/dockerdist/auth.go similarity index 58% rename from cmd/clairctl/docker/auth.go rename to cmd/clairctl/docker/dockerdist/auth.go index aa6a0866..d2dcb2af 100644 --- a/cmd/clairctl/docker/auth.go +++ b/cmd/clairctl/docker/dockerdist/auth.go @@ -1,4 +1,4 @@ -package docker +package dockerdist import ( "encoding/json" @@ -7,27 +7,14 @@ import ( "net/http" "strings" - "github.com/coreos/clair/cmd/clairctl/docker/httpclient" + "github.com/coreos/clair/cmd/clairctl/config" ) +//ErrUnauthorized is return when requested user don't have access to the resource var ErrUnauthorized = errors.New("unauthorized access") -type Authentication struct { - Username, Password string -} - -var User Authentication - -type token struct { - Value string `json:"token"` -} - -func (tok token) String() string { - return tok.Value -} - -//BearerAuthParams parse Bearer Token on Www-Authenticate header -func BearerAuthParams(r *http.Response) map[string]string { +//bearerAuthParams parse Bearer Token on Www-Authenticate header +func bearerAuthParams(r *http.Response) map[string]string { s := strings.Fields(r.Header.Get("Www-Authenticate")) if len(s) != 2 || s[0] != "Bearer" { return nil @@ -44,8 +31,9 @@ func BearerAuthParams(r *http.Response) map[string]string { return result } -func AuthenticateResponse(dockerResponse *http.Response, request *http.Request) error { - bearerToken := BearerAuthParams(dockerResponse) +//AuthenticateResponse add authentication headers on request +func AuthenticateResponse(client *http.Client, dockerResponse *http.Response, request *http.Request) error { + bearerToken := bearerAuthParams(dockerResponse) url := bearerToken["realm"] + "?service=" + bearerToken["service"] if bearerToken["scope"] != "" { url += "&scope=" + bearerToken["scope"] @@ -55,9 +43,15 @@ func AuthenticateResponse(dockerResponse *http.Response, request *http.Request) if err != nil { return err } - req.SetBasicAuth(User.Username, User.Password) - response, err := httpclient.Get().Do(req) + authConfig, err := GetAuthCredentials(config.ImageName) + if err != nil { + return err + } + + req.SetBasicAuth(authConfig.Username, authConfig.Password) + + response, err := client.Do(req) if err != nil { return err @@ -72,6 +66,9 @@ func AuthenticateResponse(dockerResponse *http.Response, request *http.Request) } defer response.Body.Close() + type token struct { + Value string `json:"token"` + } var tok token err = json.NewDecoder(response.Body).Decode(&tok) @@ -79,7 +76,7 @@ func AuthenticateResponse(dockerResponse *http.Response, request *http.Request) return err } - request.Header.Set("Authorization", "Bearer "+tok.String()) + request.Header.Set("Authorization", "Bearer "+tok.Value) return nil } diff --git a/cmd/clairctl/dockerdist/dockerdist.go b/cmd/clairctl/docker/dockerdist/dockerdist.go similarity index 95% rename from cmd/clairctl/dockerdist/dockerdist.go rename to cmd/clairctl/docker/dockerdist/dockerdist.go index a86cf693..52b9a951 100644 --- a/cmd/clairctl/dockerdist/dockerdist.go +++ b/cmd/clairctl/docker/dockerdist/dockerdist.go @@ -45,15 +45,11 @@ func getRepositoryClient(image reference.Named, insecure bool, scopes ...string) return nil, err } - // Retrieve the user's Docker configuration file (if any). - configFile, err := cliconfig.Load(cliconfig.ConfigDir()) + authConfig, err := GetAuthCredentials(image.String()) if err != nil { return nil, err } - // Resolve the authentication information for the registry specified, via the config file. - authConfig := registry.ResolveAuthConfig(configFile.AuthConfigs, indexInfo) - repoInfo := ®istry.RepositoryInfo{ image, indexInfo, @@ -62,6 +58,7 @@ func getRepositoryClient(image reference.Named, insecure bool, scopes ...string) metaHeaders := map[string][]string{} tlsConfig := tlsconfig.ServerDefault + //TODO: fix TLS tlsConfig.InsecureSkipVerify = viper.GetBool("auth.insecureSkipVerify") url, err := url.Parse("https://" + image.Hostname()) @@ -79,7 +76,6 @@ func getRepositoryClient(image reference.Named, insecure bool, scopes ...string) TrimHostname: true, TLSConfig: &tlsConfig, } - ctx := context.Background() repo, _, err := distribution.NewV2Repository(ctx, repoInfo, endpoint, metaHeaders, &authConfig, scopes...) return repo, err diff --git a/cmd/clairctl/docker/httpclient/httpclient.go b/cmd/clairctl/docker/httpclient/httpclient.go deleted file mode 100644 index 2d9f9906..00000000 --- a/cmd/clairctl/docker/httpclient/httpclient.go +++ /dev/null @@ -1,24 +0,0 @@ -package httpclient - -import ( - "crypto/tls" - "net/http" - - "github.com/spf13/viper" -) - -var client *http.Client - -//Get create a http.Client with Transport configuration -func Get() *http.Client { - - if client == nil { - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: viper.GetBool("auth.insecureSkipVerify")}, - DisableCompression: true, - } - client = &http.Client{Transport: tr} - } - - return client -} diff --git a/cmd/clairctl/docker/image.go b/cmd/clairctl/docker/image.go deleted file mode 100644 index 1acfccd6..00000000 --- a/cmd/clairctl/docker/image.go +++ /dev/null @@ -1,110 +0,0 @@ -package docker - -import ( - "encoding/json" - "errors" - "fmt" - "regexp" - "strings" - - "github.com/spf13/viper" -) - -var errDisallowed = errors.New("analysing official images is not allowed") - -//Image represent Image Manifest from Docker image, including the registry URL -type Image struct { - Name string - Tag string - Registry string - FsLayers []Layer -} - -//Layer represent the digest of a image layer -type Layer struct { - BlobSum string - History string -} - -const dockerImageRegex = "^(?:([^/]+)/)?(?:([^/]+)/)?([^@:/]+)(?:[@:](.+))?" -const DockerHub = "registry-1.docker.io" -const hubURI = "https://" + DockerHub + "/v2" - -var isLocal = false - -func TmpLocal() string { - return viper.GetString("clairctl.tempFolder") -} - -// Parse is used to parse a docker image command -// -//Example: -//"register.com:5080/wemanity-belgium/alpine" -//"register.com:5080/wemanity-belgium/alpine:latest" -//"register.com:5080/alpine" -//"register.com/wemanity-belgium/alpine" -//"register.com/alpine" -//"register.com/wemanity-belgium/alpine:latest" -//"alpine" -//"wemanity-belgium/alpine" -//"wemanity-belgium/alpine:latest" -func Parse(image string) (Image, error) { - imageRegex := regexp.MustCompile(dockerImageRegex) - - if imageRegex.MatchString(image) == false { - return Image{}, fmt.Errorf("cannot parse image name: %v", image) - } - groups := imageRegex.FindStringSubmatch(image) - - registry, repository, name, tag := groups[1], groups[2], groups[3], groups[4] - - if tag == "" { - tag = "latest" - } - - if repository == "" && !strings.ContainsAny(registry, ":.") { - repository, registry = registry, hubURI //Regex problem, if no registry in url, regex parse repository as registry, so need to invert it - - } else { - //FIXME We need to move to https. - //Maybe using a `insecure-registry` flag in configuration - if strings.Contains(registry, "docker") { - registry = "https://" + registry + "/v2" - - } else { - registry = "http://" + registry + "/v2" - } - } - - if repository != "" { - name = repository + "/" + name - } - - if strings.Contains(registry, "docker.io") && repository == "" { - return Image{}, errDisallowed - } - - return Image{ - Registry: registry, - Name: name, - Tag: tag, - }, nil -} - -// BlobsURI run Blobs URI as //blobs/ -// eg: "http://registry:5000/v2/jgsqware/ubuntu-git/blobs/sha256:13be4a52fdee2f6c44948b99b5b65ec703b1ca76c1ab5d2d90ae9bf18347082e" -func (image Image) BlobsURI(digest string) string { - return strings.Join([]string{image.Registry, image.Name, "blobs", digest}, "/") -} - -func (image Image) String() string { - return image.Registry + "/" + image.Name + ":" + image.Tag -} - -func (image Image) AsJSON() (string, error) { - b, err := json.Marshal(image) - if err != nil { - return "", fmt.Errorf("cannot marshal image: %v", err) - } - return string(b), nil -} diff --git a/cmd/clairctl/docker/image_test.go b/cmd/clairctl/docker/image_test.go deleted file mode 100644 index e5055965..00000000 --- a/cmd/clairctl/docker/image_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package docker - -import ( - "testing" - -) - -var imageNameTests = []struct { - in string - out string -}{ - {"jgsqware/ubuntu-git", hubURI + "/jgsqware/ubuntu-git:latest"}, - {"wemanity-belgium/registry-backup", hubURI + "/wemanity-belgium/registry-backup:latest"}, - {"wemanity-belgium/alpine:latest", hubURI + "/wemanity-belgium/alpine:latest"}, - {"register.com/alpine", "http://register.com/v2/alpine:latest"}, - {"register.com/wemanity-belgium/alpine", "http://register.com/v2/wemanity-belgium/alpine:latest"}, - {"register.com/wemanity-belgium/alpine:latest", "http://register.com/v2/wemanity-belgium/alpine:latest"}, - {"register.com:5080/alpine", "http://register.com:5080/v2/alpine:latest"}, - {"register.com:5080/wemanity-belgium/alpine", "http://register.com:5080/v2/wemanity-belgium/alpine:latest"}, - {"register.com:5080/wemanity-belgium/alpine:latest", "http://register.com:5080/v2/wemanity-belgium/alpine:latest"}, - {"registry:5000/google/cadvisor", "http://registry:5000/v2/google/cadvisor:latest"}, -} - -var invalidImageNameTests = []struct { - in string - out string -}{ - {"alpine", hubURI + "/alpine:latest"}, - {"docker.io/golang", hubURI + "/golang:latest"}, -} - -func TestParse(t *testing.T) { - for _, imageName := range imageNameTests { - image, err := Parse(imageName.in) - if err != nil { - t.Errorf("Parse(\"%s\") should be valid: %v", imageName.in, err) - } - if image.String() != imageName.out { - t.Errorf("Parse(\"%s\") => %v, want %v", imageName.in, image, imageName.out) - - } - } -} - -func TestParseDisallowed(t *testing.T) { - for _, imageName := range invalidImageNameTests { - _, err := Parse(imageName.in) - if err != errDisallowed { - t.Errorf("Parse(\"%s\") should failed with err \"%v\": %v", imageName.in, errDisallowed, err) - } - } -} - -func TestMBlobstURI(t *testing.T) { - image, err := Parse("localhost:5000/alpine") - - if err != nil { - t.Error(err) - } - - result := image.BlobsURI("sha256:13be4a52fdee2f6c44948b99b5b65ec703b1ca76c1ab5d2d90ae9bf18347082e") - if result != "http://localhost:5000/v2/alpine/blobs/sha256:13be4a52fdee2f6c44948b99b5b65ec703b1ca76c1ab5d2d90ae9bf18347082e" { - t.Errorf("Is %s, should be http://localhost:5000/v2/alpine/blobs/sha256:13be4a52fdee2f6c44948b99b5b65ec703b1ca76c1ab5d2d90ae9bf18347082e", result) - } -} - -func TestUniqueLayer(t *testing.T) { - image := Image{ - FsLayers: []Layer{Layer{BlobSum: "test1"}, Layer{BlobSum: "test1"}, Layer{BlobSum: "test2"}}, - } - - image.uniqueLayers() - - if len(image.FsLayers) > 2 { - t.Errorf("Layers must be unique: %v", image.FsLayers) - } -} diff --git a/cmd/clairctl/docker/local.go b/cmd/clairctl/docker/local.go deleted file mode 100644 index d00b05f4..00000000 --- a/cmd/clairctl/docker/local.go +++ /dev/null @@ -1,171 +0,0 @@ -package docker - -import ( - "bufio" - "bytes" - "encoding/json" - "errors" - "fmt" - "os" - "os/exec" - "strings" - - "github.com/Sirupsen/logrus" -) - -//Prepare populate image.FSLayers with the layer from manifest coming from `docker save` command. Layer.History will be populated with `docker history` command -func Prepare(im *Image) error { - imageName := im.Name + ":" + im.Tag - logrus.Debugf("preparing %v", imageName) - - path, err := save(imageName) - // defer os.RemoveAll(path) - if err != nil { - return fmt.Errorf("could not save image: %s", err) - } - - // Retrieve history. - logrus.Infoln("Getting image's history") - manifestLayerIDs, err := historyFromManifest(path) - - historyLayerIDs, err := historyFromCommand(imageName) - - if err != nil || (len(manifestLayerIDs) == 0 && len(historyLayerIDs) == 0) { - return fmt.Errorf("Could not get image's history: %s", err) - } - - for i, l := range manifestLayerIDs { - im.FsLayers = append(im.FsLayers, Layer{BlobSum: l, History: historyLayerIDs[i]}) - } - - return nil -} - -//FromHistory populate image.FSLayers with the layer from `docker history` command -func FromHistory(im *Image) error { - imageName := im.Name + ":" + im.Tag - layerIDs, err := historyFromCommand(imageName) - - if err != nil || len(layerIDs) == 0 { - return fmt.Errorf("Could not get image's history: %s", err) - } - - for _, l := range layerIDs { - im.FsLayers = append(im.FsLayers, Layer{BlobSum: l}) - } - - return nil -} - -func CleanLocal() error { - logrus.Debugln("cleaning temporary local repository") - err := os.RemoveAll(TmpLocal()) - - if err != nil { - return fmt.Errorf("cleaning temporary local repository: %v", err) - } - - return nil -} - -func save(imageName string) (string, error) { - path := TmpLocal() + "/" + strings.Split(imageName, ":")[0] + "/blobs" - - if _, err := os.Stat(path); os.IsExist(err) { - err := os.RemoveAll(path) - if err != nil { - return "", err - } - } - - err := os.MkdirAll(path, 0755) - if err != nil { - return "", err - } - - var stderr bytes.Buffer - logrus.Debugln("docker image to save: ", imageName) - logrus.Debugln("saving in: ", path) - save := exec.Command("docker", "save", imageName) - save.Stderr = &stderr - extract := exec.Command("tar", "xf", "-", "-C"+path) - extract.Stderr = &stderr - pipe, err := extract.StdinPipe() - if err != nil { - return "", err - } - save.Stdout = pipe - - err = extract.Start() - if err != nil { - return "", errors.New(stderr.String()) - } - err = save.Run() - if err != nil { - return "", errors.New(stderr.String()) - } - err = pipe.Close() - if err != nil { - return "", err - } - err = extract.Wait() - if err != nil { - return "", errors.New(stderr.String()) - } - return path, nil -} - -func historyFromManifest(path string) ([]string, error) { - mf, err := os.Open(path + "/manifest.json") - if err != nil { - return nil, err - } - defer mf.Close() - - // https://github.com/docker/docker/blob/master/image/tarexport/tarexport.go#L17 - type manifestItem struct { - Config string - RepoTags []string - Layers []string - } - - var manifest []manifestItem - if err = json.NewDecoder(mf).Decode(&manifest); err != nil { - return nil, err - } else if len(manifest) != 1 { - return nil, err - } - var layers []string - for _, layer := range manifest[0].Layers { - layers = append(layers, strings.TrimSuffix(layer, "/layer.tar")) - } - return layers, nil -} - -func historyFromCommand(imageName string) ([]string, error) { - var stderr bytes.Buffer - cmd := exec.Command("docker", "history", "-q", "--no-trunc", imageName) - cmd.Stderr = &stderr - stdout, err := cmd.StdoutPipe() - if err != nil { - return []string{}, err - } - - err = cmd.Start() - if err != nil { - return []string{}, errors.New(stderr.String()) - } - - var layers []string - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - layers = append(layers, scanner.Text()) - } - - for i := len(layers)/2 - 1; i >= 0; i-- { - opp := len(layers) - 1 - i - layers[i], layers[opp] = layers[opp], layers[i] - } - - return layers, nil -} diff --git a/cmd/clairctl/docker/login.go b/cmd/clairctl/docker/login.go deleted file mode 100644 index fcfe5f65..00000000 --- a/cmd/clairctl/docker/login.go +++ /dev/null @@ -1,46 +0,0 @@ -package docker - -import ( - "fmt" - "net/http" - "strings" - - "github.com/Sirupsen/logrus" - "github.com/coreos/clair/cmd/clairctl/docker/httpclient" -) - -//Pull Image from Registry or Hub depending on image name -func Login(registry string) (bool, error) { - - logrus.Info("log in: ", registry) - - if strings.Contains(registry, "docker") { - registry = "https://" + registry + "/v2" - - } else { - registry = "http://" + registry + "/v2" - } - - client := httpclient.Get() - request, err := http.NewRequest("GET", registry, nil) - response, err := client.Do(request) - if err != nil { - return false, fmt.Errorf("log in %v: %v", registry, err) - } - authorized := response.StatusCode != http.StatusUnauthorized - if !authorized { - logrus.Info("Unauthorized access") - err := AuthenticateResponse(response, request) - - if err != nil { - if err == ErrUnauthorized { - authorized = false - } - return false, err - } else { - authorized = true - } - } - - return authorized, nil -} diff --git a/cmd/clairctl/dockerdist/push.go b/cmd/clairctl/dockerdist/push.go deleted file mode 100644 index 81a6dfb0..00000000 --- a/cmd/clairctl/dockerdist/push.go +++ /dev/null @@ -1,105 +0,0 @@ -package dockerdist - -import ( - "fmt" - "strings" - - "github.com/Sirupsen/logrus" - "github.com/coreos/clair/api/v1" - "github.com/coreos/clair/cmd/clairctl/clair" - "github.com/coreos/clair/cmd/clairctl/config" - "github.com/coreos/clair/cmd/clairctl/docker" - "github.com/coreos/clair/cmd/clairctl/xstrings" - "github.com/docker/distribution/manifest/schema1" - "github.com/docker/docker/reference" -) - -var registryMapping map[string]string - -//Push image to Clair for analysis -func Push(image reference.Named, manifest schema1.SignedManifest) error { - layerCount := len(manifest.FSLayers) - - parentID := "" - - if layerCount == 0 { - logrus.Warningln("there is no layer to push") - } - localIP, err := config.LocalServerIP() - if err != nil { - return err - } - hURL := fmt.Sprintf("http://%v/v2", localIP) - if config.IsLocal { - hURL = strings.Replace(hURL, "/v2", "/local", -1) - logrus.Infof("using %v as local url", hURL) - } - - for index, layer := range manifest.FSLayers { - blobsum := layer.BlobSum.String() - if config.IsLocal { - blobsum = strings.TrimPrefix(blobsum, "sha256:") - } - - lUID := xstrings.Substr(blobsum, 0, 12) - logrus.Infof("Pushing Layer %d/%d [%v]", index+1, layerCount, lUID) - - insertRegistryMapping(blobsum, image.Hostname()) - payload := v1.LayerEnvelope{Layer: &v1.Layer{ - Name: blobsum, - Path: blobsURI(image.Hostname(), image.RemoteName(), blobsum), - ParentName: parentID, - Format: "Docker", - }} - - //FIXME Update to TLS - if config.IsLocal { - - payload.Layer.Path += "/layer.tar" - } - payload.Layer.Path = strings.Replace(payload.Layer.Path, image.Hostname(), hURL, 1) - if err := clair.Push(payload); err != nil { - logrus.Infof("adding layer %d/%d [%v]: %v", index+1, layerCount, lUID, err) - if err != clair.ErrUnanalizedLayer { - return err - } - parentID = "" - } else { - parentID = payload.Layer.Name - } - } - if config.IsLocal { - if err := docker.CleanLocal(); err != nil { - return err - } - } - return nil -} - -func blobsURI(registry string, name string, digest string) string { - return strings.Join([]string{registry, name, "blobs", digest}, "/") -} - -func insertRegistryMapping(layerDigest string, registryURI string) { - if strings.Contains(registryURI, "docker") { - registryURI = "https://" + registryURI + "/v2" - - } else { - registryURI = "http://" + registryURI + "/v2" - } - logrus.Debugf("Saving %s[%s]", layerDigest, registryURI) - registryMapping[layerDigest] = registryURI -} - -//GetRegistryMapping return the registryURI corresponding to the layerID passed as parameter -func GetRegistryMapping(layerDigest string) (string, error) { - registryURI, present := registryMapping[layerDigest] - if !present { - return "", fmt.Errorf("%v mapping not found", layerDigest) - } - return registryURI, nil -} - -func init() { - registryMapping = map[string]string{} -} diff --git a/cmd/clairctl/server/server.go b/cmd/clairctl/server/server.go index d7bd98a5..0085c188 100644 --- a/cmd/clairctl/server/server.go +++ b/cmd/clairctl/server/server.go @@ -1,6 +1,7 @@ package server import ( + "crypto/tls" "net" "net/http" "net/http/httputil" @@ -11,9 +12,9 @@ import ( "time" "github.com/Sirupsen/logrus" - "github.com/coreos/clair/cmd/clairctl/docker" - "github.com/coreos/clair/cmd/clairctl/docker/httpclient" - "github.com/coreos/clair/cmd/clairctl/dockerdist" + "github.com/coreos/clair/cmd/clairctl/clair" + "github.com/coreos/clair/cmd/clairctl/config" + "github.com/coreos/clair/cmd/clairctl/docker/dockerdist" "github.com/spf13/viper" ) @@ -22,7 +23,7 @@ func Serve(sURL string) error { go func() { http.Handle("/v2/", newSingleHostReverseProxy()) - http.Handle("/local/", http.StripPrefix("/local", restrictedFileServer(docker.TmpLocal()))) + http.Handle("/local/", http.StripPrefix("/local", restrictedFileServer(config.TmpLocal()))) listener := tcpListener(sURL) logrus.Info("Starting Server on ", listener.Addr()) @@ -68,11 +69,12 @@ func newSingleHostReverseProxy() *httputil.ReverseProxy { var validID = regexp.MustCompile(`.*/blobs/(.*)$`) u := request.URL.Path logrus.Debugf("request url: %v", u) + logrus.Debugf("request for image: %v", config.ImageName) if !validID.MatchString(u) { logrus.Errorf("cannot parse url: %v", u) } var host string - host, err := dockerdist.GetRegistryMapping(validID.FindStringSubmatch(u)[1]) + host, err := clair.GetRegistryMapping(validID.FindStringSubmatch(u)[1]) if err != nil { logrus.Errorf("response error: %v", err) return @@ -80,7 +82,10 @@ func newSingleHostReverseProxy() *httputil.ReverseProxy { out, _ := url.Parse(host) request.URL.Scheme = out.Scheme request.URL.Host = out.Host - client := httpclient.Get() + client := &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: viper.GetBool("auth.insecureSkipVerify")}, + DisableCompression: true, + }} req, _ := http.NewRequest("HEAD", request.URL.String(), nil) resp, err := client.Do(req) if err != nil { @@ -90,7 +95,7 @@ func newSingleHostReverseProxy() *httputil.ReverseProxy { if resp.StatusCode == http.StatusUnauthorized { logrus.Info("pull from clair is unauthorized") - docker.AuthenticateResponse(resp, request) + dockerdist.AuthenticateResponse(client, resp, request) } r, _ := http.NewRequest("GET", request.URL.String(), nil)