From 9df4f5bd705b554fb6b024a5ed0793d65abf186b Mon Sep 17 00:00:00 2001 From: Jean Michel MacKay Date: Tue, 4 Sep 2018 16:51:42 -0400 Subject: [PATCH] Adding httputil and version packages - Debian/RHEL/Oracle vulnsrc now use httputil to download files - httputil sets the User-Agent to the requests as Clair/ (https://github.com/coreos/clair/) - httputil holds Status2xx() which returns if the response is a http success (2xx) - GetClientAddr moved from api/httputil to pkg/httputil - the version packge holds a Version string which is set at build time from the git tag and sha - the .git directory was removed from .dockerignore so that we can use the git tag to set the version --- .dockerignore | 3 +- Dockerfile | 4 ++- api/httputil/httputil.go | 24 ------------- ext/vulnsrc/debian/debian.go | 11 ++++-- ext/vulnsrc/oracle/oracle.go | 17 ++++++++-- ext/vulnsrc/rhel/rhel.go | 18 ++++++++-- pkg/httputil/httputil.go | 65 ++++++++++++++++++++++++++++++++++++ pkg/version/version.go | 18 ++++++++++ 8 files changed, 126 insertions(+), 34 deletions(-) delete mode 100644 api/httputil/httputil.go create mode 100644 pkg/httputil/httputil.go create mode 100644 pkg/version/version.go diff --git a/.dockerignore b/.dockerignore index 090b49c0..ebd9424e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,5 @@ -.* +.dockerignore +.travis.yml *.md DCO LICENSE diff --git a/Dockerfile b/Dockerfile index 07677802..b57ff8ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,9 +13,11 @@ # limitations under the License. FROM golang:1.10-alpine AS build +RUN apk add --no-cache git ADD . /go/src/github.com/coreos/clair/ WORKDIR /go/src/github.com/coreos/clair/ -RUN go build github.com/coreos/clair/cmd/clair +RUN export CLAIR_VERSION=$(git describe --tag --always --dirty) && \ + go build -ldflags "-X github.com/coreos/clair/pkg/version.Version=$CLAIR_VERSION" github.com/coreos/clair/cmd/clair FROM alpine:3.8 COPY --from=build /go/src/github.com/coreos/clair/clair /clair diff --git a/api/httputil/httputil.go b/api/httputil/httputil.go deleted file mode 100644 index 36bfe8c1..00000000 --- a/api/httputil/httputil.go +++ /dev/null @@ -1,24 +0,0 @@ -package httputil - -import ( - "net" - "net/http" - "strings" -) - -// GetClientAddr returns the first value in X-Forwarded-For if it exists -// otherwise fall back to use RemoteAddr -func GetClientAddr(r *http.Request) string { - addr := r.RemoteAddr - if s := r.Header.Get("X-Forwarded-For"); s != "" { - ips := strings.Split(s, ",") - // assume the first one is the client address - if len(ips) != 0 { - // validate the ip - if realIP := net.ParseIP(ips[0]); realIP != nil { - addr = strings.TrimSpace(ips[0]) - } - } - } - return addr -} diff --git a/ext/vulnsrc/debian/debian.go b/ext/vulnsrc/debian/debian.go index c0efc37e..c5322593 100644 --- a/ext/vulnsrc/debian/debian.go +++ b/ext/vulnsrc/debian/debian.go @@ -22,7 +22,6 @@ import ( "encoding/json" "fmt" "io" - "net/http" "strings" log "github.com/sirupsen/logrus" @@ -32,6 +31,7 @@ import ( "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/httputil" ) const ( @@ -84,12 +84,19 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo } // Download JSON. - r, err := http.Get(url) + r, err := httputil.GetWithUserAgent(url) if err != nil { log.WithError(err).Error("could not download Debian's update") return resp, commonerr.ErrCouldNotDownload } + defer r.Body.Close() + + if !httputil.Status2xx(r) { + log.WithField("StatusCode", r.StatusCode).Error("Failed to update Debian") + return resp, commonerr.ErrCouldNotDownload + } + // Parse the JSON. resp, err = buildResponse(r.Body, latestHash) if err != nil { diff --git a/ext/vulnsrc/oracle/oracle.go b/ext/vulnsrc/oracle/oracle.go index 40dcd669..16df151b 100644 --- a/ext/vulnsrc/oracle/oracle.go +++ b/ext/vulnsrc/oracle/oracle.go @@ -20,7 +20,6 @@ import ( "bufio" "encoding/xml" "io" - "net/http" "regexp" "strconv" "strings" @@ -34,6 +33,7 @@ import ( "github.com/coreos/clair/ext/versionfmt/rpm" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/httputil" ) const ( @@ -139,13 +139,18 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo } // Fetch the update list. - r, err := http.Get(ovalURI) + r, err := httputil.GetWithUserAgent(ovalURI) if err != nil { log.WithError(err).Error("could not download Oracle's update list") return resp, commonerr.ErrCouldNotDownload } defer r.Body.Close() + if !httputil.Status2xx(r) { + log.WithField("StatusCode", r.StatusCode).Error("Failed to update Oracle") + return resp, commonerr.ErrCouldNotDownload + } + // Get the list of ELSAs that we have to process. var elsaList []int scanner := bufio.NewScanner(r.Body) @@ -162,11 +167,17 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo for _, elsa := range elsaList { // Download the ELSA's XML file. - r, err := http.Get(ovalURI + elsaFilePrefix + strconv.Itoa(elsa) + ".xml") + r, err := httputil.GetWithUserAgent(ovalURI + elsaFilePrefix + strconv.Itoa(elsa) + ".xml") if err != nil { log.WithError(err).Error("could not download Oracle's update list") return resp, commonerr.ErrCouldNotDownload } + defer r.Body.Close() + + if !httputil.Status2xx(r) { + log.WithField("StatusCode", r.StatusCode).Error("Failed to update Oracle") + return resp, commonerr.ErrCouldNotDownload + } // Parse the XML. vs, err := parseELSA(r.Body) diff --git a/ext/vulnsrc/rhel/rhel.go b/ext/vulnsrc/rhel/rhel.go index f4cbce8f..1a9548e1 100644 --- a/ext/vulnsrc/rhel/rhel.go +++ b/ext/vulnsrc/rhel/rhel.go @@ -21,7 +21,6 @@ import ( "encoding/xml" "fmt" "io" - "net/http" "regexp" "strconv" "strings" @@ -33,6 +32,7 @@ import ( "github.com/coreos/clair/ext/versionfmt/rpm" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/httputil" ) const ( @@ -116,11 +116,17 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo } // Fetch the update list. - r, err := http.Get(ovalURI) + r, err := httputil.GetWithUserAgent(ovalURI) if err != nil { log.WithError(err).Error("could not download RHEL's update list") return resp, commonerr.ErrCouldNotDownload } + defer r.Body.Close() + + if !httputil.Status2xx(r) { + log.WithField("StatusCode", r.StatusCode).Error("Failed to update RHEL") + return resp, commonerr.ErrCouldNotDownload + } // Get the list of RHSAs that we have to process. var rhsaList []int @@ -138,11 +144,17 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo for _, rhsa := range rhsaList { // Download the RHSA's XML file. - r, err := http.Get(ovalURI + rhsaFilePrefix + strconv.Itoa(rhsa) + ".xml") + r, err := httputil.GetWithUserAgent(ovalURI + rhsaFilePrefix + strconv.Itoa(rhsa) + ".xml") if err != nil { log.WithError(err).Error("could not download RHEL's update list") return resp, commonerr.ErrCouldNotDownload } + defer r.Body.Close() + + if !httputil.Status2xx(r) { + log.WithField("StatusCode", r.StatusCode).Error("Failed to update RHEL") + return resp, commonerr.ErrCouldNotDownload + } // Parse the XML. vs, err := parseRHSA(r.Body) diff --git a/pkg/httputil/httputil.go b/pkg/httputil/httputil.go new file mode 100644 index 00000000..0eae6df0 --- /dev/null +++ b/pkg/httputil/httputil.go @@ -0,0 +1,65 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package httputil implements common HTTP functionality used throughout the Clair codebase. +package httputil + +import ( + "net" + "net/http" + "strings" + + "github.com/coreos/clair/pkg/version" +) + +// GetWithUserAgent performs an HTTP GET with the proper Clair User-Agent. +func GetWithUserAgent(url string) (*http.Response, error) { + client := &http.Client{} + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + req.Header.Set("User-Agent", "Clair/"+version.Version+" (https://github.com/coreos/clair)") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + return resp, nil +} + +// GetClientAddr returns the first value in X-Forwarded-For if it exists +// otherwise fall back to use RemoteAddr +func GetClientAddr(r *http.Request) string { + addr := r.RemoteAddr + if s := r.Header.Get("X-Forwarded-For"); s != "" { + ips := strings.Split(s, ",") + // assume the first one is the client address + if len(ips) != 0 { + // validate the ip + if realIP := net.ParseIP(ips[0]); realIP != nil { + addr = strings.TrimSpace(ips[0]) + } + } + } + return addr +} + +// Status2xx returns true if the response's status code is success (2xx) +func Status2xx(resp *http.Response) bool { + return resp.StatusCode/100 == 2 +} diff --git a/pkg/version/version.go b/pkg/version/version.go new file mode 100644 index 00000000..638429c7 --- /dev/null +++ b/pkg/version/version.go @@ -0,0 +1,18 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +// Version is set at build time by the latest tag +var Version string