Merge pull request #606 from MackJM/wip/master_httputil

Adding httputil and version packages to master

.dockerignore

@@ -1,4 +1,5 @@
-.*
+.dockerignore
+.travis.yml
 *.md
 DCO
 LICENSE

Dockerfile

@@ -13,9 +13,11 @@
 # limitations under the License.
 
 FROM golang:1.10-alpine AS build
+RUN apk add --no-cache git
 ADD .   /go/src/github.com/coreos/clair/
 WORKDIR /go/src/github.com/coreos/clair/
-RUN go build github.com/coreos/clair/cmd/clair
+RUN export CLAIR_VERSION=$(git describe --tag --always --dirty) && \
+	go build -ldflags "-X github.com/coreos/clair/pkg/version.Version=$CLAIR_VERSION" github.com/coreos/clair/cmd/clair
 
 FROM alpine:3.8
 COPY --from=build /go/src/github.com/coreos/clair/clair /clair

api/httputil/httputil.go

@@ -1,24 +0,0 @@
-package httputil
-
-import (
-	"net"
-	"net/http"
-	"strings"
-)
-
-// GetClientAddr returns the first value in X-Forwarded-For if it exists
-// otherwise fall back to use RemoteAddr
-func GetClientAddr(r *http.Request) string {
-	addr := r.RemoteAddr
-	if s := r.Header.Get("X-Forwarded-For"); s != "" {
-		ips := strings.Split(s, ",")
-		// assume the first one is the client address
-		if len(ips) != 0 {
-			// validate the ip
-			if realIP := net.ParseIP(ips[0]); realIP != nil {
-				addr = strings.TrimSpace(ips[0])
-			}
-		}
-	}
-	return addr
-}

ext/vulnsrc/debian/debian.go

@@ -22,7 +22,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"net/http"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
@@ -32,6 +31,7 @@ import (
 	"github.com/coreos/clair/ext/versionfmt/dpkg"
 	"github.com/coreos/clair/ext/vulnsrc"
 	"github.com/coreos/clair/pkg/commonerr"
+	"github.com/coreos/clair/pkg/httputil"
 )
 
 const (
@@ -84,12 +84,19 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo
 	}
 
 	// Download JSON.
-	r, err := http.Get(url)
+	r, err := httputil.GetWithUserAgent(url)
 	if err != nil {
 		log.WithError(err).Error("could not download Debian's update")
 		return resp, commonerr.ErrCouldNotDownload
 	}
 
+	defer r.Body.Close()
+
+	if !httputil.Status2xx(r) {
+		log.WithField("StatusCode", r.StatusCode).Error("Failed to update Debian")
+		return resp, commonerr.ErrCouldNotDownload
+	}
+
 	// Parse the JSON.
 	resp, err = buildResponse(r.Body, latestHash)
 	if err != nil {

ext/vulnsrc/oracle/oracle.go

@@ -20,7 +20,6 @@ import (
 	"bufio"
 	"encoding/xml"
 	"io"
-	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
@@ -34,6 +33,7 @@ import (
 	"github.com/coreos/clair/ext/versionfmt/rpm"
 	"github.com/coreos/clair/ext/vulnsrc"
 	"github.com/coreos/clair/pkg/commonerr"
+	"github.com/coreos/clair/pkg/httputil"
 )
 
 const (
@@ -139,13 +139,18 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo
 	}
 
 	// Fetch the update list.
-	r, err := http.Get(ovalURI)
+	r, err := httputil.GetWithUserAgent(ovalURI)
 	if err != nil {
 		log.WithError(err).Error("could not download Oracle's update list")
 		return resp, commonerr.ErrCouldNotDownload
 	}
 	defer r.Body.Close()
 
+	if !httputil.Status2xx(r) {
+		log.WithField("StatusCode", r.StatusCode).Error("Failed to update Oracle")
+		return resp, commonerr.ErrCouldNotDownload
+	}
+
 	// Get the list of ELSAs that we have to process.
 	var elsaList []int
 	scanner := bufio.NewScanner(r.Body)
@@ -162,11 +167,17 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo
 
 	for _, elsa := range elsaList {
 		// Download the ELSA's XML file.
-		r, err := http.Get(ovalURI + elsaFilePrefix + strconv.Itoa(elsa) + ".xml")
+		r, err := httputil.GetWithUserAgent(ovalURI + elsaFilePrefix + strconv.Itoa(elsa) + ".xml")
 		if err != nil {
 			log.WithError(err).Error("could not download Oracle's update list")
 			return resp, commonerr.ErrCouldNotDownload
 		}
+		defer r.Body.Close()
+
+		if !httputil.Status2xx(r) {
+			log.WithField("StatusCode", r.StatusCode).Error("Failed to update Oracle")
+			return resp, commonerr.ErrCouldNotDownload
+		}
 
 		// Parse the XML.
 		vs, err := parseELSA(r.Body)

ext/vulnsrc/rhel/rhel.go

@@ -21,7 +21,6 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
@@ -33,6 +32,7 @@ import (
 	"github.com/coreos/clair/ext/versionfmt/rpm"
 	"github.com/coreos/clair/ext/vulnsrc"
 	"github.com/coreos/clair/pkg/commonerr"
+	"github.com/coreos/clair/pkg/httputil"
 )
 
 const (
@@ -116,11 +116,17 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo
 	}
 
 	// Fetch the update list.
-	r, err := http.Get(ovalURI)
+	r, err := httputil.GetWithUserAgent(ovalURI)
 	if err != nil {
 		log.WithError(err).Error("could not download RHEL's update list")
 		return resp, commonerr.ErrCouldNotDownload
 	}
+	defer r.Body.Close()
+
+	if !httputil.Status2xx(r) {
+		log.WithField("StatusCode", r.StatusCode).Error("Failed to update RHEL")
+		return resp, commonerr.ErrCouldNotDownload
+	}
 
 	// Get the list of RHSAs that we have to process.
 	var rhsaList []int
@@ -138,11 +144,17 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo
 
 	for _, rhsa := range rhsaList {
 		// Download the RHSA's XML file.
-		r, err := http.Get(ovalURI + rhsaFilePrefix + strconv.Itoa(rhsa) + ".xml")
+		r, err := httputil.GetWithUserAgent(ovalURI + rhsaFilePrefix + strconv.Itoa(rhsa) + ".xml")
 		if err != nil {
 			log.WithError(err).Error("could not download RHEL's update list")
 			return resp, commonerr.ErrCouldNotDownload
 		}
+		defer r.Body.Close()
+
+		if !httputil.Status2xx(r) {
+			log.WithField("StatusCode", r.StatusCode).Error("Failed to update RHEL")
+			return resp, commonerr.ErrCouldNotDownload
+		}
 
 		// Parse the XML.
 		vs, err := parseRHSA(r.Body)

pkg/httputil/httputil.go

@@ -0,0 +1,65 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package httputil implements common HTTP functionality used throughout the Clair codebase.
+package httputil
+
+import (
+	"net"
+	"net/http"
+	"strings"
+
+	"github.com/coreos/clair/pkg/version"
+)
+
+// GetWithUserAgent performs an HTTP GET with the proper Clair User-Agent.
+func GetWithUserAgent(url string) (*http.Response, error) {
+	client := &http.Client{}
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("User-Agent", "Clair/"+version.Version+" (https://github.com/coreos/clair)")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	return resp, nil
+}
+
+// GetClientAddr returns the first value in X-Forwarded-For if it exists
+// otherwise fall back to use RemoteAddr
+func GetClientAddr(r *http.Request) string {
+	addr := r.RemoteAddr
+	if s := r.Header.Get("X-Forwarded-For"); s != "" {
+		ips := strings.Split(s, ",")
+		// assume the first one is the client address
+		if len(ips) != 0 {
+			// validate the ip
+			if realIP := net.ParseIP(ips[0]); realIP != nil {
+				addr = strings.TrimSpace(ips[0])
+			}
+		}
+	}
+	return addr
+}
+
+// Status2xx returns true if the response's status code is success (2xx)
+func Status2xx(resp *http.Response) bool {
+	return resp.StatusCode/100 == 2
+}

pkg/version/version.go

@@ -0,0 +1,18 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package version
+
+// Version is set at build time by the latest tag
+var Version string