implement nodejs fetcher

Signed-off-by: liang chenye <liangchenye@huawei.com>

cmd/clair/main.go

@@ -29,6 +29,7 @@ import (
 	_ "github.com/coreos/clair/notifier/notifiers"
 
 	_ "github.com/coreos/clair/updater/fetchers/debian"
+	_ "github.com/coreos/clair/updater/fetchers/nodejs"
 	_ "github.com/coreos/clair/updater/fetchers/rhel"
 	_ "github.com/coreos/clair/updater/fetchers/ubuntu"
 	_ "github.com/coreos/clair/updater/metadata_fetchers/nvd"

updater/fetchers/nodejs/nodejs.go

@@ -0,0 +1,182 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nodejs
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/updater"
+	cerrors "github.com/coreos/clair/utils/errors"
+	"github.com/coreos/clair/utils/types"
+	"github.com/coreos/pkg/capnslog"
+)
+
+const (
+	url                  = "https://api.nodesecurity.io/advisories"
+	cveURLPrefix         = "http://cve.mitre.org/cgi-bin/cvename.cgi?name="
+	updaterFlag          = "nodejsUpdater"
+	defaultNodejsVersion = "all"
+	//FIXME: Add a suffix when an advisory is fixed `after` a certain version.
+	defaultVersionSuffix = "-1"
+)
+
+var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/nodejs")
+
+type nodejsAdvisory struct {
+	ID              int      `json:"id"`
+	UpdatedAt       string   `json:"updated_at"`
+	ModuleName      string   `json:"module_name"`
+	CVES            []string `json:"cves"`
+	PatchedVersions string   `json:"patched_versions"`
+	Overview        string   `json:"overview"`
+	CvssScore       float32  `json:"cvss_score"`
+}
+
+type nodejsAdvisories struct {
+	Total   int              `json:"total"`
+	Count   int              `json:"count"`
+	Offset  int              `json:"offset"`
+	Results []nodejsAdvisory `json:"results"`
+}
+
+// NodejsFetcher implements updater.Fetcher for the Node Security Project
+// (https://nodesecurity.io).
+type NodejsFetcher struct{}
+
+func init() {
+	updater.RegisterFetcher("nodejs", &NodejsFetcher{})
+}
+
+// FetchUpdate fetches vulnerability updates from the Node Security Project.
+func (fetcher *NodejsFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
+	log.Info("fetching Nodejs vulnerabilities")
+
+	// Download JSON.
+	r, err := http.Get(url)
+	if err != nil {
+		log.Errorf("could not download Nodejs's update: %s", err)
+		return resp, cerrors.ErrCouldNotDownload
+	}
+	defer r.Body.Close()
+
+	// Get the latest date of the latest update's JSON data
+	latestUpdate, err := datastore.GetKeyValue(updaterFlag)
+	if err != nil {
+		return resp, err
+	}
+
+	// Unmarshal JSON.
+	var advisories nodejsAdvisories
+	if err = json.NewDecoder(r.Body).Decode(&advisories); err != nil {
+		log.Errorf("could not unmarshal Nodejs's JSON: %s", err)
+		return resp, cerrors.ErrCouldNotParse
+	}
+
+	resp.Vulnerabilities, resp.FlagValue = parseNodejsAdvisories(advisories.Results, latestUpdate)
+	resp.FlagName = updaterFlag
+
+	return resp, nil
+}
+
+func parseNodejsAdvisories(advisories []nodejsAdvisory, latestUpdate string) (vulnerabilities []database.Vulnerability, newUpdated string) {
+	mvulnerabilities := make(map[string]*database.Vulnerability)
+
+	for _, advisory := range advisories {
+		if latestUpdate >= advisory.UpdatedAt {
+			break
+		}
+		if advisory.UpdatedAt > newUpdated {
+			newUpdated = advisory.UpdatedAt
+		}
+		for _, vulnName := range advisory.CVES {
+			// Get or create the vulnerability.
+			vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[vulnName]
+			if !vulnerabilityAlreadyExists {
+				vulnerability = &database.Vulnerability{
+					Name:        vulnName,
+					Link:        cveURLPrefix + strings.TrimLeft(vulnName, "CVE-"),
+					Severity:    types.Unknown,
+					Description: advisory.Overview,
+				}
+			}
+
+			// Set the priority of the vulnerability.
+			// A vulnerability has one urgency per advisory it affects.
+			// The highest urgency should be the one set.
+			if urgency := types.ScoreToPriority(advisory.CvssScore); urgency.Compare(vulnerability.Severity) > 0 {
+				vulnerability.Severity = urgency
+			}
+
+			// Create and add the feature version.
+			pkg := database.FeatureVersion{
+				Feature: database.Feature{
+					Name: advisory.ModuleName,
+					Namespace: database.Namespace{
+						Name: "nodejs:" + defaultNodejsVersion,
+					},
+				},
+			}
+			if version, err := getAdvisoryVersion(advisory.PatchedVersions); err == nil {
+				pkg.Version = version
+			}
+			vulnerability.FixedIn = append(vulnerability.FixedIn, pkg)
+
+			// Store the vulnerability.
+			mvulnerabilities[vulnName] = vulnerability
+		}
+	}
+
+	// Convert the vulnerabilities map to a slice
+	for _, v := range mvulnerabilities {
+		vulnerabilities = append(vulnerabilities, *v)
+	}
+
+	return
+}
+
+// getAdvisoryVersion parses a string containing one or multiple version ranges
+// and returns upper-bound. By nature, this simplification may lead to false-positives
+func getAdvisoryVersion(fullVersion string) (types.Version, error) {
+	fixedVersion := types.MinVersion
+
+	for _, version := range strings.Split(fullVersion, "||") {
+		ovs := getOperVersions(version)
+		for _, ov := range ovs {
+			if ov.Oper == ">" {
+				if curVersion, err := types.NewVersion(ov.Version + defaultVersionSuffix); err != nil {
+					log.Warningf("could not parse package version '%s': %s. skipping", curVersion, err.Error())
+				} else if curVersion.Compare(fixedVersion) > 0 {
+					fixedVersion = curVersion
+				}
+			} else if ov.Oper == ">=" {
+				if curVersion, err := types.NewVersion(ov.Version); err != nil {
+					log.Warningf("could not parse package version '%s': %s. skipping", curVersion, err.Error())
+				} else if curVersion.Compare(fixedVersion) > 0 {
+					fixedVersion = curVersion
+				}
+			}
+		}
+	}
+	if fixedVersion != types.MinVersion {
+		return fixedVersion, nil
+	}
+	return types.MaxVersion, cerrors.ErrNotFound
+}
+
+// Clean deletes any allocated resources.
+func (fetcher *NodejsFetcher) Clean() {}

updater/fetchers/nodejs/nodejs_test.go

@@ -0,0 +1,86 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nodejs
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/utils/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNodejsParser(t *testing.T) {
+	_, filename, _, _ := runtime.Caller(0)
+	testFile, _ := os.Open(filepath.Join(filepath.Dir(filename)) + "/testdata/fetcher_nodejs_test.json")
+	defer testFile.Close()
+
+	var advisories nodejsAdvisories
+	json.NewDecoder(testFile).Decode(&advisories)
+	assert.Len(t, advisories.Results, 4)
+
+	vulnerabilities, lastUpdated := parseNodejsAdvisories(advisories.Results, "")
+	assert.Len(t, vulnerabilities, 2)
+	assert.Equal(t, "2016-05-13T20:39:38+00:00", lastUpdated)
+
+	for _, vulnerability := range vulnerabilities {
+		if vulnerability.Name == "CVE-2015-7294" {
+			assert.Equal(t, "http://cve.mitre.org/cgi-bin/cvename.cgi?name=2015-7294", vulnerability.Link)
+			assert.Equal(t, types.Medium, vulnerability.Severity)
+			assert.Equal(t, "ldapauth versions <= 2.2.4 are vulnerable to ldap injection through the username parameter.", vulnerability.Description)
+			expectedFeatureVersions := []database.FeatureVersion{
+				{
+					Feature: database.Feature{
+						Namespace: database.Namespace{Name: "nodejs:" + defaultNodejsVersion},
+						Name:      "ldapauth",
+					},
+					Version: types.NewVersionUnsafe("2.2.4" + defaultVersionSuffix),
+				},
+				{
+					Feature: database.Feature{
+						Namespace: database.Namespace{Name: "nodejs:" + defaultNodejsVersion},
+						Name:      "ldapauth-fork",
+					},
+					Version: types.NewVersionUnsafe("2.3.3"),
+				},
+			}
+			for _, expectedFeatureVersion := range expectedFeatureVersions {
+				assert.Contains(t, vulnerability.FixedIn, expectedFeatureVersion)
+			}
+		} else if vulnerability.Name == "CVE-2016-2515" {
+			assert.Equal(t, "http://cve.mitre.org/cgi-bin/cvename.cgi?name=2016-2515", vulnerability.Link)
+			assert.Equal(t, types.Medium, vulnerability.Severity)
+			assert.Equal(t, "Specifically crafted long headers or uris can cause a minor denial of service when using hawk versions less than 4.1.1.\n\n\"The Regular expression Denial of Service (ReDoS) is a Denial of Service attack, that exploits the fact that most Regular Expression implementations may reach extreme situations that cause them to work very slowly (exponentially related to input size). An attacker can then cause a program using a Regular Expression to enter these extreme situations and then hang for a very long time.\"\n\nUpdates:\n- Updated to include fix in 3.1.3 ", vulnerability.Description)
+			expectedFeatureVersions := []database.FeatureVersion{
+				{
+					Feature: database.Feature{
+						Namespace: database.Namespace{Name: "nodejs:" + defaultNodejsVersion},
+						Name:      "hawk",
+					},
+					Version: types.NewVersionUnsafe("4.1.1"),
+				},
+			}
+			for _, expectedFeatureVersion := range expectedFeatureVersions {
+				assert.Contains(t, vulnerability.FixedIn, expectedFeatureVersion)
+			}
+		}
+	}
+
+	return
+}

updater/fetchers/nodejs/nodejs_version.go

@@ -0,0 +1,76 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nodejs
+
+import (
+	"strings"
+	"unicode"
+)
+
+type operVersion struct {
+	Oper    string
+	Version string
+}
+
+type ovState string
+
+const (
+	ovStateInit    ovState = "init"
+	ovStateOper    ovState = "operation"
+	ovStateVersion ovState = "version"
+)
+
+func isOper(ch rune) bool {
+	return ch == '>' || ch == '<' || ch == '='
+}
+
+func getOperVersions(content string) (ovs []operVersion) {
+	state := ovStateInit
+	begin := 0
+	var ov operVersion
+	for i, ch := range content {
+		if unicode.IsSpace(ch) {
+			continue
+		}
+		switch state {
+		case ovStateInit:
+			if isOper(ch) {
+				state = ovStateOper
+				begin = i
+			} else {
+				return nil
+			}
+		case ovStateOper:
+			if !isOper(ch) {
+				state = ovStateVersion
+				ov.Oper = strings.TrimSpace(content[begin:i])
+				begin = i
+			}
+		case ovStateVersion:
+			if isOper(ch) {
+				state = ovStateOper
+				ov.Version = strings.TrimSpace(content[begin:i])
+				ovs = append(ovs, ov)
+				begin = i
+			}
+		}
+	}
+	if state == ovStateVersion {
+		ov.Version = strings.TrimSpace(content[begin:len(content)])
+		ovs = append(ovs, ov)
+	}
+
+	return
+}

updater/fetchers/nodejs/nodejs_version_test.go

@@ -0,0 +1,53 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nodejs
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNodeVersion(t *testing.T) {
+	invalid_version := "3.1.3 < 4.0.0 || >=  "
+	versions := strings.Split(invalid_version, "||")
+	for _, version := range versions {
+		ovs := getOperVersions(version)
+		assert.Len(t, ovs, 0)
+	}
+
+	valid_version := ">=3.1.3 < 4.0.0 || >=4.1.1"
+	versions = strings.Split(valid_version, "||")
+	for _, version := range versions {
+		if strings.Contains(version, "4.1.1") {
+			ovs := getOperVersions(version)
+			assert.Len(t, ovs, 1)
+			assert.Equal(t, ">=", ovs[0].Oper)
+			assert.Equal(t, "4.1.1", ovs[0].Version)
+		} else {
+			ovs := getOperVersions(version)
+			assert.Len(t, ovs, 2)
+
+			for _, ov := range ovs {
+				if ov.Oper == ">=" {
+					assert.Equal(t, "3.1.3", ov.Version)
+				} else if ov.Oper == "<" {
+					assert.Equal(t, "4.0.0", ov.Version)
+				}
+			}
+		}
+	}
+}

updater/fetchers/nodejs/testdata/fetcher_nodejs_test.json

@@ -0,0 +1,105 @@
+{
+	"Total": 4,
+	"Count": 4,
+	"Offset": 0,
+	"Results": [
+		{
+			"ID": 19,
+			"Created_at": "2015-10-17T19:41:46.382+00:00",
+			"Updated_at": "2016-04-25T15:31:07+00:00",
+			"Publish_date": "2015-09-18T19:30:10+00:00",
+			"Title": "LDAP Injection",
+			"Author": "David Black, Jerome Touffe-Blin",
+			"Module_name": "ldapauth",
+			"CVES": [
+				"CVE-2015-7294"
+			],
+			"Vulnerable_versions": "<=2.2.4",
+			"Patched_versions": "> 2.2.4",
+			"Slug": "ldapauth_ldap-injection",
+			"Overview": "ldapauth versions <= 2.2.4 are vulnerable to ldap injection through the username parameter.",
+			"Recommandation": "",
+			"References": "- http://www.openwall.com/lists/oss-security/2015/09/18/4",
+			"Legacy_slug": "ldapauth-ldap-injection",
+			"Allowed_scopes": [
+				"public",
+				"admin"
+			],
+			"CVEs_vector": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:N",
+			"Cvss_score": 5.3
+		},
+		{
+			"ID": 18,
+			"Created_at": "2015-10-17T19:41:46.382+00:00",
+			"Updated_at": "2016-04-28T16:50:25+00:00",
+			"Publish_date": "2015-09-18T19:29:10+00:00",
+			"Title": "LDAP Injection",
+			"Author": "Jerome Touffe-Blin",
+			"Module_name": "ldapauth-fork",
+			"CVES": [
+				"CVE-2015-7294"
+			],
+			"Vulnerable_versions": "< 2.3.3",
+			"Patched_versions": ">= 2.3.3",
+			"Slug": "ldapauth-fork_ldap-injection",
+			"Overview": "ldapauth-fork is a module forked from node-ldapauth and is used for ldap authentication  \nThe username parameter is not filtered as per [LDAP Escape Specifications](https://tools.ietf.org/search/rfc4515#section-3)  \nA malicious user is able to change their name to certain LDAP commands and run anything that they want.",
+			"Recommandation": "",
+			"References": "- https://github.com/vesse/node-ldapauth-fork/issues/21\n- https://github.com/vesse/node-ldapauth-fork/commit/3feea43e243698bcaeffa904a7324f4d96df60e4",
+			"Legacy_slug": "ldapauth-fork-ldap-injection",
+			"Allowed_scopes": [
+				"public",
+				"admin"
+			],
+			"CVEs_vector": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:N",
+			"Cvss_score": 6.5
+		},
+		{
+			"ID": 99,
+			"Created_at": "2016-04-04T19:46:25+00:00",
+			"Updated_at": "2016-05-13T20:39:38+00:00",
+			"Publish_date": "2016-04-26T16:24:32+00:00",
+			"Title": "Insecure Defaults Allow MITM Over TLS",
+			"Author": "David Johansson",
+			"Module_name": "engine.io-client",
+			"CVES": [],
+			"Vulnerable_versions": "<= 1.6.8",
+			"Patched_versions": ">= 1.6.9",
+			"Slug": "engineio-client_tls-connections-over-websockets-vulnerable-to-mitm",
+			"Overview": "engine.io-client is the client for [engine.io](https://github.com/socketio/engine.io), the implementation of a transport-based cross-browser/cross-device bi-directional communication layer for Socket.IO.\n\nThe vulnerability is related to the way that node.js handles the `rejectUnauthorized` setting. If the value is something that evaluates to false, certificate verification will be disabled.\n\nThis is problematic as engine.io-client passes in an object for settings that includes the rejectUnauthorized property, whether it has been set or not. If the value has not been explicitly changed, it will be passed in as `null`, resulting in certificate verification being turned off:\n\n``` \n // line that causes bug\nthis.rejectUnauthorized = opts.rejectUnauthorized === undefined ? null : opts.rejectUnauthorized;\n ```",
+			"Recommandation": "",
+			"References": "- https://github.com/socketio/engine.io-client/commit/2c55b278a491bf45313ecc0825cf800e2f7ff5c1\n- https://www.cigital.com/blog/node-js-socket-io/",
+			"Legacy_slug": "",
+			"Allowed_scopes": [
+				"admin",
+				"public"
+			],
+			"CVEs_vector": "CVSS:3.0/AV:A/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:N",
+			"Cvss_score": 6.8
+		},
+		{
+			"ID": 77,
+			"Created_at": "2016-01-19T21:50:30.175+00:00",
+			"Updated_at": "2016-04-21T00:16:58+00:00",
+			"Publish_date": "2016-01-19T21:51:35.396+00:00",
+			"Title": "Regular Expression Denial of Service",
+			"Author": "Adam Baldwin",
+			"Module_name": "hawk",
+			"CVES": [
+				"CVE-2016-2515"
+			],
+			"Vulnerable_versions": "< 3.1.3  || >= 4.0.0 < 4.1.1",
+			"Patched_versions": ">=3.1.3 < 4.0.0 || >=4.1.1",
+			"Slug": "hawk_regular-expression-denial-of-service",
+			"Overview": "Specifically crafted long headers or uris can cause a minor denial of service when using hawk versions less than 4.1.1.\n\n\"The Regular expression Denial of Service (ReDoS) is a Denial of Service attack, that exploits the fact that most Regular Expression implementations may reach extreme situations that cause them to work very slowly (exponentially related to input size). An attacker can then cause a program using a Regular Expression to enter these extreme situations and then hang for a very long time.\"\n\nUpdates:\n- Updated to include fix in 3.1.3 ",
+			"Recommandation": "",
+			"References": "- https://github.com/hueniverse/hawk/issues/168\n- https://www.owasp.org/index.php/Regular_expression_Denial_of_Service_-_ReDoS",
+			"Legacy_slug": "",
+			"Allowed_scopes": [
+				"admin",
+				"public"
+			],
+			"CVEs_vector": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:L",
+			"Cvss_score": 5.3
+		}
+	]
+}

utils/types/priority.go

@@ -108,3 +108,22 @@ func (p *Priority) Scan(value interface{}) error {
 func (p *Priority) Value() (driver.Value, error) {
 	return string(*p), nil
 }
+
+// ScoreToPriority return a priority from a cvss score on the scale of 0 to 10.
+func ScoreToPriority(score float32) Priority {
+	if score < 0.0 {
+		return Unknown
+	} else if score < 1.0 {
+		return Negligible
+	} else if score <= 3.9 {
+		return Low
+	} else if score <= 6.9 {
+		return Medium
+	} else if score <= 8.9 {
+		return High
+	} else if score <= 10.0 {
+		return Critical
+	} else {
+		return Unknown
+	}
+}

utils/types/priority_test.go

@@ -30,3 +30,13 @@ func TestIsValid(t *testing.T) {
 	assert.False(t, Priority("Test").IsValid())
 	assert.True(t, Unknown.IsValid())
 }
+
+func TestScoreToPriority(t *testing.T) {
+	assert.Equal(t, ScoreToPriority(-1.0), Unknown)
+	assert.Equal(t, ScoreToPriority(0.5), Negligible)
+	assert.Equal(t, ScoreToPriority(2.0), Low)
+	assert.Equal(t, ScoreToPriority(5.0), Medium)
+	assert.Equal(t, ScoreToPriority(7.0), High)
+	assert.Equal(t, ScoreToPriority(9.0), Critical)
+	assert.Equal(t, ScoreToPriority(12.0), Unknown)
+}