Merge pull request #638 from KeyboardNerd/featureTree

Parse Source package from package information databases
master
Sida Chen 6 years ago committed by GitHub
commit fe614f2b01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -165,6 +165,12 @@ type Feature struct {
Name string
Version string
VersionFormat string
// Parent feature indicates that the vulnerability affects parent feature
// will also affect this feature.
//
// e.g. A source package is the parent feature of a binary package.
Parent *Feature
}
// NamespacedFeature is a feature with determined namespace and can be affected

@ -26,10 +26,10 @@ import (
// int keys must be the consistent with the database ID.
var (
realFeatures = map[int]database.Feature{
1: {"ourchat", "0.5", "dpkg"},
2: {"openssl", "1.0", "dpkg"},
3: {"openssl", "2.0", "dpkg"},
4: {"fake", "2.0", "rpm"},
1: {"ourchat", "0.5", "dpkg", nil},
2: {"openssl", "1.0", "dpkg", nil},
3: {"openssl", "2.0", "dpkg", nil},
4: {"fake", "2.0", "rpm", nil},
}
realNamespaces = map[int]database.Namespace{

@ -19,6 +19,7 @@ import (
"bufio"
"bytes"
"github.com/deckarep/golang-set"
log "github.com/sirupsen/logrus"
"github.com/coreos/clair/database"
@ -34,6 +35,16 @@ func init() {
type lister struct{}
func valid(pkg *featurefmt.PackageInfo) bool {
return pkg.PackageName != "" && pkg.PackageVersion != ""
}
func addSourceVersion(pkg *featurefmt.PackageInfo) {
if pkg.SourceName != "" {
pkg.SourceVersion = pkg.PackageVersion
}
}
func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) {
file, exists := files["lib/apk/db/installed"]
if !exists {
@ -43,49 +54,45 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)
// Iterate over each line in the "installed" file attempting to parse each
// package into a feature that will be stored in a set to guarantee
// uniqueness.
pkgSet := make(map[string]database.Feature)
ipkg := database.Feature{}
packages := mapset.NewSet()
pkg := featurefmt.PackageInfo{}
scanner := bufio.NewScanner(bytes.NewBuffer(file))
for scanner.Scan() {
line := scanner.Text()
if len(line) < 2 {
if valid(&pkg) {
addSourceVersion(&pkg)
packages.Add(pkg)
pkg.Reset()
}
continue
}
// Parse the package name or version.
switch {
case line[:2] == "P:":
ipkg.Name = line[2:]
case line[:2] == "V:":
switch line[:2] {
case "P:":
pkg.PackageName = line[2:]
case "V:":
version := string(line[2:])
err := versionfmt.Valid(dpkg.ParserName, version)
if err != nil {
log.WithError(err).WithField("version", version).Warning("could not parse package version. skipping")
continue
} else {
ipkg.Version = version
pkg.PackageVersion = version
}
case line == "":
// Restart if the parser reaches another package definition before
// creating a valid package.
ipkg = database.Feature{}
}
// If we have a whole feature, store it in the set and try to parse a new
// one.
if ipkg.Name != "" && ipkg.Version != "" {
pkgSet[ipkg.Name+"#"+ipkg.Version] = ipkg
ipkg = database.Feature{}
case "o:":
pkg.SourceName = line[2:]
}
}
// Convert the map into a slice and attach the version format
pkgs := make([]database.Feature, 0, len(pkgSet))
for _, pkg := range pkgSet {
pkg.VersionFormat = dpkg.ParserName
pkgs = append(pkgs, pkg)
// in case of no terminal line
if valid(&pkg) {
addSourceVersion(&pkg)
packages.Add(pkg)
}
return pkgs, nil
return featurefmt.PackageSetToFeatures(dpkg.ParserName, packages), nil
}
func (l lister) RequiredFilenames() []string {

@ -17,38 +17,30 @@ package apk
import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/pkg/tarutil"
)
func TestAPKFeatureDetection(t *testing.T) {
testFeatures := []database.Feature{
{Name: "musl", Version: "1.1.14-r10"},
{Name: "busybox", Version: "1.24.2-r9"},
{Name: "alpine-baselayout", Version: "3.0.3-r0"},
{Name: "alpine-keys", Version: "1.1-r0"},
{Name: "zlib", Version: "1.2.8-r2"},
{Name: "libcrypto1.0", Version: "1.0.2h-r1"},
{Name: "libssl1.0", Version: "1.0.2h-r1"},
{Name: "apk-tools", Version: "2.6.7-r0"},
{Name: "scanelf", Version: "1.1.6-r0"},
{Name: "musl-utils", Version: "1.1.14-r10"},
{Name: "libc-utils", Version: "0.7-r0"},
}
for i := range testFeatures {
testFeatures[i].VersionFormat = dpkg.ParserName
}
testData := []featurefmt.TestData{
for _, test := range []featurefmt.TestCase{
{
Features: testFeatures,
Files: tarutil.FilesMap{
"lib/apk/db/installed": featurefmt.LoadFileForTest("apk/testdata/installed"),
"valid case",
map[string]string{"lib/apk/db/installed": "apk/testdata/valid"},
[]featurefmt.PackageInfo{
{"musl", "1.1.14-r10", "", ""},
{"busybox", "1.24.2-r9", "", ""},
{"alpine-baselayout", "3.0.3-r0", "", ""},
{"alpine-keys", "1.1-r0", "", ""},
{"zlib", "1.2.8-r2", "", ""},
{"libcrypto1.0", "1.0.2h-r1", "openssl", "1.0.2h-r1"},
{"libssl1.0", "1.0.2h-r1", "openssl", "1.0.2h-r1"},
{"apk-tools", "2.6.7-r0", "", ""},
{"scanelf", "1.1.6-r0", "pax-utils", "1.1.6-r0"},
{"musl-utils", "1.1.14-r10", "musl", "1.1.14-r10"},
{"libc-utils", "0.7-r0", "libc-dev", "0.7-r0"},
},
},
} {
featurefmt.RunTest(t, test, lister{}, dpkg.ParserName)
}
featurefmt.TestLister(t, &lister{}, testData)
}

@ -20,6 +20,7 @@ import (
"regexp"
"strings"
"github.com/deckarep/golang-set"
log "github.com/sirupsen/logrus"
"github.com/coreos/clair/database"
@ -40,29 +41,39 @@ func init() {
featurefmt.RegisterLister("dpkg", "1.0", &lister{})
}
func valid(pkg *featurefmt.PackageInfo) bool {
return pkg.PackageName != "" && pkg.PackageVersion != ""
}
func addSourceVersion(pkg *featurefmt.PackageInfo) {
if pkg.SourceName != "" && pkg.SourceVersion == "" {
pkg.SourceVersion = pkg.PackageVersion
}
}
func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) {
f, hasFile := files["var/lib/dpkg/status"]
if !hasFile {
return []database.Feature{}, nil
}
// Create a map to store packages and ensure their uniqueness
packagesMap := make(map[string]database.Feature)
var (
pkg featurefmt.PackageInfo
pkgs = mapset.NewSet()
err error
)
var pkg database.Feature
var err error
scanner := bufio.NewScanner(strings.NewReader(string(f)))
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "Package: ") {
// Package line
// Defines the name of the package
pkg.Name = strings.TrimSpace(strings.TrimPrefix(line, "Package: "))
pkg.Version = ""
pkg.PackageName = strings.TrimSpace(strings.TrimPrefix(line, "Package: "))
pkg.PackageVersion = ""
} else if strings.HasPrefix(line, "Source: ") {
// Source line (Optionnal)
// Source line (Optional)
// Gives the name of the source package
// May also specifies a version
@ -72,50 +83,38 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)
md[dpkgSrcCaptureRegexpNames[i]] = strings.TrimSpace(n)
}
pkg.Name = md["name"]
pkg.SourceName = md["name"]
if md["version"] != "" {
version := md["version"]
err = versionfmt.Valid(dpkg.ParserName, version)
if err != nil {
if err = versionfmt.Valid(dpkg.ParserName, version); err != nil {
log.WithError(err).WithField("version", string(line[1])).Warning("could not parse package version. skipping")
} else {
pkg.Version = version
pkg.SourceVersion = version
}
}
} else if strings.HasPrefix(line, "Version: ") && pkg.Version == "" {
} else if strings.HasPrefix(line, "Version: ") {
// Version line
// Defines the version of the package
// This version is less important than a version retrieved from a Source line
// because the Debian vulnerabilities often skips the epoch from the Version field
// which is not present in the Source version, and because +bX revisions don't matter
version := strings.TrimPrefix(line, "Version: ")
err = versionfmt.Valid(dpkg.ParserName, version)
if err != nil {
if err = versionfmt.Valid(dpkg.ParserName, version); err != nil {
log.WithError(err).WithField("version", string(line[1])).Warning("could not parse package version. skipping")
} else {
pkg.Version = version
pkg.PackageVersion = version
}
} else if line == "" {
pkg.Name = ""
pkg.Version = ""
pkg.Reset()
}
// Add the package to the result array if we have all the informations
if pkg.Name != "" && pkg.Version != "" {
packagesMap[pkg.Name+"#"+pkg.Version] = pkg
pkg.Name = ""
pkg.Version = ""
if valid(&pkg) {
addSourceVersion(&pkg)
pkgs.Add(pkg)
}
}
// Convert the map to a slice and add version format.
packages := make([]database.Feature, 0, len(packagesMap))
for _, pkg := range packagesMap {
pkg.VersionFormat = dpkg.ParserName
packages = append(packages, pkg)
}
return packages, nil
return featurefmt.PackageSetToFeatures(dpkg.ParserName, pkgs), nil
}
func (l lister) RequiredFilenames() []string {

@ -17,42 +17,118 @@ package dpkg
import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/pkg/tarutil"
)
func TestDpkgFeatureDetection(t *testing.T) {
testFeatures := []database.Feature{
// Two packages from this source are installed, it should only appear one time
func TestListFeatures(t *testing.T) {
for _, test := range []featurefmt.TestCase{
{
Name: "pam",
Version: "1.1.8-3.1ubuntu3",
},
{
Name: "makedev", // The source name and the package name are equals
Version: "2.3.1-93ubuntu1", // The version comes from the "Version:" line
},
{
Name: "gcc-5",
Version: "5.1.1-12ubuntu1", // The version comes from the "Source:" line
"valid status file",
map[string]string{"var/lib/dpkg/status": "dpkg/testdata/valid"},
[]featurefmt.PackageInfo{
{"adduser", "3.116ubuntu1", "", ""},
{"apt", "1.6.3ubuntu0.1", "", ""},
{"base-files", "10.1ubuntu2.2", "", ""},
{"base-passwd", "3.5.44", "", ""},
{"bash", "4.4.18-2ubuntu1", "", ""},
{"bsdutils", "1:2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"bzip2", "1.0.6-8.1", "", ""},
{"coreutils", "8.28-1ubuntu1", "", ""},
{"dash", "0.5.8-2.10", "", ""},
{"debconf", "1.5.66", "", ""},
{"debianutils", "4.8.4", "", ""},
{"diffutils", "1:3.6-1", "", ""},
{"dpkg", "1.19.0.5ubuntu2", "", ""},
{"e2fsprogs", "1.44.1-1", "", ""},
{"fdisk", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"findutils", "4.6.0+git+20170828-2", "", ""},
{"gcc-8-base", "8-20180414-1ubuntu2", "gcc-8", "8-20180414-1ubuntu2"},
{"gpgv", "2.2.4-1ubuntu1.1", "gnupg2", "2.2.4-1ubuntu1.1"},
{"grep", "3.1-2", "", ""},
{"gzip", "1.6-5ubuntu1", "", ""},
{"hostname", "3.20", "", ""},
{"init-system-helpers", "1.51", "", ""},
{"libacl1", "2.2.52-3build1", "acl", "2.2.52-3build1"},
{"libapt-pkg5.0", "1.6.3ubuntu0.1", "apt", "1.6.3ubuntu0.1"},
{"libattr1", "1:2.4.47-2build1", "attr", "1:2.4.47-2build1"},
{"libaudit-common", "1:2.8.2-1ubuntu1", "audit", "1:2.8.2-1ubuntu1"},
{"libaudit1", "1:2.8.2-1ubuntu1", "audit", "1:2.8.2-1ubuntu1"},
{"libblkid1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"libbz2-1.0", "1.0.6-8.1", "bzip2", "1.0.6-8.1"},
{"libc-bin", "2.27-3ubuntu1", "glibc", "2.27-3ubuntu1"},
{"libc6", "2.27-3ubuntu1", "glibc", "2.27-3ubuntu1"},
{"libcap-ng0", "0.7.7-3.1", "libcap-ng", "0.7.7-3.1"},
{"libcom-err2", "1.44.1-1", "e2fsprogs", "1.44.1-1"},
{"libdb5.3", "5.3.28-13.1ubuntu1", "db5.3", "5.3.28-13.1ubuntu1"},
{"libdebconfclient0", "0.213ubuntu1", "cdebconf", "0.213ubuntu1"},
{"libext2fs2", "1.44.1-1", "e2fsprogs", "1.44.1-1"},
{"libfdisk1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"libffi6", "3.2.1-8", "libffi", "3.2.1-8"},
{"libgcc1", "1:8-20180414-1ubuntu2", "gcc-8", "8-20180414-1ubuntu2"},
{"libgcrypt20", "1.8.1-4ubuntu1.1", "", ""},
{"libgmp10", "2:6.1.2+dfsg-2", "gmp", "2:6.1.2+dfsg-2"},
{"libgnutls30", "3.5.18-1ubuntu1", "gnutls28", "3.5.18-1ubuntu1"},
{"libgpg-error0", "1.27-6", "libgpg-error", "1.27-6"},
{"libhogweed4", "3.4-1", "nettle", "3.4-1"},
{"libidn2-0", "2.0.4-1.1build2", "libidn2", "2.0.4-1.1build2"},
{"liblz4-1", "0.0~r131-2ubuntu3", "lz4", "0.0~r131-2ubuntu3"},
{"liblzma5", "5.2.2-1.3", "xz-utils", "5.2.2-1.3"},
{"libmount1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"libncurses5", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04"},
{"libncursesw5", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04"},
{"libnettle6", "3.4-1", "nettle", "3.4-1"},
{"libp11-kit0", "0.23.9-2", "p11-kit", "0.23.9-2"},
{"libpam-modules", "1.1.8-3.6ubuntu2", "pam", "1.1.8-3.6ubuntu2"},
{"libpam-modules-bin", "1.1.8-3.6ubuntu2", "pam", "1.1.8-3.6ubuntu2"},
{"libpam-runtime", "1.1.8-3.6ubuntu2", "pam", "1.1.8-3.6ubuntu2"},
{"libpam0g", "1.1.8-3.6ubuntu2", "pam", "1.1.8-3.6ubuntu2"},
{"libpcre3", "2:8.39-9", "pcre3", "2:8.39-9"},
{"libprocps6", "2:3.3.12-3ubuntu1.1", "procps", "2:3.3.12-3ubuntu1.1"},
{"libseccomp2", "2.3.1-2.1ubuntu4", "libseccomp", "2.3.1-2.1ubuntu4"},
{"libselinux1", "2.7-2build2", "libselinux", "2.7-2build2"},
{"libsemanage-common", "2.7-2build2", "libsemanage", "2.7-2build2"},
{"libsemanage1", "2.7-2build2", "libsemanage", "2.7-2build2"},
{"libsepol1", "2.7-1", "libsepol", "2.7-1"},
{"libsmartcols1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"libss2", "1.44.1-1", "e2fsprogs", "1.44.1-1"},
{"libstdc++6", "8-20180414-1ubuntu2", "gcc-8", "8-20180414-1ubuntu2"},
{"libsystemd0", "237-3ubuntu10.3", "systemd", "237-3ubuntu10.3"},
{"libtasn1-6", "4.13-2", "", ""},
{"libtinfo5", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04"},
{"libudev1", "237-3ubuntu10.3", "systemd", "237-3ubuntu10.3"},
{"libunistring2", "0.9.9-0ubuntu1", "libunistring", "0.9.9-0ubuntu1"},
{"libuuid1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"libzstd1", "1.3.3+dfsg-2ubuntu1", "libzstd", "1.3.3+dfsg-2ubuntu1"},
{"login", "1:4.5-1ubuntu1", "shadow", "1:4.5-1ubuntu1"},
{"lsb-base", "9.20170808ubuntu1", "lsb", "9.20170808ubuntu1"},
{"mawk", "1.3.3-17ubuntu3", "", ""},
{"mount", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"ncurses-base", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04"},
{"ncurses-bin", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04"},
{"passwd", "1:4.5-1ubuntu1", "shadow", "1:4.5-1ubuntu1"},
{"perl-base", "5.26.1-6ubuntu0.2", "perl", "5.26.1-6ubuntu0.2"},
{"procps", "2:3.3.12-3ubuntu1.1", "", ""},
{"sed", "4.4-2", "", ""},
{"sensible-utils", "0.0.12", "", ""},
{"sysvinit-utils", "2.88dsf-59.10ubuntu1", "sysvinit", "2.88dsf-59.10ubuntu1"},
{"tar", "1.29b-2", "", ""},
{"ubuntu-keyring", "2018.02.28", "", ""},
{"util-linux", "2.31.1-0.4ubuntu3.1", "", ""},
{"zlib1g", "1:1.2.11.dfsg-0ubuntu2", "zlib", "1:1.2.11.dfsg-0ubuntu2"},
},
},
}
for i := range testFeatures {
testFeatures[i].VersionFormat = dpkg.ParserName
}
testData := []featurefmt.TestData{
// Test an Ubuntu dpkg status file
{
Features: testFeatures,
Files: tarutil.FilesMap{
"var/lib/dpkg/status": featurefmt.LoadFileForTest("dpkg/testdata/status"),
"corrupted status file",
map[string]string{"var/lib/dpkg/status": "dpkg/testdata/corrupted"},
[]featurefmt.PackageInfo{
{"libpam-runtime", "1.1.8-3.1ubuntu3", "pam", "1.1.8-3.1ubuntu3"},
{"libpam-modules-bin", "1.1.8-3.1ubuntu3", "pam", "1.1.8-3.1ubuntu3"},
{"makedev", "2.3.1-93ubuntu1", "", ""},
{"libgcc1", "1:5.1.1-12ubuntu1", "gcc-5", "5.1.1-12ubuntu1"},
},
},
} {
featurefmt.RunTest(t, test, &lister{}, dpkg.ParserName)
}
featurefmt.TestLister(t, &lister{}, testData)
}

File diff suppressed because it is too large Load Diff

@ -17,14 +17,9 @@
package featurefmt
import (
"io/ioutil"
"path/filepath"
"runtime"
"sync"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/coreos/clair/database"
"github.com/coreos/clair/pkg/tarutil"
@ -134,30 +129,3 @@ func ListListers() []database.Detector {
}
return r
}
// TestData represents the data used to test an implementation of Lister.
type TestData struct {
Files tarutil.FilesMap
Features []database.Feature
}
// LoadFileForTest can be used in order to obtain the []byte contents of a file
// that is meant to be used for test data.
func LoadFileForTest(name string) []byte {
_, filename, _, _ := runtime.Caller(0)
d, _ := ioutil.ReadFile(filepath.Join(filepath.Dir(filename)) + "/" + name)
return d
}
// TestLister runs a Lister on each provided instance of TestData and asserts
// the ouput to be equal to the expected output.
func TestLister(t *testing.T, l Lister, testData []TestData) {
for _, td := range testData {
featureVersions, err := l.ListFeatures(td.Files)
if assert.Nil(t, err) && assert.Len(t, featureVersions, len(td.Features)) {
for _, expectedFeature := range td.Features {
assert.Contains(t, featureVersions, expectedFeature)
}
}
}
}

@ -0,0 +1,56 @@
package featurefmt
import (
"github.com/coreos/clair/database"
"github.com/deckarep/golang-set"
)
// PackageInfo is the extracted raw information from the package managers that
// can be converted to a feature.
type PackageInfo struct {
PackageName string
PackageVersion string
SourceName string
SourceVersion string
}
// Reset defaults the internal string fields to empty strings.
func (pkg *PackageInfo) Reset() {
pkg.PackageName = ""
pkg.PackageVersion = ""
pkg.SourceName = ""
pkg.SourceVersion = ""
}
func (pkg *PackageInfo) asFeature(versionFormat string) database.Feature {
feature := database.Feature{
Name: pkg.PackageName,
Version: pkg.PackageVersion,
VersionFormat: versionFormat,
}
if pkg.SourceName != "" {
parent := database.Feature{
Name: pkg.SourceName,
Version: pkg.SourceVersion,
VersionFormat: versionFormat,
}
if parent != feature {
feature.Parent = &parent
}
}
return feature
}
// PackageSetToFeatures converts a package set to feature slice
func PackageSetToFeatures(versionFormat string, pkgs mapset.Set) []database.Feature {
features := make([]database.Feature, 0, pkgs.Cardinality())
for pkg := range pkgs.Iter() {
p := pkg.(PackageInfo)
features = append(features, p.asFeature(versionFormat))
}
return features
}

@ -17,11 +17,13 @@ package rpm
import (
"bufio"
"fmt"
"io/ioutil"
"os"
"os/exec"
"strings"
"github.com/deckarep/golang-set"
log "github.com/sirupsen/logrus"
"github.com/coreos/clair/database"
@ -29,24 +31,42 @@ import (
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/rpm"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/pkg/strutil"
"github.com/coreos/clair/pkg/tarutil"
)
var ignoredPackages = []string{
"gpg-pubkey", // Ignore gpg-pubkey packages which are fake packages used to store GPG keys - they are not versionned properly.
}
type lister struct{}
func init() {
featurefmt.RegisterLister("rpm", "1.0", &lister{})
}
func isIgnored(packageName string) bool {
for _, pkg := range ignoredPackages {
if pkg == packageName {
return true
}
}
return false
}
func valid(pkg *featurefmt.PackageInfo) bool {
return pkg.PackageName != "" && pkg.PackageVersion != "" &&
((pkg.SourceName == "" && pkg.SourceVersion != "") ||
(pkg.SourceName != "" && pkg.SourceVersion != ""))
}
func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) {
f, hasFile := files["var/lib/rpm/Packages"]
if !hasFile {
return []database.Feature{}, nil
}
// Create a map to store packages and ensure their uniqueness
packagesMap := make(map[string]database.Feature)
// Write the required "Packages" file to disk
tmpDir, err := ioutil.TempDir(os.TempDir(), "rpm")
defer os.RemoveAll(tmpDir)
@ -62,7 +82,7 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)
}
// Extract binary package names because RHSA refers to binary package names.
out, err := exec.Command("rpm", "--dbpath", tmpDir, "-qa", "--qf", "%{NAME} %{EPOCH}:%{VERSION}-%{RELEASE}\n").CombinedOutput()
out, err := exec.Command("rpm", "--dbpath", tmpDir, "-qa", "--qf", "%{NAME} %{EPOCH}:%{VERSION}-%{RELEASE} %{SOURCERPM}\n").CombinedOutput()
if err != nil {
log.WithError(err).WithField("output", string(out)).Error("could not query RPM")
// Do not bubble up because we probably won't be able to fix it,
@ -70,46 +90,119 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)
return []database.Feature{}, nil
}
packages := mapset.NewSet()
scanner := bufio.NewScanner(strings.NewReader(string(out)))
for scanner.Scan() {
line := strings.Split(scanner.Text(), " ")
if len(line) != 2 {
if len(line) != 3 {
// We may see warnings on some RPM versions:
// "warning: Generating 12 missing index(es), please wait..."
continue
}
// Ignore gpg-pubkey packages which are fake packages used to store GPG keys - they are not versionned properly.
if line[0] == "gpg-pubkey" {
if isIgnored(line[0]) {
continue
}
// Parse version
version := strings.Replace(line[1], "(none):", "", -1)
err := versionfmt.Valid(rpm.ParserName, version)
if err != nil {
log.WithError(err).WithField("version", line[1]).Warning("could not parse package version. skipping")
pkg := featurefmt.PackageInfo{PackageName: line[0]}
pkg.PackageVersion = strings.Replace(line[1], "(none):", "", -1)
if err := versionfmt.Valid(rpm.ParserName, pkg.PackageVersion); err != nil {
log.WithError(err).WithField("version", line[1]).Warning("skipped unparseable package")
continue
}
// Add package
pkg := database.Feature{
Name: line[0],
Version: version,
if err := parseSourceRPM(line[2], &pkg); err != nil {
log.WithError(err).WithField("sourcerpm", line[2]).Warning("skipped unparseable package")
continue
}
packagesMap[pkg.Name+"#"+pkg.Version] = pkg
}
// Convert the map to a slice
packages := make([]database.Feature, 0, len(packagesMap))
for _, pkg := range packagesMap {
pkg.VersionFormat = rpm.ParserName
packages = append(packages, pkg)
if valid(&pkg) {
packages.Add(pkg)
}
}
return packages, nil
return featurefmt.PackageSetToFeatures(rpm.ParserName, packages), nil
}
func (l lister) RequiredFilenames() []string {
return []string{"var/lib/rpm/Packages"}
}
type rpmParserState string
const (
terminate rpmParserState = "terminate"
parseRPM rpmParserState = "RPM Token"
parseArchitecture rpmParserState = "Architecture Token"
parseRelease rpmParserState = "Release Token"
parseVersion rpmParserState = "Version Token"
)
// parseSourceRPM parses the source rpm package representation string
// http://ftp.rpm.org/max-rpm/ch-rpm-file-format.html
func parseSourceRPM(sourceRPM string, pkg *featurefmt.PackageInfo) error {
state := parseRPM
previousCheckPoint := len(sourceRPM)
release := ""
version := ""
for i := len(sourceRPM) - 1; i >= 0; i-- {
switch state {
case parseRPM:
if string(sourceRPM[i]) == "." {
state = parseArchitecture
packageType := strutil.Substring(sourceRPM, i+1, len(sourceRPM))
previousCheckPoint = i
if packageType != "rpm" {
return fmt.Errorf("unexpected package type, expect: 'rpm', got: '%s'", packageType)
}
}
case parseArchitecture:
if string(sourceRPM[i]) == "." {
state = parseRelease
architecture := strutil.Substring(sourceRPM, i+1, previousCheckPoint)
previousCheckPoint = i
if architecture != "src" && architecture != "nosrc" {
return fmt.Errorf("unexpected package architecture, expect: 'src' or 'nosrc', got: '%s'", architecture)
}
}
case parseRelease:
if string(sourceRPM[i]) == "-" {
state = parseVersion
release = strutil.Substring(sourceRPM, i+1, previousCheckPoint)
previousCheckPoint = i
if release == "" {
return fmt.Errorf("unexpected package release, expect: not empty")
}
}
case parseVersion:
if string(sourceRPM[i]) == "-" {
// terminate state
state = terminate
version = strutil.Substring(sourceRPM, i+1, previousCheckPoint)
previousCheckPoint = i
if version == "" {
return fmt.Errorf("unexpected package version, expect: not empty")
}
break
}
}
}
if state != terminate {
return fmt.Errorf("unexpected termination while parsing '%s'", state)
}
concatVersion := version + "-" + release
if err := versionfmt.Valid(rpm.ParserName, concatVersion); err != nil {
return err
}
name := strutil.Substring(sourceRPM, 0, previousCheckPoint)
if name == "" {
return fmt.Errorf("unexpected package name, expect: not empty")
}
pkg.SourceName = name
pkg.SourceVersion = concatVersion
return nil
}

@ -17,36 +17,245 @@ package rpm
import (
"testing"
"github.com/coreos/clair/database"
"github.com/stretchr/testify/require"
"github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/ext/versionfmt/rpm"
"github.com/coreos/clair/pkg/tarutil"
)
var expectedBigCaseInfo = []featurefmt.PackageInfo{
{"publicsuffix-list-dafsa", "20180514-1.fc28", "publicsuffix-list", "20180514-1.fc28"},
{"libreport-filesystem", "2.9.5-1.fc28", "libreport", "2.9.5-1.fc28"},
{"fedora-gpg-keys", "28-5", "fedora-repos", "28-5"},
{"fedora-release", "28-2", "", ""},
{"filesystem", "3.8-2.fc28", "", ""},
{"tzdata", "2018e-1.fc28", "", ""},
{"pcre2", "10.31-10.fc28", "", ""},
{"glibc-minimal-langpack", "2.27-32.fc28", "glibc", "2.27-32.fc28"},
{"glibc-common", "2.27-32.fc28", "glibc", "2.27-32.fc28"},
{"bash", "4.4.23-1.fc28", "", ""},
{"zlib", "1.2.11-8.fc28", "", ""},
{"bzip2-libs", "1.0.6-26.fc28", "bzip2", "1.0.6-26.fc28"},
{"libcap", "2.25-9.fc28", "", ""},
{"libgpg-error", "1.31-1.fc28", "", ""},
{"libzstd", "1.3.5-1.fc28", "zstd", "1.3.5-1.fc28"},
{"expat", "2.2.5-3.fc28", "", ""},
{"nss-util", "3.38.0-1.0.fc28", "", ""},
{"libcom_err", "1.44.2-0.fc28", "e2fsprogs", "1.44.2-0.fc28"},
{"libffi", "3.1-16.fc28", "", ""},
{"libgcrypt", "1.8.3-1.fc28", "", ""},
{"libxml2", "2.9.8-4.fc28", "", ""},
{"libacl", "2.2.53-1.fc28", "acl", "2.2.53-1.fc28"},
{"sed", "4.5-1.fc28", "", ""},
{"libmount", "2.32.1-1.fc28", "util-linux", "2.32.1-1.fc28"},
{"p11-kit", "0.23.12-1.fc28", "", ""},
{"libidn2", "2.0.5-1.fc28", "", ""},
{"libcap-ng", "0.7.9-4.fc28", "", ""},
{"lz4-libs", "1.8.1.2-4.fc28", "lz4", "1.8.1.2-4.fc28"},
{"libassuan", "2.5.1-3.fc28", "", ""},
{"keyutils-libs", "1.5.10-6.fc28", "keyutils", "1.5.10-6.fc28"},
{"glib2", "2.56.1-4.fc28", "", ""},
{"systemd-libs", "238-9.git0e0aa59.fc28", "systemd", "238-9.git0e0aa59.fc28"},
{"dbus-libs", "1:1.12.10-1.fc28", "dbus", "1.12.10-1.fc28"},
{"libtasn1", "4.13-2.fc28", "", ""},
{"ca-certificates", "2018.2.24-1.0.fc28", "", ""},
{"libarchive", "3.3.1-4.fc28", "", ""},
{"openssl", "1:1.1.0h-3.fc28", "openssl", "1.1.0h-3.fc28"},
{"libusbx", "1.0.22-1.fc28", "", ""},
{"libsemanage", "2.8-2.fc28", "", ""},
{"libutempter", "1.1.6-14.fc28", "", ""},
{"mpfr", "3.1.6-1.fc28", "", ""},
{"gnutls", "3.6.3-4.fc28", "", ""},
{"gzip", "1.9-3.fc28", "", ""},
{"acl", "2.2.53-1.fc28", "", ""},
{"nss-softokn-freebl", "3.38.0-1.0.fc28", "nss-softokn", "3.38.0-1.0.fc28"},
{"nss", "3.38.0-1.0.fc28", "", ""},
{"libmetalink", "0.1.3-6.fc28", "", ""},
{"libdb-utils", "5.3.28-30.fc28", "libdb", "5.3.28-30.fc28"},
{"file-libs", "5.33-7.fc28", "file", "5.33-7.fc28"},
{"libsss_idmap", "1.16.3-2.fc28", "sssd", "1.16.3-2.fc28"},
{"libsigsegv", "2.11-5.fc28", "", ""},
{"krb5-libs", "1.16.1-13.fc28", "krb5", "1.16.1-13.fc28"},
{"libnsl2", "1.2.0-2.20180605git4a062cf.fc28", "", ""},
{"python3-pip", "9.0.3-2.fc28", "python-pip", "9.0.3-2.fc28"},
{"python3", "3.6.6-1.fc28", "", ""},
{"pam", "1.3.1-1.fc28", "", ""},
{"python3-gobject-base", "3.28.3-1.fc28", "pygobject3", "3.28.3-1.fc28"},
{"python3-smartcols", "0.3.0-2.fc28", "python-smartcols", "0.3.0-2.fc28"},
{"python3-iniparse", "0.4-30.fc28", "python-iniparse", "0.4-30.fc28"},
{"openldap", "2.4.46-3.fc28", "", ""},
{"libseccomp", "2.3.3-2.fc28", "", ""},
{"npth", "1.5-4.fc28", "", ""},
{"gpgme", "1.10.0-4.fc28", "", ""},
{"json-c", "0.13.1-2.fc28", "", ""},
{"libyaml", "0.1.7-5.fc28", "", ""},
{"libpkgconf", "1.4.2-1.fc28", "pkgconf", "1.4.2-1.fc28"},
{"pkgconf-pkg-config", "1.4.2-1.fc28", "pkgconf", "1.4.2-1.fc28"},
{"iptables-libs", "1.6.2-3.fc28", "iptables", "1.6.2-3.fc28"},
{"device-mapper-libs", "1.02.146-5.fc28", "lvm2", "2.02.177-5.fc28"},
{"systemd-pam", "238-9.git0e0aa59.fc28", "systemd", "238-9.git0e0aa59.fc28"},
{"systemd", "238-9.git0e0aa59.fc28", "", ""},
{"elfutils-default-yama-scope", "0.173-1.fc28", "elfutils", "0.173-1.fc28"},
{"libcurl", "7.59.0-6.fc28", "curl", "7.59.0-6.fc28"},
{"python3-librepo", "1.8.1-7.fc28", "librepo", "1.8.1-7.fc28"},
{"rpm-plugin-selinux", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28"},
{"rpm", "4.14.1-9.fc28", "", ""},
{"libdnf", "0.11.1-3.fc28", "", ""},
{"rpm-build-libs", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28"},
{"python3-rpm", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28"},
{"dnf", "2.7.5-12.fc28", "", ""},
{"deltarpm", "3.6-25.fc28", "", ""},
{"sssd-client", "1.16.3-2.fc28", "sssd", "1.16.3-2.fc28"},
{"cracklib-dicts", "2.9.6-13.fc28", "cracklib", "2.9.6-13.fc28"},
{"tar", "2:1.30-3.fc28", "tar", "1.30-3.fc28"},
{"diffutils", "3.6-4.fc28", "", ""},
{"langpacks-en", "1.0-12.fc28", "langpacks", "1.0-12.fc28"},
{"libgcc", "8.1.1-5.fc28", "gcc", "8.1.1-5.fc28"},
{"pkgconf-m4", "1.4.2-1.fc28", "pkgconf", "1.4.2-1.fc28"},
{"dnf-conf", "2.7.5-12.fc28", "dnf", "2.7.5-12.fc28"},
{"fedora-repos", "28-5", "", ""},
{"setup", "2.11.4-1.fc28", "", ""},
{"basesystem", "11-5.fc28", "", ""},
{"ncurses-base", "6.1-5.20180224.fc28", "ncurses", "6.1-5.20180224.fc28"},
{"libselinux", "2.8-1.fc28", "", ""},
{"ncurses-libs", "6.1-5.20180224.fc28", "ncurses", "6.1-5.20180224.fc28"},
{"glibc", "2.27-32.fc28", "", ""},
{"libsepol", "2.8-1.fc28", "", ""},
{"xz-libs", "5.2.4-2.fc28", "xz", "5.2.4-2.fc28"},
{"info", "6.5-4.fc28", "texinfo", "6.5-4.fc28"},
{"libdb", "5.3.28-30.fc28", "", ""},
{"elfutils-libelf", "0.173-1.fc28", "elfutils", "0.173-1.fc28"},
{"popt", "1.16-14.fc28", "", ""},
{"nspr", "4.19.0-1.fc28", "", ""},
{"libxcrypt", "4.1.2-1.fc28", "", ""},
{"lua-libs", "5.3.4-10.fc28", "lua", "5.3.4-10.fc28"},
{"libuuid", "2.32.1-1.fc28", "util-linux", "2.32.1-1.fc28"},
{"readline", "7.0-11.fc28", "", ""},
{"libattr", "2.4.48-3.fc28", "attr", "2.4.48-3.fc28"},
{"coreutils-single", "8.29-7.fc28", "coreutils", "8.29-7.fc28"},
{"libblkid", "2.32.1-1.fc28", "util-linux", "2.32.1-1.fc28"},
{"gmp", "1:6.1.2-7.fc28", "gmp", "6.1.2-7.fc28"},
{"libunistring", "0.9.10-1.fc28", "", ""},
{"sqlite-libs", "3.22.0-4.fc28", "sqlite", "3.22.0-4.fc28"},
{"audit-libs", "2.8.4-2.fc28", "audit", "2.8.4-2.fc28"},
{"chkconfig", "1.10-4.fc28", "", ""},
{"libsmartcols", "2.32.1-1.fc28", "util-linux", "2.32.1-1.fc28"},
{"pcre", "8.42-3.fc28", "", ""},
{"grep", "3.1-5.fc28", "", ""},
{"crypto-policies", "20180425-5.git6ad4018.fc28", "", ""},
{"gdbm-libs", "1:1.14.1-4.fc28", "gdbm", "1.14.1-4.fc28"},
{"p11-kit-trust", "0.23.12-1.fc28", "p11-kit", "0.23.12-1.fc28"},
{"openssl-libs", "1:1.1.0h-3.fc28", "openssl", "1.1.0h-3.fc28"},
{"ima-evm-utils", "1.1-2.fc28", "", ""},
{"gdbm", "1:1.14.1-4.fc28", "gdbm", "1.14.1-4.fc28"},
{"gobject-introspection", "1.56.1-1.fc28", "", ""},
{"shadow-utils", "2:4.6-1.fc28", "shadow-utils", "4.6-1.fc28"},
{"libpsl", "0.20.2-2.fc28", "", ""},
{"nettle", "3.4-2.fc28", "", ""},
{"libfdisk", "2.32.1-1.fc28", "util-linux", "2.32.1-1.fc28"},
{"cracklib", "2.9.6-13.fc28", "", ""},
{"libcomps", "0.1.8-11.fc28", "", ""},
{"nss-softokn", "3.38.0-1.0.fc28", "", ""},
{"nss-sysinit", "3.38.0-1.0.fc28", "nss", "3.38.0-1.0.fc28"},
{"libksba", "1.3.5-7.fc28", "", ""},
{"kmod-libs", "25-2.fc28", "kmod", "25-2.fc28"},
{"libsss_nss_idmap", "1.16.3-2.fc28", "sssd", "1.16.3-2.fc28"},
{"libverto", "0.3.0-5.fc28", "", ""},
{"gawk", "4.2.1-1.fc28", "", ""},
{"libtirpc", "1.0.3-3.rc2.fc28", "", ""},
{"python3-libs", "3.6.6-1.fc28", "python3", "3.6.6-1.fc28"},
{"python3-setuptools", "39.2.0-6.fc28", "python-setuptools", "39.2.0-6.fc28"},
{"libpwquality", "1.4.0-7.fc28", "", ""},
{"util-linux", "2.32.1-1.fc28", "", ""},
{"python3-libcomps", "0.1.8-11.fc28", "libcomps", "0.1.8-11.fc28"},
{"python3-six", "1.11.0-3.fc28", "python-six", "1.11.0-3.fc28"},
{"cyrus-sasl-lib", "2.1.27-0.2rc7.fc28", "cyrus-sasl", "2.1.27-0.2rc7.fc28"},
{"libssh", "0.8.2-1.fc28", "", ""},
{"qrencode-libs", "3.4.4-5.fc28", "qrencode", "3.4.4-5.fc28"},
{"gnupg2", "2.2.8-1.fc28", "", ""},
{"python3-gpg", "1.10.0-4.fc28", "gpgme", "1.10.0-4.fc28"},
{"libargon2", "20161029-5.fc28", "argon2", "20161029-5.fc28"},
{"libmodulemd", "1.6.2-2.fc28", "", ""},
{"pkgconf", "1.4.2-1.fc28", "", ""},
{"libpcap", "14:1.9.0-1.fc28", "libpcap", "1.9.0-1.fc28"},
{"device-mapper", "1.02.146-5.fc28", "lvm2", "2.02.177-5.fc28"},
{"cryptsetup-libs", "2.0.4-1.fc28", "cryptsetup", "2.0.4-1.fc28"},
{"elfutils-libs", "0.173-1.fc28", "elfutils", "0.173-1.fc28"},
{"dbus", "1:1.12.10-1.fc28", "dbus", "1.12.10-1.fc28"},
{"libnghttp2", "1.32.1-1.fc28", "nghttp2", "1.32.1-1.fc28"},
{"librepo", "1.8.1-7.fc28", "", ""},
{"curl", "7.59.0-6.fc28", "", ""},
{"rpm-libs", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28"},
{"libsolv", "0.6.35-1.fc28", "", ""},
{"python3-hawkey", "0.11.1-3.fc28", "libdnf", "0.11.1-3.fc28"},
{"rpm-sign-libs", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28"},
{"python3-dnf", "2.7.5-12.fc28", "dnf", "2.7.5-12.fc28"},
{"dnf-yum", "2.7.5-12.fc28", "dnf", "2.7.5-12.fc28"},
{"rpm-plugin-systemd-inhibit", "4.14.1-9.fc28", "rpm", "4.14.1-9.fc28"},
{"nss-tools", "3.38.0-1.0.fc28", "nss", "3.38.0-1.0.fc28"},
{"openssl-pkcs11", "0.4.8-1.fc28", "", ""},
{"vim-minimal", "2:8.1.328-1.fc28", "vim", "8.1.328-1.fc28"},
{"glibc-langpack-en", "2.27-32.fc28", "glibc", "2.27-32.fc28"},
{"rootfiles", "8.1-22.fc28", "", ""},
}
func TestRpmFeatureDetection(t *testing.T) {
testData := []featurefmt.TestData{
// Test a CentOS 7 RPM database
// Memo: Use the following command on a RPM-based system to shrink a database: rpm -qa --qf "%{NAME}\n" |tail -n +3| xargs rpm -e --justdb
for _, test := range []featurefmt.TestCase{
{
Features: []database.Feature{
// Two packages from this source are installed, it should only appear once
{
Name: "centos-release",
Version: "7-1.1503.el7.centos.2.8",
VersionFormat: rpm.ParserName,
},
// Two packages from this source are installed, it should only appear once
{
Name: "filesystem",
Version: "3.2-18.el7",
VersionFormat: rpm.ParserName,
},
},
Files: tarutil.FilesMap{
"var/lib/rpm/Packages": featurefmt.LoadFileForTest("rpm/testdata/Packages"),
"valid small case",
map[string]string{"var/lib/rpm/Packages": "rpm/testdata/valid"},
[]featurefmt.PackageInfo{
{"centos-release", "7-1.1503.el7.centos.2.8", "", ""},
{"filesystem", "3.2-18.el7", "", ""},
},
},
{
"valid big case",
map[string]string{"var/lib/rpm/Packages": "rpm/testdata/valid_big"},
expectedBigCaseInfo,
},
} {
featurefmt.RunTest(t, test, lister{}, rpm.ParserName)
}
}
func TestParseSourceRPM(t *testing.T) {
for _, test := range [...]struct {
sourceRPM string
expectedName string
expectedVersion string
expectedErr string
}{
// valid cases
{"publicsuffix-list-20180514-1.fc28.src.rpm", "publicsuffix-list", "20180514-1.fc28", ""},
{"libreport-2.9.5-1.fc28.src.rpm", "libreport", "2.9.5-1.fc28", ""},
{"lua-5.3.4-10.fc28.src.rpm", "lua", "5.3.4-10.fc28", ""},
{"crypto-policies-20180425-5.git6ad4018.fc28.src.rpm", "crypto-policies", "20180425-5.git6ad4018.fc28", ""},
featurefmt.TestLister(t, &lister{}, testData)
// invalid cases
{"crypto-policies-20180425-5.git6ad4018.fc28.src.dpkg", "", "", "unexpected package type, expect: 'rpm', got: 'dpkg'"},
{"crypto-policies-20180425-5.git6ad4018.fc28.debian-8.rpm", "", "", "unexpected package architecture, expect: 'src' or 'nosrc', got: 'debian-8'"},
{"fc28.src.rpm", "", "", "unexpected termination while parsing 'Release Token'"},
{"...", "", "", "unexpected package type, expect: 'rpm', got: ''"},
// impossible case
// This illustrates the limitation of this parser, it will not find the
// error cased by extra '-' in the intended version/expect token. Based
// on the documentation, this case should never happen and indicates a
// corrupted rpm database.
// actual expected: name="lua", version="5.3.4", release="10.fc-28"
{"lua-5.3.4-10.fc-28.src.rpm", "lua-5.3.4", "10.fc-28", ""},
} {
pkg := featurefmt.PackageInfo{}
err := parseSourceRPM(test.sourceRPM, &pkg)
if test.expectedErr != "" {
require.EqualError(t, err, test.expectedErr)
continue
}
require.Nil(t, err)
require.Equal(t, test.expectedName, pkg.SourceName)
require.Equal(t, test.expectedVersion, pkg.SourceVersion)
}
}

Binary file not shown.

@ -0,0 +1,118 @@
// Copyright 2018 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package featurefmt contains utility functions for featurefmt tests
package featurefmt
import (
"io/ioutil"
"path/filepath"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/coreos/clair/database"
"github.com/coreos/clair/pkg/tarutil"
)
// LoadFileForTest can be used in order to obtain the []byte contents of a file
// that is meant to be used for test data.
func LoadFileForTest(name string) []byte {
_, filename, _, _ := runtime.Caller(0)
path := filepath.Join(filepath.Dir(filename), name)
d, err := ioutil.ReadFile(path)
if err != nil {
panic(err)
}
return d
}
func loadTestFiles(testFilePaths map[string]string) tarutil.FilesMap {
m := tarutil.FilesMap{}
for tarPath, fsPath := range testFilePaths {
m[tarPath] = LoadFileForTest(fsPath)
}
return m
}
// TestCase is used by the RunTest function to execute.
type TestCase struct {
Name string
FilePaths map[string]string
ExpectedResult []PackageInfo
}
// RunTest runs a featurefmt test by loading the package info database files and
// the expected packages.
func RunTest(t *testing.T, test TestCase, lister Lister, expectedVersionFormat string) {
t.Run(test.Name, func(t *testing.T) {
filesMap := loadTestFiles(test.FilePaths)
expected := test.ExpectedResult
features, err := lister.ListFeatures(filesMap)
require.Nil(t, err)
visited := map[PackageInfo]bool{}
// we only enforce the unique packages to match, the result features
// should be always deduplicated.
for _, pkg := range expected {
visited[pkg] = false
}
assert.Len(t, features, len(visited))
for _, f := range features {
assert.Equal(t, expectedVersionFormat, f.VersionFormat)
if f.Parent != nil {
// currently we don't have more than 2 levels deep features.
assert.Equal(t, expectedVersionFormat, f.Parent.VersionFormat)
}
pkg := convertToPackageInfo(&f)
if ok, found := visited[pkg]; ok {
assert.Fail(t, "duplicated features is not allowed", "feature=%#v", f, pkg)
} else if !found {
assert.Fail(t, "unexpected feature", "feature = %#v", pkg)
}
visited[pkg] = true
}
missingPackages := []PackageInfo{}
for pkg, ok := range visited {
if !ok {
missingPackages = append(missingPackages, pkg)
}
}
assert.Len(t, missingPackages, 0, "missing packages")
})
}
func convertToPackageInfo(feature *database.Feature) PackageInfo {
pkg := PackageInfo{
PackageName: feature.Name,
PackageVersion: feature.Version,
}
// Since in the actual package manager metadata file, there's no explicit
// tree structure, the features are converted to compare the metadata only.
if feature.Parent != nil {
pkg.SourceName = feature.Parent.Name
pkg.SourceVersion = feature.Parent.Version
}
return pkg
}

@ -57,3 +57,13 @@ func Intersect(X, Y []string) []string {
func CleanURL(str string) string {
return urlParametersRegexp.ReplaceAllString(str, "")
}
// Substring returns a substring by [start, end). If start or end are out
// of bound, it returns "".
func Substring(s string, start, end int) string {
if start > len(s) || start < 0 || end > len(s) || end < 0 || start >= end {
return ""
}
return s[start:end]
}

@ -18,8 +18,24 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSubstring(t *testing.T) {
for _, test := range [...]struct {
in string
start int
end int
out string
}{
{"", 0, 1, ""}, {"", 0, 0, ""}, {"", -1, -1, ""}, {"1", 1, 0, ""},
{"1", 1, 1, ""}, {"1", 0, 1, "1"}, {"1", 0, 2, ""},
} {
require.Equal(t, test.out, Substring(test.in, test.start, test.end))
}
}
func TestStringComparison(t *testing.T) {
cmp := Difference([]string{"a", "b", "b", "a"}, []string{"a", "c"})
assert.Len(t, cmp, 1)

Loading…
Cancel
Save