arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ext/featurefmt/dpkg: Extract source package metadata

Browse Source 
The source package metadata is extracted from the source line instead
of forcing the binary package to have source package information.
This commit is contained in:
Sida Chen 2018-10-09 17:37:16 -04:00
parent 1c40e7d016
commit 4ac046642f
4 changed files with 2246 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
ext/featurefmt/dpkg/dpkg.go
View File

@ -20,6 +20,7 @@ import (
"regexp" "regexp"
"strings" "strings"
"github.com/deckarep/golang-set"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
@ -40,29 +41,39 @@ func init() {
featurefmt.RegisterLister("dpkg", "1.0", &lister{}) featurefmt.RegisterLister("dpkg", "1.0", &lister{})
} }
func valid(pkg *featurefmt.PackageInfo) bool {
return pkg.PackageName != "" && pkg.PackageVersion != ""
}
func addSourceVersion(pkg *featurefmt.PackageInfo) {
if pkg.SourceName != "" && pkg.SourceVersion == "" {
pkg.SourceVersion = pkg.PackageVersion
}
}
func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) { func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) {
f, hasFile := files["var/lib/dpkg/status"] f, hasFile := files["var/lib/dpkg/status"]
if !hasFile { if !hasFile {
return []database.Feature{}, nil return []database.Feature{}, nil
} }
// Create a map to store packages and ensure their uniqueness var (
packagesMap := make(map[string]database.Feature) pkg featurefmt.PackageInfo
pkgs = mapset.NewSet()
err error
)
var pkg database.Feature
var err error
scanner := bufio.NewScanner(strings.NewReader(string(f))) scanner := bufio.NewScanner(strings.NewReader(string(f)))
for scanner.Scan() { for scanner.Scan() {
line := scanner.Text() line := scanner.Text()
if strings.HasPrefix(line, "Package: ") { if strings.HasPrefix(line, "Package: ") {
// Package line // Package line
// Defines the name of the package // Defines the name of the package
pkg.Name = strings.TrimSpace(strings.TrimPrefix(line, "Package: ")) pkg.PackageName = strings.TrimSpace(strings.TrimPrefix(line, "Package: "))
pkg.Version = "" pkg.PackageVersion = ""
} else if strings.HasPrefix(line, "Source: ") { } else if strings.HasPrefix(line, "Source: ") {
// Source line (Optionnal) // Source line (Optional)
// Gives the name of the source package // Gives the name of the source package
// May also specifies a version // May also specifies a version
@ -72,50 +83,38 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)
md[dpkgSrcCaptureRegexpNames[i]] = strings.TrimSpace(n) md[dpkgSrcCaptureRegexpNames[i]] = strings.TrimSpace(n)
} }
pkg.Name = md["name"] pkg.SourceName = md["name"]
if md["version"] != "" { if md["version"] != "" {
version := md["version"] version := md["version"]
err = versionfmt.Valid(dpkg.ParserName, version) if err = versionfmt.Valid(dpkg.ParserName, version); err != nil {
if err != nil {
log.WithError(err).WithField("version", string(line[1])).Warning("could not parse package version. skipping") log.WithError(err).WithField("version", string(line[1])).Warning("could not parse package version. skipping")
} else { } else {
pkg.Version = version pkg.SourceVersion = version
} }
} }
} else if strings.HasPrefix(line, "Version: ") && pkg.Version == "" { } else if strings.HasPrefix(line, "Version: ") {
// Version line // Version line
// Defines the version of the package // Defines the version of the package
// This version is less important than a version retrieved from a Source line // This version is less important than a version retrieved from a Source line
// because the Debian vulnerabilities often skips the epoch from the Version field // because the Debian vulnerabilities often skips the epoch from the Version field
// which is not present in the Source version, and because +bX revisions don't matter // which is not present in the Source version, and because +bX revisions don't matter
version := strings.TrimPrefix(line, "Version: ") version := strings.TrimPrefix(line, "Version: ")
err = versionfmt.Valid(dpkg.ParserName, version) if err = versionfmt.Valid(dpkg.ParserName, version); err != nil {
if err != nil {
log.WithError(err).WithField("version", string(line[1])).Warning("could not parse package version. skipping") log.WithError(err).WithField("version", string(line[1])).Warning("could not parse package version. skipping")
} else { } else {
pkg.Version = version pkg.PackageVersion = version
} }
} else if line == "" { } else if line == "" {
pkg.Name = "" pkg.Reset()
pkg.Version = ""
} }
// Add the package to the result array if we have all the informations if valid(&pkg) {
if pkg.Name != "" && pkg.Version != "" { addSourceVersion(&pkg)
packagesMap[pkg.Name+"#"+pkg.Version] = pkg pkgs.Add(pkg)
pkg.Name = ""
pkg.Version = ""
} }
} }
// Convert the map to a slice and add version format. return featurefmt.PackageSetToFeatures(dpkg.ParserName, pkgs), nil
packages := make([]database.Feature, 0, len(packagesMap))
for _, pkg := range packagesMap {
pkg.VersionFormat = dpkg.ParserName
packages = append(packages, pkg)
}
return packages, nil
} }
func (l lister) RequiredFilenames() []string { func (l lister) RequiredFilenames() []string {

132
ext/featurefmt/dpkg/dpkg_test.go
View File

@ -17,42 +17,118 @@ package dpkg
import ( import (
"testing" "testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/featurefmt" "github.com/coreos/clair/ext/featurefmt"
"github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/pkg/tarutil"
) )
func TestDpkgFeatureDetection(t *testing.T) { func TestListFeatures(t *testing.T) {
testFeatures := []database.Feature{ for _, test := range []featurefmt.TestCase{
// Two packages from this source are installed, it should only appear one time
{ {
Name: "pam", "valid status file",
Version: "1.1.8-3.1ubuntu3", map[string]string{"var/lib/dpkg/status": "dpkg/testdata/valid"},
[]featurefmt.PackageInfo{
{"adduser", "3.116ubuntu1", "", ""},
{"apt", "1.6.3ubuntu0.1", "", ""},
{"base-files", "10.1ubuntu2.2", "", ""},
{"base-passwd", "3.5.44", "", ""},
{"bash", "4.4.18-2ubuntu1", "", ""},
{"bsdutils", "1:2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"bzip2", "1.0.6-8.1", "", ""},
{"coreutils", "8.28-1ubuntu1", "", ""},
{"dash", "0.5.8-2.10", "", ""},
{"debconf", "1.5.66", "", ""},
{"debianutils", "4.8.4", "", ""},
{"diffutils", "1:3.6-1", "", ""},
{"dpkg", "1.19.0.5ubuntu2", "", ""},
{"e2fsprogs", "1.44.1-1", "", ""},
{"fdisk", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"findutils", "4.6.0+git+20170828-2", "", ""},
{"gcc-8-base", "8-20180414-1ubuntu2", "gcc-8", "8-20180414-1ubuntu2"},
{"gpgv", "2.2.4-1ubuntu1.1", "gnupg2", "2.2.4-1ubuntu1.1"},
{"grep", "3.1-2", "", ""},
{"gzip", "1.6-5ubuntu1", "", ""},
{"hostname", "3.20", "", ""},
{"init-system-helpers", "1.51", "", ""},
{"libacl1", "2.2.52-3build1", "acl", "2.2.52-3build1"},
{"libapt-pkg5.0", "1.6.3ubuntu0.1", "apt", "1.6.3ubuntu0.1"},
{"libattr1", "1:2.4.47-2build1", "attr", "1:2.4.47-2build1"},
{"libaudit-common", "1:2.8.2-1ubuntu1", "audit", "1:2.8.2-1ubuntu1"},
{"libaudit1", "1:2.8.2-1ubuntu1", "audit", "1:2.8.2-1ubuntu1"},
{"libblkid1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"libbz2-1.0", "1.0.6-8.1", "bzip2", "1.0.6-8.1"},
{"libc-bin", "2.27-3ubuntu1", "glibc", "2.27-3ubuntu1"},
{"libc6", "2.27-3ubuntu1", "glibc", "2.27-3ubuntu1"},
{"libcap-ng0", "0.7.7-3.1", "libcap-ng", "0.7.7-3.1"},
{"libcom-err2", "1.44.1-1", "e2fsprogs", "1.44.1-1"},
{"libdb5.3", "5.3.28-13.1ubuntu1", "db5.3", "5.3.28-13.1ubuntu1"},
{"libdebconfclient0", "0.213ubuntu1", "cdebconf", "0.213ubuntu1"},
{"libext2fs2", "1.44.1-1", "e2fsprogs", "1.44.1-1"},
{"libfdisk1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"libffi6", "3.2.1-8", "libffi", "3.2.1-8"},
{"libgcc1", "1:8-20180414-1ubuntu2", "gcc-8", "8-20180414-1ubuntu2"},
{"libgcrypt20", "1.8.1-4ubuntu1.1", "", ""},
{"libgmp10", "2:6.1.2+dfsg-2", "gmp", "2:6.1.2+dfsg-2"},
{"libgnutls30", "3.5.18-1ubuntu1", "gnutls28", "3.5.18-1ubuntu1"},
{"libgpg-error0", "1.27-6", "libgpg-error", "1.27-6"},
{"libhogweed4", "3.4-1", "nettle", "3.4-1"},
{"libidn2-0", "2.0.4-1.1build2", "libidn2", "2.0.4-1.1build2"},
{"liblz4-1", "0.0~r131-2ubuntu3", "lz4", "0.0~r131-2ubuntu3"},
{"liblzma5", "5.2.2-1.3", "xz-utils", "5.2.2-1.3"},
{"libmount1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"libncurses5", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04"},
{"libncursesw5", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04"},
{"libnettle6", "3.4-1", "nettle", "3.4-1"},
{"libp11-kit0", "0.23.9-2", "p11-kit", "0.23.9-2"},
{"libpam-modules", "1.1.8-3.6ubuntu2", "pam", "1.1.8-3.6ubuntu2"},
{"libpam-modules-bin", "1.1.8-3.6ubuntu2", "pam", "1.1.8-3.6ubuntu2"},
{"libpam-runtime", "1.1.8-3.6ubuntu2", "pam", "1.1.8-3.6ubuntu2"},
{"libpam0g", "1.1.8-3.6ubuntu2", "pam", "1.1.8-3.6ubuntu2"},
{"libpcre3", "2:8.39-9", "pcre3", "2:8.39-9"},
{"libprocps6", "2:3.3.12-3ubuntu1.1", "procps", "2:3.3.12-3ubuntu1.1"},
{"libseccomp2", "2.3.1-2.1ubuntu4", "libseccomp", "2.3.1-2.1ubuntu4"},
{"libselinux1", "2.7-2build2", "libselinux", "2.7-2build2"},
{"libsemanage-common", "2.7-2build2", "libsemanage", "2.7-2build2"},
{"libsemanage1", "2.7-2build2", "libsemanage", "2.7-2build2"},
{"libsepol1", "2.7-1", "libsepol", "2.7-1"},
{"libsmartcols1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"libss2", "1.44.1-1", "e2fsprogs", "1.44.1-1"},
{"libstdc++6", "8-20180414-1ubuntu2", "gcc-8", "8-20180414-1ubuntu2"},
{"libsystemd0", "237-3ubuntu10.3", "systemd", "237-3ubuntu10.3"},
{"libtasn1-6", "4.13-2", "", ""},
{"libtinfo5", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04"},
{"libudev1", "237-3ubuntu10.3", "systemd", "237-3ubuntu10.3"},
{"libunistring2", "0.9.9-0ubuntu1", "libunistring", "0.9.9-0ubuntu1"},
{"libuuid1", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"libzstd1", "1.3.3+dfsg-2ubuntu1", "libzstd", "1.3.3+dfsg-2ubuntu1"},
{"login", "1:4.5-1ubuntu1", "shadow", "1:4.5-1ubuntu1"},
{"lsb-base", "9.20170808ubuntu1", "lsb", "9.20170808ubuntu1"},
{"mawk", "1.3.3-17ubuntu3", "", ""},
{"mount", "2.31.1-0.4ubuntu3.1", "util-linux", "2.31.1-0.4ubuntu3.1"},
{"ncurses-base", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04"},
{"ncurses-bin", "6.1-1ubuntu1.18.04", "ncurses", "6.1-1ubuntu1.18.04"},
{"passwd", "1:4.5-1ubuntu1", "shadow", "1:4.5-1ubuntu1"},
{"perl-base", "5.26.1-6ubuntu0.2", "perl", "5.26.1-6ubuntu0.2"},
{"procps", "2:3.3.12-3ubuntu1.1", "", ""},
{"sed", "4.4-2", "", ""},
{"sensible-utils", "0.0.12", "", ""},
{"sysvinit-utils", "2.88dsf-59.10ubuntu1", "sysvinit", "2.88dsf-59.10ubuntu1"},
{"tar", "1.29b-2", "", ""},
{"ubuntu-keyring", "2018.02.28", "", ""},
{"util-linux", "2.31.1-0.4ubuntu3.1", "", ""},
{"zlib1g", "1:1.2.11.dfsg-0ubuntu2", "zlib", "1:1.2.11.dfsg-0ubuntu2"},
},
}, },
{ {
Name: "makedev", // The source name and the package name are equals "corrupted status file",
Version: "2.3.1-93ubuntu1", // The version comes from the "Version:" line map[string]string{"var/lib/dpkg/status": "dpkg/testdata/corrupted"},
}, []featurefmt.PackageInfo{
{ {"libpam-runtime", "1.1.8-3.1ubuntu3", "pam", "1.1.8-3.1ubuntu3"},
Name: "gcc-5", {"libpam-modules-bin", "1.1.8-3.1ubuntu3", "pam", "1.1.8-3.1ubuntu3"},
Version: "5.1.1-12ubuntu1", // The version comes from the "Source:" line {"makedev", "2.3.1-93ubuntu1", "", ""},
}, {"libgcc1", "1:5.1.1-12ubuntu1", "gcc-5", "5.1.1-12ubuntu1"},
}
for i := range testFeatures {
testFeatures[i].VersionFormat = dpkg.ParserName
}
testData := []featurefmt.TestData{
// Test an Ubuntu dpkg status file
{
Features: testFeatures,
Files: tarutil.FilesMap{
"var/lib/dpkg/status": featurefmt.LoadFileForTest("dpkg/testdata/status"),
}, },
}, },
} {
featurefmt.RunTest(t, test, &lister{}, dpkg.ParserName)
} }
featurefmt.TestLister(t, &lister{}, testData)
} }

0
ext/featurefmt/dpkg/testdata/status → ext/featurefmt/dpkg/testdata/corrupted vendored
View File

2110
ext/featurefmt/dpkg/testdata/valid vendored Normal file
View File

File diff suppressed because it is too large Load Diff