arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

pgsql: Split feature.go to table based files in feature module

Browse Source
This commit is contained in:
Sida Chen 2019-03-06 16:28:32 -05:00
parent 43f3ea87d8
commit c50a2339b7
6 changed files with 333 additions and 481 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

398
database/pgsql/feature.go
View File

@ -1,398 +0,0 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pgsql
import (
"database/sql"
"sort"
"github.com/lib/pq"
log "github.com/sirupsen/logrus"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/pkg/commonerr"
)
const (
soiNamespacedFeature = `
WITH new_feature_ns AS (
INSERT INTO namespaced_feature(feature_id, namespace_id)
SELECT CAST ($1 AS INTEGER), CAST ($2 AS INTEGER)
WHERE NOT EXISTS ( SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2)
RETURNING id
)
SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2
UNION
SELECT id FROM new_feature_ns`
searchPotentialAffectingVulneraibilities = `
SELECT nf.id, v.id, vaf.affected_version, vaf.id
FROM vulnerability_affected_feature AS vaf, vulnerability AS v,
namespaced_feature AS nf, feature AS f
WHERE nf.id = ANY($1)
AND nf.feature_id = f.id
AND nf.namespace_id = v.namespace_id
AND vaf.feature_name = f.name
AND vaf.feature_type = f.type
AND vaf.vulnerability_id = v.id
AND v.deleted_at IS NULL`
searchNamespacedFeaturesVulnerabilities = `
SELECT vanf.namespaced_feature_id, v.name, v.description, v.link,
v.severity, v.metadata, vaf.fixedin, n.name, n.version_format
FROM vulnerability_affected_namespaced_feature AS vanf,
Vulnerability AS v,
vulnerability_affected_feature AS vaf,
namespace AS n
WHERE vanf.namespaced_feature_id = ANY($1)
AND vaf.id = vanf.added_by
AND v.id = vanf.vulnerability_id
AND n.id = v.namespace_id
AND v.deleted_at IS NULL`
)
func (tx *pgSession) PersistFeatures(features []database.Feature) error {
if len(features) == 0 {
return nil
}
types, err := tx.getFeatureTypeMap()
if err != nil {
return err
}
// Sorting is needed before inserting into database to prevent deadlock.
sort.Slice(features, func(i, j int) bool {
return features[i].Name < features[j].Name ||
features[i].Version < features[j].Version ||
features[i].VersionFormat < features[j].VersionFormat
})
// TODO(Sida): A better interface for bulk insertion is needed.
keys := make([]interface{}, 0, len(features)*3)
for _, f := range features {
keys = append(keys, f.Name, f.Version, f.VersionFormat, types.byName[f.Type])
if f.Name == "" || f.Version == "" || f.VersionFormat == "" {
return commonerr.NewBadRequestError("Empty feature name, version or version format is not allowed")
}
}
_, err = tx.Exec(queryPersistFeature(len(features)), keys...)
return handleError("queryPersistFeature", err)
}
type namespacedFeatureWithID struct {
database.NamespacedFeature
ID int64
}
type vulnerabilityCache struct {
nsFeatureID int64
vulnID int64
vulnAffectingID int64
}
func (tx *pgSession) searchAffectingVulnerabilities(features []database.NamespacedFeature) ([]vulnerabilityCache, error) {
if len(features) == 0 {
return nil, nil
}
ids, err := tx.findNamespacedFeatureIDs(features)
if err != nil {
return nil, err
}
fMap := map[int64]database.NamespacedFeature{}
for i, f := range features {
if !ids[i].Valid {
return nil, database.ErrMissingEntities
}
fMap[ids[i].Int64] = f
}
cacheTable := []vulnerabilityCache{}
rows, err := tx.Query(searchPotentialAffectingVulneraibilities, pq.Array(ids))
if err != nil {
return nil, handleError("searchPotentialAffectingVulneraibilities", err)
}
defer rows.Close()
for rows.Next() {
var (
cache vulnerabilityCache
affected string
)
err := rows.Scan(&cache.nsFeatureID, &cache.vulnID, &affected, &cache.vulnAffectingID)
if err != nil {
return nil, err
}
if ok, err := versionfmt.InRange(fMap[cache.nsFeatureID].VersionFormat, fMap[cache.nsFeatureID].Version, affected); err != nil {
return nil, err
} else if ok {
cacheTable = append(cacheTable, cache)
}
}
return cacheTable, nil
}
func (tx *pgSession) CacheAffectedNamespacedFeatures(features []database.NamespacedFeature) error {
if len(features) == 0 {
return nil
}
_, err := tx.Exec(lockVulnerabilityAffects)
if err != nil {
return handleError("lockVulnerabilityAffects", err)
}
cache, err := tx.searchAffectingVulnerabilities(features)
if err != nil {
return err
}
keys := make([]interface{}, 0, len(cache)*3)
for _, c := range cache {
keys = append(keys, c.vulnID, c.nsFeatureID, c.vulnAffectingID)
}
if len(cache) == 0 {
return nil
}
affected, err := tx.Exec(queryPersistVulnerabilityAffectedNamespacedFeature(len(cache)), keys...)
if err != nil {
return handleError("persistVulnerabilityAffectedNamespacedFeature", err)
}
if count, err := affected.RowsAffected(); err != nil {
log.Debugf("Cached %d features in vulnerability_affected_namespaced_feature", count)
}
return nil
}
func (tx *pgSession) PersistNamespacedFeatures(features []database.NamespacedFeature) error {
if len(features) == 0 {
return nil
}
nsIDs := map[database.Namespace]sql.NullInt64{}
fIDs := map[database.Feature]sql.NullInt64{}
for _, f := range features {
nsIDs[f.Namespace] = sql.NullInt64{}
fIDs[f.Feature] = sql.NullInt64{}
}
fToFind := []database.Feature{}
for f := range fIDs {
fToFind = append(fToFind, f)
}
sort.Slice(fToFind, func(i, j int) bool {
return fToFind[i].Name < fToFind[j].Name ||
fToFind[i].Version < fToFind[j].Version ||
fToFind[i].VersionFormat < fToFind[j].VersionFormat
})
if ids, err := tx.findFeatureIDs(fToFind); err == nil {
for i, id := range ids {
if !id.Valid {
return database.ErrMissingEntities
}
fIDs[fToFind[i]] = id
}
} else {
return err
}
nsToFind := []database.Namespace{}
for ns := range nsIDs {
nsToFind = append(nsToFind, ns)
}
if ids, err := tx.findNamespaceIDs(nsToFind); err == nil {
for i, id := range ids {
if !id.Valid {
return database.ErrMissingEntities
}
nsIDs[nsToFind[i]] = id
}
} else {
return err
}
keys := make([]interface{}, 0, len(features)*2)
for _, f := range features {
keys = append(keys, fIDs[f.Feature], nsIDs[f.Namespace])
}
_, err := tx.Exec(queryPersistNamespacedFeature(len(features)), keys...)
if err != nil {
return err
}
return nil
}
// FindAffectedNamespacedFeatures retrieves vulnerabilities associated with the
// feature.
func (tx *pgSession) FindAffectedNamespacedFeatures(features []database.NamespacedFeature) ([]database.NullableAffectedNamespacedFeature, error) {
if len(features) == 0 {
return nil, nil
}
vulnerableFeatures := make([]database.NullableAffectedNamespacedFeature, len(features))
featureIDs, err := tx.findNamespacedFeatureIDs(features)
if err != nil {
return nil, err
}
for i, id := range featureIDs {
if id.Valid {
vulnerableFeatures[i].Valid = true
vulnerableFeatures[i].NamespacedFeature = features[i]
}
}
rows, err := tx.Query(searchNamespacedFeaturesVulnerabilities, pq.Array(featureIDs))
if err != nil {
return nil, handleError("searchNamespacedFeaturesVulnerabilities", err)
}
defer rows.Close()
for rows.Next() {
var (
featureID int64
vuln database.VulnerabilityWithFixedIn
)
err := rows.Scan(&featureID,
&vuln.Name,
&vuln.Description,
&vuln.Link,
&vuln.Severity,
&vuln.Metadata,
&vuln.FixedInVersion,
&vuln.Namespace.Name,
&vuln.Namespace.VersionFormat,
)
if err != nil {
return nil, handleError("searchNamespacedFeaturesVulnerabilities", err)
}
for i, id := range featureIDs {
if id.Valid && id.Int64 == featureID {
vulnerableFeatures[i].AffectedNamespacedFeature.AffectedBy = append(vulnerableFeatures[i].AffectedNamespacedFeature.AffectedBy, vuln)
}
}
}
return vulnerableFeatures, nil
}
func (tx *pgSession) findNamespacedFeatureIDs(nfs []database.NamespacedFeature) ([]sql.NullInt64, error) {
if len(nfs) == 0 {
return nil, nil
}
nfsMap := map[database.NamespacedFeature]int64{}
keys := make([]interface{}, 0, len(nfs)*5)
for _, nf := range nfs {
keys = append(keys, nf.Name, nf.Version, nf.VersionFormat, nf.Type, nf.Namespace.Name)
}
rows, err := tx.Query(querySearchNamespacedFeature(len(nfs)), keys...)
if err != nil {
return nil, handleError("searchNamespacedFeature", err)
}
defer rows.Close()
var (
id int64
nf database.NamespacedFeature
)
for rows.Next() {
err := rows.Scan(&id, &nf.Name, &nf.Version, &nf.VersionFormat, &nf.Type, &nf.Namespace.Name)
nf.Namespace.VersionFormat = nf.VersionFormat
if err != nil {
return nil, handleError("searchNamespacedFeature", err)
}
nfsMap[nf] = id
}
ids := make([]sql.NullInt64, len(nfs))
for i, nf := range nfs {
if id, ok := nfsMap[nf]; ok {
ids[i] = sql.NullInt64{id, true}
} else {
ids[i] = sql.NullInt64{}
}
}
return ids, nil
}
func (tx *pgSession) findFeatureIDs(fs []database.Feature) ([]sql.NullInt64, error) {
if len(fs) == 0 {
return nil, nil
}
types, err := tx.getFeatureTypeMap()
if err != nil {
return nil, err
}
fMap := map[database.Feature]sql.NullInt64{}
keys := make([]interface{}, 0, len(fs)*4)
for _, f := range fs {
typeID := types.byName[f.Type]
keys = append(keys, f.Name, f.Version, f.VersionFormat, typeID)
fMap[f] = sql.NullInt64{}
}
rows, err := tx.Query(querySearchFeatureID(len(fs)), keys...)
if err != nil {
return nil, handleError("querySearchFeatureID", err)
}
defer rows.Close()
var (
id sql.NullInt64
f database.Feature
)
for rows.Next() {
var typeID int
err := rows.Scan(&id, &f.Name, &f.Version, &f.VersionFormat, &typeID)
if err != nil {
return nil, handleError("querySearchFeatureID", err)
}
f.Type = types.byID[typeID]
fMap[f] = id
}
ids := make([]sql.NullInt64, len(fs))
for i, f := range fs {
ids[i] = fMap[f]
}
return ids, nil
}

121
database/pgsql/feature/feature.go Normal file
View File

@ -0,0 +1,121 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package feature
import (
"database/sql"
"fmt"
"sort"
"github.com/coreos/clair/database"
"github.com/coreos/clair/database/pgsql/util"
"github.com/coreos/clair/pkg/commonerr"
)
func queryPersistFeature(count int) string {
return util.QueryPersist(count,
"feature",
"feature_name_version_version_format_type_key",
"name",
"version",
"version_format",
"type")
}
func querySearchFeatureID(featureCount int) string {
return fmt.Sprintf(`
SELECT id, name, version, version_format, type
FROM Feature WHERE (name, version, version_format, type) IN (%s)`,
util.QueryString(4, featureCount),
)
}
func PersistFeatures(tx *sql.Tx, features []database.Feature) error {
if len(features) == 0 {
return nil
}
types, err := GetFeatureTypeMap(tx)
if err != nil {
return err
}
// Sorting is needed before inserting into database to prevent deadlock.
sort.Slice(features, func(i, j int) bool {
return features[i].Name < features[j].Name ||
features[i].Version < features[j].Version ||
features[i].VersionFormat < features[j].VersionFormat
})
// TODO(Sida): A better interface for bulk insertion is needed.
keys := make([]interface{}, 0, len(features)*3)
for _, f := range features {
keys = append(keys, f.Name, f.Version, f.VersionFormat, types.ByName[f.Type])
if f.Name == "" || f.Version == "" || f.VersionFormat == "" {
return commonerr.NewBadRequestError("Empty feature name, version or version format is not allowed")
}
}
_, err = tx.Exec(queryPersistFeature(len(features)), keys...)
return util.HandleError("queryPersistFeature", err)
}
func FindFeatureIDs(tx *sql.Tx, fs []database.Feature) ([]sql.NullInt64, error) {
if len(fs) == 0 {
return nil, nil
}
types, err := GetFeatureTypeMap(tx)
if err != nil {
return nil, err
}
fMap := map[database.Feature]sql.NullInt64{}
keys := make([]interface{}, 0, len(fs)*4)
for _, f := range fs {
typeID := types.ByName[f.Type]
keys = append(keys, f.Name, f.Version, f.VersionFormat, typeID)
fMap[f] = sql.NullInt64{}
}
rows, err := tx.Query(querySearchFeatureID(len(fs)), keys...)
if err != nil {
return nil, util.HandleError("querySearchFeatureID", err)
}
defer rows.Close()
var (
id sql.NullInt64
f database.Feature
)
for rows.Next() {
var typeID int
err := rows.Scan(&id, &f.Name, &f.Version, &f.VersionFormat, &typeID)
if err != nil {
return nil, util.HandleError("querySearchFeatureID", err)
}
f.Type = types.ByID[typeID]
fMap[f] = id
}
ids := make([]sql.NullInt64, len(fs))
for i, f := range fs {
ids[i] = fMap[f]
}
return ids, nil
}

88
database/pgsql/feature_test.go → database/pgsql/feature/feature_test.go
View File

@ -12,36 +12,38 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package pgsql package feature
import ( import (
"database/sql"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
"github.com/coreos/clair/database/pgsql/testutil"
) )
func TestPersistFeatures(t *testing.T) { func TestPersistFeatures(t *testing.T) {
tx, cleanup := createTestPgSession(t, "TestPersistFeatures") tx, cleanup := testutil.CreateTestTx(t, "TestPersistFeatures")
defer cleanup() defer cleanup()
invalid := database.Feature{} invalid := database.Feature{}
valid := *database.NewBinaryPackage("mount", "2.31.1-0.4ubuntu3.1", "dpkg") valid := *database.NewBinaryPackage("mount", "2.31.1-0.4ubuntu3.1", "dpkg")
// invalid // invalid
require.NotNil(t, tx.PersistFeatures([]database.Feature{invalid})) require.NotNil(t, PersistFeatures(tx, []database.Feature{invalid}))
// existing // existing
require.Nil(t, tx.PersistFeatures([]database.Feature{valid})) require.Nil(t, PersistFeatures(tx, []database.Feature{valid}))
require.Nil(t, tx.PersistFeatures([]database.Feature{valid})) require.Nil(t, PersistFeatures(tx, []database.Feature{valid}))
features := selectAllFeatures(t, tx) features := selectAllFeatures(t, tx)
assert.Equal(t, []database.Feature{valid}, features) assert.Equal(t, []database.Feature{valid}, features)
} }
func TestPersistNamespacedFeatures(t *testing.T) { func TestPersistNamespacedFeatures(t *testing.T) {
tx, cleanup := createTestPgSessionWithFixtures(t, "TestPersistNamespacedFeatures") tx, cleanup := testutil.CreateTestTxWithFixtures(t, "TestPersistNamespacedFeatures")
defer cleanup() defer cleanup()
// existing features // existing features
@ -58,42 +60,17 @@ func TestPersistNamespacedFeatures(t *testing.T) {
nf2 := database.NewNamespacedFeature(n2, f2) nf2 := database.NewNamespacedFeature(n2, f2)
// namespaced features with namespaces or features not in the database will // namespaced features with namespaces or features not in the database will
// generate error. // generate error.
assert.Nil(t, tx.PersistNamespacedFeatures([]database.NamespacedFeature{})) assert.Nil(t, PersistNamespacedFeatures(tx, []database.NamespacedFeature{}))
assert.NotNil(t, tx.PersistNamespacedFeatures([]database.NamespacedFeature{*nf1, *nf2})) assert.NotNil(t, PersistNamespacedFeatures(tx, []database.NamespacedFeature{*nf1, *nf2}))
// valid case: insert nf3 // valid case: insert nf3
assert.Nil(t, tx.PersistNamespacedFeatures([]database.NamespacedFeature{*nf1})) assert.Nil(t, PersistNamespacedFeatures(tx, []database.NamespacedFeature{*nf1}))
all := listNamespacedFeatures(t, tx) all := listNamespacedFeatures(t, tx)
assert.Contains(t, all, *nf1) assert.Contains(t, all, *nf1)
} }
func TestFindAffectedNamespacedFeatures(t *testing.T) { func listNamespacedFeatures(t *testing.T, tx *sql.Tx) []database.NamespacedFeature {
datastore, tx := openSessionForTest(t, "FindAffectedNamespacedFeatures", true) types, err := GetFeatureTypeMap(tx)
defer closeTest(t, datastore, tx)
ns := database.NamespacedFeature{
Feature: database.Feature{
Name: "openssl",
Version: "1.0",
VersionFormat: "dpkg",
Type: database.SourcePackage,
},
Namespace: database.Namespace{
Name: "debian:7",
VersionFormat: "dpkg",
},
}
ans, err := tx.FindAffectedNamespacedFeatures([]database.NamespacedFeature{ns})
if assert.Nil(t, err) &&
assert.Len(t, ans, 1) &&
assert.True(t, ans[0].Valid) &&
assert.Len(t, ans[0].AffectedBy, 1) {
assert.Equal(t, "CVE-OPENSSL-1-DEB7", ans[0].AffectedBy[0].Name)
}
}
func listNamespacedFeatures(t *testing.T, tx *pgSession) []database.NamespacedFeature {
types, err := tx.getFeatureTypeMap()
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -114,15 +91,15 @@ func listNamespacedFeatures(t *testing.T, tx *pgSession) []database.NamespacedFe
panic(err) panic(err)
} }
f.Type = types.byID[typeID] f.Type = types.ByID[typeID]
nf = append(nf, f) nf = append(nf, f)
} }
return nf return nf
} }
func selectAllFeatures(t *testing.T, tx *pgSession) []database.Feature { func selectAllFeatures(t *testing.T, tx *sql.Tx) []database.Feature {
types, err := tx.getFeatureTypeMap() types, err := GetFeatureTypeMap(tx)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -137,7 +114,7 @@ func selectAllFeatures(t *testing.T, tx *pgSession) []database.Feature {
f := database.Feature{} f := database.Feature{}
var typeID int var typeID int
err := rows.Scan(&f.Name, &f.Version, &f.VersionFormat, &typeID) err := rows.Scan(&f.Name, &f.Version, &f.VersionFormat, &typeID)
f.Type = types.byID[typeID] f.Type = types.ByID[typeID]
if err != nil { if err != nil {
t.FailNow() t.FailNow()
} }
@ -146,45 +123,24 @@ func selectAllFeatures(t *testing.T, tx *pgSession) []database.Feature {
return fs return fs
} }
func assertNamespacedFeatureEqual(t *testing.T, expected []database.NamespacedFeature, actual []database.NamespacedFeature) bool {
if assert.Len(t, actual, len(expected)) {
has := map[database.NamespacedFeature]bool{}
for _, nf := range expected {
has[nf] = false
}
for _, nf := range actual {
has[nf] = true
}
for nf, visited := range has {
if !assert.True(t, visited, nf.Namespace.Name+":"+nf.Name+" is expected") {
return false
}
}
return true
}
return false
}
func TestFindNamespacedFeatureIDs(t *testing.T) { func TestFindNamespacedFeatureIDs(t *testing.T) {
tx, cleanup := createTestPgSessionWithFixtures(t, "TestFindNamespacedFeatureIDs") tx, cleanup := testutil.CreateTestTxWithFixtures(t, "TestFindNamespacedFeatureIDs")
defer cleanup() defer cleanup()
features := []database.NamespacedFeature{} features := []database.NamespacedFeature{}
expectedIDs := []int{} expectedIDs := []int{}
for id, feature := range realNamespacedFeatures { for id, feature := range testutil.RealNamespacedFeatures {
features = append(features, feature) features = append(features, feature)
expectedIDs = append(expectedIDs, id) expectedIDs = append(expectedIDs, id)
} }
features = append(features, realNamespacedFeatures[1]) // test duplicated features = append(features, testutil.RealNamespacedFeatures[1]) // test duplicated
expectedIDs = append(expectedIDs, 1) expectedIDs = append(expectedIDs, 1)
namespace := realNamespaces[1] namespace := testutil.RealNamespaces[1]
features = append(features, *database.NewNamespacedFeature(&namespace, database.NewBinaryPackage("not-found", "1.0", "dpkg"))) // test not found feature features = append(features, *database.NewNamespacedFeature(&namespace, database.NewBinaryPackage("not-found", "1.0", "dpkg"))) // test not found feature
ids, err := tx.findNamespacedFeatureIDs(features) ids, err := FindNamespacedFeatureIDs(tx, features)
require.Nil(t, err) require.Nil(t, err)
require.Len(t, ids, len(expectedIDs)+1) require.Len(t, ids, len(expectedIDs)+1)
for i, id := range ids { for i, id := range ids {

24
database/pgsql/feature_type.go → database/pgsql/feature/feature_type.go
View File

@ -12,24 +12,28 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package pgsql package feature
import "github.com/coreos/clair/database" import (
"database/sql"
"github.com/coreos/clair/database"
)
const ( const (
selectAllFeatureTypes = `SELECT id, name FROM feature_type` selectAllFeatureTypes = `SELECT id, name FROM feature_type`
) )
type featureTypes struct { type FeatureTypes struct {
byID map[int]database.FeatureType ByID map[int]database.FeatureType
byName map[database.FeatureType]int ByName map[database.FeatureType]int
} }
func newFeatureTypes() *featureTypes { func newFeatureTypes() *FeatureTypes {
return &featureTypes{make(map[int]database.FeatureType), make(map[database.FeatureType]int)} return &FeatureTypes{make(map[int]database.FeatureType), make(map[database.FeatureType]int)}
} }
func (tx *pgSession) getFeatureTypeMap() (*featureTypes, error) { func GetFeatureTypeMap(tx *sql.Tx) (*FeatureTypes, error) {
rows, err := tx.Query(selectAllFeatureTypes) rows, err := tx.Query(selectAllFeatureTypes)
if err != nil { if err != nil {
return nil, err return nil, err
@ -45,8 +49,8 @@ func (tx *pgSession) getFeatureTypeMap() (*featureTypes, error) {
return nil, err return nil, err
} }
types.byID[id] = name types.ByID[id] = name
types.byName[name] = id types.ByName[name] = id
} }
return types, nil return types, nil

15
database/pgsql/feature_type_test.go → database/pgsql/feature/feature_type_test.go
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package pgsql package feature
import ( import (
"testing" "testing"
@ -20,19 +20,20 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
"github.com/coreos/clair/database/pgsql/testutil"
) )
func TestGetFeatureTypeMap(t *testing.T) { func TestGetFeatureTypeMap(t *testing.T) {
tx, cleanup := createTestPgSession(t, "TestGetFeatureTypeMap") tx, cleanup := testutil.CreateTestTx(t, "TestGetFeatureTypeMap")
defer cleanup() defer cleanup()
types, err := tx.getFeatureTypeMap() types, err := GetFeatureTypeMap(tx)
if err != nil { if err != nil {
require.Nil(t, err, err.Error()) require.Nil(t, err, err.Error())
} }
require.Equal(t, database.SourcePackage, types.byID[1]) require.Equal(t, database.SourcePackage, types.ByID[1])
require.Equal(t, database.BinaryPackage, types.byID[2]) require.Equal(t, database.BinaryPackage, types.ByID[2])
require.Equal(t, 1, types.byName[database.SourcePackage]) require.Equal(t, 1, types.ByName[database.SourcePackage])
require.Equal(t, 2, types.byName[database.BinaryPackage]) require.Equal(t, 2, types.ByName[database.BinaryPackage])
} }

168
database/pgsql/feature/namespaced_feature.go Normal file
View File

@ -0,0 +1,168 @@
// Copyright 2019 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package feature
import (
"database/sql"
"fmt"
"sort"
"github.com/coreos/clair/database"
"github.com/coreos/clair/database/pgsql/namespace"
"github.com/coreos/clair/database/pgsql/util"
)
var soiNamespacedFeature = `
WITH new_feature_ns AS (
INSERT INTO namespaced_feature(feature_id, namespace_id)
SELECT CAST ($1 AS INTEGER), CAST ($2 AS INTEGER)
WHERE NOT EXISTS ( SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2)
RETURNING id
)
SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2
UNION
SELECT id FROM new_feature_ns`
func queryPersistNamespacedFeature(count int) string {
return util.QueryPersist(count, "namespaced_feature",
"namespaced_feature_namespace_id_feature_id_key",
"feature_id",
"namespace_id")
}
func querySearchNamespacedFeature(nsfCount int) string {
return fmt.Sprintf(`
SELECT nf.id, f.name, f.version, f.version_format, t.name, n.name
FROM namespaced_feature AS nf, feature AS f, namespace AS n, feature_type AS t
WHERE nf.feature_id = f.id
AND nf.namespace_id = n.id
AND n.version_format = f.version_format
AND f.type = t.id
AND (f.name, f.version, f.version_format, t.name, n.name) IN (%s)`,
util.QueryString(5, nsfCount),
)
}
type namespacedFeatureWithID struct {
database.NamespacedFeature
ID int64
}
func PersistNamespacedFeatures(tx *sql.Tx, features []database.NamespacedFeature) error {
if len(features) == 0 {
return nil
}
nsIDs := map[database.Namespace]sql.NullInt64{}
fIDs := map[database.Feature]sql.NullInt64{}
for _, f := range features {
nsIDs[f.Namespace] = sql.NullInt64{}
fIDs[f.Feature] = sql.NullInt64{}
}
fToFind := []database.Feature{}
for f := range fIDs {
fToFind = append(fToFind, f)
}
sort.Slice(fToFind, func(i, j int) bool {
return fToFind[i].Name < fToFind[j].Name ||
fToFind[i].Version < fToFind[j].Version ||
fToFind[i].VersionFormat < fToFind[j].VersionFormat
})
if ids, err := FindFeatureIDs(tx, fToFind); err == nil {
for i, id := range ids {
if !id.Valid {
return database.ErrMissingEntities
}
fIDs[fToFind[i]] = id
}
} else {
return err
}
nsToFind := []database.Namespace{}
for ns := range nsIDs {
nsToFind = append(nsToFind, ns)
}
if ids, err := namespace.FindNamespaceIDs(tx, nsToFind); err == nil {
for i, id := range ids {
if !id.Valid {
return database.ErrMissingEntities
}
nsIDs[nsToFind[i]] = id
}
} else {
return err
}
keys := make([]interface{}, 0, len(features)*2)
for _, f := range features {
keys = append(keys, fIDs[f.Feature], nsIDs[f.Namespace])
}
_, err := tx.Exec(queryPersistNamespacedFeature(len(features)), keys...)
if err != nil {
return err
}
return nil
}
func FindNamespacedFeatureIDs(tx *sql.Tx, nfs []database.NamespacedFeature) ([]sql.NullInt64, error) {
if len(nfs) == 0 {
return nil, nil
}
nfsMap := map[database.NamespacedFeature]int64{}
keys := make([]interface{}, 0, len(nfs)*5)
for _, nf := range nfs {
keys = append(keys, nf.Name, nf.Version, nf.VersionFormat, nf.Type, nf.Namespace.Name)
}
rows, err := tx.Query(querySearchNamespacedFeature(len(nfs)), keys...)
if err != nil {
return nil, util.HandleError("searchNamespacedFeature", err)
}
defer rows.Close()
var (
id int64
nf database.NamespacedFeature
)
for rows.Next() {
err := rows.Scan(&id, &nf.Name, &nf.Version, &nf.VersionFormat, &nf.Type, &nf.Namespace.Name)
nf.Namespace.VersionFormat = nf.VersionFormat
if err != nil {
return nil, util.HandleError("searchNamespacedFeature", err)
}
nfsMap[nf] = id
}
ids := make([]sql.NullInt64, len(nfs))
for i, nf := range nfs {
if id, ok := nfsMap[nf]; ok {
ids[i] = sql.NullInt64{id, true}
} else {
ids[i] = sql.NullInt64{}
}
}
return ids, nil
}