diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go deleted file mode 100644 index bce1ef06..00000000 --- a/database/pgsql/layer.go +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright 2017 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pgsql - -import ( - "database/sql" - "sort" - - "github.com/deckarep/golang-set" - - "github.com/coreos/clair/database" - "github.com/coreos/clair/pkg/commonerr" -) - -const ( - soiLayer = ` - WITH new_layer AS ( - INSERT INTO layer (hash) - SELECT CAST ($1 AS VARCHAR) - WHERE NOT EXISTS (SELECT id FROM layer WHERE hash = $1) - RETURNING id - ) - SELECT id FROM new_Layer - UNION - SELECT id FROM layer WHERE hash = $1` - - findLayerFeatures = ` - SELECT - f.name, f.version, f.version_format, ft.name, lf.detector_id, ns.name, ns.version_format - FROM - layer_feature AS lf - LEFT JOIN feature f on f.id = lf.feature_id - LEFT JOIN feature_type ft on ft.id = f.type - LEFT JOIN namespace ns ON ns.id = lf.namespace_id - - WHERE lf.layer_id = $1` - - findLayerNamespaces = ` - SELECT ns.name, ns.version_format, ln.detector_id - FROM layer_namespace AS ln, namespace AS ns - WHERE ln.namespace_id = ns.id - AND ln.layer_id = $1` - - findLayerID = `SELECT id FROM layer WHERE hash = $1` -) - -// dbLayerNamespace represents the layer_namespace table. -type dbLayerNamespace struct { - layerID int64 - namespaceID int64 - detectorID int64 -} - -// dbLayerFeature represents the layer_feature table -type dbLayerFeature struct { - layerID int64 - featureID int64 - detectorID int64 - namespaceID sql.NullInt64 -} - -func (tx *pgSession) FindLayer(hash string) (database.Layer, bool, error) { - layer := database.Layer{Hash: hash} - if hash == "" { - return layer, false, commonerr.NewBadRequestError("non empty layer hash is expected.") - } - - layerID, ok, err := tx.findLayerID(hash) - if err != nil || !ok { - return layer, ok, err - } - - detectorMap, err := tx.findAllDetectors() - if err != nil { - return layer, false, err - } - - if layer.By, err = tx.findLayerDetectors(layerID); err != nil { - return layer, false, err - } - - if layer.Features, err = tx.findLayerFeatures(layerID, detectorMap); err != nil { - return layer, false, err - } - - if layer.Namespaces, err = tx.findLayerNamespaces(layerID, detectorMap); err != nil { - return layer, false, err - } - - return layer, true, nil -} - -func sanitizePersistLayerInput(hash string, features []database.LayerFeature, namespaces []database.LayerNamespace, detectedBy []database.Detector) error { - if hash == "" { - return commonerr.NewBadRequestError("expected non-empty layer hash") - } - - detectedBySet := mapset.NewSet() - for _, d := range detectedBy { - detectedBySet.Add(d) - } - - for _, f := range features { - if !detectedBySet.Contains(f.By) { - return database.ErrInvalidParameters - } - } - - for _, n := range namespaces { - if !detectedBySet.Contains(n.By) { - return database.ErrInvalidParameters - } - } - - return nil -} - -// PersistLayer saves the content of a layer to the database. -func (tx *pgSession) PersistLayer(hash string, features []database.LayerFeature, namespaces []database.LayerNamespace, detectedBy []database.Detector) error { - var ( - err error - id int64 - detectorIDs []int64 - ) - - if err = sanitizePersistLayerInput(hash, features, namespaces, detectedBy); err != nil { - return err - } - - if id, err = tx.soiLayer(hash); err != nil { - return err - } - - if detectorIDs, err = tx.findDetectorIDs(detectedBy); err != nil { - if err == commonerr.ErrNotFound { - return database.ErrMissingEntities - } - - return err - } - - if err = tx.persistLayerDetectors(id, detectorIDs); err != nil { - return err - } - - if err = tx.persistAllLayerFeatures(id, features); err != nil { - return err - } - - if err = tx.persistAllLayerNamespaces(id, namespaces); err != nil { - return err - } - - return nil -} - -func (tx *pgSession) persistAllLayerNamespaces(layerID int64, namespaces []database.LayerNamespace) error { - detectorMap, err := tx.findAllDetectors() - if err != nil { - return err - } - - // TODO(sidac): This kind of type conversion is very useless and wasteful, - // we need interfaces around the database models to reduce these kind of - // operations. - rawNamespaces := make([]database.Namespace, 0, len(namespaces)) - for _, ns := range namespaces { - rawNamespaces = append(rawNamespaces, ns.Namespace) - } - - rawNamespaceIDs, err := tx.findNamespaceIDs(rawNamespaces) - if err != nil { - return err - } - - dbLayerNamespaces := make([]dbLayerNamespace, 0, len(namespaces)) - for i, ns := range namespaces { - detectorID := detectorMap.byValue[ns.By] - namespaceID := rawNamespaceIDs[i].Int64 - if !rawNamespaceIDs[i].Valid { - return database.ErrMissingEntities - } - - dbLayerNamespaces = append(dbLayerNamespaces, dbLayerNamespace{layerID, namespaceID, detectorID}) - } - - return tx.persistLayerNamespaces(dbLayerNamespaces) -} - -func (tx *pgSession) persistAllLayerFeatures(layerID int64, features []database.LayerFeature) error { - detectorMap, err := tx.findAllDetectors() - if err != nil { - return err - } - var namespaces []database.Namespace - for _, feature := range features { - namespaces = append(namespaces, feature.PotentialNamespace) - } - nameSpaceIDs, _ := tx.findNamespaceIDs(namespaces) - featureNamespaceMap := map[database.Namespace]sql.NullInt64{} - rawFeatures := make([]database.Feature, 0, len(features)) - for i, f := range features { - rawFeatures = append(rawFeatures, f.Feature) - if f.PotentialNamespace.Valid() { - featureNamespaceMap[f.PotentialNamespace] = nameSpaceIDs[i] - } - } - - featureIDs, err := tx.findFeatureIDs(rawFeatures) - if err != nil { - return err - } - var namespaceID sql.NullInt64 - dbFeatures := make([]dbLayerFeature, 0, len(features)) - for i, f := range features { - detectorID := detectorMap.byValue[f.By] - if !featureIDs[i].Valid { - return database.ErrMissingEntities - } - featureID := featureIDs[i].Int64 - namespaceID = featureNamespaceMap[f.PotentialNamespace] - - dbFeatures = append(dbFeatures, dbLayerFeature{layerID, featureID, detectorID, namespaceID}) - } - - if err := tx.persistLayerFeatures(dbFeatures); err != nil { - return err - } - - return nil -} - -func (tx *pgSession) persistLayerFeatures(features []dbLayerFeature) error { - if len(features) == 0 { - return nil - } - - sort.Slice(features, func(i, j int) bool { - return features[i].featureID < features[j].featureID - }) - keys := make([]interface{}, 0, len(features)*4) - - for _, f := range features { - keys = append(keys, f.layerID, f.featureID, f.detectorID, f.namespaceID) - } - - _, err := tx.Exec(queryPersistLayerFeature(len(features)), keys...) - if err != nil { - return handleError("queryPersistLayerFeature", err) - } - return nil -} - -func (tx *pgSession) persistLayerNamespaces(namespaces []dbLayerNamespace) error { - if len(namespaces) == 0 { - return nil - } - - // for every bulk persist operation, the input data should be sorted. - sort.Slice(namespaces, func(i, j int) bool { - return namespaces[i].namespaceID < namespaces[j].namespaceID - }) - - keys := make([]interface{}, 0, len(namespaces)*3) - for _, row := range namespaces { - keys = append(keys, row.layerID, row.namespaceID, row.detectorID) - } - - _, err := tx.Exec(queryPersistLayerNamespace(len(namespaces)), keys...) - if err != nil { - return handleError("queryPersistLayerNamespace", err) - } - - return nil -} - -func (tx *pgSession) findLayerNamespaces(layerID int64, detectors detectorMap) ([]database.LayerNamespace, error) { - rows, err := tx.Query(findLayerNamespaces, layerID) - if err != nil { - return nil, handleError("findLayerNamespaces", err) - } - - namespaces := []database.LayerNamespace{} - for rows.Next() { - var ( - namespace database.LayerNamespace - detectorID int64 - ) - - if err := rows.Scan(&namespace.Name, &namespace.VersionFormat, &detectorID); err != nil { - return nil, err - } - - namespace.By = detectors.byID[detectorID] - namespaces = append(namespaces, namespace) - } - - return namespaces, nil -} - -func (tx *pgSession) findLayerFeatures(layerID int64, detectors detectorMap) ([]database.LayerFeature, error) { - rows, err := tx.Query(findLayerFeatures, layerID) - if err != nil { - return nil, handleError("findLayerFeatures", err) - } - defer rows.Close() - - features := []database.LayerFeature{} - for rows.Next() { - var ( - detectorID int64 - feature database.LayerFeature - ) - var namespaceName, namespaceVersion sql.NullString - if err := rows.Scan(&feature.Name, &feature.Version, &feature.VersionFormat, &feature.Type, &detectorID, &namespaceName, &namespaceVersion); err != nil { - return nil, handleError("findLayerFeatures", err) - } - feature.PotentialNamespace.Name = namespaceName.String - feature.PotentialNamespace.VersionFormat = namespaceVersion.String - - feature.By = detectors.byID[detectorID] - features = append(features, feature) - } - - return features, nil -} - -func (tx *pgSession) findLayerID(hash string) (int64, bool, error) { - var layerID int64 - err := tx.QueryRow(findLayerID, hash).Scan(&layerID) - if err != nil { - if err == sql.ErrNoRows { - return layerID, false, nil - } - - return layerID, false, handleError("findLayerID", err) - } - - return layerID, true, nil -} - -func (tx *pgSession) findLayerIDs(hashes []string) ([]int64, bool, error) { - layerIDs := make([]int64, 0, len(hashes)) - for _, hash := range hashes { - id, ok, err := tx.findLayerID(hash) - if !ok { - return nil, false, nil - } - - if err != nil { - return nil, false, err - } - - layerIDs = append(layerIDs, id) - } - - return layerIDs, true, nil -} - -func (tx *pgSession) soiLayer(hash string) (int64, error) { - var id int64 - if err := tx.QueryRow(soiLayer, hash).Scan(&id); err != nil { - return 0, handleError("soiLayer", err) - } - - return id, nil -} diff --git a/database/pgsql/layer/layer.go b/database/pgsql/layer/layer.go new file mode 100644 index 00000000..88fbfcb1 --- /dev/null +++ b/database/pgsql/layer/layer.go @@ -0,0 +1,177 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layer + +import ( + "database/sql" + + "github.com/deckarep/golang-set" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/database/pgsql/detector" + "github.com/coreos/clair/database/pgsql/util" + "github.com/coreos/clair/pkg/commonerr" +) + +const ( + soiLayer = ` + WITH new_layer AS ( + INSERT INTO layer (hash) + SELECT CAST ($1 AS VARCHAR) + WHERE NOT EXISTS (SELECT id FROM layer WHERE hash = $1) + RETURNING id + ) + SELECT id FROM new_Layer + UNION + SELECT id FROM layer WHERE hash = $1` + + findLayerID = `SELECT id FROM layer WHERE hash = $1` +) + +func FindLayer(tx *sql.Tx, hash string) (database.Layer, bool, error) { + layer := database.Layer{Hash: hash} + if hash == "" { + return layer, false, commonerr.NewBadRequestError("non empty layer hash is expected.") + } + + layerID, ok, err := FindLayerID(tx, hash) + if err != nil || !ok { + return layer, ok, err + } + + detectorMap, err := detector.FindAllDetectors(tx) + if err != nil { + return layer, false, err + } + + if layer.By, err = FindLayerDetectors(tx, layerID); err != nil { + return layer, false, err + } + + if layer.Features, err = FindLayerFeatures(tx, layerID, detectorMap); err != nil { + return layer, false, err + } + + if layer.Namespaces, err = FindLayerNamespaces(tx, layerID, detectorMap); err != nil { + return layer, false, err + } + + return layer, true, nil +} + +func sanitizePersistLayerInput(hash string, features []database.LayerFeature, namespaces []database.LayerNamespace, detectedBy []database.Detector) error { + if hash == "" { + return commonerr.NewBadRequestError("expected non-empty layer hash") + } + + detectedBySet := mapset.NewSet() + for _, d := range detectedBy { + detectedBySet.Add(d) + } + + for _, f := range features { + if !detectedBySet.Contains(f.By) { + return database.ErrInvalidParameters + } + } + + for _, n := range namespaces { + if !detectedBySet.Contains(n.By) { + return database.ErrInvalidParameters + } + } + + return nil +} + +// PersistLayer saves the content of a layer to the database. +func PersistLayer(tx *sql.Tx, hash string, features []database.LayerFeature, namespaces []database.LayerNamespace, detectedBy []database.Detector) error { + var ( + err error + id int64 + detectorIDs []int64 + ) + + if err = sanitizePersistLayerInput(hash, features, namespaces, detectedBy); err != nil { + return err + } + + if id, err = SoiLayer(tx, hash); err != nil { + return err + } + + if detectorIDs, err = detector.FindDetectorIDs(tx, detectedBy); err != nil { + if err == commonerr.ErrNotFound { + return database.ErrMissingEntities + } + + return err + } + + if err = PersistLayerDetectors(tx, id, detectorIDs); err != nil { + return err + } + + if err = PersistAllLayerFeatures(tx, id, features); err != nil { + return err + } + + if err = PersistAllLayerNamespaces(tx, id, namespaces); err != nil { + return err + } + + return nil +} + +func FindLayerID(tx *sql.Tx, hash string) (int64, bool, error) { + var layerID int64 + err := tx.QueryRow(findLayerID, hash).Scan(&layerID) + if err != nil { + if err == sql.ErrNoRows { + return layerID, false, nil + } + + return layerID, false, util.HandleError("findLayerID", err) + } + + return layerID, true, nil +} + +func FindLayerIDs(tx *sql.Tx, hashes []string) ([]int64, bool, error) { + layerIDs := make([]int64, 0, len(hashes)) + for _, hash := range hashes { + id, ok, err := FindLayerID(tx, hash) + if !ok { + return nil, false, nil + } + + if err != nil { + return nil, false, err + } + + layerIDs = append(layerIDs, id) + } + + return layerIDs, true, nil +} + +func SoiLayer(tx *sql.Tx, hash string) (int64, error) { + var id int64 + if err := tx.QueryRow(soiLayer, hash).Scan(&id); err != nil { + return 0, util.HandleError("soiLayer", err) + } + + return id, nil +} diff --git a/database/pgsql/layer/layer_detector.go b/database/pgsql/layer/layer_detector.go new file mode 100644 index 00000000..78941ca6 --- /dev/null +++ b/database/pgsql/layer/layer_detector.go @@ -0,0 +1,66 @@ +// Copyright 2019 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layer + +import ( + "database/sql" + + "github.com/deckarep/golang-set" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/database/pgsql/detector" + "github.com/coreos/clair/database/pgsql/util" +) + +const ( + selectLayerDetectors = ` + SELECT d.name, d.version, d.dtype + FROM layer_detector, detector AS d + WHERE layer_detector.detector_id = d.id AND layer_detector.layer_id = $1;` + + persistLayerDetector = ` + INSERT INTO layer_detector (layer_id, detector_id) + SELECT $1, $2 + WHERE NOT EXISTS (SELECT id FROM layer_detector WHERE layer_id = $1 AND detector_id = $2)` +) + +func PersistLayerDetector(tx *sql.Tx, layerID int64, detectorID int64) error { + if _, err := tx.Exec(persistLayerDetector, layerID, detectorID); err != nil { + return util.HandleError("persistLayerDetector", err) + } + + return nil +} + +func PersistLayerDetectors(tx *sql.Tx, layerID int64, detectorIDs []int64) error { + alreadySaved := mapset.NewSet() + for _, id := range detectorIDs { + if alreadySaved.Contains(id) { + continue + } + + alreadySaved.Add(id) + if err := PersistLayerDetector(tx, layerID, id); err != nil { + return err + } + } + + return nil +} + +func FindLayerDetectors(tx *sql.Tx, id int64) ([]database.Detector, error) { + detectors, err := detector.GetDetectors(tx, selectLayerDetectors, id) + return detectors, err +} diff --git a/database/pgsql/layer/layer_feature.go b/database/pgsql/layer/layer_feature.go new file mode 100644 index 00000000..3aa50a42 --- /dev/null +++ b/database/pgsql/layer/layer_feature.go @@ -0,0 +1,147 @@ +// Copyright 2019 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layer + +import ( + "database/sql" + "sort" + + "github.com/coreos/clair/database/pgsql/namespace" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/database/pgsql/detector" + "github.com/coreos/clair/database/pgsql/feature" + "github.com/coreos/clair/database/pgsql/util" +) + +const findLayerFeatures = ` +SELECT + f.name, f.version, f.version_format, ft.name, lf.detector_id, ns.name, ns.version_format +FROM + layer_feature AS lf +LEFT JOIN feature f on f.id = lf.feature_id +LEFT JOIN feature_type ft on ft.id = f.type +LEFT JOIN namespace ns ON ns.id = lf.namespace_id + +WHERE lf.layer_id = $1` + +func queryPersistLayerFeature(count int) string { + return util.QueryPersist(count, + "layer_feature", + "layer_feature_layer_id_feature_id_namespace_id_key", + "layer_id", + "feature_id", + "detector_id", + "namespace_id") +} + +// dbLayerFeature represents the layer_feature table +type dbLayerFeature struct { + layerID int64 + featureID int64 + detectorID int64 + namespaceID sql.NullInt64 +} + +func FindLayerFeatures(tx *sql.Tx, layerID int64, detectors detector.DetectorMap) ([]database.LayerFeature, error) { + rows, err := tx.Query(findLayerFeatures, layerID) + if err != nil { + return nil, util.HandleError("findLayerFeatures", err) + } + defer rows.Close() + + features := []database.LayerFeature{} + for rows.Next() { + var ( + detectorID int64 + feature database.LayerFeature + ) + var namespaceName, namespaceVersion sql.NullString + if err := rows.Scan(&feature.Name, &feature.Version, &feature.VersionFormat, &feature.Type, &detectorID, &namespaceName, &namespaceVersion); err != nil { + return nil, util.HandleError("findLayerFeatures", err) + } + feature.PotentialNamespace.Name = namespaceName.String + feature.PotentialNamespace.VersionFormat = namespaceVersion.String + + feature.By = detectors.ByID[detectorID] + features = append(features, feature) + } + + return features, nil +} + +func PersistAllLayerFeatures(tx *sql.Tx, layerID int64, features []database.LayerFeature) error { + detectorMap, err := detector.FindAllDetectors(tx) + if err != nil { + return err + } + var namespaces []database.Namespace + for _, feature := range features { + namespaces = append(namespaces, feature.PotentialNamespace) + } + nameSpaceIDs, _ := namespace.FindNamespaceIDs(tx, namespaces) + featureNamespaceMap := map[database.Namespace]sql.NullInt64{} + rawFeatures := make([]database.Feature, 0, len(features)) + for i, f := range features { + rawFeatures = append(rawFeatures, f.Feature) + if f.PotentialNamespace.Valid() { + featureNamespaceMap[f.PotentialNamespace] = nameSpaceIDs[i] + } + } + + featureIDs, err := feature.FindFeatureIDs(tx, rawFeatures) + if err != nil { + return err + } + var namespaceID sql.NullInt64 + dbFeatures := make([]dbLayerFeature, 0, len(features)) + for i, f := range features { + detectorID := detectorMap.ByValue[f.By] + featureID := featureIDs[i].Int64 + if !featureIDs[i].Valid { + return database.ErrMissingEntities + } + namespaceID = featureNamespaceMap[f.PotentialNamespace] + + dbFeatures = append(dbFeatures, dbLayerFeature{layerID, featureID, detectorID, namespaceID}) + } + + if err := PersistLayerFeatures(tx, dbFeatures); err != nil { + return err + } + + return nil +} + +func PersistLayerFeatures(tx *sql.Tx, features []dbLayerFeature) error { + if len(features) == 0 { + return nil + } + + sort.Slice(features, func(i, j int) bool { + return features[i].featureID < features[j].featureID + }) + keys := make([]interface{}, 0, len(features)*4) + + for _, f := range features { + keys = append(keys, f.layerID, f.featureID, f.detectorID, f.namespaceID) + } + + _, err := tx.Exec(queryPersistLayerFeature(len(features)), keys...) + if err != nil { + return util.HandleError("queryPersistLayerFeature", err) + } + return nil +} diff --git a/database/pgsql/layer/layer_namespace.go b/database/pgsql/layer/layer_namespace.go new file mode 100644 index 00000000..8e65e126 --- /dev/null +++ b/database/pgsql/layer/layer_namespace.go @@ -0,0 +1,127 @@ +// Copyright 2019 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layer + +import ( + "database/sql" + "sort" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/database/pgsql/detector" + "github.com/coreos/clair/database/pgsql/namespace" + "github.com/coreos/clair/database/pgsql/util" +) + +const findLayerNamespaces = ` +SELECT ns.name, ns.version_format, ln.detector_id + FROM layer_namespace AS ln, namespace AS ns + WHERE ln.namespace_id = ns.id + AND ln.layer_id = $1` + +func queryPersistLayerNamespace(count int) string { + return util.QueryPersist(count, + "layer_namespace", + "layer_namespace_layer_id_namespace_id_key", + "layer_id", + "namespace_id", + "detector_id") +} + +// dbLayerNamespace represents the layer_namespace table. +type dbLayerNamespace struct { + layerID int64 + namespaceID int64 + detectorID int64 +} + +func FindLayerNamespaces(tx *sql.Tx, layerID int64, detectors detector.DetectorMap) ([]database.LayerNamespace, error) { + rows, err := tx.Query(findLayerNamespaces, layerID) + if err != nil { + return nil, util.HandleError("findLayerNamespaces", err) + } + + namespaces := []database.LayerNamespace{} + for rows.Next() { + var ( + namespace database.LayerNamespace + detectorID int64 + ) + + if err := rows.Scan(&namespace.Name, &namespace.VersionFormat, &detectorID); err != nil { + return nil, err + } + + namespace.By = detectors.ByID[detectorID] + namespaces = append(namespaces, namespace) + } + + return namespaces, nil +} + +func PersistAllLayerNamespaces(tx *sql.Tx, layerID int64, namespaces []database.LayerNamespace) error { + detectorMap, err := detector.FindAllDetectors(tx) + if err != nil { + return err + } + + // TODO(sidac): This kind of type conversion is very useless and wasteful, + // we need interfaces around the database models to reduce these kind of + // operations. + rawNamespaces := make([]database.Namespace, 0, len(namespaces)) + for _, ns := range namespaces { + rawNamespaces = append(rawNamespaces, ns.Namespace) + } + + rawNamespaceIDs, err := namespace.FindNamespaceIDs(tx, rawNamespaces) + if err != nil { + return err + } + + dbLayerNamespaces := make([]dbLayerNamespace, 0, len(namespaces)) + for i, ns := range namespaces { + detectorID := detectorMap.ByValue[ns.By] + namespaceID := rawNamespaceIDs[i].Int64 + if !rawNamespaceIDs[i].Valid { + return database.ErrMissingEntities + } + + dbLayerNamespaces = append(dbLayerNamespaces, dbLayerNamespace{layerID, namespaceID, detectorID}) + } + + return PersistLayerNamespaces(tx, dbLayerNamespaces) +} + +func PersistLayerNamespaces(tx *sql.Tx, namespaces []dbLayerNamespace) error { + if len(namespaces) == 0 { + return nil + } + + // for every bulk persist operation, the input data should be sorted. + sort.Slice(namespaces, func(i, j int) bool { + return namespaces[i].namespaceID < namespaces[j].namespaceID + }) + + keys := make([]interface{}, 0, len(namespaces)*3) + for _, row := range namespaces { + keys = append(keys, row.layerID, row.namespaceID, row.detectorID) + } + + _, err := tx.Exec(queryPersistLayerNamespace(len(namespaces)), keys...) + if err != nil { + return util.HandleError("queryPersistLayerNamespace", err) + } + + return nil +} diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer/layer_test.go similarity index 58% rename from database/pgsql/layer_test.go rename to database/pgsql/layer/layer_test.go index 9b2be2bb..b310c041 100644 --- a/database/pgsql/layer_test.go +++ b/database/pgsql/layer/layer_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pgsql +package layer import ( "testing" @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/coreos/clair/database" + "github.com/coreos/clair/database/pgsql/testutil" ) var persistLayerTests = []struct { @@ -39,9 +40,9 @@ var persistLayerTests = []struct { { title: "layer with inconsistent feature and detectors", name: "random-forest", - by: []database.Detector{realDetectors[2]}, + by: []database.Detector{testutil.RealDetectors[2]}, features: []database.LayerFeature{ - {realFeatures[1], realDetectors[1], database.Namespace{}}, + {testutil.RealFeatures[1], testutil.RealDetectors[1], database.Namespace{}}, }, err: "parameters are not valid", }, @@ -49,70 +50,71 @@ var persistLayerTests = []struct { title: "layer with non-existing feature", name: "random-forest", err: "associated immutable entities are missing in the database", - by: []database.Detector{realDetectors[2]}, + by: []database.Detector{testutil.RealDetectors[2]}, features: []database.LayerFeature{ - {fakeFeatures[1], realDetectors[2], database.Namespace{}}, + {testutil.FakeFeatures[1], testutil.RealDetectors[2], database.Namespace{}}, }, }, { title: "layer with non-existing namespace", name: "random-forest2", err: "associated immutable entities are missing in the database", - by: []database.Detector{realDetectors[1]}, + by: []database.Detector{testutil.RealDetectors[1]}, namespaces: []database.LayerNamespace{ - {fakeNamespaces[1], realDetectors[1]}, + {testutil.FakeNamespaces[1], testutil.RealDetectors[1]}, }, }, { title: "layer with non-existing detector", name: "random-forest3", err: "associated immutable entities are missing in the database", - by: []database.Detector{fakeDetector[1]}, + by: []database.Detector{testutil.FakeDetector[1]}, }, { + title: "valid layer", name: "hamsterhouse", - by: []database.Detector{realDetectors[1], realDetectors[2]}, + by: []database.Detector{testutil.RealDetectors[1], testutil.RealDetectors[2]}, features: []database.LayerFeature{ - {realFeatures[1], realDetectors[2], database.Namespace{}}, - {realFeatures[2], realDetectors[2], database.Namespace{}}, + {testutil.RealFeatures[1], testutil.RealDetectors[2], database.Namespace{}}, + {testutil.RealFeatures[2], testutil.RealDetectors[2], database.Namespace{}}, }, namespaces: []database.LayerNamespace{ - {realNamespaces[1], realDetectors[1]}, + {testutil.RealNamespaces[1], testutil.RealDetectors[1]}, }, layer: &database.Layer{ Hash: "hamsterhouse", - By: []database.Detector{realDetectors[1], realDetectors[2]}, + By: []database.Detector{testutil.RealDetectors[1], testutil.RealDetectors[2]}, Features: []database.LayerFeature{ - {realFeatures[1], realDetectors[2], database.Namespace{}}, - {realFeatures[2], realDetectors[2], database.Namespace{}}, + {testutil.RealFeatures[1], testutil.RealDetectors[2], database.Namespace{}}, + {testutil.RealFeatures[2], testutil.RealDetectors[2], database.Namespace{}}, }, Namespaces: []database.LayerNamespace{ - {realNamespaces[1], realDetectors[1]}, + {testutil.RealNamespaces[1], testutil.RealDetectors[1]}, }, }, }, { title: "update existing layer", name: "layer-1", - by: []database.Detector{realDetectors[3], realDetectors[4]}, + by: []database.Detector{testutil.RealDetectors[3], testutil.RealDetectors[4]}, features: []database.LayerFeature{ - {realFeatures[4], realDetectors[3], database.Namespace{}}, + {testutil.RealFeatures[4], testutil.RealDetectors[3], database.Namespace{}}, }, namespaces: []database.LayerNamespace{ - {realNamespaces[3], realDetectors[4]}, + {testutil.RealNamespaces[3], testutil.RealDetectors[4]}, }, layer: &database.Layer{ Hash: "layer-1", - By: []database.Detector{realDetectors[1], realDetectors[2], realDetectors[3], realDetectors[4]}, + By: []database.Detector{testutil.RealDetectors[1], testutil.RealDetectors[2], testutil.RealDetectors[3], testutil.RealDetectors[4]}, Features: []database.LayerFeature{ - {realFeatures[1], realDetectors[2], database.Namespace{}}, - {realFeatures[2], realDetectors[2], database.Namespace{}}, - {realFeatures[4], realDetectors[3], database.Namespace{}}, + {testutil.RealFeatures[1], testutil.RealDetectors[2], database.Namespace{}}, + {testutil.RealFeatures[2], testutil.RealDetectors[2], database.Namespace{}}, + {testutil.RealFeatures[4], testutil.RealDetectors[3], database.Namespace{}}, }, Namespaces: []database.LayerNamespace{ - {realNamespaces[1], realDetectors[1]}, - {realNamespaces[3], realDetectors[4]}, + {testutil.RealNamespaces[1], testutil.RealDetectors[1]}, + {testutil.RealNamespaces[3], testutil.RealDetectors[4]}, }, }, }, @@ -120,33 +122,33 @@ var persistLayerTests = []struct { { title: "layer with potential namespace", name: "layer-potential-namespace", - by: []database.Detector{realDetectors[3]}, + by: []database.Detector{testutil.RealDetectors[3]}, features: []database.LayerFeature{ - {realFeatures[4], realDetectors[3], realNamespaces[4]}, + {testutil.RealFeatures[4], testutil.RealDetectors[3], testutil.RealNamespaces[4]}, }, namespaces: []database.LayerNamespace{ - {realNamespaces[3], realDetectors[3]}, + {testutil.RealNamespaces[3], testutil.RealDetectors[3]}, }, layer: &database.Layer{ Hash: "layer-potential-namespace", - By: []database.Detector{realDetectors[3]}, + By: []database.Detector{testutil.RealDetectors[3]}, Features: []database.LayerFeature{ - {realFeatures[4], realDetectors[3], realNamespaces[4]}, + {testutil.RealFeatures[4], testutil.RealDetectors[3], testutil.RealNamespaces[4]}, }, Namespaces: []database.LayerNamespace{ - {realNamespaces[3], realDetectors[3]}, + {testutil.RealNamespaces[3], testutil.RealDetectors[3]}, }, }, }, } func TestPersistLayer(t *testing.T) { - datastore, tx := openSessionForTest(t, "PersistLayer", true) - defer closeTest(t, datastore, tx) + tx, cleanup := testutil.CreateTestTxWithFixtures(t, "PersistLayer") + defer cleanup() for _, test := range persistLayerTests { t.Run(test.title, func(t *testing.T) { - err := tx.PersistLayer(test.name, test.features, test.namespaces, test.by) + err := PersistLayer(tx, test.name, test.features, test.namespaces, test.by) if test.err != "" { assert.EqualError(t, err, test.err, "unexpected error") return @@ -154,7 +156,7 @@ func TestPersistLayer(t *testing.T) { assert.Nil(t, err) if test.layer != nil { - layer, ok, err := tx.FindLayer(test.name) + layer, ok, err := FindLayer(tx, test.name) assert.Nil(t, err) assert.True(t, ok) database.AssertLayerEqual(t, test.layer, &layer) @@ -186,17 +188,17 @@ var findLayerTests = []struct { title: "existing layer", in: "layer-4", ok: true, - out: takeLayerPointerFromMap(realLayers, 6), + out: testutil.TakeLayerPointerFromMap(testutil.RealLayers, 6), }, } func TestFindLayer(t *testing.T) { - datastore, tx := openSessionForTest(t, "FindLayer", true) - defer closeTest(t, datastore, tx) + tx, cleanup := testutil.CreateTestTxWithFixtures(t, "FindLayer") + defer cleanup() for _, test := range findLayerTests { t.Run(test.title, func(t *testing.T) { - layer, ok, err := tx.FindLayer(test.in) + layer, ok, err := FindLayer(tx, test.in) if test.err != "" { assert.EqualError(t, err, test.err, "unexpected error") return