arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

database: disable hash/merge joins in FindLayer

Browse Source 
Our experiments have shown that PostgreSQL 9.4 makes bad
planning decisions about:
- joining the layer tree to feature versions and feature
- joining the feature versions to affected/fixed feature version and vulnerabilities
It would for instance do a merge join between affected feature versions (300 rows, estimated
3000 rows) and fixed in feature version (100k rows). In this case, it is much more
preferred to use a nested loop.
This commit is contained in:
Quentin Machu 2016-02-19 20:55:54 -05:00 committed by Jimmy Zelinskie
parent 18f2d7e672
commit 06531e01c5
2 changed files with 30 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
database/pgsql/layer.go
View File

@ -65,8 +65,30 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
// Find its features // Find its features
if withFeatures || withVulnerabilities { if withFeatures || withVulnerabilities {
// Create a transaction to disable hash/merge joins as our experiments have shown that
// PostgreSQL 9.4 makes bad planning decisions about:
// - joining the layer tree to feature versions and feature
// - joining the feature versions to affected/fixed feature version and vulnerabilities
// It would for instance do a merge join between affected feature versions (300 rows, estimated
// 3000 rows) and fixed in feature version (100k rows). In this case, it is much more
// preferred to use a nested loop.
tx, err := pgSQL.Begin()
if err != nil {
return layer, handleError("FindLayer.Begin()", err)
}
defer tx.Commit()
_, err = tx.Exec(getQuery("disable_hashjoin"))
if err != nil {
log.Warningf("FindLayer: could not disable hash join: %s", err)
}
_, err = tx.Exec(getQuery("disable_mergejoin"))
if err != nil {
log.Warningf("FindLayer: could not disable merge join: %s", err)
}
t = time.Now() t = time.Now()
featureVersions, err := pgSQL.getLayerFeatureVersions(layer.ID) featureVersions, err := getLayerFeatureVersions(tx, layer.ID)
observeQueryTime("FindLayer", "getLayerFeatureVersions", t) observeQueryTime("FindLayer", "getLayerFeatureVersions", t)
if err != nil { if err != nil {
@ -78,7 +100,7 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
if withVulnerabilities { if withVulnerabilities {
// Load the vulnerabilities that affect the FeatureVersions. // Load the vulnerabilities that affect the FeatureVersions.
t = time.Now() t = time.Now()
err := pgSQL.loadAffectedBy(layer.Features) err := loadAffectedBy(tx, layer.Features)
observeQueryTime("FindLayer", "loadAffectedBy", t) observeQueryTime("FindLayer", "loadAffectedBy", t)
if err != nil { if err != nil {
@ -91,11 +113,11 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
} }
// getLayerFeatureVersions returns list of database.FeatureVersion that a database.Layer has. // getLayerFeatureVersions returns list of database.FeatureVersion that a database.Layer has.
func (pgSQL *pgSQL) getLayerFeatureVersions(layerID int) ([]database.FeatureVersion, error) { func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion, error) {
var featureVersions []database.FeatureVersion var featureVersions []database.FeatureVersion
// Query. // Query.
rows, err := pgSQL.Query(getQuery("s_layer_featureversion"), layerID) rows, err := tx.Query(getQuery("s_layer_featureversion"), layerID)
if err != nil { if err != nil {
return featureVersions, handleError("s_layer_featureversion", err) return featureVersions, handleError("s_layer_featureversion", err)
} }
@ -140,7 +162,7 @@ func (pgSQL *pgSQL) getLayerFeatureVersions(layerID int) ([]database.FeatureVers
// loadAffectedBy returns the list of database.Vulnerability that affect the given // loadAffectedBy returns the list of database.Vulnerability that affect the given
// FeatureVersion. // FeatureVersion.
func (pgSQL *pgSQL) loadAffectedBy(featureVersions []database.FeatureVersion) error { func loadAffectedBy(tx *sql.Tx, featureVersions []database.FeatureVersion) error {
if len(featureVersions) == 0 { if len(featureVersions) == 0 {
return nil return nil
} }
@ -151,7 +173,7 @@ func (pgSQL *pgSQL) loadAffectedBy(featureVersions []database.FeatureVersion) er
featureVersionIDs = append(featureVersionIDs, featureVersions[i].ID) featureVersionIDs = append(featureVersionIDs, featureVersions[i].ID)
} }
rows, err := pgSQL.Query(getQuery("s_featureversions_vulnerabilities"), rows, err := tx.Query(getQuery("s_featureversions_vulnerabilities"),
buildInputArray(featureVersionIDs)) buildInputArray(featureVersionIDs))
if err != nil && err != sql.ErrNoRows { if err != nil && err != sql.ErrNoRows {
return handleError("s_featureversions_vulnerabilities", err) return handleError("s_featureversions_vulnerabilities", err)

2
database/pgsql/queries.go
View File

@ -25,6 +25,8 @@ func init() {
queries = make(map[string]string) queries = make(map[string]string)
queries["l_vulnerability_affects_featureversion"] = `LOCK Vulnerability_Affects_FeatureVersion IN SHARE ROW EXCLUSIVE MODE` queries["l_vulnerability_affects_featureversion"] = `LOCK Vulnerability_Affects_FeatureVersion IN SHARE ROW EXCLUSIVE MODE`
queries["disable_hashjoin"] = `SET LOCAL enable_hashjoin = off`
queries["disable_mergejoin"] = `SET LOCAL enable_mergejoin = off`
// keyvalue.go // keyvalue.go
queries["u_keyvalue"] = `UPDATE KeyValue SET value = $1 WHERE key = $2` queries["u_keyvalue"] = `UPDATE KeyValue SET value = $1 WHERE key = $2`