From 3fe3f3a4c74568699d0cbbeb9abdb28d1f249c21 Mon Sep 17 00:00:00 2001 From: Quentin Machu Date: Sat, 5 Dec 2015 02:42:37 -0500 Subject: [PATCH] database: Update cayley and use Triple instead of Quad --- Godeps/Godeps.json | 3 +- database/database.go | 11 +++-- database/database_test.go | 12 ++--- database/flag.go | 4 +- database/layer.go | 36 +++++++-------- database/lock.go | 22 ++++----- database/notification.go | 12 ++--- database/package.go | 46 +++++++++---------- database/vulnerability.go | 38 +++++++-------- vendor/github.com/google/cayley/Dockerfile | 21 +++++++++ vendor/github.com/google/cayley/README.md | 17 ++++++- .../google/cayley/cmd/cayley/cayley.go | 13 ++++++ .../google/cayley/graph/bolt/quadstore.go | 5 ++ .../google/cayley/graph/leveldb/quadstore.go | 20 ++++++-- .../google/cayley/graph/quadstore.go | 1 + .../google/cayley/graph/sql/quadstore.go | 11 ++++- vendor/github.com/google/cayley/imports.go | 4 ++ 17 files changed, 178 insertions(+), 98 deletions(-) create mode 100644 vendor/github.com/google/cayley/Dockerfile diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 3660069a..85912453 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -41,8 +41,7 @@ }, { "ImportPath": "github.com/google/cayley", - "Comment": "v0.4.1-160-gcdf0154", - "Rev": "f143602b8ae880ec975e4c154dd010047de80c1f" + "Rev": "582c4e1ca46943f2cf09c73bd12a83a6959057c9" }, { "ImportPath": "github.com/julienschmidt/httprouter", diff --git a/database/database.go b/database/database.go index c26188bd..25e46e7d 100644 --- a/database/database.go +++ b/database/database.go @@ -78,18 +78,21 @@ func Open(dbType, dbPath string) error { log.Infof("database at %s does not exist yet, creating it", dbPath) err = graph.InitQuadStore(dbType, dbPath, options) - if err != nil { + if err != nil && err != graph.ErrDatabaseExists { log.Errorf("could not create database at %s : %s", dbPath, err) return ErrCantOpen } } case "sql": // Replaces the PostgreSQL's slow COUNT query with a fast estimator. - // See: // Ref: https://wiki.postgresql.org/wiki/Count_estimate options["use_estimates"] = true - graph.InitQuadStore(dbType, dbPath, options) + err := graph.InitQuadStore(dbType, dbPath, options) + if err != nil && err != graph.ErrDatabaseExists { + log.Errorf("could not create database at %s : %s", dbPath, err) + return ErrCantOpen + } } store, err = cayley.NewGraph(dbType, dbPath, options) @@ -115,7 +118,7 @@ func Healthcheck() health.Status { var err error if store != nil { t := cayley.NewTransaction() - q := cayley.Quad("cayley", "is", "healthy", "") + q := cayley.Triple("cayley", "is", "healthy") t.AddQuad(q) t.RemoveQuad(q) glog.SetStderrThreshold("FATAL") // TODO REMOVE ME diff --git a/database/database_test.go b/database/database_test.go index 1c1f5bbe..120c46d0 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -38,12 +38,12 @@ func TestToValue(t *testing.T) { assert.Nil(t, err, "toValue should work even if the requested path leads to nothing") assert.Equal(t, "", v, "toValue should return an empty string if the requested path leads to nothing") - store.AddQuad(cayley.Quad("tests", "are", "awesome", "")) + store.AddQuad(cayley.Triple("tests", "are", "awesome")) v, err = toValue(cayley.StartPath(store, "tests").Out("are")) assert.Nil(t, err, "toValue should have worked") assert.Equal(t, "awesome", v, "toValue did not return the expected value") - store.AddQuad(cayley.Quad("tests", "are", "running", "")) + store.AddQuad(cayley.Triple("tests", "are", "running")) v, err = toValue(cayley.StartPath(store, "tests").Out("are")) assert.NotNil(t, err, "toValue should return an error and an empty string if the path leads to multiple values") assert.Equal(t, "", v, "toValue should return an error and an empty string if the path leads to multiple values") @@ -54,7 +54,7 @@ func TestToValue(t *testing.T) { assert.Len(t, vs, 0, "toValue should return an empty array if the requested path leads to nothing") words := []string{"powerful", "lightweight"} for i, word := range words { - store.AddQuad(cayley.Quad("CoreOS", fieldIs, word, "")) + store.AddQuad(cayley.Triple("CoreOS", fieldIs, word)) v, err := toValues(cayley.StartPath(store, "CoreOS").Out(fieldIs)) assert.Nil(t, err, "toValues should have worked") assert.Len(t, v, i+1, "toValues did not return the right amount of values") @@ -64,17 +64,17 @@ func TestToValue(t *testing.T) { } // toValue(s)() and empty values - store.AddQuad(cayley.Quad("bob", "likes", "", "")) + store.AddQuad(cayley.Triple("bob", "likes", "")) v, err = toValue(cayley.StartPath(store, "bob").Out("likes")) assert.Nil(t, err, "toValue should work even if the requested path leads to nothing") assert.Equal(t, "", v, "toValue should return an empty string if the requested path leads to nothing") - store.AddQuad(cayley.Quad("bob", "likes", "running", "")) + store.AddQuad(cayley.Triple("bob", "likes", "running")) v, err = toValue(cayley.StartPath(store, "bob").Out("likes")) assert.Nil(t, err, "toValue should have worked") assert.Equal(t, "running", v, "toValue did not return the expected value") - store.AddQuad(cayley.Quad("bob", "likes", "swimming", "")) + store.AddQuad(cayley.Triple("bob", "likes", "swimming")) va, err := toValues(cayley.StartPath(store, "bob").Out("likes")) assert.Nil(t, err, "toValues should have worked") assert.Len(t, va, 2, "toValues should have returned 2 values") diff --git a/database/flag.go b/database/flag.go index 823eaa30..30818198 100644 --- a/database/flag.go +++ b/database/flag.go @@ -43,9 +43,9 @@ func UpdateFlag(name, value string) error { // Build transaction name = flagNodePrefix + ":" + name if currentValue != "" { - t.RemoveQuad(cayley.Quad(name, fieldFlagValue, currentValue, "")) + t.RemoveQuad(cayley.Triple(name, fieldFlagValue, currentValue)) } - t.AddQuad(cayley.Quad(name, fieldFlagValue, value, "")) + t.AddQuad(cayley.Triple(name, fieldFlagValue, value)) // Apply transaction if err = store.ApplyTransaction(t); err != nil { diff --git a/database/layer.go b/database/layer.go index d0c64992..88116836 100644 --- a/database/layer.go +++ b/database/layer.go @@ -98,30 +98,30 @@ func InsertLayer(layer *Layer) error { if existingLayer == nil { // Create case: add permanent nodes - t.AddQuad(cayley.Quad(layer.Node, fieldIs, fieldLayerIsValue, "")) - t.AddQuad(cayley.Quad(layer.Node, FieldLayerID, layer.ID, "")) - t.AddQuad(cayley.Quad(layer.Node, FieldLayerParent, layer.ParentNode, "")) + t.AddQuad(cayley.Triple(layer.Node, fieldIs, fieldLayerIsValue)) + t.AddQuad(cayley.Triple(layer.Node, FieldLayerID, layer.ID)) + t.AddQuad(cayley.Triple(layer.Node, FieldLayerParent, layer.ParentNode)) } else { // Update case: remove everything before we add updated data - t.RemoveQuad(cayley.Quad(layer.Node, FieldLayerOS, existingLayer.OS, "")) + t.RemoveQuad(cayley.Triple(layer.Node, FieldLayerOS, existingLayer.OS)) for _, pkg := range existingLayer.InstalledPackagesNodes { - t.RemoveQuad(cayley.Quad(layer.Node, fieldLayerInstalledPackages, pkg, "")) + t.RemoveQuad(cayley.Triple(layer.Node, fieldLayerInstalledPackages, pkg)) } for _, pkg := range existingLayer.RemovedPackagesNodes { - t.RemoveQuad(cayley.Quad(layer.Node, fieldLayerRemovedPackages, pkg, "")) + t.RemoveQuad(cayley.Triple(layer.Node, fieldLayerRemovedPackages, pkg)) } - t.RemoveQuad(cayley.Quad(layer.Node, FieldLayerEngineVersion, strconv.Itoa(existingLayer.EngineVersion), "")) + t.RemoveQuad(cayley.Triple(layer.Node, FieldLayerEngineVersion, strconv.Itoa(existingLayer.EngineVersion))) } // Add OS/Packages - t.AddQuad(cayley.Quad(layer.Node, FieldLayerOS, layer.OS, "")) + t.AddQuad(cayley.Triple(layer.Node, FieldLayerOS, layer.OS)) for _, pkg := range layer.InstalledPackagesNodes { - t.AddQuad(cayley.Quad(layer.Node, fieldLayerInstalledPackages, pkg, "")) + t.AddQuad(cayley.Triple(layer.Node, fieldLayerInstalledPackages, pkg)) } for _, pkg := range layer.RemovedPackagesNodes { - t.AddQuad(cayley.Quad(layer.Node, fieldLayerRemovedPackages, pkg, "")) + t.AddQuad(cayley.Triple(layer.Node, fieldLayerRemovedPackages, pkg)) } - t.AddQuad(cayley.Quad(layer.Node, FieldLayerEngineVersion, strconv.Itoa(layer.EngineVersion), "")) + t.AddQuad(cayley.Triple(layer.Node, FieldLayerEngineVersion, strconv.Itoa(layer.EngineVersion))) // Apply transaction if err = store.ApplyTransaction(t); err != nil { @@ -163,16 +163,16 @@ func deleteLayerTreeFrom(node string, t *graph.Transaction) error { } // Remove layer. - t.RemoveQuad(cayley.Quad(layer.Node, fieldIs, fieldLayerIsValue, "")) - t.RemoveQuad(cayley.Quad(layer.Node, FieldLayerID, layer.ID, "")) - t.RemoveQuad(cayley.Quad(layer.Node, FieldLayerParent, layer.ParentNode, "")) - t.RemoveQuad(cayley.Quad(layer.Node, FieldLayerOS, layer.OS, "")) - t.RemoveQuad(cayley.Quad(layer.Node, FieldLayerEngineVersion, strconv.Itoa(layer.EngineVersion), "")) + t.RemoveQuad(cayley.Triple(layer.Node, fieldIs, fieldLayerIsValue)) + t.RemoveQuad(cayley.Triple(layer.Node, FieldLayerID, layer.ID)) + t.RemoveQuad(cayley.Triple(layer.Node, FieldLayerParent, layer.ParentNode)) + t.RemoveQuad(cayley.Triple(layer.Node, FieldLayerOS, layer.OS)) + t.RemoveQuad(cayley.Triple(layer.Node, FieldLayerEngineVersion, strconv.Itoa(layer.EngineVersion))) for _, pkg := range layer.InstalledPackagesNodes { - t.RemoveQuad(cayley.Quad(layer.Node, fieldLayerInstalledPackages, pkg, "")) + t.RemoveQuad(cayley.Triple(layer.Node, fieldLayerInstalledPackages, pkg)) } for _, pkg := range layer.RemovedPackagesNodes { - t.RemoveQuad(cayley.Quad(layer.Node, fieldLayerRemovedPackages, pkg, "")) + t.RemoveQuad(cayley.Triple(layer.Node, fieldLayerRemovedPackages, pkg)) } // Apply transaction if root call. diff --git a/database/lock.go b/database/lock.go index b33d72fb..e33008cd 100644 --- a/database/lock.go +++ b/database/lock.go @@ -52,17 +52,17 @@ func Lock(name string, duration time.Duration, owner string) (bool, time.Time) { } t := cayley.NewTransaction() - t.RemoveQuad(cayley.Quad(name, fieldLockLockedUntil, currentExpiration, "")) - t.AddQuad(cayley.Quad(name, fieldLockLockedUntil, untilString, "")) + t.RemoveQuad(cayley.Triple(name, fieldLockLockedUntil, currentExpiration)) + t.AddQuad(cayley.Triple(name, fieldLockLockedUntil, untilString)) // It is not necessary to verify if the lock is ours again in the transaction // because if someone took it, the lock's current expiration probably changed and the transaction will fail return store.ApplyTransaction(t) == nil, until } t := cayley.NewTransaction() - t.AddQuad(cayley.Quad(name, fieldLockLocked, fieldLockLockedValue, "")) // Necessary to make the transaction fails if the lock already exists (and has not been pruned) - t.AddQuad(cayley.Quad(name, fieldLockLockedUntil, untilString, "")) - t.AddQuad(cayley.Quad(name, fieldLockLockedBy, owner, "")) + t.AddQuad(cayley.Triple(name, fieldLockLocked, fieldLockLockedValue)) // Necessary to make the transaction fails if the lock already exists (and has not been pruned) + t.AddQuad(cayley.Triple(name, fieldLockLockedUntil, untilString)) + t.AddQuad(cayley.Triple(name, fieldLockLockedBy, owner)) glog.SetStderrThreshold("FATAL") success := store.ApplyTransaction(t) == nil @@ -81,9 +81,9 @@ func Unlock(name, owner string) { it.TagResults(tags) t := cayley.NewTransaction() - t.RemoveQuad(cayley.Quad(name, fieldLockLocked, fieldLockLockedValue, "")) - t.RemoveQuad(cayley.Quad(name, fieldLockLockedUntil, store.NameOf(tags[fieldLockLockedUntil]), "")) - t.RemoveQuad(cayley.Quad(name, fieldLockLockedBy, owner, "")) + t.RemoveQuad(cayley.Triple(name, fieldLockLocked, fieldLockLockedValue)) + t.RemoveQuad(cayley.Triple(name, fieldLockLockedUntil, store.NameOf(tags[fieldLockLockedUntil]))) + t.RemoveQuad(cayley.Triple(name, fieldLockLockedBy, owner)) err := store.ApplyTransaction(t) if err != nil { log.Errorf("failed transaction (Unlock): %s", err) @@ -141,9 +141,9 @@ func pruneLocks() { log.Debugf("lock %s owned by %s has expired.", n, o) tr := cayley.NewTransaction() - tr.RemoveQuad(cayley.Quad(n, fieldLockLocked, fieldLockLockedValue, "")) - tr.RemoveQuad(cayley.Quad(n, fieldLockLockedUntil, t, "")) - tr.RemoveQuad(cayley.Quad(n, fieldLockLockedBy, o, "")) + tr.RemoveQuad(cayley.Triple(n, fieldLockLocked, fieldLockLockedValue)) + tr.RemoveQuad(cayley.Triple(n, fieldLockLockedUntil, t)) + tr.RemoveQuad(cayley.Triple(n, fieldLockLockedBy, o)) err := store.ApplyTransaction(tr) if err != nil { log.Errorf("failed transaction (pruneLocks): %s", err) diff --git a/database/notification.go b/database/notification.go index e0f33c7a..926a1da2 100644 --- a/database/notification.go +++ b/database/notification.go @@ -339,10 +339,10 @@ func InsertNotifications(notifications []Notification, wrapper NotificationWrapp } node := fieldNotificationIsValue + ":" + uuid.New() - t.AddQuad(cayley.Quad(node, fieldIs, fieldNotificationIsValue, "")) - t.AddQuad(cayley.Quad(node, fieldNotificationType, wrappedNotification.Type, "")) - t.AddQuad(cayley.Quad(node, fieldNotificationData, wrappedNotification.Data, "")) - t.AddQuad(cayley.Quad(node, fieldNotificationIsSent, strconv.FormatBool(false), "")) + t.AddQuad(cayley.Triple(node, fieldIs, fieldNotificationIsValue)) + t.AddQuad(cayley.Triple(node, fieldNotificationType, wrappedNotification.Type)) + t.AddQuad(cayley.Triple(node, fieldNotificationData, wrappedNotification.Data)) + t.AddQuad(cayley.Triple(node, fieldNotificationIsSent, strconv.FormatBool(false))) } // Apply transaction @@ -401,8 +401,8 @@ func MarkNotificationAsSent(node string) { // Initialize transaction t := cayley.NewTransaction() - t.RemoveQuad(cayley.Quad(node, fieldNotificationIsSent, strconv.FormatBool(false), "")) - t.AddQuad(cayley.Quad(node, fieldNotificationIsSent, strconv.FormatBool(true), "")) + t.RemoveQuad(cayley.Triple(node, fieldNotificationIsSent, strconv.FormatBool(false))) + t.AddQuad(cayley.Triple(node, fieldNotificationIsSent, strconv.FormatBool(true))) // Apply transaction store.ApplyTransaction(t) diff --git a/database/package.go b/database/package.go index 71c99128..70477780 100644 --- a/database/package.go +++ b/database/package.go @@ -156,11 +156,11 @@ func InsertPackages(packageParameters []*Package) error { } endPackage.Node = endPackage.GetNode() - t.AddQuad(cayley.Quad(endPackage.Node, fieldIs, fieldPackageIsValue, "")) - t.AddQuad(cayley.Quad(endPackage.Node, FieldPackageOS, endPackage.OS, "")) - t.AddQuad(cayley.Quad(endPackage.Node, FieldPackageName, endPackage.Name, "")) - t.AddQuad(cayley.Quad(endPackage.Node, FieldPackageVersion, endPackage.Version.String(), "")) - t.AddQuad(cayley.Quad(endPackage.Node, FieldPackageNextVersion, "", "")) + t.AddQuad(cayley.Triple(endPackage.Node, fieldIs, fieldPackageIsValue)) + t.AddQuad(cayley.Triple(endPackage.Node, FieldPackageOS, endPackage.OS)) + t.AddQuad(cayley.Triple(endPackage.Node, FieldPackageName, endPackage.Name)) + t.AddQuad(cayley.Triple(endPackage.Node, FieldPackageVersion, endPackage.Version.String())) + t.AddQuad(cayley.Triple(endPackage.Node, FieldPackageNextVersion, "")) // Create the inserted package if it is different than a start/end package var newPackage *Package @@ -172,11 +172,11 @@ func InsertPackages(packageParameters []*Package) error { } newPackage.Node = newPackage.GetNode() - t.AddQuad(cayley.Quad(newPackage.Node, fieldIs, fieldPackageIsValue, "")) - t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageOS, newPackage.OS, "")) - t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageName, newPackage.Name, "")) - t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageVersion, newPackage.Version.String(), "")) - t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageNextVersion, endPackage.Node, "")) + t.AddQuad(cayley.Triple(newPackage.Node, fieldIs, fieldPackageIsValue)) + t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageOS, newPackage.OS)) + t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageName, newPackage.Name)) + t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageVersion, newPackage.Version.String())) + t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageNextVersion, endPackage.Node)) packageParameter.Node = newPackage.Node } @@ -189,14 +189,14 @@ func InsertPackages(packageParameters []*Package) error { } startPackage.Node = startPackage.GetNode() - t.AddQuad(cayley.Quad(startPackage.Node, fieldIs, fieldPackageIsValue, "")) - t.AddQuad(cayley.Quad(startPackage.Node, FieldPackageOS, startPackage.OS, "")) - t.AddQuad(cayley.Quad(startPackage.Node, FieldPackageName, startPackage.Name, "")) - t.AddQuad(cayley.Quad(startPackage.Node, FieldPackageVersion, startPackage.Version.String(), "")) + t.AddQuad(cayley.Triple(startPackage.Node, fieldIs, fieldPackageIsValue)) + t.AddQuad(cayley.Triple(startPackage.Node, FieldPackageOS, startPackage.OS)) + t.AddQuad(cayley.Triple(startPackage.Node, FieldPackageName, startPackage.Name)) + t.AddQuad(cayley.Triple(startPackage.Node, FieldPackageVersion, startPackage.Version.String())) if !insertingStartPackage && !insertingEndPackage { - t.AddQuad(cayley.Quad(startPackage.Node, FieldPackageNextVersion, newPackage.Node, "")) + t.AddQuad(cayley.Triple(startPackage.Node, FieldPackageNextVersion, newPackage.Node)) } else { - t.AddQuad(cayley.Quad(startPackage.Node, FieldPackageNextVersion, endPackage.Node, "")) + t.AddQuad(cayley.Triple(startPackage.Node, FieldPackageNextVersion, endPackage.Node)) } // Set package node @@ -213,10 +213,10 @@ func InsertPackages(packageParameters []*Package) error { newPackage.Node = "package:" + utils.Hash(newPackage.Key()) packageParameter.Node = newPackage.Node - t.AddQuad(cayley.Quad(newPackage.Node, fieldIs, fieldPackageIsValue, "")) - t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageOS, newPackage.OS, "")) - t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageName, newPackage.Name, "")) - t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageVersion, newPackage.Version.String(), "")) + t.AddQuad(cayley.Triple(newPackage.Node, fieldIs, fieldPackageIsValue)) + t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageOS, newPackage.OS)) + t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageName, newPackage.Name)) + t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageVersion, newPackage.Version.String())) // Sort branchPackages by version (including the new package) branchPackages = append(branchPackages, newPackage) @@ -244,13 +244,13 @@ func InsertPackages(packageParameters []*Package) error { } // Link the new packages with the branch - t.RemoveQuad(cayley.Quad(pred.Node, FieldPackageNextVersion, succ.Node, "")) + t.RemoveQuad(cayley.Triple(pred.Node, FieldPackageNextVersion, succ.Node)) pred.NextVersionNode = newPackage.Node - t.AddQuad(cayley.Quad(pred.Node, FieldPackageNextVersion, newPackage.Node, "")) + t.AddQuad(cayley.Triple(pred.Node, FieldPackageNextVersion, newPackage.Node)) newPackage.NextVersionNode = succ.Node - t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageNextVersion, succ.Node, "")) + t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageNextVersion, succ.Node)) } // Apply transaction diff --git a/database/vulnerability.go b/database/vulnerability.go index 53e92702..fd61ec6e 100644 --- a/database/vulnerability.go +++ b/database/vulnerability.go @@ -152,13 +152,13 @@ func InsertVulnerabilities(vulnerabilities []*Vulnerability) ([]Notification, er // Insert it vulnerability.Node = vulnerability.GetNode() - t.AddQuad(cayley.Quad(vulnerability.Node, fieldIs, fieldVulnerabilityIsValue, "")) - t.AddQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityID, vulnerability.ID, "")) - t.AddQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityLink, vulnerability.Link, "")) - t.AddQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority), "")) - t.AddQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description, "")) + t.AddQuad(cayley.Triple(vulnerability.Node, fieldIs, fieldVulnerabilityIsValue)) + t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityID, vulnerability.ID)) + t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityLink, vulnerability.Link)) + t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority))) + t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description)) for _, p := range vulnerability.FixedInNodes { - t.AddQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityFixedIn, p, "")) + t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityFixedIn, p)) } // Add a notification @@ -170,8 +170,8 @@ func InsertVulnerabilities(vulnerabilities []*Vulnerability) ([]Notification, er } else { // The vulnerability already exists, update it if vulnerability.Link != "" && existingVulnerability.Link != vulnerability.Link { - t.RemoveQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityLink, existingVulnerability.Link, "")) - t.AddQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityLink, vulnerability.Link, "")) + t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityLink, existingVulnerability.Link)) + t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityLink, vulnerability.Link)) existingVulnerability.Link = vulnerability.Link } @@ -197,14 +197,14 @@ func InsertVulnerabilities(vulnerabilities []*Vulnerability) ([]Notification, er } } - t.RemoveQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityPriority, string(existingVulnerability.Priority), "")) - t.AddQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority), "")) + t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityPriority, string(existingVulnerability.Priority))) + t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority))) existingVulnerability.Priority = vulnerability.Priority } if vulnerability.Description != "" && existingVulnerability.Description != vulnerability.Description { - t.RemoveQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityDescription, existingVulnerability.Description, "")) - t.AddQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description, "")) + t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityDescription, existingVulnerability.Description)) + t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description)) existingVulnerability.Description = vulnerability.Description } @@ -226,7 +226,7 @@ func InsertVulnerabilities(vulnerabilities []*Vulnerability) ([]Notification, er for _, ep := range existingVulnerabilityFixedInPackages { if p.Branch() == ep.Branch() { // A link to this package branch already exist and is not the same version, we will delete it - t.RemoveQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityFixedIn, ep.Node, "")) + t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityFixedIn, ep.Node)) var index int for i, n := range existingVulnerability.FixedInNodes { @@ -240,7 +240,7 @@ func InsertVulnerabilities(vulnerabilities []*Vulnerability) ([]Notification, er } } - t.AddQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityFixedIn, p.Node, "")) + t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityFixedIn, p.Node)) existingVulnerability.FixedInNodes = append(existingVulnerability.FixedInNodes, p.Node) addedNodes = append(addedNodes, p.Node) } @@ -280,12 +280,12 @@ func DeleteVulnerability(id string) error { } t := cayley.NewTransaction() - t.RemoveQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityID, vulnerability.ID, "")) - t.RemoveQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityLink, vulnerability.Link, "")) - t.RemoveQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority), "")) - t.RemoveQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description, "")) + t.RemoveQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityID, vulnerability.ID)) + t.RemoveQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityLink, vulnerability.Link)) + t.RemoveQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority))) + t.RemoveQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description)) for _, p := range vulnerability.FixedInNodes { - t.RemoveQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityFixedIn, p, "")) + t.RemoveQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityFixedIn, p)) } if err := store.ApplyTransaction(t); err != nil { diff --git a/vendor/github.com/google/cayley/Dockerfile b/vendor/github.com/google/cayley/Dockerfile new file mode 100644 index 00000000..55918057 --- /dev/null +++ b/vendor/github.com/google/cayley/Dockerfile @@ -0,0 +1,21 @@ +FROM golang:latest +MAINTAINER Barak Michener + +# Set up workdir +WORKDIR /go/src/github.com/google/cayley + +# Restore vendored dependencies +RUN go get github.com/tools/godep +ADD Godeps /go/src/github.com/google/cayley/Godeps +RUN godep restore + +# Add and install cayley +ADD . . +RUN go install -v github.com/google/cayley + +# Expose the port and volume for configuration and data persistence. If you're +# using a backend like bolt, make sure the file is saved to this directory. +VOLUME ["/data"] +EXPOSE 64321 + +CMD ["cayley", "http", "-config", "/data/cayley.cfg", "-init"] diff --git a/vendor/github.com/google/cayley/README.md b/vendor/github.com/google/cayley/README.md index 8deb7967..6a771fce 100644 --- a/vendor/github.com/google/cayley/README.md +++ b/vendor/github.com/google/cayley/README.md @@ -5,7 +5,8 @@ Cayley is an open-source graph inspired by the graph database behind [Freebase]( Its goal is to be a part of the developer's toolbox where [Linked Data](http://linkeddata.org/) and graph-shaped data (semantic webs, social networks, etc) in general are concerned. -[![Build Status](https://travis-ci.org/google/cayley.png?branch=master)](https://travis-ci.org/google/cayley) [Trello Board](https://trello.com/b/KioFZb5O) +[![Build Status](https://travis-ci.org/google/cayley.png?branch=master)](https://travis-ci.org/google/cayley) +[![Container Repository on Quay](https://quay.io/repository/barakmich/cayley/status "Container Repository on Quay")](https://quay.io/repository/barakmich/cayley) ## Features @@ -20,6 +21,7 @@ Its goal is to be a part of the developer's toolbox where [Linked Data](http://l * Plays well with multiple backend stores: * [LevelDB](https://github.com/google/leveldb) * [Bolt](https://github.com/boltdb/bolt) + * [PostgreSQL](http://www.postgresql.org) * [MongoDB](https://www.mongodb.org) for distributed stores * In-memory, ephemeral * Modular design; easy to extend with new languages and backends @@ -88,7 +90,7 @@ g.V("dani").Tag("source").Out("follows").Tag("target").All() ``` The visualizer expects to tag nodes as either "source" or "target." Your source is represented as a blue node. While your target is represented as an orange node. -The idea being that our node relationship goes from blue to orange (source to target). +The idea being that our node relationship goes from blue to orange (source to target). **Sample Data** @@ -162,6 +164,17 @@ g.V().Has("name", "Casablanca").Follow(filmToActor).Out("name").All() There's more in the JavaScript API Documentation, but that should give you a feel for how to walk around the graph. +## Running in a container + +A container exposing the HTTP API of cayley is available. +To run the container one must first setup a data directory that contains the configuration file and optionally contains persistent files (i.e. a boltdb database file). + +``` +mkdir data +cp my_config.cfg data/cayley.cfg +docker run -v $PWD/data:/data -p 64321:64321 -d quay.io/barakmich/cayley +``` + ## Disclaimer Not a Google project, but created and maintained [by a Googler](https://github.com/barakmich), with permission from and assignment to Google, under the [Apache License, version 2.0](http://www.apache.org/licenses/LICENSE-2.0). diff --git a/vendor/github.com/google/cayley/cmd/cayley/cayley.go b/vendor/github.com/google/cayley/cmd/cayley/cayley.go index edf0a661..20b88751 100644 --- a/vendor/github.com/google/cayley/cmd/cayley/cayley.go +++ b/vendor/github.com/google/cayley/cmd/cayley/cayley.go @@ -45,6 +45,7 @@ import ( var ( quadFile = flag.String("quads", "", "Quad file to load before going to REPL.") + initOpt = flag.Bool("init", false, "Initialize the database before using it. Equivalent to running `cayley init` followed by the given command.") quadType = flag.String("format", "cquad", `Quad format to use for loading ("cquad" or "nquad").`) cpuprofile = flag.String("prof", "", "Output profiling file.") queryLanguage = flag.String("query_lang", "gremlin", "Use this parser as the query language.") @@ -238,6 +239,12 @@ func main() { handle.Close() case "repl": + if *initOpt { + err = db.Init(cfg) + if err != nil && err != graph.ErrDatabaseExists { + break + } + } handle, err = db.Open(cfg) if err != nil { break @@ -254,6 +261,12 @@ func main() { handle.Close() case "http": + if *initOpt { + err = db.Init(cfg) + if err != nil && err != graph.ErrDatabaseExists { + break + } + } handle, err = db.Open(cfg) if err != nil { break diff --git a/vendor/github.com/google/cayley/graph/bolt/quadstore.go b/vendor/github.com/google/cayley/graph/bolt/quadstore.go index ea4b5765..92eb77ed 100644 --- a/vendor/github.com/google/cayley/graph/bolt/quadstore.go +++ b/vendor/github.com/google/cayley/graph/bolt/quadstore.go @@ -85,6 +85,11 @@ func createNewBolt(path string, _ graph.Options) error { defer db.Close() qs := &QuadStore{} qs.db = db + defer qs.Close() + err = qs.getMetadata() + if err != errNoBucket { + return graph.ErrDatabaseExists + } err = qs.createBuckets() if err != nil { return err diff --git a/vendor/github.com/google/cayley/graph/leveldb/quadstore.go b/vendor/github.com/google/cayley/graph/leveldb/quadstore.go index 8437ae62..b4f7b2ea 100644 --- a/vendor/github.com/google/cayley/graph/leveldb/quadstore.go +++ b/vendor/github.com/google/cayley/graph/leveldb/quadstore.go @@ -48,6 +48,8 @@ const ( DefaultCacheSize = 2 DefaultWriteBufferSize = 20 QuadStoreType = "leveldb" + horizonKey = "__horizon" + sizeKey = "__size" ) var ( @@ -87,6 +89,16 @@ func createNewLevelDB(path string, _ graph.Options) error { qs.writeopts = &opt.WriteOptions{ Sync: true, } + qs.readopts = &opt.ReadOptions{} + _, err = qs.db.Get([]byte(horizonKey), qs.readopts) + if err != nil && err != leveldb.ErrNotFound { + glog.Errorln("couldn't read from leveldb during init") + return err + } + if err != leveldb.ErrNotFound { + return graph.ErrDatabaseExists + } + // Write some metadata qs.Close() return nil } @@ -343,7 +355,7 @@ func (qs *QuadStore) Close() { buf := new(bytes.Buffer) err := binary.Write(buf, binary.LittleEndian, qs.size) if err == nil { - werr := qs.db.Put([]byte("__size"), buf.Bytes(), qs.writeopts) + werr := qs.db.Put([]byte(sizeKey), buf.Bytes(), qs.writeopts) if werr != nil { glog.Error("could not write size before closing!") } @@ -353,7 +365,7 @@ func (qs *QuadStore) Close() { buf.Reset() err = binary.Write(buf, binary.LittleEndian, qs.horizon) if err == nil { - werr := qs.db.Put([]byte("__horizon"), buf.Bytes(), qs.writeopts) + werr := qs.db.Put([]byte(horizonKey), buf.Bytes(), qs.writeopts) if werr != nil { glog.Error("could not write horizon before closing!") } @@ -444,11 +456,11 @@ func (qs *QuadStore) getInt64ForKey(key string, empty int64) (int64, error) { func (qs *QuadStore) getMetadata() error { var err error - qs.size, err = qs.getInt64ForKey("__size", 0) + qs.size, err = qs.getInt64ForKey(sizeKey, 0) if err != nil { return err } - qs.horizon, err = qs.getInt64ForKey("__horizon", 0) + qs.horizon, err = qs.getInt64ForKey(horizonKey, 0) return err } diff --git a/vendor/github.com/google/cayley/graph/quadstore.go b/vendor/github.com/google/cayley/graph/quadstore.go index 04b5c375..df4312d2 100644 --- a/vendor/github.com/google/cayley/graph/quadstore.go +++ b/vendor/github.com/google/cayley/graph/quadstore.go @@ -140,6 +140,7 @@ func (d Options) BoolKey(key string) (bool, bool, error) { } var ErrCannotBulkLoad = errors.New("quadstore: cannot bulk load") +var ErrDatabaseExists = errors.New("quadstore: cannot init; database already exists") type BulkLoader interface { // BulkLoad loads Quads from a quad.Unmarshaler in bulk to the QuadStore. diff --git a/vendor/github.com/google/cayley/graph/sql/quadstore.go b/vendor/github.com/google/cayley/graph/sql/quadstore.go index 09773820..e7eee2ea 100644 --- a/vendor/github.com/google/cayley/graph/sql/quadstore.go +++ b/vendor/github.com/google/cayley/graph/sql/quadstore.go @@ -67,6 +67,7 @@ func createSQLTables(addr string, options graph.Options) error { if err != nil { return err } + defer conn.Close() tx, err := conn.Begin() if err != nil { glog.Errorf("Couldn't begin creation transaction: %s", err) @@ -89,6 +90,11 @@ func createSQLTables(addr string, options graph.Options) error { UNIQUE(subject_hash, predicate_hash, object_hash, label_hash) );`) if err != nil { + tx.Rollback() + errd := err.(*pq.Error) + if errd.Code == "42P07" { + return graph.ErrDatabaseExists + } glog.Errorf("Cannot create quad table: %v", quadTable) return err } @@ -105,6 +111,7 @@ func createSQLTables(addr string, options graph.Options) error { `, factor, factor, factor)) if err != nil { glog.Errorf("Cannot create indices: %v", index) + tx.Rollback() return err } tx.Commit() @@ -310,7 +317,9 @@ func (qs *QuadStore) Horizon() graph.PrimaryKey { var horizon int64 err := qs.db.QueryRow("SELECT horizon FROM quads ORDER BY horizon DESC LIMIT 1;").Scan(&horizon) if err != nil { - glog.Errorf("Couldn't execute horizon: %v", err) + if err != sql.ErrNoRows { + glog.Errorf("Couldn't execute horizon: %v", err) + } return graph.NewSequentialKey(0) } return graph.NewSequentialKey(horizon) diff --git a/vendor/github.com/google/cayley/imports.go b/vendor/github.com/google/cayley/imports.go index c855ce9e..f47c41ff 100644 --- a/vendor/github.com/google/cayley/imports.go +++ b/vendor/github.com/google/cayley/imports.go @@ -27,6 +27,10 @@ type Handle struct { graph.QuadWriter } +func Triple(subject, predicate, object string) quad.Quad { + return quad.Quad{subject, predicate, object, ""} +} + func Quad(subject, predicate, object, label string) quad.Quad { return quad.Quad{subject, predicate, object, label} }