arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

database: Update cayley and use Triple instead of Quad

Browse Source
This commit is contained in:
Quentin Machu 2015-12-05 02:42:37 -05:00
parent 941a10cc2b
commit 3fe3f3a4c7
17 changed files with 178 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Godeps/Godeps.json generated
View File

@ -41,8 +41,7 @@
}, },
{ {
"ImportPath": "github.com/google/cayley", "ImportPath": "github.com/google/cayley",
"Comment": "v0.4.1-160-gcdf0154", "Rev": "582c4e1ca46943f2cf09c73bd12a83a6959057c9"
"Rev": "f143602b8ae880ec975e4c154dd010047de80c1f"
}, },
{ {
"ImportPath": "github.com/julienschmidt/httprouter", "ImportPath": "github.com/julienschmidt/httprouter",

11
database/database.go
View File

@ -78,18 +78,21 @@ func Open(dbType, dbPath string) error {
log.Infof("database at %s does not exist yet, creating it", dbPath) log.Infof("database at %s does not exist yet, creating it", dbPath)
err = graph.InitQuadStore(dbType, dbPath, options) err = graph.InitQuadStore(dbType, dbPath, options)
if err != nil { if err != nil && err != graph.ErrDatabaseExists {
log.Errorf("could not create database at %s : %s", dbPath, err) log.Errorf("could not create database at %s : %s", dbPath, err)
return ErrCantOpen return ErrCantOpen
} }
} }
case "sql": case "sql":
// Replaces the PostgreSQL's slow COUNT query with a fast estimator. // Replaces the PostgreSQL's slow COUNT query with a fast estimator.
// See:
// Ref: https://wiki.postgresql.org/wiki/Count_estimate // Ref: https://wiki.postgresql.org/wiki/Count_estimate
options["use_estimates"] = true options["use_estimates"] = true
graph.InitQuadStore(dbType, dbPath, options) err := graph.InitQuadStore(dbType, dbPath, options)
if err != nil && err != graph.ErrDatabaseExists {
log.Errorf("could not create database at %s : %s", dbPath, err)
return ErrCantOpen
}
} }
store, err = cayley.NewGraph(dbType, dbPath, options) store, err = cayley.NewGraph(dbType, dbPath, options)
@ -115,7 +118,7 @@ func Healthcheck() health.Status {
var err error var err error
if store != nil { if store != nil {
t := cayley.NewTransaction() t := cayley.NewTransaction()
q := cayley.Quad("cayley", "is", "healthy", "") q := cayley.Triple("cayley", "is", "healthy")
t.AddQuad(q) t.AddQuad(q)
t.RemoveQuad(q) t.RemoveQuad(q)
glog.SetStderrThreshold("FATAL") // TODO REMOVE ME glog.SetStderrThreshold("FATAL") // TODO REMOVE ME

12
database/database_test.go
View File

@ -38,12 +38,12 @@ func TestToValue(t *testing.T) {
assert.Nil(t, err, "toValue should work even if the requested path leads to nothing") assert.Nil(t, err, "toValue should work even if the requested path leads to nothing")
assert.Equal(t, "", v, "toValue should return an empty string if the requested path leads to nothing") assert.Equal(t, "", v, "toValue should return an empty string if the requested path leads to nothing")
store.AddQuad(cayley.Quad("tests", "are", "awesome", "")) store.AddQuad(cayley.Triple("tests", "are", "awesome"))
v, err = toValue(cayley.StartPath(store, "tests").Out("are")) v, err = toValue(cayley.StartPath(store, "tests").Out("are"))
assert.Nil(t, err, "toValue should have worked") assert.Nil(t, err, "toValue should have worked")
assert.Equal(t, "awesome", v, "toValue did not return the expected value") assert.Equal(t, "awesome", v, "toValue did not return the expected value")
store.AddQuad(cayley.Quad("tests", "are", "running", "")) store.AddQuad(cayley.Triple("tests", "are", "running"))
v, err = toValue(cayley.StartPath(store, "tests").Out("are")) v, err = toValue(cayley.StartPath(store, "tests").Out("are"))
assert.NotNil(t, err, "toValue should return an error and an empty string if the path leads to multiple values") assert.NotNil(t, err, "toValue should return an error and an empty string if the path leads to multiple values")
assert.Equal(t, "", v, "toValue should return an error and an empty string if the path leads to multiple values") assert.Equal(t, "", v, "toValue should return an error and an empty string if the path leads to multiple values")
@ -54,7 +54,7 @@ func TestToValue(t *testing.T) {
assert.Len(t, vs, 0, "toValue should return an empty array if the requested path leads to nothing") assert.Len(t, vs, 0, "toValue should return an empty array if the requested path leads to nothing")
words := []string{"powerful", "lightweight"} words := []string{"powerful", "lightweight"}
for i, word := range words { for i, word := range words {
store.AddQuad(cayley.Quad("CoreOS", fieldIs, word, "")) store.AddQuad(cayley.Triple("CoreOS", fieldIs, word))
v, err := toValues(cayley.StartPath(store, "CoreOS").Out(fieldIs)) v, err := toValues(cayley.StartPath(store, "CoreOS").Out(fieldIs))
assert.Nil(t, err, "toValues should have worked") assert.Nil(t, err, "toValues should have worked")
assert.Len(t, v, i+1, "toValues did not return the right amount of values") assert.Len(t, v, i+1, "toValues did not return the right amount of values")
@ -64,17 +64,17 @@ func TestToValue(t *testing.T) {
} }
// toValue(s)() and empty values // toValue(s)() and empty values
store.AddQuad(cayley.Quad("bob", "likes", "", "")) store.AddQuad(cayley.Triple("bob", "likes", ""))
v, err = toValue(cayley.StartPath(store, "bob").Out("likes")) v, err = toValue(cayley.StartPath(store, "bob").Out("likes"))
assert.Nil(t, err, "toValue should work even if the requested path leads to nothing") assert.Nil(t, err, "toValue should work even if the requested path leads to nothing")
assert.Equal(t, "", v, "toValue should return an empty string if the requested path leads to nothing") assert.Equal(t, "", v, "toValue should return an empty string if the requested path leads to nothing")
store.AddQuad(cayley.Quad("bob", "likes", "running", "")) store.AddQuad(cayley.Triple("bob", "likes", "running"))
v, err = toValue(cayley.StartPath(store, "bob").Out("likes")) v, err = toValue(cayley.StartPath(store, "bob").Out("likes"))
assert.Nil(t, err, "toValue should have worked") assert.Nil(t, err, "toValue should have worked")
assert.Equal(t, "running", v, "toValue did not return the expected value") assert.Equal(t, "running", v, "toValue did not return the expected value")
store.AddQuad(cayley.Quad("bob", "likes", "swimming", "")) store.AddQuad(cayley.Triple("bob", "likes", "swimming"))
va, err := toValues(cayley.StartPath(store, "bob").Out("likes")) va, err := toValues(cayley.StartPath(store, "bob").Out("likes"))
assert.Nil(t, err, "toValues should have worked") assert.Nil(t, err, "toValues should have worked")
assert.Len(t, va, 2, "toValues should have returned 2 values") assert.Len(t, va, 2, "toValues should have returned 2 values")

4
database/flag.go
View File

@ -43,9 +43,9 @@ func UpdateFlag(name, value string) error {
// Build transaction // Build transaction
name = flagNodePrefix + ":" + name name = flagNodePrefix + ":" + name
if currentValue != "" { if currentValue != "" {
t.RemoveQuad(cayley.Quad(name, fieldFlagValue, currentValue, "")) t.RemoveQuad(cayley.Triple(name, fieldFlagValue, currentValue))
} }
t.AddQuad(cayley.Quad(name, fieldFlagValue, value, "")) t.AddQuad(cayley.Triple(name, fieldFlagValue, value))
// Apply transaction // Apply transaction
if err = store.ApplyTransaction(t); err != nil { if err = store.ApplyTransaction(t); err != nil {

36
database/layer.go
View File

@ -98,30 +98,30 @@ func InsertLayer(layer *Layer) error {
if existingLayer == nil { if existingLayer == nil {
// Create case: add permanent nodes // Create case: add permanent nodes
t.AddQuad(cayley.Quad(layer.Node, fieldIs, fieldLayerIsValue, "")) t.AddQuad(cayley.Triple(layer.Node, fieldIs, fieldLayerIsValue))
t.AddQuad(cayley.Quad(layer.Node, FieldLayerID, layer.ID, "")) t.AddQuad(cayley.Triple(layer.Node, FieldLayerID, layer.ID))
t.AddQuad(cayley.Quad(layer.Node, FieldLayerParent, layer.ParentNode, "")) t.AddQuad(cayley.Triple(layer.Node, FieldLayerParent, layer.ParentNode))
} else { } else {
// Update case: remove everything before we add updated data // Update case: remove everything before we add updated data
t.RemoveQuad(cayley.Quad(layer.Node, FieldLayerOS, existingLayer.OS, "")) t.RemoveQuad(cayley.Triple(layer.Node, FieldLayerOS, existingLayer.OS))
for _, pkg := range existingLayer.InstalledPackagesNodes { for _, pkg := range existingLayer.InstalledPackagesNodes {
t.RemoveQuad(cayley.Quad(layer.Node, fieldLayerInstalledPackages, pkg, "")) t.RemoveQuad(cayley.Triple(layer.Node, fieldLayerInstalledPackages, pkg))
} }
for _, pkg := range existingLayer.RemovedPackagesNodes { for _, pkg := range existingLayer.RemovedPackagesNodes {
t.RemoveQuad(cayley.Quad(layer.Node, fieldLayerRemovedPackages, pkg, "")) t.RemoveQuad(cayley.Triple(layer.Node, fieldLayerRemovedPackages, pkg))
} }
t.RemoveQuad(cayley.Quad(layer.Node, FieldLayerEngineVersion, strconv.Itoa(existingLayer.EngineVersion), "")) t.RemoveQuad(cayley.Triple(layer.Node, FieldLayerEngineVersion, strconv.Itoa(existingLayer.EngineVersion)))
} }
// Add OS/Packages // Add OS/Packages
t.AddQuad(cayley.Quad(layer.Node, FieldLayerOS, layer.OS, "")) t.AddQuad(cayley.Triple(layer.Node, FieldLayerOS, layer.OS))
for _, pkg := range layer.InstalledPackagesNodes { for _, pkg := range layer.InstalledPackagesNodes {
t.AddQuad(cayley.Quad(layer.Node, fieldLayerInstalledPackages, pkg, "")) t.AddQuad(cayley.Triple(layer.Node, fieldLayerInstalledPackages, pkg))
} }
for _, pkg := range layer.RemovedPackagesNodes { for _, pkg := range layer.RemovedPackagesNodes {
t.AddQuad(cayley.Quad(layer.Node, fieldLayerRemovedPackages, pkg, "")) t.AddQuad(cayley.Triple(layer.Node, fieldLayerRemovedPackages, pkg))
} }
t.AddQuad(cayley.Quad(layer.Node, FieldLayerEngineVersion, strconv.Itoa(layer.EngineVersion), "")) t.AddQuad(cayley.Triple(layer.Node, FieldLayerEngineVersion, strconv.Itoa(layer.EngineVersion)))
// Apply transaction // Apply transaction
if err = store.ApplyTransaction(t); err != nil { if err = store.ApplyTransaction(t); err != nil {
@ -163,16 +163,16 @@ func deleteLayerTreeFrom(node string, t *graph.Transaction) error {
} }
// Remove layer. // Remove layer.
t.RemoveQuad(cayley.Quad(layer.Node, fieldIs, fieldLayerIsValue, "")) t.RemoveQuad(cayley.Triple(layer.Node, fieldIs, fieldLayerIsValue))
t.RemoveQuad(cayley.Quad(layer.Node, FieldLayerID, layer.ID, "")) t.RemoveQuad(cayley.Triple(layer.Node, FieldLayerID, layer.ID))
t.RemoveQuad(cayley.Quad(layer.Node, FieldLayerParent, layer.ParentNode, "")) t.RemoveQuad(cayley.Triple(layer.Node, FieldLayerParent, layer.ParentNode))
t.RemoveQuad(cayley.Quad(layer.Node, FieldLayerOS, layer.OS, "")) t.RemoveQuad(cayley.Triple(layer.Node, FieldLayerOS, layer.OS))
t.RemoveQuad(cayley.Quad(layer.Node, FieldLayerEngineVersion, strconv.Itoa(layer.EngineVersion), "")) t.RemoveQuad(cayley.Triple(layer.Node, FieldLayerEngineVersion, strconv.Itoa(layer.EngineVersion)))
for _, pkg := range layer.InstalledPackagesNodes { for _, pkg := range layer.InstalledPackagesNodes {
t.RemoveQuad(cayley.Quad(layer.Node, fieldLayerInstalledPackages, pkg, "")) t.RemoveQuad(cayley.Triple(layer.Node, fieldLayerInstalledPackages, pkg))
} }
for _, pkg := range layer.RemovedPackagesNodes { for _, pkg := range layer.RemovedPackagesNodes {
t.RemoveQuad(cayley.Quad(layer.Node, fieldLayerRemovedPackages, pkg, "")) t.RemoveQuad(cayley.Triple(layer.Node, fieldLayerRemovedPackages, pkg))
} }
// Apply transaction if root call. // Apply transaction if root call.

22
database/lock.go
View File

@ -52,17 +52,17 @@ func Lock(name string, duration time.Duration, owner string) (bool, time.Time) {
} }
t := cayley.NewTransaction() t := cayley.NewTransaction()
t.RemoveQuad(cayley.Quad(name, fieldLockLockedUntil, currentExpiration, "")) t.RemoveQuad(cayley.Triple(name, fieldLockLockedUntil, currentExpiration))
t.AddQuad(cayley.Quad(name, fieldLockLockedUntil, untilString, "")) t.AddQuad(cayley.Triple(name, fieldLockLockedUntil, untilString))
// It is not necessary to verify if the lock is ours again in the transaction // It is not necessary to verify if the lock is ours again in the transaction
// because if someone took it, the lock's current expiration probably changed and the transaction will fail // because if someone took it, the lock's current expiration probably changed and the transaction will fail
return store.ApplyTransaction(t) == nil, until return store.ApplyTransaction(t) == nil, until
} }
t := cayley.NewTransaction() t := cayley.NewTransaction()
t.AddQuad(cayley.Quad(name, fieldLockLocked, fieldLockLockedValue, "")) // Necessary to make the transaction fails if the lock already exists (and has not been pruned) t.AddQuad(cayley.Triple(name, fieldLockLocked, fieldLockLockedValue)) // Necessary to make the transaction fails if the lock already exists (and has not been pruned)
t.AddQuad(cayley.Quad(name, fieldLockLockedUntil, untilString, "")) t.AddQuad(cayley.Triple(name, fieldLockLockedUntil, untilString))
t.AddQuad(cayley.Quad(name, fieldLockLockedBy, owner, "")) t.AddQuad(cayley.Triple(name, fieldLockLockedBy, owner))
glog.SetStderrThreshold("FATAL") glog.SetStderrThreshold("FATAL")
success := store.ApplyTransaction(t) == nil success := store.ApplyTransaction(t) == nil
@ -81,9 +81,9 @@ func Unlock(name, owner string) {
it.TagResults(tags) it.TagResults(tags)
t := cayley.NewTransaction() t := cayley.NewTransaction()
t.RemoveQuad(cayley.Quad(name, fieldLockLocked, fieldLockLockedValue, "")) t.RemoveQuad(cayley.Triple(name, fieldLockLocked, fieldLockLockedValue))
t.RemoveQuad(cayley.Quad(name, fieldLockLockedUntil, store.NameOf(tags[fieldLockLockedUntil]), "")) t.RemoveQuad(cayley.Triple(name, fieldLockLockedUntil, store.NameOf(tags[fieldLockLockedUntil])))
t.RemoveQuad(cayley.Quad(name, fieldLockLockedBy, owner, "")) t.RemoveQuad(cayley.Triple(name, fieldLockLockedBy, owner))
err := store.ApplyTransaction(t) err := store.ApplyTransaction(t)
if err != nil { if err != nil {
log.Errorf("failed transaction (Unlock): %s", err) log.Errorf("failed transaction (Unlock): %s", err)
@ -141,9 +141,9 @@ func pruneLocks() {
log.Debugf("lock %s owned by %s has expired.", n, o) log.Debugf("lock %s owned by %s has expired.", n, o)
tr := cayley.NewTransaction() tr := cayley.NewTransaction()
tr.RemoveQuad(cayley.Quad(n, fieldLockLocked, fieldLockLockedValue, "")) tr.RemoveQuad(cayley.Triple(n, fieldLockLocked, fieldLockLockedValue))
tr.RemoveQuad(cayley.Quad(n, fieldLockLockedUntil, t, "")) tr.RemoveQuad(cayley.Triple(n, fieldLockLockedUntil, t))
tr.RemoveQuad(cayley.Quad(n, fieldLockLockedBy, o, "")) tr.RemoveQuad(cayley.Triple(n, fieldLockLockedBy, o))
err := store.ApplyTransaction(tr) err := store.ApplyTransaction(tr)
if err != nil { if err != nil {
log.Errorf("failed transaction (pruneLocks): %s", err) log.Errorf("failed transaction (pruneLocks): %s", err)

12
database/notification.go
View File

@ -339,10 +339,10 @@ func InsertNotifications(notifications []Notification, wrapper NotificationWrapp
} }
node := fieldNotificationIsValue + ":" + uuid.New() node := fieldNotificationIsValue + ":" + uuid.New()
t.AddQuad(cayley.Quad(node, fieldIs, fieldNotificationIsValue, "")) t.AddQuad(cayley.Triple(node, fieldIs, fieldNotificationIsValue))
t.AddQuad(cayley.Quad(node, fieldNotificationType, wrappedNotification.Type, "")) t.AddQuad(cayley.Triple(node, fieldNotificationType, wrappedNotification.Type))
t.AddQuad(cayley.Quad(node, fieldNotificationData, wrappedNotification.Data, "")) t.AddQuad(cayley.Triple(node, fieldNotificationData, wrappedNotification.Data))
t.AddQuad(cayley.Quad(node, fieldNotificationIsSent, strconv.FormatBool(false), "")) t.AddQuad(cayley.Triple(node, fieldNotificationIsSent, strconv.FormatBool(false)))
} }
// Apply transaction // Apply transaction
@ -401,8 +401,8 @@ func MarkNotificationAsSent(node string) {
// Initialize transaction // Initialize transaction
t := cayley.NewTransaction() t := cayley.NewTransaction()
t.RemoveQuad(cayley.Quad(node, fieldNotificationIsSent, strconv.FormatBool(false), "")) t.RemoveQuad(cayley.Triple(node, fieldNotificationIsSent, strconv.FormatBool(false)))
t.AddQuad(cayley.Quad(node, fieldNotificationIsSent, strconv.FormatBool(true), "")) t.AddQuad(cayley.Triple(node, fieldNotificationIsSent, strconv.FormatBool(true)))
// Apply transaction // Apply transaction
store.ApplyTransaction(t) store.ApplyTransaction(t)

46
database/package.go
View File

@ -156,11 +156,11 @@ func InsertPackages(packageParameters []*Package) error {
} }
endPackage.Node = endPackage.GetNode() endPackage.Node = endPackage.GetNode()
t.AddQuad(cayley.Quad(endPackage.Node, fieldIs, fieldPackageIsValue, "")) t.AddQuad(cayley.Triple(endPackage.Node, fieldIs, fieldPackageIsValue))
t.AddQuad(cayley.Quad(endPackage.Node, FieldPackageOS, endPackage.OS, "")) t.AddQuad(cayley.Triple(endPackage.Node, FieldPackageOS, endPackage.OS))
t.AddQuad(cayley.Quad(endPackage.Node, FieldPackageName, endPackage.Name, "")) t.AddQuad(cayley.Triple(endPackage.Node, FieldPackageName, endPackage.Name))
t.AddQuad(cayley.Quad(endPackage.Node, FieldPackageVersion, endPackage.Version.String(), "")) t.AddQuad(cayley.Triple(endPackage.Node, FieldPackageVersion, endPackage.Version.String()))
t.AddQuad(cayley.Quad(endPackage.Node, FieldPackageNextVersion, "", "")) t.AddQuad(cayley.Triple(endPackage.Node, FieldPackageNextVersion, ""))
// Create the inserted package if it is different than a start/end package // Create the inserted package if it is different than a start/end package
var newPackage *Package var newPackage *Package
@ -172,11 +172,11 @@ func InsertPackages(packageParameters []*Package) error {
} }
newPackage.Node = newPackage.GetNode() newPackage.Node = newPackage.GetNode()
t.AddQuad(cayley.Quad(newPackage.Node, fieldIs, fieldPackageIsValue, "")) t.AddQuad(cayley.Triple(newPackage.Node, fieldIs, fieldPackageIsValue))
t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageOS, newPackage.OS, "")) t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageOS, newPackage.OS))
t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageName, newPackage.Name, "")) t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageName, newPackage.Name))
t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageVersion, newPackage.Version.String(), "")) t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageVersion, newPackage.Version.String()))
t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageNextVersion, endPackage.Node, "")) t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageNextVersion, endPackage.Node))
packageParameter.Node = newPackage.Node packageParameter.Node = newPackage.Node
} }
@ -189,14 +189,14 @@ func InsertPackages(packageParameters []*Package) error {
} }
startPackage.Node = startPackage.GetNode() startPackage.Node = startPackage.GetNode()
t.AddQuad(cayley.Quad(startPackage.Node, fieldIs, fieldPackageIsValue, "")) t.AddQuad(cayley.Triple(startPackage.Node, fieldIs, fieldPackageIsValue))
t.AddQuad(cayley.Quad(startPackage.Node, FieldPackageOS, startPackage.OS, "")) t.AddQuad(cayley.Triple(startPackage.Node, FieldPackageOS, startPackage.OS))
t.AddQuad(cayley.Quad(startPackage.Node, FieldPackageName, startPackage.Name, "")) t.AddQuad(cayley.Triple(startPackage.Node, FieldPackageName, startPackage.Name))
t.AddQuad(cayley.Quad(startPackage.Node, FieldPackageVersion, startPackage.Version.String(), "")) t.AddQuad(cayley.Triple(startPackage.Node, FieldPackageVersion, startPackage.Version.String()))
if !insertingStartPackage && !insertingEndPackage { if !insertingStartPackage && !insertingEndPackage {
t.AddQuad(cayley.Quad(startPackage.Node, FieldPackageNextVersion, newPackage.Node, "")) t.AddQuad(cayley.Triple(startPackage.Node, FieldPackageNextVersion, newPackage.Node))
} else { } else {
t.AddQuad(cayley.Quad(startPackage.Node, FieldPackageNextVersion, endPackage.Node, "")) t.AddQuad(cayley.Triple(startPackage.Node, FieldPackageNextVersion, endPackage.Node))
} }
// Set package node // Set package node
@ -213,10 +213,10 @@ func InsertPackages(packageParameters []*Package) error {
newPackage.Node = "package:" + utils.Hash(newPackage.Key()) newPackage.Node = "package:" + utils.Hash(newPackage.Key())
packageParameter.Node = newPackage.Node packageParameter.Node = newPackage.Node
t.AddQuad(cayley.Quad(newPackage.Node, fieldIs, fieldPackageIsValue, "")) t.AddQuad(cayley.Triple(newPackage.Node, fieldIs, fieldPackageIsValue))
t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageOS, newPackage.OS, "")) t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageOS, newPackage.OS))
t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageName, newPackage.Name, "")) t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageName, newPackage.Name))
t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageVersion, newPackage.Version.String(), "")) t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageVersion, newPackage.Version.String()))
// Sort branchPackages by version (including the new package) // Sort branchPackages by version (including the new package)
branchPackages = append(branchPackages, newPackage) branchPackages = append(branchPackages, newPackage)
@ -244,13 +244,13 @@ func InsertPackages(packageParameters []*Package) error {
} }
// Link the new packages with the branch // Link the new packages with the branch
t.RemoveQuad(cayley.Quad(pred.Node, FieldPackageNextVersion, succ.Node, "")) t.RemoveQuad(cayley.Triple(pred.Node, FieldPackageNextVersion, succ.Node))
pred.NextVersionNode = newPackage.Node pred.NextVersionNode = newPackage.Node
t.AddQuad(cayley.Quad(pred.Node, FieldPackageNextVersion, newPackage.Node, "")) t.AddQuad(cayley.Triple(pred.Node, FieldPackageNextVersion, newPackage.Node))
newPackage.NextVersionNode = succ.Node newPackage.NextVersionNode = succ.Node
t.AddQuad(cayley.Quad(newPackage.Node, FieldPackageNextVersion, succ.Node, "")) t.AddQuad(cayley.Triple(newPackage.Node, FieldPackageNextVersion, succ.Node))
} }
// Apply transaction // Apply transaction

38
database/vulnerability.go
View File

@ -152,13 +152,13 @@ func InsertVulnerabilities(vulnerabilities []*Vulnerability) ([]Notification, er
// Insert it // Insert it
vulnerability.Node = vulnerability.GetNode() vulnerability.Node = vulnerability.GetNode()
t.AddQuad(cayley.Quad(vulnerability.Node, fieldIs, fieldVulnerabilityIsValue, "")) t.AddQuad(cayley.Triple(vulnerability.Node, fieldIs, fieldVulnerabilityIsValue))
t.AddQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityID, vulnerability.ID, "")) t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityID, vulnerability.ID))
t.AddQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityLink, vulnerability.Link, "")) t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityLink, vulnerability.Link))
t.AddQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority), "")) t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority)))
t.AddQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description, "")) t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description))
for _, p := range vulnerability.FixedInNodes { for _, p := range vulnerability.FixedInNodes {
t.AddQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityFixedIn, p, "")) t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityFixedIn, p))
} }
// Add a notification // Add a notification
@ -170,8 +170,8 @@ func InsertVulnerabilities(vulnerabilities []*Vulnerability) ([]Notification, er
} else { } else {
// The vulnerability already exists, update it // The vulnerability already exists, update it
if vulnerability.Link != "" && existingVulnerability.Link != vulnerability.Link { if vulnerability.Link != "" && existingVulnerability.Link != vulnerability.Link {
t.RemoveQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityLink, existingVulnerability.Link, "")) t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityLink, existingVulnerability.Link))
t.AddQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityLink, vulnerability.Link, "")) t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityLink, vulnerability.Link))
existingVulnerability.Link = vulnerability.Link existingVulnerability.Link = vulnerability.Link
} }
@ -197,14 +197,14 @@ func InsertVulnerabilities(vulnerabilities []*Vulnerability) ([]Notification, er
} }
} }
t.RemoveQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityPriority, string(existingVulnerability.Priority), "")) t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityPriority, string(existingVulnerability.Priority)))
t.AddQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority), "")) t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority)))
existingVulnerability.Priority = vulnerability.Priority existingVulnerability.Priority = vulnerability.Priority
} }
if vulnerability.Description != "" && existingVulnerability.Description != vulnerability.Description { if vulnerability.Description != "" && existingVulnerability.Description != vulnerability.Description {
t.RemoveQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityDescription, existingVulnerability.Description, "")) t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityDescription, existingVulnerability.Description))
t.AddQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description, "")) t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description))
existingVulnerability.Description = vulnerability.Description existingVulnerability.Description = vulnerability.Description
} }
@ -226,7 +226,7 @@ func InsertVulnerabilities(vulnerabilities []*Vulnerability) ([]Notification, er
for _, ep := range existingVulnerabilityFixedInPackages { for _, ep := range existingVulnerabilityFixedInPackages {
if p.Branch() == ep.Branch() { if p.Branch() == ep.Branch() {
// A link to this package branch already exist and is not the same version, we will delete it // A link to this package branch already exist and is not the same version, we will delete it
t.RemoveQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityFixedIn, ep.Node, "")) t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityFixedIn, ep.Node))
var index int var index int
for i, n := range existingVulnerability.FixedInNodes { for i, n := range existingVulnerability.FixedInNodes {
@ -240,7 +240,7 @@ func InsertVulnerabilities(vulnerabilities []*Vulnerability) ([]Notification, er
} }
} }
t.AddQuad(cayley.Quad(existingVulnerability.Node, FieldVulnerabilityFixedIn, p.Node, "")) t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityFixedIn, p.Node))
existingVulnerability.FixedInNodes = append(existingVulnerability.FixedInNodes, p.Node) existingVulnerability.FixedInNodes = append(existingVulnerability.FixedInNodes, p.Node)
addedNodes = append(addedNodes, p.Node) addedNodes = append(addedNodes, p.Node)
} }
@ -280,12 +280,12 @@ func DeleteVulnerability(id string) error {
} }
t := cayley.NewTransaction() t := cayley.NewTransaction()
t.RemoveQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityID, vulnerability.ID, "")) t.RemoveQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityID, vulnerability.ID))
t.RemoveQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityLink, vulnerability.Link, "")) t.RemoveQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityLink, vulnerability.Link))
t.RemoveQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority), "")) t.RemoveQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority)))
t.RemoveQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description, "")) t.RemoveQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description))
for _, p := range vulnerability.FixedInNodes { for _, p := range vulnerability.FixedInNodes {
t.RemoveQuad(cayley.Quad(vulnerability.Node, FieldVulnerabilityFixedIn, p, "")) t.RemoveQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityFixedIn, p))
} }
if err := store.ApplyTransaction(t); err != nil { if err := store.ApplyTransaction(t); err != nil {

21
vendor/github.com/google/cayley/Dockerfile generated vendored Normal file
View File

@ -0,0 +1,21 @@
FROM golang:latest
MAINTAINER Barak Michener <me@barakmich.com>
# Set up workdir
WORKDIR /go/src/github.com/google/cayley
# Restore vendored dependencies
RUN go get github.com/tools/godep
ADD Godeps /go/src/github.com/google/cayley/Godeps
RUN godep restore
# Add and install cayley
ADD . .
RUN go install -v github.com/google/cayley
# Expose the port and volume for configuration and data persistence. If you're
# using a backend like bolt, make sure the file is saved to this directory.
VOLUME ["/data"]
EXPOSE 64321
CMD ["cayley", "http", "-config", "/data/cayley.cfg", "-init"]

17
vendor/github.com/google/cayley/README.md generated vendored
View File

@ -5,7 +5,8 @@ Cayley is an open-source graph inspired by the graph database behind [Freebase](
Its goal is to be a part of the developer's toolbox where [Linked Data](http://linkeddata.org/) and graph-shaped data (semantic webs, social networks, etc) in general are concerned. Its goal is to be a part of the developer's toolbox where [Linked Data](http://linkeddata.org/) and graph-shaped data (semantic webs, social networks, etc) in general are concerned.
[![Build Status](https://travis-ci.org/google/cayley.png?branch=master)](https://travis-ci.org/google/cayley) [Trello Board](https://trello.com/b/KioFZb5O) [![Build Status](https://travis-ci.org/google/cayley.png?branch=master)](https://travis-ci.org/google/cayley)
[![Container Repository on Quay](https://quay.io/repository/barakmich/cayley/status "Container Repository on Quay")](https://quay.io/repository/barakmich/cayley)
## Features ## Features
@ -20,6 +21,7 @@ Its goal is to be a part of the developer's toolbox where [Linked Data](http://l
* Plays well with multiple backend stores: * Plays well with multiple backend stores:
* [LevelDB](https://github.com/google/leveldb) * [LevelDB](https://github.com/google/leveldb)
* [Bolt](https://github.com/boltdb/bolt) * [Bolt](https://github.com/boltdb/bolt)
* [PostgreSQL](http://www.postgresql.org)
* [MongoDB](https://www.mongodb.org) for distributed stores * [MongoDB](https://www.mongodb.org) for distributed stores
* In-memory, ephemeral * In-memory, ephemeral
* Modular design; easy to extend with new languages and backends * Modular design; easy to extend with new languages and backends
@ -88,7 +90,7 @@ g.V("dani").Tag("source").Out("follows").Tag("target").All()
``` ```
The visualizer expects to tag nodes as either "source" or "target." Your source is represented as a blue node. The visualizer expects to tag nodes as either "source" or "target." Your source is represented as a blue node.
While your target is represented as an orange node. While your target is represented as an orange node.
The idea being that our node relationship goes from blue to orange (source to target). The idea being that our node relationship goes from blue to orange (source to target).
**Sample Data** **Sample Data**
@ -162,6 +164,17 @@ g.V().Has("name", "Casablanca").Follow(filmToActor).Out("name").All()
There's more in the JavaScript API Documentation, but that should give you a feel for how to walk around the graph. There's more in the JavaScript API Documentation, but that should give you a feel for how to walk around the graph.
## Running in a container
A container exposing the HTTP API of cayley is available.
To run the container one must first setup a data directory that contains the configuration file and optionally contains persistent files (i.e. a boltdb database file).
```
mkdir data
cp my_config.cfg data/cayley.cfg
docker run -v $PWD/data:/data -p 64321:64321 -d quay.io/barakmich/cayley
```
## Disclaimer ## Disclaimer
Not a Google project, but created and maintained [by a Googler](https://github.com/barakmich), with permission from and assignment to Google, under the [Apache License, version 2.0](http://www.apache.org/licenses/LICENSE-2.0). Not a Google project, but created and maintained [by a Googler](https://github.com/barakmich), with permission from and assignment to Google, under the [Apache License, version 2.0](http://www.apache.org/licenses/LICENSE-2.0).

13
vendor/github.com/google/cayley/cmd/cayley/cayley.go generated vendored
View File

@ -45,6 +45,7 @@ import (
var ( var (
quadFile = flag.String("quads", "", "Quad file to load before going to REPL.") quadFile = flag.String("quads", "", "Quad file to load before going to REPL.")
initOpt = flag.Bool("init", false, "Initialize the database before using it. Equivalent to running `cayley init` followed by the given command.")
quadType = flag.String("format", "cquad", `Quad format to use for loading ("cquad" or "nquad").`) quadType = flag.String("format", "cquad", `Quad format to use for loading ("cquad" or "nquad").`)
cpuprofile = flag.String("prof", "", "Output profiling file.") cpuprofile = flag.String("prof", "", "Output profiling file.")
queryLanguage = flag.String("query_lang", "gremlin", "Use this parser as the query language.") queryLanguage = flag.String("query_lang", "gremlin", "Use this parser as the query language.")
@ -238,6 +239,12 @@ func main() {
handle.Close() handle.Close()
case "repl": case "repl":
if *initOpt {
err = db.Init(cfg)
if err != nil && err != graph.ErrDatabaseExists {
break
}
}
handle, err = db.Open(cfg) handle, err = db.Open(cfg)
if err != nil { if err != nil {
break break
@ -254,6 +261,12 @@ func main() {
handle.Close() handle.Close()
case "http": case "http":
if *initOpt {
err = db.Init(cfg)
if err != nil && err != graph.ErrDatabaseExists {
break
}
}
handle, err = db.Open(cfg) handle, err = db.Open(cfg)
if err != nil { if err != nil {
break break

5
vendor/github.com/google/cayley/graph/bolt/quadstore.go generated vendored
View File

@ -85,6 +85,11 @@ func createNewBolt(path string, _ graph.Options) error {
defer db.Close() defer db.Close()
qs := &QuadStore{} qs := &QuadStore{}
qs.db = db qs.db = db
defer qs.Close()
err = qs.getMetadata()
if err != errNoBucket {
return graph.ErrDatabaseExists
}
err = qs.createBuckets() err = qs.createBuckets()
if err != nil { if err != nil {
return err return err

20
vendor/github.com/google/cayley/graph/leveldb/quadstore.go generated vendored
View File

@ -48,6 +48,8 @@ const (
DefaultCacheSize = 2 DefaultCacheSize = 2
DefaultWriteBufferSize = 20 DefaultWriteBufferSize = 20
QuadStoreType = "leveldb" QuadStoreType = "leveldb"
horizonKey = "__horizon"
sizeKey = "__size"
) )
var ( var (
@ -87,6 +89,16 @@ func createNewLevelDB(path string, _ graph.Options) error {
qs.writeopts = &opt.WriteOptions{ qs.writeopts = &opt.WriteOptions{
Sync: true, Sync: true,
} }
qs.readopts = &opt.ReadOptions{}
_, err = qs.db.Get([]byte(horizonKey), qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorln("couldn't read from leveldb during init")
return err
}
if err != leveldb.ErrNotFound {
return graph.ErrDatabaseExists
}
// Write some metadata
qs.Close() qs.Close()
return nil return nil
} }
@ -343,7 +355,7 @@ func (qs *QuadStore) Close() {
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
err := binary.Write(buf, binary.LittleEndian, qs.size) err := binary.Write(buf, binary.LittleEndian, qs.size)
if err == nil { if err == nil {
werr := qs.db.Put([]byte("__size"), buf.Bytes(), qs.writeopts) werr := qs.db.Put([]byte(sizeKey), buf.Bytes(), qs.writeopts)
if werr != nil { if werr != nil {
glog.Error("could not write size before closing!") glog.Error("could not write size before closing!")
} }
@ -353,7 +365,7 @@ func (qs *QuadStore) Close() {
buf.Reset() buf.Reset()
err = binary.Write(buf, binary.LittleEndian, qs.horizon) err = binary.Write(buf, binary.LittleEndian, qs.horizon)
if err == nil { if err == nil {
werr := qs.db.Put([]byte("__horizon"), buf.Bytes(), qs.writeopts) werr := qs.db.Put([]byte(horizonKey), buf.Bytes(), qs.writeopts)
if werr != nil { if werr != nil {
glog.Error("could not write horizon before closing!") glog.Error("could not write horizon before closing!")
} }
@ -444,11 +456,11 @@ func (qs *QuadStore) getInt64ForKey(key string, empty int64) (int64, error) {
func (qs *QuadStore) getMetadata() error { func (qs *QuadStore) getMetadata() error {
var err error var err error
qs.size, err = qs.getInt64ForKey("__size", 0) qs.size, err = qs.getInt64ForKey(sizeKey, 0)
if err != nil { if err != nil {
return err return err
} }
qs.horizon, err = qs.getInt64ForKey("__horizon", 0) qs.horizon, err = qs.getInt64ForKey(horizonKey, 0)
return err return err
} }

1
vendor/github.com/google/cayley/graph/quadstore.go generated vendored
View File

@ -140,6 +140,7 @@ func (d Options) BoolKey(key string) (bool, bool, error) {
} }
var ErrCannotBulkLoad = errors.New("quadstore: cannot bulk load") var ErrCannotBulkLoad = errors.New("quadstore: cannot bulk load")
var ErrDatabaseExists = errors.New("quadstore: cannot init; database already exists")
type BulkLoader interface { type BulkLoader interface {
// BulkLoad loads Quads from a quad.Unmarshaler in bulk to the QuadStore. // BulkLoad loads Quads from a quad.Unmarshaler in bulk to the QuadStore.

11
vendor/github.com/google/cayley/graph/sql/quadstore.go generated vendored
View File

@ -67,6 +67,7 @@ func createSQLTables(addr string, options graph.Options) error {
if err != nil { if err != nil {
return err return err
} }
defer conn.Close()
tx, err := conn.Begin() tx, err := conn.Begin()
if err != nil { if err != nil {
glog.Errorf("Couldn't begin creation transaction: %s", err) glog.Errorf("Couldn't begin creation transaction: %s", err)
@ -89,6 +90,11 @@ func createSQLTables(addr string, options graph.Options) error {
UNIQUE(subject_hash, predicate_hash, object_hash, label_hash) UNIQUE(subject_hash, predicate_hash, object_hash, label_hash)
);`) );`)
if err != nil { if err != nil {
tx.Rollback()
errd := err.(*pq.Error)
if errd.Code == "42P07" {
return graph.ErrDatabaseExists
}
glog.Errorf("Cannot create quad table: %v", quadTable) glog.Errorf("Cannot create quad table: %v", quadTable)
return err return err
} }
@ -105,6 +111,7 @@ func createSQLTables(addr string, options graph.Options) error {
`, factor, factor, factor)) `, factor, factor, factor))
if err != nil { if err != nil {
glog.Errorf("Cannot create indices: %v", index) glog.Errorf("Cannot create indices: %v", index)
tx.Rollback()
return err return err
} }
tx.Commit() tx.Commit()
@ -310,7 +317,9 @@ func (qs *QuadStore) Horizon() graph.PrimaryKey {
var horizon int64 var horizon int64
err := qs.db.QueryRow("SELECT horizon FROM quads ORDER BY horizon DESC LIMIT 1;").Scan(&horizon) err := qs.db.QueryRow("SELECT horizon FROM quads ORDER BY horizon DESC LIMIT 1;").Scan(&horizon)
if err != nil { if err != nil {
glog.Errorf("Couldn't execute horizon: %v", err) if err != sql.ErrNoRows {
glog.Errorf("Couldn't execute horizon: %v", err)
}
return graph.NewSequentialKey(0) return graph.NewSequentialKey(0)
} }
return graph.NewSequentialKey(horizon) return graph.NewSequentialKey(horizon)

4
vendor/github.com/google/cayley/imports.go generated vendored
View File

@ -27,6 +27,10 @@ type Handle struct {
graph.QuadWriter graph.QuadWriter
} }
func Triple(subject, predicate, object string) quad.Quad {
return quad.Quad{subject, predicate, object, ""}
}
func Quad(subject, predicate, object, label string) quad.Quad { func Quad(subject, predicate, object, label string) quad.Quad {
return quad.Quad{subject, predicate, object, label} return quad.Quad{subject, predicate, object, label}
} }