implement db backend for votes/likes #5

isso/__init__.py

@@ -55,6 +55,7 @@ url_map = Map([
     Rule('/', methods=['HEAD', 'GET'], endpoint=views.comment.get),
     Rule('/', methods=['PUT', 'DELETE'], endpoint=views.comment.modify),
     Rule('/new', methods=['POST'], endpoint=views.comment.create),
+    Rule('/like', methods=['POST'], endpoint=views.comment.like),
 
     Rule('/admin/', endpoint=views.admin.index)
 ])

isso/db.py

@@ -7,6 +7,7 @@ import abc
 import time
 import sqlite3
 
+from isso.utils import Bloomfilter
 from isso.models import Comment
 
 
@@ -65,8 +66,12 @@ class Abstract:
         """
         return
 
-
+    @abc.abstractmethod
-
+    def like(self, path, id, remote_addr):
+        """+1 a given comment. Returns the new like count (may not change because
+        the creater can't vote on his/her own comment and multiple votes from the
+        same ip address are ignored as well)."""
+        return
 
 class SQLite(Abstract):
     """A basic :class:`Abstract` implementation using SQLite3.  All comments
@@ -76,7 +81,7 @@ class SQLite(Abstract):
 
     fields = [
         'path', 'id', 'created', 'modified',
-        'text', 'author', 'hash', 'website', 'parent', 'mode'
+        'text', 'author', 'hash', 'website', 'parent', 'mode', 'voters'
     ]
 
     def __init__(self, dbpath, moderation):
@@ -89,6 +94,7 @@ class SQLite(Abstract):
                    'created FLOAT NOT NULL, modified FLOAT, text VARCHAR,'
                    'author VARCHAR(64), hash VARCHAR(32), website VARCHAR(64),'
                    'parent INTEGER, mode INTEGER,'
+                   'likes INTEGER DEFAULT 0, dislikes INTEGER DEFAULT 0, voters BLOB NOT NULL,'
                    'PRIMARY KEY (id, path))')
             con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql)
 
@@ -101,22 +107,29 @@ class SQLite(Abstract):
                     WHERE rowid=NEW.rowid;
                 END;""")
 
+            # threads (path -> title for now)
+            sql = ('main.threads (path VARCHAR(255) NOT NULL, title TEXT'
+                   'PRIMARY KEY path)')
+            con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql)
+
     def query2comment(self, query):
         if query is None:
             return None
 
         return Comment(
             text=query[4], author=query[5], hash=query[6], website=query[7], parent=query[8],
-            path=query[0], id=query[1], created=query[2], modified=query[3], mode=query[9]
+            path=query[0], id=query[1], created=query[2], modified=query[3], mode=query[9],
+            votes=query[10]
         )
 
-    def add(self, path, c):
+    def add(self, path, c, remote_addr):
+        voters = buffer(Bloomfilter(iterable=[remote_addr]).array)
         with sqlite3.connect(self.dbpath) as con:
             keys = ','.join(self.fields)
             values = ','.join('?' * len(self.fields))
             con.execute('INSERT INTO comments (%s) VALUES (%s);' % (keys, values), (
                 path, 0, c.created, c.modified, c.text, c.author, c.hash, c.website,
-                c.parent, self.mode)
+                c.parent, self.mode, voters)
             )
 
         with sqlite3.connect(self.dbpath) as con:
@@ -159,6 +172,27 @@ class SQLite(Abstract):
                     (None, path, id))
         return self.get(path, id)
 
+    def like(self, path, id, remote_addr):
+        with sqlite3.connect(self.dbpath) as con:
+            rv = con.execute("SELECT likes, dislikes, voters FROM comments" \
+               + " WHERE path=? AND id=?", (path, id)).fetchone()
+
+        likes, dislikes, voters = rv
+        if likes + dislikes >= 142:
+            return likes
+
+        bf = Bloomfilter(bytearray(voters), likes + dislikes)
+        if remote_addr in bf:
+            return likes
+
+        bf.add(remote_addr)
+        with sqlite3.connect(self.dbpath) as con:
+            con.execute("UPDATE comments SET likes = likes + 1 WHERE path=? AND id=?", (path, id))
+            con.execute("UPDATE comments SET voters = ? WHERE path=? AND id=?", (
+                buffer(bf.array), path, id))
+
+        return likes + 1
+
     def retrieve(self, path, mode=5):
         with sqlite3.connect(self.dbpath) as con:
             rv = con.execute("SELECT * FROM comments WHERE path=? AND (? | mode) = ?" \
@@ -181,3 +215,4 @@ class SQLite(Abstract):
 
         for item in rv:
             yield self.query2comment(item)
+

isso/models.py

@@ -25,7 +25,7 @@ class Comment(object):
     normal and queued using MODE=3.
     """
 
-    protected = ['path', 'id', 'mode', 'created', 'modified', 'hash']
+    protected = ['path', 'id', 'mode', 'created', 'modified', 'hash', 'votes']
     fields = ['text', 'author', 'website', 'parent']
 
     def __init__(self, **kw):

isso/utils.py

@@ -3,12 +3,16 @@
 # Copyright 2012, Martin Zimmermann <info@posativ.org>. All rights reserved.
 # License: BSD Style, 2 clauses. see isso/__init__.py
 
+from __future__ import division
+
 import json
 
 import socket
 import httplib
 
+import math
 import random
+import hashlib
 import contextlib
 
 from string import ascii_letters, digits
@@ -46,3 +50,51 @@ def normalize(host):
 
 def mksecret(length):
     return ''.join(random.choice(ascii_letters + digits) for x in range(length))
+
+
+class Bloomfilter:
+    """A space-efficient probabilistic data structure. False-positive rate:
+
+        * 1e-05 for  <80 elements
+        * 1e-04 for <105 elements
+        * 1e-03 for <142 elements
+
+    Uses a 256 byte array (2048 bits) and 11 hash functions. 256 byte because
+    of space efficiency (array is saved for each comment) and 11 hash functions
+    because of best overall false-positive rate in that range.
+
+    -- via Raymond Hettinger
+       http://code.activestate.com/recipes/577684-bloom-filter/
+    """
+
+    def __init__(self, array=bytearray(256), elements=0, iterable=()):
+        self.array = array
+        self.elements = elements
+        self.k = 11
+        self.m = len(array) * 8
+
+        for item in iterable:
+            self.add(item)
+
+    def get_probes(self, key):
+        h = int(hashlib.sha256(key.encode()).hexdigest(), 16)
+        for _ in range(self.k):
+            yield h & self.m - 1
+            h >>= self.k
+
+    def add(self, key):
+        for i in self.get_probes(key):
+            self.array[i//8] |= 2 ** (i%8)
+        self.elements += 1
+
+    @property
+    def density(self):
+        c = ''.join(format(x, '08b') for x in self.array)
+        return c.count('1') / len(c)
+
+    def __contains__(self, key):
+        return all(self.array[i//8] & (2 ** (i%8)) for i in self.get_probes(key))
+
+    def __len__(self):
+        return self.elements
+

isso/views/comment.py

@@ -55,7 +55,7 @@ def create(app, environ, request, uri):
                 Response('', 400)
 
     try:
-        rv = app.db.add(uri, comment)
+        rv = app.db.add(uri, comment, request.remote_addr)
     except ValueError:
         abort(400)  # FIXME: custom exception class, error descr
 
@@ -120,6 +120,14 @@ def modify(app, environ, request, uri, id):
         return resp
 
 
+@requires(str, 'uri')
+@requires(int, 'id')
+def like(app, environ, request, uri, id):
+
+    nv = app.db.like(uri, id, request.remote_addr)
+    return Response(str(nv), 200)
+
+
 def approve(app, environ, request, path, id):
 
     try:

specs/test_db.py

@@ -23,7 +23,7 @@ class TestSQLite(unittest.TestCase):
 
     def test_get(self):
 
-        rv = self.db.add('/', comment(text='Spam'))
+        rv = self.db.add('/', comment(text='Spam'), '')
         c = self.db.get('/', rv.id)
 
         assert c.id == 1
@@ -31,9 +31,9 @@ class TestSQLite(unittest.TestCase):
 
     def test_add(self):
 
-        self.db.add('/', comment(text='Foo'))
+        self.db.add('/', comment(text='Foo'), '')
-        self.db.add('/', comment(text='Bar'))
+        self.db.add('/', comment(text='Bar'), '')
-        self.db.add('/path/', comment(text='Baz'))
+        self.db.add('/path/', comment(text='Baz'), '')
 
         rv = list(self.db.retrieve('/'))
         assert rv[0].id == 1
@@ -50,14 +50,14 @@ class TestSQLite(unittest.TestCase):
 
     def test_add_return(self):
 
-        self.db.add('/', comment(text='1'))
+        self.db.add('/', comment(text='1'), '')
-        self.db.add('/', comment(text='2'))
+        self.db.add('/', comment(text='2'), '')
 
-        assert self.db.add('/path/', comment(text='1')).id == 1
+        assert self.db.add('/path/', comment(text='1'), '').id == 1
 
     def test_update(self):
 
-        rv = self.db.add('/', comment(text='Foo'))
+        rv = self.db.add('/', comment(text='Foo'), '')
         time.sleep(0.1)
         rv = self.db.update('/', rv.id, comment(text='Bla'))
         c = self.db.get('/', rv.id)
@@ -69,15 +69,15 @@ class TestSQLite(unittest.TestCase):
     def test_delete(self):
 
         rv = self.db.add('/', comment(
-            text='F**CK', author='P*NIS', website='http://somebadhost.org/'))
+            text='F**CK', author='P*NIS', website='http://somebadhost.org/'), '')
         assert self.db.delete('/', rv.id) == None
 
     def test_recent(self):
 
-        self.db.add('/path/', comment(text='2'))
+        self.db.add('/path/', comment(text='2'), '')
 
         for x in range(5):
-            self.db.add('/', comment(text='%i' % (x+1)))
+            self.db.add('/', comment(text='%i' % (x+1)), '')
 
         assert len(list(self.db.recent(mode=7))) == 6
         assert len(list(self.db.recent(mode=7, limit=5))) == 5
@@ -95,13 +95,13 @@ class TestSQLitePending(unittest.TestCase):
 
     def test_retrieve(self):
 
-        self.db.add('/', comment(text='Foo'))
+        self.db.add('/', comment(text='Foo'), '')
         assert len(list(self.db.retrieve('/'))) == 0
 
     def test_activate(self):
 
-        self.db.add('/', comment(text='Foo'))
+        self.db.add('/', comment(text='Foo'), '')
-        self.db.add('/', comment(text='Bar'))
+        self.db.add('/', comment(text='Bar'), '')
         self.db.activate('/', 2)
 
         assert len(list(self.db.retrieve('/'))) == 1

specs/test_vote.py

@@ -0,0 +1,80 @@
+
+import os
+import json
+import tempfile
+import unittest
+
+from werkzeug.test import Client
+from werkzeug.wrappers import Response
+
+from isso import Isso
+
+
+class FakeIP(object):
+
+    def __init__(self, app, ip):
+        self.app = app
+        self.ip = ip
+
+    def __call__(self, environ, start_response):
+        environ['REMOTE_ADDR'] = self.ip
+        return self.app(environ, start_response)
+
+
+class TestVote(unittest.TestCase):
+
+    def setUp(self):
+        fd, self.path = tempfile.mkstemp()
+
+    def tearDown(self):
+        os.unlink(self.path)
+
+    def makeClient(self, ip):
+
+        app = Isso(self.path, '...', '...', 15*60, "...")
+        app.wsgi_app = FakeIP(app.wsgi_app, ip)
+
+        return Client(app, Response)
+
+    def testZeroLikes(self):
+
+        rv = self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."}))
+        assert json.loads(rv.data)['votes'] == 0
+
+    def testSingleLike(self):
+
+        self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."}))
+        rv = self.makeClient("0.0.0.0").post("/like?uri=test&id=1")
+
+        assert rv.status_code == 200
+        assert rv.data == "1"
+
+    def testSelfLike(self):
+
+        bob = self.makeClient("127.0.0.1")
+        bob.post("/new?uri=test", data=json.dumps({"text": "..."}))
+        rv = bob.post('/like?uri=test&id=1')
+
+        assert rv.status_code == 200
+        assert rv.data == "0"
+
+    def testMultipleLikes(self):
+
+        self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."}))
+        for num in range(15):
+            rv = self.makeClient("1.2.3.%i" % num).post('/like?uri=test&id=1')
+            assert rv.status_code == 200
+            assert rv.data == str(num + 1)
+
+    def testTooManyLikes(self):
+
+        self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."}))
+        for num in range(256):
+            rv = self.makeClient("1.2.3.%i" % num).post('/like?uri=test&id=1')
+            assert rv.status_code == 200
+
+            if num >= 142:
+                assert rv.data == "142"
+            else:
+                assert rv.data == str(num + 1)
+