diff --git a/isso/__init__.py b/isso/__init__.py index 12b3d44..cb10721 100644 --- a/isso/__init__.py +++ b/isso/__init__.py @@ -55,6 +55,7 @@ url_map = Map([ Rule('/', methods=['HEAD', 'GET'], endpoint=views.comment.get), Rule('/', methods=['PUT', 'DELETE'], endpoint=views.comment.modify), Rule('/new', methods=['POST'], endpoint=views.comment.create), + Rule('/like', methods=['POST'], endpoint=views.comment.like), Rule('/admin/', endpoint=views.admin.index) ]) diff --git a/isso/db.py b/isso/db.py index d7dc0e0..428a983 100644 --- a/isso/db.py +++ b/isso/db.py @@ -7,6 +7,7 @@ import abc import time import sqlite3 +from isso.utils import Bloomfilter from isso.models import Comment @@ -65,8 +66,12 @@ class Abstract: """ return - - + @abc.abstractmethod + def like(self, path, id, remote_addr): + """+1 a given comment. Returns the new like count (may not change because + the creater can't vote on his/her own comment and multiple votes from the + same ip address are ignored as well).""" + return class SQLite(Abstract): """A basic :class:`Abstract` implementation using SQLite3. All comments @@ -76,7 +81,7 @@ class SQLite(Abstract): fields = [ 'path', 'id', 'created', 'modified', - 'text', 'author', 'hash', 'website', 'parent', 'mode' + 'text', 'author', 'hash', 'website', 'parent', 'mode', 'voters' ] def __init__(self, dbpath, moderation): @@ -89,6 +94,7 @@ class SQLite(Abstract): 'created FLOAT NOT NULL, modified FLOAT, text VARCHAR,' 'author VARCHAR(64), hash VARCHAR(32), website VARCHAR(64),' 'parent INTEGER, mode INTEGER,' + 'likes INTEGER DEFAULT 0, dislikes INTEGER DEFAULT 0, voters BLOB NOT NULL,' 'PRIMARY KEY (id, path))') con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql) @@ -101,22 +107,29 @@ class SQLite(Abstract): WHERE rowid=NEW.rowid; END;""") + # threads (path -> title for now) + sql = ('main.threads (path VARCHAR(255) NOT NULL, title TEXT' + 'PRIMARY KEY path)') + con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql) + def query2comment(self, query): if query is None: return None return Comment( text=query[4], author=query[5], hash=query[6], website=query[7], parent=query[8], - path=query[0], id=query[1], created=query[2], modified=query[3], mode=query[9] + path=query[0], id=query[1], created=query[2], modified=query[3], mode=query[9], + votes=query[10] ) - def add(self, path, c): + def add(self, path, c, remote_addr): + voters = buffer(Bloomfilter(iterable=[remote_addr]).array) with sqlite3.connect(self.dbpath) as con: keys = ','.join(self.fields) values = ','.join('?' * len(self.fields)) con.execute('INSERT INTO comments (%s) VALUES (%s);' % (keys, values), ( path, 0, c.created, c.modified, c.text, c.author, c.hash, c.website, - c.parent, self.mode) + c.parent, self.mode, voters) ) with sqlite3.connect(self.dbpath) as con: @@ -159,6 +172,27 @@ class SQLite(Abstract): (None, path, id)) return self.get(path, id) + def like(self, path, id, remote_addr): + with sqlite3.connect(self.dbpath) as con: + rv = con.execute("SELECT likes, dislikes, voters FROM comments" \ + + " WHERE path=? AND id=?", (path, id)).fetchone() + + likes, dislikes, voters = rv + if likes + dislikes >= 142: + return likes + + bf = Bloomfilter(bytearray(voters), likes + dislikes) + if remote_addr in bf: + return likes + + bf.add(remote_addr) + with sqlite3.connect(self.dbpath) as con: + con.execute("UPDATE comments SET likes = likes + 1 WHERE path=? AND id=?", (path, id)) + con.execute("UPDATE comments SET voters = ? WHERE path=? AND id=?", ( + buffer(bf.array), path, id)) + + return likes + 1 + def retrieve(self, path, mode=5): with sqlite3.connect(self.dbpath) as con: rv = con.execute("SELECT * FROM comments WHERE path=? AND (? | mode) = ?" \ @@ -181,3 +215,4 @@ class SQLite(Abstract): for item in rv: yield self.query2comment(item) + diff --git a/isso/models.py b/isso/models.py index 324f07d..c615ca1 100644 --- a/isso/models.py +++ b/isso/models.py @@ -25,7 +25,7 @@ class Comment(object): normal and queued using MODE=3. """ - protected = ['path', 'id', 'mode', 'created', 'modified', 'hash'] + protected = ['path', 'id', 'mode', 'created', 'modified', 'hash', 'votes'] fields = ['text', 'author', 'website', 'parent'] def __init__(self, **kw): diff --git a/isso/utils.py b/isso/utils.py index b37b2f7..be31c87 100644 --- a/isso/utils.py +++ b/isso/utils.py @@ -3,12 +3,16 @@ # Copyright 2012, Martin Zimmermann . All rights reserved. # License: BSD Style, 2 clauses. see isso/__init__.py +from __future__ import division + import json import socket import httplib +import math import random +import hashlib import contextlib from string import ascii_letters, digits @@ -46,3 +50,51 @@ def normalize(host): def mksecret(length): return ''.join(random.choice(ascii_letters + digits) for x in range(length)) + + +class Bloomfilter: + """A space-efficient probabilistic data structure. False-positive rate: + + * 1e-05 for <80 elements + * 1e-04 for <105 elements + * 1e-03 for <142 elements + + Uses a 256 byte array (2048 bits) and 11 hash functions. 256 byte because + of space efficiency (array is saved for each comment) and 11 hash functions + because of best overall false-positive rate in that range. + + -- via Raymond Hettinger + http://code.activestate.com/recipes/577684-bloom-filter/ + """ + + def __init__(self, array=bytearray(256), elements=0, iterable=()): + self.array = array + self.elements = elements + self.k = 11 + self.m = len(array) * 8 + + for item in iterable: + self.add(item) + + def get_probes(self, key): + h = int(hashlib.sha256(key.encode()).hexdigest(), 16) + for _ in range(self.k): + yield h & self.m - 1 + h >>= self.k + + def add(self, key): + for i in self.get_probes(key): + self.array[i//8] |= 2 ** (i%8) + self.elements += 1 + + @property + def density(self): + c = ''.join(format(x, '08b') for x in self.array) + return c.count('1') / len(c) + + def __contains__(self, key): + return all(self.array[i//8] & (2 ** (i%8)) for i in self.get_probes(key)) + + def __len__(self): + return self.elements + diff --git a/isso/views/comment.py b/isso/views/comment.py index b188b41..10ba58d 100644 --- a/isso/views/comment.py +++ b/isso/views/comment.py @@ -55,7 +55,7 @@ def create(app, environ, request, uri): Response('', 400) try: - rv = app.db.add(uri, comment) + rv = app.db.add(uri, comment, request.remote_addr) except ValueError: abort(400) # FIXME: custom exception class, error descr @@ -120,6 +120,14 @@ def modify(app, environ, request, uri, id): return resp +@requires(str, 'uri') +@requires(int, 'id') +def like(app, environ, request, uri, id): + + nv = app.db.like(uri, id, request.remote_addr) + return Response(str(nv), 200) + + def approve(app, environ, request, path, id): try: diff --git a/specs/test_db.py b/specs/test_db.py index 4165f9a..547b0bb 100644 --- a/specs/test_db.py +++ b/specs/test_db.py @@ -23,7 +23,7 @@ class TestSQLite(unittest.TestCase): def test_get(self): - rv = self.db.add('/', comment(text='Spam')) + rv = self.db.add('/', comment(text='Spam'), '') c = self.db.get('/', rv.id) assert c.id == 1 @@ -31,9 +31,9 @@ class TestSQLite(unittest.TestCase): def test_add(self): - self.db.add('/', comment(text='Foo')) - self.db.add('/', comment(text='Bar')) - self.db.add('/path/', comment(text='Baz')) + self.db.add('/', comment(text='Foo'), '') + self.db.add('/', comment(text='Bar'), '') + self.db.add('/path/', comment(text='Baz'), '') rv = list(self.db.retrieve('/')) assert rv[0].id == 1 @@ -50,14 +50,14 @@ class TestSQLite(unittest.TestCase): def test_add_return(self): - self.db.add('/', comment(text='1')) - self.db.add('/', comment(text='2')) + self.db.add('/', comment(text='1'), '') + self.db.add('/', comment(text='2'), '') - assert self.db.add('/path/', comment(text='1')).id == 1 + assert self.db.add('/path/', comment(text='1'), '').id == 1 def test_update(self): - rv = self.db.add('/', comment(text='Foo')) + rv = self.db.add('/', comment(text='Foo'), '') time.sleep(0.1) rv = self.db.update('/', rv.id, comment(text='Bla')) c = self.db.get('/', rv.id) @@ -69,15 +69,15 @@ class TestSQLite(unittest.TestCase): def test_delete(self): rv = self.db.add('/', comment( - text='F**CK', author='P*NIS', website='http://somebadhost.org/')) + text='F**CK', author='P*NIS', website='http://somebadhost.org/'), '') assert self.db.delete('/', rv.id) == None def test_recent(self): - self.db.add('/path/', comment(text='2')) + self.db.add('/path/', comment(text='2'), '') for x in range(5): - self.db.add('/', comment(text='%i' % (x+1))) + self.db.add('/', comment(text='%i' % (x+1)), '') assert len(list(self.db.recent(mode=7))) == 6 assert len(list(self.db.recent(mode=7, limit=5))) == 5 @@ -95,13 +95,13 @@ class TestSQLitePending(unittest.TestCase): def test_retrieve(self): - self.db.add('/', comment(text='Foo')) + self.db.add('/', comment(text='Foo'), '') assert len(list(self.db.retrieve('/'))) == 0 def test_activate(self): - self.db.add('/', comment(text='Foo')) - self.db.add('/', comment(text='Bar')) + self.db.add('/', comment(text='Foo'), '') + self.db.add('/', comment(text='Bar'), '') self.db.activate('/', 2) assert len(list(self.db.retrieve('/'))) == 1 diff --git a/specs/test_vote.py b/specs/test_vote.py new file mode 100644 index 0000000..6047039 --- /dev/null +++ b/specs/test_vote.py @@ -0,0 +1,80 @@ + +import os +import json +import tempfile +import unittest + +from werkzeug.test import Client +from werkzeug.wrappers import Response + +from isso import Isso + + +class FakeIP(object): + + def __init__(self, app, ip): + self.app = app + self.ip = ip + + def __call__(self, environ, start_response): + environ['REMOTE_ADDR'] = self.ip + return self.app(environ, start_response) + + +class TestVote(unittest.TestCase): + + def setUp(self): + fd, self.path = tempfile.mkstemp() + + def tearDown(self): + os.unlink(self.path) + + def makeClient(self, ip): + + app = Isso(self.path, '...', '...', 15*60, "...") + app.wsgi_app = FakeIP(app.wsgi_app, ip) + + return Client(app, Response) + + def testZeroLikes(self): + + rv = self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."})) + assert json.loads(rv.data)['votes'] == 0 + + def testSingleLike(self): + + self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."})) + rv = self.makeClient("0.0.0.0").post("/like?uri=test&id=1") + + assert rv.status_code == 200 + assert rv.data == "1" + + def testSelfLike(self): + + bob = self.makeClient("127.0.0.1") + bob.post("/new?uri=test", data=json.dumps({"text": "..."})) + rv = bob.post('/like?uri=test&id=1') + + assert rv.status_code == 200 + assert rv.data == "0" + + def testMultipleLikes(self): + + self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."})) + for num in range(15): + rv = self.makeClient("1.2.3.%i" % num).post('/like?uri=test&id=1') + assert rv.status_code == 200 + assert rv.data == str(num + 1) + + def testTooManyLikes(self): + + self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."})) + for num in range(256): + rv = self.makeClient("1.2.3.%i" % num).post('/like?uri=test&id=1') + assert rv.status_code == 200 + + if num >= 142: + assert rv.data == "142" + else: + assert rv.data == str(num + 1) +