implement db backend for votes/likes #5
This commit is contained in:
Martin Zimmermann
parent
11c6e4d720
commit
c7ee35423b
@ -55,6 +55,7 @@ url_map = Map([
|
||||
Rule('/', methods=['HEAD', 'GET'], endpoint=views.comment.get),
|
||||
Rule('/', methods=['PUT', 'DELETE'], endpoint=views.comment.modify),
|
||||
Rule('/new', methods=['POST'], endpoint=views.comment.create),
|
||||
Rule('/like', methods=['POST'], endpoint=views.comment.like),
|
||||
|
||||
Rule('/admin/', endpoint=views.admin.index)
|
||||
])
|
||||
|
||||
47
isso/db.py
47
isso/db.py
@ -7,6 +7,7 @@ import abc
|
||||
import time
|
||||
import sqlite3
|
||||
|
||||
from isso.utils import Bloomfilter
|
||||
from isso.models import Comment
|
||||
|
||||
|
||||
@ -65,8 +66,12 @@ class Abstract:
|
||||
"""
|
||||
return
|
||||
|
||||
|
||||
|
||||
@abc.abstractmethod
|
||||
def like(self, path, id, remote_addr):
|
||||
"""+1 a given comment. Returns the new like count (may not change because
|
||||
the creater can't vote on his/her own comment and multiple votes from the
|
||||
same ip address are ignored as well)."""
|
||||
return
|
||||
|
||||
class SQLite(Abstract):
|
||||
"""A basic :class:`Abstract` implementation using SQLite3. All comments
|
||||
@ -76,7 +81,7 @@ class SQLite(Abstract):
|
||||
|
||||
fields = [
|
||||
'path', 'id', 'created', 'modified',
|
||||
'text', 'author', 'hash', 'website', 'parent', 'mode'
|
||||
'text', 'author', 'hash', 'website', 'parent', 'mode', 'voters'
|
||||
]
|
||||
|
||||
def __init__(self, dbpath, moderation):
|
||||
@ -89,6 +94,7 @@ class SQLite(Abstract):
|
||||
'created FLOAT NOT NULL, modified FLOAT, text VARCHAR,'
|
||||
'author VARCHAR(64), hash VARCHAR(32), website VARCHAR(64),'
|
||||
'parent INTEGER, mode INTEGER,'
|
||||
'likes INTEGER DEFAULT 0, dislikes INTEGER DEFAULT 0, voters BLOB NOT NULL,'
|
||||
'PRIMARY KEY (id, path))')
|
||||
con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql)
|
||||
|
||||
@ -101,22 +107,29 @@ class SQLite(Abstract):
|
||||
WHERE rowid=NEW.rowid;
|
||||
END;""")
|
||||
|
||||
# threads (path -> title for now)
|
||||
sql = ('main.threads (path VARCHAR(255) NOT NULL, title TEXT'
|
||||
'PRIMARY KEY path)')
|
||||
con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql)
|
||||
|
||||
def query2comment(self, query):
|
||||
if query is None:
|
||||
return None
|
||||
|
||||
return Comment(
|
||||
text=query[4], author=query[5], hash=query[6], website=query[7], parent=query[8],
|
||||
path=query[0], id=query[1], created=query[2], modified=query[3], mode=query[9]
|
||||
path=query[0], id=query[1], created=query[2], modified=query[3], mode=query[9],
|
||||
votes=query[10]
|
||||
)
|
||||
|
||||
def add(self, path, c):
|
||||
def add(self, path, c, remote_addr):
|
||||
voters = buffer(Bloomfilter(iterable=[remote_addr]).array)
|
||||
with sqlite3.connect(self.dbpath) as con:
|
||||
keys = ','.join(self.fields)
|
||||
values = ','.join('?' * len(self.fields))
|
||||
con.execute('INSERT INTO comments (%s) VALUES (%s);' % (keys, values), (
|
||||
path, 0, c.created, c.modified, c.text, c.author, c.hash, c.website,
|
||||
c.parent, self.mode)
|
||||
c.parent, self.mode, voters)
|
||||
)
|
||||
|
||||
with sqlite3.connect(self.dbpath) as con:
|
||||
@ -159,6 +172,27 @@ class SQLite(Abstract):
|
||||
(None, path, id))
|
||||
return self.get(path, id)
|
||||
|
||||
def like(self, path, id, remote_addr):
|
||||
with sqlite3.connect(self.dbpath) as con:
|
||||
rv = con.execute("SELECT likes, dislikes, voters FROM comments" \
|
||||
+ " WHERE path=? AND id=?", (path, id)).fetchone()
|
||||
|
||||
likes, dislikes, voters = rv
|
||||
if likes + dislikes >= 142:
|
||||
return likes
|
||||
|
||||
bf = Bloomfilter(bytearray(voters), likes + dislikes)
|
||||
if remote_addr in bf:
|
||||
return likes
|
||||
|
||||
bf.add(remote_addr)
|
||||
with sqlite3.connect(self.dbpath) as con:
|
||||
con.execute("UPDATE comments SET likes = likes + 1 WHERE path=? AND id=?", (path, id))
|
||||
con.execute("UPDATE comments SET voters = ? WHERE path=? AND id=?", (
|
||||
buffer(bf.array), path, id))
|
||||
|
||||
return likes + 1
|
||||
|
||||
def retrieve(self, path, mode=5):
|
||||
with sqlite3.connect(self.dbpath) as con:
|
||||
rv = con.execute("SELECT * FROM comments WHERE path=? AND (? | mode) = ?" \
|
||||
@ -181,3 +215,4 @@ class SQLite(Abstract):
|
||||
|
||||
for item in rv:
|
||||
yield self.query2comment(item)
|
||||
|
||||
|
||||
@ -25,7 +25,7 @@ class Comment(object):
|
||||
normal and queued using MODE=3.
|
||||
"""
|
||||
|
||||
protected = ['path', 'id', 'mode', 'created', 'modified', 'hash']
|
||||
protected = ['path', 'id', 'mode', 'created', 'modified', 'hash', 'votes']
|
||||
fields = ['text', 'author', 'website', 'parent']
|
||||
|
||||
def __init__(self, **kw):
|
||||
|
||||
@ -3,12 +3,16 @@
|
||||
# Copyright 2012, Martin Zimmermann <info@posativ.org>. All rights reserved.
|
||||
# License: BSD Style, 2 clauses. see isso/__init__.py
|
||||
|
||||
from __future__ import division
|
||||
|
||||
import json
|
||||
|
||||
import socket
|
||||
import httplib
|
||||
|
||||
import math
|
||||
import random
|
||||
import hashlib
|
||||
import contextlib
|
||||
|
||||
from string import ascii_letters, digits
|
||||
@ -46,3 +50,51 @@ def normalize(host):
|
||||
|
||||
def mksecret(length):
|
||||
return ''.join(random.choice(ascii_letters + digits) for x in range(length))
|
||||
|
||||
|
||||
class Bloomfilter:
|
||||
"""A space-efficient probabilistic data structure. False-positive rate:
|
||||
|
||||
* 1e-05 for <80 elements
|
||||
* 1e-04 for <105 elements
|
||||
* 1e-03 for <142 elements
|
||||
|
||||
Uses a 256 byte array (2048 bits) and 11 hash functions. 256 byte because
|
||||
of space efficiency (array is saved for each comment) and 11 hash functions
|
||||
because of best overall false-positive rate in that range.
|
||||
|
||||
-- via Raymond Hettinger
|
||||
http://code.activestate.com/recipes/577684-bloom-filter/
|
||||
"""
|
||||
|
||||
def __init__(self, array=bytearray(256), elements=0, iterable=()):
|
||||
self.array = array
|
||||
self.elements = elements
|
||||
self.k = 11
|
||||
self.m = len(array) * 8
|
||||
|
||||
for item in iterable:
|
||||
self.add(item)
|
||||
|
||||
def get_probes(self, key):
|
||||
h = int(hashlib.sha256(key.encode()).hexdigest(), 16)
|
||||
for _ in range(self.k):
|
||||
yield h & self.m - 1
|
||||
h >>= self.k
|
||||
|
||||
def add(self, key):
|
||||
for i in self.get_probes(key):
|
||||
self.array[i//8] |= 2 ** (i%8)
|
||||
self.elements += 1
|
||||
|
||||
@property
|
||||
def density(self):
|
||||
c = ''.join(format(x, '08b') for x in self.array)
|
||||
return c.count('1') / len(c)
|
||||
|
||||
def __contains__(self, key):
|
||||
return all(self.array[i//8] & (2 ** (i%8)) for i in self.get_probes(key))
|
||||
|
||||
def __len__(self):
|
||||
return self.elements
|
||||
|
||||
|
||||
@ -55,7 +55,7 @@ def create(app, environ, request, uri):
|
||||
Response('', 400)
|
||||
|
||||
try:
|
||||
rv = app.db.add(uri, comment)
|
||||
rv = app.db.add(uri, comment, request.remote_addr)
|
||||
except ValueError:
|
||||
abort(400) # FIXME: custom exception class, error descr
|
||||
|
||||
@ -120,6 +120,14 @@ def modify(app, environ, request, uri, id):
|
||||
return resp
|
||||
|
||||
|
||||
@requires(str, 'uri')
|
||||
@requires(int, 'id')
|
||||
def like(app, environ, request, uri, id):
|
||||
|
||||
nv = app.db.like(uri, id, request.remote_addr)
|
||||
return Response(str(nv), 200)
|
||||
|
||||
|
||||
def approve(app, environ, request, path, id):
|
||||
|
||||
try:
|
||||
|
||||
@ -23,7 +23,7 @@ class TestSQLite(unittest.TestCase):
|
||||
|
||||
def test_get(self):
|
||||
|
||||
rv = self.db.add('/', comment(text='Spam'))
|
||||
rv = self.db.add('/', comment(text='Spam'), '')
|
||||
c = self.db.get('/', rv.id)
|
||||
|
||||
assert c.id == 1
|
||||
@ -31,9 +31,9 @@ class TestSQLite(unittest.TestCase):
|
||||
|
||||
def test_add(self):
|
||||
|
||||
self.db.add('/', comment(text='Foo'))
|
||||
self.db.add('/', comment(text='Bar'))
|
||||
self.db.add('/path/', comment(text='Baz'))
|
||||
self.db.add('/', comment(text='Foo'), '')
|
||||
self.db.add('/', comment(text='Bar'), '')
|
||||
self.db.add('/path/', comment(text='Baz'), '')
|
||||
|
||||
rv = list(self.db.retrieve('/'))
|
||||
assert rv[0].id == 1
|
||||
@ -50,14 +50,14 @@ class TestSQLite(unittest.TestCase):
|
||||
|
||||
def test_add_return(self):
|
||||
|
||||
self.db.add('/', comment(text='1'))
|
||||
self.db.add('/', comment(text='2'))
|
||||
self.db.add('/', comment(text='1'), '')
|
||||
self.db.add('/', comment(text='2'), '')
|
||||
|
||||
assert self.db.add('/path/', comment(text='1')).id == 1
|
||||
assert self.db.add('/path/', comment(text='1'), '').id == 1
|
||||
|
||||
def test_update(self):
|
||||
|
||||
rv = self.db.add('/', comment(text='Foo'))
|
||||
rv = self.db.add('/', comment(text='Foo'), '')
|
||||
time.sleep(0.1)
|
||||
rv = self.db.update('/', rv.id, comment(text='Bla'))
|
||||
c = self.db.get('/', rv.id)
|
||||
@ -69,15 +69,15 @@ class TestSQLite(unittest.TestCase):
|
||||
def test_delete(self):
|
||||
|
||||
rv = self.db.add('/', comment(
|
||||
text='F**CK', author='P*NIS', website='http://somebadhost.org/'))
|
||||
text='F**CK', author='P*NIS', website='http://somebadhost.org/'), '')
|
||||
assert self.db.delete('/', rv.id) == None
|
||||
|
||||
def test_recent(self):
|
||||
|
||||
self.db.add('/path/', comment(text='2'))
|
||||
self.db.add('/path/', comment(text='2'), '')
|
||||
|
||||
for x in range(5):
|
||||
self.db.add('/', comment(text='%i' % (x+1)))
|
||||
self.db.add('/', comment(text='%i' % (x+1)), '')
|
||||
|
||||
assert len(list(self.db.recent(mode=7))) == 6
|
||||
assert len(list(self.db.recent(mode=7, limit=5))) == 5
|
||||
@ -95,13 +95,13 @@ class TestSQLitePending(unittest.TestCase):
|
||||
|
||||
def test_retrieve(self):
|
||||
|
||||
self.db.add('/', comment(text='Foo'))
|
||||
self.db.add('/', comment(text='Foo'), '')
|
||||
assert len(list(self.db.retrieve('/'))) == 0
|
||||
|
||||
def test_activate(self):
|
||||
|
||||
self.db.add('/', comment(text='Foo'))
|
||||
self.db.add('/', comment(text='Bar'))
|
||||
self.db.add('/', comment(text='Foo'), '')
|
||||
self.db.add('/', comment(text='Bar'), '')
|
||||
self.db.activate('/', 2)
|
||||
|
||||
assert len(list(self.db.retrieve('/'))) == 1
|
||||
|
||||
80
specs/test_vote.py
Normal file
80
specs/test_vote.py
Normal file
@ -0,0 +1,80 @@
|
||||
|
||||
import os
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from werkzeug.test import Client
|
||||
from werkzeug.wrappers import Response
|
||||
|
||||
from isso import Isso
|
||||
|
||||
|
||||
class FakeIP(object):
|
||||
|
||||
def __init__(self, app, ip):
|
||||
self.app = app
|
||||
self.ip = ip
|
||||
|
||||
def __call__(self, environ, start_response):
|
||||
environ['REMOTE_ADDR'] = self.ip
|
||||
return self.app(environ, start_response)
|
||||
|
||||
|
||||
class TestVote(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
fd, self.path = tempfile.mkstemp()
|
||||
|
||||
def tearDown(self):
|
||||
os.unlink(self.path)
|
||||
|
||||
def makeClient(self, ip):
|
||||
|
||||
app = Isso(self.path, '...', '...', 15*60, "...")
|
||||
app.wsgi_app = FakeIP(app.wsgi_app, ip)
|
||||
|
||||
return Client(app, Response)
|
||||
|
||||
def testZeroLikes(self):
|
||||
|
||||
rv = self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."}))
|
||||
assert json.loads(rv.data)['votes'] == 0
|
||||
|
||||
def testSingleLike(self):
|
||||
|
||||
self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."}))
|
||||
rv = self.makeClient("0.0.0.0").post("/like?uri=test&id=1")
|
||||
|
||||
assert rv.status_code == 200
|
||||
assert rv.data == "1"
|
||||
|
||||
def testSelfLike(self):
|
||||
|
||||
bob = self.makeClient("127.0.0.1")
|
||||
bob.post("/new?uri=test", data=json.dumps({"text": "..."}))
|
||||
rv = bob.post('/like?uri=test&id=1')
|
||||
|
||||
assert rv.status_code == 200
|
||||
assert rv.data == "0"
|
||||
|
||||
def testMultipleLikes(self):
|
||||
|
||||
self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."}))
|
||||
for num in range(15):
|
||||
rv = self.makeClient("1.2.3.%i" % num).post('/like?uri=test&id=1')
|
||||
assert rv.status_code == 200
|
||||
assert rv.data == str(num + 1)
|
||||
|
||||
def testTooManyLikes(self):
|
||||
|
||||
self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."}))
|
||||
for num in range(256):
|
||||
rv = self.makeClient("1.2.3.%i" % num).post('/like?uri=test&id=1')
|
||||
assert rv.status_code == 200
|
||||
|
||||
if num >= 142:
|
||||
assert rv.data == "142"
|
||||
else:
|
||||
assert rv.data == str(num + 1)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user