rewrite db backend

pull/16/head
Martin Zimmermann 11 years ago
parent 3905e84af1
commit 560e73cc0a

@ -68,10 +68,6 @@ You must encode `path`, e.g. to retrieve comments for `/hello-world/`:
GET /?uri=%2Fhello-world%2F
You can also pass an `id` to fetch a specific comment:
GET /?uri=%2Fhello-world%2F&id=1
To disable automatic Markdown-to-HTML conversion, pass `plain=1` to the
query URL:

@ -28,7 +28,6 @@ dist = pkg_resources.get_distribution("isso")
import sys
import io
import os
import json
import socket
import httplib
import urlparse
@ -50,16 +49,17 @@ from werkzeug.contrib.fixers import ProxyFix
from jinja2 import Environment, FileSystemLoader
from isso import db, utils, migrate, views, wsgi, notify, colors
from isso import db, migrate, views, wsgi, notify, colors
from isso.views import comment, admin
url_map = Map([
Rule('/', methods=['HEAD', 'GET'], endpoint=views.comment.get),
Rule('/', methods=['PUT', 'DELETE'], endpoint=views.comment.modify),
Rule('/new', methods=['POST'], endpoint=views.comment.create),
Rule('/like', methods=['POST'], endpoint=views.comment.like),
Rule('/count', methods=['GET'], endpoint=views.comment.count),
Rule('/new', methods=['POST'], endpoint=views.comment.new),
Rule('/id/<int:id>', methods=['GET', 'PUT', 'DELETE'], endpoint=views.comment.single),
Rule('/id/<int:id>/like', methods=['POST'], endpoint=views.comment.like),
Rule('/', methods=['GET'], endpoint=views.comment.fetch),
Rule('/count', methods=['GET'], endpoint=views.comment.count),
Rule('/admin/', endpoint=views.admin.index)
])
@ -75,7 +75,7 @@ class Isso(object):
self.PASSPHRASE = passphrase
self.MAX_AGE = max_age
self.db = db.SQLite(dbpath, moderation=False)
self.db = db.SQLite3(dbpath)
self.signer = URLSafeTimedSerializer(secret)
self.j2env = Environment(loader=FileSystemLoader(join(dirname(__file__), 'templates/')))
self.notify = lambda *args, **kwargs: mailer.sendmail(*args, **kwargs)
@ -95,10 +95,6 @@ class Isso(object):
tt = self.j2env.get_template(tt)
return tt.render(**ctx)
@classmethod
def dumps(cls, obj, **kw):
return json.dumps(obj, cls=utils.IssoEncoder, **kw)
def dispatch(self, request, start_response):
adapter = url_map.bind_to_environ(request.environ)
try:
@ -159,7 +155,7 @@ def main():
conf.read(args.conf)
if args.command == "import":
migrate.disqus(db.SQLite(conf.get("general", "dbpath"), False), args.dump)
migrate.disqus(db.SQLite3(conf.get("general", "dbpath"), False), args.dump)
sys.exit(0)
if not conf.get("general", "host").startswith(("http://", "https://")):

@ -1,277 +0,0 @@
# -*- encoding: utf-8 -*-
#
# Copyright 2012, Martin Zimmermann <info@posativ.org>. All rights reserved.
# License: BSD Style, 2 clauses. see isso/__init__.py
import abc
import time
import sqlite3
from isso.utils import Bloomfilter
from isso.models import Comment
class Abstract:
__metaclass__ = abc.ABCMeta
@abc.abstractmethod
def __init__(self, dbpath, moderation):
return
@abc.abstractmethod
def add(self, path, comment):
"""Add a new comment to the database. Returns a Comment object."""
return
@abc.abstractmethod
def activate(self, path, id):
"""Activate comment id if pending and return comment for (path, id)."""
return
@abc.abstractmethod
def update(self, path, id, comment):
"""
Update an existing comment, but only writeable fields such as text,
author, email, website and parent. This method should set the modified
field to the current time.
"""
return
@abc.abstractmethod
def delete(self, path, id):
"""
Delete a comment. There are two distinctions: a comment is referenced
by another valid comment's parent attribute or stand-a-lone. In this
case the comment can't be removed without losing depending comments.
Hence, delete removes all visible data such as text, author, email,
website sets the mode field to 4.
In the second case this comment can be safely removed without any side
effects."""
return
@abc.abstractmethod
def retrieve(self, path, mode):
"""
Return all comments for `path` with `mode`.
"""
return
@abc.abstractmethod
def recent(self, mode=7, limit=None):
"""
Return most recent comments with `mode`. If `limit` is None, return
*all* comments that are currently stored, otherwise limit by `limit`.
"""
return
@abc.abstractmethod
def like(self, path, id, remote_addr):
"""+1 a given comment. Returns the new like count (may not change because
the creater can't vote on his/her own comment and multiple votes from the
same ip address are ignored as well)."""
return
@abc.abstractmethod
def count(self, path=None, mode=1):
"""return count of comments for path (optional) and mode (defaults to
visible, not deleted or moderated comments)."""
return
class Threads(object):
def __init__(self, dbpath):
self.dbpath = dbpath
with sqlite3.connect(self.dbpath) as con:
sql = ('main.threads (path VARCHAR(255) NOT NULL, title TEXT'
'PRIMARY KEY path)')
con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql)
def get(self, path):
with sqlite3.connect(self.dbpath) as con:
rv = con.execute("SELECT title FROM threads WHERE path=?", (path,)).fetchone()
if rv is not None:
return rv[0]
return None
def add(self, path, title):
with sqlite3.connect(self.dbpath) as con:
con.execute("INSERT INTO threads (path, title) VALUES (?, ?)", (path, title))
return title
class SQLite(Abstract):
"""A basic :class:`Abstract` implementation using SQLite3. All comments
share a single database. The tuple (id, path) acts as unique identifier
for a comment. Multiple comments per path (= that is the URI to your blog
post) are ordered by that id."""
fields = [
'path', 'id', 'created', 'modified',
'text', 'author', 'hash', 'website', 'parent', 'mode', 'voters'
]
def __init__(self, dbpath, moderation):
self.dbpath = dbpath
self.mode = 2 if moderation else 1
with sqlite3.connect(self.dbpath) as con:
sql = ('main.comments (path VARCHAR(255) NOT NULL, id INTEGER NOT NULL,'
'created FLOAT NOT NULL, modified FLOAT, text VARCHAR,'
'author VARCHAR(64), hash VARCHAR(32), website VARCHAR(64),'
'parent INTEGER, mode INTEGER,'
'likes INTEGER DEFAULT 0, dislikes INTEGER DEFAULT 0, voters BLOB NOT NULL,'
'PRIMARY KEY (id, path))')
con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql)
# increment id if (id, path) is no longer unique
con.execute("""\
CREATE TRIGGER IF NOT EXISTS increment AFTER INSERT ON comments
BEGIN
UPDATE comments SET
id=(SELECT MAX(id)+1 FROM comments WHERE path=NEW.path)
WHERE rowid=NEW.rowid;
END;""")
self.threads = Threads(self.dbpath)
def query2comment(self, query):
if query is None:
return None
return Comment(
text=query[4], author=query[5], hash=query[6], website=query[7], parent=query[8],
path=query[0], id=query[1], created=query[2], modified=query[3], mode=query[9],
votes=query[10]
)
def add(self, uri, c, remote_addr):
voters = buffer(Bloomfilter(iterable=[remote_addr]).array)
with sqlite3.connect(self.dbpath) as con:
keys = ','.join(self.fields)
values = ','.join('?' * len(self.fields))
con.execute('INSERT INTO comments (%s) VALUES (%s);' % (keys, values), (
uri, 0, c['created'] or time.time(), None, c["text"], c["author"],
c["hash"], c["website"], c["parent"], self.mode, voters)
)
with sqlite3.connect(self.dbpath) as con:
return self.query2comment(
con.execute('SELECT *, MAX(id) FROM comments WHERE path=?;', (uri, )).fetchone())
def activate(self, path, id):
with sqlite3.connect(self.dbpath) as con:
con.execute("UPDATE comments SET mode=1 WHERE path=? AND id=? AND mode=2", (path, id))
return self.get(path, id)
def update(self, path, id, values):
with sqlite3.connect(self.dbpath) as con:
for key, value in values.iteritems():
con.execute('UPDATE comments SET %s=? WHERE path=? AND id=?;' % key,
(value, path, id))
with sqlite3.connect(self.dbpath) as con:
con.execute('UPDATE comments SET modified=? WHERE path=? AND id=?',
(time.time(), path, id))
return self.get(path, id)
def get(self, path, id):
with sqlite3.connect(self.dbpath) as con:
return self.query2comment(con.execute(
'SELECT * FROM comments WHERE path=? AND id=?;', (path, id)).fetchone())
def _remove_stale(self, con, path):
sql = ('DELETE FROM',
' comments',
'WHERE',
' path=? AND mode=4 AND id NOT IN (',
' SELECT',
' parent',
' FROM',
' comments',
' WHERE path=? AND parent IS NOT NULL)')
while con.execute(' '.join(sql), (path, path)).rowcount:
continue
def delete(self, path, id):
with sqlite3.connect(self.dbpath) as con:
sql = 'SELECT * FROM comments WHERE path=? AND parent=?'
refs = con.execute(sql, (path, id)).fetchone()
if refs is None:
con.execute('DELETE FROM comments WHERE path=? AND id=?', (path, id))
self._remove_stale(con, path)
return None
con.execute('UPDATE comments SET text=? WHERE path=? AND id=?', ('', path, id))
con.execute('UPDATE comments SET mode=? WHERE path=? AND id=?', (4, path, id))
for field in ('author', 'website'):
con.execute('UPDATE comments SET %s=? WHERE path=? AND id=?' % field,
(None, path, id))
self._remove_stale(con, path)
return self.get(path, id)
def like(self, path, id, remote_addr):
with sqlite3.connect(self.dbpath) as con:
rv = con.execute("SELECT likes, dislikes, voters FROM comments" \
+ " WHERE path=? AND id=?", (path, id)).fetchone()
likes, dislikes, voters = rv
if likes + dislikes >= 142:
return likes
bf = Bloomfilter(bytearray(voters), likes + dislikes)
if remote_addr in bf:
return likes
bf.add(remote_addr)
with sqlite3.connect(self.dbpath) as con:
con.execute("UPDATE comments SET likes = likes + 1 WHERE path=? AND id=?", (path, id))
con.execute("UPDATE comments SET voters = ? WHERE path=? AND id=?", (
buffer(bf.array), path, id))
return likes + 1
def count(self, path=None, mode=1):
if path is not None:
with sqlite3.connect(self.dbpath) as con:
return con.execute("SELECT COUNT(*) FROM comments WHERE path=?" \
+ " AND (? | mode) = ?", (path, mode, mode)).fetchone()
with sqlite3.connect(self.dbpath) as con:
return con.execute("SELECT COUNT(*) FROM comments WHERE" \
+ "(? | mode) = ?", (path, mode, mode)).fetchone()
def retrieve(self, path, mode=5):
with sqlite3.connect(self.dbpath) as con:
rv = con.execute("SELECT * FROM comments WHERE path=? AND (? | mode) = ?" \
+ " ORDER BY id ASC;", (path, mode, mode)).fetchall()
for item in rv:
yield self.query2comment(item)
def recent(self, mode=7, limit=None):
sql = 'SELECT * FROM comments WHERE (? | mode) = ? ORDER BY created DESC'
args = [mode, mode]
if limit:
sql += ' LIMIT ?'
args.append(limit)
with sqlite3.connect(self.dbpath) as con:
rv = con.execute(sql + ';', args).fetchall()
for item in rv:
yield self.query2comment(item)

@ -0,0 +1,27 @@
# -*- encoding: utf-8 -*-
import sqlite3
from isso.db.comments import Comments
from isso.db.threads import Threads
class SQLite3:
connection = None
def __init__(self, path, moderation=False):
self.path = path
self.mode = 2 if moderation else 1
self.threads = Threads(self)
self.comments = Comments(self)
def execute(self, sql, args=()):
if isinstance(sql, (list, tuple)):
sql = ' '.join(sql)
with sqlite3.connect(self.path) as con:
return con.execute(sql, args)

@ -0,0 +1,179 @@
# -*- encoding: utf-8 -*-
import time
import sqlite3
from isso.utils import Bloomfilter
class Comments:
"""Hopefully DB-independend SQL to store, modify and retrieve all
comment-related actions. Here's a short scheme overview:
| tid (thread id) | cid (comment id) | parent | ... | likes | remote_addr |
+-----------------+------------------+--------+-----+-------+-------------+
| 1 | 1 | null | ... | BLOB | 127.0.0.0 |
| 1 | 2 | 1 | ... | BLOB | 127.0.0.0 |
+-----------------+------------------+--------+-----+-------+-------------+
The tuple (tid, cid) is unique and thus primary key.
"""
fields = ['tid', 'id', 'parent', 'created', 'modified', 'mode', 'remote_addr',
'text', 'author', 'email', 'website', 'likes', 'dislikes', 'voters']
def __init__(self, db):
self.db = db
self.db.execute([
'CREATE TABLE IF NOT EXISTS comments (',
' tid REFERENCES threads(id), id INTEGER PRIMARY KEY, parent INTEGER,',
' created FLOAT NOT NULL, modified FLOAT, mode INTEGER, remote_addr VARCHAR,',
' text VARCHAR, author VARCHAR, email VARCHAR, website VARCHAR,',
' likes INTEGER DEFAULT 0, dislikes INTEGER DEFAULT 0, voters BLOB NOT NULL);'])
def add(self, uri, c):
"""
Add a new comment to the database and return public fields as dict.
Initializes voter bloom array with provided :param:`remote_addr` and
adds a new thread to the `main.threads` table.
"""
self.db.execute([
'INSERT INTO comments (',
' tid, parent,'
' created, modified, mode, remote_addr,',
' text, author, email, website, voters )',
'SELECT',
' threads.id, ?,',
' ?, ?, ?, ?,',
' ?, ?, ?, ?, ?',
'FROM threads WHERE threads.uri = ?;'], (
c.get('parent'),
c.get('created') or time.time(), None, self.db.mode, c['remote_addr'],
c['text'], c.get('author'), c.get('email'), c.get('website'), buffer(
Bloomfilter(iterable=[c['remote_addr']]).array),
uri)
)
return dict(zip(Comments.fields, self.db.execute(
'SELECT *, MAX(c.id) FROM comments AS c INNER JOIN threads ON threads.uri = ?',
(uri, )).fetchone()))
# def activate(self, path, id):
# """Activate comment id if pending and return comment for (path, id)."""
# with sqlite3.connect(self.dbpath) as con:
# con.execute("UPDATE comments SET mode=1 WHERE path=? AND id=? AND mode=2", (path, id))
# return self.get(path, id)
def update(self, id, data):
"""
Update an existing comment, but only writeable fields such as text,
author, email, website and parent. This method should set the modified
field to the current time.
"""
self.db.execute([
'UPDATE comments SET',
','.join(key + '=' + '?' for key in data),
'WHERE id=?;'],
data.values() + [id])
return self.get(id)
def get(self, id):
rv = self.db.execute('SELECT * FROM comments WHERE id=?', (id, )).fetchone()
if rv:
return dict(zip(Comments.fields, rv))
return None
def fetch(self, uri, mode=5):
"""
Return all comments for `path` with `mode`.
"""
rv = self.db.execute([
'SELECT comments.* FROM comments INNER JOIN threads ON',
' threads.uri=? AND comments.tid=threads.id AND (? | comments.mode) = ?'
'ORDER BY id ASC;'], (uri, mode, mode)).fetchall()
for item in rv:
yield dict(zip(Comments.fields, item))
def _remove_stale(self):
sql = ('DELETE FROM',
' comments',
'WHERE',
' mode=4 AND id NOT IN (',
' SELECT',
' parent',
' FROM',
' comments',
' WHERE parent IS NOT NULL)')
while self.db.execute(sql).rowcount:
continue
def delete(self, id):
"""
Delete a comment. There are two distinctions: a comment is referenced
by another valid comment's parent attribute or stand-a-lone. In this
case the comment can't be removed without losing depending comments.
Hence, delete removes all visible data such as text, author, email,
website sets the mode field to 4.
In the second case this comment can be safely removed without any side
effects."""
refs = self.db.execute('SELECT * FROM comments WHERE parent=?', (id, )).fetchone()
if refs is None:
self.db.execute('DELETE FROM comments WHERE id=?', (id, ))
self._remove_stale()
return None
self.db.execute('UPDATE comments SET text=? WHERE id=?', ('', id))
self.db.execute('UPDATE comments SET mode=? WHERE id=?', (4, id))
for field in ('author', 'website'):
self.db.execute('UPDATE comments SET %s=? WHERE id=?' % field, (None, id))
self._remove_stale()
return self.get(id)
def like(self, id, remote_addr):
"""+1 a given comment. Returns the new like count (may not change because
the creater can't vote on his/her own comment and multiple votes from the
same ip address are ignored as well)."""
rv = self.db.execute(
'SELECT likes, dislikes, voters FROM comments WHERE id=?', (id, )) \
.fetchone()
if rv is None:
return 0
likes, dislikes, voters = rv
if likes + dislikes >= 142:
return likes
bf = Bloomfilter(bytearray(voters), likes + dislikes)
if remote_addr in bf:
return likes
bf.add(remote_addr)
self.db.execute([
'UPDATE comments SET',
' likes = likes + 1, voters = ?',
'WHERE id=?;'], (buffer(bf.array), id))
return likes + 1
def count(self, uri):
"""
return count of comments for uri.
"""
return self.db.execute([
'SELECT COUNT(comments.id) FROM comments INNER JOIN threads ON',
' threads.uri=? AND comments.tid=threads.id AND comments.mode=1;'],
(uri, )).fetchone()

@ -0,0 +1,29 @@
# -*- encoding: utf-8 -*-
class Thread(object):
def __init__(self, id, uri, title):
self.id = id
self.uri = uri
self.title = title
class Threads(object):
def __init__(self, db):
self.db = db
self.db.execute([
'CREATE TABLE IF NOT EXISTS threads (',
' id INTEGER PRIMARY KEY, uri VARCHAR(256) UNIQUE, title VARCHAR(256))'])
def __contains__(self, uri):
return self.db.execute("SELECT title FROM threads WHERE uri=?", (uri, )) \
.fetchone() is not None
def __getitem__(self, uri):
return Thread(*self.db.execute("SELECT * FROM threads WHERE uri=?", (uri, )).fetchone())
def new(self, uri, title):
self.db.execute("INSERT INTO threads (uri, title) VALUES (?, ?)", (uri, title))

@ -90,7 +90,7 @@ define(["lib/q"], function(Q) {
}
var remove = function(id) {
return curl("DELETE", endpoint + "/?" + qs({uri: location, id: id}), null)
return curl("DELETE", endpoint + "/id/" + id, null)
.then(function(rv) {
if (rv.status == 200) {
return JSON.parse(rv.body) == null;

@ -67,7 +67,7 @@ define(["lib/q", "lib/HTML", "helper/utils", "helper/identicons", "./api", "./fo
node.scrollIntoView(false);
}
if (utils.read(window.location.pathname + "-" + comment.id)) {
if (utils.read(comment.id)) {
node.footer.add("a.delete{Löschen}").href = "#";
node.footer.add("a.edit{Bearbeiten}").href = "#";

@ -12,7 +12,6 @@ from __future__ import division
import sys
import os
import hashlib
from time import mktime, strptime
from urlparse import urlparse
@ -20,8 +19,6 @@ from collections import defaultdict
from xml.etree import ElementTree
from isso.models import Comment
ns = '{http://disqus.com}'
dsq = '{http://disqus.com/disqus-internals}'
@ -32,20 +29,15 @@ def insert(db, thread, comments):
path = urlparse(thread.find('%sid' % ns).text).path
remap = dict()
for item in sorted(comments, key=lambda k: k['created']):
parent = remap.get(item.get('dsq:parent'))
comment = Comment(created=item['created'], text=item['text'],
author=item['author'], parent=parent,
hash=hashlib.md5(item["email"] or '').hexdigest())
if path not in db.threads:
db.threads.new(path, thread.find('%stitle' % ns).text.strip())
rv = db.add(path, comment, '127.0.0.1')
remap[item['dsq:id']] = rv["id"]
for item in sorted(comments, key=lambda k: k['created']):
try:
db.threads.get(path)
except KeyError:
db.threads.add(path, thread.find('%stitle' % ns).text.strip())
dsq_id = item.pop('dsq:id')
item['parent'] = remap.get(item.pop('dsq:parent', None))
rv = db.comments.add(path, item)
remap[dsq_id] = rv["id"]
def disqus(db, xmlfile):
@ -61,7 +53,8 @@ def disqus(db, xmlfile):
'author': post.find('%sauthor/%sname' % (ns, ns)).text,
'email': post.find('%sauthor/%semail' % (ns, ns)).text,
'created': mktime(strptime(
post.find('%screatedAt' % ns).text, '%Y-%m-%dT%H:%M:%SZ'))
post.find('%screatedAt' % ns).text, '%Y-%m-%dT%H:%M:%SZ')),
'remote_addr': '127.0.0.0'
}
if post.find(ns + 'parent') is not None:

@ -1,63 +0,0 @@
# -*- encoding: utf-8 -*-
#
# Copyright 2012, Martin Zimmermann <info@posativ.org>. All rights reserved.
# License: BSD Style, 2 clauses. see isso/__init__.py
from __future__ import unicode_literals
import hashlib
class Comment(object):
"""This class represents a regular comment. It needs at least a text
field, all other fields are optional (or automatically set by the
database driver.
The field `mode` has a special meaning:
1: normal
2: in moderation queue
4: deleted
You can query for them like with UNIX permission bits, so you get both
normal and queued using MODE=3.
"""
fields = ["text", "author", "website", "votes", "hash", "parent", "mode", "id",
"created", "modified"]
def __init__(self, **kwargs):
self.values = {}
for key in Comment.fields:
self.values[key] = kwargs.get(key, None)
def __getitem__(self, key):
return self.values[key]
def __setitem__(self, key, value):
self.values[key] = value
def iteritems(self):
for key in Comment.fields:
yield key, self.values[key]
@property
def pending(self):
return self.values["mode"] == 2
@property
def deleted(self):
return self.values["mode"] == 4
@property
def md5(self):
hv = hashlib.md5()
for key, value in self.iteritems():
if key == "parent" or value is None:
continue
hv.update(unicode(self.values.get(key, "")).encode("utf-8", "replace")) # XXX
return hv.hexdigest()

@ -4,8 +4,7 @@
<title>Hello World</title>
<meta charset="utf-8" />
<link rel="stylesheet" href="/static/isso.css" />
<!--<script src="/js/embed.js"></script>-->
<script data-main="/js/main" src="/js/require.js"></script>
<script data-main="/js/embed" src="/js/require.js"></script>
<style type="text/css">
body {
margin: 0 auto;

@ -5,8 +5,6 @@
from __future__ import division
import json
import socket
import httplib
@ -20,17 +18,6 @@ from contextlib import closing
import html5lib
import ipaddress
from isso.models import Comment
class IssoEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Comment):
return dict((field, value) for field, value in obj.iteritems())
return json.JSONEncoder.default(self, obj)
def normalize(host):
@ -111,15 +98,14 @@ def heading(host, path):
def anonymize(remote_addr):
ip = ipaddress.ip_address(remote_addr)
try:
ipv4 = ipaddress.IPv4Address(remote_addr)
return ''.join(ip.exploded.rsplit('.', 1)[0]) + '.' + '0'
return ''.join(ipv4.exploded.rsplit('.', 1)[0]) + '.' + '0'
except ipaddress.AddressValueError:
ipv6 = ipaddress.IPv6Address(remote_addr)
if ipv6.ipv4_mapped is not None:
return anonymize(ipv6.ipv4_mapped)
return ip.exploded.rsplit(':', 5)[0]
return ipv6.exploded.rsplit(':', 5)[0] + ':' + ':'.join(['0000']*3)
def salt(value, s=u'\x082@t9*\x17\xad\xc1\x1c\xa5\x98'):

@ -5,6 +5,7 @@
import cgi
import json
import time
import thread
import hashlib
import sqlite3
@ -15,7 +16,10 @@ from itsdangerous import SignatureExpired, BadSignature
from werkzeug.wrappers import Response
from werkzeug.exceptions import abort, BadRequest
from isso import models, utils, notify
from isso import utils, notify
FIELDS = set(['id', 'parent', 'text', 'author', 'website', 'email', 'mode',
'created', 'modified', 'likes', 'dislikes', 'hash'])
class requires:
@ -41,9 +45,9 @@ class requires:
@requires(str, 'uri')
def create(app, environ, request, uri):
def new(app, environ, request, uri):
if app.db.threads.get(uri) is None and not utils.urlexists(app.ORIGIN, uri):
if uri not in app.db.threads and not utils.urlexists(app.ORIGIN, uri):
return Response('URI does not exist', 404)
try:
@ -51,80 +55,81 @@ def create(app, environ, request, uri):
except ValueError:
return Response("No JSON object could be decoded", 400)
for field in set(data.keys()) - set(['text', 'author', 'website', 'email', 'parent']):
data.pop(field)
if not data.get("text"):
return Response("No text given.", 400)
if "id" in data and not isinstance(data["id"], int):
return Response("Parent ID must be an integer.")
if data.get("email"):
hash = data["email"]
else:
hash = utils.salt(utils.anonymize(unicode(request.remote_addr)))
comment = models.Comment(
text=data["text"], parent=data.get("parent"),
for field in ("author", "email"):
if data.get(field):
data[field] = cgi.escape(data[field])
author=data.get("author") and cgi.escape(data.get("author")),
website=data.get("website") and cgi.escape(data.get("website")),
data['remote_addr'] = utils.anonymize(unicode(request.remote_addr))
hash=hashlib.md5(hash).hexdigest())
title = app.db.threads.get(uri)
if title is None:
title = app.db.threads.add(uri, utils.heading(app.ORIGIN, uri))
if uri not in app.db.threads:
app.db.threads.new(uri, utils.heading(app.ORIGIN, uri))
title = app.db.threads[uri].title
try:
rv = app.db.add(uri, comment, request.remote_addr)
rv = app.db.comments.add(uri, data)
except sqlite3.Error:
logging.exception('uncaught SQLite3 exception')
abort(400)
else:
rv["text"] = app.markdown(rv["text"])
href = (app.ORIGIN.rstrip("/") + uri + "#isso-%i" % rv["id"])
thread.start_new_thread(app.notify, notify.create(comment, title, href, request.remote_addr))
thread.start_new_thread(
app.notify,
notify.create(rv, title, href, utils.anonymize(unicode(request.remote_addr))))
resp = Response(app.dumps(rv), 202 if rv.pending else 201,
# save checksum of text into cookie, so mallory can't modify/delete a comment, if
# he add a comment, then removed it but not the signed cookie.
checksum = hashlib.md5(rv["text"]).hexdigest()
rv["text"] = app.markdown(rv["text"])
rv["hash"] = hashlib.md5(rv.get('email') or utils.salt(rv['remote_addr'])).hexdigest()
for key in set(rv.keys()) - FIELDS:
rv.pop(key)
resp = Response(json.dumps(rv), 202 if rv["mode"] == 2 else 201,
content_type='application/json')
resp.set_cookie('%s-%s' % (uri, rv["id"]), app.sign([uri, rv["id"], rv.md5]),
resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]),
max_age=app.MAX_AGE, path='/')
return resp
@requires(str, 'uri')
def get(app, environ, request, uri):
def single(app, environ, request, id):
id = request.args.get('id', None)
if request.method == 'GET':
rv = app.db.comments.get(id)
if rv is None:
abort(404)
rv = list(app.db.retrieve(uri)) if id is None else app.db.get(uri, id)
if not rv:
abort(404)
for key in set(rv.keys()) - FIELDS:
rv.pop(key)
if request.args.get('plain', '0') == '0':
if isinstance(rv, list):
for item in rv:
item['text'] = app.markdown(item['text'])
else:
if request.args.get('plain', '0') == '0':
rv['text'] = app.markdown(rv['text'])
return Response(app.dumps(rv), 200, content_type='application/json')
@requires(str, 'uri')
@requires(int, 'id')
def modify(app, environ, request, uri, id):
return Response(json.dumps(rv), 200, content_type='application/json')
try:
rv = app.unsign(request.cookies.get('%s-%s' % (uri, id), ''))
rv = app.unsign(request.cookies.get(str(id), ''))
except (SignatureExpired, BadSignature):
try:
rv = app.unsign(request.cookies.get('admin', ''))
except (SignatureExpired, BadSignature):
abort(403)
if rv[0] != id:
abort(403)
# verify checksum, mallory might skip cookie deletion when he deletes a comment
if not (rv == '*' or rv[0:2] == [uri, id] or app.db.get(uri, id).md5 != rv[2]):
if rv[1] != hashlib.md5(app.db.comments.get(id)["text"]).hexdigest():
abort(403)
if request.method == 'PUT':
@ -133,55 +138,69 @@ def modify(app, environ, request, uri, id):
except ValueError:
return Response("No JSON object could be decoded", 400)
if not data.get("text"):
if data.get("text") is not None and len(data['text']) < 3:
return Response("No text given.", 400)
for key in data.keys():
if key not in ("text", "author", "website"):
data.pop(key)
for key in set(data.keys()) - set(["text", "author", "website"]):
data.pop(key)
data['modified'] = time.time()
try:
rv = app.db.update(uri, id, data)
rv = app.db.comments.update(id, data)
except sqlite3.Error:
logging.exception('uncaught SQLite3 exception')
abort(400)
for key in set(rv.keys()) - FIELDS:
rv.pop(key)
rv["text"] = app.markdown(rv["text"])
return Response(app.dumps(rv), 200, content_type='application/json')
return Response(json.dumps(rv), 200, content_type='application/json')
if request.method == 'DELETE':
rv = app.db.delete(uri, id)
resp = Response(app.dumps(rv), 200, content_type='application/json')
resp.delete_cookie(uri + '-' + str(id), path='/')
rv = app.db.comments.delete(id)
if rv:
for key in set(rv.keys()) - FIELDS:
rv.pop(key)
resp = Response(json.dumps(rv), 200, content_type='application/json')
resp.delete_cookie(str(id), path='/')
return resp
@requires(str, 'uri')
@requires(int, 'id')
def like(app, environ, request, uri, id):
def fetch(app, environ, request, uri):
nv = app.db.like(uri, id, request.remote_addr)
return Response(str(nv), 200)
rv = list(app.db.comments.fetch(uri))
if not rv:
abort(404)
for item in rv:
def approve(app, environ, request, path, id):
item['hash'] = hashlib.md5(item['email'] or utils.salt(item['remote_addr'])).hexdigest()
try:
if app.unsign(request.cookies.get('admin', '')) != '*':
abort(403)
except (SignatureExpired, BadSignature):
abort(403)
for key in set(item.keys()) - FIELDS:
item.pop(key)
app.db.activate(path, id)
return Response(app.dumps(app.db.get(path, id)), 200,
content_type='application/json')
if request.args.get('plain', '0') == '0':
for item in rv:
item['text'] = app.markdown(item['text'])
return Response(json.dumps(rv), 200, content_type='application/json')
def like(app, environ, request, id):
nv = app.db.comments.like(id, utils.anonymize(unicode(request.remote_addr)))
return Response(str(nv), 200)
@requires(str, 'uri')
def count(app, environ, request, uri):
rv = app.db.count(uri, mode=1)[0]
rv = app.db.comments.count(uri)[0]
if rv == 0:
abort(404)

@ -10,8 +10,8 @@ import unittest
from werkzeug.test import Client
from werkzeug.wrappers import Response
from isso import Isso, notify, utils
from isso.models import Comment
from isso import Isso, notify, utils, views
from isso.views import comment
utils.heading = lambda *args: "Untitled."
utils.urlexists = lambda *args: True
@ -47,7 +47,7 @@ class TestComments(unittest.TestCase):
def testGet(self):
self.post('/new?uri=%2Fpath%2F', data=json.dumps({'text': 'Lorem ipsum ...'}))
r = self.get('/?uri=%2Fpath%2F&id=1')
r = self.get('/id/1')
assert r.status_code == 200
rv = json.loads(r.data)
@ -67,6 +67,16 @@ class TestComments(unittest.TestCase):
assert rv["mode"] == 1
assert rv["text"] == '<p>Lorem ipsum ...</p>\n'
def testCreateMultiple(self):
a = self.post('/new?uri=test', data=json.dumps({'text': '...'}))
b = self.post('/new?uri=test', data=json.dumps({'text': '...'}))
c = self.post('/new?uri=test', data=json.dumps({'text': '...'}))
assert json.loads(a.data)["id"] == 1
assert json.loads(b.data)["id"] == 2
assert json.loads(c.data)["id"] == 3
def testCreateAndGetMultiple(self):
for i in range(20):
@ -87,10 +97,10 @@ class TestComments(unittest.TestCase):
def testUpdate(self):
self.post('/new?uri=%2Fpath%2F', data=json.dumps({'text': 'Lorem ipsum ...'}))
self.put('/?uri=%2Fpath%2F&id=1', data=json.dumps({
self.put('/id/1', data=json.dumps({
'text': 'Hello World', 'author': 'me', 'website': 'http://example.com/'}))
r = self.get('/?uri=%2Fpath%2F&id=1&plain=1')
r = self.get('/id/1?plain=1')
assert r.status_code == 200
rv = json.loads(r.data)
@ -102,10 +112,10 @@ class TestComments(unittest.TestCase):
def testDelete(self):
self.post('/new?uri=%2Fpath%2F', data=json.dumps({'text': 'Lorem ipsum ...'}))
r = self.delete('/?uri=%2Fpath%2F&id=1')
r = self.delete('/id/1')
assert r.status_code == 200
assert json.loads(r.data) == None
assert self.get('/?uri=%2Fpath%2F&id=1').status_code == 404
assert self.get('/id/1').status_code == 404
def testDeleteWithReference(self):
@ -113,14 +123,15 @@ class TestComments(unittest.TestCase):
client.post('/new?uri=%2Fpath%2F', data=json.dumps({'text': 'First'}))
client.post('/new?uri=%2Fpath%2F', data=json.dumps({'text': 'First', 'parent': 1}))
r = client.delete('/?uri=%2Fpath%2F&id=1')
r = client.delete('/id/1')
assert r.status_code == 200
assert Comment(**json.loads(r.data)).deleted
print r.data
assert json.loads(r.data)['mode'] == 4
assert self.get('/?uri=%2Fpath%2F&id=1').status_code == 200
assert self.get('/?uri=%2Fpath%2F&id=2').status_code == 200
r = client.delete('/?uri=%2Fpath%2F&id=2')
r = client.delete('/id/2')
assert self.get('/?uri=%2Fpath%2F').status_code == 404
def testDeleteWithMultipleReferences(self):
@ -142,15 +153,15 @@ class TestComments(unittest.TestCase):
client.post('/new?uri=%2Fpath%2F', data=json.dumps({'text': 'Third 2', 'parent': 2}))
client.post('/new?uri=%2Fpath%2F', data=json.dumps({'text': '...'}))
client.delete('/?uri=%2Fpath%2F&id=1')
client.delete('/id/1')
assert self.get('/?uri=%2Fpath%2F').status_code == 200
client.delete('/?uri=%2Fpath%2F&id=2')
client.delete('/id/2')
assert self.get('/?uri=%2Fpath%2F').status_code == 200
client.delete('/?uri=%2Fpath%2F&id=3')
client.delete('/id/3')
assert self.get('/?uri=%2Fpath%2F').status_code == 200
client.delete('/?uri=%2Fpath%2F&id=4')
client.delete('/id/4')
assert self.get('/?uri=%2Fpath%2F').status_code == 200
client.delete('/?uri=%2Fpath%2F&id=5')
client.delete('/id/5')
assert self.get('/?uri=%2Fpath%2F').status_code == 404
def testPathVariations(self):
@ -161,21 +172,21 @@ class TestComments(unittest.TestCase):
assert self.post('/new?' + urllib.urlencode({'uri': path}),
data=json.dumps({'text': '...'})).status_code == 201
for path in paths:
for i, path in enumerate(paths):
assert self.get('/?' + urllib.urlencode({'uri': path})).status_code == 200
assert self.get('/?' + urllib.urlencode({'uri': path, id: 1})).status_code == 200
assert self.get('/id/%i' % (i + 1)).status_code == 200
def testDeleteAndCreateByDifferentUsersButSamePostId(self):
mallory = Client(self.app, Response)
mallory.post('/new?uri=%2Fpath%2F', data=json.dumps({'text': 'Foo'}))
mallory.delete('/?uri=%2Fpath%2F&id=1')
mallory.delete('/id/1')
bob = Client(self.app, Response)
bob.post('/new?uri=%2Fpath%2F', data=json.dumps({'text': 'Bar'}))
assert mallory.delete('/?uri=%2Fpath%2F&id=1').status_code == 403
assert bob.delete('/?uri=%2Fpath%2F&id=1').status_code == 200
assert mallory.delete('/id/1').status_code == 403
assert bob.delete('/id/1').status_code == 200
def testHash(self):
@ -199,7 +210,7 @@ class TestComments(unittest.TestCase):
rv = json.loads(rv.data)
for key in Comment.fields:
for key in comment.FIELDS:
rv.pop(key)
assert rv.keys() == []
@ -221,8 +232,7 @@ class TestComments(unittest.TestCase):
assert json.loads(rv.data) == 4
for x in range(4):
self.delete('/?uri=%%2Fpath%%2F&id=%i' % (x + 1))
self.delete('/id/%i' % (x + 1))
rv = self.get('/count?uri=%2Fpath%2F')
assert rv.status_code == 404

@ -1,103 +0,0 @@
import os
import time
import tempfile
import unittest
from isso.models import Comment
from isso.db import SQLite
class TestSQLite(unittest.TestCase):
def setUp(self):
fd, self.path = tempfile.mkstemp()
self.db = SQLite(self.path, False)
def test_get(self):
rv = self.db.add('/', Comment(text='Spam'), '')
c = self.db.get('/', rv["id"])
assert c["id"] == 1
assert c["text"] == 'Spam'
def test_add(self):
self.db.add('/', Comment(text='Foo'), '')
self.db.add('/', Comment(text='Bar'), '')
self.db.add('/path/', Comment(text='Baz'), '')
rv = list(self.db.retrieve('/'))
assert rv[0]["id"] == 1
assert rv[0]["text"] == 'Foo'
assert rv[1]["id"] == 2
assert rv[1]["text"] == 'Bar'
rv = list(self.db.retrieve('/path/'))
assert rv[0]["id"] == 1
assert rv[0]["text"] == 'Baz'
def test_add_return(self):
self.db.add('/', Comment(text='1'), '')
self.db.add('/', Comment(text='2'), '')
assert self.db.add('/path/', Comment(text='1'), '')["id"] == 1
def test_update(self):
rv = self.db.add('/', Comment(text='Foo'), '')
time.sleep(0.1)
rv = self.db.update('/', rv["id"], {"text": "Bla"})
c = self.db.get('/', rv["id"])
assert c["id"] == 1
assert c["text"] == 'Bla'
assert c["created"] < c["modified"]
def test_delete(self):
rv = self.db.add('/', Comment(
text='F**CK', author='P*NIS', website='http://somebadhost.org/'), '')
assert self.db.delete('/', rv["id"]) == None
def test_recent(self):
self.db.add('/path/', Comment(text='2'), '')
for x in range(5):
self.db.add('/', Comment(text='%i' % (x+1)), '')
assert len(list(self.db.recent(mode=7))) == 6
assert len(list(self.db.recent(mode=7, limit=5))) == 5
def tearDown(self):
os.unlink(self.path)
class TestSQLitePending(unittest.TestCase):
def setUp(self):
fd, self.path = tempfile.mkstemp()
self.db = SQLite(self.path, True)
def test_retrieve(self):
self.db.add('/', Comment(text='Foo'), '')
assert len(list(self.db.retrieve('/'))) == 0
def test_activate(self):
self.db.add('/', Comment(text='Foo'), '')
self.db.add('/', Comment(text='Bar'), '')
self.db.activate('/', 2)
assert len(list(self.db.retrieve('/'))) == 1
assert len(list(self.db.retrieve('/', mode=3))) == 2
def tearDown(self):
os.unlink(self.path)

@ -44,12 +44,12 @@ class TestVote(unittest.TestCase):
def testZeroLikes(self):
rv = self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."}))
assert json.loads(rv.data)['votes'] == 0
assert json.loads(rv.data)['likes'] == json.loads(rv.data)['dislikes'] == 0
def testSingleLike(self):
self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."}))
rv = self.makeClient("0.0.0.0").post("/like?uri=test&id=1")
rv = self.makeClient("0.0.0.0").post("/id/1/like")
assert rv.status_code == 200
assert rv.data == "1"
@ -58,7 +58,7 @@ class TestVote(unittest.TestCase):
bob = self.makeClient("127.0.0.1")
bob.post("/new?uri=test", data=json.dumps({"text": "..."}))
rv = bob.post('/like?uri=test&id=1')
rv = bob.post('/id/1/like')
assert rv.status_code == 200
assert rv.data == "0"
@ -67,15 +67,20 @@ class TestVote(unittest.TestCase):
self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."}))
for num in range(15):
rv = self.makeClient("1.2.3.%i" % num).post('/like?uri=test&id=1')
rv = self.makeClient("1.2.%i.0" % num).post('/id/1/like')
assert rv.status_code == 200
assert rv.data == str(num + 1)
def testVoteOnNonexistentComment(self):
rv = self.makeClient("1.2.3.4").post('/id/1/like')
assert rv.status_code == 200
assert rv.data == '0'
def testTooManyLikes(self):
self.makeClient("127.0.0.1").post("/new?uri=test", data=json.dumps({"text": "..."}))
for num in range(256):
rv = self.makeClient("1.2.3.%i" % num).post('/like?uri=test&id=1')
rv = self.makeClient("1.2.%i.0" % num).post('/id/1/like')
assert rv.status_code == 200
if num >= 142:

Loading…
Cancel
Save