diff --git a/docs/API.md b/docs/API.md index 29d3972..1d25a48 100644 --- a/docs/API.md +++ b/docs/API.md @@ -68,10 +68,6 @@ You must encode `path`, e.g. to retrieve comments for `/hello-world/`: GET /?uri=%2Fhello-world%2F -You can also pass an `id` to fetch a specific comment: - - GET /?uri=%2Fhello-world%2F&id=1 - To disable automatic Markdown-to-HTML conversion, pass `plain=1` to the query URL: diff --git a/isso/__init__.py b/isso/__init__.py index 5aab314..42a2aab 100644 --- a/isso/__init__.py +++ b/isso/__init__.py @@ -28,7 +28,6 @@ dist = pkg_resources.get_distribution("isso") import sys import io import os -import json import socket import httplib import urlparse @@ -50,16 +49,17 @@ from werkzeug.contrib.fixers import ProxyFix from jinja2 import Environment, FileSystemLoader -from isso import db, utils, migrate, views, wsgi, notify, colors +from isso import db, migrate, views, wsgi, notify, colors from isso.views import comment, admin url_map = Map([ - Rule('/', methods=['HEAD', 'GET'], endpoint=views.comment.get), - Rule('/', methods=['PUT', 'DELETE'], endpoint=views.comment.modify), - Rule('/new', methods=['POST'], endpoint=views.comment.create), - Rule('/like', methods=['POST'], endpoint=views.comment.like), - Rule('/count', methods=['GET'], endpoint=views.comment.count), + Rule('/new', methods=['POST'], endpoint=views.comment.new), + + Rule('/id/', methods=['GET', 'PUT', 'DELETE'], endpoint=views.comment.single), + Rule('/id//like', methods=['POST'], endpoint=views.comment.like), + Rule('/', methods=['GET'], endpoint=views.comment.fetch), + Rule('/count', methods=['GET'], endpoint=views.comment.count), Rule('/admin/', endpoint=views.admin.index) ]) @@ -75,7 +75,7 @@ class Isso(object): self.PASSPHRASE = passphrase self.MAX_AGE = max_age - self.db = db.SQLite(dbpath, moderation=False) + self.db = db.SQLite3(dbpath) self.signer = URLSafeTimedSerializer(secret) self.j2env = Environment(loader=FileSystemLoader(join(dirname(__file__), 'templates/'))) self.notify = lambda *args, **kwargs: mailer.sendmail(*args, **kwargs) @@ -95,10 +95,6 @@ class Isso(object): tt = self.j2env.get_template(tt) return tt.render(**ctx) - @classmethod - def dumps(cls, obj, **kw): - return json.dumps(obj, cls=utils.IssoEncoder, **kw) - def dispatch(self, request, start_response): adapter = url_map.bind_to_environ(request.environ) try: @@ -159,7 +155,7 @@ def main(): conf.read(args.conf) if args.command == "import": - migrate.disqus(db.SQLite(conf.get("general", "dbpath"), False), args.dump) + migrate.disqus(db.SQLite3(conf.get("general", "dbpath"), False), args.dump) sys.exit(0) if not conf.get("general", "host").startswith(("http://", "https://")): diff --git a/isso/db.py b/isso/db.py deleted file mode 100644 index 21dcd12..0000000 --- a/isso/db.py +++ /dev/null @@ -1,277 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# Copyright 2012, Martin Zimmermann . All rights reserved. -# License: BSD Style, 2 clauses. see isso/__init__.py - -import abc -import time -import sqlite3 - -from isso.utils import Bloomfilter -from isso.models import Comment - - -class Abstract: - - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def __init__(self, dbpath, moderation): - return - - @abc.abstractmethod - def add(self, path, comment): - """Add a new comment to the database. Returns a Comment object.""" - return - - @abc.abstractmethod - def activate(self, path, id): - """Activate comment id if pending and return comment for (path, id).""" - return - - @abc.abstractmethod - def update(self, path, id, comment): - """ - Update an existing comment, but only writeable fields such as text, - author, email, website and parent. This method should set the modified - field to the current time. - """ - return - - @abc.abstractmethod - def delete(self, path, id): - """ - Delete a comment. There are two distinctions: a comment is referenced - by another valid comment's parent attribute or stand-a-lone. In this - case the comment can't be removed without losing depending comments. - Hence, delete removes all visible data such as text, author, email, - website sets the mode field to 4. - - In the second case this comment can be safely removed without any side - effects.""" - return - - @abc.abstractmethod - def retrieve(self, path, mode): - """ - Return all comments for `path` with `mode`. - """ - return - - @abc.abstractmethod - def recent(self, mode=7, limit=None): - """ - Return most recent comments with `mode`. If `limit` is None, return - *all* comments that are currently stored, otherwise limit by `limit`. - """ - return - - @abc.abstractmethod - def like(self, path, id, remote_addr): - """+1 a given comment. Returns the new like count (may not change because - the creater can't vote on his/her own comment and multiple votes from the - same ip address are ignored as well).""" - return - - @abc.abstractmethod - def count(self, path=None, mode=1): - """return count of comments for path (optional) and mode (defaults to - visible, not deleted or moderated comments).""" - return - - -class Threads(object): - - def __init__(self, dbpath): - - self.dbpath = dbpath - - with sqlite3.connect(self.dbpath) as con: - sql = ('main.threads (path VARCHAR(255) NOT NULL, title TEXT' - 'PRIMARY KEY path)') - con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql) - - def get(self, path): - with sqlite3.connect(self.dbpath) as con: - rv = con.execute("SELECT title FROM threads WHERE path=?", (path,)).fetchone() - if rv is not None: - return rv[0] - return None - - def add(self, path, title): - with sqlite3.connect(self.dbpath) as con: - con.execute("INSERT INTO threads (path, title) VALUES (?, ?)", (path, title)) - return title - - -class SQLite(Abstract): - """A basic :class:`Abstract` implementation using SQLite3. All comments - share a single database. The tuple (id, path) acts as unique identifier - for a comment. Multiple comments per path (= that is the URI to your blog - post) are ordered by that id.""" - - fields = [ - 'path', 'id', 'created', 'modified', - 'text', 'author', 'hash', 'website', 'parent', 'mode', 'voters' - ] - - def __init__(self, dbpath, moderation): - - self.dbpath = dbpath - self.mode = 2 if moderation else 1 - - with sqlite3.connect(self.dbpath) as con: - sql = ('main.comments (path VARCHAR(255) NOT NULL, id INTEGER NOT NULL,' - 'created FLOAT NOT NULL, modified FLOAT, text VARCHAR,' - 'author VARCHAR(64), hash VARCHAR(32), website VARCHAR(64),' - 'parent INTEGER, mode INTEGER,' - 'likes INTEGER DEFAULT 0, dislikes INTEGER DEFAULT 0, voters BLOB NOT NULL,' - 'PRIMARY KEY (id, path))') - con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql) - - # increment id if (id, path) is no longer unique - con.execute("""\ - CREATE TRIGGER IF NOT EXISTS increment AFTER INSERT ON comments - BEGIN - UPDATE comments SET - id=(SELECT MAX(id)+1 FROM comments WHERE path=NEW.path) - WHERE rowid=NEW.rowid; - END;""") - - self.threads = Threads(self.dbpath) - - def query2comment(self, query): - if query is None: - return None - - return Comment( - text=query[4], author=query[5], hash=query[6], website=query[7], parent=query[8], - path=query[0], id=query[1], created=query[2], modified=query[3], mode=query[9], - votes=query[10] - ) - - def add(self, uri, c, remote_addr): - voters = buffer(Bloomfilter(iterable=[remote_addr]).array) - with sqlite3.connect(self.dbpath) as con: - keys = ','.join(self.fields) - values = ','.join('?' * len(self.fields)) - con.execute('INSERT INTO comments (%s) VALUES (%s);' % (keys, values), ( - uri, 0, c['created'] or time.time(), None, c["text"], c["author"], - c["hash"], c["website"], c["parent"], self.mode, voters) - ) - - with sqlite3.connect(self.dbpath) as con: - return self.query2comment( - con.execute('SELECT *, MAX(id) FROM comments WHERE path=?;', (uri, )).fetchone()) - - def activate(self, path, id): - with sqlite3.connect(self.dbpath) as con: - con.execute("UPDATE comments SET mode=1 WHERE path=? AND id=? AND mode=2", (path, id)) - return self.get(path, id) - - def update(self, path, id, values): - with sqlite3.connect(self.dbpath) as con: - for key, value in values.iteritems(): - con.execute('UPDATE comments SET %s=? WHERE path=? AND id=?;' % key, - (value, path, id)) - - with sqlite3.connect(self.dbpath) as con: - con.execute('UPDATE comments SET modified=? WHERE path=? AND id=?', - (time.time(), path, id)) - return self.get(path, id) - - def get(self, path, id): - with sqlite3.connect(self.dbpath) as con: - return self.query2comment(con.execute( - 'SELECT * FROM comments WHERE path=? AND id=?;', (path, id)).fetchone()) - - def _remove_stale(self, con, path): - - sql = ('DELETE FROM', - ' comments', - 'WHERE', - ' path=? AND mode=4 AND id NOT IN (', - ' SELECT', - ' parent', - ' FROM', - ' comments', - ' WHERE path=? AND parent IS NOT NULL)') - - while con.execute(' '.join(sql), (path, path)).rowcount: - continue - - def delete(self, path, id): - with sqlite3.connect(self.dbpath) as con: - sql = 'SELECT * FROM comments WHERE path=? AND parent=?' - refs = con.execute(sql, (path, id)).fetchone() - - if refs is None: - con.execute('DELETE FROM comments WHERE path=? AND id=?', (path, id)) - self._remove_stale(con, path) - return None - - con.execute('UPDATE comments SET text=? WHERE path=? AND id=?', ('', path, id)) - con.execute('UPDATE comments SET mode=? WHERE path=? AND id=?', (4, path, id)) - for field in ('author', 'website'): - con.execute('UPDATE comments SET %s=? WHERE path=? AND id=?' % field, - (None, path, id)) - - self._remove_stale(con, path) - - return self.get(path, id) - - def like(self, path, id, remote_addr): - with sqlite3.connect(self.dbpath) as con: - rv = con.execute("SELECT likes, dislikes, voters FROM comments" \ - + " WHERE path=? AND id=?", (path, id)).fetchone() - - likes, dislikes, voters = rv - if likes + dislikes >= 142: - return likes - - bf = Bloomfilter(bytearray(voters), likes + dislikes) - if remote_addr in bf: - return likes - - bf.add(remote_addr) - with sqlite3.connect(self.dbpath) as con: - con.execute("UPDATE comments SET likes = likes + 1 WHERE path=? AND id=?", (path, id)) - con.execute("UPDATE comments SET voters = ? WHERE path=? AND id=?", ( - buffer(bf.array), path, id)) - - return likes + 1 - - def count(self, path=None, mode=1): - - if path is not None: - with sqlite3.connect(self.dbpath) as con: - return con.execute("SELECT COUNT(*) FROM comments WHERE path=?" \ - + " AND (? | mode) = ?", (path, mode, mode)).fetchone() - - with sqlite3.connect(self.dbpath) as con: - return con.execute("SELECT COUNT(*) FROM comments WHERE" \ - + "(? | mode) = ?", (path, mode, mode)).fetchone() - - def retrieve(self, path, mode=5): - with sqlite3.connect(self.dbpath) as con: - rv = con.execute("SELECT * FROM comments WHERE path=? AND (? | mode) = ?" \ - + " ORDER BY id ASC;", (path, mode, mode)).fetchall() - - for item in rv: - yield self.query2comment(item) - - def recent(self, mode=7, limit=None): - - sql = 'SELECT * FROM comments WHERE (? | mode) = ? ORDER BY created DESC' - args = [mode, mode] - - if limit: - sql += ' LIMIT ?' - args.append(limit) - - with sqlite3.connect(self.dbpath) as con: - rv = con.execute(sql + ';', args).fetchall() - - for item in rv: - yield self.query2comment(item) - diff --git a/isso/db/__init__.py b/isso/db/__init__.py new file mode 100644 index 0000000..4621765 --- /dev/null +++ b/isso/db/__init__.py @@ -0,0 +1,27 @@ +# -*- encoding: utf-8 -*- + +import sqlite3 + +from isso.db.comments import Comments +from isso.db.threads import Threads + + +class SQLite3: + + connection = None + + def __init__(self, path, moderation=False): + + self.path = path + self.mode = 2 if moderation else 1 + + self.threads = Threads(self) + self.comments = Comments(self) + + def execute(self, sql, args=()): + + if isinstance(sql, (list, tuple)): + sql = ' '.join(sql) + + with sqlite3.connect(self.path) as con: + return con.execute(sql, args) diff --git a/isso/db/comments.py b/isso/db/comments.py new file mode 100644 index 0000000..25b5466 --- /dev/null +++ b/isso/db/comments.py @@ -0,0 +1,179 @@ +# -*- encoding: utf-8 -*- + +import time +import sqlite3 + +from isso.utils import Bloomfilter + + +class Comments: + """Hopefully DB-independend SQL to store, modify and retrieve all + comment-related actions. Here's a short scheme overview: + + | tid (thread id) | cid (comment id) | parent | ... | likes | remote_addr | + +-----------------+------------------+--------+-----+-------+-------------+ + | 1 | 1 | null | ... | BLOB | 127.0.0.0 | + | 1 | 2 | 1 | ... | BLOB | 127.0.0.0 | + +-----------------+------------------+--------+-----+-------+-------------+ + + The tuple (tid, cid) is unique and thus primary key. + """ + + fields = ['tid', 'id', 'parent', 'created', 'modified', 'mode', 'remote_addr', + 'text', 'author', 'email', 'website', 'likes', 'dislikes', 'voters'] + + def __init__(self, db): + + self.db = db + self.db.execute([ + 'CREATE TABLE IF NOT EXISTS comments (', + ' tid REFERENCES threads(id), id INTEGER PRIMARY KEY, parent INTEGER,', + ' created FLOAT NOT NULL, modified FLOAT, mode INTEGER, remote_addr VARCHAR,', + ' text VARCHAR, author VARCHAR, email VARCHAR, website VARCHAR,', + ' likes INTEGER DEFAULT 0, dislikes INTEGER DEFAULT 0, voters BLOB NOT NULL);']) + + def add(self, uri, c): + """ + Add a new comment to the database and return public fields as dict. + Initializes voter bloom array with provided :param:`remote_addr` and + adds a new thread to the `main.threads` table. + """ + self.db.execute([ + 'INSERT INTO comments (', + ' tid, parent,' + ' created, modified, mode, remote_addr,', + ' text, author, email, website, voters )', + 'SELECT', + ' threads.id, ?,', + ' ?, ?, ?, ?,', + ' ?, ?, ?, ?, ?', + 'FROM threads WHERE threads.uri = ?;'], ( + c.get('parent'), + c.get('created') or time.time(), None, self.db.mode, c['remote_addr'], + c['text'], c.get('author'), c.get('email'), c.get('website'), buffer( + Bloomfilter(iterable=[c['remote_addr']]).array), + uri) + ) + + return dict(zip(Comments.fields, self.db.execute( + 'SELECT *, MAX(c.id) FROM comments AS c INNER JOIN threads ON threads.uri = ?', + (uri, )).fetchone())) + + # def activate(self, path, id): + # """Activate comment id if pending and return comment for (path, id).""" + # with sqlite3.connect(self.dbpath) as con: + # con.execute("UPDATE comments SET mode=1 WHERE path=? AND id=? AND mode=2", (path, id)) + # return self.get(path, id) + + def update(self, id, data): + """ + Update an existing comment, but only writeable fields such as text, + author, email, website and parent. This method should set the modified + field to the current time. + """ + + self.db.execute([ + 'UPDATE comments SET', + ','.join(key + '=' + '?' for key in data), + 'WHERE id=?;'], + data.values() + [id]) + + return self.get(id) + + def get(self, id): + + rv = self.db.execute('SELECT * FROM comments WHERE id=?', (id, )).fetchone() + if rv: + return dict(zip(Comments.fields, rv)) + + return None + + def fetch(self, uri, mode=5): + """ + Return all comments for `path` with `mode`. + """ + rv = self.db.execute([ + 'SELECT comments.* FROM comments INNER JOIN threads ON', + ' threads.uri=? AND comments.tid=threads.id AND (? | comments.mode) = ?' + 'ORDER BY id ASC;'], (uri, mode, mode)).fetchall() + + for item in rv: + yield dict(zip(Comments.fields, item)) + + def _remove_stale(self): + + sql = ('DELETE FROM', + ' comments', + 'WHERE', + ' mode=4 AND id NOT IN (', + ' SELECT', + ' parent', + ' FROM', + ' comments', + ' WHERE parent IS NOT NULL)') + + while self.db.execute(sql).rowcount: + continue + + def delete(self, id): + """ + Delete a comment. There are two distinctions: a comment is referenced + by another valid comment's parent attribute or stand-a-lone. In this + case the comment can't be removed without losing depending comments. + Hence, delete removes all visible data such as text, author, email, + website sets the mode field to 4. + + In the second case this comment can be safely removed without any side + effects.""" + + refs = self.db.execute('SELECT * FROM comments WHERE parent=?', (id, )).fetchone() + + if refs is None: + self.db.execute('DELETE FROM comments WHERE id=?', (id, )) + self._remove_stale() + return None + + self.db.execute('UPDATE comments SET text=? WHERE id=?', ('', id)) + self.db.execute('UPDATE comments SET mode=? WHERE id=?', (4, id)) + for field in ('author', 'website'): + self.db.execute('UPDATE comments SET %s=? WHERE id=?' % field, (None, id)) + + self._remove_stale() + return self.get(id) + + def like(self, id, remote_addr): + """+1 a given comment. Returns the new like count (may not change because + the creater can't vote on his/her own comment and multiple votes from the + same ip address are ignored as well).""" + + rv = self.db.execute( + 'SELECT likes, dislikes, voters FROM comments WHERE id=?', (id, )) \ + .fetchone() + + if rv is None: + return 0 + + likes, dislikes, voters = rv + if likes + dislikes >= 142: + return likes + + bf = Bloomfilter(bytearray(voters), likes + dislikes) + if remote_addr in bf: + return likes + + bf.add(remote_addr) + self.db.execute([ + 'UPDATE comments SET', + ' likes = likes + 1, voters = ?', + 'WHERE id=?;'], (buffer(bf.array), id)) + + return likes + 1 + + def count(self, uri): + """ + return count of comments for uri. + """ + return self.db.execute([ + 'SELECT COUNT(comments.id) FROM comments INNER JOIN threads ON', + ' threads.uri=? AND comments.tid=threads.id AND comments.mode=1;'], + (uri, )).fetchone() diff --git a/isso/db/threads.py b/isso/db/threads.py new file mode 100644 index 0000000..236ae2f --- /dev/null +++ b/isso/db/threads.py @@ -0,0 +1,29 @@ +# -*- encoding: utf-8 -*- + + +class Thread(object): + + def __init__(self, id, uri, title): + self.id = id + self.uri = uri + self.title = title + + +class Threads(object): + + def __init__(self, db): + + self.db = db + self.db.execute([ + 'CREATE TABLE IF NOT EXISTS threads (', + ' id INTEGER PRIMARY KEY, uri VARCHAR(256) UNIQUE, title VARCHAR(256))']) + + def __contains__(self, uri): + return self.db.execute("SELECT title FROM threads WHERE uri=?", (uri, )) \ + .fetchone() is not None + + def __getitem__(self, uri): + return Thread(*self.db.execute("SELECT * FROM threads WHERE uri=?", (uri, )).fetchone()) + + def new(self, uri, title): + self.db.execute("INSERT INTO threads (uri, title) VALUES (?, ?)", (uri, title)) diff --git a/isso/js/app/api.js b/isso/js/app/api.js index 2cf7f2e..3b7f712 100644 --- a/isso/js/app/api.js +++ b/isso/js/app/api.js @@ -90,7 +90,7 @@ define(["lib/q"], function(Q) { } var remove = function(id) { - return curl("DELETE", endpoint + "/?" + qs({uri: location, id: id}), null) + return curl("DELETE", endpoint + "/id/" + id, null) .then(function(rv) { if (rv.status == 200) { return JSON.parse(rv.body) == null; diff --git a/isso/js/app/isso.js b/isso/js/app/isso.js index 87a2c94..b56b8b5 100644 --- a/isso/js/app/isso.js +++ b/isso/js/app/isso.js @@ -67,7 +67,7 @@ define(["lib/q", "lib/HTML", "helper/utils", "helper/identicons", "./api", "./fo node.scrollIntoView(false); } - if (utils.read(window.location.pathname + "-" + comment.id)) { + if (utils.read(comment.id)) { node.footer.add("a.delete{Löschen}").href = "#"; node.footer.add("a.edit{Bearbeiten}").href = "#"; diff --git a/isso/migrate.py b/isso/migrate.py index 56db54c..5ffe726 100644 --- a/isso/migrate.py +++ b/isso/migrate.py @@ -12,7 +12,6 @@ from __future__ import division import sys import os -import hashlib from time import mktime, strptime from urlparse import urlparse @@ -20,8 +19,6 @@ from collections import defaultdict from xml.etree import ElementTree -from isso.models import Comment - ns = '{http://disqus.com}' dsq = '{http://disqus.com/disqus-internals}' @@ -32,20 +29,15 @@ def insert(db, thread, comments): path = urlparse(thread.find('%sid' % ns).text).path remap = dict() - for item in sorted(comments, key=lambda k: k['created']): - - parent = remap.get(item.get('dsq:parent')) - comment = Comment(created=item['created'], text=item['text'], - author=item['author'], parent=parent, - hash=hashlib.md5(item["email"] or '').hexdigest()) + if path not in db.threads: + db.threads.new(path, thread.find('%stitle' % ns).text.strip()) - rv = db.add(path, comment, '127.0.0.1') - remap[item['dsq:id']] = rv["id"] + for item in sorted(comments, key=lambda k: k['created']): - try: - db.threads.get(path) - except KeyError: - db.threads.add(path, thread.find('%stitle' % ns).text.strip()) + dsq_id = item.pop('dsq:id') + item['parent'] = remap.get(item.pop('dsq:parent', None)) + rv = db.comments.add(path, item) + remap[dsq_id] = rv["id"] def disqus(db, xmlfile): @@ -61,7 +53,8 @@ def disqus(db, xmlfile): 'author': post.find('%sauthor/%sname' % (ns, ns)).text, 'email': post.find('%sauthor/%semail' % (ns, ns)).text, 'created': mktime(strptime( - post.find('%screatedAt' % ns).text, '%Y-%m-%dT%H:%M:%SZ')) + post.find('%screatedAt' % ns).text, '%Y-%m-%dT%H:%M:%SZ')), + 'remote_addr': '127.0.0.0' } if post.find(ns + 'parent') is not None: diff --git a/isso/models.py b/isso/models.py deleted file mode 100644 index d9d9c86..0000000 --- a/isso/models.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# Copyright 2012, Martin Zimmermann . All rights reserved. -# License: BSD Style, 2 clauses. see isso/__init__.py - -from __future__ import unicode_literals - -import hashlib - - -class Comment(object): - """This class represents a regular comment. It needs at least a text - field, all other fields are optional (or automatically set by the - database driver. - - The field `mode` has a special meaning: - - 1: normal - 2: in moderation queue - 4: deleted - - You can query for them like with UNIX permission bits, so you get both - normal and queued using MODE=3. - """ - - fields = ["text", "author", "website", "votes", "hash", "parent", "mode", "id", - "created", "modified"] - - def __init__(self, **kwargs): - - self.values = {} - - for key in Comment.fields: - self.values[key] = kwargs.get(key, None) - - def __getitem__(self, key): - return self.values[key] - - def __setitem__(self, key, value): - self.values[key] = value - - def iteritems(self): - for key in Comment.fields: - yield key, self.values[key] - - @property - def pending(self): - return self.values["mode"] == 2 - - @property - def deleted(self): - return self.values["mode"] == 4 - - @property - def md5(self): - hv = hashlib.md5() - - for key, value in self.iteritems(): - if key == "parent" or value is None: - continue - hv.update(unicode(self.values.get(key, "")).encode("utf-8", "replace")) # XXX - - return hv.hexdigest() diff --git a/isso/static/post.html b/isso/static/post.html index 5985185..0bbd738 100644 --- a/isso/static/post.html +++ b/isso/static/post.html @@ -4,8 +4,7 @@ Hello World - - +