From 48b4c9f9a58f335220c5e2eeb434a105e7ca6131 Mon Sep 17 00:00:00 2001 From: Martin Zimmermann Date: Thu, 24 Oct 2013 14:16:14 +0200 Subject: [PATCH] purge comments in moderation queue after given time, closes #13 --- docs/CONFIGURATION.md | 11 +++- isso/__init__.py | 30 +++++++-- isso/core.py | 99 ++++++++++++++++++---------- isso/db/comments.py | 6 ++ isso/{utils.py => utils/__init__.py} | 0 isso/utils/parse.py | 63 ++++++++++++++++++ isso/views/comment.py | 4 +- specs/test_comment.py | 34 +++++++++- tox.ini | 1 + 9 files changed, 203 insertions(+), 45 deletions(-) rename isso/{utils.py => utils/__init__.py} (100%) create mode 100644 isso/utils/parse.py diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index f8d2005..cba2f8a 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -39,8 +39,7 @@ establish a connection to your website (a simple HEAD request). If this check fails, none can comment on your website. max-age -: time to allow users to remove or edit their comments. Defaults to `900` -seconds (15 minutes). +: time to allow users to remove or edit their comments. Defaults to `15m`. ## server (not applicable for uWSGI) @@ -54,6 +53,14 @@ reload : reload application, when editing the source code (only useful for developers), disabled by default. +## moderation + +enabled +: enable comment moderation queue, disabled by default + +purge-after +: remove unprocessed comments in moderation queue after, by default, `30d`. + ## SMTP username diff --git a/isso/__init__.py b/isso/__init__.py index a5cc851..0df63bd 100644 --- a/isso/__init__.py +++ b/isso/__init__.py @@ -32,10 +32,18 @@ dist = pkg_resources.get_distribution("isso") import sys import os +import socket from os.path import dirname, join from argparse import ArgumentParser +try: + import httplib + import urlparse +except ImportError: + import http.client as httplib + import urllib.parse as urlparse + import misaka from itsdangerous import URLSafeTimedSerializer @@ -49,8 +57,8 @@ from werkzeug.contrib.fixers import ProxyFix from jinja2 import Environment, FileSystemLoader -from isso import db, migrate, views, wsgi -from isso.core import NaiveMixin, uWSGIMixin, Config +from isso import db, migrate, views, wsgi, colors +from isso.core import ThreadedMixin, uWSGIMixin, Config from isso.views import comment, admin rules = Map([ @@ -76,13 +84,13 @@ class Isso(object): def __init__(self, conf): - super(Isso, self).__init__(conf) - self.conf = conf self.db = db.SQLite3(conf.get('general', 'dbpath'), conf) self.signer = URLSafeTimedSerializer(conf.get('general', 'secretkey')) self.j2env = Environment(loader=FileSystemLoader(join(dirname(__file__), 'templates/'))) + super(Isso, self).__init__(conf) + def sign(self, obj): return self.signer.dumps(obj) @@ -128,7 +136,7 @@ def make_app(conf=None): try: import uwsgi except ImportError: - class App(Isso, NaiveMixin): + class App(Isso, ThreadedMixin): pass else: class App(Isso, uWSGIMixin): @@ -136,6 +144,18 @@ def make_app(conf=None): isso = App(conf) + if not conf.get("general", "host").startswith(("http://", "https://")): + raise SystemExit("error: host must start with http:// or https://") + + try: + print(" * connecting to HTTP server", end=" ") + rv = urlparse.urlparse(conf.get("general", "host")) + host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc + httplib.HTTPConnection(host, timeout=5).request('GET', rv.path) + print("[%s]" % colors.green("ok")) + except (httplib.HTTPException, socket.error): + print("[%s]" % colors.red("failed")) + app = ProxyFix(wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, { '/static': join(dirname(__file__), 'static/'), '/js': join(dirname(__file__), 'js/'), diff --git a/isso/core.py b/isso/core.py index 838bbbd..f9c5c63 100644 --- a/isso/core.py +++ b/isso/core.py @@ -22,16 +22,29 @@ from isso.compat import PY2K if PY2K: import thread - - import httplib - import urlparse else: import _thread as thread - import http.client as httplib - import urllib.parse as urlparse - from isso import notify, colors +from isso.utils import parse + + +class IssoParser(ConfigParser): + + @classmethod + def _total_seconds(cls, td): + return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 + + def getint(self, section, key): + try: + delta = parse.timedelta(self.get(section, key)) + except ValueError: + return super(IssoParser, self).getint(section, key) + else: + try: + return int(delta.total_seconds()) + except AttributeError: + return int(IssoParser._total_seconds(delta)) class Config: @@ -40,7 +53,10 @@ class Config: "[general]", "dbpath = /tmp/isso.db", "secretkey = %r" % binascii.b2a_hex(os.urandom(24)), "host = http://localhost:8080/", "passphrase = p@$$w0rd", - "max-age = 900", "moderated = false", + "max-age = 15m", + "[moderation]", + "enabled = false", + "purge-after = 30d", "[server]", "host = localhost", "port = 8080", "reload = off", "[SMTP]", @@ -56,7 +72,7 @@ class Config: @classmethod def load(cls, configfile): - rv = ConfigParser(allow_no_value=True) + rv = IssoParser(allow_no_value=True) rv.read_file(io.StringIO(u'\n'.join(Config.default))) if configfile: @@ -65,50 +81,47 @@ class Config: return rv -def threaded(func): +def SMTP(conf): - def dec(self, *args, **kwargs): - thread.start_new_thread(func, (self, ) + args, kwargs) + try: + print(" * connecting to SMTP server", end=" ") + mailer = notify.SMTPMailer(conf) + print("[%s]" % colors.green("ok")) + except (socket.error, smtplib.SMTPException): + print("[%s]" % colors.red("failed")) + mailer = notify.NullMailer() - return dec + return mailer class Mixin(object): - def __init__(self, *args): + def __init__(self, conf): self.lock = threading.Lock() def notify(self, subject, body, retries=5): pass -class NaiveMixin(Mixin): +def threaded(func): - def __init__(self, conf): + def dec(self, *args, **kwargs): + thread.start_new_thread(func, (self, ) + args, kwargs) - super(NaiveMixin, self).__init__() + return dec - try: - print(" * connecting to SMTP server", end=" ") - mailer = notify.SMTPMailer(conf) - print("[%s]" % colors.green("ok")) - except (socket.error, smtplib.SMTPException): - print("[%s]" % colors.red("failed")) - mailer = notify.NullMailer() - self.mailer = mailer +class ThreadedMixin(Mixin): - if not conf.get("general", "host").startswith(("http://", "https://")): - raise SystemExit("error: host must start with http:// or https://") + def __init__(self, conf): + + super(ThreadedMixin, self).__init__(conf) + + if conf.getboolean("moderation", "enabled"): + self.purge(conf.getint("moderation", "purge-after")) + + self.mailer = SMTP(conf) - try: - print(" * connecting to HTTP server", end=" ") - rv = urlparse.urlparse(conf.get("general", "host")) - host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc - httplib.HTTPConnection(host, timeout=5).request('GET', rv.path) - print("[%s]" % colors.green("ok")) - except (httplib.HTTPException, socket.error): - print("[%s]" % colors.red("failed")) @threaded def notify(self, subject, body, retries=5): @@ -121,8 +134,15 @@ class NaiveMixin(Mixin): else: break + @threaded + def purge(self, delta): + while True: + with self.lock: + self.db.comments.purge(delta) + time.sleep(delta) -class uWSGIMixin(NaiveMixin): + +class uWSGIMixin(Mixin): def __init__(self, conf): @@ -148,7 +168,16 @@ class uWSGIMixin(NaiveMixin): return uwsgi.SPOOL_OK self.lock = Lock() + self.mailer = SMTP(conf) uwsgi.spooler = spooler + timedelta = conf.getint("moderation", "purge-after") + purge = lambda signum: self.db.comments.purge(timedelta) + uwsgi.register_signal(1, "", purge) + uwsgi.add_timer(1, timedelta) + + # run purge once + purge(1) + def notify(self, subject, body, retries=5): uwsgi.spool({"subject": subject.encode('utf-8'), "body": body.encode('utf-8')}) diff --git a/isso/db/comments.py b/isso/db/comments.py index 37ffcd8..2a549ba 100644 --- a/isso/db/comments.py +++ b/isso/db/comments.py @@ -183,3 +183,9 @@ class Comments: 'SELECT COUNT(comments.id) FROM comments INNER JOIN threads ON', ' threads.uri=? AND comments.tid=threads.id AND comments.mode=1;'], (uri, )).fetchone() + + def purge(self, delta): + self.db.execute([ + 'DELETE FROM comments WHERE mode = 2 AND ? - created > ?;' + ], (time.time(), delta)) + self._remove_stale() diff --git a/isso/utils.py b/isso/utils/__init__.py similarity index 100% rename from isso/utils.py rename to isso/utils/__init__.py diff --git a/isso/utils/parse.py b/isso/utils/parse.py new file mode 100644 index 0000000..3bc1aca --- /dev/null +++ b/isso/utils/parse.py @@ -0,0 +1,63 @@ + +from __future__ import print_function + +import re +import datetime + +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse + + +def timedelta(value): + """ + Parse :param value: into :class:`datetime.timedelta`, you can use any (logical) + combination of Nw, Nd, Nh and Nm, e.g. `1h30m` for 1 hour, 30 minutes or `3w` for + 3 weeks. Raises a ValueError if the input is invalid/unparseable. + + >>> print(timedelta("3w")) + 21 days, 0:00:00 + >>> print(timedelta("3w 12h 57m")) + 21 days, 12:57:00 + >>> print(timedelta("1h30m37s")) + 1:30:37 + >>> print(timedelta("1asdf3w")) + Traceback (most recent call last): + ... + ValueError: invalid human-readable timedelta + """ + + keys = ["weeks", "days", "hours", "minutes", "seconds"] + regex = "".join(["((?P<%s>\d+)%s ?)?" % (k, k[0]) for k in keys]) + kwargs = {} + for k, v in re.match(regex, value).groupdict(default="0").items(): + kwargs[k] = int(v) + + rv = datetime.timedelta(**kwargs) + if rv == datetime.timedelta(): + raise ValueError("invalid human-readable timedelta") + return datetime.timedelta(**kwargs) + + +def host(name): + """ + Parse :param name: into `httplib`-compatible host:port. + + >>> print(host("http://example.tld/")) + ('example.tld', 80) + >>> print(host("https://example.tld/")) + ('example.tld', 443) + >>> print(host("example.tld")) + ('example.tld', 80) + >>> print(host("example.tld:42")) + ('example.tld', 42) + """ + + if not name.startswith(('http://', 'https://')): + name = 'http://' + name + + rv = urlparse(name) + if rv.scheme == 'https': + return (rv.netloc, 443) + return (rv.netloc.rsplit(':')[0], rv.port or 80) diff --git a/isso/views/comment.py b/isso/views/comment.py index d6813c4..3cdb3e8 100644 --- a/isso/views/comment.py +++ b/isso/views/comment.py @@ -67,7 +67,7 @@ def new(app, environ, request, uri): if data.get(field): data[field] = cgi.escape(data[field]) - data['mode'] = (app.conf.getboolean('general', 'moderated') and 2) or 1 + data['mode'] = (app.conf.getboolean('moderation', 'enabled') and 2) or 1 data['remote_addr'] = utils.anonymize(str(request.remote_addr)) with app.lock: @@ -90,7 +90,7 @@ def new(app, environ, request, uri): deletion = host + environ["SCRIPT_NAME"] + "/delete/" + app.sign(str(rv["id"])) activation = None - if app.conf.getboolean('general', 'moderated'): + if app.conf.getboolean('moderation', 'enabled'): activation = host + environ["SCRIPT_NAME"] + "/activate/" + app.sign(str(rv["id"])) app.notify(title, notify.format(rv, href, utils.anonymize(str(request.remote_addr)), diff --git a/specs/test_comment.py b/specs/test_comment.py index 6d47042..40ae196 100644 --- a/specs/test_comment.py +++ b/specs/test_comment.py @@ -264,7 +264,7 @@ class TestModeratedComments(unittest.TestCase): fd, self.path = tempfile.mkstemp() conf = core.Config.load(None) conf.set("general", "dbpath", self.path) - conf.set("general", "moderated", "true") + conf.set("moderation", "enabled", "true") conf.set("guard", "enabled", "off") class App(Isso, core.Mixin): @@ -287,3 +287,35 @@ class TestModeratedComments(unittest.TestCase): self.app.db.comments.activate(1) assert self.client.get('/?uri=test').status_code == 200 + + +class TestPurgeComments(unittest.TestCase): + + def setUp(self): + fd, self.path = tempfile.mkstemp() + conf = core.Config.load(None) + conf.set("general", "dbpath", self.path) + conf.set("moderation", "enabled", "true") + conf.set("guard", "enabled", "off") + + class App(Isso, core.Mixin): + pass + + self.app = App(conf) + self.app.wsgi_app = FakeIP(self.app.wsgi_app) + self.client = Client(self.app, Response) + + def testPurgeDoesNoHarm(self): + self.client.post('/new?uri=test', data=json.dumps({"text": "..."})) + self.app.db.comments.activate(1) + self.app.db.comments.purge(0) + assert self.client.get('/?uri=test').status_code == 200 + + def testPurgeWorks(self): + self.client.post('/new?uri=test', data=json.dumps({"text": "..."})) + self.app.db.comments.purge(0) + assert self.client.get('/id/1').status_code == 404 + + self.client.post('/new?uri=test', data=json.dumps({"text": "..."})) + self.app.db.comments.purge(3600) + assert self.client.get('/id/1').status_code == 200 diff --git a/tox.ini b/tox.ini index 317a221..4d836e2 100755 --- a/tox.ini +++ b/tox.ini @@ -16,4 +16,5 @@ deps = nose ipaddress commands= + nosetests --with-doctest isso/ nosetests specs/