purge comments in moderation queue after given time, closes #13

docs/CONFIGURATION.md

@@ -39,8 +39,7 @@ establish a connection to your website (a simple HEAD request). If this
 check fails, none can comment on your website.
 
 max-age
-: time to allow users to remove or edit their comments. Defaults to `900`
+: time to allow users to remove or edit their comments. Defaults to `15m`.
-seconds (15 minutes).
 
 ## server (not applicable for uWSGI)
 
@@ -54,6 +53,14 @@ reload
 : reload application, when editing the source code (only useful for developers),
 disabled by default.
 
+## moderation
+
+enabled
+: enable comment moderation queue, disabled by default
+
+purge-after
+: remove unprocessed comments in moderation queue after, by default, `30d`.
+
 ## SMTP
 
 username

isso/__init__.py

@@ -32,10 +32,18 @@ dist = pkg_resources.get_distribution("isso")
 
 import sys
 import os
+import socket
 
 from os.path import dirname, join
 from argparse import ArgumentParser
 
+try:
+    import httplib
+    import urlparse
+except ImportError:
+    import http.client as httplib
+    import urllib.parse as urlparse
+
 import misaka
 from itsdangerous import URLSafeTimedSerializer
 
@@ -49,8 +57,8 @@ from werkzeug.contrib.fixers import ProxyFix
 
 from jinja2 import Environment, FileSystemLoader
 
-from isso import db, migrate, views, wsgi
+from isso import db, migrate, views, wsgi, colors
-from isso.core import NaiveMixin, uWSGIMixin, Config
+from isso.core import ThreadedMixin, uWSGIMixin, Config
 from isso.views import comment, admin
 
 rules = Map([
@@ -76,13 +84,13 @@ class Isso(object):
 
     def __init__(self, conf):
 
-        super(Isso, self).__init__(conf)
-
         self.conf = conf
         self.db = db.SQLite3(conf.get('general', 'dbpath'), conf)
         self.signer = URLSafeTimedSerializer(conf.get('general', 'secretkey'))
         self.j2env = Environment(loader=FileSystemLoader(join(dirname(__file__), 'templates/')))
 
+        super(Isso, self).__init__(conf)
+
     def sign(self, obj):
         return self.signer.dumps(obj)
 
@@ -128,7 +136,7 @@ def make_app(conf=None):
     try:
         import uwsgi
     except ImportError:
-        class App(Isso, NaiveMixin):
+        class App(Isso, ThreadedMixin):
             pass
     else:
         class App(Isso, uWSGIMixin):
@@ -136,6 +144,18 @@ def make_app(conf=None):
 
     isso = App(conf)
 
+    if not conf.get("general", "host").startswith(("http://", "https://")):
+        raise SystemExit("error: host must start with http:// or https://")
+
+    try:
+        print(" * connecting to HTTP server", end=" ")
+        rv = urlparse.urlparse(conf.get("general", "host"))
+        host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc
+        httplib.HTTPConnection(host, timeout=5).request('GET', rv.path)
+        print("[%s]" % colors.green("ok"))
+    except (httplib.HTTPException, socket.error):
+        print("[%s]" % colors.red("failed"))
+
     app = ProxyFix(wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, {
         '/static': join(dirname(__file__), 'static/'),
         '/js': join(dirname(__file__), 'js/'),

isso/core.py

@@ -22,16 +22,29 @@ from isso.compat import PY2K
 
 if PY2K:
     import thread
-
-    import httplib
-    import urlparse
 else:
     import _thread as thread
 
-    import http.client as httplib
-    import urllib.parse as urlparse
-
 from isso import notify, colors
+from isso.utils import parse
+
+
+class IssoParser(ConfigParser):
+
+    @classmethod
+    def _total_seconds(cls, td):
+        return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6
+
+    def getint(self, section, key):
+        try:
+            delta = parse.timedelta(self.get(section, key))
+        except ValueError:
+            return super(IssoParser, self).getint(section, key)
+        else:
+            try:
+                return int(delta.total_seconds())
+            except AttributeError:
+                return int(IssoParser._total_seconds(delta))
 
 
 class Config:
@@ -40,7 +53,10 @@ class Config:
         "[general]",
         "dbpath = /tmp/isso.db", "secretkey = %r" % binascii.b2a_hex(os.urandom(24)),
         "host = http://localhost:8080/", "passphrase = p@$$w0rd",
-        "max-age = 900", "moderated = false",
+        "max-age = 15m",
+        "[moderation]",
+        "enabled = false",
+        "purge-after = 30d",
         "[server]",
         "host = localhost", "port = 8080", "reload = off",
         "[SMTP]",
@@ -56,7 +72,7 @@ class Config:
     @classmethod
     def load(cls, configfile):
 
-        rv = ConfigParser(allow_no_value=True)
+        rv = IssoParser(allow_no_value=True)
         rv.read_file(io.StringIO(u'\n'.join(Config.default)))
 
         if configfile:
@@ -65,6 +81,28 @@ class Config:
         return rv
 
 
+def SMTP(conf):
+
+    try:
+        print(" * connecting to SMTP server", end=" ")
+        mailer = notify.SMTPMailer(conf)
+        print("[%s]" % colors.green("ok"))
+    except (socket.error, smtplib.SMTPException):
+        print("[%s]" % colors.red("failed"))
+        mailer = notify.NullMailer()
+
+    return mailer
+
+
+class Mixin(object):
+
+    def __init__(self, conf):
+        self.lock = threading.Lock()
+
+    def notify(self, subject, body, retries=5):
+        pass
+
+
 def threaded(func):
 
     def dec(self, *args, **kwargs):
@@ -73,42 +111,17 @@ def threaded(func):
     return dec
 
 
-class Mixin(object):
+class ThreadedMixin(Mixin):
-
-    def __init__(self, *args):
-        self.lock = threading.Lock()
-
-    def notify(self, subject, body, retries=5):
-        pass
-
-
-class NaiveMixin(Mixin):
 
     def __init__(self, conf):
 
-        super(NaiveMixin, self).__init__()
+        super(ThreadedMixin, self).__init__(conf)
 
-        try:
+        if conf.getboolean("moderation", "enabled"):
-             print(" * connecting to SMTP server", end=" ")
+            self.purge(conf.getint("moderation", "purge-after"))
-             mailer = notify.SMTPMailer(conf)
-             print("[%s]" % colors.green("ok"))
-        except (socket.error, smtplib.SMTPException):
-             print("[%s]" % colors.red("failed"))
-             mailer = notify.NullMailer()
 
-        self.mailer = mailer
+        self.mailer = SMTP(conf)
 
-        if not conf.get("general", "host").startswith(("http://", "https://")):
-            raise SystemExit("error: host must start with http:// or https://")
-
-        try:
-             print(" * connecting to HTTP server", end=" ")
-             rv = urlparse.urlparse(conf.get("general", "host"))
-             host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc
-             httplib.HTTPConnection(host, timeout=5).request('GET', rv.path)
-             print("[%s]" % colors.green("ok"))
-        except (httplib.HTTPException, socket.error):
-             print("[%s]" % colors.red("failed"))
 
     @threaded
     def notify(self, subject, body, retries=5):
@@ -121,8 +134,15 @@ class NaiveMixin(Mixin):
             else:
                 break
 
+    @threaded
+    def purge(self, delta):
+        while True:
+            with self.lock:
+                self.db.comments.purge(delta)
+            time.sleep(delta)
 
-class uWSGIMixin(NaiveMixin):
+
+class uWSGIMixin(Mixin):
 
     def __init__(self, conf):
 
@@ -148,7 +168,16 @@ class uWSGIMixin(NaiveMixin):
                 return uwsgi.SPOOL_OK
 
         self.lock = Lock()
+        self.mailer = SMTP(conf)
         uwsgi.spooler = spooler
 
+        timedelta = conf.getint("moderation", "purge-after")
+        purge = lambda signum: self.db.comments.purge(timedelta)
+        uwsgi.register_signal(1, "", purge)
+        uwsgi.add_timer(1, timedelta)
+
+        # run purge once
+        purge(1)
+
     def notify(self, subject, body, retries=5):
         uwsgi.spool({"subject": subject.encode('utf-8'), "body": body.encode('utf-8')})

isso/db/comments.py

@@ -183,3 +183,9 @@ class Comments:
             'SELECT COUNT(comments.id) FROM comments INNER JOIN threads ON',
             '    threads.uri=? AND comments.tid=threads.id AND comments.mode=1;'],
             (uri, )).fetchone()
+
+    def purge(self, delta):
+        self.db.execute([
+            'DELETE FROM comments WHERE mode = 2 AND ? - created > ?;'
+        ], (time.time(), delta))
+        self._remove_stale()

isso/utils/parse.py

@@ -0,0 +1,63 @@
+
+from __future__ import print_function
+
+import re
+import datetime
+
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+
+def timedelta(value):
+    """
+    Parse :param value: into :class:`datetime.timedelta`, you can use any (logical)
+    combination of Nw, Nd, Nh and Nm, e.g. `1h30m` for 1 hour, 30 minutes or `3w` for
+    3 weeks. Raises a ValueError if the input is invalid/unparseable.
+
+    >>> print(timedelta("3w"))
+    21 days, 0:00:00
+    >>> print(timedelta("3w 12h 57m"))
+    21 days, 12:57:00
+    >>> print(timedelta("1h30m37s"))
+    1:30:37
+    >>> print(timedelta("1asdf3w"))
+    Traceback (most recent call last):
+        ...
+    ValueError: invalid human-readable timedelta
+    """
+
+    keys = ["weeks", "days", "hours", "minutes", "seconds"]
+    regex = "".join(["((?P<%s>\d+)%s ?)?" % (k, k[0]) for k in keys])
+    kwargs = {}
+    for k, v in re.match(regex, value).groupdict(default="0").items():
+        kwargs[k] = int(v)
+
+    rv = datetime.timedelta(**kwargs)
+    if rv == datetime.timedelta():
+        raise ValueError("invalid human-readable timedelta")
+    return datetime.timedelta(**kwargs)
+
+
+def host(name):
+    """
+    Parse :param name: into `httplib`-compatible host:port.
+
+    >>> print(host("http://example.tld/"))
+    ('example.tld', 80)
+    >>> print(host("https://example.tld/"))
+    ('example.tld', 443)
+    >>> print(host("example.tld"))
+    ('example.tld', 80)
+    >>> print(host("example.tld:42"))
+    ('example.tld', 42)
+    """
+
+    if not name.startswith(('http://', 'https://')):
+        name = 'http://' + name
+
+    rv = urlparse(name)
+    if rv.scheme == 'https':
+        return (rv.netloc, 443)
+    return (rv.netloc.rsplit(':')[0], rv.port or 80)

isso/views/comment.py

@@ -67,7 +67,7 @@ def new(app, environ, request, uri):
         if data.get(field):
             data[field] = cgi.escape(data[field])
 
-    data['mode'] = (app.conf.getboolean('general', 'moderated') and 2) or 1
+    data['mode'] = (app.conf.getboolean('moderation', 'enabled') and 2) or 1
     data['remote_addr'] = utils.anonymize(str(request.remote_addr))
 
     with app.lock:
@@ -90,7 +90,7 @@ def new(app, environ, request, uri):
     deletion = host + environ["SCRIPT_NAME"] + "/delete/" + app.sign(str(rv["id"]))
     activation = None
 
-    if app.conf.getboolean('general', 'moderated'):
+    if app.conf.getboolean('moderation', 'enabled'):
         activation = host + environ["SCRIPT_NAME"] + "/activate/" + app.sign(str(rv["id"]))
 
     app.notify(title, notify.format(rv, href, utils.anonymize(str(request.remote_addr)),

specs/test_comment.py

@@ -264,7 +264,7 @@ class TestModeratedComments(unittest.TestCase):
         fd, self.path = tempfile.mkstemp()
         conf = core.Config.load(None)
         conf.set("general", "dbpath", self.path)
-        conf.set("general", "moderated", "true")
+        conf.set("moderation", "enabled", "true")
         conf.set("guard", "enabled", "off")
 
         class App(Isso, core.Mixin):
@@ -287,3 +287,35 @@ class TestModeratedComments(unittest.TestCase):
 
         self.app.db.comments.activate(1)
         assert self.client.get('/?uri=test').status_code == 200
+
+
+class TestPurgeComments(unittest.TestCase):
+
+    def setUp(self):
+        fd, self.path = tempfile.mkstemp()
+        conf = core.Config.load(None)
+        conf.set("general", "dbpath", self.path)
+        conf.set("moderation", "enabled", "true")
+        conf.set("guard", "enabled", "off")
+
+        class App(Isso, core.Mixin):
+            pass
+
+        self.app = App(conf)
+        self.app.wsgi_app = FakeIP(self.app.wsgi_app)
+        self.client = Client(self.app, Response)
+
+    def testPurgeDoesNoHarm(self):
+        self.client.post('/new?uri=test', data=json.dumps({"text": "..."}))
+        self.app.db.comments.activate(1)
+        self.app.db.comments.purge(0)
+        assert self.client.get('/?uri=test').status_code == 200
+
+    def testPurgeWorks(self):
+        self.client.post('/new?uri=test', data=json.dumps({"text": "..."}))
+        self.app.db.comments.purge(0)
+        assert self.client.get('/id/1').status_code == 404
+
+        self.client.post('/new?uri=test', data=json.dumps({"text": "..."}))
+        self.app.db.comments.purge(3600)
+        assert self.client.get('/id/1').status_code == 200

tox.ini

@@ -16,4 +16,5 @@ deps =
     nose
     ipaddress
 commands=
+    nosetests --with-doctest isso/
     nosetests specs/