From ab618ad89856fcd80e9c26ed621598549d52f092 Mon Sep 17 00:00:00 2001 From: Martin Zimmermann Date: Sun, 6 Oct 2013 18:37:05 +0200 Subject: [PATCH] add basic spam protection --- isso/__init__.py | 29 ++++++++++---------------- isso/core.py | 43 +++++++++++++++++++++++++++++++++------ isso/db/__init__.py | 10 +++++---- isso/db/comments.py | 2 ++ isso/db/spam.py | 47 +++++++++++++++++++++++++++++++++++++++++++ isso/views/comment.py | 11 ++++++---- specs/test_comment.py | 10 +++++++-- specs/test_vote.py | 11 ++++++++-- 8 files changed, 127 insertions(+), 36 deletions(-) create mode 100644 isso/db/spam.py diff --git a/isso/__init__.py b/isso/__init__.py index 3123b6a..4c056b2 100644 --- a/isso/__init__.py +++ b/isso/__init__.py @@ -79,20 +79,8 @@ class Isso(object): super(Isso, self).__init__(conf) - if not conf.get("general", "host").startswith(("http://", "https://")): - sys.exit("error: host must start with http:// or https://") - - try: - print(" * connecting to HTTP server", end=" ") - rv = urlparse.urlparse(conf.get("general", "host")) - host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc - httplib.HTTPConnection(host, timeout=5).request('GET', rv.path) - print("[%s]" % colors.green("ok")) - except (httplib.HTTPException, socket.error): - print("[%s]" % colors.red("failed")) - self.conf = conf - self.db = db.SQLite3(conf.get('general', 'dbpath')) + self.db = db.SQLite3(conf.get('general', 'dbpath'), conf) self.signer = URLSafeTimedSerializer(conf.get('general', 'secretkey')) self.j2env = Environment(loader=FileSystemLoader(join(dirname(__file__), 'templates/'))) @@ -100,7 +88,7 @@ class Isso(object): return self.signer.dumps(obj) def unsign(self, obj): - return self.signer.loads(obj, max_age=self.MAX_AGE) + return self.signer.loads(obj, max_age=self.conf.getint('general', 'max-age')) def markdown(self, text): return misaka.html(text, extensions=misaka.EXT_STRIKETHROUGH \ @@ -121,7 +109,7 @@ class Isso(object): except MethodNotAllowed: return Response("Yup.", 200) except HTTPException as e: - return Response(e, 500) + return e def wsgi_app(self, environ, start_response): response = self.dispatch(Request(environ), start_response) @@ -141,9 +129,13 @@ def make_app(conf=None): try: import uwsgi except ImportError: - isso = type("Isso", (Isso, NaiveMixin), {})(conf) + class App(Isso, NaiveMixin): + pass else: - isso = type("Isso", (Isso, uWSGIMixin), {})(conf) + class App(Isso, uWSGIMixin): + pass + + isso = App(conf) app = ProxyFix(wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, { '/static': join(dirname(__file__), 'static/'), @@ -172,7 +164,8 @@ def main(): conf = Config.load(args.conf) if args.command == "import": - migrate.disqus(db.SQLite3(conf.get('general', 'dbpath')), args.dump) + conf.set("guard", "enabled", "off") + migrate.disqus(db.SQLite3(conf.get('general', 'dbpath'), conf), args.dump) sys.exit(0) run_simple(conf.get('server', 'host'), conf.getint('server', 'port'), make_app(conf), diff --git a/isso/core.py b/isso/core.py index dad20f6..2c102cf 100644 --- a/isso/core.py +++ b/isso/core.py @@ -12,6 +12,9 @@ import threading import socket import smtplib +import httplib +import urlparse + from ConfigParser import ConfigParser try: @@ -21,19 +24,24 @@ except ImportError: from isso import notify, colors + class Config: default = [ "[general]", "dbpath = /tmp/isso.db", "secretkey = %r" % os.urandom(24), "host = http://localhost:8080/", "passphrase = p@$$w0rd", - "max_age = 450", + "max-age = 900", "[server]", "host = localhost", "port = 8080", "reload = off", "[SMTP]", "username = ", "password = ", "host = localhost", "port = 465", "ssl = on", - "to = ", "from = " + "to = ", "from = ", + "[guard]", + "enabled = on", + "ratelimit = 2" + "" ] @classmethod @@ -56,10 +64,21 @@ def threaded(func): return dec -class NaiveMixin(object): +class Mixin(object): + + def __init__(self, *args): + self.lock = threading.Lock() + + def notify(self, subject, body, retries=5): + pass + + +class NaiveMixin(Mixin): def __init__(self, conf): + super(NaiveMixin, self).__init__() + try: print(" * connecting to SMTP server", end=" ") mailer = notify.SMTPMailer(conf) @@ -69,7 +88,18 @@ class NaiveMixin(object): mailer = notify.NullMailer() self.mailer = mailer - self.lock = threading.Lock() + + if not conf.get("general", "host").startswith(("http://", "https://")): + raise SystemExit("error: host must start with http:// or https://") + + try: + print(" * connecting to HTTP server", end=" ") + rv = urlparse.urlparse(conf.get("general", "host")) + host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc + httplib.HTTPConnection(host, timeout=5).request('GET', rv.path) + print("[%s]" % colors.green("ok")) + except (httplib.HTTPException, socket.error): + print("[%s]" % colors.red("failed")) @threaded def notify(self, subject, body, retries=5): @@ -86,15 +116,16 @@ class NaiveMixin(object): class uWSGIMixin(NaiveMixin): def __init__(self, conf): + super(uWSGIMixin, self).__init__(conf) class Lock(): def __enter__(self): while uwsgi.queue_get(0) == "LOCK": - time.sleep(0.1) + time.sleep(0.01) - uwsgi.queue_set(uwsgi.queue_slot(), "LOCK") + uwsgi.queue_set(0, "LOCK") def __exit__(self, exc_type, exc_val, exc_tb): uwsgi.queue_pop() diff --git a/isso/db/__init__.py b/isso/db/__init__.py index 4621765..8ebcecb 100644 --- a/isso/db/__init__.py +++ b/isso/db/__init__.py @@ -2,18 +2,20 @@ import sqlite3 +class IssoDBException(Exception): + pass + from isso.db.comments import Comments from isso.db.threads import Threads class SQLite3: - connection = None - - def __init__(self, path, moderation=False): + def __init__(self, path, conf): self.path = path - self.mode = 2 if moderation else 1 + self.conf = conf + self.mode = 1 self.threads = Threads(self) self.comments = Comments(self) diff --git a/isso/db/comments.py b/isso/db/comments.py index 95cffae..154aa70 100644 --- a/isso/db/comments.py +++ b/isso/db/comments.py @@ -2,6 +2,7 @@ import time +from isso.db import spam from isso.utils import Bloomfilter @@ -31,6 +32,7 @@ class Comments: ' text VARCHAR, author VARCHAR, email VARCHAR, website VARCHAR,', ' likes INTEGER DEFAULT 0, dislikes INTEGER DEFAULT 0, voters BLOB NOT NULL);']) + @spam.check def add(self, uri, c): """ Add a new comment to the database and return public fields as dict. diff --git a/isso/db/spam.py b/isso/db/spam.py new file mode 100644 index 0000000..5713bce --- /dev/null +++ b/isso/db/spam.py @@ -0,0 +1,47 @@ +# -*- encoding: utf-8 -*- + +import time + +from isso.db import IssoDBException + + +class TooManyComments(IssoDBException): + pass + + +def check(func): + + def dec(self, uri, c): + + if not self.db.conf.getboolean("guard", "enabled"): + return func(self, uri, c) + + # block more than two comments per minute + rv = self.db.execute([ + 'SELECT id FROM comments WHERE remote_addr = ? AND created > ?;' + ], (c["remote_addr"], time.time() + 60)).fetchall() + + if len(rv) >= 2: + raise TooManyComments + + # block more than three comments as direct response to the post + rv = self.db.execute([ + 'SELECT id FROM comments WHERE remote_addr = ? AND parent IS NULL;' + ], (c["remote_addr"], )).fetchall() + + if len(rv) >= 3: + raise TooManyComments + + # block reply to own comment if the cookie is still available (max age) + if "parent" in c: + rv = self.db.execute([ + 'SELECT id FROM comments WHERE remote_addr = ? AND id = ? AND ? - created < ?;' + ], (c["remote_addr"], c["parent"], time.time(), + self.db.conf.getint("general", "max-age"))).fetchall() + + if len(rv) > 0: + raise TooManyComments + + return func(self, uri, c) + + return dec diff --git a/isso/views/comment.py b/isso/views/comment.py index c7ef0bf..384c911 100644 --- a/isso/views/comment.py +++ b/isso/views/comment.py @@ -12,7 +12,7 @@ from itsdangerous import SignatureExpired, BadSignature from werkzeug.wrappers import Response from werkzeug.exceptions import abort, BadRequest -from isso import utils, notify +from isso import utils, notify, db from isso.crypto import pbkdf2 FIELDS = {'id', 'parent', 'text', 'author', 'website', 'email', 'mode', 'created', @@ -73,10 +73,13 @@ def new(app, environ, request, uri): title = app.db.threads[uri].title try: - rv = app.db.comments.add(uri, data) + with app.lock: + rv = app.db.comments.add(uri, data) except sqlite3.Error: logging.exception('uncaught SQLite3 exception') abort(400) + except db.IssoDBException: + abort(403) href = (app.conf.get('general', 'host').rstrip("/") + uri + "#isso-%i" % rv["id"]) app.notify(title, notify.format(rv, href, utils.anonymize(unicode(request.remote_addr)))) @@ -93,7 +96,7 @@ def new(app, environ, request, uri): resp = Response(json.dumps(rv), 202 if rv["mode"] == 2 else 201, content_type='application/json') - resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]), max_age=app.conf.getint('general', 'max_age')) + resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]), max_age=app.conf.getint('general', 'max-age')) return resp @@ -154,7 +157,7 @@ def single(app, environ, request, id): rv["text"] = app.markdown(rv["text"]) resp = Response(json.dumps(rv), 200, content_type='application/json') - resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]), max_age=app.MAX_AGE) + resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]), max_age=app.conf.getint('general', 'max-age')) return resp if request.method == 'DELETE': diff --git a/specs/test_comment.py b/specs/test_comment.py index a613028..cf12993 100644 --- a/specs/test_comment.py +++ b/specs/test_comment.py @@ -10,7 +10,7 @@ import unittest from werkzeug.test import Client from werkzeug.wrappers import Response -from isso import Isso, notify, utils, views +from isso import Isso, notify, utils, views, core from isso.views import comment utils.heading = lambda *args: "Untitled." @@ -31,8 +31,14 @@ class TestComments(unittest.TestCase): def setUp(self): fd, self.path = tempfile.mkstemp() + conf = core.Config.load(None) + conf.set("general", "dbpath", self.path) + conf.set("guard", "enabled", "off") - self.app = Isso(self.path, '...', '...', 15*60, "...", notify.NullMailer()) + class App(Isso, core.Mixin): + pass + + self.app = App(conf) self.app.wsgi_app = FakeIP(self.app.wsgi_app) self.client = Client(self.app, Response) diff --git a/specs/test_vote.py b/specs/test_vote.py index e405a89..2426fe8 100644 --- a/specs/test_vote.py +++ b/specs/test_vote.py @@ -9,7 +9,7 @@ import unittest from werkzeug.test import Client from werkzeug.wrappers import Response -from isso import Isso, notify, utils +from isso import Isso, notify, utils, core utils.heading = lambda *args: "Untitled." utils.urlexists = lambda *args: True @@ -36,7 +36,14 @@ class TestVote(unittest.TestCase): def makeClient(self, ip): - app = Isso(self.path, '...', '...', 15*60, "...", notify.NullMailer()) + conf = core.Config.load(None) + conf.set("general", "dbpath", self.path) + conf.set("guard", "enabled", "off") + + class App(Isso, core.Mixin): + pass + + app = App(conf) app.wsgi_app = FakeIP(app.wsgi_app, ip) return Client(app, Response)