support for uWSGI

* naive uWSGI fallback which spawns one thread per request and
  one thread per mail notification
* uWSGI backend which utilize queues and spooling to handle
  simultanous requests and mail notifications

This also fixes a bug where N concurrent POSTs on a new topic
failed for N-1 requests (db integrity error).

isso/__init__.py

@@ -31,7 +31,6 @@ import pkg_resources
 dist = pkg_resources.get_distribution("isso")
 
 import sys
-import io
 import os
 import socket
 import httplib
@@ -39,14 +38,13 @@ import urlparse
 
 from os.path import dirname, join
 from argparse import ArgumentParser
-from ConfigParser import ConfigParser
 
 import misaka
 from itsdangerous import URLSafeTimedSerializer
 
 from werkzeug.routing import Map, Rule
 from werkzeug.wrappers import Response, Request
-from werkzeug.exceptions import HTTPException, NotFound, InternalServerError, MethodNotAllowed
+from werkzeug.exceptions import HTTPException, NotFound, MethodNotAllowed
 
 from werkzeug.wsgi import SharedDataMiddleware
 from werkzeug.serving import run_simple
@@ -54,10 +52,11 @@ from werkzeug.contrib.fixers import ProxyFix
 
 from jinja2 import Environment, FileSystemLoader
 
-from isso import db, migrate, views, wsgi, notify, colors, utils
+from isso import db, migrate, views, wsgi, colors
+from isso.core import NaiveMixin, uWSGIMixin, Config
 from isso.views import comment, admin
 
-url_map = Map([
+rules = Map([
     Rule('/new', methods=['POST'], endpoint=views.comment.new),
 
     Rule('/id/<int:id>', methods=['GET', 'PUT', 'DELETE'], endpoint=views.comment.single),
@@ -74,20 +73,28 @@ url_map = Map([
 
 class Isso(object):
 
-    PRODUCTION = False
+    salt = "Eech7co8Ohloopo9Ol6baimi"
-    SALT = "Eech7co8Ohloopo9Ol6baimi"
 
-    def __init__(self, dbpath, secret, origin, max_age, passphrase, mailer):
+    def __init__(self, conf):
 
-        self.DBPATH = dbpath
+        super(Isso, self).__init__(conf)
-        self.ORIGIN = origin
-        self.PASSPHRASE = passphrase
-        self.MAX_AGE = max_age
 
-        self.db = db.SQLite3(dbpath)
+        if not conf.get("general", "host").startswith(("http://", "https://")):
-        self.signer = URLSafeTimedSerializer(secret)
+            sys.exit("error: host must start with http:// or https://")
+
+        try:
+             print(" * connecting to HTTP server", end=" ")
+             rv = urlparse.urlparse(conf.get("general", "host"))
+             host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc
+             httplib.HTTPConnection(host, timeout=5).request('GET', rv.path)
+             print("[%s]" % colors.green("ok"))
+        except (httplib.HTTPException, socket.error):
+             print("[%s]" % colors.red("failed"))
+
+        self.conf = conf
+        self.db = db.SQLite3(conf.get('general', 'dbpath'))
+        self.signer = URLSafeTimedSerializer(conf.get('general', 'secretkey'))
         self.j2env = Environment(loader=FileSystemLoader(join(dirname(__file__), 'templates/')))
-        self.notify = lambda *args, **kwargs: mailer.sendmail(*args, **kwargs)
 
     def sign(self, obj):
         return self.signer.dumps(obj)
@@ -105,7 +112,7 @@ class Isso(object):
         return tt.render(**ctx)
 
     def dispatch(self, request, start_response):
-        adapter = url_map.bind_to_environ(request.environ)
+        adapter = rules.bind_to_environ(request.environ)
         try:
             handler, values = adapter.match()
             return handler(self, request.environ, request, **values)
@@ -114,14 +121,12 @@ class Isso(object):
         except MethodNotAllowed:
             return Response("Yup.", 200)
         except HTTPException as e:
-            return e
-        except InternalServerError as e:
             return Response(e, 500)
 
     def wsgi_app(self, environ, start_response):
         response = self.dispatch(Request(environ), start_response)
         if hasattr(response, 'headers'):
-            response.headers["Access-Control-Allow-Origin"] = self.ORIGIN.rstrip('/')
+            response.headers["Access-Control-Allow-Origin"] = self.conf.get('general', 'host').rstrip('/')
             response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type"
             response.headers["Access-Control-Allow-Credentials"] = "true"
             response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE"
@@ -131,6 +136,24 @@ class Isso(object):
         return self.wsgi_app(environ, start_response)
 
 
+def make_app(conf=None):
+
+    try:
+        import uwsgi
+    except ImportError:
+        isso = type("Isso", (Isso, NaiveMixin), {})(conf)
+    else:
+        isso = type("Isso", (Isso, uWSGIMixin), {})(conf)
+
+    app = ProxyFix(wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, {
+        '/static': join(dirname(__file__), 'static/'),
+        '/js': join(dirname(__file__), 'js/'),
+        '/css': join(dirname(__file__), 'css/')
+        })))
+
+    return app
+
+
 def main():
 
     parser = ArgumentParser(description="a blog comment hosting service")
@@ -145,62 +168,20 @@ def main():
 
     serve = subparser.add_parser("run", help="run server")
 
-    defaultcfg = [
-        "[general]",
-        "dbpath = /tmp/isso.db", "secretkey = %r" % os.urandom(24),
-        "host = http://localhost:8080/", "passphrase = p@$$w0rd",
-        "max_age = 450",
-        "[server]",
-        "host = localhost", "port = 8080", "reload = off",
-        "[SMTP]",
-        "username = ", "password = ",
-        "host = localhost", "port = 465", "ssl = on",
-        "to = ", "from = "
-    ]
-
     args = parser.parse_args()
-    conf = ConfigParser(allow_no_value=True)
+    conf = Config.load(args.conf)
-    conf.readfp(io.StringIO(u'\n'.join(defaultcfg)))
-    conf.read(args.conf)
 
     if args.command == "import":
-        migrate.disqus(db.SQLite3(conf.get("general", "dbpath"), False), args.dump)
+        migrate.disqus(db.SQLite3(conf.get('general', 'dbpath')), args.dump)
         sys.exit(0)
 
-    if not conf.get("general", "host").startswith(("http://", "https://")):
+    run_simple(conf.get('server', 'host'), conf.getint('server', 'port'), make_app(conf),
-        sys.exit("error: host must start with http:// or https://")
+               threaded=True, use_reloader=conf.getboolean('server', 'reload'))
 
-    try:
-        print(" * connecting to SMTP server", end=" ")
-        mailer = notify.SMTPMailer(conf)
-        print("[%s]" % colors.green("ok"))
-    except (socket.error, notify.SMTPException):
-        print("[%s]" % colors.red("failed"))
-        mailer = notify.NullMailer()
 
-    try:
+try:
-        print(" * connecting to HTTP server", end=" ")
+    import uwsgi
-        rv = urlparse.urlparse(conf.get("general", "host"))
+except ImportError:
-        host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc
+    pass
-        httplib.HTTPConnection(host, timeout=5).request('GET', rv.path)
+else:
-        print("[%s]" % colors.green("ok"))
+    application = make_app(Config.load(os.environ.get('ISSO_SETTINGS')))
-    except (httplib.HTTPException, socket.error):
-        print("[%s]" % colors.red("failed"))
-
-    isso = Isso(
-        dbpath=conf.get('general', 'dbpath'),
-        secret=conf.get('general', 'secretkey'),
-        origin=conf.get('general', 'host'),
-        max_age=conf.getint('general', 'max_age'),
-        passphrase=conf.get('general', 'passphrase'),
-        mailer=mailer
-    )
-
-    app = ProxyFix(wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, {
-        '/static': join(dirname(__file__), 'static/'),
-        '/js': join(dirname(__file__), 'js/'),
-        '/css': join(dirname(__file__), 'css/')
-        })))
-
-    run_simple(conf.get('server', 'host'), conf.getint('server', 'port'),
-        app, threaded=True, use_reloader=conf.getboolean('server', 'reload'))

isso/core.py

@@ -0,0 +1,114 @@
+# -*- encoding: utf-8 -*-
+
+from __future__ import print_function
+
+import io
+import os
+import time
+
+import thread
+import threading
+
+import socket
+import smtplib
+
+from ConfigParser import ConfigParser
+
+try:
+    import uwsgi
+except ImportError:
+    uwsgi = None
+
+from isso import notify, colors
+
+class Config:
+
+    default = [
+        "[general]",
+        "dbpath = /tmp/isso.db", "secretkey = %r" % os.urandom(24),
+        "host = http://localhost:8080/", "passphrase = p@$$w0rd",
+        "max_age = 450",
+        "[server]",
+        "host = localhost", "port = 8080", "reload = off",
+        "[SMTP]",
+        "username = ", "password = ",
+        "host = localhost", "port = 465", "ssl = on",
+        "to = ", "from = "
+    ]
+
+    @classmethod
+    def load(cls, configfile):
+
+        rv = ConfigParser(allow_no_value=True)
+        rv.readfp(io.StringIO(u'\n'.join(Config.default)))
+
+        if configfile:
+            rv.read(configfile)
+
+        return rv
+
+
+def threaded(func):
+
+    def dec(self, *args, **kwargs):
+        thread.start_new_thread(func, (self, ) + args, kwargs)
+
+    return dec
+
+
+class NaiveMixin(object):
+
+    def __init__(self, conf):
+
+        try:
+             print(" * connecting to SMTP server", end=" ")
+             mailer = notify.SMTPMailer(conf)
+             print("[%s]" % colors.green("ok"))
+        except (socket.error, smtplib.SMTPException):
+             print("[%s]" % colors.red("failed"))
+             mailer = notify.NullMailer()
+
+        self.mailer = mailer
+        self.lock = threading.Lock()
+
+    @threaded
+    def notify(self, subject, body, retries=5):
+
+        for x in range(retries):
+            try:
+                self.mailer.sendmail(subject, body)
+            except Exception:
+                time.sleep(60)
+            else:
+                break
+
+
+class uWSGIMixin(NaiveMixin):
+
+    def __init__(self, conf):
+        super(uWSGIMixin, self).__init__(conf)
+
+        class Lock():
+
+            def __enter__(self):
+                while uwsgi.queue_get(0) == "LOCK":
+                    time.sleep(0.1)
+
+                uwsgi.queue_set(uwsgi.queue_slot(), "LOCK")
+
+            def __exit__(self, exc_type, exc_val, exc_tb):
+                uwsgi.queue_pop()
+
+        def spooler(args):
+            try:
+                self.mailer.sendmail(args["subject"].decode('utf-8'), args["body"].decode('utf-8'))
+            except smtplib.SMTPConnectError:
+                return uwsgi.SPOOL_RETRY
+            else:
+                return uwsgi.SPOOL_OK
+
+        self.lock = Lock()
+        uwsgi.spooler = spooler
+
+    def notify(self, subject, body, retries=5):
+        uwsgi.spool({"subject": subject.encode('utf-8'), "body": body.encode('utf-8')})

isso/crypto.py

@@ -4,7 +4,6 @@
 # All rights reserved.
 
 import hmac
-import time
 import struct
 import base64
 import hashlib

isso/db/comments.py

@@ -1,7 +1,6 @@
 # -*- encoding: utf-8 -*-
 
 import time
-import sqlite3
 
 from isso.utils import Bloomfilter
 

isso/notify.py

@@ -1,13 +1,10 @@
 # -*- encoding: utf-8 -*-
 
-import time
+from smtplib import SMTP, SMTP_SSL
-import logging
-
-from smtplib import SMTP, SMTP_SSL, SMTPException
 from email.mime.text import MIMEText
 
 
-def create(comment, subject, permalink, remote_addr):
+def format(comment, permalink, remote_addr):
 
     rv = []
     rv.append("%s schrieb:" % (comment["author"] or "Jemand"))
@@ -21,7 +18,7 @@ def create(comment, subject, permalink, remote_addr):
     rv.append("IP Adresse: %s" % remote_addr)
     rv.append("Link zum Kommentar: %s" % permalink)
 
-    return subject, u'\n'.join(rv)
+    return u'\n'.join(rv)
 
 
 class Connection(object):
@@ -54,23 +51,15 @@ class SMTPMailer(object):
         with Connection(self.conf):
             pass
 
-    def sendmail(self, subject, body, retries=5):
+    def sendmail(self, subject, body):
 
         msg = MIMEText(body, 'plain', 'utf-8')
         msg['From'] = "Ich schrei sonst! <%s>" % self.from_addr
         msg['To'] = self.to_addr
         msg['Subject'] = subject.encode('utf-8')
 
-        for i in range(retries):
-            try:
         with Connection(self.conf) as con:
             con.sendmail(self.from_addr, self.to_addr, msg.as_string())
-            except SMTPException:
-                logging.exception("uncaught exception, %i of %i:", i + 1, retries)
-            else:
-                return
-
-            time.sleep(60)
 
 
 class NullMailer(object):

isso/utils.py

@@ -59,7 +59,7 @@ def heading(host, path):
               filter(lambda i: i.attributes.has_key("id"), html.getElementsByTagName("div"))))
 
     if not el:
-        return None
+        return "Untitled"
 
     el = el[0]
     visited = []
@@ -91,7 +91,7 @@ def heading(host, path):
 
         el = el.parentNode
 
-    return None
+    return "Untitled."
 
 
 def anonymize(remote_addr):

isso/views/comment.py

@@ -3,10 +3,9 @@
 import cgi
 import json
 import time
-import thread
 import hashlib
-import sqlite3
 import logging
+import sqlite3
 
 from itsdangerous import SignatureExpired, BadSignature
 
@@ -16,8 +15,8 @@ from werkzeug.exceptions import abort, BadRequest
 from isso import utils, notify
 from isso.crypto import pbkdf2
 
-FIELDS = set(['id', 'parent', 'text', 'author', 'website', 'email', 'mode',
+FIELDS = {'id', 'parent', 'text', 'author', 'website', 'email', 'mode', 'created',
-              'created', 'modified', 'likes', 'dislikes', 'hash'])
+          'modified', 'likes', 'dislikes', 'hash'}
 
 
 class requires:
@@ -45,7 +44,7 @@ class requires:
 @requires(str, 'uri')
 def new(app, environ, request, uri):
 
-    if uri not in app.db.threads and not utils.urlexists(app.ORIGIN, uri):
+    if uri not in app.db.threads and not utils.urlexists(app.conf.get('general', 'host'), uri):
         return Response('URI does not exist', 404)
 
     try:
@@ -68,8 +67,9 @@ def new(app, environ, request, uri):
 
     data['remote_addr'] = utils.anonymize(unicode(request.remote_addr))
 
+    with app.lock:
         if uri not in app.db.threads:
-        app.db.threads.new(uri, utils.heading(app.ORIGIN, uri))
+            app.db.threads.new(uri, utils.heading(app.conf.get('general', 'host'), uri))
     title = app.db.threads[uri].title
 
     try:
@@ -78,24 +78,22 @@ def new(app, environ, request, uri):
         logging.exception('uncaught SQLite3 exception')
         abort(400)
 
-    href = (app.ORIGIN.rstrip("/") + uri + "#isso-%i" % rv["id"])
+    href = (app.conf.get('general', 'host').rstrip("/") + uri + "#isso-%i" % rv["id"])
-    thread.start_new_thread(
+    app.notify(title, notify.format(rv, href, utils.anonymize(unicode(request.remote_addr))))
-        app.notify,
-        notify.create(rv, title, href, utils.anonymize(unicode(request.remote_addr))))
 
     # save checksum of text into cookie, so mallory can't modify/delete a comment, if
     # he add a comment, then removed it but not the signed cookie.
     checksum = hashlib.md5(rv["text"].encode('utf-8')).hexdigest()
 
     rv["text"] = app.markdown(rv["text"])
-    rv["hash"] = pbkdf2(rv.get('email') or rv['remote_addr'], app.SALT, 1000, 6)
+    rv["hash"] = pbkdf2(rv.get('email') or rv['remote_addr'], app.salt, 1000, 6)
 
     for key in set(rv.keys()) - FIELDS:
         rv.pop(key)
 
     resp = Response(json.dumps(rv), 202 if rv["mode"] == 2 else 201,
         content_type='application/json')
-    resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]), max_age=app.MAX_AGE)
+    resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]), max_age=app.conf.getint('general', 'max_age'))
     return resp
 
 
@@ -180,7 +178,7 @@ def fetch(app, environ, request, uri):
 
     for item in rv:
 
-        item['hash'] = pbkdf2(item['email'] or item['remote_addr'], app.SALT, 1000, 6)
+        item['hash'] = pbkdf2(item['email'] or item['remote_addr'], app.salt, 1000, 6)
 
         for key in set(item.keys()) - FIELDS:
             item.pop(key)