From 6eab8ad5ca4c4ed3d0743d3ca19aab7276d4fa6b Mon Sep 17 00:00:00 2001 From: Martin Zimmermann Date: Sun, 6 Oct 2013 15:09:53 +0200 Subject: [PATCH 1/2] support for uWSGI * naive uWSGI fallback which spawns one thread per request and one thread per mail notification * uWSGI backend which utilize queues and spooling to handle simultanous requests and mail notifications This also fixes a bug where N concurrent POSTs on a new topic failed for N-1 requests (db integrity error). --- isso/__init__.py | 123 ++++++++++++++++++------------------------ isso/core.py | 114 +++++++++++++++++++++++++++++++++++++++ isso/crypto.py | 1 - isso/db/comments.py | 1 - isso/notify.py | 23 +++----- isso/utils.py | 4 +- isso/views/comment.py | 26 +++++---- 7 files changed, 186 insertions(+), 106 deletions(-) create mode 100644 isso/core.py diff --git a/isso/__init__.py b/isso/__init__.py index 8186054..3123b6a 100644 --- a/isso/__init__.py +++ b/isso/__init__.py @@ -31,7 +31,6 @@ import pkg_resources dist = pkg_resources.get_distribution("isso") import sys -import io import os import socket import httplib @@ -39,14 +38,13 @@ import urlparse from os.path import dirname, join from argparse import ArgumentParser -from ConfigParser import ConfigParser import misaka from itsdangerous import URLSafeTimedSerializer from werkzeug.routing import Map, Rule from werkzeug.wrappers import Response, Request -from werkzeug.exceptions import HTTPException, NotFound, InternalServerError, MethodNotAllowed +from werkzeug.exceptions import HTTPException, NotFound, MethodNotAllowed from werkzeug.wsgi import SharedDataMiddleware from werkzeug.serving import run_simple @@ -54,10 +52,11 @@ from werkzeug.contrib.fixers import ProxyFix from jinja2 import Environment, FileSystemLoader -from isso import db, migrate, views, wsgi, notify, colors, utils +from isso import db, migrate, views, wsgi, colors +from isso.core import NaiveMixin, uWSGIMixin, Config from isso.views import comment, admin -url_map = Map([ +rules = Map([ Rule('/new', methods=['POST'], endpoint=views.comment.new), Rule('/id/', methods=['GET', 'PUT', 'DELETE'], endpoint=views.comment.single), @@ -74,20 +73,28 @@ url_map = Map([ class Isso(object): - PRODUCTION = False - SALT = "Eech7co8Ohloopo9Ol6baimi" + salt = "Eech7co8Ohloopo9Ol6baimi" - def __init__(self, dbpath, secret, origin, max_age, passphrase, mailer): + def __init__(self, conf): - self.DBPATH = dbpath - self.ORIGIN = origin - self.PASSPHRASE = passphrase - self.MAX_AGE = max_age + super(Isso, self).__init__(conf) - self.db = db.SQLite3(dbpath) - self.signer = URLSafeTimedSerializer(secret) + if not conf.get("general", "host").startswith(("http://", "https://")): + sys.exit("error: host must start with http:// or https://") + + try: + print(" * connecting to HTTP server", end=" ") + rv = urlparse.urlparse(conf.get("general", "host")) + host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc + httplib.HTTPConnection(host, timeout=5).request('GET', rv.path) + print("[%s]" % colors.green("ok")) + except (httplib.HTTPException, socket.error): + print("[%s]" % colors.red("failed")) + + self.conf = conf + self.db = db.SQLite3(conf.get('general', 'dbpath')) + self.signer = URLSafeTimedSerializer(conf.get('general', 'secretkey')) self.j2env = Environment(loader=FileSystemLoader(join(dirname(__file__), 'templates/'))) - self.notify = lambda *args, **kwargs: mailer.sendmail(*args, **kwargs) def sign(self, obj): return self.signer.dumps(obj) @@ -105,7 +112,7 @@ class Isso(object): return tt.render(**ctx) def dispatch(self, request, start_response): - adapter = url_map.bind_to_environ(request.environ) + adapter = rules.bind_to_environ(request.environ) try: handler, values = adapter.match() return handler(self, request.environ, request, **values) @@ -114,14 +121,12 @@ class Isso(object): except MethodNotAllowed: return Response("Yup.", 200) except HTTPException as e: - return e - except InternalServerError as e: return Response(e, 500) def wsgi_app(self, environ, start_response): response = self.dispatch(Request(environ), start_response) if hasattr(response, 'headers'): - response.headers["Access-Control-Allow-Origin"] = self.ORIGIN.rstrip('/') + response.headers["Access-Control-Allow-Origin"] = self.conf.get('general', 'host').rstrip('/') response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type" response.headers["Access-Control-Allow-Credentials"] = "true" response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE" @@ -131,6 +136,24 @@ class Isso(object): return self.wsgi_app(environ, start_response) +def make_app(conf=None): + + try: + import uwsgi + except ImportError: + isso = type("Isso", (Isso, NaiveMixin), {})(conf) + else: + isso = type("Isso", (Isso, uWSGIMixin), {})(conf) + + app = ProxyFix(wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, { + '/static': join(dirname(__file__), 'static/'), + '/js': join(dirname(__file__), 'js/'), + '/css': join(dirname(__file__), 'css/') + }))) + + return app + + def main(): parser = ArgumentParser(description="a blog comment hosting service") @@ -145,62 +168,20 @@ def main(): serve = subparser.add_parser("run", help="run server") - defaultcfg = [ - "[general]", - "dbpath = /tmp/isso.db", "secretkey = %r" % os.urandom(24), - "host = http://localhost:8080/", "passphrase = p@$$w0rd", - "max_age = 450", - "[server]", - "host = localhost", "port = 8080", "reload = off", - "[SMTP]", - "username = ", "password = ", - "host = localhost", "port = 465", "ssl = on", - "to = ", "from = " - ] - args = parser.parse_args() - conf = ConfigParser(allow_no_value=True) - conf.readfp(io.StringIO(u'\n'.join(defaultcfg))) - conf.read(args.conf) + conf = Config.load(args.conf) if args.command == "import": - migrate.disqus(db.SQLite3(conf.get("general", "dbpath"), False), args.dump) + migrate.disqus(db.SQLite3(conf.get('general', 'dbpath')), args.dump) sys.exit(0) - if not conf.get("general", "host").startswith(("http://", "https://")): - sys.exit("error: host must start with http:// or https://") + run_simple(conf.get('server', 'host'), conf.getint('server', 'port'), make_app(conf), + threaded=True, use_reloader=conf.getboolean('server', 'reload')) - try: - print(" * connecting to SMTP server", end=" ") - mailer = notify.SMTPMailer(conf) - print("[%s]" % colors.green("ok")) - except (socket.error, notify.SMTPException): - print("[%s]" % colors.red("failed")) - mailer = notify.NullMailer() - try: - print(" * connecting to HTTP server", end=" ") - rv = urlparse.urlparse(conf.get("general", "host")) - host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc - httplib.HTTPConnection(host, timeout=5).request('GET', rv.path) - print("[%s]" % colors.green("ok")) - except (httplib.HTTPException, socket.error): - print("[%s]" % colors.red("failed")) - - isso = Isso( - dbpath=conf.get('general', 'dbpath'), - secret=conf.get('general', 'secretkey'), - origin=conf.get('general', 'host'), - max_age=conf.getint('general', 'max_age'), - passphrase=conf.get('general', 'passphrase'), - mailer=mailer - ) - - app = ProxyFix(wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, { - '/static': join(dirname(__file__), 'static/'), - '/js': join(dirname(__file__), 'js/'), - '/css': join(dirname(__file__), 'css/') - }))) - - run_simple(conf.get('server', 'host'), conf.getint('server', 'port'), - app, threaded=True, use_reloader=conf.getboolean('server', 'reload')) +try: + import uwsgi +except ImportError: + pass +else: + application = make_app(Config.load(os.environ.get('ISSO_SETTINGS'))) diff --git a/isso/core.py b/isso/core.py new file mode 100644 index 0000000..dad20f6 --- /dev/null +++ b/isso/core.py @@ -0,0 +1,114 @@ +# -*- encoding: utf-8 -*- + +from __future__ import print_function + +import io +import os +import time + +import thread +import threading + +import socket +import smtplib + +from ConfigParser import ConfigParser + +try: + import uwsgi +except ImportError: + uwsgi = None + +from isso import notify, colors + +class Config: + + default = [ + "[general]", + "dbpath = /tmp/isso.db", "secretkey = %r" % os.urandom(24), + "host = http://localhost:8080/", "passphrase = p@$$w0rd", + "max_age = 450", + "[server]", + "host = localhost", "port = 8080", "reload = off", + "[SMTP]", + "username = ", "password = ", + "host = localhost", "port = 465", "ssl = on", + "to = ", "from = " + ] + + @classmethod + def load(cls, configfile): + + rv = ConfigParser(allow_no_value=True) + rv.readfp(io.StringIO(u'\n'.join(Config.default))) + + if configfile: + rv.read(configfile) + + return rv + + +def threaded(func): + + def dec(self, *args, **kwargs): + thread.start_new_thread(func, (self, ) + args, kwargs) + + return dec + + +class NaiveMixin(object): + + def __init__(self, conf): + + try: + print(" * connecting to SMTP server", end=" ") + mailer = notify.SMTPMailer(conf) + print("[%s]" % colors.green("ok")) + except (socket.error, smtplib.SMTPException): + print("[%s]" % colors.red("failed")) + mailer = notify.NullMailer() + + self.mailer = mailer + self.lock = threading.Lock() + + @threaded + def notify(self, subject, body, retries=5): + + for x in range(retries): + try: + self.mailer.sendmail(subject, body) + except Exception: + time.sleep(60) + else: + break + + +class uWSGIMixin(NaiveMixin): + + def __init__(self, conf): + super(uWSGIMixin, self).__init__(conf) + + class Lock(): + + def __enter__(self): + while uwsgi.queue_get(0) == "LOCK": + time.sleep(0.1) + + uwsgi.queue_set(uwsgi.queue_slot(), "LOCK") + + def __exit__(self, exc_type, exc_val, exc_tb): + uwsgi.queue_pop() + + def spooler(args): + try: + self.mailer.sendmail(args["subject"].decode('utf-8'), args["body"].decode('utf-8')) + except smtplib.SMTPConnectError: + return uwsgi.SPOOL_RETRY + else: + return uwsgi.SPOOL_OK + + self.lock = Lock() + uwsgi.spooler = spooler + + def notify(self, subject, body, retries=5): + uwsgi.spool({"subject": subject.encode('utf-8'), "body": body.encode('utf-8')}) diff --git a/isso/crypto.py b/isso/crypto.py index 8ebd6f6..1cdb82e 100644 --- a/isso/crypto.py +++ b/isso/crypto.py @@ -4,7 +4,6 @@ # All rights reserved. import hmac -import time import struct import base64 import hashlib diff --git a/isso/db/comments.py b/isso/db/comments.py index 0be5156..95cffae 100644 --- a/isso/db/comments.py +++ b/isso/db/comments.py @@ -1,7 +1,6 @@ # -*- encoding: utf-8 -*- import time -import sqlite3 from isso.utils import Bloomfilter diff --git a/isso/notify.py b/isso/notify.py index 1d97774..a5f4ccc 100644 --- a/isso/notify.py +++ b/isso/notify.py @@ -1,13 +1,10 @@ # -*- encoding: utf-8 -*- -import time -import logging - -from smtplib import SMTP, SMTP_SSL, SMTPException +from smtplib import SMTP, SMTP_SSL from email.mime.text import MIMEText -def create(comment, subject, permalink, remote_addr): +def format(comment, permalink, remote_addr): rv = [] rv.append("%s schrieb:" % (comment["author"] or "Jemand")) @@ -21,7 +18,7 @@ def create(comment, subject, permalink, remote_addr): rv.append("IP Adresse: %s" % remote_addr) rv.append("Link zum Kommentar: %s" % permalink) - return subject, u'\n'.join(rv) + return u'\n'.join(rv) class Connection(object): @@ -54,23 +51,15 @@ class SMTPMailer(object): with Connection(self.conf): pass - def sendmail(self, subject, body, retries=5): + def sendmail(self, subject, body): msg = MIMEText(body, 'plain', 'utf-8') msg['From'] = "Ich schrei sonst! <%s>" % self.from_addr msg['To'] = self.to_addr msg['Subject'] = subject.encode('utf-8') - for i in range(retries): - try: - with Connection(self.conf) as con: - con.sendmail(self.from_addr, self.to_addr, msg.as_string()) - except SMTPException: - logging.exception("uncaught exception, %i of %i:", i + 1, retries) - else: - return - - time.sleep(60) + with Connection(self.conf) as con: + con.sendmail(self.from_addr, self.to_addr, msg.as_string()) class NullMailer(object): diff --git a/isso/utils.py b/isso/utils.py index a571b46..f5e4b44 100644 --- a/isso/utils.py +++ b/isso/utils.py @@ -59,7 +59,7 @@ def heading(host, path): filter(lambda i: i.attributes.has_key("id"), html.getElementsByTagName("div")))) if not el: - return None + return "Untitled" el = el[0] visited = [] @@ -91,7 +91,7 @@ def heading(host, path): el = el.parentNode - return None + return "Untitled." def anonymize(remote_addr): diff --git a/isso/views/comment.py b/isso/views/comment.py index 00b2043..c7ef0bf 100644 --- a/isso/views/comment.py +++ b/isso/views/comment.py @@ -3,10 +3,9 @@ import cgi import json import time -import thread import hashlib -import sqlite3 import logging +import sqlite3 from itsdangerous import SignatureExpired, BadSignature @@ -16,8 +15,8 @@ from werkzeug.exceptions import abort, BadRequest from isso import utils, notify from isso.crypto import pbkdf2 -FIELDS = set(['id', 'parent', 'text', 'author', 'website', 'email', 'mode', - 'created', 'modified', 'likes', 'dislikes', 'hash']) +FIELDS = {'id', 'parent', 'text', 'author', 'website', 'email', 'mode', 'created', + 'modified', 'likes', 'dislikes', 'hash'} class requires: @@ -45,7 +44,7 @@ class requires: @requires(str, 'uri') def new(app, environ, request, uri): - if uri not in app.db.threads and not utils.urlexists(app.ORIGIN, uri): + if uri not in app.db.threads and not utils.urlexists(app.conf.get('general', 'host'), uri): return Response('URI does not exist', 404) try: @@ -68,8 +67,9 @@ def new(app, environ, request, uri): data['remote_addr'] = utils.anonymize(unicode(request.remote_addr)) - if uri not in app.db.threads: - app.db.threads.new(uri, utils.heading(app.ORIGIN, uri)) + with app.lock: + if uri not in app.db.threads: + app.db.threads.new(uri, utils.heading(app.conf.get('general', 'host'), uri)) title = app.db.threads[uri].title try: @@ -78,24 +78,22 @@ def new(app, environ, request, uri): logging.exception('uncaught SQLite3 exception') abort(400) - href = (app.ORIGIN.rstrip("/") + uri + "#isso-%i" % rv["id"]) - thread.start_new_thread( - app.notify, - notify.create(rv, title, href, utils.anonymize(unicode(request.remote_addr)))) + href = (app.conf.get('general', 'host').rstrip("/") + uri + "#isso-%i" % rv["id"]) + app.notify(title, notify.format(rv, href, utils.anonymize(unicode(request.remote_addr)))) # save checksum of text into cookie, so mallory can't modify/delete a comment, if # he add a comment, then removed it but not the signed cookie. checksum = hashlib.md5(rv["text"].encode('utf-8')).hexdigest() rv["text"] = app.markdown(rv["text"]) - rv["hash"] = pbkdf2(rv.get('email') or rv['remote_addr'], app.SALT, 1000, 6) + rv["hash"] = pbkdf2(rv.get('email') or rv['remote_addr'], app.salt, 1000, 6) for key in set(rv.keys()) - FIELDS: rv.pop(key) resp = Response(json.dumps(rv), 202 if rv["mode"] == 2 else 201, content_type='application/json') - resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]), max_age=app.MAX_AGE) + resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]), max_age=app.conf.getint('general', 'max_age')) return resp @@ -180,7 +178,7 @@ def fetch(app, environ, request, uri): for item in rv: - item['hash'] = pbkdf2(item['email'] or item['remote_addr'], app.SALT, 1000, 6) + item['hash'] = pbkdf2(item['email'] or item['remote_addr'], app.salt, 1000, 6) for key in set(item.keys()) - FIELDS: item.pop(key) From 59353c43f93e9454dd52bad05b0b0a0b24968af0 Mon Sep 17 00:00:00 2001 From: Martin Zimmermann Date: Sun, 6 Oct 2013 15:19:53 +0200 Subject: [PATCH 2/2] add uWSGI development ini $ mkdir mail $ uwsgi isso.ini --- isso.ini | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 isso.ini diff --git a/isso.ini b/isso.ini new file mode 100644 index 0000000..fcdf194 --- /dev/null +++ b/isso.ini @@ -0,0 +1,9 @@ +[uwsgi] +http = :8080 +master = true +processes = 4 +spooler = %v/mail +queue = 1 +module = isso +virtualenv = . +env = ISSO_SETTINGS=%v/sample.cfg