arno/isso
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'feature/uwsgi'

Browse Source
This commit is contained in:
Martin Zimmermann 2013-10-06 15:42:42 +02:00
commit 382dd3487e
8 changed files with 195 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
isso.ini Normal file
View File

@ -0,0 +1,9 @@
[uwsgi]
http = :8080
master = true
processes = 4
spooler = %v/mail
queue = 1
module = isso
virtualenv = .
env = ISSO_SETTINGS=%v/sample.cfg

123
isso/__init__.py
View File

@ -31,7 +31,6 @@ import pkg_resources
dist = pkg_resources.get_distribution("isso")
import sys
import io
import os
import socket
import httplib
@ -39,14 +38,13 @@ import urlparse
from os.path import dirname, join
from argparse import ArgumentParser
from ConfigParser import ConfigParser
import misaka
from itsdangerous import URLSafeTimedSerializer
from werkzeug.routing import Map, Rule
from werkzeug.wrappers import Response, Request
from werkzeug.exceptions import HTTPException, NotFound, InternalServerError, MethodNotAllowed
from werkzeug.exceptions import HTTPException, NotFound, MethodNotAllowed
from werkzeug.wsgi import SharedDataMiddleware
from werkzeug.serving import run_simple
@ -54,10 +52,11 @@ from werkzeug.contrib.fixers import ProxyFix
from jinja2 import Environment, FileSystemLoader
from isso import db, migrate, views, wsgi, notify, colors, utils
from isso import db, migrate, views, wsgi, colors
from isso.core import NaiveMixin, uWSGIMixin, Config
from isso.views import comment, admin
url_map = Map([
rules = Map([
Rule('/new', methods=['POST'], endpoint=views.comment.new),
Rule('/id/<int:id>', methods=['GET', 'PUT', 'DELETE'], endpoint=views.comment.single),
@ -74,20 +73,28 @@ url_map = Map([
class Isso(object):
PRODUCTION = False
SALT = "Eech7co8Ohloopo9Ol6baimi"
salt = "Eech7co8Ohloopo9Ol6baimi"
def __init__(self, dbpath, secret, origin, max_age, passphrase, mailer):
def __init__(self, conf):
self.DBPATH = dbpath
self.ORIGIN = origin
self.PASSPHRASE = passphrase
self.MAX_AGE = max_age
super(Isso, self).__init__(conf)
self.db = db.SQLite3(dbpath)
self.signer = URLSafeTimedSerializer(secret)
if not conf.get("general", "host").startswith(("http://", "https://")):
sys.exit("error: host must start with http:// or https://")
try:
print(" * connecting to HTTP server", end=" ")
rv = urlparse.urlparse(conf.get("general", "host"))
host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc
httplib.HTTPConnection(host, timeout=5).request('GET', rv.path)
print("[%s]" % colors.green("ok"))
except (httplib.HTTPException, socket.error):
print("[%s]" % colors.red("failed"))
self.conf = conf
self.db = db.SQLite3(conf.get('general', 'dbpath'))
self.signer = URLSafeTimedSerializer(conf.get('general', 'secretkey'))
self.j2env = Environment(loader=FileSystemLoader(join(dirname(__file__), 'templates/')))
self.notify = lambda *args, **kwargs: mailer.sendmail(*args, **kwargs)
def sign(self, obj):
return self.signer.dumps(obj)
@ -105,7 +112,7 @@ class Isso(object):
return tt.render(**ctx)
def dispatch(self, request, start_response):
adapter = url_map.bind_to_environ(request.environ)
adapter = rules.bind_to_environ(request.environ)
try:
handler, values = adapter.match()
return handler(self, request.environ, request, **values)
@ -114,14 +121,12 @@ class Isso(object):
except MethodNotAllowed:
return Response("Yup.", 200)
except HTTPException as e:
return e
except InternalServerError as e:
return Response(e, 500)
def wsgi_app(self, environ, start_response):
response = self.dispatch(Request(environ), start_response)
if hasattr(response, 'headers'):
response.headers["Access-Control-Allow-Origin"] = self.ORIGIN.rstrip('/')
response.headers["Access-Control-Allow-Origin"] = self.conf.get('general', 'host').rstrip('/')
response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type"
response.headers["Access-Control-Allow-Credentials"] = "true"
response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE"
@ -131,6 +136,24 @@ class Isso(object):
return self.wsgi_app(environ, start_response)
def make_app(conf=None):
try:
import uwsgi
except ImportError:
isso = type("Isso", (Isso, NaiveMixin), {})(conf)
else:
isso = type("Isso", (Isso, uWSGIMixin), {})(conf)
app = ProxyFix(wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, {
'/static': join(dirname(__file__), 'static/'),
'/js': join(dirname(__file__), 'js/'),
'/css': join(dirname(__file__), 'css/')
})))
return app
def main():
parser = ArgumentParser(description="a blog comment hosting service")
@ -145,62 +168,20 @@ def main():
serve = subparser.add_parser("run", help="run server")
defaultcfg = [
"[general]",
"dbpath = /tmp/isso.db", "secretkey = %r" % os.urandom(24),
"host = http://localhost:8080/", "passphrase = p@$$w0rd",
"max_age = 450",
"[server]",
"host = localhost", "port = 8080", "reload = off",
"[SMTP]",
"username = ", "password = ",
"host = localhost", "port = 465", "ssl = on",
"to = ", "from = "
]
args = parser.parse_args()
conf = ConfigParser(allow_no_value=True)
conf.readfp(io.StringIO(u'\n'.join(defaultcfg)))
conf.read(args.conf)
conf = Config.load(args.conf)
if args.command == "import":
migrate.disqus(db.SQLite3(conf.get("general", "dbpath"), False), args.dump)
migrate.disqus(db.SQLite3(conf.get('general', 'dbpath')), args.dump)
sys.exit(0)
if not conf.get("general", "host").startswith(("http://", "https://")):
sys.exit("error: host must start with http:// or https://")
run_simple(conf.get('server', 'host'), conf.getint('server', 'port'), make_app(conf),
threaded=True, use_reloader=conf.getboolean('server', 'reload'))
try:
print(" * connecting to SMTP server", end=" ")
mailer = notify.SMTPMailer(conf)
print("[%s]" % colors.green("ok"))
except (socket.error, notify.SMTPException):
print("[%s]" % colors.red("failed"))
mailer = notify.NullMailer()
try:
print(" * connecting to HTTP server", end=" ")
rv = urlparse.urlparse(conf.get("general", "host"))
host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc
httplib.HTTPConnection(host, timeout=5).request('GET', rv.path)
print("[%s]" % colors.green("ok"))
except (httplib.HTTPException, socket.error):
print("[%s]" % colors.red("failed"))
isso = Isso(
dbpath=conf.get('general', 'dbpath'),
secret=conf.get('general', 'secretkey'),
origin=conf.get('general', 'host'),
max_age=conf.getint('general', 'max_age'),
passphrase=conf.get('general', 'passphrase'),
mailer=mailer
)
app = ProxyFix(wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, {
'/static': join(dirname(__file__), 'static/'),
'/js': join(dirname(__file__), 'js/'),
'/css': join(dirname(__file__), 'css/')
})))
run_simple(conf.get('server', 'host'), conf.getint('server', 'port'),
app, threaded=True, use_reloader=conf.getboolean('server', 'reload'))
import uwsgi
except ImportError:
pass
else:
application = make_app(Config.load(os.environ.get('ISSO_SETTINGS')))

114
isso/core.py Normal file
View File

@ -0,0 +1,114 @@
# -*- encoding: utf-8 -*-
from __future__ import print_function
import io
import os
import time
import thread
import threading
import socket
import smtplib
from ConfigParser import ConfigParser
try:
import uwsgi
except ImportError:
uwsgi = None
from isso import notify, colors
class Config:
default = [
"[general]",
"dbpath = /tmp/isso.db", "secretkey = %r" % os.urandom(24),
"host = http://localhost:8080/", "passphrase = p@$$w0rd",
"max_age = 450",
"[server]",
"host = localhost", "port = 8080", "reload = off",
"[SMTP]",
"username = ", "password = ",
"host = localhost", "port = 465", "ssl = on",
"to = ", "from = "
]
@classmethod
def load(cls, configfile):
rv = ConfigParser(allow_no_value=True)
rv.readfp(io.StringIO(u'\n'.join(Config.default)))
if configfile:
rv.read(configfile)
return rv
def threaded(func):
def dec(self, *args, **kwargs):
thread.start_new_thread(func, (self, ) + args, kwargs)
return dec
class NaiveMixin(object):
def __init__(self, conf):
try:
print(" * connecting to SMTP server", end=" ")
mailer = notify.SMTPMailer(conf)
print("[%s]" % colors.green("ok"))
except (socket.error, smtplib.SMTPException):
print("[%s]" % colors.red("failed"))
mailer = notify.NullMailer()
self.mailer = mailer
self.lock = threading.Lock()
@threaded
def notify(self, subject, body, retries=5):
for x in range(retries):
try:
self.mailer.sendmail(subject, body)
except Exception:
time.sleep(60)
else:
break
class uWSGIMixin(NaiveMixin):
def __init__(self, conf):
super(uWSGIMixin, self).__init__(conf)
class Lock():
def __enter__(self):
while uwsgi.queue_get(0) == "LOCK":
time.sleep(0.1)
uwsgi.queue_set(uwsgi.queue_slot(), "LOCK")
def __exit__(self, exc_type, exc_val, exc_tb):
uwsgi.queue_pop()
def spooler(args):
try:
self.mailer.sendmail(args["subject"].decode('utf-8'), args["body"].decode('utf-8'))
except smtplib.SMTPConnectError:
return uwsgi.SPOOL_RETRY
else:
return uwsgi.SPOOL_OK
self.lock = Lock()
uwsgi.spooler = spooler
def notify(self, subject, body, retries=5):
uwsgi.spool({"subject": subject.encode('utf-8'), "body": body.encode('utf-8')})

1
isso/crypto.py
View File

@ -4,7 +4,6 @@
# All rights reserved.
import hmac
import time
import struct
import base64
import hashlib

1
isso/db/comments.py
View File

@ -1,7 +1,6 @@
# -*- encoding: utf-8 -*-
import time
import sqlite3
from isso.utils import Bloomfilter

19
isso/notify.py
View File

@ -1,13 +1,10 @@
# -*- encoding: utf-8 -*-
import time
import logging
from smtplib import SMTP, SMTP_SSL, SMTPException
from smtplib import SMTP, SMTP_SSL
from email.mime.text import MIMEText
def create(comment, subject, permalink, remote_addr):
def format(comment, permalink, remote_addr):
rv = []
rv.append("%s schrieb:" % (comment["author"] or "Jemand"))
@ -21,7 +18,7 @@ def create(comment, subject, permalink, remote_addr):
rv.append("IP Adresse: %s" % remote_addr)
rv.append("Link zum Kommentar: %s" % permalink)
return subject, u'\n'.join(rv)
return u'\n'.join(rv)
class Connection(object):
@ -54,23 +51,15 @@ class SMTPMailer(object):
with Connection(self.conf):
pass
def sendmail(self, subject, body, retries=5):
def sendmail(self, subject, body):
msg = MIMEText(body, 'plain', 'utf-8')
msg['From'] = "Ich schrei sonst! <%s>" % self.from_addr
msg['To'] = self.to_addr
msg['Subject'] = subject.encode('utf-8')
for i in range(retries):
try:
with Connection(self.conf) as con:
con.sendmail(self.from_addr, self.to_addr, msg.as_string())
except SMTPException:
logging.exception("uncaught exception, %i of %i:", i + 1, retries)
else:
return
time.sleep(60)
class NullMailer(object):

4
isso/utils.py
View File

@ -59,7 +59,7 @@ def heading(host, path):
filter(lambda i: i.attributes.has_key("id"), html.getElementsByTagName("div"))))
if not el:
return None
return "Untitled"
el = el[0]
visited = []
@ -91,7 +91,7 @@ def heading(host, path):
el = el.parentNode
return None
return "Untitled."
def anonymize(remote_addr):

24
isso/views/comment.py
View File

@ -3,10 +3,9 @@
import cgi
import json
import time
import thread
import hashlib
import sqlite3
import logging
import sqlite3
from itsdangerous import SignatureExpired, BadSignature
@ -16,8 +15,8 @@ from werkzeug.exceptions import abort, BadRequest
from isso import utils, notify
from isso.crypto import pbkdf2
FIELDS = set(['id', 'parent', 'text', 'author', 'website', 'email', 'mode',
'created', 'modified', 'likes', 'dislikes', 'hash'])
FIELDS = {'id', 'parent', 'text', 'author', 'website', 'email', 'mode', 'created',
'modified', 'likes', 'dislikes', 'hash'}
class requires:
@ -45,7 +44,7 @@ class requires:
@requires(str, 'uri')
def new(app, environ, request, uri):
if uri not in app.db.threads and not utils.urlexists(app.ORIGIN, uri):
if uri not in app.db.threads and not utils.urlexists(app.conf.get('general', 'host'), uri):
return Response('URI does not exist', 404)
try:
@ -68,8 +67,9 @@ def new(app, environ, request, uri):
data['remote_addr'] = utils.anonymize(unicode(request.remote_addr))
with app.lock:
if uri not in app.db.threads:
app.db.threads.new(uri, utils.heading(app.ORIGIN, uri))
app.db.threads.new(uri, utils.heading(app.conf.get('general', 'host'), uri))
title = app.db.threads[uri].title
try:
@ -78,24 +78,22 @@ def new(app, environ, request, uri):
logging.exception('uncaught SQLite3 exception')
abort(400)
href = (app.ORIGIN.rstrip("/") + uri + "#isso-%i" % rv["id"])
thread.start_new_thread(
app.notify,
notify.create(rv, title, href, utils.anonymize(unicode(request.remote_addr))))
href = (app.conf.get('general', 'host').rstrip("/") + uri + "#isso-%i" % rv["id"])
app.notify(title, notify.format(rv, href, utils.anonymize(unicode(request.remote_addr))))
# save checksum of text into cookie, so mallory can't modify/delete a comment, if
# he add a comment, then removed it but not the signed cookie.
checksum = hashlib.md5(rv["text"].encode('utf-8')).hexdigest()
rv["text"] = app.markdown(rv["text"])
rv["hash"] = pbkdf2(rv.get('email') or rv['remote_addr'], app.SALT, 1000, 6)
rv["hash"] = pbkdf2(rv.get('email') or rv['remote_addr'], app.salt, 1000, 6)
for key in set(rv.keys()) - FIELDS:
rv.pop(key)
resp = Response(json.dumps(rv), 202 if rv["mode"] == 2 else 201,
content_type='application/json')
resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]), max_age=app.MAX_AGE)
resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]), max_age=app.conf.getint('general', 'max_age'))
return resp
@ -180,7 +178,7 @@ def fetch(app, environ, request, uri):
for item in rv:
item['hash'] = pbkdf2(item['email'] or item['remote_addr'], app.SALT, 1000, 6)
item['hash'] = pbkdf2(item['email'] or item['remote_addr'], app.salt, 1000, 6)
for key in set(item.keys()) - FIELDS:
item.pop(key)