Merge branch 'feature/uwsgi'
This commit is contained in:
Martin Zimmermann
commit
382dd3487e
9
isso.ini
Normal file
9
isso.ini
Normal file
@ -0,0 +1,9 @@
|
||||
[uwsgi]
|
||||
http = :8080
|
||||
master = true
|
||||
processes = 4
|
||||
spooler = %v/mail
|
||||
queue = 1
|
||||
module = isso
|
||||
virtualenv = .
|
||||
env = ISSO_SETTINGS=%v/sample.cfg
|
||||
123
isso/__init__.py
123
isso/__init__.py
@ -31,7 +31,6 @@ import pkg_resources
|
||||
dist = pkg_resources.get_distribution("isso")
|
||||
|
||||
import sys
|
||||
import io
|
||||
import os
|
||||
import socket
|
||||
import httplib
|
||||
@ -39,14 +38,13 @@ import urlparse
|
||||
|
||||
from os.path import dirname, join
|
||||
from argparse import ArgumentParser
|
||||
from ConfigParser import ConfigParser
|
||||
|
||||
import misaka
|
||||
from itsdangerous import URLSafeTimedSerializer
|
||||
|
||||
from werkzeug.routing import Map, Rule
|
||||
from werkzeug.wrappers import Response, Request
|
||||
from werkzeug.exceptions import HTTPException, NotFound, InternalServerError, MethodNotAllowed
|
||||
from werkzeug.exceptions import HTTPException, NotFound, MethodNotAllowed
|
||||
|
||||
from werkzeug.wsgi import SharedDataMiddleware
|
||||
from werkzeug.serving import run_simple
|
||||
@ -54,10 +52,11 @@ from werkzeug.contrib.fixers import ProxyFix
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
from isso import db, migrate, views, wsgi, notify, colors, utils
|
||||
from isso import db, migrate, views, wsgi, colors
|
||||
from isso.core import NaiveMixin, uWSGIMixin, Config
|
||||
from isso.views import comment, admin
|
||||
|
||||
url_map = Map([
|
||||
rules = Map([
|
||||
Rule('/new', methods=['POST'], endpoint=views.comment.new),
|
||||
|
||||
Rule('/id/<int:id>', methods=['GET', 'PUT', 'DELETE'], endpoint=views.comment.single),
|
||||
@ -74,20 +73,28 @@ url_map = Map([
|
||||
|
||||
class Isso(object):
|
||||
|
||||
PRODUCTION = False
|
||||
SALT = "Eech7co8Ohloopo9Ol6baimi"
|
||||
salt = "Eech7co8Ohloopo9Ol6baimi"
|
||||
|
||||
def __init__(self, dbpath, secret, origin, max_age, passphrase, mailer):
|
||||
def __init__(self, conf):
|
||||
|
||||
self.DBPATH = dbpath
|
||||
self.ORIGIN = origin
|
||||
self.PASSPHRASE = passphrase
|
||||
self.MAX_AGE = max_age
|
||||
super(Isso, self).__init__(conf)
|
||||
|
||||
self.db = db.SQLite3(dbpath)
|
||||
self.signer = URLSafeTimedSerializer(secret)
|
||||
if not conf.get("general", "host").startswith(("http://", "https://")):
|
||||
sys.exit("error: host must start with http:// or https://")
|
||||
|
||||
try:
|
||||
print(" * connecting to HTTP server", end=" ")
|
||||
rv = urlparse.urlparse(conf.get("general", "host"))
|
||||
host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc
|
||||
httplib.HTTPConnection(host, timeout=5).request('GET', rv.path)
|
||||
print("[%s]" % colors.green("ok"))
|
||||
except (httplib.HTTPException, socket.error):
|
||||
print("[%s]" % colors.red("failed"))
|
||||
|
||||
self.conf = conf
|
||||
self.db = db.SQLite3(conf.get('general', 'dbpath'))
|
||||
self.signer = URLSafeTimedSerializer(conf.get('general', 'secretkey'))
|
||||
self.j2env = Environment(loader=FileSystemLoader(join(dirname(__file__), 'templates/')))
|
||||
self.notify = lambda *args, **kwargs: mailer.sendmail(*args, **kwargs)
|
||||
|
||||
def sign(self, obj):
|
||||
return self.signer.dumps(obj)
|
||||
@ -105,7 +112,7 @@ class Isso(object):
|
||||
return tt.render(**ctx)
|
||||
|
||||
def dispatch(self, request, start_response):
|
||||
adapter = url_map.bind_to_environ(request.environ)
|
||||
adapter = rules.bind_to_environ(request.environ)
|
||||
try:
|
||||
handler, values = adapter.match()
|
||||
return handler(self, request.environ, request, **values)
|
||||
@ -114,14 +121,12 @@ class Isso(object):
|
||||
except MethodNotAllowed:
|
||||
return Response("Yup.", 200)
|
||||
except HTTPException as e:
|
||||
return e
|
||||
except InternalServerError as e:
|
||||
return Response(e, 500)
|
||||
|
||||
def wsgi_app(self, environ, start_response):
|
||||
response = self.dispatch(Request(environ), start_response)
|
||||
if hasattr(response, 'headers'):
|
||||
response.headers["Access-Control-Allow-Origin"] = self.ORIGIN.rstrip('/')
|
||||
response.headers["Access-Control-Allow-Origin"] = self.conf.get('general', 'host').rstrip('/')
|
||||
response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type"
|
||||
response.headers["Access-Control-Allow-Credentials"] = "true"
|
||||
response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE"
|
||||
@ -131,6 +136,24 @@ class Isso(object):
|
||||
return self.wsgi_app(environ, start_response)
|
||||
|
||||
|
||||
def make_app(conf=None):
|
||||
|
||||
try:
|
||||
import uwsgi
|
||||
except ImportError:
|
||||
isso = type("Isso", (Isso, NaiveMixin), {})(conf)
|
||||
else:
|
||||
isso = type("Isso", (Isso, uWSGIMixin), {})(conf)
|
||||
|
||||
app = ProxyFix(wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, {
|
||||
'/static': join(dirname(__file__), 'static/'),
|
||||
'/js': join(dirname(__file__), 'js/'),
|
||||
'/css': join(dirname(__file__), 'css/')
|
||||
})))
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
parser = ArgumentParser(description="a blog comment hosting service")
|
||||
@ -145,62 +168,20 @@ def main():
|
||||
|
||||
serve = subparser.add_parser("run", help="run server")
|
||||
|
||||
defaultcfg = [
|
||||
"[general]",
|
||||
"dbpath = /tmp/isso.db", "secretkey = %r" % os.urandom(24),
|
||||
"host = http://localhost:8080/", "passphrase = p@$$w0rd",
|
||||
"max_age = 450",
|
||||
"[server]",
|
||||
"host = localhost", "port = 8080", "reload = off",
|
||||
"[SMTP]",
|
||||
"username = ", "password = ",
|
||||
"host = localhost", "port = 465", "ssl = on",
|
||||
"to = ", "from = "
|
||||
]
|
||||
|
||||
args = parser.parse_args()
|
||||
conf = ConfigParser(allow_no_value=True)
|
||||
conf.readfp(io.StringIO(u'\n'.join(defaultcfg)))
|
||||
conf.read(args.conf)
|
||||
conf = Config.load(args.conf)
|
||||
|
||||
if args.command == "import":
|
||||
migrate.disqus(db.SQLite3(conf.get("general", "dbpath"), False), args.dump)
|
||||
migrate.disqus(db.SQLite3(conf.get('general', 'dbpath')), args.dump)
|
||||
sys.exit(0)
|
||||
|
||||
if not conf.get("general", "host").startswith(("http://", "https://")):
|
||||
sys.exit("error: host must start with http:// or https://")
|
||||
run_simple(conf.get('server', 'host'), conf.getint('server', 'port'), make_app(conf),
|
||||
threaded=True, use_reloader=conf.getboolean('server', 'reload'))
|
||||
|
||||
try:
|
||||
print(" * connecting to SMTP server", end=" ")
|
||||
mailer = notify.SMTPMailer(conf)
|
||||
print("[%s]" % colors.green("ok"))
|
||||
except (socket.error, notify.SMTPException):
|
||||
print("[%s]" % colors.red("failed"))
|
||||
mailer = notify.NullMailer()
|
||||
|
||||
try:
|
||||
print(" * connecting to HTTP server", end=" ")
|
||||
rv = urlparse.urlparse(conf.get("general", "host"))
|
||||
host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc
|
||||
httplib.HTTPConnection(host, timeout=5).request('GET', rv.path)
|
||||
print("[%s]" % colors.green("ok"))
|
||||
except (httplib.HTTPException, socket.error):
|
||||
print("[%s]" % colors.red("failed"))
|
||||
|
||||
isso = Isso(
|
||||
dbpath=conf.get('general', 'dbpath'),
|
||||
secret=conf.get('general', 'secretkey'),
|
||||
origin=conf.get('general', 'host'),
|
||||
max_age=conf.getint('general', 'max_age'),
|
||||
passphrase=conf.get('general', 'passphrase'),
|
||||
mailer=mailer
|
||||
)
|
||||
|
||||
app = ProxyFix(wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, {
|
||||
'/static': join(dirname(__file__), 'static/'),
|
||||
'/js': join(dirname(__file__), 'js/'),
|
||||
'/css': join(dirname(__file__), 'css/')
|
||||
})))
|
||||
|
||||
run_simple(conf.get('server', 'host'), conf.getint('server', 'port'),
|
||||
app, threaded=True, use_reloader=conf.getboolean('server', 'reload'))
|
||||
try:
|
||||
import uwsgi
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
application = make_app(Config.load(os.environ.get('ISSO_SETTINGS')))
|
||||
|
||||
114
isso/core.py
Normal file
114
isso/core.py
Normal file
@ -0,0 +1,114 @@
|
||||
# -*- encoding: utf-8 -*-
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
|
||||
import thread
|
||||
import threading
|
||||
|
||||
import socket
|
||||
import smtplib
|
||||
|
||||
from ConfigParser import ConfigParser
|
||||
|
||||
try:
|
||||
import uwsgi
|
||||
except ImportError:
|
||||
uwsgi = None
|
||||
|
||||
from isso import notify, colors
|
||||
|
||||
class Config:
|
||||
|
||||
default = [
|
||||
"[general]",
|
||||
"dbpath = /tmp/isso.db", "secretkey = %r" % os.urandom(24),
|
||||
"host = http://localhost:8080/", "passphrase = p@$$w0rd",
|
||||
"max_age = 450",
|
||||
"[server]",
|
||||
"host = localhost", "port = 8080", "reload = off",
|
||||
"[SMTP]",
|
||||
"username = ", "password = ",
|
||||
"host = localhost", "port = 465", "ssl = on",
|
||||
"to = ", "from = "
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def load(cls, configfile):
|
||||
|
||||
rv = ConfigParser(allow_no_value=True)
|
||||
rv.readfp(io.StringIO(u'\n'.join(Config.default)))
|
||||
|
||||
if configfile:
|
||||
rv.read(configfile)
|
||||
|
||||
return rv
|
||||
|
||||
|
||||
def threaded(func):
|
||||
|
||||
def dec(self, *args, **kwargs):
|
||||
thread.start_new_thread(func, (self, ) + args, kwargs)
|
||||
|
||||
return dec
|
||||
|
||||
|
||||
class NaiveMixin(object):
|
||||
|
||||
def __init__(self, conf):
|
||||
|
||||
try:
|
||||
print(" * connecting to SMTP server", end=" ")
|
||||
mailer = notify.SMTPMailer(conf)
|
||||
print("[%s]" % colors.green("ok"))
|
||||
except (socket.error, smtplib.SMTPException):
|
||||
print("[%s]" % colors.red("failed"))
|
||||
mailer = notify.NullMailer()
|
||||
|
||||
self.mailer = mailer
|
||||
self.lock = threading.Lock()
|
||||
|
||||
@threaded
|
||||
def notify(self, subject, body, retries=5):
|
||||
|
||||
for x in range(retries):
|
||||
try:
|
||||
self.mailer.sendmail(subject, body)
|
||||
except Exception:
|
||||
time.sleep(60)
|
||||
else:
|
||||
break
|
||||
|
||||
|
||||
class uWSGIMixin(NaiveMixin):
|
||||
|
||||
def __init__(self, conf):
|
||||
super(uWSGIMixin, self).__init__(conf)
|
||||
|
||||
class Lock():
|
||||
|
||||
def __enter__(self):
|
||||
while uwsgi.queue_get(0) == "LOCK":
|
||||
time.sleep(0.1)
|
||||
|
||||
uwsgi.queue_set(uwsgi.queue_slot(), "LOCK")
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
uwsgi.queue_pop()
|
||||
|
||||
def spooler(args):
|
||||
try:
|
||||
self.mailer.sendmail(args["subject"].decode('utf-8'), args["body"].decode('utf-8'))
|
||||
except smtplib.SMTPConnectError:
|
||||
return uwsgi.SPOOL_RETRY
|
||||
else:
|
||||
return uwsgi.SPOOL_OK
|
||||
|
||||
self.lock = Lock()
|
||||
uwsgi.spooler = spooler
|
||||
|
||||
def notify(self, subject, body, retries=5):
|
||||
uwsgi.spool({"subject": subject.encode('utf-8'), "body": body.encode('utf-8')})
|
||||
@ -4,7 +4,6 @@
|
||||
# All rights reserved.
|
||||
|
||||
import hmac
|
||||
import time
|
||||
import struct
|
||||
import base64
|
||||
import hashlib
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
# -*- encoding: utf-8 -*-
|
||||
|
||||
import time
|
||||
import sqlite3
|
||||
|
||||
from isso.utils import Bloomfilter
|
||||
|
||||
|
||||
@ -1,13 +1,10 @@
|
||||
# -*- encoding: utf-8 -*-
|
||||
|
||||
import time
|
||||
import logging
|
||||
|
||||
from smtplib import SMTP, SMTP_SSL, SMTPException
|
||||
from smtplib import SMTP, SMTP_SSL
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
|
||||
def create(comment, subject, permalink, remote_addr):
|
||||
def format(comment, permalink, remote_addr):
|
||||
|
||||
rv = []
|
||||
rv.append("%s schrieb:" % (comment["author"] or "Jemand"))
|
||||
@ -21,7 +18,7 @@ def create(comment, subject, permalink, remote_addr):
|
||||
rv.append("IP Adresse: %s" % remote_addr)
|
||||
rv.append("Link zum Kommentar: %s" % permalink)
|
||||
|
||||
return subject, u'\n'.join(rv)
|
||||
return u'\n'.join(rv)
|
||||
|
||||
|
||||
class Connection(object):
|
||||
@ -54,23 +51,15 @@ class SMTPMailer(object):
|
||||
with Connection(self.conf):
|
||||
pass
|
||||
|
||||
def sendmail(self, subject, body, retries=5):
|
||||
def sendmail(self, subject, body):
|
||||
|
||||
msg = MIMEText(body, 'plain', 'utf-8')
|
||||
msg['From'] = "Ich schrei sonst! <%s>" % self.from_addr
|
||||
msg['To'] = self.to_addr
|
||||
msg['Subject'] = subject.encode('utf-8')
|
||||
|
||||
for i in range(retries):
|
||||
try:
|
||||
with Connection(self.conf) as con:
|
||||
con.sendmail(self.from_addr, self.to_addr, msg.as_string())
|
||||
except SMTPException:
|
||||
logging.exception("uncaught exception, %i of %i:", i + 1, retries)
|
||||
else:
|
||||
return
|
||||
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
class NullMailer(object):
|
||||
|
||||
@ -59,7 +59,7 @@ def heading(host, path):
|
||||
filter(lambda i: i.attributes.has_key("id"), html.getElementsByTagName("div"))))
|
||||
|
||||
if not el:
|
||||
return None
|
||||
return "Untitled"
|
||||
|
||||
el = el[0]
|
||||
visited = []
|
||||
@ -91,7 +91,7 @@ def heading(host, path):
|
||||
|
||||
el = el.parentNode
|
||||
|
||||
return None
|
||||
return "Untitled."
|
||||
|
||||
|
||||
def anonymize(remote_addr):
|
||||
|
||||
@ -3,10 +3,9 @@
|
||||
import cgi
|
||||
import json
|
||||
import time
|
||||
import thread
|
||||
import hashlib
|
||||
import sqlite3
|
||||
import logging
|
||||
import sqlite3
|
||||
|
||||
from itsdangerous import SignatureExpired, BadSignature
|
||||
|
||||
@ -16,8 +15,8 @@ from werkzeug.exceptions import abort, BadRequest
|
||||
from isso import utils, notify
|
||||
from isso.crypto import pbkdf2
|
||||
|
||||
FIELDS = set(['id', 'parent', 'text', 'author', 'website', 'email', 'mode',
|
||||
'created', 'modified', 'likes', 'dislikes', 'hash'])
|
||||
FIELDS = {'id', 'parent', 'text', 'author', 'website', 'email', 'mode', 'created',
|
||||
'modified', 'likes', 'dislikes', 'hash'}
|
||||
|
||||
|
||||
class requires:
|
||||
@ -45,7 +44,7 @@ class requires:
|
||||
@requires(str, 'uri')
|
||||
def new(app, environ, request, uri):
|
||||
|
||||
if uri not in app.db.threads and not utils.urlexists(app.ORIGIN, uri):
|
||||
if uri not in app.db.threads and not utils.urlexists(app.conf.get('general', 'host'), uri):
|
||||
return Response('URI does not exist', 404)
|
||||
|
||||
try:
|
||||
@ -68,8 +67,9 @@ def new(app, environ, request, uri):
|
||||
|
||||
data['remote_addr'] = utils.anonymize(unicode(request.remote_addr))
|
||||
|
||||
with app.lock:
|
||||
if uri not in app.db.threads:
|
||||
app.db.threads.new(uri, utils.heading(app.ORIGIN, uri))
|
||||
app.db.threads.new(uri, utils.heading(app.conf.get('general', 'host'), uri))
|
||||
title = app.db.threads[uri].title
|
||||
|
||||
try:
|
||||
@ -78,24 +78,22 @@ def new(app, environ, request, uri):
|
||||
logging.exception('uncaught SQLite3 exception')
|
||||
abort(400)
|
||||
|
||||
href = (app.ORIGIN.rstrip("/") + uri + "#isso-%i" % rv["id"])
|
||||
thread.start_new_thread(
|
||||
app.notify,
|
||||
notify.create(rv, title, href, utils.anonymize(unicode(request.remote_addr))))
|
||||
href = (app.conf.get('general', 'host').rstrip("/") + uri + "#isso-%i" % rv["id"])
|
||||
app.notify(title, notify.format(rv, href, utils.anonymize(unicode(request.remote_addr))))
|
||||
|
||||
# save checksum of text into cookie, so mallory can't modify/delete a comment, if
|
||||
# he add a comment, then removed it but not the signed cookie.
|
||||
checksum = hashlib.md5(rv["text"].encode('utf-8')).hexdigest()
|
||||
|
||||
rv["text"] = app.markdown(rv["text"])
|
||||
rv["hash"] = pbkdf2(rv.get('email') or rv['remote_addr'], app.SALT, 1000, 6)
|
||||
rv["hash"] = pbkdf2(rv.get('email') or rv['remote_addr'], app.salt, 1000, 6)
|
||||
|
||||
for key in set(rv.keys()) - FIELDS:
|
||||
rv.pop(key)
|
||||
|
||||
resp = Response(json.dumps(rv), 202 if rv["mode"] == 2 else 201,
|
||||
content_type='application/json')
|
||||
resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]), max_age=app.MAX_AGE)
|
||||
resp.set_cookie(str(rv["id"]), app.sign([rv["id"], checksum]), max_age=app.conf.getint('general', 'max_age'))
|
||||
return resp
|
||||
|
||||
|
||||
@ -180,7 +178,7 @@ def fetch(app, environ, request, uri):
|
||||
|
||||
for item in rv:
|
||||
|
||||
item['hash'] = pbkdf2(item['email'] or item['remote_addr'], app.SALT, 1000, 6)
|
||||
item['hash'] = pbkdf2(item['email'] or item['remote_addr'], app.salt, 1000, 6)
|
||||
|
||||
for key in set(item.keys()) - FIELDS:
|
||||
item.pop(key)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user