From adb3d40c0375bdaf24876521b72f1c66e28c2472 Mon Sep 17 00:00:00 2001 From: Martin Zimmermann Date: Fri, 13 Sep 2013 15:21:18 +0200 Subject: [PATCH] send notification for new comments This commit also introduces a new db which maps path to thread title. The title is read by parsing the HTML for a related

tag using `html5lib`. You can set up SMTP in your configuration (here the defaults): [SMTP] host = localhost port = 465 ssl = on username = password = recipient = sender = In short, by default Isso uses a local SMTP server using SSL without any authentication. An email is send on comment creation to "recipient" from "Ich schrei sonst ". This commit also uses a simple ANSI colorization module from my static blog compiler project. On server startup, Isso will connect to the SMTP server and fall back to a null mailer. It also tries to connect to your website, so if that doesn't work, you probably can't comment on your website either. --- isso/__init__.py | 52 +++++++++++++++++++++++++-------- isso/colors.py | 51 ++++++++++++++++++++++++++++++++ isso/db.py | 29 ++++++++++++++++--- isso/js/helper/utils.js | 2 +- isso/migrate.py | 5 ++++ isso/notify.py | 60 ++++++++++++++++++++++++++++++++++++++ isso/utils.py | 64 +++++++++++++++++++++++++++++++++++++---- isso/views/comment.py | 15 ++++++++-- setup.py | 2 +- specs/test_comment.py | 4 +-- specs/test_vote.py | 4 +-- 11 files changed, 257 insertions(+), 31 deletions(-) create mode 100644 isso/colors.py create mode 100644 isso/notify.py diff --git a/isso/__init__.py b/isso/__init__.py index 8c26aa4..8da079a 100644 --- a/isso/__init__.py +++ b/isso/__init__.py @@ -29,6 +29,9 @@ import sys import io import os import json +import socket +import httplib +import urlparse from os.path import dirname, join from argparse import ArgumentParser @@ -47,7 +50,7 @@ from werkzeug.contrib.fixers import ProxyFix from jinja2 import Environment, FileSystemLoader -from isso import db, utils, migrate, views, wsgi +from isso import db, utils, migrate, views, wsgi, notify, colors from isso.views import comment, admin url_map = Map([ @@ -65,7 +68,7 @@ class Isso(object): PRODUCTION = False - def __init__(self, dbpath, secret, origin, max_age, passphrase): + def __init__(self, dbpath, secret, origin, max_age, passphrase, mailer): self.DBPATH = dbpath self.ORIGIN = origin @@ -75,6 +78,7 @@ class Isso(object): self.db = db.SQLite(dbpath, moderation=False) self.signer = URLSafeTimedSerializer(secret) self.j2env = Environment(loader=FileSystemLoader(join(dirname(__file__), 'templates/'))) + self.notify = lambda *args, **kwargs: mailer.sendmail(*args, **kwargs) def sign(self, obj): return self.signer.dumps(obj) @@ -146,11 +150,15 @@ def main(): defaultcfg = [ "[general]", - "dbpath = /tmp/isso.db", "secret = %r" % os.urandom(24), + "dbpath = /tmp/isso.db", "secretkey = %r" % os.urandom(24), "host = http://localhost:8080/", "passphrase = p@$$w0rd", "max_age = 450", "[server]", - "host = localhost", "port = 8080" + "host = localhost", "port = 8080", "reload = off", + "[SMTP]", + "username = ", "password = ", + "host = localhost", "port = 465", "ssl = on", + "recipient = ", "sender = " ] args = parser.parse_args() @@ -158,23 +166,43 @@ def main(): conf.readfp(io.StringIO(u'\n'.join(defaultcfg))) conf.read(args.conf) + if args.command == "import": + migrate.disqus(db.SQLite(conf.get("general", "dbpath"), False), args.dump) + sys.exit(0) + + if not conf.get("general", "host").startswith(("http://", "https://")): + sys.exit("error: host must start with http:// or https://") + + try: + print(" * connecting to SMTP server", end=" ") + mailer = notify.SMTPMailer(conf) + print("[%s]" % colors.green("ok")) + except (socket.error, notify.SMTPException): + print("[%s]" % colors.red("failed")) + mailer = notify.NullMailer() + + try: + print(" * connecting to HTTP server", end=" ") + rv = urlparse.urlparse(conf.get("general", "host")) + host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc + httplib.HTTPConnection(host, timeout=5).request('GET', rv.path) + print("[%s]" % colors.green("ok")) + except (httplib.HTTPException, socket.error): + print("[%s]" % colors.red("failed")) + isso = Isso( dbpath=conf.get('general', 'dbpath'), - secret=conf.get('general', 'secret'), + secret=conf.get('general', 'secretkey'), origin=conf.get('general', 'host'), max_age=conf.getint('general', 'max_age'), - passphrase=conf.get('general', 'passphrase') + passphrase=conf.get('general', 'passphrase'), + mailer=mailer ) - if args.command == "import": - migrate.disqus(isso.db, args.dump) - sys.exit(0) - app = wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, { '/static': join(dirname(__file__), 'static/'), '/js': join(dirname(__file__), 'js/') })) run_simple(conf.get('server', 'host'), conf.getint('server', 'port'), - app, processes=2) - + app, threaded=True, use_reloader=conf.getboolean('server', 'reload')) diff --git a/isso/colors.py b/isso/colors.py new file mode 100644 index 0000000..1d1eb0b --- /dev/null +++ b/isso/colors.py @@ -0,0 +1,51 @@ +# -*- encoding: utf-8 -*- + +# from isso import compat +# from isso.compat import text_type as str, string_types + +str = unicode +string_types = (unicode, str) + + +# @compat.implements_to_string +class ANSIString(object): + + style = 0 + color = 30 + + def __init__(self, obj, style=None, color=None): + + if isinstance(obj, ANSIString): + if style is None: + style = obj.style + if color is None: + color = obj.color + obj = obj.obj + elif not isinstance(obj, string_types): + obj = str(obj) + + self.obj = obj + if style: + self.style = style + if color: + self.color = color + + def __str__(self): + return '\033[%i;%im' % (self.style, self.color) + self.obj + '\033[0m' + + def __add__(self, other): + return str.__add__(str(self), other) + + def __radd__(self, other): + return other + str(self) + + def encode(self, encoding): + return str(self).encode(encoding) + + +normal, bold, underline = [lambda obj, x=x: ANSIString(obj, style=x) + for x in (0, 1, 4)] + +black, red, green, yellow, blue, \ +magenta, cyan, white = [lambda obj, y=y: ANSIString(obj, color=y) + for y in range(30, 38)] diff --git a/isso/db.py b/isso/db.py index fcc9360..ab0246b 100644 --- a/isso/db.py +++ b/isso/db.py @@ -80,6 +80,30 @@ class Abstract: return +class Threads(object): + + def __init__(self, dbpath): + + self.dbpath = dbpath + + with sqlite3.connect(self.dbpath) as con: + sql = ('main.threads (path VARCHAR(255) NOT NULL, title TEXT' + 'PRIMARY KEY path)') + con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql) + + def get(self, path): + with sqlite3.connect(self.dbpath) as con: + rv = con.execute("SELECT title FROM threads WHERE path=?", (path,)).fetchone() + if rv is not None: + return rv[0] + raise KeyError + + def add(self, path, title): + with sqlite3.connect(self.dbpath) as con: + con.execute("INSERT INTO threads (path, title) VALUES (?, ?)", (path, title)) + return title + + class SQLite(Abstract): """A basic :class:`Abstract` implementation using SQLite3. All comments share a single database. The tuple (id, path) acts as unique identifier @@ -114,10 +138,7 @@ class SQLite(Abstract): WHERE rowid=NEW.rowid; END;""") - # threads (path -> title for now) - sql = ('main.threads (path VARCHAR(255) NOT NULL, title TEXT' - 'PRIMARY KEY path)') - con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql) + self.threads = Threads(self.dbpath) def query2comment(self, query): if query is None: diff --git a/isso/js/helper/utils.js b/isso/js/helper/utils.js index 08d729d..ea00a65 100644 --- a/isso/js/helper/utils.js +++ b/isso/js/helper/utils.js @@ -62,7 +62,7 @@ define({ } } - while (true) { + while (el != null) { visited.push(el); diff --git a/isso/migrate.py b/isso/migrate.py index e157f64..56db54c 100644 --- a/isso/migrate.py +++ b/isso/migrate.py @@ -42,6 +42,11 @@ def insert(db, thread, comments): rv = db.add(path, comment, '127.0.0.1') remap[item['dsq:id']] = rv["id"] + try: + db.threads.get(path) + except KeyError: + db.threads.add(path, thread.find('%stitle' % ns).text.strip()) + def disqus(db, xmlfile): diff --git a/isso/notify.py b/isso/notify.py new file mode 100644 index 0000000..df2a605 --- /dev/null +++ b/isso/notify.py @@ -0,0 +1,60 @@ + +import time +import logging + +from smtplib import SMTP, SMTP_SSL, SMTPException +from email.mime.text import MIMEText + + +def create(comment, subject, permalink, remote_addr): + + rv = [] + rv.append("%s schrieb:" % (comment["author"] or "Jemand")) + rv.append("") + rv.append(comment["text"]) + rv.append("") + + if comment["website"]: + rv.append("Webseite des Kommentators: %s" % comment["website"]) + + rv.append("IP Adresse: %s" % remote_addr) + rv.append("Link zum Kommentar: %s" % permalink) + + return subject, u'\n'.join(rv) + + +class SMTPMailer(object): + + def __init__(self, conf): + + self.server = (SMTP_SSL if conf.getboolean('SMTP', 'ssl') else SMTP)( + host=conf.get('SMTP', 'host'), port=conf.getint('SMTP', 'port')) + + if conf.get('SMTP', 'username') and conf.get('SMTP', 'password'): + self.server.login(conf.get('SMTP', 'username'), conf.get('SMTP', 'password')) + + self.from_addr = conf.get('SMTP', 'from') + self.to_addr = conf.get('SMTP', 'to') + + def sendmail(self, subject, body, retries=5): + + msg = MIMEText(body, 'plain', 'utf-8') + msg['From'] = "Ich schrei sonst! <%s>" % self.from_addr + msg['To'] = self.to_addr + msg['Subject'] = subject.encode('utf-8') + + for i in range(retries): + try: + self.server.sendmail(self.from_addr, self.to_addr, msg.as_string()) + except SMTPException: + logging.exception("uncaught exception, %i of %i:", i + 1, retries) + else: + return + + time.sleep(60) + + +class NullMailer(object): + + def sendmail(self, subject, body, retries=5): + pass diff --git a/isso/utils.py b/isso/utils.py index f1b3cfb..772ecdb 100644 --- a/isso/utils.py +++ b/isso/utils.py @@ -9,14 +9,16 @@ import json import socket import httplib -import ipaddress import random import hashlib -import contextlib from string import ascii_letters, digits from urlparse import urlparse +from contextlib import closing + +import html5lib +import ipaddress from isso.models import Comment @@ -31,14 +33,65 @@ class IssoEncoder(json.JSONEncoder): def urlexists(host, path): - with contextlib.closing(httplib.HTTPConnection(host)) as con: + with closing(httplib.HTTPConnection(normalize(host), timeout=3)) as con: try: - con.request('HEAD', normalize(path)) - except socket.error: + con.request('HEAD', path) + except (httplib.HTTPException, socket.error): return False return con.getresponse().status == 200 +def heading(host, path): + """Connect to `host`, GET path and start from #isso-thread to search for + a possible heading (h1). Returns `None` if nothing found.""" + + with closing(httplib.HTTPConnection(normalize(host), timeout=15)) as con: + con.request('GET', path) + html = html5lib.parse(con.getresponse().read(), treebuilder="dom") + + assert html.lastChild.nodeName == "html" + html = html.lastChild + + # aka getElementById + el = list(filter(lambda i: i.attributes["id"].value == "isso-thread", + filter(lambda i: i.attributes.has_key("id"), html.getElementsByTagName("div")))) + + if not el: + return None + + el = el[0] + visited = [] + + def recurse(node): + for child in node.childNodes: + if child.nodeType != child.ELEMENT_NODE: + continue + if child.nodeName.upper() == "H1": + return child + if child not in visited: + return recurse(child) + + def gettext(rv): + for child in rv.childNodes: + if child.nodeType == child.TEXT_NODE: + yield child.nodeValue + if child.nodeType == child.ELEMENT_NODE: + for item in gettext(child): + yield item + + while el is not None: # el.parentNode is None in the very end + + visited.append(el) + rv = recurse(el) + + if rv: + return ''.join(gettext(rv)).strip() + + el = el.parentNode + + return None + + def normalize(host): """Make `host` compatible with :py:mod:`httplib`.""" @@ -108,4 +161,3 @@ class Bloomfilter: def __len__(self): return self.elements - diff --git a/isso/views/comment.py b/isso/views/comment.py index 8923a04..b914410 100644 --- a/isso/views/comment.py +++ b/isso/views/comment.py @@ -5,6 +5,7 @@ import cgi import json +import thread import hashlib import sqlite3 import logging @@ -14,7 +15,7 @@ from itsdangerous import SignatureExpired, BadSignature from werkzeug.wrappers import Response from werkzeug.exceptions import abort, BadRequest -from isso import models, utils +from isso import models, utils, notify class requires: @@ -42,7 +43,7 @@ class requires: @requires(str, 'uri') def create(app, environ, request, uri): - if app.PRODUCTION and not utils.urlexists(app.HOST, uri): + if not utils.urlexists(app.ORIGIN, uri): return Response('URI does not exist', 400) try: @@ -69,13 +70,21 @@ def create(app, environ, request, uri): hash=hashlib.md5(hash).hexdigest()) + try: + title = app.db.threads.get(uri) + except KeyError: + title = app.db.threads.add(uri, utils.heading(app.ORIGIN, uri)) + try: rv = app.db.add(uri, comment, utils.anonymize(unicode(request.remote_addr))) except sqlite3.Error: logging.exception('uncaught SQLite3 exception') abort(400) + else: + rv["text"] = app.markdown(rv["text"]) - rv["text"] = app.markdown(rv["text"]) + href = (app.ORIGIN.rstrip("/") + uri + "#isso-%i" % rv["id"]) + thread.start_new_thread(app.notify, notify.create(comment, title, href, request.remote_addr)) resp = Response(app.dumps(rv), 202 if rv.pending else 201, content_type='application/json') diff --git a/setup.py b/setup.py index acf77c6..b403d61 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ setup( "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7" ], - install_requires=['Jinja2>=2.7', 'werkzeug>=0.9', 'itsdangerous', 'misaka'], + install_requires=['Jinja2>=2.7', 'werkzeug>=0.9', 'itsdangerous', 'misaka', 'html5lib'], entry_points={ 'console_scripts': ['isso = isso:main'], diff --git a/specs/test_comment.py b/specs/test_comment.py index e7174ab..694a419 100644 --- a/specs/test_comment.py +++ b/specs/test_comment.py @@ -10,7 +10,7 @@ import unittest from werkzeug.test import Client from werkzeug.wrappers import Response -from isso import Isso +from isso import Isso, notify from isso.models import Comment @@ -29,7 +29,7 @@ class TestComments(unittest.TestCase): def setUp(self): fd, self.path = tempfile.mkstemp() - self.app = Isso(self.path, '...', '...', 15*60, "...") + self.app = Isso(self.path, '...', '...', 15*60, "...", notify.NullMailer()) self.app.wsgi_app = FakeIP(self.app.wsgi_app) self.client = Client(self.app, Response) diff --git a/specs/test_vote.py b/specs/test_vote.py index 3f9dca3..ea8b2bd 100644 --- a/specs/test_vote.py +++ b/specs/test_vote.py @@ -9,7 +9,7 @@ import unittest from werkzeug.test import Client from werkzeug.wrappers import Response -from isso import Isso +from isso import Isso, notify class FakeIP(object): @@ -33,7 +33,7 @@ class TestVote(unittest.TestCase): def makeClient(self, ip): - app = Isso(self.path, '...', '...', 15*60, "...") + app = Isso(self.path, '...', '...', 15*60, "...", notify.NullMailer()) app.wsgi_app = FakeIP(app.wsgi_app, ip) return Client(app, Response)