send notification for new comments

This commit also introduces a new db which maps path to thread title.
The title is read by parsing the HTML for a related <h1> tag using
`html5lib`.

You can set up SMTP in your configuration (here the defaults):

    [SMTP]
    host = localhost
    port = 465
    ssl = on
    username =
    password =
    recipient =
    sender =

In short, by default Isso uses a local SMTP server using SSL without
any authentication. An email is send on comment creation to "recipient"
from "Ich schrei sonst <sender>".

This commit also uses a simple ANSI colorization module from my static
blog compiler project.

On server startup, Isso will connect to the SMTP server and fall back to
a null mailer. It also tries to connect to your website, so if that
doesn't work, you probably can't comment on your website either.

isso/__init__.py

@@ -29,6 +29,9 @@ import sys
 import io
 import os
 import json
+import socket
+import httplib
+import urlparse
 
 from os.path import dirname, join
 from argparse import ArgumentParser
@@ -47,7 +50,7 @@ from werkzeug.contrib.fixers import ProxyFix
 
 from jinja2 import Environment, FileSystemLoader
 
-from isso import db, utils, migrate, views, wsgi
+from isso import db, utils, migrate, views, wsgi, notify, colors
 from isso.views import comment, admin
 
 url_map = Map([
@@ -65,7 +68,7 @@ class Isso(object):
 
     PRODUCTION = False
 
-    def __init__(self, dbpath, secret, origin, max_age, passphrase):
+    def __init__(self, dbpath, secret, origin, max_age, passphrase, mailer):
 
         self.DBPATH = dbpath
         self.ORIGIN = origin
@@ -75,6 +78,7 @@ class Isso(object):
         self.db = db.SQLite(dbpath, moderation=False)
         self.signer = URLSafeTimedSerializer(secret)
         self.j2env = Environment(loader=FileSystemLoader(join(dirname(__file__), 'templates/')))
+        self.notify = lambda *args, **kwargs: mailer.sendmail(*args, **kwargs)
 
     def sign(self, obj):
         return self.signer.dumps(obj)
@@ -146,11 +150,15 @@ def main():
 
     defaultcfg = [
         "[general]",
-        "dbpath = /tmp/isso.db", "secret = %r" % os.urandom(24),
+        "dbpath = /tmp/isso.db", "secretkey = %r" % os.urandom(24),
         "host = http://localhost:8080/", "passphrase = p@$$w0rd",
         "max_age = 450",
         "[server]",
-        "host = localhost", "port = 8080"
+        "host = localhost", "port = 8080", "reload = off",
+        "[SMTP]",
+        "username = ", "password = ",
+        "host = localhost", "port = 465", "ssl = on",
+        "recipient = ", "sender = "
     ]
 
     args = parser.parse_args()
@@ -158,23 +166,43 @@ def main():
     conf.readfp(io.StringIO(u'\n'.join(defaultcfg)))
     conf.read(args.conf)
 
+    if args.command == "import":
+        migrate.disqus(db.SQLite(conf.get("general", "dbpath"), False), args.dump)
+        sys.exit(0)
+
+    if not conf.get("general", "host").startswith(("http://", "https://")):
+        sys.exit("error: host must start with http:// or https://")
+
+    try:
+        print(" * connecting to SMTP server", end=" ")
+        mailer = notify.SMTPMailer(conf)
+        print("[%s]" % colors.green("ok"))
+    except (socket.error, notify.SMTPException):
+        print("[%s]" % colors.red("failed"))
+        mailer = notify.NullMailer()
+
+    try:
+        print(" * connecting to HTTP server", end=" ")
+        rv = urlparse.urlparse(conf.get("general", "host"))
+        host = (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc
+        httplib.HTTPConnection(host, timeout=5).request('GET', rv.path)
+        print("[%s]" % colors.green("ok"))
+    except (httplib.HTTPException, socket.error):
+        print("[%s]" % colors.red("failed"))
+
     isso = Isso(
         dbpath=conf.get('general', 'dbpath'),
-        secret=conf.get('general', 'secret'),
+        secret=conf.get('general', 'secretkey'),
         origin=conf.get('general', 'host'),
         max_age=conf.getint('general', 'max_age'),
-        passphrase=conf.get('general', 'passphrase')
+        passphrase=conf.get('general', 'passphrase'),
+        mailer=mailer
     )
 
-    if args.command == "import":
-        migrate.disqus(isso.db, args.dump)
-        sys.exit(0)
-
     app = wsgi.SubURI(SharedDataMiddleware(isso.wsgi_app, {
         '/static': join(dirname(__file__), 'static/'),
         '/js': join(dirname(__file__), 'js/')
         }))
 
     run_simple(conf.get('server', 'host'), conf.getint('server', 'port'),
-        app, processes=2)
+        app, threaded=True, use_reloader=conf.getboolean('server', 'reload'))
-

isso/colors.py

@@ -0,0 +1,51 @@
+# -*- encoding: utf-8 -*-
+
+# from isso import compat
+# from isso.compat import text_type as str, string_types
+
+str = unicode
+string_types = (unicode, str)
+
+
+# @compat.implements_to_string
+class ANSIString(object):
+
+    style = 0
+    color = 30
+
+    def __init__(self, obj, style=None, color=None):
+
+        if isinstance(obj, ANSIString):
+            if style is None:
+                style = obj.style
+            if color is None:
+                color = obj.color
+            obj = obj.obj
+        elif not isinstance(obj, string_types):
+            obj = str(obj)
+
+        self.obj = obj
+        if style:
+            self.style = style
+        if color:
+            self.color = color
+
+    def __str__(self):
+        return '\033[%i;%im' % (self.style, self.color) + self.obj + '\033[0m'
+
+    def __add__(self, other):
+        return str.__add__(str(self), other)
+
+    def __radd__(self, other):
+        return other + str(self)
+
+    def encode(self, encoding):
+        return str(self).encode(encoding)
+
+
+normal, bold, underline = [lambda obj, x=x: ANSIString(obj, style=x)
+    for x in (0, 1, 4)]
+
+black, red, green, yellow, blue, \
+magenta, cyan, white = [lambda obj, y=y: ANSIString(obj, color=y)
+    for y in range(30, 38)]

isso/db.py

@@ -80,6 +80,30 @@ class Abstract:
         return
 
 
+class Threads(object):
+
+    def __init__(self, dbpath):
+
+        self.dbpath = dbpath
+
+        with sqlite3.connect(self.dbpath) as con:
+            sql = ('main.threads (path VARCHAR(255) NOT NULL, title TEXT'
+                   'PRIMARY KEY path)')
+            con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql)
+
+    def get(self, path):
+        with sqlite3.connect(self.dbpath) as con:
+            rv = con.execute("SELECT title FROM threads WHERE path=?", (path,)).fetchone()
+            if rv is not None:
+                return rv[0]
+            raise KeyError
+
+    def add(self, path, title):
+        with sqlite3.connect(self.dbpath) as con:
+            con.execute("INSERT INTO threads (path, title) VALUES (?, ?)", (path, title))
+        return title
+
+
 class SQLite(Abstract):
     """A basic :class:`Abstract` implementation using SQLite3.  All comments
     share a single database. The tuple (id, path) acts as unique identifier
@@ -114,10 +138,7 @@ class SQLite(Abstract):
                     WHERE rowid=NEW.rowid;
                 END;""")
 
-            # threads (path -> title for now)
+        self.threads = Threads(self.dbpath)
-            sql = ('main.threads (path VARCHAR(255) NOT NULL, title TEXT'
-                   'PRIMARY KEY path)')
-            con.execute("CREATE TABLE IF NOT EXISTS %s;" % sql)
 
     def query2comment(self, query):
         if query is None:

isso/js/helper/utils.js

@@ -62,7 +62,7 @@ define({
             }
         }
 
-        while (true) {
+        while (el != null) {
 
             visited.push(el);
 

isso/migrate.py

@@ -42,6 +42,11 @@ def insert(db, thread, comments):
         rv = db.add(path, comment, '127.0.0.1')
         remap[item['dsq:id']] = rv["id"]
 
+        try:
+            db.threads.get(path)
+        except KeyError:
+            db.threads.add(path, thread.find('%stitle' % ns).text.strip())
+
 
 def disqus(db, xmlfile):
 

isso/notify.py

@@ -0,0 +1,60 @@
+
+import time
+import logging
+
+from smtplib import SMTP, SMTP_SSL, SMTPException
+from email.mime.text import MIMEText
+
+
+def create(comment, subject, permalink, remote_addr):
+
+    rv = []
+    rv.append("%s schrieb:" % (comment["author"] or "Jemand"))
+    rv.append("")
+    rv.append(comment["text"])
+    rv.append("")
+
+    if comment["website"]:
+        rv.append("Webseite des Kommentators: %s" % comment["website"])
+
+    rv.append("IP Adresse: %s" % remote_addr)
+    rv.append("Link zum Kommentar: %s" % permalink)
+
+    return subject, u'\n'.join(rv)
+
+
+class SMTPMailer(object):
+
+    def __init__(self, conf):
+
+        self.server = (SMTP_SSL if conf.getboolean('SMTP', 'ssl') else SMTP)(
+            host=conf.get('SMTP', 'host'), port=conf.getint('SMTP', 'port'))
+
+        if conf.get('SMTP', 'username') and conf.get('SMTP', 'password'):
+            self.server.login(conf.get('SMTP', 'username'), conf.get('SMTP', 'password'))
+
+        self.from_addr = conf.get('SMTP', 'from')
+        self.to_addr = conf.get('SMTP', 'to')
+
+    def sendmail(self, subject, body, retries=5):
+
+        msg = MIMEText(body, 'plain', 'utf-8')
+        msg['From'] = "Ich schrei sonst! <%s>" % self.from_addr
+        msg['To'] = self.to_addr
+        msg['Subject'] = subject.encode('utf-8')
+
+        for i in range(retries):
+            try:
+                self.server.sendmail(self.from_addr, self.to_addr, msg.as_string())
+            except SMTPException:
+                logging.exception("uncaught exception, %i of %i:", i + 1, retries)
+            else:
+                return
+
+            time.sleep(60)
+
+
+class NullMailer(object):
+
+    def sendmail(self, subject, body, retries=5):
+        pass

isso/utils.py

@@ -9,14 +9,16 @@ import json
 
 import socket
 import httplib
-import ipaddress
 
 import random
 import hashlib
-import contextlib
 
 from string import ascii_letters, digits
 from urlparse import urlparse
+from contextlib import closing
+
+import html5lib
+import ipaddress
 
 from isso.models import Comment
 
@@ -31,14 +33,65 @@ class IssoEncoder(json.JSONEncoder):
 
 
 def urlexists(host, path):
-    with contextlib.closing(httplib.HTTPConnection(host)) as con:
+    with closing(httplib.HTTPConnection(normalize(host), timeout=3)) as con:
         try:
-            con.request('HEAD', normalize(path))
+            con.request('HEAD', path)
-        except socket.error:
+        except (httplib.HTTPException, socket.error):
             return False
         return con.getresponse().status == 200
 
 
+def heading(host, path):
+    """Connect to `host`, GET path and start from #isso-thread to search for
+    a possible heading (h1). Returns `None` if nothing found."""
+
+    with closing(httplib.HTTPConnection(normalize(host), timeout=15)) as con:
+        con.request('GET', path)
+        html = html5lib.parse(con.getresponse().read(), treebuilder="dom")
+
+    assert html.lastChild.nodeName == "html"
+    html = html.lastChild
+
+    # aka getElementById
+    el = list(filter(lambda i: i.attributes["id"].value == "isso-thread",
+              filter(lambda i: i.attributes.has_key("id"), html.getElementsByTagName("div"))))
+
+    if not el:
+        return None
+
+    el = el[0]
+    visited = []
+
+    def recurse(node):
+        for child in node.childNodes:
+            if child.nodeType != child.ELEMENT_NODE:
+                continue
+            if child.nodeName.upper() == "H1":
+                return child
+            if child not in visited:
+                return recurse(child)
+
+    def gettext(rv):
+        for child in rv.childNodes:
+            if child.nodeType == child.TEXT_NODE:
+                yield child.nodeValue
+            if child.nodeType == child.ELEMENT_NODE:
+                for item in gettext(child):
+                    yield item
+
+    while el is not None:  # el.parentNode is None in the very end
+
+        visited.append(el)
+        rv = recurse(el)
+
+        if rv:
+            return ''.join(gettext(rv)).strip()
+
+        el = el.parentNode
+
+    return None
+
+
 def normalize(host):
     """Make `host` compatible with :py:mod:`httplib`."""
 
@@ -108,4 +161,3 @@ class Bloomfilter:
 
     def __len__(self):
         return self.elements
-

isso/views/comment.py

@@ -5,6 +5,7 @@
 
 import cgi
 import json
+import thread
 import hashlib
 import sqlite3
 import logging
@@ -14,7 +15,7 @@ from itsdangerous import SignatureExpired, BadSignature
 from werkzeug.wrappers import Response
 from werkzeug.exceptions import abort, BadRequest
 
-from isso import models, utils
+from isso import models, utils, notify
 
 
 class requires:
@@ -42,7 +43,7 @@ class requires:
 @requires(str, 'uri')
 def create(app, environ, request, uri):
 
-    if app.PRODUCTION and not utils.urlexists(app.HOST, uri):
+    if not utils.urlexists(app.ORIGIN, uri):
         return Response('URI does not exist', 400)
 
     try:
@@ -69,14 +70,22 @@ def create(app, environ, request, uri):
 
         hash=hashlib.md5(hash).hexdigest())
 
+    try:
+        title = app.db.threads.get(uri)
+    except KeyError:
+        title = app.db.threads.add(uri, utils.heading(app.ORIGIN, uri))
+
     try:
         rv = app.db.add(uri, comment, utils.anonymize(unicode(request.remote_addr)))
     except sqlite3.Error:
         logging.exception('uncaught SQLite3 exception')
         abort(400)
-
+    else:
         rv["text"] = app.markdown(rv["text"])
 
+    href = (app.ORIGIN.rstrip("/") + uri + "#isso-%i" % rv["id"])
+    thread.start_new_thread(app.notify, notify.create(comment, title, href, request.remote_addr))
+
     resp = Response(app.dumps(rv), 202 if rv.pending else 201,
         content_type='application/json')
     resp.set_cookie('%s-%s' % (uri, rv["id"]), app.sign([uri, rv["id"], rv.md5]),

setup.py

@@ -23,7 +23,7 @@ setup(
         "Programming Language :: Python :: 2.6",
         "Programming Language :: Python :: 2.7"
     ],
-    install_requires=['Jinja2>=2.7', 'werkzeug>=0.9', 'itsdangerous', 'misaka'],
+    install_requires=['Jinja2>=2.7', 'werkzeug>=0.9', 'itsdangerous', 'misaka', 'html5lib'],
     entry_points={
         'console_scripts':
             ['isso = isso:main'],

specs/test_comment.py

@@ -10,7 +10,7 @@ import unittest
 from werkzeug.test import Client
 from werkzeug.wrappers import Response
 
-from isso import Isso
+from isso import Isso, notify
 from isso.models import Comment
 
 
@@ -29,7 +29,7 @@ class TestComments(unittest.TestCase):
     def setUp(self):
         fd, self.path = tempfile.mkstemp()
 
-        self.app = Isso(self.path, '...', '...', 15*60, "...")
+        self.app = Isso(self.path, '...', '...', 15*60, "...", notify.NullMailer())
         self.app.wsgi_app = FakeIP(self.app.wsgi_app)
 
         self.client = Client(self.app, Response)

specs/test_vote.py

@@ -9,7 +9,7 @@ import unittest
 from werkzeug.test import Client
 from werkzeug.wrappers import Response
 
-from isso import Isso
+from isso import Isso, notify
 
 
 class FakeIP(object):
@@ -33,7 +33,7 @@ class TestVote(unittest.TestCase):
 
     def makeClient(self, ip):
 
-        app = Isso(self.path, '...', '...', 15*60, "...")
+        app = Isso(self.path, '...', '...', 15*60, "...", notify.NullMailer())
         app.wsgi_app = FakeIP(app.wsgi_app, ip)
 
         return Client(app, Response)