Back to development: 0.6.2

To be compatible with comments from Disqus (and users unfamiliar with
Markdown), Misaka no longer disables user-inputted HTML, but the
generated HTML is now post-processed and all "unsafe" tags (not
possible with Markdown) are discarded.

Whitelist: p, a, pre, blockquote, h1-h6, em, sub, sup, del, ins, math,
           dl, ol, ul, li

This commit also removes an unnecessary newline generated by
Misaka/Sundown.

Conflicts:
	isso/utils/__init__.py

CHANGES.rst

@@ -1,13 +1,19 @@
 Changelog for Isso
 ==================
 
-0.6 (unreleased)
-----------------
+0.6.1 (2014-01-12)
+------------------
 
 Major improvements:
 
+- override thread discovery with data-isso-id="...", #27
+
+  To use the same thread for different URLs, you can now add a custom
+  ``data-isso-id="my-id"`` attribute which is used to identify and retrieve
+  comments (defaults to current URL aka `window.location.pathname`).
+
 - `isso.dispatch` now dispatches multiple websites (= configurations) based on
-  a URL prefix
+  URL prefixes
 
 - fix a cross-site request forgery vulnerability for comment creation, voting,
   editing and deletion, #40
@@ -18,15 +24,16 @@ Major improvements:
   http://posativ.org/docs (or docs/ in the repository). Also includes an
   annotated `example.conf`, #43
 
+- new italian and russian translations
+
 Minor improvements:
 
 - move `isso:application` to `isso.run:application` to avoid uneccessary
-  initialization in some cases (change the module if you use uWSGI or Gunicorn)
+  initialization in some cases (change module if you use uWSGI or Gunicorn)
 - add Date header to email notifications, #42
 - check for blank text in new comment, #41
-- work around IE10 HTML5 abilities for custom data-attributes
-- add support for Gunicorn (and other pre-forking wsgi servers)
-- new russian translation
+- work around IE10's HTML5 abilities for custom data-attributes
+- add support for Gunicorn (and other pre-forking WSGI servers)
 
 
 0.5 (2013-11-17)

isso/utils/__init__.py

@@ -5,12 +5,19 @@ from __future__ import division
 import pkg_resources
 werkzeug = pkg_resources.get_distribution("werkzeug")
 
+import io
 import json
 import random
 import hashlib
 
 from string import ascii_letters, digits
 
+try:
+    from html.parser import HTMLParser, HTMLParseError
+except ImportError:
+    from HTMLParser import HTMLParser, HTMLParseError
+
+from werkzeug.utils import escape
 from werkzeug.wrappers import Request
 from werkzeug.exceptions import BadRequest
 
@@ -126,10 +133,84 @@ class JSONRequest(Request):
             raise BadRequest('Unable to read JSON request')
 
 
+class Sanitizer(HTMLParser, object):
+    """Sanitize HTML output: remove unsafe HTML tags such as iframe or
+    script based on a whitelist of allowed tags."""
+
+    safe = set([
+        "p", "a", "pre", "blockquote",
+        "h1", "h2", "h3", "h4", "h5", "h6",
+        "em", "sub", "sup", "del", "ins", "math",
+        "dl", "ol", "ul", "li"])
+
+    @classmethod
+    def format(cls, attrs):
+        res = []
+        for key, value in attrs:
+            if value is None:
+                res.append(key)
+            else:
+                res.append(u'{0}="{1}"'.format(key, escape(value)))
+        return ' '.join(res)
+
+    def __init__(self, html):
+        super(Sanitizer, self).__init__()
+        self.result = io.StringIO()
+        self.feed(html)
+        self.result.seek(0)
+
+    def handle_starttag(self, tag, attrs):
+        if tag in Sanitizer.safe:
+            self.result.write(u"<" + tag)
+            if attrs:
+                self.result.write(" " + Sanitizer.format(attrs))
+            self.result.write(u">")
+
+    def handle_data(self, data):
+        self.result.write(data)
+
+    def handle_endtag(self, tag):
+        if tag in Sanitizer.safe:
+            self.result.write(u"</" + tag + ">")
+
+    def handle_startendtag(self, tag, attrs):
+        if tag in Sanitizer.safe:
+            self.result.write(u"<" + tag)
+            if attrs:
+                self.result.write(" " + Sanitizer.format(attrs))
+            self.result.write(u"/>")
+
+    def handle_entityref(self, name):
+        self.result.write(u'&' + name + ';')
+
+    def handle_charref(self, char):
+        self.result.write(u'&#' + char + ';')
+
+
 def markdown(text):
-    return misaka.html(text, extensions= misaka.EXT_STRIKETHROUGH
-        | misaka.EXT_SUPERSCRIPT | misaka.EXT_AUTOLINK
-        | misaka.HTML_SKIP_HTML  | misaka.HTML_SKIP_IMAGES | misaka.HTML_SAFELINK)
+    """Convert Markdown to (safe) HTML.
+
+    >>> markdown("*Ohai!*") # doctest: +IGNORE_UNICODE
+    '<p><em>Ohai!</em></p>'
+    >>> markdown("<em>Hi</em>") # doctest: +IGNORE_UNICODE
+    '<p><em>Hi</em></p>'
+    >>> markdown("<script>alert('Onoe')</script>") # doctest: +IGNORE_UNICODE
+    "<p>alert('Onoe')</p>"
+    >>> markdown("http://example.org/ and sms:+1234567890") # doctest: +IGNORE_UNICODE
+    '<p><a href="http://example.org/">http://example.org/</a> and sms:+1234567890</p>'
+    """
+
+    # ~~strike through~~, sub script: 2^(nd) and http://example.org/ auto-link
+    exts = misaka.EXT_STRIKETHROUGH | misaka.EXT_SUPERSCRIPT | misaka.EXT_AUTOLINK
+
+    # remove HTML tags, skip <img> (for now) and only render "safe" protocols
+    html = misaka.HTML_SKIP_STYLE | misaka.HTML_SKIP_IMAGES | misaka.HTML_SAFELINK
+
+    rv = misaka.html(text, extensions=exts, render_flags=html).rstrip("\n")
+    if not rv.startswith("<p>") and not rv.endswith("</p>"):
+        rv = "<p>" + rv + "</p>"
+
+    return Sanitizer(rv).result.read()
 
 
 def origin(hosts):

setup.py

@@ -17,7 +17,7 @@ else:
 
 setup(
     name='isso',
-    version='0.6.dev0',
+    version='0.6.2.dev0',
     author='Martin Zimmermann',
     author_email='info@posativ.org',
     packages=find_packages(),

specs/test_comments.py

@@ -54,7 +54,7 @@ class TestComments(unittest.TestCase):
         rv = loads(r.data)
 
         assert rv['id'] == 1
-        assert rv['text'] == '<p>Lorem ipsum ...</p>\n'
+        assert rv['text'] == '<p>Lorem ipsum ...</p>'
 
     def testCreate(self):
 
@@ -66,7 +66,7 @@ class TestComments(unittest.TestCase):
         rv = loads(rv.data)
 
         assert rv["mode"] == 1
-        assert rv["text"] == '<p>Lorem ipsum ...</p>\n'
+        assert rv["text"] == '<p>Lorem ipsum ...</p>'
 
     def textCreateWithNonAsciiText(self):
 
@@ -78,7 +78,7 @@ class TestComments(unittest.TestCase):
         rv = loads(rv.data)
 
         assert rv["mode"] == 1
-        assert rv["text"] == '<p>Здравствуй, мир!</p>\n'
+        assert rv["text"] == '<p>Здравствуй, мир!</p>'
 
     def testCreateMultiple(self):
 
@@ -261,10 +261,10 @@ class TestComments(unittest.TestCase):
         self.post('/new?uri=test', data=json.dumps({"text": "Tpyo"}))
 
         self.put('/id/1', data=json.dumps({"text": "Tyop"}))
-        assert loads(self.get('/id/1').data)["text"] == "<p>Tyop</p>\n"
+        assert loads(self.get('/id/1').data)["text"] == "<p>Tyop</p>"
 
         self.put('/id/1', data=json.dumps({"text": "Typo"}))
-        assert loads(self.get('/id/1').data)["text"] == "<p>Typo</p>\n"
+        assert loads(self.get('/id/1').data)["text"] == "<p>Typo</p>"
 
     def testDeleteCommentRemovesThread(self):