isso import FILE can import Disqus export

Readme.md

@@ -17,14 +17,25 @@ Current status: `nosetests specs/`. Ran 11 tests in 0.570s.
 - simple JSON API (hence comments are JavaScript-only)
 - create comments and modify/delete within a time range
 - Ping/Trackback support (not implemented yet)
-- simple admin interface (not implemented yet)
+- simple admin interface (work in progress)
-- easy integration, similar to Disqus (not implemented yet)
+- easy integration, similar to Disqus (work in progress)
 - spam filtering using [http:bl](https://www.projecthoneypot.org/) (not implemented yet)
 
 ## Installation
 
 TODO
 
+## Migrating from Disqus
+
+Go to [disqus.com](https://disqus.com/) and export your "forum" as XML. If you
+use Firefox and you get a 403, try a webkit browser, Disqus did something very
+weird with that download link. Next:
+
+    $ isso import /path/to/ur/dump.xml
+
+That's it. Visit your admin page to see all threads. If it doesn't work for you,
+please file in a bug report \*including\* your dump.
+
 ## API
 
 ### fetch comments for /foo-bar/

isso/__init__.py

@@ -22,16 +22,24 @@
 
 __version__ = '0.2'
 
+import sys; reload(sys)
+sys.setdefaultencoding('utf-8')  # we only support UTF-8 and python 2.X :-)
+
+import io
 import json
 
+from os.path import join, dirname
+from optparse import OptionParser, make_option, SUPPRESS_HELP
+
 from itsdangerous import URLSafeTimedSerializer
 
+from werkzeug.wsgi import SharedDataMiddleware
 from werkzeug.routing import Map, Rule
 from werkzeug.serving import run_simple
 from werkzeug.wrappers import Request, Response
 from werkzeug.exceptions import HTTPException, NotFound, InternalServerError
 
-from isso import admin, comment, db
+from isso import admin, comment, db, migrate
 from isso.utils import determine, import_object, RegexConverter, IssoEncoder
 
 # override default json :func:`dumps`.
@@ -106,10 +114,32 @@ class Isso:
 
 def main():
 
-    from os.path import join, dirname
+    options = [
-    from werkzeug.wsgi import SharedDataMiddleware
+        make_option("--version", action="store_true", help="print version info and exit"),
+        make_option("--sqlite", dest="sqlite", metavar='FILE', default="/tmp/sqlite.db",
+            help="use SQLite3 database"),
+        make_option("--port", dest="port", default=8000, help="webserver port"),
+        make_option("--test", dest="production", action="store_false", default=True,
+            help=SUPPRESS_HELP),
+    ]
 
-    app = Isso({'SQLITE': '/tmp/sqlite.db', 'PRODUCTION': False})
+    parser = OptionParser(option_list=options)
-    app = SharedDataMiddleware(app,{
+    options, args = parser.parse_args()
-        '/static': join(dirname(__file__), 'static')})
+
-    run_simple('127.0.0.1', 8000, app, use_reloader=True)
+    if options.version:
+        print 'isso', __version__
+        sys.exit(0)
+
+    app = Isso({'SQLITE': options.sqlite, 'PRODUCTION': options.production})
+
+    if len(args) > 0 and args[0] == 'import':
+        if len(args) < 2:
+            print 'usage: isso import FILE'
+            sys.exit(2)
+
+        with io.open(args[1], encoding='utf-8') as fp:
+            migrate.disqus(app.db, fp.read())
+    else:
+        app = SharedDataMiddleware(app, {
+            '/static': join(dirname(__file__), 'static')})
+        run_simple('127.0.0.1', 8000, app, use_reloader=True)

isso/db.py

@@ -107,7 +107,7 @@ class SQLite(Abstract):
 
         with sqlite3.connect(self.dbpath) as con:
             return self.query2comment(
-                con.execute('SELECT *, MAX(id) FROM comments;').fetchone())
+                con.execute('SELECT *, MAX(id) FROM comments WHERE path=?;', (path, )).fetchone())
 
     def activate(self, path, id):
         with sqlite3.connect(self.dbpath) as con:

isso/migrate.py

@@ -0,0 +1,66 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2012, Martin Zimmermann <info@posativ.org>. All rights reserved.
+# License: BSD Style, 2 clauses. see isso/__init__.py
+#
+# TODO
+#
+# - export does not include website from commenters
+# - Disqus includes already deleted comments
+
+from time import mktime, strptime
+from urlparse import urlparse
+from collections import defaultdict
+
+from xml.etree import ElementTree
+
+from isso.models import Comment
+
+
+ns = '{http://disqus.com}'
+dsq = '{http://disqus.com/disqus-internals}'
+
+
+def insert(db, thread, comments):
+
+    path = urlparse(thread.find('%sid' % ns).text).path
+    remap = dict()
+
+    for item in sorted(comments, key=lambda k: k['created']):
+
+        parent = remap.get(item.get('dsq:parent'))
+        comment = Comment(created=item['created'], text=item['text'],
+                          author=item['author'], email=item['email'], parent=parent)
+
+        rv = db.add(path, comment)
+        print rv.id, rv.text[:25], rv.author
+        remap[item['dsq:id']] = rv.id
+
+
+def disqus(db, xml):
+
+    tree = ElementTree.fromstring(xml)
+    res = defaultdict(list)
+
+    for post in tree.findall('%spost' % ns):
+
+        item = {
+            'dsq:id': post.attrib.get(dsq + 'id'),
+            'text': post.find('%smessage' % ns).text,
+            'author': post.find('%sauthor/%sname' % (ns, ns)).text,
+            'email': post.find('%sauthor/%semail' % (ns, ns)).text,
+            'created': mktime(strptime(
+                post.find('%screatedAt' % ns).text, '%Y-%m-%dT%H:%M:%SZ'))
+        }
+
+        if post.find(ns + 'parent') is not None:
+            item['dsq:parent'] = post.find(ns + 'parent').attrib.get(dsq + 'id')
+
+        res[post.find('%sthread' % ns).attrib.get(dsq + 'id')].append(item)
+
+    for thread in tree.findall('%sthread' % ns):
+        id = thread.attrib.get(dsq + 'id')
+        if id in res:
+            insert(db, thread, res[id])
+            # for comment in res[_id]:
+            #     print '  ', comment['author'], comment['text'][:25]

specs/test_db.py

@@ -46,6 +46,13 @@ class TestSQLite(unittest.TestCase):
         assert rv[0].id == 1
         assert rv[0].text == 'Baz'
 
+    def test_add_return(self):
+
+        self.db.add('/', comment(text='1'))
+        self.db.add('/', comment(text='2'))
+
+        assert self.db.add('/path/', comment(text='1')).id == 1
+
     def test_update(self):
 
         rv = self.db.add('/', comment(text='Foo'))