[core, modules] Python 3 support!

Issue #236

I'm happy to announce that with this commit, all basic Willie functionality
can be ran on Python 3!

There are few bugs remaining to iron out and few modules which fail to load
and need to be fixed, but all the basic functionality now works well with
Python3.

No extra dependencies introduced, Python 2.7 is obviously still supported.

willie/bot.py

@@ -26,11 +26,14 @@
 import willie.irc as irc
 from willie.db import WillieDB
 from willie.tools import (stderr, Nick, PriorityQueue, released,
-                   get_command_regexp)
+                   get_command_regexp, iteritems, itervalues)
 import willie.module as module
 if sys.version_info.major >= 3:
     unicode = str
     basestring = str
+    py3 = True
+else:
+    py3 = False
 
 
 class Willie(irc.Bot):
@@ -304,7 +307,7 @@ def setup(self):
 
         modules = []
         error_count = 0
-        for name, filename in filenames.iteritems():
+        for name, filename in iteritems(filenames):
             try:
                 module = imp.load_source(name, filename)
             except Exception as e:
@@ -378,7 +381,7 @@ def register(self, variables):
         quitting.
 
         """
-        for obj in variables.itervalues():
+        for obj in itervalues(variables):
             if self.is_callable(obj):
                 self.callables.add(obj)
             if self.is_shutdown(obj):
@@ -398,16 +401,16 @@ def unregister(self, variables):
 
         def remove_func(func, commands):
             """Remove all traces of func from commands."""
-            for func_list in commands.itervalues():
+            for func_list in itervalues(commands):
                 if func in func_list:
                     func_list.remove(func)
 
         hostmask = "%s!%s@%s" % (self.nick, self.user, socket.gethostname())
         willie = self.WillieWrapper(self, irc.Origin(self, hostmask, [], {}))
-        for obj in variables.itervalues():
+        for obj in itervalues(variables):
             if obj in self.callables:
                 self.callables.remove(obj)
-                for commands in self.commands.itervalues():
+                for commands in itervalues(self.commands):
                     remove_func(obj, commands)
             if obj in self.shutdown_methods:
                 try:
@@ -567,7 +570,7 @@ def say(self, string, max_messages=1):
             self.bot.msg(self.origin.sender, string, max_messages)
 
         def reply(self, string, notice=False):
-            if isinstance(string, str):
+            if isinstance(string, str) and not py3:
                 string = string.decode('utf8')
             if notice:
                 self.notice(

willie/config.py

@@ -40,6 +40,7 @@
 from __future__ import absolute_import
 
 import willie.db as db
+from willie.tools import iteritems
 import os
 import sys
 try:
@@ -321,7 +322,7 @@ def _modules(self):
         modules_dir = os.path.join(home, 'modules')
         filenames = self.enumerate_modules()
         os.sys.path.insert(0, modules_dir)
-        for name, filename in filenames.iteritems():
+        for name, filename in iteritems(filenames):
             try:
                 module = imp.load_source(name, filename)
             except Exception as e:

willie/coretasks.py

@@ -19,7 +19,7 @@
 import re
 import time
 import willie
-from willie.tools import Nick
+from willie.tools import Nick, iteritems
 import base64
 
 
@@ -132,7 +132,7 @@ def handle_names(bot, trigger):
 
     for name in names:
         priv = 0
-        for prefix, value in mapping.iteritems():
+        for prefix, value in iteritems(mapping):
             if prefix in name:
                 priv = priv | value
         nick = Nick(name.lstrip(''.join(mapping.keys())))
@@ -364,7 +364,7 @@ def recieve_cap_ls_reply(bot, trigger):
         # parse it, so we don't need to worry if it fails.
         bot._cap_reqs['multi-prefix'] = (['', 'coretasks', None],)
 
-    for cap, reqs in bot._cap_reqs.iteritems():
+    for cap, reqs in iteritems(bot._cap_reqs):
         # At this point, we know mandatory and prohibited don't co-exist, but
         # we need to call back for optionals if they're also prohibited
         prefix = ''

willie/modules/help.py

@@ -8,6 +8,7 @@
 http://willie.dftba.net
 """
 from willie.module import commands, rule, example, priority
+from willie.tools import iterkeys
 
 
 @rule('$nick' '(?i)(help|doc) +([A-Za-z]+)(?:\?+)?$')
@@ -32,7 +33,7 @@ def help(bot, trigger):
 @priority('low')
 def commands(bot, trigger):
     """Return a list of bot's commands"""
-    names = ', '.join(sorted(bot.doc.iterkeys()))
+    names = ', '.join(sorted(iterkeys(bot.doc)))
     if not trigger.is_privmsg:
         bot.reply("I am sending you a private message of all my commands!")
     bot.msg(trigger.nick, 'Commands I recognise: ' + names + '.', max_messages=10)
@@ -49,7 +50,3 @@ def help2(bot, trigger):
         'general details. My owner is %s.'
     ) % bot.config.owner
     bot.reply(response)
-
-
-if __name__ == '__main__':
-    print __doc__.strip()

willie/modules/reload.py

@@ -10,6 +10,7 @@
 import os.path
 import time
 import imp
+from willie.tools import iteritems
 import willie.module
 import subprocess
 
@@ -38,7 +39,7 @@ def f_reload(bot, trigger):
     old_module = sys.modules[name]
 
     old_callables = {}
-    for obj_name, obj in vars(old_module).iteritems():
+    for obj_name, obj in iteritems(vars(old_module)):
         if bot.is_callable(obj) or bot.is_shutdown(obj):
             old_callables[obj_name] = obj
 
@@ -151,7 +152,3 @@ def pm_f_load(bot, trigger):
     """Wrapper for allowing delivery of .load command via PM"""
     if trigger.is_privmsg:
         f_load(bot, trigger)
-
-
-if __name__ == '__main__':
-    print __doc__.strip()

willie/modules/remind.py

@@ -45,7 +45,7 @@ def load_database(name):
 
 def dump_database(name, data):
     f = codecs.open(name, 'w', encoding='utf-8')
-    for unixtime, reminders in data.iteritems():
+    for unixtime, reminders in willie.tools.iteritems(data):
         for channel, nick, message in reminders:
             f.write('%s\t%s\t%s\t%s\n' % (unixtime, channel, nick, message))
     f.close()

willie/modules/tell.py

@@ -13,7 +13,7 @@
 import datetime
 import willie.tools
 import threading
-from willie.tools import Nick
+from willie.tools import Nick, iterkeys
 from willie.module import commands, nickname_commands, rule, priority, example
 
 maximum = 4
@@ -42,7 +42,7 @@ def dumpReminders(fn, data, lock):
     lock.acquire()
     try:
         f = open(fn, 'w')
-        for tellee in data.iterkeys():
+        for tellee in iterkeys(data):
             for remindon in data[tellee]:
                 line = '\t'.join((tellee,) + remindon)
                 try:

willie/modules/translate.py

@@ -73,7 +73,7 @@ def translate(text, input='auto', output='en'):
     return ''.join(x[0] for x in data[0]), language
 
 
-@rule(ur'$nickname[,:]\s+(?:([a-z]{2}) +)?(?:([a-z]{2}|en-raw) +)?["“](.+?)["”]\? *$')
+@rule(u'$nickname[,:]\s+(?:([a-z]{2}) +)?(?:([a-z]{2}|en-raw) +)?["“](.+?)["”]\? *$')
 @example('$nickname: "mon chien"? or $nickname: fr "mon chien"?')
 @priority('low')
 def tr(bot, trigger):

willie/modules/url.py

@@ -9,10 +9,16 @@
 """
 
 import re
-from htmlentitydefs import name2codepoint
+import sys
+if sys.version_info.major < 3:
+    from htmlentitydefs import name2codepoint
+    import urlparse
+else:
+    from html.entities import name2codepoint
+    import urllib.parse as urlparse
 from willie import web, tools
 from willie.module import commands, rule, example
-import urlparse
+
 
 url_finder = None
 exclusion_char = '!'
@@ -117,7 +123,6 @@ def title_auto(bot, trigger):
     """
     if re.match(bot.config.core.prefix + 'title', trigger):
         return
-
     urls = re.findall(url_finder, trigger)
     results = process_urls(bot, trigger, urls)
     bot.memory['last_seen_url'][trigger.sender] = urls[-1]
@@ -188,7 +193,7 @@ def check_callbacks(bot, trigger, url, run=True):
     # Check if it matches the exclusion list first
     matched = any(regex.search(url) for regex in bot.memory['url_exclude'])
     # Then, check if there's anything in the callback list
-    for regex, function in bot.memory['url_callbacks'].iteritems():
+    for regex, function in tools.iteritems(bot.memory['url_callbacks']):
         match = regex.search(url)
         if match:
             if run:

willie/test_tools.py

@@ -104,7 +104,7 @@ def test():
         if hasattr(module, 'setup'):
             module.setup(bot)
 
-        for _i in xrange(repeat):
+        for _i in range(repeat):
             wrapper = MockWillieWrapper(bot, origin)
             tested_func(wrapper, trigger)
             assert len(wrapper.output) == len(results)
@@ -123,7 +123,7 @@ def insert_into_module(func, module_name, base_name, prefix):
     func.__module__ = module_name
     module = sys.modules[module_name]
     # Make sure the func method does not overwrite anything.
-    for i in xrange(1000):
+    for i in range(1000):
         func.__name__ = str("%s_%s_%s" % (prefix, base_name, i))
         if not hasattr(module, func.__name__):
             break

willie/tools.py

@@ -34,6 +34,13 @@
 import operator
 if sys.version_info.major >= 3:
     unicode = str
+    iteritems  = dict.items
+    itervalues = dict.values
+    iterkeys   = dict.keys
+else:
+    iteritems  = dict.iteritems
+    itervalues = dict.itervalues
+    iterkeys   = dict.iterkeys
 
 
 class ExpressionEvaluator:

willie/web.py

@@ -157,10 +157,13 @@ def get_urllib_object(uri, timeout, headers=None, verify_ssl=True, data=None):
     For more information, refer to the urllib2 documentation.
 
     """
-    try:
-        uri = uri.encode("utf-8")
-    except:
-        pass
+    if sys.version_info.major < 3:
+        try:
+            uri = uri.encode("utf-8")
+        except:
+            pass
+    else:
+        uri = str(uri)
     original_headers = {'Accept': '*/*', 'User-Agent': 'Mozilla/5.0 (Willie)'}
     if headers is not None:
         original_headers.update(headers)