From ae4e719b29aed26888538c0828eb25040e60ee47 Mon Sep 17 00:00:00 2001
From: Elad Alfassa <elad@fedoraproject.org>
Date: Sat, 22 Feb 2014 18:36:10 +0200
Subject: [PATCH] [core, modules] Python 3 support!

Issue #236

I'm happy to announce that with this commit, all basic Willie functionality
can be ran on Python 3!

There are few bugs remaining to iron out and few modules which fail to load
and need to be fixed, but all the basic functionality now works well with
Python3.

No extra dependencies introduced, Python 2.7 is obviously still supported.
---
 willie/bot.py               | 17 ++++++++++-------
 willie/config.py            |  3 ++-
 willie/coretasks.py         |  6 +++---
 willie/modules/help.py      |  7 ++-----
 willie/modules/reload.py    |  7 ++-----
 willie/modules/remind.py    |  2 +-
 willie/modules/tell.py      |  4 ++--
 willie/modules/translate.py |  2 +-
 willie/modules/url.py       | 13 +++++++++----
 willie/test_tools.py        |  4 ++--
 willie/tools.py             |  7 +++++++
 willie/web.py               | 11 +++++++----
 12 files changed, 48 insertions(+), 35 deletions(-)

diff --git a/willie/bot.py b/willie/bot.py
index 4dd09481ec..a400bbc37e 100644
--- a/willie/bot.py
+++ b/willie/bot.py
@@ -26,11 +26,14 @@
 import willie.irc as irc
 from willie.db import WillieDB
 from willie.tools import (stderr, Nick, PriorityQueue, released,
-                   get_command_regexp)
+                   get_command_regexp, iteritems, itervalues)
 import willie.module as module
 if sys.version_info.major >= 3:
     unicode = str
     basestring = str
+    py3 = True
+else:
+    py3 = False
 
 
 class Willie(irc.Bot):
@@ -304,7 +307,7 @@ def setup(self):
 
         modules = []
         error_count = 0
-        for name, filename in filenames.iteritems():
+        for name, filename in iteritems(filenames):
             try:
                 module = imp.load_source(name, filename)
             except Exception as e:
@@ -378,7 +381,7 @@ def register(self, variables):
         quitting.
 
         """
-        for obj in variables.itervalues():
+        for obj in itervalues(variables):
             if self.is_callable(obj):
                 self.callables.add(obj)
             if self.is_shutdown(obj):
@@ -398,16 +401,16 @@ def unregister(self, variables):
 
         def remove_func(func, commands):
             """Remove all traces of func from commands."""
-            for func_list in commands.itervalues():
+            for func_list in itervalues(commands):
                 if func in func_list:
                     func_list.remove(func)
 
         hostmask = "%s!%s@%s" % (self.nick, self.user, socket.gethostname())
         willie = self.WillieWrapper(self, irc.Origin(self, hostmask, [], {}))
-        for obj in variables.itervalues():
+        for obj in itervalues(variables):
             if obj in self.callables:
                 self.callables.remove(obj)
-                for commands in self.commands.itervalues():
+                for commands in itervalues(self.commands):
                     remove_func(obj, commands)
             if obj in self.shutdown_methods:
                 try:
@@ -567,7 +570,7 @@ def say(self, string, max_messages=1):
             self.bot.msg(self.origin.sender, string, max_messages)
 
         def reply(self, string, notice=False):
-            if isinstance(string, str):
+            if isinstance(string, str) and not py3:
                 string = string.decode('utf8')
             if notice:
                 self.notice(
diff --git a/willie/config.py b/willie/config.py
index ee5a7bc86b..06dd8524c8 100644
--- a/willie/config.py
+++ b/willie/config.py
@@ -40,6 +40,7 @@
 from __future__ import absolute_import
 
 import willie.db as db
+from willie.tools import iteritems
 import os
 import sys
 try:
@@ -321,7 +322,7 @@ def _modules(self):
         modules_dir = os.path.join(home, 'modules')
         filenames = self.enumerate_modules()
         os.sys.path.insert(0, modules_dir)
-        for name, filename in filenames.iteritems():
+        for name, filename in iteritems(filenames):
             try:
                 module = imp.load_source(name, filename)
             except Exception as e:
diff --git a/willie/coretasks.py b/willie/coretasks.py
index b1b9886e4a..8c84fc1ad0 100644
--- a/willie/coretasks.py
+++ b/willie/coretasks.py
@@ -19,7 +19,7 @@
 import re
 import time
 import willie
-from willie.tools import Nick
+from willie.tools import Nick, iteritems
 import base64
 
 
@@ -132,7 +132,7 @@ def handle_names(bot, trigger):
 
     for name in names:
         priv = 0
-        for prefix, value in mapping.iteritems():
+        for prefix, value in iteritems(mapping):
             if prefix in name:
                 priv = priv | value
         nick = Nick(name.lstrip(''.join(mapping.keys())))
@@ -364,7 +364,7 @@ def recieve_cap_ls_reply(bot, trigger):
         # parse it, so we don't need to worry if it fails.
         bot._cap_reqs['multi-prefix'] = (['', 'coretasks', None],)
 
-    for cap, reqs in bot._cap_reqs.iteritems():
+    for cap, reqs in iteritems(bot._cap_reqs):
         # At this point, we know mandatory and prohibited don't co-exist, but
         # we need to call back for optionals if they're also prohibited
         prefix = ''
diff --git a/willie/modules/help.py b/willie/modules/help.py
index b8e1baca27..6b6c741002 100644
--- a/willie/modules/help.py
+++ b/willie/modules/help.py
@@ -8,6 +8,7 @@
 http://willie.dftba.net
 """
 from willie.module import commands, rule, example, priority
+from willie.tools import iterkeys
 
 
 @rule('$nick' '(?i)(help|doc) +([A-Za-z]+)(?:\?+)?$')
@@ -32,7 +33,7 @@ def help(bot, trigger):
 @priority('low')
 def commands(bot, trigger):
     """Return a list of bot's commands"""
-    names = ', '.join(sorted(bot.doc.iterkeys()))
+    names = ', '.join(sorted(iterkeys(bot.doc)))
     if not trigger.is_privmsg:
         bot.reply("I am sending you a private message of all my commands!")
     bot.msg(trigger.nick, 'Commands I recognise: ' + names + '.', max_messages=10)
@@ -49,7 +50,3 @@ def help2(bot, trigger):
         'general details. My owner is %s.'
     ) % bot.config.owner
     bot.reply(response)
-
-
-if __name__ == '__main__':
-    print __doc__.strip()
diff --git a/willie/modules/reload.py b/willie/modules/reload.py
index 93ead4f4ad..9da03c9a27 100644
--- a/willie/modules/reload.py
+++ b/willie/modules/reload.py
@@ -10,6 +10,7 @@
 import os.path
 import time
 import imp
+from willie.tools import iteritems
 import willie.module
 import subprocess
 
@@ -38,7 +39,7 @@ def f_reload(bot, trigger):
     old_module = sys.modules[name]
 
     old_callables = {}
-    for obj_name, obj in vars(old_module).iteritems():
+    for obj_name, obj in iteritems(vars(old_module)):
         if bot.is_callable(obj) or bot.is_shutdown(obj):
             old_callables[obj_name] = obj
 
@@ -151,7 +152,3 @@ def pm_f_load(bot, trigger):
     """Wrapper for allowing delivery of .load command via PM"""
     if trigger.is_privmsg:
         f_load(bot, trigger)
-
-
-if __name__ == '__main__':
-    print __doc__.strip()
diff --git a/willie/modules/remind.py b/willie/modules/remind.py
index ea6738867e..78b0a325d9 100644
--- a/willie/modules/remind.py
+++ b/willie/modules/remind.py
@@ -45,7 +45,7 @@ def load_database(name):
 
 def dump_database(name, data):
     f = codecs.open(name, 'w', encoding='utf-8')
-    for unixtime, reminders in data.iteritems():
+    for unixtime, reminders in willie.tools.iteritems(data):
         for channel, nick, message in reminders:
             f.write('%s\t%s\t%s\t%s\n' % (unixtime, channel, nick, message))
     f.close()
diff --git a/willie/modules/tell.py b/willie/modules/tell.py
index 9aa8f6c12e..630fd67bce 100644
--- a/willie/modules/tell.py
+++ b/willie/modules/tell.py
@@ -13,7 +13,7 @@
 import datetime
 import willie.tools
 import threading
-from willie.tools import Nick
+from willie.tools import Nick, iterkeys
 from willie.module import commands, nickname_commands, rule, priority, example
 
 maximum = 4
@@ -42,7 +42,7 @@ def dumpReminders(fn, data, lock):
     lock.acquire()
     try:
         f = open(fn, 'w')
-        for tellee in data.iterkeys():
+        for tellee in iterkeys(data):
             for remindon in data[tellee]:
                 line = '\t'.join((tellee,) + remindon)
                 try:
diff --git a/willie/modules/translate.py b/willie/modules/translate.py
index 1caabac1fd..c3c0d1ca3e 100644
--- a/willie/modules/translate.py
+++ b/willie/modules/translate.py
@@ -73,7 +73,7 @@ def translate(text, input='auto', output='en'):
     return ''.join(x[0] for x in data[0]), language
 
 
-@rule(ur'$nickname[,:]\s+(?:([a-z]{2}) +)?(?:([a-z]{2}|en-raw) +)?["“](.+?)["”]\? *$')
+@rule(u'$nickname[,:]\s+(?:([a-z]{2}) +)?(?:([a-z]{2}|en-raw) +)?["“](.+?)["”]\? *$')
 @example('$nickname: "mon chien"? or $nickname: fr "mon chien"?')
 @priority('low')
 def tr(bot, trigger):
diff --git a/willie/modules/url.py b/willie/modules/url.py
index cfe4140de8..4391db3876 100644
--- a/willie/modules/url.py
+++ b/willie/modules/url.py
@@ -9,10 +9,16 @@
 """
 
 import re
-from htmlentitydefs import name2codepoint
+import sys
+if sys.version_info.major < 3:
+    from htmlentitydefs import name2codepoint
+    import urlparse
+else:
+    from html.entities import name2codepoint
+    import urllib.parse as urlparse
 from willie import web, tools
 from willie.module import commands, rule, example
-import urlparse
+
 
 url_finder = None
 exclusion_char = '!'
@@ -117,7 +123,6 @@ def title_auto(bot, trigger):
     """
     if re.match(bot.config.core.prefix + 'title', trigger):
         return
-
     urls = re.findall(url_finder, trigger)
     results = process_urls(bot, trigger, urls)
     bot.memory['last_seen_url'][trigger.sender] = urls[-1]
@@ -188,7 +193,7 @@ def check_callbacks(bot, trigger, url, run=True):
     # Check if it matches the exclusion list first
     matched = any(regex.search(url) for regex in bot.memory['url_exclude'])
     # Then, check if there's anything in the callback list
-    for regex, function in bot.memory['url_callbacks'].iteritems():
+    for regex, function in tools.iteritems(bot.memory['url_callbacks']):
         match = regex.search(url)
         if match:
             if run:
diff --git a/willie/test_tools.py b/willie/test_tools.py
index 897f3e60aa..fcc3262658 100644
--- a/willie/test_tools.py
+++ b/willie/test_tools.py
@@ -104,7 +104,7 @@ def test():
         if hasattr(module, 'setup'):
             module.setup(bot)
 
-        for _i in xrange(repeat):
+        for _i in range(repeat):
             wrapper = MockWillieWrapper(bot, origin)
             tested_func(wrapper, trigger)
             assert len(wrapper.output) == len(results)
@@ -123,7 +123,7 @@ def insert_into_module(func, module_name, base_name, prefix):
     func.__module__ = module_name
     module = sys.modules[module_name]
     # Make sure the func method does not overwrite anything.
-    for i in xrange(1000):
+    for i in range(1000):
         func.__name__ = str("%s_%s_%s" % (prefix, base_name, i))
         if not hasattr(module, func.__name__):
             break
diff --git a/willie/tools.py b/willie/tools.py
index 8b819c0b78..b24cfed44d 100644
--- a/willie/tools.py
+++ b/willie/tools.py
@@ -34,6 +34,13 @@
 import operator
 if sys.version_info.major >= 3:
     unicode = str
+    iteritems  = dict.items
+    itervalues = dict.values
+    iterkeys   = dict.keys
+else:
+    iteritems  = dict.iteritems
+    itervalues = dict.itervalues
+    iterkeys   = dict.iterkeys
 
 
 class ExpressionEvaluator:
diff --git a/willie/web.py b/willie/web.py
index 17f331f2e4..fde797da47 100644
--- a/willie/web.py
+++ b/willie/web.py
@@ -157,10 +157,13 @@ def get_urllib_object(uri, timeout, headers=None, verify_ssl=True, data=None):
     For more information, refer to the urllib2 documentation.
 
     """
-    try:
-        uri = uri.encode("utf-8")
-    except:
-        pass
+    if sys.version_info.major < 3:
+        try:
+            uri = uri.encode("utf-8")
+        except:
+            pass
+    else:
+        uri = str(uri)
     original_headers = {'Accept': '*/*', 'User-Agent': 'Mozilla/5.0 (Willie)'}
     if headers is not None:
         original_headers.update(headers)