From 821dcc66a2cec7a896cfefea6e7c1db387099e8a Mon Sep 17 00:00:00 2001 From: Edward Powell Date: Wed, 5 Jun 2013 18:13:04 -0400 Subject: [PATCH 01/14] [seen, version, xkcd] Update to API 4.0, issue #276 --- seen.py | 18 ++++++++--------- version.py | 58 +++++++++++++++++++++++++++--------------------------- xkcd.py | 39 ++++++++++++++++++------------------ 3 files changed, 58 insertions(+), 57 deletions(-) diff --git a/seen.py b/seen.py index 69821aeb5b..a52744750b 100644 --- a/seen.py +++ b/seen.py @@ -12,8 +12,10 @@ import datetime import pytz from willie.tools import Ddict, Nick +from willie.module import command, rule + +seen_dict = Ddict(dict) -seen_dict=Ddict(dict) def get_user_time(willie, nick): tz = 'UTC' @@ -25,13 +27,15 @@ def get_user_time(willie, nick): tz = 'UTC' return (pytz.timezone(tz.strip()), tformat or '%Y-%m-%d %H:%M:%S %Z') + +@command('seen') def seen(willie, trigger): """Reports when and where the user was last seen.""" if not trigger.group(2): willie.say(".seen - Reports when was last seen.") return nick = Nick(trigger.group(2).strip()) - if seen_dict.has_key(nick): + if nick in seen_dict: timestamp = seen_dict[nick]['timestamp'] channel = seen_dict[nick]['channel'] message = seen_dict[nick]['message'] @@ -44,17 +48,13 @@ def seen(willie, trigger): willie.say(str(trigger.nick) + ': ' + msg) else: willie.say("Sorry, I haven't seen %s around." % nick) -seen.commands = ['seen'] + +@rule('(.*)') +@priority('low') def note(willie, trigger): if trigger.sender.startswith('#'): nick = Nick(trigger.nick) seen_dict[nick]['timestamp'] = time.time() seen_dict[nick]['channel'] = trigger.sender seen_dict[nick]['message'] = trigger - -note.rule = r'(.*)' -note.priority = 'low' - -if __name__ == '__main__': - print __doc__.strip() diff --git a/version.py b/version.py index b8a72bfa47..0276170399 100644 --- a/version.py +++ b/version.py @@ -8,7 +8,7 @@ from datetime import datetime from subprocess import * -from willie import __version__ +import willie def git_info(): @@ -20,49 +20,49 @@ def git_info(): return commit, author, date -def version(willie, trigger): +@willie.module.command('version') +def version(bot, trigger): """Display the latest commit version, if Willie is running in a git repo.""" commit, author, date = git_info() - + if not commit.strip(): - willie.reply("Willie v. " + __version__) + bot.reply("Willie v. " + willie.__version__) return - willie.say(str(trigger.nick) + ": Willie v. %s at commit:" % __version__) - willie.say(" " + commit) - willie.say(" " + author) - willie.say(" " + date) -version.commands = ['version'] + bot.say(str(trigger.nick) + ": Willie v. %s at commit:" % willie.__version__) + bot.say(" " + commit) + bot.say(" " + author) + bot.say(" " + date) -def ctcp_version(willie, trigger): - willie.write(('NOTICE', trigger.nick), - '\x01VERSION Willie IRC Bot version %s\x01' % __version__) -ctcp_version.rule = '\x01VERSION\x01' -ctcp_version.rate = 20 +@willie.module.rule('\x01VERSION\x01') +@willie.module.rate(20) +def ctcp_version(bot, trigger): + bot.write(('NOTICE', trigger.nick), + '\x01VERSION Willie IRC Bot version %s\x01' % willie.__version__) -def ctcp_source(willie, trigger): - willie.write(('NOTICE', trigger.nick), - '\x01SOURCE https://github.com/Embolalia/willie/\x01') -ctcp_source.rule = '\x01SOURCE\x01' -ctcp_source.rate = 20 +@willie.module.rule('\x01SOURCE\x01') +@willie.module.rate(20) +def ctcp_source(bot, trigger): + bot.write(('NOTICE', trigger.nick), + '\x01SOURCE https://github.com/Embolalia/willie/\x01') -def ctcp_ping(willie, trigger): +@willie.module.rule('\x01PING\s(.*)\x01') +@willie.module.rate(10) +def ctcp_ping(bot, trigger): text = trigger.group() text = text.replace("PING ", "") text = text.replace("\x01", "") - willie.write(('NOTICE', trigger.nick), - '\x01PING {0}\x01'.format(text)) -ctcp_ping.rule = '\x01PING\s(.*)\x01' -ctcp_ping.rate = 10 + bot.write(('NOTICE', trigger.nick), + '\x01PING {0}\x01'.format(text)) -def ctcp_time(willie, trigger): +@willie.module.rule('\x01TIME\x01') +@willie.module.rate(20) +def ctcp_time(bot, trigger): dt = datetime.now() current_time = dt.strftime("%A, %d. %B %Y %I:%M%p") - willie.write(('NOTICE', trigger.nick), - '\x01TIME {0}\x01'.format(current_time)) -ctcp_time.rule = '\x01TIME\x01' -ctcp_time.rate = 20 + bot.write(('NOTICE', trigger.nick), + '\x01TIME {0}\x01'.format(current_time)) diff --git a/xkcd.py b/xkcd.py index 610880aaff..9b413bdd7e 100644 --- a/xkcd.py +++ b/xkcd.py @@ -10,11 +10,14 @@ import random from willie.modules.search import google_search from willie.modules.url import find_title +from willie.module import command import urllib2 from lxml import etree import re -def xkcd(willie, trigger): + +@command('xkcd') +def xkcd(bot, trigger): """ .xkcd - Finds an xkcd comic strip. Takes one of 3 inputs: If no input is provided it will return a random comic @@ -28,39 +31,37 @@ def xkcd(willie, trigger): max_int = int(newest.split("/")[-3]) # if no input is given (pre - lior's edits code) - if not trigger.group(2): # get rand comic + if not trigger.group(2): # get rand comic random.seed() - website = "http://xkcd.com/%d/" % random.randint(0,max_int+1) + website = "http://xkcd.com/%d/" % random.randint(0, max_int + 1) else: query = trigger.group(2).strip() # numeric input! get that comic number if it exists if (query.isdigit()): if (int(query) > max_int): - willie.say("Sorry, comic #" + query + " hasn't been posted yet. The last comic was #%d" % max_int) + bot.say("Sorry, comic #" + query + " hasn't been posted yet. The last comic was #%d" % max_int) return - else: website = "http://xkcd.com/" + query - + else: + website = "http://xkcd.com/" + query + # non-numeric input! code lifted from search.g else: - if (query.lower() == "latest" or query.lower() == "newest"): # special commands + if (query.lower() == "latest" or query.lower() == "newest"): # special commands website = "https://xkcd.com/" - else: # just google + else: # just google try: query = query.encode('utf-8') except: pass - website = google_search("site:xkcd.com "+ query) - chkForum = re.match(re.compile(r'.*?([0-9].*?):.*'), find_title(website)) # regex for comic specific forum threads + website = google_search("site:xkcd.com " + query) + chkForum = re.match(re.compile(r'.*?([0-9].*?):.*'), find_title(website)) # regex for comic specific forum threads if (chkForum): website = "http://xkcd.com/" + chkForum.groups()[0].lstrip('0') - if website: # format and say result + if website: # format and say result website += ' [' + find_title(website)[6:] + ']' - willie.say(website) - elif website is False: willie.say("Problem getting data from Google.") - else: willie.say("No results found for '%s'." % query) -xkcd.commands = ['xkcd'] -xkcd.priority = 'low' - -if __name__ == '__main__': - print __doc__.strip() + bot.say(website) + elif website is False: + bot.say("Problem getting data from Google.") + else: + bot.say("No results found for '%s'." % query) From 7a1302324b60760cee7097f45577064c11d11e68 Mon Sep 17 00:00:00 2001 From: Edward Powell Date: Wed, 5 Jun 2013 18:15:35 -0400 Subject: [PATCH 02/14] [url] Update to 4.0 standards, fix .title --- url.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/url.py b/url.py index 78d44545b9..2047a0696e 100644 --- a/url.py +++ b/url.py @@ -10,6 +10,7 @@ import re from htmlentitydefs import name2codepoint import willie.web as web +from willie.module import command, rule import urllib2 import urlparse @@ -74,6 +75,7 @@ def setup(willie): (exclusion_char)) +@command('title') def title_command(willie, trigger): """ Show the title or URL information for the given URL, or the last URL seen @@ -93,9 +95,11 @@ def title_command(willie, trigger): urls = re.findall(url_finder, trigger) results = process_urls(willie, trigger, urls) -title_command.commands = ['title'] + for result in results[:4]: + message = '[ %s ] - %s' % tuple(result) +@rule('(?u).*(https?://\S+).*') def title_auto(willie, trigger): """ Automatically show titles for URLs. For shortened URLs/redirects, find @@ -113,7 +117,6 @@ def title_auto(willie, trigger): message = '[ %s ] - %s' % tuple(result) if message != trigger: willie.say(message) -title_auto.rule = '(?u).*(https?://\S+).*' def process_urls(willie, trigger, urls): From ce03dc3a4fbe876578ed51563aed82c20cfe2a72 Mon Sep 17 00:00:00 2001 From: Edward Powell Date: Wed, 5 Jun 2013 18:16:01 -0400 Subject: [PATCH 03/14] [search] Update to 4.0 standards, fix registration of last url --- search.py | 68 +++++++++++++++++++++++++------------------------------ 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/search.py b/search.py index dc3cba41d3..5eb74810fe 100644 --- a/search.py +++ b/search.py @@ -50,33 +50,31 @@ def formatnumber(n): return ''.join(parts) -def g(willie, trigger): +def g(bot, trigger): """Queries Google for the specified input.""" query = trigger.group(2) if not query: - return willie.reply('.g what?') + return bot.reply('.g what?') uri = google_search(query) if uri: - willie.reply(uri) - if not hasattr(willie.bot, 'last_seen_uri'): - willie.bot.last_seen_uri = {} - willie.bot.last_seen_uri[trigger.sender] = uri + bot.reply(uri) + bot.memory['last_seen_url'][trigger.sender] = uri elif uri is False: - willie.reply("Problem getting data from Google.") + bot.reply("Problem getting data from Google.") else: - willie.reply("No results found for '%s'." % query) + bot.reply("No results found for '%s'." % query) g.commands = ['g', 'google'] g.priority = 'high' g.example = '.g swhack' -def gc(willie, trigger): +def gc(bot, trigger): """Returns the number of Google results for the specified input.""" query = trigger.group(2) if not query: - return willie.reply('.gc what?') + return bot.reply('.gc what?') num = formatnumber(google_count(query)) - willie.say(query + ': ' + num) + bot.say(query + ': ' + num) gc.commands = ['gc'] gc.priority = 'high' gc.example = '.gc extrapolate' @@ -86,13 +84,13 @@ def gc(willie, trigger): ) -def gcs(willie, trigger): +def gcs(bot, trigger): """Compare the number of Google search results""" if not trigger.group(2): - return willie.reply("Nothing to compare.") + return bot.reply("Nothing to compare.") queries = r_query.findall(trigger.group(2)) if len(queries) > 6: - return willie.reply('Sorry, can only compare up to six things.') + return bot.reply('Sorry, can only compare up to six things.') results = [] for i, query in enumerate(queries): @@ -106,7 +104,7 @@ def gcs(willie, trigger): results = [(term, n) for (n, term) in reversed(sorted(results))] reply = ', '.join('%s (%s)' % (t, formatnumber(n)) for (t, n) in results) - willie.say(reply) + bot.say(reply) gcs.commands = ['gcs', 'comp'] gcs.example = '.gcs foo bar' @@ -122,7 +120,7 @@ def bing_search(query, lang='en-GB'): return m.group(1) -def bing(willie, trigger): +def bing(bot, trigger): """Queries Bing for the specified input.""" query = trigger.group(2) if query.startswith(':'): @@ -131,16 +129,14 @@ def bing(willie, trigger): else: lang = 'en-GB' if not query: - return willie.reply('.bing what?') + return bot.reply('.bing what?') uri = bing_search(query, lang) if uri: - willie.reply(uri) - if not hasattr(willie.bot, 'last_seen_uri'): - willie.bot.last_seen_uri = {} - willie.bot.last_seen_uri[trigger.sender] = uri + bot.reply(uri) + bot.memory['last_seen_url'][trigger.sender] = uri else: - willie.reply("No results found for '%s'." % query) + bot.reply("No results found for '%s'." % query) bing.commands = ['bing'] bing.example = '.bing swhack' @@ -171,35 +167,33 @@ def duck_api(query): return None -def duck(willie, trigger): +def duck(bot, trigger): """Queries Duck Duck Go for the specified input.""" query = trigger.group(2) if not query: - return willie.reply('.ddg what?') + return bot.reply('.ddg what?') #If the API gives us something, say it and stop result = duck_api(query) if result: - willie.reply(result) + bot.reply(result) return #Otherwise, look it up on the HTMl version uri = duck_search(query) if uri: - willie.reply(uri) - if not hasattr(willie.bot, 'last_seen_uri'): - willie.bot.last_seen_uri = {} - willie.bot.last_seen_uri[trigger.sender] = uri + bot.reply(uri) + bot.memory['last_seen_url'][trigger.sender] = uri else: - willie.reply("No results found for '%s'." % query) + bot.reply("No results found for '%s'." % query) duck.commands = ['duck', 'ddg'] -def search(willie, trigger): +def search(bot, trigger): """Searches Google, Bing, and Duck Duck Go.""" if not trigger.group(2): - return willie.reply('.search for what?') + return bot.reply('.search for what?') query = trigger.group(2) gu = google_search(query) or '-' bu = bing_search(query) or '-' @@ -222,22 +216,22 @@ def search(willie, trigger): du = '(extremely long link)' result = '%s (g), %s (b), %s (d)' % (gu, bu, du) - willie.reply(result) + bot.reply(result) search.commands = ['search'] search.example = '.search nerdfighter' -def suggest(willie, trigger): +def suggest(bot, trigger): """Suggest terms starting with given input""" if not trigger.group(2): - return willie.reply("No query term.") + return bot.reply("No query term.") query = trigger.group(2) uri = 'http://websitedev.de/temp-bin/suggest.pl?q=' answer = web.get(uri + web.quote(query).replace('+', '%2B')) if answer: - willie.say(answer) + bot.say(answer) else: - willie.reply('Sorry, no result.') + bot.reply('Sorry, no result.') suggest.commands = ['suggest'] if __name__ == '__main__': From 2e3a9dad9e00d45cb3e3c396df7ebc8468c82d9b Mon Sep 17 00:00:00 2001 From: Edward Powell Date: Wed, 5 Jun 2013 19:44:06 -0400 Subject: [PATCH 04/14] [oblique] Move module to -extras The code is a mess, half of it doesn't work, half of what does work is duplicated by other, better modules, and everything else is so confusing to use that it's just not worth it. --- oblique.py | 119 ----------------------------------------------------- 1 file changed, 119 deletions(-) delete mode 100644 oblique.py diff --git a/oblique.py b/oblique.py deleted file mode 100644 index 4fd8547678..0000000000 --- a/oblique.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -oblique.py - Web Services Interface -Copyright 2008-9, Sean B. Palmer, inamidst.com -Licensed under the Eiffel Forum License 2. - -http://willie.dftba.net -""" - -import re, urllib -import willie.web as web - -definitions = 'https://github.com/nslater/oblique/wiki' - -r_item = re.compile(r'(?i)
  • (.*?)
    • ') -r_tag = re.compile(r'<[^>]+>') - -def mappings(uri): - result = {} - bytes = web.get(uri) - for item in r_item.findall(bytes): - item = r_tag.sub('', item).strip(' \t\r\n') - if not ' ' in item: continue - - command, template = item.split(' ', 1) - if not command.isalnum(): continue - if not template.startswith('http://'): continue - result[command] = template.replace('&', '&') - return result - -def service(willie, trigger, command, args): - t = o.services[command] - template = t.replace('${args}', urllib.quote(args.encode('utf-8'), '')) - template = template.replace('${nick}', urllib.quote(trigger.nick, '')) - uri = template.replace('${sender}', urllib.quote(trigger.sender, '')) - - info = web.head(uri) - if isinstance(info, list): - info = info[0] - if not 'text/plain' in info.get('content-type', '').lower(): - return willie.reply("Sorry, the service didn't respond in plain text.") - bytes = web.get(uri) - lines = bytes.splitlines() - if not lines: - return willie.reply("Sorry, the service didn't respond any output.") - willie.say(lines[0][:350]) - -def refresh(willie): - if hasattr(willie.config, 'services'): - services = willie.config.services - else: services = definitions - - old = o.services - o.serviceURI = services - o.services = mappings(o.serviceURI) - return len(o.services), set(o.services) - set(old) - -def o(willie, trigger): - """Call a webservice.""" - if trigger.group(1) == 'urban': - text = 'ud '+ trigger.group(2) - else: - text = trigger.group(2) - - if (not o.services) or (text == 'refresh'): - length, added = refresh(willie) - if text == 'refresh': - msg = 'Okay, found %s services.' % length - if added: - msg += ' Added: ' + ', '.join(sorted(added)[:5]) - if len(added) > 5: msg += ', &c.' - return willie.reply(msg) - - if not text: - return willie.reply('Try %s for details.' % o.serviceURI) - - if ' ' in text: - command, args = text.split(' ', 1) - else: command, args = text, '' - command = command.lower() - - if command == 'service': - msg = o.services.get(args, 'No such service!') - return willie.reply(msg) - - if not o.services.has_key(command): - return willie.reply('Service not found in %s' % o.serviceURI) - - if hasattr(willie.config, 'external'): - default = willie.config.external.get('*') - manifest = willie.config.external.get(trigger.sender, default) - if manifest: - commands = set(manifest) - if (command not in commands) and (manifest[0] != '!'): - return willie.reply('Sorry, %s is not whitelisted' % command) - elif (command in commands) and (manifest[0] == '!'): - return willie.reply('Sorry, %s is blacklisted' % command) - service(willie, trigger, command, args) -o.commands = ['o','urban'] -o.example = '.o servicename arg1 arg2 arg3' -o.services = {} -o.serviceURI = None - -def snippet(willie, trigger): - if not o.services: - refresh(willie) - - search = urllib.quote(trigger.group(2).encode('utf-8')) - py = "BeautifulSoup.BeautifulSoup(re.sub('<.*?>|(?<= ) +', '', " + \ - "''.join(chr(ord(c)) for c in " + \ - "eval(urllib.urlopen('http://ajax.googleapis.com/ajax/serv" + \ - "ices/search/web?v=1.0&q=" + search + "').read()" + \ - ".replace('null', 'None'))['responseData']['resul" + \ - "ts'][0]['content'].decode('unicode-escape')).replace(" + \ - "'"', '\x22')), convertEntities=True)" - service(willie, trigger, 'py', py) -snippet.commands = ['snippet'] - -if __name__ == '__main__': - print __doc__.strip() From f8ce5af81fc07c2a987e8e74424950ab7af92fa7 Mon Sep 17 00:00:00 2001 From: Elad Alfassa Date: Thu, 6 Jun 2013 13:26:17 +0300 Subject: [PATCH 05/14] [search] remove Bing support It barely works: outputs unnecessarily long links which are never relevant to what you're looking for. Also, nobody uses bing apart from Microsoft employees (and they only do so because company policy forces them to do so) --- search.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/search.py b/search.py index 5eb74810fe..e28743a1d0 100644 --- a/search.py +++ b/search.py @@ -119,30 +119,8 @@ def bing_search(query, lang='en-GB'): if m: return m.group(1) - -def bing(bot, trigger): - """Queries Bing for the specified input.""" - query = trigger.group(2) - if query.startswith(':'): - lang, query = query.split(' ', 1) - lang = lang[1:] - else: - lang = 'en-GB' - if not query: - return bot.reply('.bing what?') - - uri = bing_search(query, lang) - if uri: - bot.reply(uri) - bot.memory['last_seen_url'][trigger.sender] = uri - else: - bot.reply("No results found for '%s'." % query) -bing.commands = ['bing'] -bing.example = '.bing swhack' - r_duck = re.compile(r'nofollow" class="[^"]+" href="(.*?)">') - def duck_search(query): query = query.replace('!', '') query = web.quote(query) From c5ebfa192263530a292bbefa0626fa2b6abf9c4f Mon Sep 17 00:00:00 2001 From: Elad Alfassa Date: Thu, 6 Jun 2013 14:07:48 +0300 Subject: [PATCH 06/14] [units] Fix syntax error Someone changed the conditions and forgot to test his code before committing --- units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/units.py b/units.py index 3d7e1b87f5..c6a5883c6a 100644 --- a/units.py +++ b/units.py @@ -66,7 +66,7 @@ def distance(bot, trigger): meter = numeric / 39.370 elif unit in ("centimeters","centimeter","cm"): meter = numeric / 100 - elif unit in ("feet","foot","ft": + elif unit in ("feet","foot","ft"): meter = numeric / 3.2808 elif unit in ("yards", "yard", "yd"): meter = numeric / (3.2808 * 3) From 736bcb0e8b2e3a696ad179c16f7236545f7686aa Mon Sep 17 00:00:00 2001 From: Elad Alfassa Date: Fri, 7 Jun 2013 12:30:34 +0300 Subject: [PATCH 07/14] [seen] Fix import --- seen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seen.py b/seen.py index a52744750b..497c57bab5 100644 --- a/seen.py +++ b/seen.py @@ -12,7 +12,7 @@ import datetime import pytz from willie.tools import Ddict, Nick -from willie.module import command, rule +from willie.module import command, rule, priority seen_dict = Ddict(dict) From e1211af98e3990b02125770b629d00be15173937 Mon Sep 17 00:00:00 2001 From: Edward Powell Date: Sat, 8 Jun 2013 15:48:28 -0400 Subject: [PATCH 08/14] [url] Fix error on Unicode pages --- url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/url.py b/url.py index 2047a0696e..b7a3c1fd8a 100644 --- a/url.py +++ b/url.py @@ -186,7 +186,7 @@ def check_callbacks(willie, trigger, url, run=True): def find_title(url): """Return the title for the given URL.""" - content = web.get(url) + content = web.get(url).decode('utf8') # Some cleanup that I don't really grok, but was in the original, so # we'll keep it (with the compiled regexes made global) for now. content = title_tag_data.sub(r'<\1title>', content) From b1243015249a7633a7644e1ee3de310aef1f2fcf Mon Sep 17 00:00:00 2001 From: Edward Powell Date: Sat, 8 Jun 2013 16:40:48 -0400 Subject: [PATCH 09/14] [url] Respect page's reported encoding, otherwise try UTF-8 Also, fail silently if the above don't work. --- url.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/url.py b/url.py index b7a3c1fd8a..8e8ee8e539 100644 --- a/url.py +++ b/url.py @@ -186,7 +186,24 @@ def check_callbacks(willie, trigger, url, run=True): def find_title(url): """Return the title for the given URL.""" - content = web.get(url).decode('utf8') + # We want to try for UTF-8 if we can + content = web.get(url, headers={'Accept-Charset': 'utf-8'}) + headers = web.head(url, headers={'Accept-Charset': 'utf-8'}) + content_type = headers.get('Content-Type') + encoding = re.match('.*?charset *= *(\S+)', content_type).group(1) + # If they gave us something else instead, try that + if encoding: + try: + content = content.decode(encoding) + except: + encoding = None + # They didn't tell us what they gave us, so go with UTF-8 or fail silently. + if not encoding: + try: + content = content.decode('utf-8') + except: + return + # Some cleanup that I don't really grok, but was in the original, so # we'll keep it (with the compiled regexes made global) for now. content = title_tag_data.sub(r'<\1title>', content) From bfa7c690d76e5fb87dd9ae47bef2b55c18217052 Mon Sep 17 00:00:00 2001 From: Elad Alfassa Date: Sun, 9 Jun 2013 00:22:36 +0300 Subject: [PATCH 10/14] [core,url] Fix error where title was not shown for wikipedia pages Also remove cruft from web.py, urllib knows how to handle redirects itself --- url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/url.py b/url.py index 8e8ee8e539..1407f49c40 100644 --- a/url.py +++ b/url.py @@ -158,7 +158,7 @@ def follow_redirects(url): there's a problem. """ try: - connection = urllib2.urlopen(url) + connection = web.get_urllib_object(url, 60) url = connection.geturl() or url connection.close() except: From 4211d63b49b3779e41e2841308cbcedc4a3bfed7 Mon Sep 17 00:00:00 2001 From: Edward Powell Date: Sat, 8 Jun 2013 17:33:30 -0400 Subject: [PATCH 11/14] [url] Handle when a page does not specify its encoding --- url.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/url.py b/url.py index 1407f49c40..388450a1bb 100644 --- a/url.py +++ b/url.py @@ -190,15 +190,15 @@ def find_title(url): content = web.get(url, headers={'Accept-Charset': 'utf-8'}) headers = web.head(url, headers={'Accept-Charset': 'utf-8'}) content_type = headers.get('Content-Type') - encoding = re.match('.*?charset *= *(\S+)', content_type).group(1) + encoding_match = re.match('.*?charset *= *(\S+)', content_type) # If they gave us something else instead, try that - if encoding: + if encoding_match: try: - content = content.decode(encoding) + content = content.decode(encoding_match.group(1)) except: - encoding = None + encoding_match = None # They didn't tell us what they gave us, so go with UTF-8 or fail silently. - if not encoding: + if not encoding_match: try: content = content.decode('utf-8') except: From 0497fa2437aeb2c8c0fe22d9b2122c94429adf93 Mon Sep 17 00:00:00 2001 From: Edward Powell Date: Sat, 8 Jun 2013 17:35:23 -0400 Subject: [PATCH 12/14] [url] Handle when page doesn't give Content-Type at all --- url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/url.py b/url.py index 388450a1bb..f6728c259d 100644 --- a/url.py +++ b/url.py @@ -189,7 +189,7 @@ def find_title(url): # We want to try for UTF-8 if we can content = web.get(url, headers={'Accept-Charset': 'utf-8'}) headers = web.head(url, headers={'Accept-Charset': 'utf-8'}) - content_type = headers.get('Content-Type') + content_type = headers.get('Content-Type') or '' encoding_match = re.match('.*?charset *= *(\S+)', content_type) # If they gave us something else instead, try that if encoding_match: From f9b7bbaf46949b9f89977be9a36447cf41456d16 Mon Sep 17 00:00:00 2001 From: Elad Alfassa Date: Sun, 9 Jun 2013 00:45:05 +0300 Subject: [PATCH 13/14] [url,web] Only preform one request for each URL Doing both GET and HEAD is silly. This commit changes web.get a bit, to allow it to return headers if needed. Also, remove Accept-Charset because this seems to break certain misconfigured webservers --- url.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/url.py b/url.py index f6728c259d..0c377bf83f 100644 --- a/url.py +++ b/url.py @@ -186,9 +186,7 @@ def check_callbacks(willie, trigger, url, run=True): def find_title(url): """Return the title for the given URL.""" - # We want to try for UTF-8 if we can - content = web.get(url, headers={'Accept-Charset': 'utf-8'}) - headers = web.head(url, headers={'Accept-Charset': 'utf-8'}) + content, headers = web.get(url, return_headers=True) content_type = headers.get('Content-Type') or '' encoding_match = re.match('.*?charset *= *(\S+)', content_type) # If they gave us something else instead, try that From 6eb85cf7131680f601e1745d55fcd318e22992f9 Mon Sep 17 00:00:00 2001 From: Edward Powell Date: Sat, 8 Jun 2013 23:31:10 -0400 Subject: [PATCH 14/14] [calc, bugzilla, radio] Update to 4.0 standards, issue #276 --- bugzilla.py | 27 +++++++++-------- calc.py | 43 +++++++++++++-------------- radio.py | 86 +++++++++++++++++++++++++++-------------------------- 3 files changed, 78 insertions(+), 78 deletions(-) diff --git a/bugzilla.py b/bugzilla.py index de1dddd87e..bf152e9856 100644 --- a/bugzilla.py +++ b/bugzilla.py @@ -9,6 +9,7 @@ from lxml import etree import re from willie import web +from willie.module import rule import urllib import urllib2 @@ -28,27 +29,30 @@ def configure(config): 'Domain:') -def setup(willie): +def setup(bot): regexes = [] - if not (willie.config.has_option('bugzilla', 'domains') - and willie.config.bugzilla.get_list('domains')): + if not (bot.config.has_option('bugzilla', 'domains') + and bot.config.bugzilla.get_list('domains')): return - if not willie.memory.contains('url_callbacks'): - willie.memory['url_callbacks'] = {} + if not bot.memory.contains('url_callbacks'): + bot.memory['url_callbacks'] = {} - domains = '|'.join(willie.config.bugzilla.get_list('domains')) + domains = '|'.join(bot.config.bugzilla.get_list('domains')) regex = re.compile((r'https?://(%s)' '(/show_bug.cgi\?\S*?)' '(id=\d+)') % domains) - willie.memory['url_callbacks'][regex] = show_bug + bot.memory['url_callbacks'][regex] = show_bug -def show_bug(willie, trigger, match=None): +@rule(r'.*https?://(\S+?)' + '(/show_bug.cgi\?\S*?)' + '(id=\d+).*') +def show_bug(bot, trigger, match=None): """Show information about a Bugzilla bug.""" match = match or trigger domain = match.group(1) - if domain not in willie.config.bugzilla.get_list('domains'): + if domain not in bot.config.bugzilla.get_list('domains'): return url = 'https://%s%sctype=xml&%s' % match.groups() data = web.get(url) @@ -70,7 +74,4 @@ def show_bug(willie, trigger, match=None): (bug.find('priority').text + ' ' + bug.find('bug_severity').text), status, bug.find('assigned_to').text, bug.find('creation_ts').text, bug.find('delta_ts').text) - willie.say(message) -show_bug.rule = (r'.*https?://(\S+?)' - '(/show_bug.cgi\?\S*?)' - '(id=\d+).*') + bot.say(message) diff --git a/calc.py b/calc.py index 7693370cc2..c52dd848d0 100644 --- a/calc.py +++ b/calc.py @@ -8,7 +8,8 @@ """ import re -import willie.web as web +from willie import web +from willie.module import command, commands, example from socket import timeout import string import HTMLParser @@ -35,39 +36,41 @@ def calculate(q): return 'Sorry, no result.' -def c(willie, trigger): +@commands('c', 'calc') +@example('.c 5 + 3') +def c(bot, trigger): """Google calculator.""" if not trigger.group(2): - return willie.reply("Nothing to calculate.") + return bot.reply("Nothing to calculate.") result = calculate(trigger.group(2)) - willie.reply(result) -c.commands = ['c', 'calc'] -c.example = '.c 5 + 3' + bot.reply(result) -def py(willie, trigger): +@command('py') +@example('.py len([1,2,3])') +def py(bot, trigger): """Evaluate a Python expression.""" query = trigger.group(2) uri = 'http://tumbolia.appspot.com/py/' answer = web.get(uri + web.quote(query)) if answer: - willie.say(answer) + bot.say(answer) else: - willie.reply('Sorry, no result.') -py.commands = ['py'] -py.example = '.py len([1,2,3])' + bot.reply('Sorry, no result.') -def wa(willie, trigger): +@commands('wa', 'wolfram') +@example('.wa circumference of the sun * pi') +def wa(bot, trigger): """Wolfram Alpha calculator""" if not trigger.group(2): - return willie.reply("No search term.") + return bot.reply("No search term.") query = trigger.group(2) uri = 'http://tumbolia.appspot.com/wa/' try: answer = web.get(uri + web.quote(query.replace('+', '%2B')), 45) except timeout as e: - return willie.say('[WOLFRAM ERROR] Request timed out') + return bot.say('[WOLFRAM ERROR] Request timed out') if answer: answer = answer.decode('string_escape') answer = HTMLParser.HTMLParser().unescape(answer) @@ -81,17 +84,11 @@ def wa(willie, trigger): answer = answer.replace('\:' + char_code, char) waOutputArray = string.split(answer, ";") if(len(waOutputArray) < 2): - willie.say('[WOLFRAM ERROR]' + answer) + bot.say('[WOLFRAM ERROR]' + answer) else: - willie.say('[WOLFRAM] ' + waOutputArray[0] + " = " + bot.say('[WOLFRAM] ' + waOutputArray[0] + " = " + waOutputArray[1]) waOutputArray = [] else: - willie.reply('Sorry, no result.') -wa.commands = ['wa', 'wolfram'] -wa.example = '.wa circumference of the sun * pi' -wa.commands = ['wa'] - -if __name__ == '__main__': - print __doc__.strip() + bot.reply('Sorry, no result.') diff --git a/radio.py b/radio.py index 9623b71559..0b7947d7c6 100644 --- a/radio.py +++ b/radio.py @@ -9,8 +9,10 @@ from time import sleep from xml.dom.minidom import parseString import willie.web as web +from willie.module import command import xml.dom.minidom + def configure(config): """ | [radio] | example | purpose | @@ -23,83 +25,88 @@ def configure(config): config.interactive_add('radio', 'url', 'URL to the ShoutCAST administration page', 'http://127.0.0.1:8000/') config.interactive_add('radio', 'sid', 'Stream ID (only required for multi-stream servers.)', '1') -radioURL = '' # Set once, after the first .radio request. +radioURL = '' # Set once, after the first .radio request. checkSongs = 0 current_song = '' -def getAPI(willie, trigger): + +def getAPI(bot, trigger): #contact the 'heavyweight' XML API try: raw = web.get(radioURL % 'stats') except Exception as e: - willie.say('The radio is not responding to the stats request.') + bot.say('The radio is not responding to the stats request.') return 0 - + #Parse the XML XML = parseString(raw).documentElement status = XML.getElementsByTagName('STREAMSTATUS')[0].firstChild.nodeValue if status == '0': - willie.say('The radio is currently offline.') + bot.say('The radio is currently offline.') return 0 status = 'Online' servername = XML.getElementsByTagName('SERVERTITLE')[0].firstChild.nodeValue curlist = XML.getElementsByTagName('CURRENTLISTENERS')[0].firstChild.nodeValue maxlist = XML.getElementsByTagName('MAXLISTENERS')[0].firstChild.nodeValue - + #Garbage disposal XML.unlink() #print results - willie.say('[%s]Status: %s. Listeners: %s/%s.' % (servername, status, curlist, maxlist)) + bot.say('[%s]Status: %s. Listeners: %s/%s.' % (servername, status, curlist, maxlist)) return 1 -def currentSong(willie, trigger): + +def currentSong(bot, trigger): # This function uses the PLAINTEXT API to get the current song only. try: song = web.get(radioURL % 'currentsong') except Exception as e: - willie.say('The radio is not responding to the song request.') - willie.debug('radio', 'Exception while trying to get current song: %s' % e, 'warning') + bot.say('The radio is not responding to the song request.') + bot.debug('radio', 'Exception while trying to get current song: %s' % e, 'warning') if song: - willie.say('Now playing: '+song) + bot.say('Now playing: ' + song) else: - willie.say('The radio is currently offline.') + bot.say('The radio is currently offline.') -def nextSong(willie, trigger): + +def nextSong(bot, trigger): # This function uses the PLAINTEXT API to get the next song only. try: song = web.get(radioURL % 'nextsong') except Exception as e: - willie.say('The radio is not responding to the song request.') - willie.debug('radio', 'Exception while trying to get next song: %s' % e, 'warning') + bot.say('The radio is not responding to the song request.') + bot.debug('radio', 'Exception while trying to get next song: %s' % e, 'warning') if song: - willie.say('Next up: '+song) + bot.say('Next up: ' + song) else: - willie.say('No songs are queued up.') + bot.say('No songs are queued up.') + -def radio(willie, trigger): +@command('radio') +def radio(bot, trigger): """ Radio functions, valid parameters: on, off, song, now, next, soon, stats. """ global checkSongs, current_song, radioURL if not radioURL: - if not hasattr(willie.config, 'radio'): - willie.say('Radio module not configured') + if not hasattr(bot.config, 'radio'): + bot.say('Radio module not configured') return else: - radioURL = willie.config.radio.url+'%s?sid='+willie.config.radio.sid + radioURL = bot.config.radio.url + '%s?sid=' + bot.config.radio.sid try: args = trigger.group(2).lower().split(' ') except AttributeError: - willie.say('Usage: .radio (next|now|off|on|song|soon|stats)') + bot.say('Usage: .radio (next|now|off|on|song|soon|stats)') return if args[0] == 'on': if not trigger.isop: return if checkSongs != 0: - return willie.reply('Radio data checking is already on.') - if not getAPI(willie, trigger): + return bot.reply('Radio data checking is already on.') + if not getAPI(bot, trigger): checkSongs = 0 - return willie.say('Radio data checking not enabled.') + return bot.say('Radio data checking not enabled.') checkSongs = 10 while checkSongs: last = current_song @@ -109,37 +116,32 @@ def radio(willie, trigger): except Exception as e: checkSongs -= 1 if checkSongs == 0: - willie.debug('radio', 'Exception while trying to get periodic radio data: %s' % e, 'warning') - willie.say('The radio is not responding to the song request.') - willie.say('Turning off radio data checking.') + bot.debug('radio', 'Exception while trying to get periodic radio data: %s' % e, 'warning') + bot.say('The radio is not responding to the song request.') + bot.say('Turning off radio data checking.') break if not current_song == last: if not current_song: csong = 'The radio is currently offline.' else: - csong = 'Now Playing: '+current_song + csong = 'Now Playing: ' + current_song if nextsong and current_song: - willie.say(csong+' | Coming Up: '+nextsong) + bot.say(csong + ' | Coming Up: ' + nextsong) else: - willie.say(csong) + bot.say(csong) sleep(5) elif args[0] == 'off': if not trigger.isop: - return; + return if checkSongs == 0: - willie.reply('Radio data checking is already off.') + bot.reply('Radio data checking is already off.') return checkSongs = 0 current_song = '' - willie.reply('Turning off radio data checking.') + bot.reply('Turning off radio data checking.') elif args[0] == 'song' or args[0] == 'now': - currentSong(willie, trigger) + currentSong(bot, trigger) elif args[0] == 'next' or args[0] == 'soon': - nextSong(willie, trigger) + nextSong(bot, trigger) elif args[0] == 'stats': - getAPI(willie, trigger) -radio.commands = ['radio'] -radio.priority = 'medium' - -if __name__ == '__main__': - print __doc__.strip() + getAPI(bot, trigger)