diff --git a/willie/bot.py b/willie/bot.py
index 6f1d4944a3..d05a1c8cce 100644
--- a/willie/bot.py
+++ b/willie/bot.py
@@ -262,7 +262,10 @@ def setup(self):
                 module = imp.load_source(name, filename)
             except Exception, e:
                 error_count = error_count + 1
-                stderr("Error loading %s: %s (in bot.py)" % (name, e))
+                filename, lineno = tools.get_raising_file_and_line()
+                rel_path = os.path.relpath(filename, os.path.dirname(__file__))
+                raising_stmt = "%s:%d" % (rel_path, lineno)
+                stderr("Error loading %s: %s (%s)" % (name, e, raising_stmt))
             else:
                 try:
                     if hasattr(module, 'setup'):
@@ -271,8 +274,11 @@ def setup(self):
                     modules.append(name)
                 except Exception, e:
                     error_count = error_count + 1
-                    stderr("Error in %s setup procedure: %s (in bot.py)"
-                           % (name, e))
+                    filename, lineno = tools.get_raising_file_and_line()
+                    rel_path = os.path.relpath(filename, os.path.dirname(__file__))
+                    raising_stmt = "%s:%d" % (rel_path, lineno)
+                    stderr("Error in %s setup procedure: %s (%s)"
+                           % (name, e, raising_stmt))
 
         if modules:
             stderr('\n\nRegistered %d modules,' % (len(modules) - 1))
diff --git a/willie/tools.py b/willie/tools.py
index b2080e5338..5939ce6b2d 100644
--- a/willie/tools.py
+++ b/willie/tools.py
@@ -28,6 +28,19 @@
 import copy
 
 
+def get_raising_file_and_line(tb=None):
+    """Return the file and line number of the statement that raised the tb
+
+    Returns: (filename, lineno) tuple
+    """
+    if not tb:
+        tb = sys.exc_info()[2]
+
+    filename, lineno, _context, _line = traceback.extract_tb(tb)[-1]
+
+    return filename, lineno
+
+
 def get_command_regexp(prefix, command):
     """Return a compiled regexp object that implements the command."""
     # Escape all whitespace with a single backslash. This ensures that regexp