Skip to content

Commit

Permalink
ENH: Revised loglevels
Browse files Browse the repository at this point in the history
fixes #614

Signed-off-by: Sebastian Wagner <[email protected]>
  • Loading branch information
sebix committed Nov 16, 2016
1 parent 06cde54 commit d4dd349
Show file tree
Hide file tree
Showing 12 changed files with 62 additions and 94 deletions.
7 changes: 1 addition & 6 deletions intelmq/bots/collectors/blueliv/collector_crimeserver.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,8 @@ def init(self):
self.logger.error('Could not import BluelivAPI. Please install it.')
self.stop()

def init(self):
if BluelivAPI is None:
self.logger.error('Could not import sdk.blueliv_api.BluelivAPI. Please install it.')
self.stop()

def process(self):
self.logger.info("Downloading report through API")
self.logger.debug("Downloading report through API.")
http_proxy = getattr(self.parameters, 'http_proxy', None)
https_proxy = getattr(self.parameters, 'http_ssl_proxy', None)
proxy = None
Expand Down
26 changes: 12 additions & 14 deletions intelmq/bots/collectors/ftp/collector_ftp.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@

class FTPCollectorBot(CollectorBot):
def process(self):
self.logger.info("Downloading report from %s" %
self.parameters.ftp_host + ':' +
str(self.parameters.ftp_port))
self.logger.info("Downloading report from %s." %
(self.parameters.ftp_host + ':' +
str(self.parameters.ftp_port)))

ftp = FTP()
ftp.connect(host=self.parameters.ftp_host,
Expand All @@ -37,39 +37,37 @@ def process(self):
passwd=self.parameters.ftp_password)
cwd = '/'
if hasattr(self.parameters, 'ftp_directory'):
self.logger.info('Changing working directory to: ' +
self.parameters.ftp_directory)
self.logger.debug('Changing working directory to: %r.'
'' % self.parameters.ftp_directory)
cwd = self.parameters.ftp_directory
ftp.cwd(cwd)

filemask = '*'
if hasattr(self.parameters, 'ftp_file'):
self.logger.info('Setting filemask to to: ' +
self.parameters.ftp_file)
self.logger.debug('Setting filemask to to: %r.'
'' % self.parameters.ftp_file)
filemask = self.parameters.ftp_file

mem = io.BytesIO()
files = fnmatch.filter(ftp.nlst(), filemask)
self.logger.info('Found following files in the directory: ' +
repr(files))
self.logger.info('Looking for latest file matching following pattern: ' + filemask)

if files:
self.logger.info('Retrieving file: ' + files[-1])
self.logger.info('Retrieving file: %r.' % files[-1])
ftp.retrbinary("RETR " + files[-1], mem.write)
else:
self.logger.error("No file found, terminating download")
self.logger.info("No file found, terminating download.")
return

self.logger.info("Report downloaded.")
self.logger.debug("Report downloaded.")

raw_reports = []
try:
zfp = zipfile.ZipFile(mem, "r")
except zipfile.BadZipfile:
raw_reports.append(mem.getvalue())
else:
self.logger.info('Downloaded zip file, extracting following files: ' + ', '.join(zfp.namelist()))
self.logger.info('Downloaded zip file, extracting following files: %r'
'' % zfp.namelist())
for filename in zfp.namelist():
raw_reports.append(zfp.read(filename))

Expand Down
20 changes: 9 additions & 11 deletions intelmq/bots/collectors/ftp/collector_ftps.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,9 @@ def connect(self, host='', port=0, timeout=-999):

class FTPSCollectorBot(CollectorBot):
def process(self):
self.logger.info("Downloading report from %s" %
self.parameters.ftps_host + ':' +
str(self.parameters.ftps_port))
self.logger.info("Downloading report from %s." %
(self.parameters.ftp_host + ':' +
str(self.parameters.ftp_port)))

ftps = FTPS()
ftps.connect(host=self.parameters.ftps_host,
Expand All @@ -73,22 +73,19 @@ def process(self):

cwd = '/'
if hasattr(self.parameters, 'ftps_directory'):
self.logger.info('Changing working directory to: ' +
self.parameters.ftps_directory)
self.logger.debug('Changing working directory to: %r.'
'' % self.parameters.ftp_directory)
cwd = self.parameters.ftps_directory
ftps.cwd(cwd)

filemask = '*'
if hasattr(self.parameters, 'ftps_file'):
self.logger.info('Setting filemask to to: ' +
self.parameters.ftps_file)
self.logger.debug('Setting filemask to to: %r.'
'' % self.parameters.ftp_file)
filemask = self.parameters.ftps_file

mem = io.BytesIO()
files = fnmatch.filter(ftps.nlst(), filemask)
self.logger.info('Found following files in the directory: ' +
repr(files))
self.logger.info('Looking for latest file matching following pattern: ' + filemask)

if files:
self.logger.info('Retrieving file: ' + files[-1])
Expand All @@ -105,7 +102,8 @@ def process(self):
except zipfile.BadZipfile:
raw_reports.append(mem.getvalue())
else:
self.logger.info('Downloaded zip file, extracting following files: ' + ', '.join(zfp.namelist()))
self.logger.info('Downloaded zip file, extracting following files: %r'
'' % zfp.namelist())
for filename in zfp.namelist():
raw_reports.append(zfp.read(filename))

Expand Down
4 changes: 1 addition & 3 deletions intelmq/bots/collectors/mail/collector_mail_attach.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@ def process(self):
message.subject)):
continue

self.logger.info("Reading email report")

for attach in message.attachments:
if not attach:
continue
Expand All @@ -61,7 +59,7 @@ def process(self):
# so other instances watching this mailbox will still
# check it.
mailbox.mark_seen(uid)
self.logger.info("Email report read")
self.logger.debug("Email report read.")
mailbox.logout()


Expand Down
6 changes: 2 additions & 4 deletions intelmq/bots/collectors/mail/collector_mail_url.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@ def process(self):
message.subject)):
continue

self.logger.info("Reading email report")

for body in message.body['plain']:
match = re.search(self.parameters.url_regex, str(body))
if match:
Expand Down Expand Up @@ -66,7 +64,7 @@ def process(self):

self.http_header['User-agent'] = self.parameters.http_user_agent

self.logger.info("Downloading report from %s" % url)
self.logger.info("Downloading report from %r." % url)
resp = requests.get(url=url,
auth=self.auth, proxies=self.proxy,
headers=self.http_header,
Expand All @@ -86,7 +84,7 @@ def process(self):
# so other instances watching this mailbox will still
# check it.
mailbox.mark_seen(uid)
self.logger.info("Email report read")
self.logger.info("Email report read.")
mailbox.logout()


Expand Down
12 changes: 6 additions & 6 deletions intelmq/bots/collectors/n6/collector_stomp.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@ def __init__(self, n6stompcollector):
self.n6stomper = n6stompcollector

def on_heartbeat_timeout(self):
self.n6stomper.logger.warn("Heartbeat timeout. Attempting to re-connect")
self.n6stomper.logger.warn("Heartbeat timeout. Attempting to re-connect.")
self.n6stomper.conn.disconnect()
status = self.n6stomper.conn.connect(wait=False)
self.n6stomper.logger.warn("Re-connected: {}".format(status))
self.n6stomper.logger.info("Re-connected: {}.".format(status))

def on_error(self, headers, message):
self.n6stomper.logger.warn('Received an error :"%s".' % repr(message))
self.n6stomper.logger.error('Received an error :"%s".' % repr(message))

def on_message(self, headers, message):
self.n6stomper.logger.debug('Receive message '
Expand All @@ -34,7 +34,6 @@ def on_message(self, headers, message):
":" + str(self.n6stomper.parameters.port) +
"/" + self.n6stomper.parameters.exchange)
self.n6stomper.send_message(report)
self.n6stomper.logger.debug('Receiving Message.')
except ImportError:
stomp = None

Expand Down Expand Up @@ -64,7 +63,7 @@ def init(self):
# check if certificates exist
for f in [self.ssl_ca_cert, self.ssl_cl_cert, self.ssl_cl_cert_key]:
if not os.path.isfile(f):
raise ValueError("Could not open file '%s'." % f)
raise ValueError("Could not open file %r." % f)

_host = [(self.server, self.port)]
self.conn = stomp.Connection(host_and_ports=_host, use_ssl=True,
Expand All @@ -79,7 +78,8 @@ def init(self):
self.conn.start()
self.conn.connect(wait=False)
self.conn.subscribe(destination=self.exchange, id=1, ack='auto')
self.logger.info('Successfully connected and subscribed to {}:{}'.format(self.server, self.port))
self.logger.info('Successfully connected and subscribed to {}:{}.'
''.format(self.server, self.port))

def disconnect(self):
self.conn.disconnect()
Expand Down
15 changes: 7 additions & 8 deletions intelmq/bots/collectors/xmpp/collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def __init__(self,
sleekxmpp.ClientXMPP.__init__(self, jid, password)

self.logger = logger
self.logger.info("Initiated")
self.logger.info("Initiated.")
self.xmpp_room = room
self.xmpp_room_nick = room_nick
self.xmpp_room_password = room_password
Expand All @@ -51,20 +51,20 @@ def __init__(self,

def session_start(self, event):
self.send_presence()
self.logger.debug("Session started")
self.logger.debug("Session started.")

try:
self.get_roster()
except sleekxmpp.exceptions.IqError as err:
self.logger.error('There was an error getting the roster')
self.logger.error('There was an error getting the roster.')
self.logger.error(err.iq['error']['condition'])
self.disconnect()
except sleekxmpp.exceptions.IqTimeout:
self.logger.error('Server is taking too long to respond')
self.logger.error('Server is taking too long to respond.')
self.disconnect()

if self.xmpp_room and self.plugin.get('xep_0045'):
self.logger.debug("Joining room: %s", self.xmpp_room)
self.logger.debug("Joining room: %s." % self.xmpp_room)
pwd = self.xmpp_room_password if self.xmpp_room_password else ""
self.plugin['xep_0045'].joinMUC(self.xmpp_room,
self.xmpp_room_nick,
Expand Down Expand Up @@ -105,7 +105,7 @@ def process(self):
def stop(self):
if self.xmpp:
self.xmpp.disconnect()
self.logger.info("Disconnected from XMPP")
self.logger.info("Disconnected from XMPP.")

super(XMPPCollectorBot, self).stop()
else:
Expand All @@ -125,8 +125,7 @@ def log_message(self, msg):
else:
tmp_body = body

self.logger.debug("Received Stanza: %r from %r", tmp_body,
msg['from'])
self.logger.debug("Received Stanza: %r from %r." % (tmp_body, msg['from']))

raw_msg = body

Expand Down
2 changes: 1 addition & 1 deletion intelmq/bots/experts/generic_db_lookup/expert.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ def process(self):
# Skip events with missing match-keys
for key in self.match.keys():
if key not in event:
self.logger.warning('%s not present in event. Skipping event' % key)
self.logger.debug('%s not present in event. Skipping event.' % key)
self.send_message(event)
self.acknowledge_message()
return
Expand Down
5 changes: 0 additions & 5 deletions intelmq/bots/experts/reverse_dns/expert.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,6 @@ def process(self):
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6

else:
self.logger.warning("Invalid IP version {}".format(ip_version))
self.send_message(event)
self.acknowledge_message()

cache_key = bin(ip_integer)[2: minimum + 2]
cachevalue = self.cache.get(cache_key)

Expand Down
2 changes: 1 addition & 1 deletion intelmq/bots/outputs/mongodb/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def init(self):
self.connect()

def connect(self):
self.logger.info('Connecting to mongodb server.')
self.logger.debug('Connecting to mongodb server.')
try:
self.client = pymongo.MongoClient(self.parameters.host,
int(self.parameters.port))
Expand Down
18 changes: 8 additions & 10 deletions intelmq/bots/outputs/xmpp/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,22 +32,22 @@ def __init__(self, jid, password, logger):
sleekxmpp.ClientXMPP.__init__(self, jid, password)

self.logger = logger
self.logger.info("Connected to Server")
self.logger.info("Connected to Server.")

self.add_event_handler("session_start", self.session_start)

def session_start(self, event):
self.send_presence()
self.logger.debug("Session started")
self.logger.debug("Session started.")

try:
self.get_roster()
except sleekxmpp.exceptions.IqError as err:
self.logger.error('There was an error getting the roster')
self.logger.error('There was an error getting the roster.')
self.logger.error(err.iq['error']['condition'])
self.disconnect()
except sleekxmpp.exceptions.IqTimeout:
self.logger.error('Server is taking too long to respond')
self.logger.error('Server is taking too long to respond.')
self.disconnect()
except ImportError:
sleekxmpp = None
Expand Down Expand Up @@ -79,24 +79,22 @@ def process(self):
try:
# TODO: proper error handling. Right now it cannot be
# detected if the message was sent successfully.
self.logger.debug("Trying to send Event: %r to %r",
jevent,
receiver)
self.logger.debug("Trying to send Event: %r to %r." % (jevent, receiver))
self.xmpp.send_message(mto=receiver, mbody=jevent)
except sleekxmpp.exceptions.XMPPError as err:
self.logger.error('There was an error when sending the event')
self.logger.error('There was an error when sending the event.')
self.logger.error(err.iq['error']['condition'])

self.acknowledge_message()

def stop(self):
if self.xmpp:
self.xmpp.disconnect()
self.logger.info("Disconnected from XMPP")
self.logger.info("Disconnected from XMPP.")

super(XMPPOutputBot, self).stop()
else:
self.logger.info("There was no XMPPClient I could stop")
self.logger.info("There was no XMPPClient I could stop.")


if __name__ == "__main__":
Expand Down
Loading

0 comments on commit d4dd349

Please sign in to comment.