logchecks.py

#! /usr/bin/env python
from xml.dom.minidom import Document
from datetime import datetime
from collections import deque
from socket import gethostname
import getpass
import re
import socket
import string
import getopt,sys
from urlparse import urlsplit
import urllib2
import os
#,pwd
import time
import smtplib


""" Class to represent log aka CAL Messagws
Any values that are not found in the message are set to "" """ 
class CalMessage(object):
  lineNumber=0
	calClass="" # can be t, T, A, E
	calTimestamp="" #A string of form HH:MM:SS.ff
	calType="" #Message type
	calName="" #Message name
	calStatus="" #Status
	calDuration="" #Duration
	calPayload="" #Payload 
	def getLineNumber(self):
		return self.lineNumber
	def getTimeStamp(self):
		return self.calTimestamp
	def getCalClass(self):
		return self.calClass
	def getCalType(self):
		return self.calType
	def getCalName(self):
		return self.calName
	def getCalStatus(self):
		return self.calStatus
	def getCalDuration(self):
		 return self.calDuration
	def getCalPayload(self):
		return self.calPayload
	def isRootTransactionEnd(self):
		pl=self.getCalPayload()
		if ("corr_id_=" in pl ) and ("session_id_=" in pl ):
			return True
		else:
			return False
	#Convenience function to print all the class members
	def printMessage(self):
		print self.getLineNumber() 
		print self.getCalClass() 
		print self.getTimeStamp()
		print self.getCalType()
		print self.getCalName()
		print self.getCalStatus()
		print self.getCalDuration()
		print self.getCalPayload()
	#Returns a string representation of comma seperated class member values
	def toString(self):
		return str(self.getLineNumber())+" "+self.getCalClass()+" "+self.getTimeStamp()+" "+ self.getCalType()+" "+self.getCalName()+" "+self.getCalStatus()+" "+self.getCalDuration()+" "+self.getCalPayload()

	#Calculate length of cal message - sum of lengths of each field
	def calMesgLen(self):
                CalLen=None
                CalLen=len(self.getTimeStamp())+len(self.getCalClass())+len(self.getCalType())+len(self.getCalName())+len(self.getCalStatus())+len(self.getCalDuration())+len(self.getCalPayload())
                return CalLen

	#Create a object cal message from a string. The string must be a space/tab seperated CAL message
	def __init__(self,num,line):
		self.lineNumber=num
		words=line.split()
		if (words[0]=="[calmsg]"):
			words.remove(words[0])
		nwords=len(words)
		if (nwords>0):
			 self.calClass=words[0][0]
			 self.calTimestamp=words[0][1:]
		if (nwords>1):
			 self.calType=words[1]
		if (nwords>2):
			 self.calName=words[2]
		if (nwords>3):
			 self.calStatus=words[3]
		if (nwords>4):
			 self.calDuration=words[4]
		if (nwords>5):
			 self.calPayload=words[5]

class CalError(object):
	
	global errmap
	errmap={
		"E01":["Bad nesting. Some messages don't have a matching start/end transaction","Error"],
		"E02":["Duplicate names within transaction hierarchy","Error"],
		"W03":["Bad Instrumentation messages generated by CalClient are seen in logs","Error"],
		"W04":["Unclosed Transaction messages generated by CalClient are seen in logs","Error"],
		"W05":["Invalid character data found in the data field","Error"],
		"W06":["Cal message size exceeds a maxiumum allowed of 4096 bytes","Warning"],
		"E07":["Cal messages of Type SEND or RECV must contain the size field in Payload","Error"],
		"E08":["CAL messages of Type WARN,ERROR,FATAL,and EXCEPTION must have non zero Status","Error"],
		"E09":["CAL messages of Type WARN,ERROR,FATAL,and EXCEPTION must be of Event Class","Error"],
		"E10":["CAL message Status is not in proper format","Warning"],
		"E11":["CAL message Name is not in proper format","Warning"],
		"E12":["CAL message Type is not in proper format","Warning"],
		"E13":["CAL Transaction should be of at least 10 millisecs duration","Error"],
		"E14":["CAL Log file must start with a transaction","Warning"],
		"W15":["Wrong Chronological order","Warning"],
		"E16":["CAL Type of API or URL must be used only by root transactions","Error"]
               }
	id=""
	desc=""

	def __init__(self,id):
		self.id=id
		self.desc=(errmap[id])[0]

	#Print the error in XML format
	def printError(self,errq):
		err= doc.createElement("Error")
		id= doc.createElement("Id")
		desc= doc.createElement("Description")
		msglist= doc.createElement("MessageList")

		err.appendChild(id)
		id.appendChild( doc.createTextNode(self.id) )
		err.appendChild(desc)
		desc.appendChild( doc.createTextNode(self.desc) )
		msgs=""
	 	for m in errq:
			msg=doc.createElement("Message")
			if isinstance(m, CalMessage):
				msg.appendChild( doc.createTextNode(m.toString()) )
			else:
				msg.appendChild( doc.createTextNode(m) )
			msglist.appendChild(msg)
		err.appendChild(msglist)
		root.appendChild(err)
		
	
#---- Tests ----
class CalChecker(object):
	# Create the minidom document
	global doc, root
	doc = Document()
	# Create the base element
	root= doc.createElement("CalVal")
	doc.appendChild(root)
	pi = doc.createProcessingInstruction('xml-stylesheet', 'type="text/xsl" href="calval.xsl"')
	doc.insertBefore(pi, root)

	def getReport(self,reportFile):
		#Write XML report
	 	outfile=open(reportFile,"w")
		doc.writexml(outfile,indent="",addindent=" ", newl="\n")
		outfile.close()
		

	def printHeader(self):
		#generate header info
		start= doc.createElement("StartTime")
		root.appendChild(start)
		start.appendChild( doc.createTextNode(str(datetime.now())) )
		input= doc.createElement("InputLocation")
		root.appendChild(input)
		input.appendChild( doc.createTextNode("location TBD") )
		host= doc.createElement("Host")
		root.appendChild(host)
		host.appendChild( doc.createTextNode(gethostname()) )
		user= doc.createElement("User")
		root.appendChild(user)
		user.appendChild( doc.createTextNode(getpass.getuser()) )

	def printFooter(self,idx):
		#generate footer info
		nmsgs= doc.createElement("NumMessages")
		root.appendChild(nmsgs)
		nmsgs.appendChild( doc.createTextNode(str(idx)) )
		end= doc.createElement("EndTime")
		root.appendChild(end)
		end.appendChild( doc.createTextNode(str(datetime.now())) )
		linkstr="Refer to CAL Guidelines document at: https://dev.paypal.com/wiki/Base/CalMsgPrinciples"
		foot= doc.createElement("Footer")
		foot.appendChild( doc.createTextNode(linkstr) )
		root.appendChild(foot)
		
		
	def  checkFileStart(self,m):
                if (m.getCalClass()!="t") :
			ce=CalError("E14")
			ce.printError(deque([m]))

	#Check if all CAL messages are in right chronlogical order 
	def checkChrono(self, msg1, msg2):
		tf="%H:%M:%S.%f" #time format example 12:59:45.05
		t1=datetime.strptime(msg1.getTimeStamp(),tf)
		t2=datetime.strptime(msg2.getTimeStamp(),tf)
		if ((t1)>(t2)):
			ce=CalError("W15")
			ce.printError(deque([msg1,msg2]))

	#Check if API and URL are used only for root transaction
	def checkRootType(self,msgq):
		errq=deque([])
		nmsg=len(msgq) - 2 #Skip first and last - root transaction messages
		for i in range(1,nmsg):
			if ( msgq[i].getCalType()in ["API","URL"] ): 
				errq.append(msgq[i])
		if (len(errq)>0):
			ce=CalError("E16")
			ce.printError(errq)

	#Check if every t and T match
	def checkNesting(self,msgq):
		errq=deque([]) #Check bad nesting
		lgth=len(msgq) - 1
		for i in (0,lgth):
			m=msgq[i]
			# Improper nesting check - create queue here, check after loop
			if (m.getCalClass() in ['T']): 
				errq.append(m) #Add T to nested transaction queue
			if (m.getCalClass() in ['t']): 
				try:
					errq.pop() #If t remove top most "t" from nested transaction queue
				except IndexError:
					pass
		if ( len(errq) > 0 ):
			ce=CalError("E01")
			ce.printError(errq)

	#Check duplicate names with a transaction's hierarchy
	def checkDuplicateName(self,msgq):
		errq=deque([]) 
		lst=list()
		lgth=len(msgq) - 1
		item=""
		# To make search more efficient copy the names to a list
		for j in range(0,lgth):
			if (msgq[j].getCalClass()!="H"): #Skip heartbeats
				lst.append(msgq[j].getCalName())
		uniqueSet = set(item for item in lst) #Create an uniqueSet based on the lists
		for item in uniqueSet:
			if lst.count(item) > 1: # if there is more than one occurence (duplicate)
				errq.append(item) # add to errors queue
		if ( len(errq) > 0 ):
			ce=CalError("E02")
			ce.printError(errq)

	# Returns true if the strign contains only digits else false
	def is_digit(self,s):
		"""return false if string contains atleast one non-digit"""
        	if s.isdigit():
        	        return True
        	return False
	# grep functionality in python		
	def grep(self,pattern, file_obj, include_line_nums=False):
	    """grep functionality"""
	    grepper = re.compile(pattern)
	    for line_num, line in enumerate(file_obj):
	        if grepper.search(line):
	            if include_line_nums:
	                yield (line_num, line)
	            else:
	                yield line
	#is avalid calfile
	def isValidcalfile(self,file_obj):
		err=[]
		gen=self.grep("Environment:|SQLLog|Label:|Start:|\$[0-9]+|\[calclient\]|^\[calmsg\]",file_obj,False)
		try:		
			gen.next()
		except StopIteration:
				return False
		else:
			print "is a valid calfile"
			return True
			
	#Checks for BadInstrumentation messages in log file
	def checkBadInstrumentationMsg(self,file_obj):
		"""Check  for BadInstrumentation messages in logfile  """
		print file_obj
		err=[]
		for elem in self.grep('BadInstrumentation', file_obj,True):
			err.append(repr(elem))
			print elem
		if ( len(err) > 0 ):
			ce=CalError("W03")
			ce.printError(err)
	#Checks for UnclosedTransaction in log file
	def checkUnclosedTransaction(self,file_obj):
		"""Check  for UnclosedTransaction from Cal Daemon"""
		err=[]
		for elem in self.grep('UnclosedTransaction', file_obj, True):
			err.append(repr(elem))
                if ( len(err) > 0 ):
			 ce=CalError("W04")
			 ce.printError(err)
	#Checks for invalidCALData in log file
	def checkInvalidCharData(self,file_obj):
		"""Check for InvalidCALData from Cal"""
		err=[]
		for elem in self.grep('__InvalidCALData__',file_obj,True):
			err.append(repr(elem))
                if ( len(err) > 0 ):
			 ce=CalError("W05")
			 ce.printError(err)

	#valides the cal message size
	def checkCalMsgSize(self,mesg):
		"""CAL message size is limited to 4k bytes. 12 byte internal cal message header + (4*1024)byte CAL message body. Message body exceeding 4K will be chopped to 4K"""
		err=[]
		if mesg.calMesgLen() + 12 >= 4096:
			ce=CalError("W06")
			err.append(mesg)
			ce.printError(err)
		else:
			pass		
	# CAL messages of type SEND, RECV should have payload size in data field
	def checkPayloadSendRecv(self,mesg):
		"""CAL messages of type SEND, RECV should have payload size in data field."""
		pat=r'(len|ilen|olen)=\d+(&|\S)'
		prog=re.compile(pat)
		if mesg.getCalType() in ("SEND","RECV"):
			if not prog.search(mesg.getCalPayload()):
				ce=CalError("E07")
				ce.printError([mesg])
			else:
				pass
		else:
			pass		
	#Checks for CAL Transaction  duration
	def checkTenMilliSecondsDuration(self,mesg):
		"""CAL Transaction should be of at least 10millisecs duration """
		if mesg.isRootTransactionEnd():
			if not float(mesg.getCalDuration())>10:
				ce=CalError("E13")
				ce.printError([mesg])
			
	#CAL messages of type WARN, ERROR or FATAL,EXCEPTION must have non zero status.(line)
	def checkNonZeroStatusCheckWarnErrorExceptionFatal(self,m1):
	        """Check CAL messages of type WARN, ERROR or FATAL,EXCEPTION must have non zero status.(line)"""
		err=[]
	        if  m1.getCalType() in ("WARN","ERROR","FATAL","EXCEPTION"):
	                if m1.getCalClass() == 'E':
	                        status=m1.getCalStatus()
	                        if status != "0" and status!="":
	                                pass
	                        else:
					ce=CalError("E08")
					err.append(m1)
					ce.printError(err)
	                        return -1
	                else:
					ce=CalError("E09")
					err.append(m1)
					ce.printError(err)
	                                return -1
	        else:
	                pass
	        return 0
	#return true if string starts with alpha char else false
	def startWithAlphachar(self,s):
	        """Starts with alpha char"""
	        prog=re.compile(r"^[a-zA-Z]")
	        if not prog.match(s):
	                return False
	        return True
	#return true if string is alpha numeric else false
	def is_alphanumeric(self,s):
	        """Check for alpha numberic"""
		prog=re.compile(r"[a-zA-Z0-9]+")
	        if not prog.search(s):
	                return False
	        return True
	#return true if string contains a special char else false
	def is_specialchar(self,s):
	        pat=r'[~!`@#$%^&*()+={}|\[\]/<>,\\ ]'
	        prog=re.compile(pat)
	        if prog.search(s):
	           return True
	        else:
	           return False
	#check for - _ in the string.
	def is_underscorHypen(self,s):
		pat=r'[_-]'
		prog=re.compile(pat)
		if prog.search(s):
                        return True
	#return true if the string contains any control chars else flase
	def is_controlchar(self,s):
	        prog=re.compile("[\a\f\n\r\t\v]")
	        if(prog.search(s)):
	                return True
	        return False
	#return false if contains odd number of colons continuouslsy
	def is_doublecolonOp(self,s):
		pattern=r'((\w+)::)+\w+$'
		prog=re.compile(pattern)
		if not prog.search(s):
			return False
		return True
	#returns true if the ip address is in proper format else false
	def is_IPAddress(self,addr):
	    address=addr.split(":")	
	    try:
	        addr= socket.inet_pton(socket.AF_INET, address[0])
	    except AttributeError: # no inet_pton here, sorry
	        try:
	            addr= socket.inet_aton(address[0])
	        except socket.error:
	            return False
	        return address[0].count('.') == 3
	    except socket.error: # not a valid address
	        return False
	    return True

	#check the cal messges status format based on cal guidelines.
	def checkStatusFormat(self,mesg):
		ce=CalError("E10")
		err=[]
		if not len(mesg.getCalStatus())>64:  #checks the length of status field whose max length=64
			 if self.is_specialchar(mesg.getCalStatus()): #check for special chars in the status field
				err.insert(0,mesg)
				ce.desc += ": Has special char"			
			 if self.is_controlchar(mesg.getCalStatus()): #check for the control chars in the status field
				err.insert(0,mesg)
				ce.desc += ": Has control char"
			 if mesg.getCalStatus()!="0" and mesg.getCalStatus()!="": 
				format=mesg.getCalStatus().split(".") #if status field is not 0,check for status error format 1.x.x.x
				if len(format)==4:
					if not re.compile("[123]").match(format[0]):  #first field should be 1 or 2 or 3
						err.insert(0,mesg)
						ce.desc +": Severity must be 1 or 2 or 3"
					if not self.is_alphanumeric(format[1]):#check for callling module
						err.insert(0,mesg)
						ce.desc +": Calling module must be alphanumeric"
					if not re.compile("DATA|INTERNAL|UNKNOWN").search(format[2]):#check for cal error field
						err.insert(0,mesg)
						ce.desc += ": CAL Error Status must be DATA or INTERNAL or UNKNOWN"			
					if not self.is_alphanumeric(format[3]): #check for return code.
						err.insert(0,mesg)
						ce.desc += ": Return Code must be alphanumeric "			
				else:
					err.insert(0,mesg)
					ce.desc += ": must be in the format x.x.x.x"			
		else:
			ce.desc += ": Status must be less than 64 characters long"			
			err.insert(0,mesg)
		if(len(err)>0):
                        ce.printError(err)
	
	#checks cal message name format based on cal guide lines
	def checkNameFormat(self,mesg):
		err=[]
		ce=CalError("E11")
		if not (self.is_IPAddress(mesg.getCalName()) or self.is_digit(mesg.getCalName())):
			if not self.startWithAlphachar(mesg.getCalName()):
				err.insert(0,mesg)
				ce.desc +=": Must start with an alphabet"
			if not self.is_alphanumeric(mesg.getCalName()):
				err.insert(0,mesg)
				ce.desc +=": Must be alphanumeric"
			if self.is_specialchar(mesg.getCalName()):
				err.insert(0,mesg)
				ce.desc +=": Must not contain special characters"
			if self.is_controlchar(mesg.getCalName()):
				err.insert(0,mesg)
				ce.desc +=": Must not contain control characters"
			if self.is_underscorHypen(mesg.getCalName()):
				pass
		if(len(err)>0):
                        ce.printError(err)

	#checks cal message type format based on cal guide lines
	def checkTypeFormat(self,mesg):
                err=[]
		ce=CalError("E12")
                if not self.startWithAlphachar(mesg.getCalType()):
			ce.desc +=": Must start with an alphabet"
			err.insert(0,mesg)
                if not self.is_alphanumeric(mesg.getCalType()):
			ce.desc +=": Must be alphanumeric"
			err.insert(0,mesg)
                if self.is_specialchar(mesg.getCalType()):
			ce.desc +=": Must not contain special characters"
			err.insert(0,mesg)
                if self.is_controlchar(mesg.getCalType()):
			ce.desc +=": Must not contain control characters"
			err.insert(0,mesg)
                if self.is_underscorHypen(mesg.getCalType()):
                        pass
                if(len(err)>0):
                        ce.printError(err)