diff --git a/adpengine/dmonadpengine.py b/adpengine/dmonadpengine.py index 6b91952..594522a 100644 --- a/adpengine/dmonadpengine.py +++ b/adpengine/dmonadpengine.py @@ -1,12 +1,11 @@ -import os from dmonconnector import * from dmonpoint import * from util import queryParser, nodesParse, str2Bool, cfilterparse, rfilterparse, pointThraesholds, parseDelay, parseMethodSettings, ut2hum -import pandas as pd from threadRun import AdpDetectThread, AdpPointThread, AdpTrainThread from dmonweka import dweka from time import sleep import tempfile +from dmonscikit import dmonscilearncluster as sdmon class AdpEngine: def __init__(self, settingsDict, dataDir, modelsDir): @@ -32,17 +31,18 @@ def __init__(self, settingsDict, dataDir, modelsDir): self.smemory = settingsDict['smemory'] self.snetwork = settingsDict['snetwork'] self.methodSettings = settingsDict['MethodSettings'] + self.resetIndex = settingsDict['resetindex'] self.dataDir = dataDir self.modelsDir = modelsDir self.anomalyIndex = "anomalies" - self.dmonConnector = Connector(self.esendpoint) - self.qConstructor = QueryConstructor() - self.dformat = DataFormatter(self.dataDir) - self.dweka = dweka(self.dataDir, self.modelsDir) self.regnodeList = [] - self.allowedMethodsClustering = ['skm', 'em', 'dbscan'] + self.allowedMethodsClustering = ['skm', 'em', 'dbscan', 'sdbscan', 'isoforest'] self.allowefMethodsClassification = [] # TODO self.heap = settingsDict['heap'] + self.dmonConnector = Connector(self.esendpoint) + self.qConstructor = QueryConstructor() + self.dformat = DataFormatter(self.dataDir) + self.dweka = dweka(self.dataDir, self.modelsDir, wHeap=self.heap) self.cfilter = settingsDict['cfilter'] self.rfilter = settingsDict['rfilter'] self.dfilter = settingsDict['dfilter'] @@ -56,6 +56,9 @@ def __init__(self, settingsDict, dataDir, modelsDir): self.systemReturn = 0 self.mapmetrics = 0 self.reducemetrics = 0 + self.mrapp = 0 + self.dataNodeTraining = 0 + self.dataNodeDetecting = 0 def initConnector(self): print "Establishing connection with dmon ....." @@ -86,6 +89,12 @@ def initConnector(self): self.regnodeList = self.dmonConnector.getNodeList() print "Nodes found -> %s" %self.regnodeList self.desiredNodesList = self.getDesiredNodes() + if str2Bool(self.resetIndex): + print "Reseting index %s" %self.anomalyIndex + self.dmonConnector.deleteIndex(self.anomalyIndex) + logger.warning('[%s] : [WARN] Reset index %s complete', + datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), self.anomalyIndex) + print "Reseting index %s complete" % self.anomalyIndex def getDesiredNodes(self): desNodes = [] @@ -366,7 +375,7 @@ def getData(self, detect=False): merged_DFS = self.dformat.chainMergeDFS(dfs=df_dfs, dfsfs=df_dfsFs, fsop=df_fsop) merged_cluster = self.dformat.chainMergeCluster(clusterMetrics=df_cluster, queue=df_queue, - jvmRM=df_jvmResourceManager, jvmmrapp=df_jvmMrapp) + jvmRM=df_jvmResourceManager) merge_nodemanager, jvmNode_manager, mShuffle= self.dformat.chainMergeNM(lNM=lNM, lNMJvm=lNMJvm, lShuffle=lShuffle) datanode_merge = self.dformat.chainMergeDN(lDN=lDataNode) df_jvmNameNode = self.dformat.dict2csv(gjvmNameNode, qjvmNameNode, jvmNameNode_file, df=checkpoint) @@ -382,6 +391,7 @@ def getData(self, detect=False): self.yarnReturn = final_merge self.mapmetrics = lmap self.reducemetrics = lreduce + self.mrapp = df_jvmMrapp else: # cluster, nn, nm, dfs, dn, mr mCluster = mNameNode = mNodeManager = mNodeManagerJVM = mShuffle = mDFS = mDataNode = mMap = mReduce = 0 @@ -397,7 +407,7 @@ def getData(self, detect=False): if el == 'dn': mDataNode = self.getDataNode(desNodes, detect=detect) if el == 'mr': - mMap, mReduce = self.getMapnReduce(desNodes, detect=detect) + mMap, mReduce, mMRApp = self.getMapnReduce(desNodes, detect=detect) if el not in ['cluster', 'nn', 'nm', 'dfs', 'dn', 'mr']: logger.error('[%s] : [ERROR] Unknown metrics context %s', datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), el) @@ -413,6 +423,7 @@ def getData(self, detect=False): self.yarnReturn = final_merge self.reducemetrics = mReduce self.mapmetrics = mMap + self.mrapp = mMRApp self.dformat.df2csv(final_merge, os.path.join(self.dataDir, 'cTest.csv')) print "Finished query and merge for yarn metrics" @@ -423,7 +434,7 @@ def getData(self, detect=False): print "Storm metrics" #todo self.stormReturn = 0 - return self.systemReturn, self.yarnReturn, self.reducemetrics, self.mapmetrics, self.sparkReturn, self.stormReturn + return self.systemReturn, self.yarnReturn, self.reducemetrics, self.mapmetrics, self.mrapp, self.sparkReturn, self.stormReturn def filterData(self, df, m=False): ''' @@ -481,20 +492,26 @@ def filterData(self, df, m=False): def trainMethod(self): if str2Bool(self.train): print "Getting data ..." - systemReturn, yarnReturn, reducemetrics, mapmetrics, sparkReturn, stormReturn = self.getData() - #yarnReturn = self.filterData(yarnReturn) #todo + checkpoint = str2Bool(self.checkpoint) + systemReturn, yarnReturn, reducemetrics, mapmetrics, mrapp, sparkReturn, stormReturn = self.getData() + yarnReturn = self.filterData(yarnReturn) #todo if self.type == 'clustering': if self.method in self.allowedMethodsClustering: print "Training with selected method %s of type %s" % (self.method, self.type) + if checkpoint: + dataf = tempfile.NamedTemporaryFile(suffix='.csv') + self.dformat.df2csv(yarnReturn, dataf.name) + data = dataf.name + else: + dataf = os.path.join(self.dataDir, 'Final_Merge.csv') + data = dataf if self.method == 'skm': print "Method %s settings detected -> %s" % (self.method, str(self.methodSettings)) opt = parseMethodSettings(self.methodSettings) if not opt: opt = ['-S', '10', '-N', '10'] - dataf = tempfile.NamedTemporaryFile(suffix='.csv') - self.dformat.df2csv(yarnReturn, dataf.name) try: - self.dweka.simpleKMeansTrain(dataf=dataf.name, options=opt, mname=self.export) + self.dweka.simpleKMeansTrain(dataf=data, options=opt, mname=self.export) except Exception as inst: print "Unable to run training for method %s exited with %s and %s" % (self.method, type(inst), inst.args) logger.error('[%s] : [ERROR] Unable to run training for method %s exited with %s and %s', @@ -506,10 +523,8 @@ def trainMethod(self): opt = parseMethodSettings(self.methodSettings) if not opt: opt = ["-I", "1000", "-N", "6", "-M", "1.0E-6", "-num-slots", "1", "-S", "100"] - dataf = tempfile.NamedTemporaryFile(suffix='.csv') - self.dformat.df2csv(yarnReturn, dataf.name) try: - self.dweka.emTrain(dataf=dataf.name, options=opt, mname=self.export) + self.dweka.emTrain(dataf=data, options=opt, mname=self.export) except Exception as inst: print "Unable to run training for method %s exited with %s and %s" % ( self.method, type(inst), inst.args) @@ -523,10 +538,8 @@ def trainMethod(self): opt = parseMethodSettings(self.methodSettings) if not opt: opt = ["-E", "0.9", "-M", "6", "-D", "weka.clusterers.forOPTICSAndDBScan.DataObjects.EuclideanDataObject"] - dataf = tempfile.NamedTemporaryFile(suffix='.csv') - self.dformat.df2csv(yarnReturn, dataf.name) try: - self.dweka.dbscanTrain(dataf=dataf.name, options=opt, mname=self.export) + self.dweka.dbscanTrain(dataf=data, options=opt, mname=self.export) except Exception as inst: print "Unable to run training for method %s exited with %s and %s" % ( self.method, type(inst), inst.args) @@ -535,7 +548,25 @@ def trainMethod(self): type(inst), inst.args) sys.exit(1) print "Saving model with name %s" % self.modelName(self.method, self.export) - + elif self.method == 'sdbscan': + opt = self.methodSettings + if not opt: + opt = {'eps': 0.9, 'min_samples': 10, 'metric': 'euclidean', + 'algorithm': 'auto', 'leaf_size': 30, 'p': 0.2, 'n_jobs': 1} + print "Using settings for sdbscan -> %s" % str(opt) + logger.info('[%s] : [INFO] Using settings for sdbscan -> %s ', + datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), str(opt)) + db = sdmon.SciCluster(self.modelsDir) + dbmodel = db.sdbscanTrain(settings=opt, mname=self.export, data=yarnReturn) + elif self.method == 'isoforest': + opt = self.methodSettings + if not opt: + opt = {'n_estimators': 100, 'max_samples': 100, 'contamination': 0.01, 'bootstrap': False, 'max_features': 1.0, 'n_jobs': -1, 'random_state': None, 'verbose': 0} + print "Using settings for isoForest -> %s" % str(opt) + logger.info('[%s] : [INFO] Using settings for isoForest -> %s ', + datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), str(opt)) + isofrst = sdmon.SciCluster(self.modelsDir) + isofrstmodel = isofrst.isolationForest(settings=opt, mname=self.export, data=yarnReturn) # Once training finished set training to false self.train = False return self.modelName(self.method, self.export) @@ -619,39 +650,62 @@ def detectPointAnomalies(self): # print dict_system for th in all: for type, val in th.iteritems(): + responseD = {} if val['bound'] == 'gd': anomalies = self.adppoint.detpoint(dict_system, type=type, threashold=val['threashold'], lt=False) - for a in anomalies: - self.reportAnomaly(a) - sleep(parseDelay(self.delay)) + responseD['anomalies'] = anomalies + self.reportAnomaly(responseD) else: anomalies = self.adppoint.detpoint(dict_system, type=type, threashold=val['threashold'], lt=True) - for a in anomalies: - self.reportAnomaly(a) - sleep(parseDelay(self.delay)) + responseD['anomalies'] = anomalies + self.reportAnomaly(responseD) + sleep(parseDelay(self.delay)) def detectAnomalies(self): if str2Bool(self.detect): + checkpoint = str2Bool(self.checkpoint) if self.type == 'clustering': print "Collect data ..." - systemReturn, yarnReturn, reducemetrics, mapmetrics, sparkReturn, stormReturn = self.getData(detect=True) - # yarnReturn = self.filterData(yarnReturn) #todo + systemReturn, yarnReturn, reducemetrics, mapmetrics, mrapp, sparkReturn, stormReturn = self.getData(detect=True) + # if list(set(self.dformat.fmHead) - set(list(yarnReturn.columns.values))): + # print "Mismatch between desired and loaded data" + # sys.exit() + # if self.dataNodeTraining != self.dataNodeDetecting: + # print "Detected datanode discrepancies; training %s, detecting %s" %(self.dataNodeTraining, self.dataNodeDetecting) + # logger.error('[%s] : [ERROR]Detected datanode discrepancies; training %s, detecting %s', + # datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), self.dataNodeTraining, self.dataNodeDetecting) + # sys.exit(1) + yarnReturn = self.filterData(yarnReturn) #todo + if checkpoint: + dataf = tempfile.NamedTemporaryFile(suffix='.csv') + self.dformat.df2csv(yarnReturn, dataf.name) + data = dataf.name + else: + dataf = os.path.join(self.dataDir, 'Final_Merge.csv') + data = dataf if self.method in self.allowedMethodsClustering: print "Detecting with selected method %s of type %s" % (self.method, self.type) if os.path.isfile(os.path.join(self.modelsDir, self.modelName(self.method, self.load))): print "Model found at %s" % str( os.path.join(self.modelsDir, self.modelName(self.method, self.load))) - dataf = tempfile.NamedTemporaryFile(suffix='.csv') - self.dformat.df2csv(yarnReturn, dataf.name) - anomalies = self.dweka.runclustermodel(self.method, self.load) - for e in anomalies: - print ut2hum(e) - a = {"type": self.method, "time": ut2hum(e)} + wekaList = ['skm', 'em', 'dbscan'] + if self.method in wekaList: + anomalies = self.dweka.runclustermodel(self.method, self.load, data) + # print ut2hum(e) + a = {"method": self.method, "qinterval": self.qinterval, "anomalies": anomalies} self.reportAnomaly(a) + else: + smodel = sdmon.SciCluster(modelDir=self.modelDir) + anomalies = smodel.detect(self.method, self.load, yarnReturn) + anomalies['method'] = self.method + anomalies['qinterval'] = self.qinterval + self.reportAnomaly(anomalies) + else: logger.error('[%s] : [ERROR] Model %s not found at %s ', datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), self.load, str(os.path.join(self.modelsDir, self.modelName(self.method, self.load)))) + print "Model not found %s" % self.modelName(self.method, self.load) sys.exit(1) else: print "Unknown method %s of type %s" % (self.method, self.type) @@ -669,23 +723,22 @@ def detectAnomalies(self): else: print "Detect is set to false, skipping..." logger.warning('[%s] : [WARN] Detect is set to false, skipping...', - datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), str(self.detect)) + datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')) sleep(parseDelay(self.interval)) def run(self, engine): try: - threadPoint = AdpPointThread(engine, 'Thread Point') - threadTrain = AdpTrainThread(engine, 'Thread Train') - threadDetect = AdpDetectThread(engine, 'Thread Detect') + threadPoint = AdpPointThread(engine, 'Point') + threadTrain = AdpTrainThread(engine, 'Train') + threadDetect = AdpDetectThread(engine, 'Detect') threadPoint.start() threadTrain.start() - # threadDetect.start() + threadDetect.start() except Exception as inst: logger.error('[%s] : [ERROR] Exception %s with %s during point thread execution, halting', datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst), inst.args) sys.exit(1) - return 0 def modelName(self, methodname, modelName): @@ -704,6 +757,9 @@ def compareModel(self): return "Compare models" def reportAnomaly(self, body): + now = datetime.utcnow() + itime = now.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % (now.microsecond / 1000) + "Z" + body["timestamp"] = itime self.dmonConnector.pushAnomaly(anomalyIndex=self.anomalyIndex, doc_type='anomaly', body=body) def getDFS(self, detect=False): @@ -876,23 +932,23 @@ def getCluster(self, detect=False): checkpoint = str2Bool(self.checkpoint) queue, queue_file = self.qConstructor.queueResourceString() cluster, cluster_file = self.qConstructor.clusterMetricsSring() - jvmMrapp, jvmMrapp_file = self.qConstructor.jvmMrappmasterString() + # jvmMrapp, jvmMrapp_file = self.qConstructor.jvmMrappmasterString() jvmResMng, jvmResMng_file = self.qConstructor.jvmResourceManagerString() - qjvmMrapp = self.qConstructor.jvmNNquery(jvmMrapp, tfrom, to, self.qsize, self.qinterval) + # qjvmMrapp = self.qConstructor.jvmNNquery(jvmMrapp, tfrom, to, self.qsize, self.qinterval) qqueue = self.qConstructor.resourceQueueQuery(queue, tfrom, to, self.qsize, self.qinterval) qcluster = self.qConstructor.clusterMetricsQuery(cluster, tfrom, to, self.qsize, self.qinterval) qjvmResMng = self.qConstructor.jvmNNquery(jvmResMng, tfrom, to, self.qsize, self.qinterval) gqueue = self.dmonConnector.aggQuery(qqueue) gcluster = self.dmonConnector.aggQuery(qcluster) - gjvmMrapp = self.dmonConnector.aggQuery(qjvmMrapp) + # gjvmMrapp = self.dmonConnector.aggQuery(qjvmMrapp) gjvmResourceManager = self.dmonConnector.aggQuery(qjvmResMng) if not checkpoint: self.dformat.dict2csv(gcluster, qcluster, cluster_file) self.dformat.dict2csv(gqueue, qqueue, queue_file) - self.dformat.dict2csv(gjvmMrapp, qjvmMrapp, jvmMrapp_file) + # self.dformat.dict2csv(gjvmMrapp, qjvmMrapp, jvmMrapp_file) self.dformat.dict2csv(gjvmResourceManager, qjvmResMng, jvmResMng_file) print "Starting cluster merge ..." @@ -902,11 +958,11 @@ def getCluster(self, detect=False): else: df_cluster = self.dformat.dict2csv(gcluster, qcluster, cluster_file, df=checkpoint) df_queue = self.dformat.dict2csv(gqueue, qqueue, queue_file, df=checkpoint) - df_jvmMrapp = self.dformat.dict2csv(gjvmMrapp, qjvmMrapp, jvmMrapp_file, df=checkpoint) + # df_jvmMrapp = self.dformat.dict2csv(gjvmMrapp, qjvmMrapp, jvmMrapp_file, df=checkpoint) df_jvmResourceManager = self.dformat.dict2csv(gjvmResourceManager, qjvmResMng, jvmResMng_file, df=checkpoint) print "Starting cluster merge ..." merged_cluster = self.dformat.chainMergeCluster(clusterMetrics=df_cluster, queue=df_queue, - jvmRM=df_jvmResourceManager, jvmmrapp=df_jvmMrapp) + jvmRM=df_jvmResourceManager) clusterReturn = merged_cluster print "Querying Cluster metrics complete" logger.info('[%s] : [INFO] Querying Name Node metrics complete', @@ -1004,7 +1060,23 @@ def getMapnReduce(self, nodes, detect=False): print "Querying Mapper metrics complete" logger.info('[%s] : [INFO] Querying Mapper metrics complete', datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')) - return lMP, lRD + print "Querying MRApp metrics ... " + logger.info('[%s] : [INFO] Querying MRApp metrics ...', + datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')) + jvmMrapp, jvmMrapp_file = self.qConstructor.jvmMrappmasterString() + + qjvmMrapp = self.qConstructor.jvmNNquery(jvmMrapp, tfrom, to, self.qsize, self.qinterval) + gjvmMrapp = self.dmonConnector.aggQuery(qjvmMrapp) + + if not checkpoint: + self.dformat.dict2csv(gjvmMrapp, qjvmMrapp, jvmMrapp_file) + df_jvmMrapp = 0 + else: + df_jvmMrapp = self.dformat.dict2csv(gjvmMrapp, qjvmMrapp, jvmMrapp_file, df=checkpoint) + logger.info('[%s] : [INFO] Querying MRApp metrics complete', + datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')) + print "Querying MRApp metrics complete " + return lMP, lRD, df_jvmMrapp def getDataNode(self, nodes, detect=False): if detect: @@ -1032,6 +1104,10 @@ def getDataNode(self, nodes, detect=False): logger.info('[%s] : [INFO] Empty response from %s no datanode metrics!', datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), node) print "Querying Data Node metrics complete" + if detect: + self.dataNodeDetecting = len(lDN) + else: + self.dataNodeTraining = len(lDN) logger.info('[%s] : [INFO] Querying Data Node metrics complete', datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')) diff --git a/adplogger.py b/adplogger.py index 42d9d52..a5abaa4 100644 --- a/adplogger.py +++ b/adplogger.py @@ -1,13 +1,24 @@ import logging -from logging.handlers import TimedRotatingFileHandler +from logging.handlers import RotatingFileHandler import os import datetime -logger = logging.getLogger("Rotating Log") -logger.setLevel(logging.ERROR) +logger = logging.getLogger("ADP Log") +logger.setLevel(logging.WARN) + +loggerESt = logging.getLogger('elasticsearch.trace') +loggerESt.setLevel(logging.WARN) +loggerES = logging.getLogger('elasticsearch') +loggerES.setLevel(logging.WARN) +loggerurl3 = logging.getLogger("urllib3") +loggerurl3.setLevel(logging.WARN) + # add a rotating handler -logFile = os.path.join(os.path.dirname(os.path.abspath('')), 'dmonadp.log') -handler = TimedRotatingFileHandler(logFile, when="d", interval=1, backupCount=5) +logFile = os.path.join('dmonadp.log') +handler = RotatingFileHandler(logFile, maxBytes=100000000, backupCount=5) logger.addHandler(handler) +loggerESt.addHandler(handler) +loggerES.addHandler(handler) +loggerurl3.addHandler(handler) diff --git a/dataformatter.py b/dataformatter.py index 3376481..46dca81 100644 --- a/dataformatter.py +++ b/dataformatter.py @@ -33,6 +33,7 @@ class DataFormatter: def __init__(self, dataDir): self.dataDir = dataDir + self.fmHead = 0 def getJson(self): return 'load Json' @@ -100,7 +101,7 @@ def dropColumns(self, df, lColumns, cp=True): return 0 def fillMissing(self, df): - df.fillna(0) + df.fillna(0, inplace=True) def dropMissing(self, df): df.dropna(axis=1, how='all', inplace=True) @@ -204,17 +205,17 @@ def chainMergeDFS(self, dfs=None, dfsfs=None, fsop=None): lFiles = [dfs, dfsfs, fsop] return self.listMerge(lFiles) - def chainMergeCluster(self, clusterMetrics=None, queue=None, jvmRM=None, jvmmrapp=None): + def chainMergeCluster(self, clusterMetrics=None, queue=None, jvmRM=None): ''' :return: -> merged cluster metrics ''' - if clusterMetrics is None and queue is None and jvmRM is None and jvmmrapp is None: + if clusterMetrics is None and queue is None and jvmRM is None: clusterMetrics = os.path.join(self.dataDir, "ClusterMetrics.csv") queue = os.path.join(self.dataDir, "ResourceManagerQueue.csv") jvmRM = os.path.join(self.dataDir, "JVM_RM.csv") - jvmmrapp = os.path.join(self.dataDir, "JVM_MRAPP.csv") + # jvmmrapp = os.path.join(self.dataDir, "JVM_MRAPP.csv") - lFiles = [clusterMetrics, queue, jvmRM, jvmmrapp] + lFiles = [clusterMetrics, queue, jvmRM] return self.listMerge(lFiles) @@ -395,7 +396,9 @@ def mergeFinal(self, dfs=None, cluster=None, nodeMng=None, jvmnodeMng=None, data merged_df = self.listMerge(lFile) merged_df.sort_index(axis=1, inplace=True) # merged_df.set_index('key', inplace=True) - self.dropMissing(merged_df) + #self.dropMissing(merged_df) + self.fillMissing(merged_df) + self.fmHead = list(merged_df.columns.values) return merged_df def dict2csv(self, response, query, filename, df=False): diff --git a/dmonadp.ini b/dmonadp.ini index fed4bce..80ba8f9 100644 --- a/dmonadp.ini +++ b/dmonadp.ini @@ -2,22 +2,22 @@ ESEndpoint:85.120.206.27 ESPort:9200 DMonPort:5001 -From:1477561800000 -To:1477562100000 +From:1479105362284 +To:1479119769978 Query:yarn:cluster, nn, nm, dfs, dn, mr;system Nodes: QSize:0 QInterval:10s [Mode] -Training:True +Training:false Validate:False -Detect:True +Detect:false [Filter] -Columns: -Rows: -DColumns: +#Columns:colname;colname2;colname3 +#Rows:ld:145607979;gd:145607979 +#DColumns:colname;colname2;colname3 [Detect] @@ -40,10 +40,5 @@ Network: tx:gd:34344;rx:ld:323434 heap:512m checkpoint:false delay:2m -interval:30m - - - - - - +interval:15m +resetindex:false \ No newline at end of file diff --git a/dmonadp.py b/dmonadp.py index 5a8f6fd..8878537 100644 --- a/dmonadp.py +++ b/dmonadp.py @@ -63,7 +63,8 @@ def main(argv): "heap": None, "checkpoint": None, "delay": None, - "interval": None + "interval": None, + "resetindex": None } # Only for testing @@ -347,7 +348,7 @@ def main(argv): try: print "Settings for method %s: " % settings['method'] - settings['MethodSettings'] = {} + settings['MethodSettings'] = {} #todo read settings from commandline ? for name, value in readCnf['MethodSettings'].iteritems(): print "%s -> %s" % (name, value) settings['MethodSettings'][name] = value @@ -482,7 +483,17 @@ def main(argv): logger.info('[%s] : [INFO] Interval is set to %s', datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), settings['interval']) - + if settings["resetindex"] is None: + try: + print "Reset index set to %s" % readCnf['Misc']['resetindex'] + settings["resetindex"] = readCnf['Misc']['resetindex'] + except: + print "Reset index not set, skipping." + settings["resetindex"] = False + else: + print "Reset index set to %s" % settings["resetindex"] + logger.info('[%s] : [INFO] Reset index set to %s"', + datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), settings['resetindex']) #if settings["esendpoint"] == None: @@ -530,7 +541,7 @@ def main(argv): # engine.trainMethod() # engine.detectAnomalies(30) # engine.printTest() - + print "\n" print "#" * 100 diff --git a/dmonconnector.py b/dmonconnector.py index 33ba1ab..db115ce 100644 --- a/dmonconnector.py +++ b/dmonconnector.py @@ -119,7 +119,7 @@ def closeIndex(self, indexName): def deleteIndex(self, indexName): try: - res = self.esInstance.delete(index=indexName, ignore=[400, 404]) + res = self.esInstance.indices.delete(index=indexName, ignore=[400, 404]) logger.info('[%s] : [INFO] Deleted index %s', datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), indexName) except Exception as inst: diff --git a/dmonweka/dweka.py b/dmonweka/dweka.py index 98337f8..877d3e8 100644 --- a/dmonweka/dweka.py +++ b/dmonweka/dweka.py @@ -39,12 +39,18 @@ def loadClusterModel(self, method, mname): def runclustermodel(self, model, method, dataf, temp=True): anomalies = [] try: - jvm.start() + jvm.start(max_heap_size=self.wHeap) data = self.loadData(dataf, temp) cluster = self.loadClusterModel(model, method) clusterMembership = [] + print cluster.number_of_clusters for inst in data: - cl = cluster.cluster_instance(inst) + try: + cl = cluster.cluster_instance(inst) + except Exception as inst: + logger.error('[%s] : [ERROR] Mismatch model and data attributes', + datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')) + dist = cluster.distribution_for_instance(inst) print ("cluster=" + str(cl) + ", distribution=" + str(dist)) clusterMembership.append(cl) @@ -54,10 +60,15 @@ def runclustermodel(self, model, method, dataf, temp=True): # print data.get_instance(3) pa = self.calcThreashold(dict(Counter(clusterMembership)), 21) - for a in pa: - # print data.get_instance(a).get_value(0) #todo always set key as first atribute - anomalies.append(data.get_instance(a).get_value(0)) - print "Detected using %s anomalies at timestamp(s) %s" % (model, str(anomalies)) + if pa == 0: + logger.warning('[%s] : [WARN] Most instances are computed as anomalies, possible error encountered!', + datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'),) + print "Most instances are computed as anomalies, possible error encountered!" + else: + for a in pa: + # print data.get_instance(a).get_value(0) #todo always set key as first atribute + anomalies.append(data.get_instance(a).get_value(0)) + print "Detected using %s anomalies at timestamp(s) %s" % (model, str(anomalies)) except Exception, e: print(traceback.format_exc()) finally: @@ -77,7 +88,7 @@ def simpleKMeansTrain(self, dataf, options, mname, temp=True): :return: ''' try: - jvm.start() + jvm.start(max_heap_size=self.wHeap) data = self.loadData(dataf, temp=True) clusterer = Clusterer(classname="weka.clusterers.SimpleKMeans", options=options) clusterer.build_clusterer(data) @@ -106,7 +117,7 @@ def dbscanTrain(self, dataf, options, mname, temp=True): ''' try: - jvm.start() + jvm.start(max_heap_size=self.wHeap) data = self.loadData(dataf, temp) clusterDBSCAN = Clusterer(classname="weka.clusterers.DBSCAN", options=options) clusterDBSCAN.build_clusterer(data) @@ -132,7 +143,7 @@ def emTrain(self, dataf, options, mname, temp=True): :return: ''' try: - jvm.start() + jvm.start(max_heap_size=self.wHeap) data = self.loadData(dataf, temp) clusterEM = Clusterer(classname="weka.clusterers.EM", options=options) diff --git a/pyQueryConstructor.py b/pyQueryConstructor.py index 84e6684..01e4c1a 100644 --- a/pyQueryConstructor.py +++ b/pyQueryConstructor.py @@ -102,7 +102,7 @@ def jvmRedProcessString(self, host): def jvmMapProcessingString(self, host): qstring = "type:\"maptask-metrics\" AND serviceType:\"jvm\" AND hostname:\"%s\"" % host - return qstring + return qstring #TODO def jvmRedProcessbyNameString(self, host, process): qstring = "type:\"reducetask-metrics\" AND serviceType:\"jvm\" AND hostname:\"%s\" AND ProcessName:\"%s\"" %(host, process) diff --git a/pyadp.py b/pyadp.py index e3ad528..9afad98 100644 --- a/pyadp.py +++ b/pyadp.py @@ -162,7 +162,7 @@ pass # Get non host based metrics queries and file strings - dfs, dfs_file = qConstructor.dfsFString() + dfs, dfs_file = qConstructor.dfsString() dfsFs, dfsFs_file = qConstructor.dfsFString() jvmNameNodeString, jvmNameNode_file = qConstructor.jvmNameNodeString() queue, queue_file = qConstructor.queueResourceString() diff --git a/test/adpPointTest.py b/test/adpPointTest.py index e69de29..f49a2d7 100644 --- a/test/adpPointTest.py +++ b/test/adpPointTest.py @@ -0,0 +1,57 @@ +from dmonpoint import * +import datetime + +# testWatcher = AdpPoint('85.120.206.27') +# +# +# print testWatcher.watcherInfo() + +test = {'free_slave1': {1477561840000: 292937728.0, 1477561880000: 285921280.0, 1477561930000: 270848000.0, 1477562050000: 261517312.0, 1477562000000: 274919424.0, 1477561950000: 270499840.0, 1477561860000: 287571968.0, 1477561990000: 277430272.0, 1477561970000: 268066816.0, 1477562060000: 261300224.0, 1477561870000: 287920128.0, 1477562010000: 270602240.0, 1477561960000: 268353536.0, 1477561910000: 259362816.0, 1477561800000: 295542784.0, 1477561940000: 270782464.0, 1477562070000: 261234688.0, 1477561810000: 293310464.0, 1477562020000: 270422016.0, 1477561890000: 285839360.0, 1477561920000: 272781312.0, 1477562080000: 261185536.0, 1477561900000: 277942272.0, 1477561850000: 291295232.0, 1477562030000: 268972032.0, 1477561980000: 277131264.0, 1477561820000: 292610048.0, 1477561830000: 291241984.0, 1477562090000: 260681728.0, 1477562040000: 262090752.0}, 'shortterm_master': {1477561840000: 0.01, 1477561880000: 0.0, 1477561930000: 0.040000000000000001, 1477562050000: 0.01, 1477562000000: 0.01, 1477561950000: 0.029999999999999999, 1477561860000: 0.0, 1477561990000: 0.02, 1477561970000: 0.02, 1477562060000: 0.0, 1477561870000: 0.0, 1477562010000: 0.01, 1477561960000: 0.029999999999999999, 1477561910000: 0.059999999999999998, 1477561800000: 0.01, 1477561940000: 0.040000000000000001, 1477562070000: 0.0, 1477561810000: 0.01, 1477562020000: 0.01, 1477561890000: 0.080000000000000002, 1477561920000: 0.050000000000000003, 1477562080000: 0.0, 1477561900000: 0.070000000000000007, 1477561850000: 0.0, 1477562030000: 0.01, 1477561980000: 0.02, 1477561820000: 0.01, 1477561830000: 0.01, 1477562090000: 0.0, 1477562040000: 0.01}, 'free_slave3': {1477561840000: 2775625728.0, 1477561880000: 2788417536.0, 1477561930000: 2786336768.0, 1477562050000: 2776641536.0, 1477562000000: 2779774976.0, 1477561950000: 2784944128.0, 1477561860000: 2775887872.0, 1477561990000: 2777755648.0, 1477561970000: 2780868608.0, 1477562060000: 2776449024.0, 1477561870000: 2775953408.0, 1477562010000: 2779881472.0, 1477561960000: 2782715904.0, 1477561910000: 2788425728.0, 1477561800000: 2794987520.0, 1477561940000: 2785345536.0, 1477562070000: 3296321536.0, 1477561810000: 2792759296.0, 1477562020000: 2779656192.0, 1477561890000: 2788331520.0, 1477561920000: 2786435072.0, 1477562080000: 2953306112.0, 1477561900000: 2788438016.0, 1477561850000: 2776104960.0, 1477562030000: 2778222592.0, 1477561980000: 2779754496.0, 1477561820000: 2792468480.0, 1477561830000: 2775904256.0, 1477562090000: 2933829632.0, 1477562040000: 2778767360.0}, 'free_slave2': {1477561840000: 4367798272.0, 1477561880000: 4324278272.0, 1477561930000: 4326248448.0, 1477562050000: 4311019520.0, 1477562000000: 4315004928.0, 1477561950000: 4321456128.0, 1477561860000: 4344754176.0, 1477561990000: 4317200384.0, 1477561970000: 4319621120.0, 1477562060000: 4309491712.0, 1477561870000: 4343193600.0, 1477562010000: 4315000832.0, 1477561960000: 4321263616.0, 1477561910000: 4325994496.0, 1477561800000: 4412100608.0, 1477561940000: 4323360768.0, 1477562070000: 4313718784.0, 1477561810000: 4409942016.0, 1477562020000: 4312748032.0, 1477561890000: 4328116224.0, 1477561920000: 4325928960.0, 1477562080000: 4311556096.0, 1477561900000: 4325998592.0, 1477561850000: 4367757312.0, 1477562030000: 4313444352.0, 1477561980000: 4317331456.0, 1477561820000: 4370337792.0, 1477561830000: 4370509824.0, 1477562090000: 4311781376.0, 1477562040000: 4311597056.0}, 'midterm_slave2': {1477561840000: 1.98, 1477561880000: 1.99, 1477561930000: 1.99, 1477562050000: 1.99, 1477562000000: 2.0, 1477561950000: 2.0, 1477561860000: 1.98, 1477561990000: 2.0, 1477561970000: 2.0, 1477562060000: 1.99, 1477561870000: 1.99, 1477562010000: 2.0, 1477561960000: 2.0, 1477561910000: 1.99, 1477561800000: 1.98, 1477561940000: 2.0, 1477562070000: 1.99, 1477561810000: 1.98, 1477562020000: 1.99, 1477561890000: 1.99, 1477561920000: 1.99, 1477562080000: 1.99, 1477561900000: 1.99, 1477561850000: 1.98, 1477562030000: 1.99, 1477561980000: 2.0, 1477561820000: 1.98, 1477561830000: 1.98, 1477562090000: 2.0, 1477562040000: 1.99}, 'free_slave4': {1477561840000: 3246813184.0, 1477561880000: 3244154880.0, 1477561930000: 3218280448.0, 1477562050000: 3206406144.0, 1477562000000: 3212689408.0, 1477561950000: 3214737408.0, 1477561860000: 3244240896.0, 1477561990000: 3212181504.0, 1477561970000: 3211563008.0, 1477562060000: 3202109440.0, 1477561870000: 3244240896.0, 1477562010000: 3211440128.0, 1477561960000: 3213754368.0, 1477561910000: 3228971008.0, 1477561800000: 3248189440.0, 1477561940000: 3218604032.0, 1477562070000: 3193905152.0, 1477561810000: 3247140864.0, 1477562020000: 3209576448.0, 1477561890000: 3238490112.0, 1477561920000: 3219488768.0, 1477562080000: 3185516544.0, 1477561900000: 3238723584.0, 1477561850000: 3246465024.0, 1477562030000: 3207520256.0, 1477561980000: 3212148736.0, 1477561820000: 3247345664.0, 1477561830000: 3247083520.0, 1477562090000: 3170959360.0, 1477562040000: 3208790016.0}, 'midterm_slave1': {1477561840000: 2.0899999999999999, 1477561880000: 2.0800000000000001, 1477561930000: 2.0800000000000001, 1477562050000: 2.23, 1477562000000: 2.1499999999999999, 1477561950000: 2.1099999999999999, 1477561860000: 2.0899999999999999, 1477561990000: 2.1600000000000001, 1477561970000: 2.1299999999999999, 1477562060000: 2.2200000000000002, 1477561870000: 2.0899999999999999, 1477562010000: 2.2599999999999998, 1477561960000: 2.1400000000000001, 1477561910000: 2.0899999999999999, 1477561800000: 2.1099999999999999, 1477561940000: 2.1099999999999999, 1477562070000: 2.2200000000000002, 1477561810000: 2.1000000000000001, 1477562020000: 2.25, 1477561890000: 2.0800000000000001, 1477561920000: 2.0899999999999999, 1477562080000: 2.21, 1477561900000: 2.0899999999999999, 1477561850000: 2.0899999999999999, 1477562030000: 2.25, 1477561980000: 2.1299999999999999, 1477561820000: 2.1000000000000001, 1477561830000: 2.1000000000000001, 1477562090000: 2.2000000000000002, 1477562040000: 2.2400000000000002}, 'cached_master': {1477561840000: 1512923136.0, 1477561880000: 1512972288.0, 1477561930000: 1513029632.0, 1477562050000: 1513189376.0, 1477562000000: 1513119744.0, 1477561950000: 1513062400.0, 1477561860000: 1512947712.0, 1477561990000: 1513111552.0, 1477561970000: 1513086976.0, 1477562060000: 1513193472.0, 1477561870000: 1512960000.0, 1477562010000: 1513132032.0, 1477561960000: 1513070592.0, 1477561910000: 1513005056.0, 1477561800000: 1512878080.0, 1477561940000: 1513046016.0, 1477562070000: 1513205760.0, 1477561810000: 1512886272.0, 1477562020000: 1513140224.0, 1477561890000: 1512984576.0, 1477561920000: 1513021440.0, 1477562080000: 1513291776.0, 1477561900000: 1512992768.0, 1477561850000: 1512935424.0, 1477562030000: 1513156608.0, 1477561980000: 1513095168.0, 1477561820000: 1512898560.0, 1477561830000: 1512910848.0, 1477562090000: 1513320448.0, 1477562040000: 1513172992.0}, 'used_slave2': {1477561840000: 3034705920.0, 1477561880000: 3078176768.0, 1477561930000: 3076153344.0, 1477562050000: 3091247104.0, 1477562000000: 3087319040.0, 1477561950000: 3080925184.0, 1477561860000: 3057725440.0, 1477561990000: 3085139968.0, 1477561970000: 3082735616.0, 1477562060000: 3092762624.0, 1477561870000: 3059269632.0, 1477562010000: 3087314944.0, 1477561960000: 3081101312.0, 1477561910000: 3076423680.0, 1477561800000: 2990448640.0, 1477561940000: 3079024640.0, 1477562070000: 3088519168.0, 1477561810000: 2992594944.0, 1477562020000: 3089555456.0, 1477561890000: 3074326528.0, 1477561920000: 3076481024.0, 1477562080000: 3090673664.0, 1477561900000: 3076435968.0, 1477561850000: 3034734592.0, 1477562030000: 3088834560.0, 1477561980000: 3085012992.0, 1477561820000: 3032190976.0, 1477561830000: 3032002560.0, 1477562090000: 3090427904.0, 1477562040000: 3090677760.0}, 'used_slave3': {1477561840000: 3682643968.0, 1477561880000: 3669774336.0, 1477561930000: 3671699456.0, 1477562050000: 3680440320.0, 1477562000000: 3677798400.0, 1477561950000: 3672723456.0, 1477561860000: 3682340864.0, 1477561990000: 3679830016.0, 1477561970000: 3676749824.0, 1477562060000: 3680153600.0, 1477561870000: 3682254848.0, 1477562010000: 3677667328.0, 1477561960000: 3674931200.0, 1477561910000: 3669712896.0, 1477561800000: 3663491072.0, 1477561940000: 3672334336.0, 1477562070000: 3160342528.0, 1477561810000: 3665698816.0, 1477562020000: 3677872128.0, 1477561890000: 3669839872.0, 1477561920000: 3671613440.0, 1477562080000: 3503218688.0, 1477561900000: 3669716992.0, 1477561850000: 3682144256.0, 1477562030000: 3679293440.0, 1477561980000: 3677851648.0, 1477561820000: 3665842176.0, 1477561830000: 3682377728.0, 1477562090000: 3522662400.0, 1477562040000: 3678724096.0}, 'used_slave1': {1477561840000: 3964420096.0, 1477561880000: 3969478656.0, 1477561930000: 3985526784.0, 1477562050000: 3993726976.0, 1477562000000: 3982487552.0, 1477561950000: 3985784832.0, 1477561860000: 3968053248.0, 1477561990000: 3980054528.0, 1477561970000: 3988185088.0, 1477562060000: 3993870336.0, 1477561870000: 3967500288.0, 1477562010000: 3986415616.0, 1477561960000: 3987922944.0, 1477561910000: 3996000256.0, 1477561800000: 3961925632.0, 1477561940000: 3985514496.0, 1477562070000: 3993911296.0, 1477561810000: 3964149760.0, 1477562020000: 3986690048.0, 1477561890000: 3969556480.0, 1477561920000: 3983740928.0, 1477562080000: 3993952256.0, 1477561900000: 3977437184.0, 1477561850000: 3966050304.0, 1477562030000: 3988127744.0, 1477561980000: 3980369920.0, 1477561820000: 3964768256.0, 1477561830000: 3966128128.0, 1477562090000: 3994443776.0, 1477562040000: 3993161728.0}, 'shortterm_slave1': {1477561840000: 2.0299999999999998, 1477561880000: 2.02, 1477561930000: 2.0499999999999998, 1477562050000: 2.3900000000000001, 1477562000000: 2.27, 1477561950000: 2.1600000000000001, 1477561860000: 2.02, 1477561990000: 2.3199999999999998, 1477561970000: 2.2400000000000002, 1477562060000: 2.3300000000000001, 1477561870000: 2.02, 1477562010000: 2.75, 1477561960000: 2.2799999999999998, 1477561910000: 2.0699999999999998, 1477561800000: 2.0699999999999998, 1477561940000: 2.1899999999999999, 1477562070000: 2.2799999999999998, 1477561810000: 2.0600000000000001, 1477562020000: 2.6400000000000001, 1477561890000: 2.0099999999999998, 1477561920000: 2.0600000000000001, 1477562080000: 2.23, 1477561900000: 2.0899999999999999, 1477561850000: 2.0299999999999998, 1477562030000: 2.54, 1477561980000: 2.2000000000000002, 1477561820000: 2.0499999999999998, 1477561830000: 2.04, 1477562090000: 2.2000000000000002, 1477562040000: 2.46}, 'shortterm_slave2': {1477561840000: 1.97, 1477561880000: 1.99, 1477561930000: 1.99, 1477562050000: 1.97, 1477562000000: 2.0299999999999998, 1477561950000: 2.0600000000000001, 1477561860000: 1.98, 1477561990000: 2.0299999999999998, 1477561970000: 2.0499999999999998, 1477562060000: 1.97, 1477561870000: 1.98, 1477562010000: 2.02, 1477561960000: 2.0499999999999998, 1477561910000: 1.99, 1477561800000: 1.95, 1477561940000: 2.0699999999999998, 1477562070000: 1.98, 1477561810000: 1.95, 1477562020000: 1.95, 1477561890000: 1.99, 1477561920000: 1.99, 1477562080000: 1.98, 1477561900000: 1.99, 1477561850000: 1.98, 1477562030000: 1.95, 1477561980000: 2.04, 1477561820000: 1.96, 1477561830000: 1.97, 1477562090000: 2.0600000000000001, 1477562040000: 1.96}, 'shortterm_slave3': {1477561840000: 1.01, 1477561880000: 1.0, 1477561930000: 1.0, 1477562050000: 1.0, 1477562000000: 1.0, 1477561950000: 1.01, 1477561860000: 1.0, 1477561990000: 1.0, 1477561970000: 1.01, 1477562060000: 1.0, 1477561870000: 1.0, 1477562010000: 1.0, 1477561960000: 1.01, 1477561910000: 1.0, 1477561800000: 1.01, 1477561940000: 1.0, 1477562070000: 1.0800000000000001, 1477561810000: 1.01, 1477562020000: 1.0, 1477561890000: 1.0, 1477561920000: 1.0, 1477562080000: 1.0600000000000001, 1477561900000: 1.0, 1477561850000: 1.0, 1477562030000: 1.0, 1477561980000: 1.01, 1477561820000: 1.01, 1477561830000: 1.01, 1477562090000: 1.1299999999999999, 1477562040000: 1.0}, 'shortterm_slave4': {1477561840000: 1.04, 1477561880000: 1.02, 1477561930000: 1.0700000000000001, 1477562050000: 1.01, 1477562000000: 1.02, 1477561950000: 1.05, 1477561860000: 1.03, 1477561990000: 1.02, 1477561970000: 1.03, 1477562060000: 1.01, 1477561870000: 1.02, 1477562010000: 1.02, 1477561960000: 1.04, 1477561910000: 1.0900000000000001, 1477561800000: 1.0800000000000001, 1477561940000: 1.0600000000000001, 1477562070000: 1.01, 1477561810000: 1.0600000000000001, 1477562020000: 1.01, 1477561890000: 1.02, 1477561920000: 1.0800000000000001, 1477562080000: 1.01, 1477561900000: 1.01, 1477561850000: 1.03, 1477562030000: 1.01, 1477561980000: 1.03, 1477561820000: 1.05, 1477561830000: 1.05, 1477562090000: 1.0, 1477562040000: 1.01}, 'midterm_master': {1477561840000: 0.040000000000000001, 1477561880000: 0.040000000000000001, 1477561930000: 0.050000000000000003, 1477562050000: 0.029999999999999999, 1477562000000: 0.040000000000000001, 1477561950000: 0.040000000000000001, 1477561860000: 0.040000000000000001, 1477561990000: 0.040000000000000001, 1477561970000: 0.040000000000000001, 1477562060000: 0.029999999999999999, 1477561870000: 0.040000000000000001, 1477562010000: 0.040000000000000001, 1477561960000: 0.040000000000000001, 1477561910000: 0.050000000000000003, 1477561800000: 0.050000000000000003, 1477561940000: 0.040000000000000001, 1477562070000: 0.029999999999999999, 1477561810000: 0.050000000000000003, 1477562020000: 0.040000000000000001, 1477561890000: 0.050000000000000003, 1477561920000: 0.050000000000000003, 1477562080000: 0.029999999999999999, 1477561900000: 0.050000000000000003, 1477561850000: 0.040000000000000001, 1477562030000: 0.040000000000000001, 1477561980000: 0.040000000000000001, 1477561820000: 0.040000000000000001, 1477561830000: 0.040000000000000001, 1477562090000: 0.029999999999999999, 1477562040000: 0.029999999999999999}, 'rx_slave4_x': {1477561840000: 892817621.0, 1477561880000: 892905021.0, 1477561930000: 893013178.0, 1477562050000: 893273065.0, 1477562000000: 893164789.0, 1477561950000: 893055173.0, 1477561860000: 892862746.0, 1477561990000: 893143084.0, 1477561970000: 893101252.0, 1477562060000: 893294929.0, 1477561870000: 892884235.0, 1477562010000: 893186738.0, 1477561960000: 893076474.0, 1477561910000: 892971343.0, 1477561800000: 892732670.0, 1477561940000: 893035005.0, 1477562070000: 893317628.0, 1477561810000: 892754237.0, 1477562020000: 893208604.0, 1477561890000: 892926386.0, 1477561920000: 892991632.0, 1477562080000: 893337908.0, 1477561900000: 892947657.0, 1477561850000: 892841438.0, 1477562030000: 893231497.0, 1477561980000: 893121622.0, 1477561820000: 892775244.0, 1477561830000: 892796443.0, 1477562090000: 893360716.0, 1477562040000: 893251980.0}, 'rx_slave4_y': {1477561840000: 5550434.0, 1477561880000: 5550973.0, 1477561930000: 5551632.0, 1477562050000: 5553253.0, 1477562000000: 5552578.0, 1477561950000: 5551892.0, 1477561860000: 5550713.0, 1477561990000: 5552443.0, 1477561970000: 5552186.0, 1477562060000: 5553390.0, 1477561870000: 5550844.0, 1477562010000: 5552717.0, 1477561960000: 5552021.0, 1477561910000: 5551379.0, 1477561800000: 5549917.0, 1477561940000: 5551768.0, 1477562070000: 5553532.0, 1477561810000: 5550049.0, 1477562020000: 5552854.0, 1477561890000: 5551100.0, 1477561920000: 5551499.0, 1477562080000: 5553655.0, 1477561900000: 5551228.0, 1477561850000: 5550586.0, 1477562030000: 5553001.0, 1477561980000: 5552312.0, 1477561820000: 5550179.0, 1477561830000: 5550306.0, 1477562090000: 5553796.0, 1477562040000: 5553127.0}, 'longterm_master': {1477561840000: 0.059999999999999998, 1477561880000: 0.059999999999999998, 1477561930000: 0.059999999999999998, 1477562050000: 0.050000000000000003, 1477562000000: 0.050000000000000003, 1477561950000: 0.059999999999999998, 1477561860000: 0.059999999999999998, 1477561990000: 0.050000000000000003, 1477561970000: 0.059999999999999998, 1477562060000: 0.050000000000000003, 1477561870000: 0.059999999999999998, 1477562010000: 0.050000000000000003, 1477561960000: 0.059999999999999998, 1477561910000: 0.059999999999999998, 1477561800000: 0.070000000000000007, 1477561940000: 0.059999999999999998, 1477562070000: 0.050000000000000003, 1477561810000: 0.070000000000000007, 1477562020000: 0.050000000000000003, 1477561890000: 0.059999999999999998, 1477561920000: 0.059999999999999998, 1477562080000: 0.050000000000000003, 1477561900000: 0.059999999999999998, 1477561850000: 0.059999999999999998, 1477562030000: 0.050000000000000003, 1477561980000: 0.050000000000000003, 1477561820000: 0.059999999999999998, 1477561830000: 0.059999999999999998, 1477562090000: 0.050000000000000003, 1477562040000: 0.050000000000000003}, 'midterm_slave3': {1477561840000: 1.03, 1477561880000: 1.03, 1477561930000: 1.02, 1477562050000: 1.01, 1477562000000: 1.01, 1477561950000: 1.02, 1477561860000: 1.03, 1477561990000: 1.02, 1477561970000: 1.02, 1477562060000: 1.01, 1477561870000: 1.03, 1477562010000: 1.01, 1477561960000: 1.02, 1477561910000: 1.02, 1477561800000: 1.03, 1477561940000: 1.02, 1477562070000: 1.03, 1477561810000: 1.03, 1477562020000: 1.01, 1477561890000: 1.02, 1477561920000: 1.02, 1477562080000: 1.03, 1477561900000: 1.02, 1477561850000: 1.03, 1477562030000: 1.01, 1477561980000: 1.02, 1477561820000: 1.03, 1477561830000: 1.03, 1477562090000: 1.04, 1477562040000: 1.01}, 'rx_slave3_y': {1477561840000: 3911125.0, 1477561880000: 3911860.0, 1477561930000: 3912777.0, 1477562050000: 3915037.0, 1477562000000: 3914086.0, 1477561950000: 3913137.0, 1477561860000: 3911491.0, 1477561990000: 3913900.0, 1477561970000: 3913514.0, 1477562060000: 3915220.0, 1477561870000: 3911681.0, 1477562010000: 3914274.0, 1477561960000: 3913320.0, 1477561910000: 3912402.0, 1477561800000: 3910402.0, 1477561940000: 3912958.0, 1477562070000: 3915417.0, 1477561810000: 3910592.0, 1477562020000: 3914456.0, 1477561890000: 3912036.0, 1477561920000: 3912583.0, 1477562080000: 3915611.0, 1477561900000: 3912217.0, 1477561850000: 3911307.0, 1477562030000: 3914655.0, 1477561980000: 3913709.0, 1477561820000: 3910767.0, 1477561830000: 3910946.0, 1477562090000: 3915806.0, 1477562040000: 3914842.0}, 'rx_slave3_x': {1477561840000: 727012876.0, 1477561880000: 727172778.0, 1477561930000: 727368143.0, 1477562050000: 727842284.0, 1477562000000: 727644398.0, 1477561950000: 727445154.0, 1477561860000: 727088875.0, 1477561990000: 727604314.0, 1477561970000: 727521367.0, 1477562060000: 727882158.0, 1477561870000: 727133345.0, 1477562010000: 727683027.0, 1477561960000: 727483104.0, 1477561910000: 727286133.0, 1477561800000: 726854235.0, 1477561940000: 727407675.0, 1477562070000: 727921743.0, 1477561810000: 726898794.0, 1477562020000: 727720006.0, 1477561890000: 727210189.0, 1477561920000: 727324087.0, 1477562080000: 727963353.0, 1477561900000: 727247383.0, 1477561850000: 727050687.0, 1477562030000: 727759481.0, 1477561980000: 727560715.0, 1477561820000: 726937576.0, 1477561830000: 726975110.0, 1477562090000: 728002284.0, 1477562040000: 727797699.0}, 'used_master': {1477561840000: 4225744896.0, 1477561880000: 4225695744.0, 1477561930000: 4229361664.0, 1477562050000: 4232347648.0, 1477562000000: 4228026368.0, 1477561950000: 4229775360.0, 1477561860000: 4226297856.0, 1477561990000: 4228034560.0, 1477561970000: 4229574656.0, 1477562060000: 4232323072.0, 1477561870000: 4225777664.0, 1477562010000: 4228005888.0, 1477561960000: 4229607424.0, 1477561910000: 4227104768.0, 1477561800000: 4225892352.0, 1477561940000: 4229591040.0, 1477562070000: 4232699904.0, 1477561810000: 4225536000.0, 1477562020000: 4228472832.0, 1477561890000: 4226572288.0, 1477561920000: 4229775360.0, 1477562080000: 4232708096.0, 1477561900000: 4226936832.0, 1477561850000: 4225961984.0, 1477562030000: 4229189632.0, 1477561980000: 4228759552.0, 1477561820000: 4225650688.0, 1477561830000: 4225630208.0, 1477562090000: 4232359936.0, 1477562040000: 4229849088.0}, 'rx_slave2_x': {1477561840000: 224884818.0, 1477561880000: 224933146.0, 1477561930000: 224991936.0, 1477562050000: 225133934.0, 1477562000000: 225074006.0, 1477561950000: 225015325.0, 1477561860000: 224909279.0, 1477561990000: 225062068.0, 1477561970000: 225038357.0, 1477562060000: 225145571.0, 1477561870000: 224921004.0, 1477562010000: 225085640.0, 1477561960000: 225026572.0, 1477561910000: 224968886.0, 1477561800000: 224839493.0, 1477561940000: 225003971.0, 1477562070000: 225164125.0, 1477561810000: 224851271.0, 1477562020000: 225096929.0, 1477561890000: 224944072.0, 1477561920000: 224980097.0, 1477562080000: 225175779.0, 1477561900000: 224955343.0, 1477561850000: 224898142.0, 1477562030000: 225110442.0, 1477561980000: 225050086.0, 1477561820000: 224862611.0, 1477561830000: 224873554.0, 1477562090000: 225187914.0, 1477562040000: 225122036.0}, 'rx_slave2_y': {1477561840000: 2063700.0, 1477561880000: 2063998.0, 1477561930000: 2064355.0, 1477562050000: 2065249.0, 1477562000000: 2064868.0, 1477561950000: 2064501.0, 1477561860000: 2063851.0, 1477561990000: 2064792.0, 1477561970000: 2064642.0, 1477562060000: 2065321.0, 1477561870000: 2063921.0, 1477562010000: 2064944.0, 1477561960000: 2064568.0, 1477561910000: 2064216.0, 1477561800000: 2063428.0, 1477561940000: 2064431.0, 1477562070000: 2065399.0, 1477561810000: 2063497.0, 1477562020000: 2065012.0, 1477561890000: 2064061.0, 1477561920000: 2064282.0, 1477562080000: 2065477.0, 1477561900000: 2064128.0, 1477561850000: 2063786.0, 1477562030000: 2065105.0, 1477561980000: 2064718.0, 1477561820000: 2063570.0, 1477561830000: 2063635.0, 1477562090000: 2065550.0, 1477562040000: 2065177.0}, 'rx_master_y': {1477561840000: 7311979.0, 1477561880000: 7312515.0, 1477561930000: 7313192.0, 1477562050000: 7314820.0, 1477562000000: 7314117.0, 1477561950000: 7313410.0, 1477561860000: 7312286.0, 1477561990000: 7313998.0, 1477561970000: 7313720.0, 1477562060000: 7314934.0, 1477561870000: 7312407.0, 1477562010000: 7314234.0, 1477561960000: 7313557.0, 1477561910000: 7312926.0, 1477561800000: 7311496.0, 1477561940000: 7313306.0, 1477562070000: 7315051.0, 1477561810000: 7311610.0, 1477562020000: 7314379.0, 1477561890000: 7312646.0, 1477561920000: 7313069.0, 1477562080000: 7315219.0, 1477561900000: 7312762.0, 1477561850000: 7312137.0, 1477562030000: 7314544.0, 1477561980000: 7313876.0, 1477561820000: 7311720.0, 1477561830000: 7311859.0, 1477562090000: 7315385.0, 1477562040000: 7314690.0}, 'rx_master_x': {1477561840000: 1459653141.0, 1477561880000: 1459754693.0, 1477561930000: 1459883051.0, 1477562050000: 1460186933.0, 1477562000000: 1460057510.0, 1477561950000: 1459926068.0, 1477561860000: 1459708873.0, 1477561990000: 1460034096.0, 1477561970000: 1459984931.0, 1477562060000: 1460209314.0, 1477561870000: 1459732152.0, 1477562010000: 1460079199.0, 1477561960000: 1459955218.0, 1477561910000: 1459834139.0, 1477561800000: 1459557774.0, 1477561940000: 1459905432.0, 1477562070000: 1460231571.0, 1477561810000: 1459580136.0, 1477562020000: 1460107219.0, 1477561890000: 1459781054.0, 1477561920000: 1459859594.0, 1477562080000: 1460262631.0, 1477561900000: 1459803727.0, 1477561850000: 1459682755.0, 1477562030000: 1460136999.0, 1477561980000: 1460010711.0, 1477561820000: 1459602878.0, 1477561830000: 1459630365.0, 1477562090000: 1460292714.0, 1477562040000: 1460162548.0}, 'tx_slave4_x': {1477561840000: 1045742353.0, 1477561880000: 1045892836.0, 1477561930000: 1046067071.0, 1477562050000: 1046481990.0, 1477562000000: 1046309571.0, 1477561950000: 1046132230.0, 1477561860000: 1045826840.0, 1477561990000: 1046276291.0, 1477561970000: 1046215363.0, 1477562060000: 1046520640.0, 1477561870000: 1045856391.0, 1477562010000: 1046343147.0, 1477561960000: 1046161769.0, 1477561910000: 1046005586.0, 1477561800000: 1045618349.0, 1477561940000: 1046101795.0, 1477562070000: 1046555269.0, 1477561810000: 1045648282.0, 1477562020000: 1046376006.0, 1477561890000: 1045925135.0, 1477561920000: 1046034229.0, 1477562080000: 1046580410.0, 1477561900000: 1045953383.0, 1477561850000: 1045797533.0, 1477562030000: 1046423727.0, 1477561980000: 1046241367.0, 1477561820000: 1045684901.0, 1477561830000: 1045711980.0, 1477562090000: 1046634697.0, 1477562040000: 1046455570.0}, 'tx_slave4_y': {1477561840000: 4007845.0, 1477561880000: 4008334.0, 1477561930000: 4008935.0, 1477562050000: 4010367.0, 1477562000000: 4009775.0, 1477561950000: 4009171.0, 1477561860000: 4008099.0, 1477561990000: 4009657.0, 1477561970000: 4009432.0, 1477562060000: 4010495.0, 1477561870000: 4008214.0, 1477562010000: 4009898.0, 1477561960000: 4009285.0, 1477561910000: 4008704.0, 1477561800000: 4007378.0, 1477561940000: 4009056.0, 1477562070000: 4010626.0, 1477561810000: 4007498.0, 1477562020000: 4010022.0, 1477561890000: 4008455.0, 1477561920000: 4008817.0, 1477562080000: 4010734.0, 1477561900000: 4008569.0, 1477561850000: 4007985.0, 1477562030000: 4010143.0, 1477561980000: 4009537.0, 1477561820000: 4007619.0, 1477561830000: 4007730.0, 1477562090000: 4010864.0, 1477562040000: 4010258.0}, 'free_master': {1477561840000: 2537127936.0, 1477561880000: 2537127936.0, 1477561930000: 2533404672.0, 1477562050000: 2530258944.0, 1477562000000: 2534649856.0, 1477561950000: 2532958208.0, 1477561860000: 2536550400.0, 1477561990000: 2534649856.0, 1477561970000: 2533134336.0, 1477562060000: 2530279424.0, 1477561870000: 2537058304.0, 1477562010000: 2534658048.0, 1477561960000: 2533117952.0, 1477561910000: 2535686144.0, 1477561800000: 2537025536.0, 1477561940000: 2533158912.0, 1477562070000: 2529890304.0, 1477561810000: 2537373696.0, 1477562020000: 2534182912.0, 1477561890000: 2536239104.0, 1477561920000: 2532999168.0, 1477562080000: 2529796096.0, 1477561900000: 2535866368.0, 1477561850000: 2536898560.0, 1477562030000: 2533449728.0, 1477561980000: 2533941248.0, 1477561820000: 2537246720.0, 1477561830000: 2537254912.0, 1477562090000: 2530115584.0, 1477562040000: 2532773888.0}, 'tx_master_y': {1477561840000: 5681111.0, 1477561880000: 5681547.0, 1477561930000: 5682104.0, 1477562050000: 5683395.0, 1477562000000: 5682839.0, 1477561950000: 5682279.0, 1477561860000: 5681357.0, 1477561990000: 5682746.0, 1477561970000: 5682513.0, 1477562060000: 5683485.0, 1477561870000: 5681461.0, 1477562010000: 5682931.0, 1477561960000: 5682398.0, 1477561910000: 5681875.0, 1477561800000: 5680709.0, 1477561940000: 5682193.0, 1477562070000: 5683581.0, 1477561810000: 5680808.0, 1477562020000: 5683046.0, 1477561890000: 5681660.0, 1477561920000: 5682001.0, 1477562080000: 5683712.0, 1477561900000: 5681754.0, 1477561850000: 5681227.0, 1477562030000: 5683163.0, 1477561980000: 5682645.0, 1477561820000: 5680897.0, 1477561830000: 5681014.0, 1477562090000: 5683827.0, 1477562040000: 5683287.0}, 'used_slave4': {1477561840000: 3492433920.0, 1477561880000: 3495010304.0, 1477561930000: 3520782336.0, 1477562050000: 3532402688.0, 1477562000000: 3526221824.0, 1477561950000: 3524280320.0, 1477561860000: 3494973440.0, 1477561990000: 3526750208.0, 1477561970000: 3527413760.0, 1477562060000: 3536674816.0, 1477561870000: 3494952960.0, 1477562010000: 3527450624.0, 1477561960000: 3525242880.0, 1477561910000: 3510120448.0, 1477561800000: 3491151872.0, 1477561940000: 3520442368.0, 1477562070000: 3544858624.0, 1477561810000: 3492175872.0, 1477562020000: 3529297920.0, 1477561890000: 3500662784.0, 1477561920000: 3519594496.0, 1477562080000: 3553226752.0, 1477561900000: 3500408832.0, 1477561850000: 3492773888.0, 1477562030000: 3531325440.0, 1477561980000: 3526815744.0, 1477561820000: 3491958784.0, 1477561830000: 3492196352.0, 1477562090000: 3567763456.0, 1477562040000: 3530035200.0}, 'tx_slave1_y': {1477561840000: 6434114.0, 1477561880000: 6434655.0, 1477561930000: 6435313.0, 1477562050000: 6436837.0, 1477562000000: 6436184.0, 1477561950000: 6435516.0, 1477561860000: 6434439.0, 1477561990000: 6436069.0, 1477561970000: 6435824.0, 1477562060000: 6436953.0, 1477561870000: 6434558.0, 1477562010000: 6436280.0, 1477561960000: 6435640.0, 1477561910000: 6435076.0, 1477561800000: 6433667.0, 1477561940000: 6435423.0, 1477562070000: 6437063.0, 1477561810000: 6433780.0, 1477562020000: 6436420.0, 1477561890000: 6434782.0, 1477561920000: 6435205.0, 1477562080000: 6437197.0, 1477561900000: 6434885.0, 1477561850000: 6434293.0, 1477562030000: 6436592.0, 1477561980000: 6435967.0, 1477561820000: 6433875.0, 1477561830000: 6434007.0, 1477562090000: 6437373.0, 1477562040000: 6436725.0}, 'tx_slave1_x': {1477561840000: 1211506352.0, 1477561880000: 1211621650.0, 1477561930000: 1211768529.0, 1477562050000: 1212107180.0, 1477562000000: 1211964866.0, 1477561950000: 1211816717.0, 1477561860000: 1211568330.0, 1477561990000: 1211936000.0, 1477561970000: 1211880647.0, 1477562060000: 1212134841.0, 1477561870000: 1211598659.0, 1477562010000: 1211986098.0, 1477561960000: 1211845366.0, 1477561910000: 1211714101.0, 1477561800000: 1211399458.0, 1477561940000: 1211792689.0, 1477562070000: 1212164851.0, 1477561810000: 1211427999.0, 1477562020000: 1212019201.0, 1477561890000: 1211651829.0, 1477561920000: 1211739046.0, 1477562080000: 1212196685.0, 1477561900000: 1211677612.0, 1477561850000: 1211541111.0, 1477562030000: 1212051181.0, 1477561980000: 1211910682.0, 1477561820000: 1211449781.0, 1477561830000: 1211482474.0, 1477562090000: 1212228593.0, 1477562040000: 1212079413.0}, 'longterm_slave3': {1477561840000: 1.05, 1477561880000: 1.05, 1477561930000: 1.05, 1477562050000: 1.05, 1477562000000: 1.05, 1477561950000: 1.05, 1477561860000: 1.05, 1477561990000: 1.05, 1477561970000: 1.05, 1477562060000: 1.05, 1477561870000: 1.05, 1477562010000: 1.05, 1477561960000: 1.05, 1477561910000: 1.05, 1477561800000: 1.05, 1477561940000: 1.05, 1477562070000: 1.05, 1477561810000: 1.05, 1477562020000: 1.05, 1477561890000: 1.05, 1477561920000: 1.05, 1477562080000: 1.05, 1477561900000: 1.05, 1477561850000: 1.05, 1477562030000: 1.05, 1477561980000: 1.05, 1477561820000: 1.05, 1477561830000: 1.05, 1477562090000: 1.05, 1477562040000: 1.05}, 'longterm_slave2': {1477561840000: 2.0299999999999998, 1477561880000: 2.0299999999999998, 1477561930000: 2.0299999999999998, 1477562050000: 2.0299999999999998, 1477562000000: 2.0299999999999998, 1477561950000: 2.0299999999999998, 1477561860000: 2.0299999999999998, 1477561990000: 2.0299999999999998, 1477561970000: 2.0299999999999998, 1477562060000: 2.0299999999999998, 1477561870000: 2.0299999999999998, 1477562010000: 2.0299999999999998, 1477561960000: 2.0299999999999998, 1477561910000: 2.0299999999999998, 1477561800000: 2.0299999999999998, 1477561940000: 2.0299999999999998, 1477562070000: 2.0299999999999998, 1477561810000: 2.0299999999999998, 1477562020000: 2.0299999999999998, 1477561890000: 2.0299999999999998, 1477561920000: 2.0299999999999998, 1477562080000: 2.0299999999999998, 1477561900000: 2.0299999999999998, 1477561850000: 2.0299999999999998, 1477562030000: 2.0299999999999998, 1477561980000: 2.0299999999999998, 1477561820000: 2.0299999999999998, 1477561830000: 2.0299999999999998, 1477562090000: 2.0299999999999998, 1477562040000: 2.0299999999999998}, 'longterm_slave1': {1477561840000: 2.1200000000000001, 1477561880000: 2.1200000000000001, 1477561930000: 2.1200000000000001, 1477562050000: 2.1699999999999999, 1477562000000: 2.1400000000000001, 1477561950000: 2.1299999999999999, 1477561860000: 2.1200000000000001, 1477561990000: 2.1400000000000001, 1477561970000: 2.1400000000000001, 1477562060000: 2.1699999999999999, 1477561870000: 2.1200000000000001, 1477562010000: 2.1800000000000002, 1477561960000: 2.1400000000000001, 1477561910000: 2.1200000000000001, 1477561800000: 2.1299999999999999, 1477561940000: 2.1299999999999999, 1477562070000: 2.1699999999999999, 1477561810000: 2.1299999999999999, 1477562020000: 2.1800000000000002, 1477561890000: 2.1200000000000001, 1477561920000: 2.1200000000000001, 1477562080000: 2.1600000000000001, 1477561900000: 2.1200000000000001, 1477561850000: 2.1200000000000001, 1477562030000: 2.1699999999999999, 1477561980000: 2.1299999999999999, 1477561820000: 2.1200000000000001, 1477561830000: 2.1200000000000001, 1477562090000: 2.1600000000000001, 1477562040000: 2.1699999999999999}, 'rx_slave1_y': {1477561840000: 5307418.0, 1477561880000: 5307845.0, 1477561930000: 5308357.0, 1477562050000: 5309585.0, 1477562000000: 5309051.0, 1477561950000: 5308504.0, 1477561860000: 5307683.0, 1477561990000: 5308964.0, 1477561970000: 5308760.0, 1477562060000: 5309673.0, 1477561870000: 5307772.0, 1477562010000: 5309127.0, 1477561960000: 5308606.0, 1477561910000: 5308165.0, 1477561800000: 5307088.0, 1477561940000: 5308441.0, 1477562070000: 5309758.0, 1477561810000: 5307170.0, 1477562020000: 5309236.0, 1477561890000: 5307941.0, 1477561920000: 5308270.0, 1477562080000: 5309854.0, 1477561900000: 5308012.0, 1477561850000: 5307564.0, 1477562030000: 5309387.0, 1477561980000: 5308880.0, 1477561820000: 5307241.0, 1477561830000: 5307343.0, 1477562090000: 5309991.0, 1477562040000: 5309497.0}, 'rx_slave1_x': {1477561840000: 1511398439.0, 1477561880000: 1511516882.0, 1477561930000: 1511658180.0, 1477562050000: 1511973533.0, 1477562000000: 1511831790.0, 1477561950000: 1511684158.0, 1477561860000: 1511481350.0, 1477561990000: 1511814181.0, 1477561970000: 1511773667.0, 1477562060000: 1511991244.0, 1477561870000: 1511503153.0, 1477562010000: 1511843892.0, 1477561960000: 1511712520.0, 1477561910000: 1511618349.0, 1477561800000: 1511322911.0, 1477561940000: 1511675618.0, 1477562070000: 1512010610.0, 1477561810000: 1511344181.0, 1477562020000: 1511872656.0, 1477561890000: 1511544718.0, 1477561920000: 1511636654.0, 1477562080000: 1512035826.0, 1477561900000: 1511556904.0, 1477561850000: 1511459169.0, 1477562030000: 1511933326.0, 1477561980000: 1511792817.0, 1477561820000: 1511357906.0, 1477561830000: 1511385886.0, 1477562090000: 1512096097.0, 1477562040000: 1511951810.0}, 'longterm_slave4': {1477561840000: 1.6399999999999999, 1477561880000: 1.6200000000000001, 1477561930000: 1.5900000000000001, 1477562050000: 1.52, 1477562000000: 1.55, 1477561950000: 1.5800000000000001, 1477561860000: 1.6299999999999999, 1477561990000: 1.55, 1477561970000: 1.5600000000000001, 1477562060000: 1.51, 1477561870000: 1.6200000000000001, 1477562010000: 1.54, 1477561960000: 1.5700000000000001, 1477561910000: 1.6000000000000001, 1477561800000: 1.6699999999999999, 1477561940000: 1.5800000000000001, 1477562070000: 1.51, 1477561810000: 1.6599999999999999, 1477562020000: 1.54, 1477561890000: 1.6100000000000001, 1477561920000: 1.5900000000000001, 1477562080000: 1.5, 1477561900000: 1.6000000000000001, 1477561850000: 1.6399999999999999, 1477562030000: 1.53, 1477561980000: 1.5600000000000001, 1477561820000: 1.6599999999999999, 1477561830000: 1.6499999999999999, 1477562090000: 1.5, 1477562040000: 1.52}, 'buffered_master': {1477561840000: 97255424.0, 1477561880000: 97255424.0, 1477561930000: 97255424.0, 1477562050000: 97255424.0, 1477562000000: 97255424.0, 1477561950000: 97255424.0, 1477561860000: 97255424.0, 1477561990000: 97255424.0, 1477561970000: 97255424.0, 1477562060000: 97255424.0, 1477561870000: 97255424.0, 1477562010000: 97255424.0, 1477561960000: 97255424.0, 1477561910000: 97255424.0, 1477561800000: 97255424.0, 1477561940000: 97255424.0, 1477562070000: 97255424.0, 1477561810000: 97255424.0, 1477562020000: 97255424.0, 1477561890000: 97255424.0, 1477561920000: 97255424.0, 1477562080000: 97255424.0, 1477561900000: 97255424.0, 1477561850000: 97255424.0, 1477562030000: 97255424.0, 1477561980000: 97255424.0, 1477561820000: 97255424.0, 1477561830000: 97255424.0, 1477562090000: 97255424.0, 1477562040000: 97255424.0}, 'buffered_slave2': {1477561840000: 94973952.0, 1477561880000: 94978048.0, 1477561930000: 94978048.0, 1477562050000: 94978048.0, 1477562000000: 94978048.0, 1477561950000: 94978048.0, 1477561860000: 94978048.0, 1477561990000: 94978048.0, 1477561970000: 94978048.0, 1477562060000: 94978048.0, 1477561870000: 94978048.0, 1477562010000: 94978048.0, 1477561960000: 94978048.0, 1477561910000: 94978048.0, 1477561800000: 94973952.0, 1477561940000: 94978048.0, 1477562070000: 94978048.0, 1477561810000: 94973952.0, 1477562020000: 94978048.0, 1477561890000: 94978048.0, 1477561920000: 94978048.0, 1477562080000: 94978048.0, 1477561900000: 94978048.0, 1477561850000: 94978048.0, 1477562030000: 94978048.0, 1477561980000: 94978048.0, 1477561820000: 94973952.0, 1477561830000: 94973952.0, 1477562090000: 94978048.0, 1477562040000: 94978048.0}, 'tx_master_x': {1477561840000: 1430564125.0, 1477561880000: 1430671597.0, 1477561930000: 1430812775.0, 1477562050000: 1431128149.0, 1477562000000: 1430988824.0, 1477561950000: 1430847864.0, 1477561860000: 1430628130.0, 1477561990000: 1430969828.0, 1477561970000: 1430920777.0, 1477562060000: 1431146732.0, 1477561870000: 1430654935.0, 1477562010000: 1431006524.0, 1477561960000: 1430881233.0, 1477561910000: 1430763782.0, 1477561800000: 1430469559.0, 1477561940000: 1430829771.0, 1477562070000: 1431164933.0, 1477561810000: 1430494834.0, 1477562020000: 1431039320.0, 1477561890000: 1430704568.0, 1477561920000: 1430785849.0, 1477562080000: 1431202624.0, 1477561900000: 1430723372.0, 1477561850000: 1430605452.0, 1477562030000: 1431080453.0, 1477561980000: 1430944577.0, 1477561820000: 1430513053.0, 1477561830000: 1430546471.0, 1477562090000: 1431242575.0, 1477562040000: 1431102401.0}, 'tx_slave2_x': {1477561840000: 644526119.0, 1477561880000: 644636093.0, 1477561930000: 644767073.0, 1477562050000: 645086489.0, 1477562000000: 644953375.0, 1477561950000: 644820921.0, 1477561860000: 644578241.0, 1477561990000: 644927898.0, 1477561970000: 644868468.0, 1477562060000: 645109942.0, 1477561870000: 644607781.0, 1477562010000: 644976808.0, 1477561960000: 644843579.0, 1477561910000: 644712128.0, 1477561800000: 644418144.0, 1477561940000: 644795121.0, 1477562070000: 645138016.0, 1477561810000: 644450688.0, 1477562020000: 645000975.0, 1477561890000: 644659260.0, 1477561920000: 644738980.0, 1477562080000: 645163936.0, 1477561900000: 644682154.0, 1477561850000: 644552851.0, 1477562030000: 645030113.0, 1477561980000: 644896348.0, 1477561820000: 644477479.0, 1477561830000: 644498914.0, 1477562090000: 645188214.0, 1477562040000: 645053022.0}, 'tx_slave2_y': {1477561840000: 3105690.0, 1477561880000: 3106185.0, 1477561930000: 3106781.0, 1477562050000: 3108208.0, 1477562000000: 3107612.0, 1477561950000: 3107024.0, 1477561860000: 3105940.0, 1477561990000: 3107492.0, 1477561970000: 3107249.0, 1477562060000: 3108322.0, 1477561870000: 3106057.0, 1477562010000: 3107726.0, 1477561960000: 3107133.0, 1477561910000: 3106548.0, 1477561800000: 3105221.0, 1477561940000: 3106907.0, 1477562070000: 3108445.0, 1477561810000: 3105344.0, 1477562020000: 3107836.0, 1477561890000: 3106294.0, 1477561920000: 3106666.0, 1477562080000: 3108568.0, 1477561900000: 3106405.0, 1477561850000: 3105824.0, 1477562030000: 3107971.0, 1477561980000: 3107370.0, 1477561820000: 3105467.0, 1477561830000: 3105573.0, 1477562090000: 3108690.0, 1477562040000: 3108086.0}, 'buffered_slave1': {1477561840000: 75751424.0, 1477561880000: 75751424.0, 1477561930000: 75751424.0, 1477562050000: 75751424.0, 1477562000000: 75751424.0, 1477561950000: 75751424.0, 1477561860000: 75751424.0, 1477561990000: 75751424.0, 1477561970000: 75751424.0, 1477562060000: 75751424.0, 1477561870000: 75751424.0, 1477562010000: 75751424.0, 1477561960000: 75751424.0, 1477561910000: 75751424.0, 1477561800000: 75751424.0, 1477561940000: 75751424.0, 1477562070000: 75751424.0, 1477561810000: 75751424.0, 1477562020000: 75751424.0, 1477561890000: 75751424.0, 1477561920000: 75751424.0, 1477562080000: 75751424.0, 1477561900000: 75751424.0, 1477561850000: 75751424.0, 1477562030000: 75751424.0, 1477561980000: 75751424.0, 1477561820000: 75751424.0, 1477561830000: 75751424.0, 1477562090000: 75751424.0, 1477562040000: 75751424.0}, 'cached_slave3': {1477561840000: 1814450176.0, 1477561880000: 1814528000.0, 1477561930000: 1814683648.0, 1477562050000: 1815638016.0, 1477562000000: 1815146496.0, 1477561950000: 1815052288.0, 1477561860000: 1814491136.0, 1477561990000: 1815134208.0, 1477561970000: 1815101440.0, 1477562060000: 1816117248.0, 1477561870000: 1814511616.0, 1477562010000: 1815171072.0, 1477561960000: 1815072768.0, 1477561910000: 1814581248.0, 1477561800000: 1814241280.0, 1477561940000: 1815040000.0, 1477562070000: 1816047616.0, 1477561810000: 1814261760.0, 1477562020000: 1815191552.0, 1477561890000: 1814548480.0, 1477561920000: 1814671360.0, 1477562080000: 1816186880.0, 1477561900000: 1814564864.0, 1477561850000: 1814470656.0, 1477562030000: 1815203840.0, 1477561980000: 1815113728.0, 1477561820000: 1814409216.0, 1477561830000: 1814437888.0, 1477562090000: 1816219648.0, 1477562040000: 1815228416.0}, 'cached_slave2': {1477561840000: 875573248.0, 1477561880000: 875618304.0, 1477561930000: 875671552.0, 1477562050000: 875806720.0, 1477562000000: 875749376.0, 1477561950000: 875692032.0, 1477561860000: 875593728.0, 1477561990000: 875732992.0, 1477561970000: 875716608.0, 1477562060000: 875819008.0, 1477561870000: 875610112.0, 1477562010000: 875757568.0, 1477561960000: 875708416.0, 1477561910000: 875655168.0, 1477561800000: 875528192.0, 1477561940000: 875687936.0, 1477562070000: 875835392.0, 1477561810000: 875540480.0, 1477562020000: 875769856.0, 1477561890000: 875630592.0, 1477561920000: 875663360.0, 1477562080000: 875843584.0, 1477561900000: 875638784.0, 1477561850000: 875581440.0, 1477562030000: 875794432.0, 1477561980000: 875728896.0, 1477561820000: 875548672.0, 1477561830000: 875565056.0, 1477562090000: 875864064.0, 1477562040000: 875798528.0}, 'cached_slave1': {1477561840000: 4039942144.0, 1477561880000: 4041900032.0, 1477561930000: 4040925184.0, 1477562050000: 4042055680.0, 1477562000000: 4039892992.0, 1477561950000: 4041015296.0, 1477561860000: 4041674752.0, 1477561990000: 4039815168.0, 1477561970000: 4041048064.0, 1477562060000: 4042129408.0, 1477561870000: 4041879552.0, 1477562010000: 4040282112.0, 1477561960000: 4041023488.0, 1477561910000: 4041936896.0, 1477561800000: 4039831552.0, 1477561940000: 4041003008.0, 1477562070000: 4042153984.0, 1477561810000: 4039839744.0, 1477562020000: 4040187904.0, 1477561890000: 4041904128.0, 1477561920000: 4040777728.0, 1477562080000: 4042162176.0, 1477561900000: 4041920512.0, 1477561850000: 4039954432.0, 1477562030000: 4040200192.0, 1477561980000: 4039798784.0, 1477561820000: 4039921664.0, 1477561830000: 4039929856.0, 1477562090000: 4042174464.0, 1477562040000: 4042047488.0}, 'tx_slave3_y': {1477561840000: 4841133.0, 1477561880000: 4841796.0, 1477561930000: 4842630.0, 1477562050000: 4844638.0, 1477562000000: 4843790.0, 1477561950000: 4842948.0, 1477561860000: 4841473.0, 1477561990000: 4843626.0, 1477561970000: 4843289.0, 1477562060000: 4844795.0, 1477561870000: 4841638.0, 1477562010000: 4843956.0, 1477561960000: 4843109.0, 1477561910000: 4842292.0, 1477561800000: 4840481.0, 1477561940000: 4842788.0, 1477562070000: 4844981.0, 1477561810000: 4840657.0, 1477562020000: 4844114.0, 1477561890000: 4841959.0, 1477561920000: 4842459.0, 1477562080000: 4845162.0, 1477561900000: 4842118.0, 1477561850000: 4841298.0, 1477562030000: 4844295.0, 1477561980000: 4843460.0, 1477561820000: 4840810.0, 1477561830000: 4840974.0, 1477562090000: 4845350.0, 1477562040000: 4844463.0}, 'tx_slave3_x': {1477561840000: 1043426369.0, 1477561880000: 1043571389.0, 1477561930000: 1043752324.0, 1477562050000: 1044187544.0, 1477562000000: 1044002490.0, 1477561950000: 1043821173.0, 1477561860000: 1043505724.0, 1477561990000: 1043967522.0, 1477561970000: 1043895665.0, 1477562060000: 1044218361.0, 1477561870000: 1043537600.0, 1477562010000: 1044040771.0, 1477561960000: 1043855033.0, 1477561910000: 1043678239.0, 1477561800000: 1043287878.0, 1477561940000: 1043786069.0, 1477562070000: 1044265184.0, 1477561810000: 1043326593.0, 1477562020000: 1044071827.0, 1477561890000: 1043608777.0, 1477561920000: 1043716639.0, 1477562080000: 1044298750.0, 1477561900000: 1043639850.0, 1477561850000: 1043461430.0, 1477562030000: 1044111869.0, 1477561980000: 1043932946.0, 1477561820000: 1043357050.0, 1477561830000: 1043395224.0, 1477562090000: 1044340601.0, 1477562040000: 1044150240.0}, 'cached_slave4': {1477561840000: 1538547712.0, 1477561880000: 1538629632.0, 1477561930000: 1538732032.0, 1477562050000: 1538985984.0, 1477562000000: 1538883584.0, 1477561950000: 1538777088.0, 1477561860000: 1538580480.0, 1477561990000: 1538863104.0, 1477561970000: 1538818048.0, 1477562060000: 1539010560.0, 1477561870000: 1538600960.0, 1477562010000: 1538904064.0, 1477561960000: 1538797568.0, 1477561910000: 1538703360.0, 1477561800000: 1538453504.0, 1477561940000: 1538748416.0, 1477562070000: 1539031040.0, 1477561810000: 1538478080.0, 1477562020000: 1538920448.0, 1477561890000: 1538641920.0, 1477561920000: 1538711552.0, 1477562080000: 1539051520.0, 1477561900000: 1538662400.0, 1477561850000: 1538555904.0, 1477562030000: 1538949120.0, 1477561980000: 1538830336.0, 1477561820000: 1538490368.0, 1477561830000: 1538514944.0, 1477562090000: 1539072000.0, 1477562040000: 1538969600.0}, 'buffered_slave3': {1477561840000: 100331520.0, 1477561880000: 100331520.0, 1477561930000: 100331520.0, 1477562050000: 100331520.0, 1477562000000: 100331520.0, 1477561950000: 100331520.0, 1477561860000: 100331520.0, 1477561990000: 100331520.0, 1477561970000: 100331520.0, 1477562060000: 100331520.0, 1477561870000: 100331520.0, 1477562010000: 100331520.0, 1477561960000: 100331520.0, 1477561910000: 100331520.0, 1477561800000: 100331520.0, 1477561940000: 100331520.0, 1477562070000: 100339712.0, 1477561810000: 100331520.0, 1477562020000: 100331520.0, 1477561890000: 100331520.0, 1477561920000: 100331520.0, 1477562080000: 100339712.0, 1477561900000: 100331520.0, 1477561850000: 100331520.0, 1477562030000: 100331520.0, 1477561980000: 100331520.0, 1477561820000: 100331520.0, 1477561830000: 100331520.0, 1477562090000: 100339712.0, 1477562040000: 100331520.0}, 'buffered_slave4': {1477561840000: 95256576.0, 1477561880000: 95256576.0, 1477561930000: 95256576.0, 1477562050000: 95256576.0, 1477562000000: 95256576.0, 1477561950000: 95256576.0, 1477561860000: 95256576.0, 1477561990000: 95256576.0, 1477561970000: 95256576.0, 1477562060000: 95256576.0, 1477561870000: 95256576.0, 1477562010000: 95256576.0, 1477561960000: 95256576.0, 1477561910000: 95256576.0, 1477561800000: 95256576.0, 1477561940000: 95256576.0, 1477562070000: 95256576.0, 1477561810000: 95256576.0, 1477562020000: 95256576.0, 1477561890000: 95256576.0, 1477561920000: 95256576.0, 1477562080000: 95256576.0, 1477561900000: 95256576.0, 1477561850000: 95256576.0, 1477562030000: 95256576.0, 1477561980000: 95256576.0, 1477561820000: 95256576.0, 1477561830000: 95256576.0, 1477562090000: 95256576.0, 1477562040000: 95256576.0}, 'midterm_slave4': {1477561840000: 1.25, 1477561880000: 1.22, 1477561930000: 1.2, 1477562050000: 1.1299999999999999, 1477562000000: 1.1599999999999999, 1477561950000: 1.1899999999999999, 1477561860000: 1.23, 1477561990000: 1.1599999999999999, 1477561970000: 1.1699999999999999, 1477562060000: 1.1299999999999999, 1477561870000: 1.22, 1477562010000: 1.1499999999999999, 1477561960000: 1.1799999999999999, 1477561910000: 1.21, 1477561800000: 1.28, 1477561940000: 1.1899999999999999, 1477562070000: 1.1200000000000001, 1477561810000: 1.27, 1477562020000: 1.1499999999999999, 1477561890000: 1.21, 1477561920000: 1.21, 1477562080000: 1.1200000000000001, 1477561900000: 1.2, 1477561850000: 1.24, 1477562030000: 1.1399999999999999, 1477561980000: 1.1699999999999999, 1477561820000: 1.26, 1477561830000: 1.26, 1477562090000: 1.1200000000000001, 1477562040000: 1.1399999999999999}} + + +def pacheck(data, type, threashold, lt): + anomalies = [] + for k, v in data.iteritems(): + if type in k: + for t, vl in v.iteritems(): + time = datetime.datetime.fromtimestamp(t / 1000).strftime('%Y-%m-%d %H:%M:%S') + if lt: + if vl > threashold: + print "Found anomaly for %s at %s with value %f" % (k, time, vl) + anomalies.append({"type": "point", "metric": k, "time": time, "value": vl, "threashold": threashold, "threashold_type": "upper"}) + else: + if vl < threashold: + print "Found anomaly for %s at %s with value %f" % (k, t, vl) + anomalies.append( + {"type": "point", "metric": k, "time": time, "value": vl, "threashold": threashold, + "threashold_type": "lower"}) + return anomalies + + + + + +print datetime.datetime.fromtimestamp( + int("1477561960") + ).strftime('%Y-%m-%d %H:%M:%S') + + + +t = pacheck(test, 'shortterm', 2.0, True) +t1 = pacheck(test, 'midterm', 2.0, True) +t2 = pacheck(test, 'longterm', 2.0, True) + +t3 = pacheck(test, 'tx', 568300, True) +t4 = pacheck(test, 'rx', 568300, True) + +t5 = pacheck(test, 'cached', 1512935424, True) +t6 = pacheck(test, 'buffered', 1512935424, True) +t7 = pacheck(test, 'used', 1512935424, True) +t8 = pacheck(test, 'free', 1512935424, True) + + +print len(t) +for p in t: + print p +print len(t8) + diff --git a/util.py b/util.py index 4f3f5ae..1e8f0de 100644 --- a/util.py +++ b/util.py @@ -228,15 +228,15 @@ def parseMethodSettings(st): return mSettings -def wait4Model(count): - if count < 10: +def wait4Model(count=0): + test = False + if test or count < 10: time.sleep(1) count += 1 - print count return wait4Model(count) - print "Done" + return 1 -# wait4Model(0) +# wait4Model() # test = {'s': '10', 'n': '10'} # print parseMethodSettings(test)