Skip to content

Commit

Permalink
merge from origin
Browse files Browse the repository at this point in the history
  • Loading branch information
@chuckablack
chuckablack committed Dec 2, 2020
2 parents bf73936 + 665af5e commit d8cf096
Show file tree
Hide file tree
Showing 13 changed files with 134 additions and 125 deletions.
4 changes: 4 additions & 0 deletions quokka-ui/src/components/Capture.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ class Capture extends Component {
renderDevices(dashboard) {
dashboard.setState({show: "devices"})
}
renderWorkers(dashboard) {
dashboard.setState({show: "workers"})
}

render() {

Expand All @@ -100,6 +103,7 @@ class Capture extends Component {
<Button variant="contained" style={{width: '100%'}} onClick={() => this.renderDevices(this.state.dashboard)}>Return to Devices</Button>
<Button variant="contained" style={{width: '100%'}} onClick={() => this.renderHosts(this.state.dashboard)}>Return to Hosts</Button>
<Button variant="contained" style={{width: '100%'}} onClick={() => this.renderServices(this.state.dashboard)}>Return to Services</Button>
<Button variant="contained" style={{width: '100%'}} onClick={() => this.renderWorkers(this.state.dashboard)}>Return to Workers</Button>
</Grid>

<Grid item style={{width: '85%', paddingRight: '10px'}}>
Expand Down
136 changes: 79 additions & 57 deletions quokka-ui/src/components/WorkerStatus.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import {FlexibleXYPlot, HorizontalGridLines, LineMarkSeries, LineSeries, XAxis, YAxis} from 'react-vis'
import React, {Component} from "react";
import Grid from "@material-ui/core/Grid";
import Button from "@material-ui/core/Button";

class WorkerStatus extends Component {

Expand Down Expand Up @@ -85,6 +86,10 @@ class WorkerStatus extends Component {
return {tsData: tsData, maxY: maxY};
}

renderWorkers(dashboard) {
dashboard.setState({show: "workers"})
}

render() {

let data = this.getTSData("RSP_TIME");
Expand All @@ -100,64 +105,81 @@ class WorkerStatus extends Component {
const tsMemoryData = data.tsData;
const maxYMemory = data.maxY;
return (
<Grid item style={{padding: '10px'}}>
<h6 align='right'>Time until refresh: {this.state.countdownValue} seconds</h6>
<Grid container direction="row">
<Grid item style={{width: '50%'}}>
<Grid item>
<h5>Response Time (establish connection)</h5>
<FlexibleXYPlot
height={300}
xType="time"
yDomain={[0,maxYRspTime+(maxYRspTime/5)]}>
<HorizontalGridLines />
<LineSeries
data={tsRspTimeData} />
<XAxis title="Time of Day"/>
<YAxis title="Response Time"/>
</FlexibleXYPlot>
</Grid>
<Grid item>
<h5>Availability</h5>
<FlexibleXYPlot
height={300}
xType="time"
yDomain={[0,maxYAvailability]}>
<HorizontalGridLines />
<LineMarkSeries
color="green"
data={tsAvailabilityData} />
<XAxis title="Time of Day"/>
<YAxis title="Availability"/>
</FlexibleXYPlot>
</Grid>
<Grid container direction="column">
<Grid container direction="row" style={{paddingTop: '10px'}}>
<Grid item style={{width: '15%', paddingLeft: '10px'}}>
<b>WORKER NAME</b>:<br />{this.state.workerData.worker.name}
<br /><br />
<b>Host</b>:<br />{this.state.workerData.worker.host}
<br /><br />
<b>Worker Type</b>:<br />{this.state.workerData.worker.worker_type}
<br /><br />
<b>Last heard</b>:<br />{this.state.workerData.worker.last_heard}
<br /><br /> <br /><br />
<b>REFRESH IN</b>:<br/>{this.state.countdownValue} seconds
<br/><br/> <br/><br/>
<Button variant="contained" style={{width: '100%'}} onClick={() => this.renderWorkers(this.state.dashboard)}>Return to Workers</Button>
</Grid>
<Grid item style={{width: '50%'}}>
<Grid item>
<h5>CPU Utilization</h5>
<FlexibleXYPlot
height={300}
xType="time"
yDomain={[0,100]}>
<HorizontalGridLines />
<LineSeries
data={tsCpuData} />
<XAxis title="Time of Day"/>
<YAxis title="CPU"/>
</FlexibleXYPlot>
</Grid>
<Grid item>
<h5>Memory Utilization</h5>
<FlexibleXYPlot
height={300}
xType="time"
yDomain={[0,100]}>
<HorizontalGridLines />
<LineSeries
data={tsMemoryData} />
<XAxis title="Time of Day"/>
<YAxis title="Memory"/>
</FlexibleXYPlot>
<Grid item style={{width: '85%', paddingRight: '10px'}}>
<h6 align='right'>Time until refresh: {this.state.countdownValue} seconds</h6>
<Grid container direction="row">
<Grid item style={{width: '50%'}}>
<Grid item>
<h5>Response Time</h5>
<FlexibleXYPlot
height={300}
xType="time"
yDomain={[0,maxYRspTime+(maxYRspTime/5)]}>
<HorizontalGridLines />
<LineSeries
data={tsRspTimeData} />
<XAxis title="Time of Day"/>
<YAxis title="Response Time"/>
</FlexibleXYPlot>
</Grid>
<Grid item>
<h5>Availability</h5>
<FlexibleXYPlot
height={300}
xType="time"
yDomain={[0,maxYAvailability]}>
<HorizontalGridLines />
<LineMarkSeries
color="green"
data={tsAvailabilityData} />
<XAxis title="Time of Day"/>
<YAxis title="Availability"/>
</FlexibleXYPlot>
</Grid>
</Grid>
<Grid item style={{width: '50%'}}>
<Grid item>
<h5>CPU Utilization</h5>
<FlexibleXYPlot
height={300}
xType="time"
yDomain={[0,100]}>
<HorizontalGridLines />
<LineSeries
data={tsCpuData} />
<XAxis title="Time of Day"/>
<YAxis title="CPU"/>
</FlexibleXYPlot>
</Grid>
<Grid item>
<h5>Memory Utilization</h5>
<FlexibleXYPlot
height={300}
xType="time"
yDomain={[0,100]}>
<HorizontalGridLines />
<LineSeries
data={tsMemoryData} />
<XAxis title="Time of Day"/>
<YAxis title="Memory"/>
</FlexibleXYPlot>
</Grid>
</Grid>
</Grid>
</Grid>
</Grid>
Expand Down
14 changes: 10 additions & 4 deletions quokka-ui/src/components/Workers.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@ class Workers extends Component {
}

renderCapture(ip) {
if(ip === 'localhost') {
return;
}
this.state.dashboard.setState({ip: ip, protocol: null, port: null, show: "capture"})
}

Expand All @@ -103,6 +106,9 @@ class Workers extends Component {
}

renderTraceRouteDialog(target) {
if(target === 'localhost') {
return;
}
this.initiateTraceRoute(target)
this.setState({openTraceRouteDialog: true, target: target})
}
Expand Down Expand Up @@ -175,15 +181,15 @@ class Workers extends Component {
icon: 'pageview',
tooltip: 'Capture packets for worker',
onClick: (event, rowData) => {
this.renderCapture(rowData.ip_address)
}
this.renderCapture(rowData.host)
},
},
{
icon: AccountTreeTwoToneIcon,
tooltip: 'Trace-route to worker',
onClick: (event, rowData) => {
this.renderTraceRouteDialog(rowData.hostname)
}
this.renderTraceRouteDialog(rowData.host)
},
}
]}
/>
Expand Down
4 changes: 2 additions & 2 deletions quokka/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@
ThreadManager.start_discovery_thread(discovery_interval)
ThreadManager.start_host_thread(host_monitor_interval)
ThreadManager.start_summaries_thread()
# ThreadManager.start_worker_thread(worker_monitor_interval)
ThreadManager.start_worker_thread(worker_monitor_interval)

from quokka.controller.CaptureManager import CaptureManager
capture_manager = CaptureManager()
Expand All @@ -119,7 +119,7 @@ def shutdown():
ThreadManager.stop_host_thread()
ThreadManager.stop_service_thread()
ThreadManager.stop_summaries_thread()
# ThreadManager.stop_worker_thread()
ThreadManager.stop_worker_thread()
ThreadManager.stop_device_threads()

log_console("\n---> all threads shut down, terminating.")
Expand Down
4 changes: 2 additions & 2 deletions quokka/controller/CaptureManager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ def find_monitor(ip=None):
def initiate_capture(ip, protocol, port, count):

monitor = CaptureManager.find_monitor(ip)
result, worker = get_worker(host=monitor, worker_type=CaptureManager.worker_type)
if result != "success":
worker = get_worker(host=monitor, worker_type=CaptureManager.worker_type)
if worker is None:
log_console(
f"Capture Manager: could not find worker, host={monitor}, worker_type={CaptureManager.worker_type} in DB"
)
Expand Down
4 changes: 2 additions & 2 deletions quokka/controller/PortscanManager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ def find_monitor(ip=None):
def initiate_portscan(host_ip, host_name, token):

monitor = PortscanManager.find_monitor(host_ip)
result, worker = get_worker(host=monitor, worker_type=PortscanManager.worker_type)
if result != "success":
worker = get_worker(host=monitor, worker_type=PortscanManager.worker_type)
if worker is None:
log_console(
f"Portscan Manager: could not find worker, host={monitor}, worker_type={PortscanManager.worker_type} in DB"
)
Expand Down
17 changes: 12 additions & 5 deletions quokka/controller/SummariesTask.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,13 @@ def get_summaries(self, items, item_type, get_hour_data_function):
rsp_time_in_seconds = hourly_summary["response_time"] / 1000
if "sla_response_time" in item and rsp_time_in_seconds > item["sla_response_time"]:
info = f"SLA response time violation, {rsp_time_in_seconds:.2f} > {item['sla_response_time']}"
log_event(str(datetime.now())[:-3], item_type, item['name'], "WARNING", info)
if "sla_availability" in item and hourly_summary["availability"] < item["sla_availability"]:
log_event(str(datetime.now())[:-3], item_type, item["name"], "WARNING", info)
if (
"sla_availability" in item
and hourly_summary["availability"] < item["sla_availability"]
):
info = f"SLA availability violation, {hourly_summary['availability']:.2f} < {item['sla_availability']}"
log_event(str(datetime.now())[:-3], item_type, item['name'], "WARNING", info)
log_event(str(datetime.now())[:-3], item_type, item["name"], "WARNING", info)

return hourly_summaries

Expand All @@ -80,9 +83,13 @@ def start(self, interval):
time.sleep(60)
continue

service_hourly_summaries = self.get_summaries(get_all_services(), "services", get_service_status_data_for_hour)
service_hourly_summaries = self.get_summaries(
get_all_services(), "services", get_service_status_data_for_hour
)
record_service_hourly_summaries(service_hourly_summaries)
host_hourly_summaries = self.get_summaries(get_all_hosts(), "hosts", get_host_status_data_for_hour)
host_hourly_summaries = self.get_summaries(
get_all_hosts(), "hosts", get_host_status_data_for_hour
)
record_host_hourly_summaries(host_hourly_summaries)
self.get_summaries(get_all_devices(), "devices", get_device_status_data_for_hour)

Expand Down
4 changes: 2 additions & 2 deletions quokka/controller/TracerouteManager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ def initiate_traceroute(target, token):
target = parsed_target.netloc

monitor = TracerouteManager.find_monitor(target)
result, worker = get_worker(host=monitor, worker_type=TracerouteManager.worker_type)
if result != "success":
worker = get_worker(host=monitor, worker_type=TracerouteManager.worker_type)
if worker is None:
log_console(
f"Traceroute Manager: could not find worker, host={monitor}, worker_type={TracerouteManager.worker_type} in DB"
)
Expand Down
51 changes: 10 additions & 41 deletions quokka/controller/WorkerMonitorTask.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
import subprocess
from datetime import datetime, timedelta

import time
from datetime import datetime, timedelta

from quokka.controller.utils import get_response_time
from quokka.models.apis import get_all_workers, set_worker, record_worker_status, log_event
from quokka.controller.utils import log_console
from quokka.models.apis import get_all_workers, set_worker, record_worker_status

MAX_NOT_HEARD_SECONDS = 90 # For http workers, time to have not seen a heartbeat

Expand All @@ -30,43 +27,15 @@ def monitor(self, interval):
if self.terminate:
break

if worker["connection_type"] == "rabbitmq":

log_console(f"--- monitor:worker pinging {worker['host']}")
try:
ping_output = subprocess.check_output(
["ping", "-c3", "-n", "-i0.5", "-W2", str(worker["host"])]
)
worker["availability"] = True
worker["response_time"] = get_response_time(str(ping_output))
worker["last_heard"] = str(datetime.now())[:-3]

except subprocess.CalledProcessError:
worker["availability"] = False
log_event(
str(datetime.now())[:-3],
"worker monitor",
worker["host"],
"INFO",
f"Availability failed for worker: {worker['host']}",
)

elif worker["connection_type"] == "http":

if not worker["last_heard"]:
continue

last_heard_time = datetime.strptime(worker["last_heard"], "%Y-%m-%d %H:%M:%S.%f")
print(f"now: {datetime.now()}, last_heard: {last_heard_time}")
if (datetime.now() - last_heard_time) > timedelta(seconds=MAX_NOT_HEARD_SECONDS):
worker["availability"] = False
record_worker_status(worker)
set_worker(worker)

continue # HTTP-REST devices (e.g. sdwan) communicate to us, we don't poll them
if not worker["last_heard"]:
continue

record_worker_status(worker)
set_worker(worker)
last_heard_time = datetime.strptime(worker["last_heard"], "%Y-%m-%d %H:%M:%S.%f")
print(f"now: {datetime.now()}, last_heard: {last_heard_time}")
if (datetime.now() - last_heard_time) > timedelta(seconds=MAX_NOT_HEARD_SECONDS):
worker["availability"] = False
record_worker_status(worker)
set_worker(worker)

for _ in range(0, int(interval / 10)):
time.sleep(10)
Expand Down
6 changes: 3 additions & 3 deletions quokka/models/apis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -910,9 +910,9 @@ def get_worker(worker_id=None, serial=None, host=None, worker_type=None):

worker_obj = db.session.query(Worker).filter_by(**search).one_or_none()
if not worker_obj:
return "failed", "Could not find worker in DB"

return "success", get_model_as_dict(worker_obj)
return None
else:
return get_model_as_dict(worker_obj)


def get_all_workers():
Expand Down
1 change: 1 addition & 0 deletions quokka/views/ui_views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from datetime import datetime

from quokka.controller.device.device_info import get_device_info
from quokka.controller.utils import get_this_ip
from quokka.models.apis import (
get_device,
get_all_devices,
Expand Down
Loading

0 comments on commit d8cf096

Please sign in to comment.