Merge pull request #2 from marko-k0/develop

Dockerfile & Backoff Spamming Fix

.travis.yml

@@ -1,3 +1,7 @@
+branches:
+  only:
+    - master
+    - develop
 language: python
 python:
   - "3.7"

Dockerfile

@@ -0,0 +1,31 @@
+FROM alpine as build
+
+RUN apk add --no-cache python3 python3-dev build-base libffi-dev openssl-dev linux-headers git && \
+    python3 -m ensurepip && \
+    rm -r /usr/lib/python*/ensurepip && \
+    pip3 install --upgrade pip setuptools wheel && \
+    if [ ! -e /usr/bin/pip ]; then ln -s pip3 /usr/bin/pip ; fi && \
+    if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi && \
+    rm -r /root/.cache
+
+COPY . /sonny
+WORKDIR /sonny
+
+RUN pip install -e . && python setup.py bdist_wheel
+
+
+
+FROM alpine
+
+COPY --from=build /sonny/dist/*.whl .
+RUN apk add --no-cache python3 python3-dev build-base libffi-dev openssl-dev linux-headers git nmap && \
+    python3 -m ensurepip && \
+    rm -r /usr/lib/python*/ensurepip && \
+    pip3 install --upgrade pip setuptools wheel && \
+    pip3 install --upgrade *.whl && \
+    if [ ! -e /usr/bin/pip ]; then ln -s pip3 /usr/bin/pip ; fi && \
+    if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi && \
+    rm -r /root/.cache && rm *.whl
+
+WORKDIR /root
+CMD ["true"]

src/sonny/monitor.py

@@ -90,6 +90,8 @@ def __init__(self):
         signal.signal(signal.SIGINT, self.signal_catch)
         signal.signal(signal.SIGTERM, self.signal_catch)
 
+        self.last_run_backed_off = 0
+
     def signal_catch(self, signum, frame):
         _logger.warning('monitor terminating')
         sys.exit()
@@ -136,8 +138,14 @@ def run_step(self):
 
             s_hvs = self.get_suspicious_hypervisors()
             if s_hvs:
+                backoff = len(s_hvs) > SUSPICIOUS_BACKOFF
+                if backoff and self.last_run_backed_off:
+                    self.last_run_backed_off += 1
+                    return
+
                 _logger.warning(f'suspicious hypervisors: {s_hvs}')
-                if len(s_hvs) > SUSPICIOUS_BACKOFF:
+                if backoff:
+                    self.last_run_backed_off += 1
                     _logger.warning(
                         'too many suspicious hypervisors, backing off')
                     return
@@ -160,7 +168,13 @@ def run_step(self):
                 else:
                     _logger.info('tcp scan check shows hypervisors are ok')
             else:
-                _logger.debug('no suspicious hypervisors')
+                if self.last_run_backed_off:
+                    n = self.last_run_backed_off
+                    _logger.info('no suspicious hypervisors')
+                    _logger.info(f'backed off {n} times')
+                    self.last_run_backed_off = 0
+                else:
+                    _logger.debug('no suspicious hypervisors')
         elif not self.api_alive:
             if job.result:
                 _logger.warning(f'issues within the worker: {job.result}')
@@ -312,7 +326,7 @@ def get_suspicious_hypervisors(self):
 
             if all([(current_time - t) > HEARTBEAT_PERIOD for t in ts_list]):
                 hypervisor_list.append(hv_name)
-                _logger.info(f'hypervisor {hv_name} is suspicious')
+                _logger.debug(f'hypervisor {hv_name} is suspicious')
                 for a, t in agent_dict.items():
                     tt = strptime(t, "%Y-%m-%d %H:%M:%S").timestamp()
                     tt_d = int(current_time - tt)

src/sonny/sonny.py

@@ -90,7 +90,7 @@ def run(self):
                         self.post_message('sonny initialized')
                         self.post_message(f'subscribed to clouds {CLOUDS}')
                     else:
-                        self.post_message('sonny re-initialized')
+                        self.handle_command('status', self.channel)
 
                     delay = 2
                     while True:

tests/test_monitor.py

@@ -47,6 +47,7 @@ def monitor():
         monitor.redis = sonny.monitor.redis
         monitor.redis.flushall()
         monitor.api_alive = True
+        monitor.last_run_backed_off = 0
 
         monitor.wait_for_job = MagicMock(return_value=True)