Skip to content

Commit

Permalink
don't ship external logs from the main thread of the dispatcher
Browse files Browse the repository at this point in the history
this is a fairly esoteric change that attempts to work around a bug
we've discovered in cpython itself

context: ansible#4181
  • Loading branch information
ryanpetrello committed Jun 27, 2019
1 parent 5c338e5 commit 955b3d0
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 2 deletions.
7 changes: 7 additions & 0 deletions awx/main/management/commands/run_dispatcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from django.db import connection as django_connection, connections
from kombu import Exchange, Queue

from awx.main.utils.handlers import AWXProxyHandler
from awx.main.dispatch import get_local_queuename, reaper
from awx.main.dispatch.control import Control
from awx.main.dispatch.kombu import Connection
Expand Down Expand Up @@ -121,6 +122,12 @@ def handle(self, *arg, **options):

reaper.reap()
consumer = None

# don't ship external logs inside the dispatcher's parent process
# this exists to work around a race condition + deadlock bug on fork
# in cpython itself:
# https://bugs.python.org/issue37429
AWXProxyHandler.disable()
with Connection(settings.BROKER_URL) as conn:
try:
bcast = 'tower_broadcast_all'
Expand Down
14 changes: 12 additions & 2 deletions awx/main/utils/handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import json
import requests
import time
import threading
import socket
import select
from urllib import parse as urlparse
Expand Down Expand Up @@ -286,6 +287,8 @@ class AWXProxyHandler(logging.Handler):
Parameters match same parameters in the actualized handler classes.
'''

thread_local = threading.local()

def __init__(self, **kwargs):
# TODO: process 'level' kwarg
super(AWXProxyHandler, self).__init__(**kwargs)
Expand Down Expand Up @@ -322,8 +325,9 @@ def get_handler(self, custom_settings=None, force_create=False):
return self._handler

def emit(self, record):
actual_handler = self.get_handler()
return actual_handler.emit(record)
if AWXProxyHandler.thread_local.enabled:
actual_handler = self.get_handler()
return actual_handler.emit(record)

def perform_test(self, custom_settings):
"""
Expand Down Expand Up @@ -353,6 +357,12 @@ def perform_test(self, custom_settings):
except RequestException as e:
raise LoggingConnectivityException(str(e))

@classmethod
def disable(cls):
cls.thread_local.enabled = False

AWXProxyHandler.thread_local.enabled = True


ColorHandler = logging.StreamHandler

Expand Down

0 comments on commit 955b3d0

Please sign in to comment.