Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hot fix for failing video downloads on OSX. #4115

Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 19 additions & 32 deletions python-packages/fle_utils/django_utils/command.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
from django.utils.translation import ugettext as _



def call_command_with_output(cmd, *args, **kwargs):
"""
Run call_command while capturing stdout/stderr and calls to sys.exit
Expand Down Expand Up @@ -51,36 +50,6 @@ def call_command_with_output(cmd, *args, **kwargs):
sys.exit = backups[2]


def call_command_subprocess(cmd, *args, **kwargs):
assert "manage_py_dir" in kwargs, "don't forget to specify the manage_py_dir"

manage_py_dir = kwargs["manage_py_dir"]
del kwargs["manage_py_dir"]
wait_for_exit = kwargs.get("wait_for_exit", True)
if "wait_for_exit" in kwargs: del kwargs["wait_for_exit"]

# Use sys to get the same executable running as is running this process.
# Make sure to call the manage.py from this project.
call_args = [sys.executable, os.path.join(manage_py_dir, "manage.py"), cmd]
call_args += list(args)
for key,val in kwargs.iteritems():
if isinstance(val, bool):
call_args.append("--%s" % key)
else:
call_args.append("--%s=%s" % (key, val))

# We don't need to hold onto the process handle.
# we expect all commands to return eventually, on their own--
# we have no way to deal with a rogue process.
# But, because they're subprocesses of this process, when the
# server stops, so do these processes.
# Note that this is also OK because chronograph does all "stopping"
# using messaging through the database
p = subprocess.Popen(call_args)
if wait_for_exit:
p.communicate()
return p

class CommandProcess(multiprocessing.Process):
def __init__(self, cmd, *args, **kwargs):

This comment was marked as spam.

super(CommandProcess, self).__init__()
Expand All @@ -91,6 +60,7 @@ def __init__(self, cmd, *args, **kwargs):
def run(self):
call_command(self.cmd, *self.args, **self.kwargs)


class CommandThread(threading.Thread):
def __init__(self, cmd, *args, **kwargs):
super(CommandThread, self).__init__()
Expand All @@ -102,6 +72,7 @@ def run(self):
#logging.debug("Starting command %s with parameters %s, %s)" % (self.cmd, self.args, self.kwargs))
call_command(self.cmd, *self.args, **self.kwargs)


JOB_THREADS = {}
def call_command_subprocess(cmd, *args, **kwargs):
p = CommandProcess(cmd, *args, **kwargs)
Expand Down Expand Up @@ -138,11 +109,27 @@ def call_command_async(cmd, *args, **kwargs):
# and have everyone else spawn threads.
is_osx = sys.platform == 'darwin'
in_proc = kwargs.pop('in_proc', not is_osx)
#in_proc = kwargs.pop('in_proc', True)S

if in_proc:
return call_command_threaded(cmd, *args, **kwargs)
else:
# MUST: Check if running on PyRun to prevent crashing the server since it doesn't have

This comment was marked as spam.

# the `multiprocessing` module by default.
if hasattr(sys, 'pyrun'):
# MUST: Do this since PyRun doesn't include the `multiprocessing` module
# by default so let's call `call_outside_command_with_output` which uses
# the `subprocess` module.

if settings.IS_SOURCE and 'kalite_dir' not in kwargs:
kwargs['kalite_dir'] = settings.SOURCE_DIR

if 'wait' not in kwargs:
# We are supposed to be an async call so let's not wait.
kwargs['wait'] = False

return call_outside_command_with_output(cmd, *args, **kwargs)

# Let's use the OS's python interpreter.
return call_command_subprocess(cmd, *args, **kwargs)


Expand Down