diff --git a/bin/kalite b/bin/kalite index ffc0496231..9481f8f92b 100755 --- a/bin/kalite +++ b/bin/kalite @@ -28,6 +28,13 @@ import subprocess from distutils import spawn import warnings + +# Python 2+3 definition +try: + input = raw_input # @ReservedAssignment +except NameError: + pass + kalite_path = None # Ensure that PATH is set because distutils.find_executable fails on @@ -78,6 +85,30 @@ if not os.path.exists(kalite_path): # Converts to normal string for Windows users, where it gets turned into a unicode string and errors out in subprocess env['KALITE_DIR'] = str(kalite_path) +# TODO(benjaoming): Delegate responsibility to the .deb package +# This always evaluates to False on Windows, so no problem with the +# system-specific path +if os.name == "posix" and os.path.isfile("/etc/ka-lite/username"): + username = open("/etc/ka-lite/username", "r").read() + username = username.split("\n")[0] if username else None + if username and username != os.environ["USER"]: + default_home = os.environ.get("KALITE_HOME", os.path.expanduser("~/.kalite")) + if not os.path.exists(default_home) or not os.listdir(default_home): + sys.stderr.write(( + "You are not running kalite as the default user {0}. " + "This is recommended unless you want to re-create the database " + "and start storing new videos/exercises etc. for a different " + "user account\n\n" + ).format(username)) + sys.stderr.write(( + "To run the command as the default user, run this instead:\n\n" + " sudo su {user} -s /bin/sh -c {command}\n\n" + ).format(user=username, command=" ".join(sys.argv))) + cont = input("Do you wish to continue? [y/N] ") + if not cont.lower() == "y": + sys.exit(0) + + if 'KALITE_PYTHON' in os.environ: python_executable = os.environ['KALITE_PYTHON'] if not spawn.find_executable(python_executable): diff --git a/docs/installguide/install_all.rst b/docs/installguide/install_all.rst index 79bbc427af..bf92082948 100644 --- a/docs/installguide/install_all.rst +++ b/docs/installguide/install_all.rst @@ -120,7 +120,10 @@ On Linux and other Unix-like systems, downloaded videos and database files are i Raspberry Pi ============ -For a Raspberry Pi running a Debian system, you can install the Debian package. +For a Raspberry Pi running a Debian system, you can install the special Debian +package, 'ka-lite-raspberry-pi'. + +You can find here (TODO) @@ -141,30 +144,26 @@ In our tests, we found that the WiPi adaptor supported a higher number tablet co * Afterwards, insert the wireless USB adaptor. * Lastly, switch the Raspberry Pi on. -#. Make sure the Raspberry Pi operating system is up-to-date. - * Login with the account used to install KA Lite - * Update the Raspberry Pi operating system by: - * *sudo apt-get update* - * *sudo apt-get upgrade* -#. Get the installation scripts. - * *cd /opt* - * *sudo git clone https://github.com/learningequality/ka-lite-pi-scripts.git* +#. Install the .deb package: ``dpkg -i /path/to/ka-lite-raspberry-pi.deb`` +#. Get the network configuration scripts. + * ``cd /opt`` + * ``sudo git clone https://github.com/learningequality/ka-lite-pi-scripts.git`` #. Install and configure the access point. - * *cd /opt/ka-lite-pi-scripts* - * *sudo ./configure.sh* + * ``cd /opt/ka-lite-pi-scripts`` + * ``sudo ./configure.sh`` .. note:: If using the Edimax EW-7811UN, ignore the "hostapdSegmentation fault" error. #. Install the USB adaptor software. * If using the WiPi, run this command: - * *cd /opt/ka-lite-pi-scripts* - * *sudo ./use_wipi.sh* + * ``cd /opt/ka-lite-pi-scripts`` + * ``sudo ./use_wipi.sh`` * If using the Edimax EW-7811Un, run this command: - * *cd /opt/ka-lite-pi-scripts* - * *sudo ./use_edimax.sh* + * ``cd /opt/ka-lite-pi-scripts`` + * ``sudo ./use_edimax.sh`` #. Complete the access point configuration - * *sudo python ./configure_network_interfaces.py* - * *sudo insserv hostapd* + * ``sudo python ./configure_network_interfaces.py`` + * ``sudo insserv hostapd`` #. Finally - * *sudo reboot* + * ``sudo reboot`` * A wireless network named "kalite" should be available. * Connect to this network * If the KA Lite server is started, browse to 1.1.1.1 diff --git a/kalite/distributed/management/commands/initialize_kalite.py b/kalite/distributed/management/commands/initialize_kalite.py index 6617db7ec8..da22a76790 100644 --- a/kalite/distributed/management/commands/initialize_kalite.py +++ b/kalite/distributed/management/commands/initialize_kalite.py @@ -22,17 +22,19 @@ class Command(BaseCommand): ) def setup_server_if_needed(self): - """Run the setup command, if necessary.""" - - # Ensure that the database has been synced and a Device has been created + """Run the setup command, if necessary. + It's necessary if the Settings model doesn't have a "database_version" or if that version doesn't match + kalite.version.VERSION, indicating the source has been changed. Then setup is run to create/migrate the db. + """ + try: - assert Settings.get("private_key") and Device.objects.count() + from kalite.version import VERSION + assert Settings.get("database_version") == VERSION except (DatabaseError, AssertionError): - # Otherwise, run the setup command - self.stdout.write("Setting up KA Lite; this may take a few minutes; please wait!\n") + logging.info("Setting up KA Lite; this may take a few minutes; please wait!\n") call_command("setup", interactive=False) - # Double check that the setup process successfully created a Device - assert Settings.get("private_key") and Device.objects.count(), "There was an error configuring the server. Please report the output of this command to Learning Equality." + # Double check the setup process worked ok. + assert Settings.get("database_version") == VERSION, "There was an error configuring the server. Please report the output of this command to Learning Equality." def reinitialize_server(self): """Reset the server state.""" diff --git a/kalite/distributed/management/commands/setup.py b/kalite/distributed/management/commands/setup.py index deeac047d2..e679684f5f 100644 --- a/kalite/distributed/management/commands/setup.py +++ b/kalite/distributed/management/commands/setup.py @@ -365,6 +365,15 @@ def handle(self, *args, **options): # Should clean_pyc for (clean) reinstall purposes # call_command("clean_pyc", interactive=False, verbosity=options.get("verbosity"), path=os.path.join(settings.PROJECT_PATH, "..")) + # Migrate the database + call_command( + "syncdb", interactive=False, verbosity=options.get("verbosity")) + call_command("migrate", merge=True, verbosity=options.get("verbosity")) + # Create *.json and friends database + call_command("syncdb", interactive=False, verbosity=options.get( + "verbosity"), database="assessment_items") + Settings.set("database_version", VERSION) + # download assessment items # This can take a long time and lead to Travis stalling. None of this # is required for tests, and does not apply to the central server. @@ -374,10 +383,6 @@ def handle(self, *args, **options): else: - # Migrate the database - call_command( - "syncdb", interactive=False, verbosity=options.get("verbosity")) - call_command("migrate", merge=True, verbosity=options.get("verbosity")) # Outdated location of assessment items - move assessment items from their # old location (CONTENT_ROOT/khan where they were mixed with other content # items) diff --git a/kalite/settings/__init__.py b/kalite/settings/__init__.py index 718962ab6d..c9db9eb4b2 100644 --- a/kalite/settings/__init__.py +++ b/kalite/settings/__init__.py @@ -46,6 +46,8 @@ def package_selected(package_name): RemovedInKALite_v015_Warning ) +DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP = getattr(local_settings, "DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP", False) + # Config for Raspberry Pi distributed server if package_selected("RPi"): diff --git a/kalite/version.py b/kalite/version.py index bb4464030c..a397f77c10 100644 --- a/kalite/version.py +++ b/kalite/version.py @@ -7,7 +7,7 @@ # Must also be of the form N.N.N for internal use, where N is a non-negative integer MAJOR_VERSION = "0" MINOR_VERSION = "14" -PATCH_VERSION = "0" +PATCH_VERSION = "alpha9" VERSION = "%s.%s.%s" % (MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION) SHORTVERSION = "%s.%s" % (MAJOR_VERSION, MINOR_VERSION) diff --git a/python-packages/fle_utils/django_utils/command.py b/python-packages/fle_utils/django_utils/command.py index 44a1b403b3..2e15fc6d39 100644 --- a/python-packages/fle_utils/django_utils/command.py +++ b/python-packages/fle_utils/django_utils/command.py @@ -14,7 +14,6 @@ from django.utils.translation import ugettext as _ - def call_command_with_output(cmd, *args, **kwargs): """ Run call_command while capturing stdout/stderr and calls to sys.exit @@ -51,36 +50,6 @@ def call_command_with_output(cmd, *args, **kwargs): sys.exit = backups[2] -def call_command_subprocess(cmd, *args, **kwargs): - assert "manage_py_dir" in kwargs, "don't forget to specify the manage_py_dir" - - manage_py_dir = kwargs["manage_py_dir"] - del kwargs["manage_py_dir"] - wait_for_exit = kwargs.get("wait_for_exit", True) - if "wait_for_exit" in kwargs: del kwargs["wait_for_exit"] - - # Use sys to get the same executable running as is running this process. - # Make sure to call the manage.py from this project. - call_args = [sys.executable, os.path.join(manage_py_dir, "manage.py"), cmd] - call_args += list(args) - for key,val in kwargs.iteritems(): - if isinstance(val, bool): - call_args.append("--%s" % key) - else: - call_args.append("--%s=%s" % (key, val)) - - # We don't need to hold onto the process handle. - # we expect all commands to return eventually, on their own-- - # we have no way to deal with a rogue process. - # But, because they're subprocesses of this process, when the - # server stops, so do these processes. - # Note that this is also OK because chronograph does all "stopping" - # using messaging through the database - p = subprocess.Popen(call_args) - if wait_for_exit: - p.communicate() - return p - class CommandProcess(multiprocessing.Process): def __init__(self, cmd, *args, **kwargs): super(CommandProcess, self).__init__() @@ -91,6 +60,7 @@ def __init__(self, cmd, *args, **kwargs): def run(self): call_command(self.cmd, *self.args, **self.kwargs) + class CommandThread(threading.Thread): def __init__(self, cmd, *args, **kwargs): super(CommandThread, self).__init__() @@ -102,6 +72,7 @@ def run(self): #logging.debug("Starting command %s with parameters %s, %s)" % (self.cmd, self.args, self.kwargs)) call_command(self.cmd, *self.args, **self.kwargs) + JOB_THREADS = {} def call_command_subprocess(cmd, *args, **kwargs): p = CommandProcess(cmd, *args, **kwargs) @@ -138,11 +109,27 @@ def call_command_async(cmd, *args, **kwargs): # and have everyone else spawn threads. is_osx = sys.platform == 'darwin' in_proc = kwargs.pop('in_proc', not is_osx) - #in_proc = kwargs.pop('in_proc', True)S if in_proc: return call_command_threaded(cmd, *args, **kwargs) else: + # MUST: Check if running on PyRun to prevent crashing the server since it doesn't have + # the `multiprocessing` module by default. + if hasattr(sys, 'pyrun'): + # MUST: Do this since PyRun doesn't include the `multiprocessing` module + # by default so let's call `call_outside_command_with_output` which uses + # the `subprocess` module. + + if settings.IS_SOURCE and 'kalite_dir' not in kwargs: + kwargs['kalite_dir'] = settings.SOURCE_DIR + + if 'wait' not in kwargs: + # We are supposed to be an async call so let's not wait. + kwargs['wait'] = False + + return call_outside_command_with_output(cmd, *args, **kwargs) + + # Let's use the OS's python interpreter. return call_command_subprocess(cmd, *args, **kwargs) diff --git a/scripts/raspberry_pi_setup.sh b/scripts/raspberry_pi_setup.sh deleted file mode 100755 index f155a838c5..0000000000 --- a/scripts/raspberry_pi_setup.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/bin/bash - -if [ `whoami` == "root" ] -then - echo "Cannot run as root!" - exit 1 -fi - -# Create configuration directory in case we never ran before -mkdir -p ~/.kalite - -USER_CONFIG=~/.kalite/settings.py - -if ! grep -Pq "^from kalite.project.settings.raspberry_pi" $USER_CONFIG -then - echo "Changing user config to use raspberry pi settings: $USER_CONFIG" - echo "from kalite.project.settings.raspberry_pi import *" > $USER_CONFIG -fi - -if [ ! -f "/etc/init.d/kalite" ] -then - read -p "Do you want to run ka-lite automatically at boot ? [Y/n] " yn - if [[ ! $yn == "n" ]]; then echo "skipping" - else - kalite manage initdconfig > /etc/init.d/kalite - chmod 755 /etc/init.d/kalite - update-rc.d kalite defaults - fi -fi - -echo "Step 1 - Installing M2Crypto, psutil and nginx" - -# discover if packages are already installed -to_install="" -python -c "import M2Crypto" >/dev/null 2>&1 -if [ $? != 0 ] ; then to_install="python-m2crypto"; fi -python -c "import psutil" >/dev/null 2>&1 -if [ $? != 0 ] ; then to_install="$to_install python-psutil"; fi -nginx -v >/dev/null 2>&1 -if [ $? != 0 ] ; then to_install="$to_install nginx"; fi - -# check network (by trying some likely sites), but only if packages need installing -if [ "$to_install" != "" ] ; then - echo "Info: Need to install: $to_install, testing connection" - wget -q http://adhocsync.org >/dev/null 2>&1 - if [ $? != 0 ] ; then wget -q http://mirrordirector.raspbian.org >/dev/null 2>&1 - if [ $? != 0 ] ; then wget -q http://google.com >/dev/null 2>&1 - if [ $? != 0 ] ; then - echo "Error: internet connection isn't working, cannot continue" - read WAITING - exit 1 - fi - fi - fi -else - echo "Info: Everything is already installed" -fi - -if [ "$to_install" != "" ] ; then sudo apt-get -y install $to_install; fi - -# Finally, check the packages are installed, incase there was an apt-get failure -sanity_check_ok="False" -python -c "import M2Crypto" >/dev/null 2>&1 -if [ $? = 0 ] ; then python -c "import psutil" >/dev/null 2>&1 - if [ $? = 0 ] ; then nginx -v >/dev/null 2>&1 - if [ $? = 0 ] ; then sanity_check_ok="True"; fi - fi -fi - -if [ $sanity_check_ok != "True" ] ; then - echo "Error: One or more modules are missing, cannot continue" - read WAITING - exit 1 -fi - -echo "Step 2 - Configure or reconfigure nginx to work with KA Lite" - -if [ -f /etc/nginx/sites-enabled/default ]; then - sudo rm /etc/nginx/sites-enabled/default -fi -if [ -f /etc/nginx/sites-enabled/kalite ]; then - sudo rm /etc/nginx/sites-enabled/kalite -fi - -sudo touch /etc/nginx/sites-available/kalite -sudo sh -c "kalite manage nginxconfig > /etc/nginx/sites-available/kalite" -sudo ln -s /etc/nginx/sites-available/kalite /etc/nginx/sites-enabled/kalite - -echo "Step 3 - Optimize nginx configuration" - -sudo rm /etc/nginx/nginx.conf -sudo touch /etc/nginx/nginx.conf - -sudo sh -c "cat > /etc/nginx/nginx.conf" <<'NGINX' -user www-data; -pid /var/run/nginx.pid; - -### -# we have 1 cpu so only need 1 worker process -worker_processes 1; - -events { - ### - # good overall speed on RPi with this setting - worker_connections 1536; - - ### - # Activate the optimised polling for linux - use epoll; - - ### - # Keep multi_accept off - RPi+KA Lite is slowed if "on" - multi_accept off; -} - -http { - ### - # RPi+KA Lite is faster with sendfile "off" - sendfile off; - tcp_nopush off; - - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - ### - # Speed up landing page by caching open file descriptors - open_file_cache max=2048; - - ## - # Logging Settings - # don't log, we don't need to know normally - access_log off; - error_log off; - - ## - # Gzip Settings - # We are CPU limited, not bandwidth limited, so don't gzip - gzip off; - - ## - # Virtual Host Configs - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; -} - -NGINX - -echo 'Step 4 - Finally... stopping and starting the background servers' - -sudo service kalite stop -sudo service kalite start -sudo service nginx stop -sudo service nginx start - -echo 'Now you can access KA-Lite through port 8008 (which will use optimizations)' -echo 'or directly through port 7007 (which will not use optimizations)'