Merge pull request #8026 from rtibbles/schoolbus

Remove cherrypy dependency

.github/workflows/no_zombies.yml

@@ -42,5 +42,5 @@ jobs:
         pip install .
 
         # ensure kolibri stops within 20 seconds 10 times in a row
-        ./test/ensure_kolibri_stops_within_time.sh 20 10 8082
+        ./test/ensure_kolibri_stops_within_time.sh 20 10 8082 8083
         ./test/ensure_no_kolibris_running_on_port.sh 8082

kolibri/core/auth/management/commands/deprovision.py

@@ -51,12 +51,14 @@ def deprovision(self):
     def handle_async(self, *args, **options):
 
         # safest not to run this command while the server is running
-        status_code, _ = server.get_urls()
-        if status_code == server.STATUS_RUNNING:
+        try:
+            server.get_status()
             logger.error(
                 "The Kolibri server is currently running. Please stop it and then re-run this command."
             )
             sys.exit(1)
+        except server.NotRunning:
+            pass
 
         # ensure the user REALLY wants to do this!
         confirm_or_exit(

kolibri/core/content/utils/paths.py

@@ -8,6 +8,7 @@
 
 from kolibri.core.content.errors import InvalidStorageFilenameError
 from kolibri.utils import conf
+from kolibri.utils.server import get_zip_port
 
 
 # valid storage filenames consist of 32-char hex plus a file extension
@@ -253,6 +254,20 @@ def get_file_checksums_url(channel_id, baseurl, version="1"):
 ZIPCONTENT = "zipcontent/"
 
 
+def get_zip_content_config():
+    zip_content_origin = conf.OPTIONS["Deployment"]["ZIP_CONTENT_ORIGIN"]
+    if not zip_content_origin:
+        zip_content_port = str(
+            get_zip_port() or conf.OPTIONS["Deployment"]["ZIP_CONTENT_PORT"]
+        )
+    elif type(zip_content_origin) is int:
+        zip_content_port = str(zip_content_origin)
+        zip_content_origin = ""
+    else:
+        zip_content_port = ""
+    return zip_content_origin, zip_content_port
+
+
 def zip_content_path_prefix():
     path_prefix = conf.OPTIONS["Deployment"]["ZIP_CONTENT_URL_PATH_PREFIX"]
 

kolibri/core/discovery/utils/network/search.py

@@ -201,7 +201,11 @@ def register_zeroconf_service(port):
     if ZEROCONF_STATE["service"] is not None:
         unregister_zeroconf_service()
 
-    logger.info("Registering ourselves to zeroconf network with id '%s'..." % id)
+    logger.info(
+        "Registering ourselves to zeroconf network with id '{}' and port '{}'".format(
+            id, port
+        )
+    )
     data = device_info
     ZEROCONF_STATE["service"] = KolibriZeroconfService(id=id, port=port, data=data)
     ZEROCONF_STATE["service"].register()

kolibri/core/kolibri_plugin.py

@@ -19,6 +19,7 @@
 from kolibri.core.content.utils.paths import get_content_storage_url
 from kolibri.core.content.utils.paths import get_hashi_path
 from kolibri.core.content.utils.paths import get_zip_content_base_path
+from kolibri.core.content.utils.paths import get_zip_content_config
 from kolibri.core.device.models import ContentCacheKey
 from kolibri.core.device.utils import allow_other_browsers_to_connect
 from kolibri.core.hooks import NavigationHook
@@ -53,14 +54,7 @@ def url_tag(self):
             # For some reason the js_name gets escaped going into the template
             # so this was the easiest way to inject it.
         ).replace("__placeholder__", js_name)
-        zip_content_origin = OPTIONS["Deployment"]["ZIP_CONTENT_ORIGIN"]
-        if not zip_content_origin:
-            zip_content_port = str(OPTIONS["Deployment"]["ZIP_CONTENT_PORT"])
-        elif type(zip_content_origin) is int:
-            zip_content_port = str(zip_content_origin)
-            zip_content_origin = ""
-        else:
-            zip_content_port = ""
+        zip_content_origin, zip_content_port = get_zip_content_config()
         return [
             mark_safe(
                 """<script type="text/javascript">"""

kolibri/core/tasks/test/test_worker.py

@@ -40,7 +40,7 @@ def test_enqueue_job_runs_job(self, worker):
         job = Job(id, 9)
         worker.storage.enqueue_job(job, QUEUE)
 
-        while job.state == State.QUEUED:
+        while job.state != State.COMPLETED:
             job = worker.storage.get_job(job.job_id)
             time.sleep(0.5)
         try:

kolibri/core/tasks/utils.py

@@ -49,7 +49,7 @@ def import_stringified_func(funcstring):
 class InfiniteLoopThread(compat.Thread):
     """A class that runs a given function an infinite number of times, until told to shut down."""
 
-    DEFAULT_TIMEOUT_SECONDS = 0.2
+    DEFAULT_TIMEOUT_SECONDS = 0.001
 
     def __init__(self, func, thread_name, wait_between_runs=1, *args, **kwargs):
         """

kolibri/utils/cli.py

@@ -28,11 +28,9 @@
 from . import sanity_checks
 from . import server
 from .debian_check import check_debian_user
-from .system import become_daemon
 from kolibri.core.deviceadmin.utils import IncompatibleDatabase
 from kolibri.core.upgrade import matches_version
 from kolibri.core.upgrade import run_upgrades
-from kolibri.deployment.default.cache import recreate_diskcache
 from kolibri.plugins import config
 from kolibri.plugins import DEFAULT_PLUGINS
 from kolibri.plugins.utils import autoremove_unavailable_plugins
@@ -326,6 +324,9 @@ def initialize(skip_update=False):  # noqa: max-complexity=12
     # we attempt to set up django.
     autoremove_unavailable_plugins()
 
+    # Check if there is an options.ini file exist inside the KOLIBRI_HOME folder
+    sanity_checks.check_default_options_exist()
+
     version = get_version()
 
     updated = version_updated(kolibri.__version__, version)
@@ -446,93 +447,34 @@ def main(ctx):
         pass
 
 
-def create_startup_lock(port):
-    try:
-        server._write_pid_file(server.STARTUP_LOCK, port)
-    except (IOError, OSError):
-        logger.warn("Impossible to create file lock to communicate starting process")
-
-
 @main.command(cls=KolibriDjangoCommand, help="Start the Kolibri process")
 @click.option(
     "--port",
     default=OPTIONS["Deployment"]["HTTP_PORT"],
     type=int,
     help="Port on which Kolibri is being served",
 )
+@click.option(
+    "--zip-port",
+    default=OPTIONS["Deployment"]["ZIP_CONTENT_PORT"],
+    type=int,
+    help="Port on which zip content server is being served",
+)
 @click.option(
     "--background/--foreground",
     default=True,
     help="Run Kolibri as a background process",
 )
-def start(port, background):
+def start(port, zip_port, background):
     """
     Start the server on given port.
     """
-
-    # Check if there is an options.ini file exist inside the KOLIBRI_HOME folder
-    sanity_checks.check_default_options_exist()
-
-    serve_http = OPTIONS["Server"]["CHERRYPY_START"]
-
-    if serve_http:
-        # Check if the port is occupied
-        sanity_checks.check_other_kolibri_running(port)
-
-    create_startup_lock(port)
-
-    # Clear old sessions up
-    call_command("clearsessions")
-    recreate_diskcache()
-
-    # On Mac, Python crashes when forking the process, so prevent daemonization until we can figure out
-    # a better fix. See https://github.com/learningequality/kolibri/issues/4821
-    if sys.platform == "darwin":
-        background = False
-
-    if not background:
-        logger.info("Running Kolibri")
-
-    else:
-        logger.info("Running Kolibri as background process")
-
-    if serve_http:
-
-        __, urls = server.get_urls(listen_port=port)
-        if not urls:
-            logger.error(
-                "Could not detect an IP address that Kolibri binds to, but try "
-                "opening up the following addresses:\n"
-            )
-            urls = [
-                "http://{}:{}".format(ip, port) for ip in ("localhost", "127.0.0.1")
-            ]
-        else:
-            logger.info("Kolibri running on:\n")
-        for addr in urls:
-            sys.stderr.write("\t{}\n".format(addr))
-        sys.stderr.write("\n")
-    else:
-        logger.info("Starting Kolibri background workers")
-
-    # Daemonize at this point, no more user output is needed
-    if background:
-
-        from django.conf import settings
-
-        kolibri_log = settings.LOGGING["handlers"]["file"]["filename"]
-        logger.info("Going to background mode, logging to {0}".format(kolibri_log))
-
-        kwargs = {}
-        # Truncate the file
-        if os.path.isfile(server.DAEMON_LOG):
-            open(server.DAEMON_LOG, "w").truncate()
-        kwargs["out_log"] = server.DAEMON_LOG
-        kwargs["err_log"] = server.DAEMON_LOG
-
-        become_daemon(**kwargs)
-
-    server.start(port=port, serve_http=serve_http)
+    server.start(
+        port=port,
+        zip_port=zip_port,
+        serve_http=OPTIONS["Server"]["CHERRYPY_START"],
+        background=background,
+    )
 
 
 @main.command(cls=KolibriCommand, help="Stop the Kolibri process")
@@ -541,41 +483,31 @@ def stop():
     Stops the server unless it isn't running
     """
     try:
-        pid, __, __ = server.get_status()
-        server.stop(pid=pid)
-        stopped = True
+        server.get_status()
+    except server.NotRunning as e:
+        if e.status_code == server.STATUS_STOPPED:
+            logging.info(
+                "Already stopped: {}".format(
+                    server.status_messages[server.STATUS_STOPPED]
+                )
+            )
+            sys.exit(0)
+    status = server.stop()
+    if status == server.STATUS_STOPPED:
         if OPTIONS["Server"]["CHERRYPY_START"]:
             logger.info("Kolibri server has successfully been stopped.")
         else:
             logger.info("Kolibri background services have successfully been stopped.")
-    except server.NotRunning as e:
-        verbose_status = "{msg:s} ({code:d})".format(
-            code=e.status_code, msg=status.codes[e.status_code]
-        )
-        if e.status_code == server.STATUS_STOPPED:
-            logger.info("Already stopped: {}".format(verbose_status))
-            stopped = True
-        elif e.status_code == server.STATUS_STARTING_UP:
-            logger.error("Not stopped: {}".format(verbose_status))
-            sys.exit(e.status_code)
-        else:
-            logger.error(
-                "During graceful shutdown, server says: {}".format(verbose_status)
-            )
-            logger.error("Not responding, killing with force")
-            server.stop(force=True)
-            stopped = True
-
-    if stopped:
         sys.exit(0)
+    sys.exit(status)
 
 
 @main.command(cls=KolibriCommand, help="Show the status of the Kolibri process")
 def status():
     """
     How is Kolibri doing?
     Check the server's status. For possible statuses, see the status dictionary
-    status.codes
+    server.status_messages
 
     Status *always* outputs the current status in the first line of stderr.
     The following lines contain optional information such as the addresses where
@@ -584,47 +516,30 @@ def status():
     TODO: We can't guarantee the above behavior because of the django stack
     being loaded regardless
 
-    Exits with status_code, key has description in status.codes
+    Exits with status_code, key has description in server.status_messages
     """
     status_code, urls = server.get_urls()
 
     if status_code == server.STATUS_RUNNING:
-        sys.stderr.write("{msg:s} (0)\n".format(msg=status.codes[0]))
+        sys.stderr.write("{msg:s} (0)\n".format(msg=server.status_messages[0]))
         if urls:
             sys.stderr.write("Kolibri running on:\n\n")
             for addr in urls:
                 sys.stderr.write("\t{}\n".format(addr))
     else:
-        verbose_status = status.codes[status_code]
+        verbose_status = server.status_messages[status_code]
         sys.stderr.write(
             "{msg:s} ({code:d})\n".format(code=status_code, msg=verbose_status)
         )
     sys.exit(status_code)
 
 
-status.codes = {
-    server.STATUS_RUNNING: "OK, running",
-    server.STATUS_STOPPED: "Stopped",
-    server.STATUS_STARTING_UP: "Starting up",
-    server.STATUS_NOT_RESPONDING: "Not responding",
-    server.STATUS_FAILED_TO_START: "Failed to start (check log file: {0})".format(
-        server.DAEMON_LOG
-    ),
-    server.STATUS_UNCLEAN_SHUTDOWN: "Unclean shutdown",
-    server.STATUS_UNKNOWN_INSTANCE: "Unknown Kolibri running on port",
-    server.STATUS_SERVER_CONFIGURATION_ERROR: "Kolibri server configuration error",
-    server.STATUS_PID_FILE_READ_ERROR: "Could not read PID file",
-    server.STATUS_PID_FILE_INVALID: "Invalid PID file",
-    server.STATUS_UNKNOWN: "Could not determine status",
-}
-
-
 @main.command(cls=KolibriDjangoCommand, help="Start worker processes")
 @click.option(
     "--port",
     default=OPTIONS["Deployment"]["HTTP_PORT"],
     type=int,
-    help="Port on which to run Kolibri services",
+    help="Port on which Kolibri is running to inform services",
 )
 @click.option(
     "--background/--foreground",
@@ -636,29 +551,9 @@ def services(port, background):
     Start the kolibri background services.
     """
 
-    create_startup_lock(None)
-    recreate_diskcache()
-
     logger.info("Starting Kolibri background services")
 
-    # Daemonize at this point, no more user output is needed
-    if background:
-
-        from django.conf import settings
-
-        kolibri_log = settings.LOGGING["handlers"]["file"]["filename"]
-        logger.info("Going to background mode, logging to {0}".format(kolibri_log))
-
-        kwargs = {}
-        # Truncate the file
-        if os.path.isfile(server.DAEMON_LOG):
-            open(server.DAEMON_LOG, "w").truncate()
-        kwargs["out_log"] = server.DAEMON_LOG
-        kwargs["err_log"] = server.DAEMON_LOG
-
-        become_daemon(**kwargs)
-
-    server.start(port=port, serve_http=False)
+    server.start(port=port, zip_port=None, serve_http=False, background=background)
 
 
 def setup_logging(debug=False, debug_database=False):

kolibri/utils/logger.py

@@ -224,21 +224,6 @@ def get_default_logging_config(LOG_ROOT, debug=False, debug_database=False):
                 "level": DEFAULT_LEVEL,
                 "propagate": False,
             },
-            "cherrypy.access": {
-                "handlers": ["file", "console", "file_debug"],
-                "level": DEFAULT_LEVEL,
-                "propagate": False,
-            },
-            "cherrypy.error": {
-                "handlers": ["file", "console", "file_debug"],
-                "level": DEFAULT_LEVEL,
-                "propagate": False,
-            },
-            "cherrypy": {
-                "handlers": ["file", "console", "file_debug"],
-                "level": DEFAULT_LEVEL,
-                "propagate": False,
-            },
         },
     }