Switch from file-based caching to Django's lru_cache for sqlite max v…

…ars (#73)

Switch from file-based caching to Django's lru_cache for sqlite max vars

morango/__init__.py

@@ -3,4 +3,4 @@
 from __future__ import unicode_literals
 
 default_app_config = "morango.apps.MorangoConfig"
-__version__ = "0.4.5"
+__version__ = "0.4.7"

morango/apps.py

@@ -3,9 +3,7 @@
 import logging as logger
 
 from django.apps import AppConfig
-from django.db import connection
 
-from morango.util import max_parameter_substitution
 from morango.utils.register_models import add_syncable_models
 
 logging = logger.getLogger(__name__)
@@ -20,5 +18,3 @@ def ready(self):
 
         # add models to be synced by profile
         add_syncable_models()
-        if "sqlite" in connection.vendor:
-            max_parameter_substitution()

morango/util.py

@@ -1,10 +1,10 @@
 import functools
 import logging
-import os
 import sqlite3
 
+from django.utils.lru_cache import lru_cache
+
 from morango.constants.capabilities import GZIP_BUFFER_POST
-from morango.constants.file import SQLITE_VARIABLE_FILE_CACHE
 
 logger = logging.getLogger(__name__)
 
@@ -79,14 +79,14 @@ def wrapper(*args, **kwargs):
         return wrapper
 
 
-def max_parameter_substitution():
+@lru_cache(maxsize=1)
+def calculate_max_sqlite_variables():
     """
     SQLite has a limit on the max number of variables allowed for parameter substitution. This limit is usually 999, but
     can be compiled to a different number. This function calculates what the max is for the sqlite version running on the device.
     We use the calculated value to chunk our SQL bulk insert statements when deserializing from the store to the app layer.
+    Source: https://stackoverflow.com/questions/17872665/determine-maximum-number-of-columns-from-sqlite3
     """
-    if os.path.isfile(SQLITE_VARIABLE_FILE_CACHE):
-        return
     conn = sqlite3.connect(":memory:")
     low = 1
     high = (
@@ -109,5 +109,4 @@ def max_parameter_substitution():
         else:
             low = guess
     conn.close()
-    with open(SQLITE_VARIABLE_FILE_CACHE, "w") as file:
-        file.write(str(low))
+    return low

morango/utils/backends/sqlite.py

@@ -1,23 +1,16 @@
 import os
 
-from morango.constants.file import SQLITE_VARIABLE_FILE_CACHE
 from morango.models import Buffer
 from morango.models import RecordMaxCounter
 from morango.models import RecordMaxCounterBuffer
 from morango.models import Store
+from morango.util import calculate_max_sqlite_variables
 from morango.utils.backends.base import BaseSQLWrapper
 
 
 class SQLWrapper(BaseSQLWrapper):
     backend = "sqlite"
 
-    def __init__(self):
-        if os.path.isfile(SQLITE_VARIABLE_FILE_CACHE):
-            with open(SQLITE_VARIABLE_FILE_CACHE) as file:
-                self.SQLITE_MAX_VARIABLE_NUMBER = int(file.read())
-        else:
-            self.SQLITE_MAX_VARIABLE_NUMBER = 999
-
     def _bulk_insert_into_app_models(
         self, cursor, app_model, fields, db_values, placeholder_list
     ):
@@ -27,7 +20,7 @@ def _bulk_insert_into_app_models(
         where values=[1,2,3,4,5,6,7,8,9]
         """
         # calculate and create equal sized chunks of data to insert incrementally
-        num_of_rows_able_to_insert = self.SQLITE_MAX_VARIABLE_NUMBER // len(fields)
+        num_of_rows_able_to_insert = calculate_max_sqlite_variables() // len(fields)
         num_of_values_able_to_insert = num_of_rows_able_to_insert * len(fields)
         value_chunks = [
             db_values[x : x + num_of_values_able_to_insert]