Upgrade Django (and related packages) to 4.2

CONTRIBUTING.md

@@ -14,7 +14,7 @@ If you find a bug in the source code or a mistake in the documentation, you can
 with a fix.
 
 #### Submitting an Issue
-Before you submit your issue search the archive, maybe your question was already answered. Please take a minute to search through the open issues in the repo, and if you find one that matches your own, feel free to comment and your thoughts.
+Before you submit your issue search the archive, maybe your question was already answered. Please take a minute to search through the open issues in the repo, and if you find one that matches your own, feel free to comment and add your thoughts.
 If your issue appears to be a bug, and hasn't been reported, open a new issue. Be sure to create your issue in the Arches repo and not a fork of Arches so that it is visible to the community.
 
 Please add the [appropriate labels](https://github.com/archesproject/arches/wiki/Issue-and-Pull-Request-Labels) to the issue (multiple labels are ok) to help us keep track of them.

README.md

@@ -45,7 +45,7 @@ to create a frontend asset bundle. This process should complete in less than 2 m
 
 Finally, visit `localhost:8000` in a browser (only Chrome is fully supported at this time).
 
-If you run into problems, please review our the full [installation documentation](http://arches.readthedocs.io/en/stable/installation/)
+If you run into problems, please review our full [installation documentation](http://arches.readthedocs.io/en/stable/installation/)
 
 #### Release Cycle
 

arches/app/datatypes/base.py

@@ -3,7 +3,7 @@
 from arches.app.models import models
 from arches.app.models.system_settings import settings
 from arches.app.search.elasticsearch_dsl_builder import Dsl, Bool, Terms, Exists, Nested
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 import logging
 
 logger = logging.getLogger(__name__)

arches/app/datatypes/concept_types.py

@@ -3,7 +3,7 @@
 import csv
 import logging
 from django.core.exceptions import ObjectDoesNotExist
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from arches.app.models import models
 from arches.app.models import concept
 from django.core.cache import cache
@@ -184,7 +184,7 @@ def transform_value_for_tile(self, value, **kwargs):
                 try:
                     value = self.lookup_labelid_from_label(value, kwargs)
                 except:
-                    logger.warn(_("Unable to convert {0} to concept label".format(value)))
+                    logger.warning(_("Unable to convert {0} to concept label".format(value)))
         return value
 
     def transform_export_values(self, value, *args, **kwargs):
@@ -328,7 +328,7 @@ def transform_value_for_tile(self, value, **kwargs):
                             ret.append(self.lookup_labelid_from_label(v, kwargs))
                         except:
                             ret.append(v)
-                            logger.warn(_("Unable to convert {0} to concept label".format(v)))
+                            logger.warning(_("Unable to convert {0} to concept label".format(v)))
         return ret
 
     def transform_export_values(self, value, *args, **kwargs):

arches/app/datatypes/datatypes.py

@@ -48,15 +48,14 @@
 from django.core.files import File
 from django.core.files.base import ContentFile
 from django.core.files.storage import FileSystemStorage, default_storage
-from django.utils.translation import ugettext as _
 from django.contrib.gis.geos import GEOSGeometry
 from django.contrib.gis.geos import GeometryCollection
 from django.contrib.gis.geos import fromstr
 from django.contrib.gis.geos import Polygon
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.exceptions import ValidationError
 from django.db import connection, transaction
-from django.utils.translation import get_language, ugettext as _
+from django.utils.translation import get_language, gettext as _
 
 from elasticsearch import Elasticsearch
 from elasticsearch.exceptions import NotFoundError

arches/app/datatypes/url.py

@@ -27,7 +27,7 @@
 from rdflib import ConjunctiveGraph as Graph
 from rdflib import URIRef, Literal, Namespace
 from rdflib.namespace import RDF, RDFS, XSD, DC, DCTERMS
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 
 archesproject = Namespace(settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT)
 cidoc_nm = Namespace("http://www.cidoc-crm.org/cidoc-crm/")
@@ -41,7 +41,7 @@
 try:
     default_url_widget = Widget.objects.get(name=default_widget_name)
 except Widget.DoesNotExist as e:
-    logger.warn("Setting 'url' datatype's default widget to None ({0} widget not found).".format(default_widget_name))
+    logger.warning("Setting 'url' datatype's default widget to None ({0} widget not found).".format(default_widget_name))
 
 details = {
     "datatype": "url",

arches/app/etl_modules/base_data_editor.py

@@ -5,7 +5,7 @@
 import uuid
 from django.db import connection
 from django.http import HttpRequest
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from arches.app.datatypes.datatypes import DataTypeFactory
 from arches.app.etl_modules.base_import_module import BaseImportModule
 from arches.app.models.models import GraphModel, Node

arches/app/etl_modules/base_import_module.py

@@ -2,7 +2,7 @@
 import os
 from django.core.files.storage import default_storage
 from django.db.utils import IntegrityError, ProgrammingError
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from django.utils.decorators import method_decorator
 from django.db import connection
 from arches.app.utils.decorators import user_created_transaction_match
@@ -43,7 +43,7 @@ def reverse(self, request, **kwargs):
         except Exception as e:
             response["data"] = e
             logger.error(e)
-        logger.warn(response)
+        logger.warning(response)
         return response
 
     def save_to_tiles(self, loadid, finalize_import=True, multiprocessing=True):

arches/app/etl_modules/branch_csv_importer.py

@@ -10,7 +10,7 @@
 from django.http import HttpResponse
 from openpyxl import load_workbook
 from django.db import connection
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from django.core.files.storage import default_storage
 from arches.app.datatypes.datatypes import DataTypeFactory
 from arches.app.models.models import Node

arches/app/etl_modules/import_single_csv.py

@@ -9,7 +9,7 @@
 from django.core.files.storage import default_storage
 from django.db import connection
 from django.db.models.functions import Lower
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from arches.app.datatypes.datatypes import DataTypeFactory
 from arches.app.models.models import GraphModel, Node, NodeGroup
 from arches.app.models.system_settings import settings

arches/app/functions/primary_descriptors.py

@@ -4,7 +4,7 @@
 from arches.app.functions.base import BaseFunction
 from arches.app.models import models
 from arches.app.datatypes.datatypes import DataTypeFactory
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 
 logger = logging.getLogger(__name__)
 

arches/app/models/concept.py

@@ -30,8 +30,7 @@
 from arches.app.search.mappings import CONCEPTS_INDEX
 from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer
 from arches.app.utils.i18n import rank_label
-from django.utils.translation import ugettext as _
-from django.utils.translation import get_language
+from django.utils.translation import get_language, gettext as _
 from django.db import IntegrityError
 from psycopg2.extensions import AsIs
 
@@ -1441,7 +1440,7 @@ def exact_val_match(val, conceptid=None):
                 }
             }
 
-    concept_label_results = se.search(index=CONCEPTS_INDEX, body=exact_val_match(label, conceptid))
+    concept_label_results = se.search(index=CONCEPTS_INDEX, **exact_val_match(label, conceptid))
     if concept_label_results is None:
         print("Found no matches for label:'{0}' and concept_id: '{1}'".format(label, conceptid))
         return

arches/app/models/fields/i18n.py

@@ -5,10 +5,13 @@
 from django.utils.translation import gettext_lazy as _
 from django.db.models import JSONField
 from django.db.models.sql.compiler import SQLInsertCompiler
+from django.db.models.sql.where import NothingNode
 from django.utils.translation import get_language
 
 
-class I18n_String(object):
+class I18n_String(NothingNode):
+    """Subclassing NothingNode works around https://code.djangoproject.com/ticket/34745."""
+
     def __init__(self, value=None, lang=None, use_nulls=False, attname=None):
         self.attname = attname
         self.value = value
@@ -212,8 +215,13 @@ def get_prep_value(self, value):
 
         return I18n_String(value, attname=self.attname, use_nulls=self.use_nulls)
 
+    def get_db_prep_save(self, value, connection):
+        """Override to avoid the optimization from Django 4.2 that
+        immediately returns `value` if it is None."""
+        return self.get_db_prep_value(value, connection)
+
 
-class I18n_JSON(object):
+class I18n_JSON(NothingNode):
     def __init__(self, value=None, lang=None, use_nulls=False, attname=None):
         self.attname = attname
         self.value = value
@@ -397,3 +405,8 @@ def get_prep_value(self, value):
         """
 
         return I18n_JSON(value, attname=self.attname)
+
+    def get_db_prep_save(self, value, connection):
+        """Override to avoid the optimization from Django 4.2 that
+        immediately returns `value` if it is None."""
+        return self.get_db_prep_value(value, connection)

arches/app/models/graph.py

@@ -32,7 +32,7 @@
 from arches.app.datatypes.datatypes import DataTypeFactory
 from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer
 from arches.app.search.search_engine_factory import SearchEngineFactory
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from pyld.jsonld import compact, JsonLdError
 from django.db.models.base import Deferred
 from django.utils import translation

arches/app/models/migrations/9748_alter_graphmodel_functions_alter_icon_id.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.4 on 2023-08-16 07:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('models', '9747_alter_maplayer_options'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='graphmodel',
+            name='functions',
+            field=models.ManyToManyField(through='models.FunctionXGraph', to='models.function'),
+        ),
+        migrations.AlterField(
+            model_name='icon',
+            name='id',
+            field=models.AutoField(primary_key=True, serialize=False),
+        ),
+    ]

arches/app/models/models.py

@@ -27,7 +27,7 @@
 from django.db.models.signals import post_delete, pre_save, post_save
 from django.dispatch import receiver
 from django.utils import translation
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from django.contrib.auth.models import User
 from django.contrib.auth.models import Group
 from django.contrib.contenttypes.models import ContentType
@@ -1519,7 +1519,7 @@ def send_email_on_save(sender, instance, **kwargs):
                 instance.save()
         except Exception as e:
             logger = logging.getLogger(__name__)
-            logger.warn("Email Server not correctly set up. See settings to configure.")
+            logger.warning("Email Server not correctly set up. See settings to configure.")
 
     return False
 

arches/app/models/resource.py

@@ -26,7 +26,7 @@
 from django.contrib.auth.models import User, Group
 from django.forms.models import model_to_dict
 from django.core.exceptions import ObjectDoesNotExist
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from django.utils.translation import get_language
 from arches.app.models import models
 from arches.app.models.models import EditLog

arches/app/models/tile.py

@@ -30,7 +30,7 @@
 from django.db.models import Q
 from django.contrib.auth.models import User
 from django.utils import timezone
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from arches.app.models import models
 from arches.app.models.resource import Resource
 from arches.app.models.resource import EditLog

arches/app/search/base_index.py

@@ -1,6 +1,6 @@
 import pyprind
 from datetime import datetime
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from arches.app.models import models
 from arches.app.models.resource import Resource
 from arches.app.models.system_settings import settings
@@ -32,7 +32,7 @@ def prepare_index(self):
         """
 
         if self.index_metadata is not None:
-            self.se.create_index(index=self.index_name, body=self.index_metadata)
+            self.se.create_index(index=self.index_name, **self.index_metadata)
         else:
             raise SearchIndexError("No index metadata defined.")
 
@@ -84,7 +84,7 @@ def index_resources(self, resources=None, batch_size=settings.BULK_IMPORT_BATCH_
         start = datetime.now()
         q = Query(se=self.se)
         self.se.refresh(index=self.index_name)
-        count_before = self.se.count(index=self.index_name, body=q.dsl)
+        count_before = self.se.count(index=self.index_name, **q.dsl)
         result_summary = {"database": len(resources), "indexed": 0}
         if quiet is False:
             bar = pyprind.ProgBar(len(resources), bar_char="█") if len(resources) > 1 else None
@@ -98,7 +98,7 @@ def index_resources(self, resources=None, batch_size=settings.BULK_IMPORT_BATCH_
                     indexer.add(index=self.index_name, id=doc_id, data=document)
 
         self.se.refresh(index=self.index_name)
-        result_summary["indexed"] = self.se.count(index=self.index_name, body=q.dsl) - count_before
+        result_summary["indexed"] = self.se.count(index=self.index_name, **q.dsl) - count_before
         status = "Passed" if result_summary["database"] == result_summary["indexed"] else "Failed"
         print(f"Custom Index - {settings.ELASTICSEARCH_PREFIX}_{self.index_name}")
         print(

arches/app/search/components/map_filter.py

@@ -1,7 +1,7 @@
 import logging
 from django.contrib.gis.geos import GEOSGeometry
 from django.db import connection
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from arches.app.models.system_settings import settings
 from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer
 from arches.app.search.elasticsearch_dsl_builder import Bool, Nested, Terms, GeoShape

arches/app/search/elasticsearch_dsl_builder.py

@@ -17,7 +17,7 @@
 """
 
 from arches.app.utils.betterJSONSerializer import JSONSerializer
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 
 
 class Dsl(object):
@@ -98,10 +98,10 @@ def search(self, index="", **kwargs):
             return self.se.search(index=index, scroll=self.scroll, **self.dsl)
 
     def count(self, index="", **kwargs):
-        return self.se.count(index=index, body=self.dsl)
+        return self.se.count(index=index, **self.dsl)
 
     def delete(self, index="", **kwargs):
-        return self.se.delete(index=index, body=self.dsl, **kwargs)
+        return self.se.delete(index=index, query=self.dsl, **kwargs)
 
     def prepare(self, scroll=False):
         if self.scroll is None:

arches/app/search/mappings.py

@@ -57,7 +57,7 @@ def prepare_terms_index(create=False):
 
     if create:
         se = SearchEngineFactory().create()
-        se.create_index(index=TERMS_INDEX, body=index_settings)
+        se.create_index(index=TERMS_INDEX, **index_settings)
 
     return index_settings
 
@@ -90,7 +90,7 @@ def prepare_concepts_index(create=False):
 
     if create:
         se = SearchEngineFactory().create()
-        se.create_index(index=CONCEPTS_INDEX, body=index_settings)
+        se.create_index(index=CONCEPTS_INDEX, **index_settings)
 
     return index_settings
 
@@ -253,7 +253,7 @@ def prepare_search_index(create=False):
 
     if create:
         se = SearchEngineFactory().create()
-        se.create_index(index=RESOURCES_INDEX, body=index_settings)
+        se.create_index(index=RESOURCES_INDEX, **index_settings)
 
     return index_settings
 
@@ -292,7 +292,7 @@ def prepare_resource_relations_index(create=False):
 
     if create:
         se = SearchEngineFactory().create()
-        se.create_index(index=RESOURCE_RELATIONS_INDEX, body=index_settings)
+        se.create_index(index=RESOURCE_RELATIONS_INDEX, **index_settings)
 
     return index_settings