✅ Add tests / fix mocks ES

project/geosource/tests/test_model_exceptions.py

@@ -25,8 +25,10 @@ def decode(self):
         raise UnicodeDecodeError("wrong")
 
 
+@patch("elasticsearch.client.IndicesClient.create")
+@patch("elasticsearch.client.IndicesClient.delete")
 class CSVSourceExceptionsTestCase(TestCase):
-    def test_csv_with_wrong_x_coord(self):
+    def test_csv_with_wrong_x_coord(self, mocked_es_delete, mocked_es_create):
         source = CSVSource.objects.create(
             file=get_file("source.csv"),
             geom_type=0,
@@ -48,7 +50,7 @@ def test_csv_with_wrong_x_coord(self):
             source._get_records()
             self.assertIn(msg, source.report.get("message", []))
 
-    def test_csv_with_wrong_y_coord(self):
+    def test_csv_with_wrong_y_coord(self, mocked_es_delete, mocked_es_create):
         source = CSVSource.objects.create(
             file=get_file("source.csv"),
             geom_type=0,
@@ -70,7 +72,9 @@ def test_csv_with_wrong_y_coord(self):
             source._get_records()
             self.assertIn(msg, source.report.get("message", []))
 
-    def test_invalid_csv_file_raise_value_error(self):
+    def test_invalid_csv_file_raise_value_error(
+        self, mocked_es_delete, mocked_es_create
+    ):
         source = CSVSource.objects.create(
             file=SimpleUploadedFile("not_a_csv", b"some content"),
             geom_type=0,
@@ -95,7 +99,9 @@ def test_invalid_csv_file_raise_value_error(self):
             source._get_records()
             self.assertIn(msg, source.report.get("message", []))
 
-    def test_invalid_coordinate_format_raise_error(self):
+    def test_invalid_coordinate_format_raise_error(
+        self, mocked_es_delete, mocked_es_create
+    ):
         source = CSVSource.objects.create(
             file=get_file("source.csv"),
             geom_type=0,
@@ -118,7 +124,9 @@ def test_invalid_coordinate_format_raise_error(self):
             "coordxy is not a valid coordinate field", source.report["message"]
         )
 
-    def test_coordinates_system_without_digit_srid_raise_value_error(self):
+    def test_coordinates_system_without_digit_srid_raise_value_error(
+        self, mocked_es_delete, mocked_es_create
+    ):
         source = CSVSource.objects.create(
             file=get_file("source.csv"),
             geom_type=0,
@@ -138,7 +146,9 @@ def test_coordinates_system_without_digit_srid_raise_value_error(self):
         with self.assertRaises(ValueError):
             source._get_records()
 
-    def test_coordinates_systems_malformed_raise_index_error(self):
+    def test_coordinates_systems_malformed_raise_index_error(
+        self, mocked_es_delete, mocked_es_create
+    ):
         source = CSVSource.objects.create(
             file=get_file("source.csv"),
             geom_type=0,
@@ -158,8 +168,11 @@ def test_coordinates_systems_malformed_raise_index_error(self):
         with self.assertRaises(IndexError):
             source._get_records()
 
-    def test_invalid_id_field_raise_value_error_when_refreshing_data(self):
+    def test_invalid_id_field_raise_value_error_when_refreshing_data(
+        self, mocked_es_delete, mocked_es_create
+    ):
         source = CSVSource.objects.create(
+            name="csv-source",
             file=get_file("source.csv"),
             geom_type=0,
             id_field="identifier",
@@ -181,8 +194,12 @@ def test_invalid_id_field_raise_value_error_when_refreshing_data(self):
             self.assertIn(msg, source.report.get("message", []))
 
 
+@patch("elasticsearch.client.IndicesClient.create")
+@patch("elasticsearch.client.IndicesClient.delete")
 class GeoJSONSourceExceptionsTestCase(TestCase):
-    def test_source_geojson_with_wrong_id_raise_value_error(self):
+    def test_source_geojson_with_wrong_id_raise_value_error(
+        self, mocked_es_delete, mocked_es_create
+    ):
         geodict = {
             "type": "FeatureCollection",
             "features": [
@@ -209,6 +226,8 @@ def test_source_geojson_with_wrong_id_raise_value_error(self):
             self.assertIn(msg, source.report.get("message", []))
 
 
+@patch("elasticsearch.client.IndicesClient.create")
+@patch("elasticsearch.client.IndicesClient.delete")
 class PostGISSourceExceptionsTestCase(TestCase):
     @classmethod
     def setUpTestData(cls):
@@ -219,15 +238,21 @@ def setUpTestData(cls):
         )
 
     @patch("psycopg2.connect", side_effect=OperationalError("Connection error"))
-    def test_operationalerror_on_db_connect_is_reported(self, mocked_connect):
+    def test_operationalerror_on_db_connect_is_reported(
+        self, mocked_es_delete, mocked_es_create, mocked_connect
+    ):
         with self.assertRaises(OperationalError):
             self.source._db_connection()
             mocked_connect.assert_called_once()
             self.assertIn("Connection error", self.source.report.get("message", []))
 
 
+@patch("elasticsearch.client.IndicesClient.create")
+@patch("elasticsearch.client.IndicesClient.delete")
 class ShapefileSourceExceptionsTestCase(TestCase):
-    def test_wrong_id_raise_exception_on_refresh_and_get_reported(self):
+    def test_wrong_id_raise_exception_on_refresh_and_get_reported(
+        self, mocked_es_delete, mocked_es_create
+    ):
         source = ShapefileSource.objects.create(
             name="test_shapefile",
             geom_type=GeometryTypes.Point,

project/geosource/tests/test_models.py

@@ -103,7 +103,9 @@ def test_query_filter(self):
     def test_geojsonsource_type(self):
         self.assertEqual(self.geojson_source.type, self.geojson_source.__class__)
 
-    def test_wrong_identifier_refresh(self):
+    @mock.patch("elasticsearch.client.IndicesClient.create")
+    @mock.patch("elasticsearch.client.IndicesClient.delete")
+    def test_wrong_identifier_refresh(self, mocked_es_delete, mocked_es_create):
         self.geojson_source.id_field = "wrong_identifier"
         self.geojson_source.save()
         with self.assertRaisesRegexp(Exception, "Failed to refresh data"):
@@ -234,7 +236,8 @@ def setUp(self):
     @mock.patch("sys.stdout", new_callable=StringIO)
     def test_refresh_data(self, mocked_stdout, mock_index):
         mock_index.return_value = True
-        self.source.refresh_data()
+        with self.assertNumQueries(31):
+            self.source.refresh_data()
         self.assertIn("Start refresh", mocked_stdout.getvalue())
 
     def test_get_records(self):
@@ -289,8 +292,8 @@ def test_get_records_with_two_columns_coordinates(self, mock_index):
 
         records = source._get_records()
         self.assertEqual(len(records), 6, len(records))
-
-        row_count = source.refresh_data()
+        with self.assertNumQueries(54):
+            row_count = source.refresh_data()
         self.assertEqual(row_count["count"], len(records), row_count)
 
     @mock.patch("project.geosource.elasticsearch.index.LayerESIndex.index")
@@ -311,7 +314,8 @@ def test_get_records_with_one_column_coordinates(self, mock_index):
 
         records = source._get_records()
         self.assertEqual(len(records), 9, len(records))
-        row_count = source.refresh_data()
+        with self.assertNumQueries(72):
+            row_count = source.refresh_data()
         self.assertEqual(row_count["count"], len(records), row_count)
 
     @mock.patch("project.geosource.elasticsearch.index.LayerESIndex.index")
@@ -337,7 +341,8 @@ def test_get_records_with_decimal_separator_as_comma(self, mock_index):
         )
         records = source._get_records()
         self.assertEqual(len(records), 9, len(records))
-        row_count = source.refresh_data()
+        with self.assertNumQueries(72):
+            row_count = source.refresh_data()
         self.assertEqual(row_count["count"], len(records), row_count)
 
     @mock.patch("project.geosource.elasticsearch.index.LayerESIndex.index")
@@ -363,7 +368,8 @@ def test_get_records_with_nulled_columns_ignored(self, mock_index):
             if record.get("photoEtablissement")
         ]
         self.assertEqual(len(empty_entry), 0, empty_entry)
-        row_count = source.refresh_data()
+        with self.assertNumQueries(54):
+            row_count = source.refresh_data()
         self.assertEqual(row_count["count"], len(records), row_count)
 
     @mock.patch("project.geosource.elasticsearch.index.LayerESIndex.index")
@@ -384,8 +390,8 @@ def test_get_records_with_no_header_and_yx_csv(self, mock_index):
         )
         records = source._get_records()
         self.assertEqual(len(records), 9, len(records))
-
-        row_count = source.refresh_data()
+        with self.assertNumQueries(72):
+            row_count = source.refresh_data()
         self.assertEqual(row_count["count"], len(records), row_count)
 
     @mock.patch("project.geosource.elasticsearch.index.LayerESIndex.index")
@@ -406,7 +412,8 @@ def test_get_records_with_no_header_and_two_columns_csv(self, mock_index):
         )
         records = source._get_records()
         self.assertEqual(len(records), 10, len(records))
-        row_count = source.refresh_data()
+        with self.assertNumQueries(78):
+            row_count = source.refresh_data()
         self.assertEqual(row_count["count"], len(records), row_count)
 
     def test_update_fields_keep_order(self):
@@ -424,7 +431,7 @@ def test_update_fields_keep_order(self):
         sheet = source.get_file_as_sheet()
         sheet.name_columns_by_row(0)
         colnames = [name for name in sheet.colnames if name not in ("XCOORD", "YCOORD")]
-
-        source.update_fields()
+        with self.assertNumQueries(698):
+            source.update_fields()
         fields = [f.name for f in Field.objects.filter(source=source)]
         self.assertTrue(fields == colnames)

project/geosource/tests/test_periodics.py

@@ -58,8 +58,10 @@ def test_should_refresh(self, mock_timezone):
 
         self.assertEqual(self.source2.should_refresh(), True)
 
+    @mock.patch("elasticsearch.client.IndicesClient.create")
+    @mock.patch("elasticsearch.client.IndicesClient.delete")
     @mock.patch("django.utils.timezone.now")
-    def test_auto_refresh(self, mock_timezone):
+    def test_auto_refresh(self, mock_timezone, mocked_es_delete, mocked_es_create):
         with mock.patch(
             "project.geosource.models.Source._refresh_data"
         ) as mocked, mock.patch(

project/geosource/tests/test_tasks.py

@@ -10,6 +10,8 @@
 from project.geosource.tests.helpers import get_file
 
 
+@mock.patch("elasticsearch.client.IndicesClient.create")
+@mock.patch("elasticsearch.client.IndicesClient.delete")
 class TaskTestCase(TestCase):
     @classmethod
     def setUpTestData(cls):
@@ -21,7 +23,7 @@ def setUpTestData(cls):
             settings={"groups": [cls.group.pk]},
         )
 
-    def test_task_refresh_data_method(self):
+    def test_task_refresh_data_method(self, mocked_es_delete, mocked_es_create):
         run_model_object_method.apply(
             (
                 self.element._meta.app_label,
@@ -35,7 +37,9 @@ def test_task_refresh_data_method(self):
         self.assertEqual(Feature.objects.first().properties, {"id": 1, "test": 5})
         self.assertEqual(Layer.objects.first().authorized_groups.first().name, "Group")
 
-    def test_task_refresh_data_method_wrong_pk(self):
+    def test_task_refresh_data_method_wrong_pk(
+        self, mocked_es_delete, mocked_es_create
+    ):
         logging.disable(logging.WARNING)
         run_model_object_method.apply(
             (
@@ -47,7 +51,7 @@ def test_task_refresh_data_method_wrong_pk(self):
         )
         self.assertEqual(Layer.objects.count(), 0)
 
-    def test_task_wrong_method(self):
+    def test_task_wrong_method(self, mocked_es_delete, mocked_es_create):
         logging.disable(logging.ERROR)
         run_model_object_method.apply(
             (
@@ -60,7 +64,9 @@ def test_task_wrong_method(self):
         self.assertEqual(Layer.objects.count(), 0)
 
     @mock.patch("project.geosource.models.Source.objects")
-    def test_task_good_method_error(self, mock_source):
+    def test_task_good_method_error(
+        self, mocked_es_delete, mocked_es_create, mock_source
+    ):
         mock_source.get.side_effect = ValueError
         logging.disable(logging.ERROR)
         run_model_object_method.apply(