From ea24bf5f705b89986b434d4919eee306d7631371 Mon Sep 17 00:00:00 2001 From: Christopher Byrd Date: Fri, 12 May 2023 12:14:09 -0700 Subject: [PATCH 01/44] updates jquery to track minor/patch versions #9527 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 68d8c5d39de..1fa0ca4d9d2 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "font-awesome": "~4.6.3", "ionicons": "~2.0.1", "jqtree": "1.3.4", - "jquery": "3.6.1", + "jquery": "^3.6.1", "jquery-migrate": "3.4.0", "jquery-validation": "1.19.5", "jqueryui": "1.11.1", From f1c6d60d0da48090179c63340b9ac9e14132cc91 Mon Sep 17 00:00:00 2001 From: Cyrus Hiatt Date: Mon, 19 Jun 2023 14:01:26 -0700 Subject: [PATCH 02/44] Simplify value check, re #9648 --- arches/app/models/tile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arches/app/models/tile.py b/arches/app/models/tile.py index a9b25872faf..314b15a1b31 100644 --- a/arches/app/models/tile.py +++ b/arches/app/models/tile.py @@ -566,7 +566,7 @@ def after_update_all(self): def is_blank(self): if self.data != {}: - if len([item for item in list(self.data.values()) if item is not None]) > 0: + if any(self.data.values()): return False child_tiles_are_blank = True From bc5551c485dedd75b8e64fc8e8e413286bd2cb97 Mon Sep 17 00:00:00 2001 From: Cyrus Hiatt Date: Mon, 19 Jun 2023 14:02:01 -0700 Subject: [PATCH 03/44] Add check for prov tiles to model, re #9648 --- arches/app/models/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arches/app/models/models.py b/arches/app/models/models.py index ca0ec38132f..0f31c9334da 100644 --- a/arches/app/models/models.py +++ b/arches/app/models/models.py @@ -1100,8 +1100,12 @@ def __init__(self, *args, **kwargs): if not self.tileid: self.tileid = uuid.uuid4() + def is_fully_provisional(self): + has_data = any(self.data.values()) + self.provisionaledits is not None and has_data is False + def save(self, *args, **kwargs): - if self.sortorder is None or (self.provisionaledits is not None and self.data == {}): + if self.sortorder is None or self.is_fully_provisional(): sortorder_max = TileModel.objects.filter( nodegroup_id=self.nodegroup_id, resourceinstance_id=self.resourceinstance_id ).aggregate(Max("sortorder"))["sortorder__max"] From b27a04d2a8eebe9f6a968303008c9c81afbd14bb Mon Sep 17 00:00:00 2001 From: Christopher Byrd Date: Thu, 29 Jun 2023 13:53:03 -0700 Subject: [PATCH 04/44] creates empty key/value pairs for all nodes in a tile #9648 --- arches/app/models/models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arches/app/models/models.py b/arches/app/models/models.py index 0f31c9334da..c216a5630b4 100644 --- a/arches/app/models/models.py +++ b/arches/app/models/models.py @@ -1101,10 +1101,13 @@ def __init__(self, *args, **kwargs): self.tileid = uuid.uuid4() def is_fully_provisional(self): - has_data = any(self.data.values()) - self.provisionaledits is not None and has_data is False + return bool(self.provisionaledits and not any(self.data.values())) def save(self, *args, **kwargs): + for node in Node.objects.filter(nodegroup_id=self.nodegroup_id): + if not str(node.pk) in self.data: + self.data[str(node.pk)] = None + if self.sortorder is None or self.is_fully_provisional(): sortorder_max = TileModel.objects.filter( nodegroup_id=self.nodegroup_id, resourceinstance_id=self.resourceinstance_id From 46f67548f2beeb17900b7acc68c902403d7661d2 Mon Sep 17 00:00:00 2001 From: Christopher Byrd Date: Thu, 29 Jun 2023 14:06:29 -0700 Subject: [PATCH 05/44] adds migration for tiledata #9648 --- ...9648_add_empty_key_value_pairs_to_tiles.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 arches/app/models/migrations/9648_add_empty_key_value_pairs_to_tiles.py diff --git a/arches/app/models/migrations/9648_add_empty_key_value_pairs_to_tiles.py b/arches/app/models/migrations/9648_add_empty_key_value_pairs_to_tiles.py new file mode 100644 index 00000000000..3dbaf350ab3 --- /dev/null +++ b/arches/app/models/migrations/9648_add_empty_key_value_pairs_to_tiles.py @@ -0,0 +1,20 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("models", "9477_fix_for_spatial_view_dbf_function_edtf_displaying_null"), + ] + + def forwards_func(apps, schema_editor): + TileModel = apps.get_model("models", "TileModel") + + for tile in TileModel.objects.all(): + tile.save() + + def reverse_func(apps, schema_editor): + pass + + operations = [ + migrations.RunPython(forwards_func, reverse_func), + ] From 62e0caffd3625e792319592ab26bc9cf2a6bd8bf Mon Sep 17 00:00:00 2001 From: Christopher Byrd Date: Thu, 29 Jun 2023 14:07:47 -0700 Subject: [PATCH 06/44] nit #9648 --- .../migrations/9648_add_empty_key_value_pairs_to_tiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arches/app/models/migrations/9648_add_empty_key_value_pairs_to_tiles.py b/arches/app/models/migrations/9648_add_empty_key_value_pairs_to_tiles.py index 3dbaf350ab3..3a7a633b9aa 100644 --- a/arches/app/models/migrations/9648_add_empty_key_value_pairs_to_tiles.py +++ b/arches/app/models/migrations/9648_add_empty_key_value_pairs_to_tiles.py @@ -10,7 +10,7 @@ def forwards_func(apps, schema_editor): TileModel = apps.get_model("models", "TileModel") for tile in TileModel.objects.all(): - tile.save() + tile.save() # should trigger new code in `TileModel.save` method that adds empty key/value pairs to tile data def reverse_func(apps, schema_editor): pass From ad12852d083de652437858dcbc3b2d4d2a50cb1c Mon Sep 17 00:00:00 2001 From: Christopher Byrd Date: Fri, 30 Jun 2023 11:20:29 -0700 Subject: [PATCH 07/44] updates failing tests #9648" gst --- arches/app/models/tile.py | 3 +- tests/importer/jsonld_import_tests.py | 50 +++++++++++++++++++-------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/arches/app/models/tile.py b/arches/app/models/tile.py index 314b15a1b31..0ee1ec91e6f 100644 --- a/arches/app/models/tile.py +++ b/arches/app/models/tile.py @@ -210,9 +210,8 @@ def is_provisional(self): been approved by a user in the resource reviewer group """ - result = False - if self.provisionaledits is not None and len(self.data) == 0: + if self.provisionaledits is not None and not any(self.data.values()): result = True return result diff --git a/tests/importer/jsonld_import_tests.py b/tests/importer/jsonld_import_tests.py index 94aaaa7b840..cbb2c2bad92 100644 --- a/tests/importer/jsonld_import_tests.py +++ b/tests/importer/jsonld_import_tests.py @@ -1275,14 +1275,26 @@ def get_tile_by_id(tiles, tileid): tile1 = get_tiles_by_nodegroup(tiles, "e717dda0-1540-11ea-b353-acde48001122", "f025e108-1540-11ea-b353-acde48001122", 6) tile1_parent = get_tile_by_id(tiles, tile1.parenttile_id) - self.assertEqual(tile1_parent.data, {}) + self.assertEqual( + tile1_parent.data, + { + 'c5a9174c-1540-11ea-b353-acde48001122': None, + 'd155a4c0-1540-11ea-b353-acde48001122': None, + 'ddc44d9c-1540-11ea-b353-acde48001122': None + } + ) tile1_grandparent = get_tile_by_id(tiles, tile1_parent.parenttile_id) self.assertEqual(tile1_grandparent.data["02ec0ace-1541-11ea-b353-acde48001122"]["en"], {"direction": "ltr", "value": "second part"}) tile2 = get_tiles_by_nodegroup(tiles, "e717dda0-1540-11ea-b353-acde48001122", "f025e108-1540-11ea-b353-acde48001122", 1) tile2_parent = get_tile_by_id(tiles, tile2.parenttile_id) self.assertEqual( - tile2_parent.data, {"d155a4c0-1540-11ea-b353-acde48001122": "2019-12-03", "ddc44d9c-1540-11ea-b353-acde48001122": "2019-12-05"} + tile2_parent.data, + { + 'c5a9174c-1540-11ea-b353-acde48001122': None, + "d155a4c0-1540-11ea-b353-acde48001122": "2019-12-03", + "ddc44d9c-1540-11ea-b353-acde48001122": "2019-12-05" + } ) tile2_grandparent = get_tile_by_id(tiles, tile2_parent.parenttile_id) self.assertEqual(tile2_grandparent.data["02ec0ace-1541-11ea-b353-acde48001122"]["en"], {"direction": "ltr", "value": "asdf"}) @@ -1290,24 +1302,30 @@ def get_tile_by_id(tiles, tileid): tile2_grandparent.data["26927e22-1541-11ea-b353-acde48001122"][0]["resourceId"], "5e9baff0-109b-11ea-957a-acde48001122" ) tile2_greatgrandparent = get_tile_by_id(tiles, tile2_grandparent.parenttile_id) - self.assertEqual(tile2_greatgrandparent.data, {}) + self.assertEqual(tile2_greatgrandparent.data, {'af6c5d36-1540-11ea-b353-acde48001122': None}) tile1_greatgrandparent = get_tile_by_id(tiles, tile1_grandparent.parenttile_id) self.assertEqual(tile2_greatgrandparent, tile1_greatgrandparent) tile3 = get_tiles_by_nodegroup(tiles, "c5a9174c-1540-11ea-b353-acde48001122", "d155a4c0-1540-11ea-b353-acde48001122", "2019-12-07") self.assertEqual( - tile3.data, {"d155a4c0-1540-11ea-b353-acde48001122": "2019-12-07", "ddc44d9c-1540-11ea-b353-acde48001122": "2019-12-08"} + tile3.data, + { + 'c5a9174c-1540-11ea-b353-acde48001122': None, + "d155a4c0-1540-11ea-b353-acde48001122": "2019-12-07", + "ddc44d9c-1540-11ea-b353-acde48001122": "2019-12-08" + } ) tile3_parent = get_tile_by_id(tiles, tile3.parenttile_id) self.assertEqual(tile3_parent.data["02ec0ace-1541-11ea-b353-acde48001122"]["en"], {"direction": "ltr", "value": "bar"}) tile3_grandparent = get_tile_by_id(tiles, tile3_parent.parenttile_id) - self.assertEqual(tile3_grandparent.data, {}) + self.assertEqual(tile3_grandparent.data, {'af6c5d36-1540-11ea-b353-acde48001122': None}) tile4 = get_tiles_by_nodegroup(tiles, "48f3ab3a-1541-11ea-b353-acde48001122")[0] self.assertEqual( tile4.data, { + '48f3ab3a-1541-11ea-b353-acde48001122': None, "54272cd4-1541-11ea-b353-acde48001122": ["d2908ab9-19a6-4a82-953d-5ebe8d164e85"], "8e08a496-1541-11ea-b353-acde48001122": None, }, @@ -1472,17 +1490,21 @@ def test_8181_import_bug_2(self): actual_tiledata.append(tile.data) expected_tiledata = [ - {"f13ffd9c-3e76-11ec-9a49-faffc210b420": {"en": {"value": "ALLAN, DAVID", "direction": "ltr"}}}, { - "f1420740-3e76-11ec-9a49-faffc210b420": None, - "f1420f92-3e76-11ec-9a49-faffc210b420": None, - "f1417e1a-3e76-11ec-9a49-faffc210b420": ["b83cab06-1cfe-4aeb-9653-cb9f0cc45595"], - }, + 'f1403424-3e76-11ec-9a49-faffc210b420': None, + 'f1417e1a-3e76-11ec-9a49-faffc210b420': ['b83cab06-1cfe-4aeb-9653-cb9f0cc45595'], + 'f1420740-3e76-11ec-9a49-faffc210b420': None, + 'f1420f92-3e76-11ec-9a49-faffc210b420': None + }, { - "f1420740-3e76-11ec-9a49-faffc210b420": None, - "f1420f92-3e76-11ec-9a49-faffc210b420": None, - "f1417e1a-3e76-11ec-9a49-faffc210b420": ["fc3559c4-03c4-428c-a48e-0f480c8a3751"], - }, + 'f1403424-3e76-11ec-9a49-faffc210b420': None, + 'f1417e1a-3e76-11ec-9a49-faffc210b420': ['fc3559c4-03c4-428c-a48e-0f480c8a3751'], + 'f1420740-3e76-11ec-9a49-faffc210b420': None, + 'f1420f92-3e76-11ec-9a49-faffc210b420': None + }, + { + 'f13ffd9c-3e76-11ec-9a49-faffc210b420': {'en': {'value': 'ALLAN, DAVID', 'direction': 'ltr'}} + } ] self.assertCountEqual(actual_tiledata, expected_tiledata) From faa92943a5702aac306fa6c94547ee24ce85f328 Mon Sep 17 00:00:00 2001 From: Christopher Byrd Date: Fri, 30 Jun 2023 12:38:04 -0700 Subject: [PATCH 08/44] fixes failing tests #9648 --- .../resource_export_business_data_truth.json | 402 +++++++++++------- tests/importer/jsonld_import_tests.py | 20 +- 2 files changed, 252 insertions(+), 170 deletions(-) diff --git a/tests/fixtures/data/json/resource_export_business_data_truth.json b/tests/fixtures/data/json/resource_export_business_data_truth.json index 9b6af72f347..cb0214e861a 100644 --- a/tests/fixtures/data/json/resource_export_business_data_truth.json +++ b/tests/fixtures/data/json/resource_export_business_data_truth.json @@ -2,287 +2,367 @@ "business_data": { "resources": [ { + "resourceinstance": { + "descriptors": { + "ar": { + "description": "Undefined", + "map_popup": "Undefined", + "name": "Undefined" + }, + "de": { + "description": "Undefined", + "map_popup": "Undefined", + "name": "Undefined" + }, + "en": { + "description": "Undefined", + "map_popup": "Undefined", + "name": "Prehistoric building, 181-183 High Street" + }, + "en-gb": { + "description": "Undefined", + "map_popup": "Undefined", + "name": "Undefined" + }, + "es": { + "description": "Undefined", + "map_popup": "Undefined", + "name": "Undefined" + } + }, + "graph_id": "ab74af76-fa0e-11e6-9e3e-026d961c88e6", + "legacyid": "b546621a-9556-4955-8997-8be88dcabf2b", + "name": "Prehistoric building, 181-183 High Street", + "publication_id": "f546e4de-aa0d-11ec-99a5-8e0f431ffc6d", + "resourceinstanceid": "b546621a-9556-4955-8997-8be88dcabf2b" + }, "tiles": [ { - "data": {}, - "provisionaledits": null, - "parenttile_id": null, + "data": { + "677f236e-09cc-11e7-8ff7-6c4008b05c4c": null + }, "nodegroup_id": "677f236e-09cc-11e7-8ff7-6c4008b05c4c", - "sortorder": 0, + "parenttile_id": null, + "provisionaledits": null, "resourceinstance_id": "b546621a-9556-4955-8997-8be88dcabf2b", + "sortorder": 0, "tileid": "074c5da1-9aad-495a-a5f7-043679f44de6" }, { "data": { - "677f39a8-09cc-11e7-834a-6c4008b05c4c": "7949d7b5-6e57-469a-8f38-87aac08e1788", - "677f303d-09cc-11e7-9aa6-6c4008b05c4c": {"en": {"value": "Prehistoric building, 181-183 High Street", "direction": "ltr"}} + "ab74afc8-fa0e-11e6-9e3e-026d961c88e6": { + "en": { + "direction": "ltr", + "value": "2" + } + }, + "ab74affa-fa0e-11e6-9e3e-026d961c88e6": "7949d7b5-6e57-469a-8f38-87aac08e1788", + "ab74b016-fa0e-11e6-9e3e-026d961c88e6": null }, + "nodegroup_id": "ab74afc8-fa0e-11e6-9e3e-026d961c88e6", + "parenttile_id": null, "provisionaledits": null, - "parenttile_id": "074c5da1-9aad-495a-a5f7-043679f44de6", - "nodegroup_id": "677f303d-09cc-11e7-9aa6-6c4008b05c4c", - "sortorder": 0, "resourceinstance_id": "b546621a-9556-4955-8997-8be88dcabf2b", - "tileid": "ce97c18b-c8da-4e77-b542-d18dea1a7ab0" + "sortorder": 0, + "tileid": "18249662-5ad3-4c9c-bcc3-39033bfb2a3e" }, { "data": { - "677f3c4f-09cc-11e7-92a9-6c4008b05c4c": null, - "677f3a97-09cc-11e7-813b-6c4008b05c4c": "1965-02-02", - "677f384a-09cc-11e7-868f-6c4008b05c4c": "1964-02-02", - "677f1d5e-09cc-11e7-8716-6c4008b05c4c": null, - "677f3457-09cc-11e7-aef4-6c4008b05c4c": null, - "677f36cf-09cc-11e7-98fd-6c4008b05c4c": "fad6f17d-f7c8-4fa1-b358-e8626571599e", - "677f3b75-09cc-11e7-ac1e-6c4008b05c4c": null, - "677f3d2b-09cc-11e7-ba91-6c4008b05c4c": null, - "677f35b5-09cc-11e7-8978-6c4008b05c4c": "fad6f17d-f7c8-4fa1-b358-e8626571599e" + "ab74af95-fa0e-11e6-9e3e-026d961c88e6": null, + "ab74afe2-fa0e-11e6-9e3e-026d961c88e6": null, + "ab74affd-fa0e-11e6-9e3e-026d961c88e6": "9fa56006-6828-480f-8395-ad5c5a84726b" }, + "nodegroup_id": "ab74af95-fa0e-11e6-9e3e-026d961c88e6", + "parenttile_id": null, "provisionaledits": null, - "parenttile_id": "074c5da1-9aad-495a-a5f7-043679f44de6", - "nodegroup_id": "677f1d5e-09cc-11e7-8716-6c4008b05c4c", - "sortorder": 0, "resourceinstance_id": "b546621a-9556-4955-8997-8be88dcabf2b", - "tileid": "8f0f6d90-a132-47cb-bd51-b2e644bd127c" - }, - { - "data": {}, - "provisionaledits": null, - "parenttile_id": null, - "nodegroup_id": "e456023d-fa36-11e6-9e3e-026d961c88e6", "sortorder": 0, - "resourceinstance_id": "b546621a-9556-4955-8997-8be88dcabf2b", - "tileid": "fcc6272f-06fd-49ad-869b-ae0ea284a0cb" + "tileid": "229e13a5-becc-4a01-9e41-4bc6777e3751" }, { "data": { - "e456024f-fa36-11e6-9e3e-026d961c88e6": null, - "e4560243-fa36-11e6-9e3e-026d961c88e6": null, - "e4560245-fa36-11e6-9e3e-026d961c88e6": null, - "e4560246-fa36-11e6-9e3e-026d961c88e6": "9fa56006-6828-480f-8395-ad5c5a84726b", - "e4560237-fa36-11e6-9e3e-026d961c88e6": null, - "e4560249-fa36-11e6-9e3e-026d961c88e6": null + "6ead3ef0-09cc-11e7-a37d-6c4008b05c4c": null }, + "nodegroup_id": "6ead3ef0-09cc-11e7-a37d-6c4008b05c4c", + "parenttile_id": null, "provisionaledits": null, - "parenttile_id": "fcc6272f-06fd-49ad-869b-ae0ea284a0cb", - "nodegroup_id": "e4560237-fa36-11e6-9e3e-026d961c88e6", - "sortorder": 0, "resourceinstance_id": "b546621a-9556-4955-8997-8be88dcabf2b", - "tileid": "e0085315-dd30-4f6c-ace8-d4f78f4ebb99" - }, - { - "data": {}, - "provisionaledits": null, - "parenttile_id": null, - "nodegroup_id": "6ead3ef0-09cc-11e7-a37d-6c4008b05c4c", "sortorder": 0, - "resourceinstance_id": "b546621a-9556-4955-8997-8be88dcabf2b", "tileid": "726ccc5b-38de-4a05-b565-2192cea8fd12" }, { "data": { "6ead34dc-09cc-11e7-be0d-6c4008b05c4c": { - "type": "FeatureCollection", "features": [ { "geometry": { - "type": "Point", "coordinates": [ -0.542000315593716, 53.2268402605504 - ] + ], + "type": "Point" }, - "type": "Feature", "id": "16432b04-dc6b-4c09-94a6-6017cefd21e1", - "properties": {} + "properties": {}, + "type": "Feature" } - ] + ], + "type": "FeatureCollection" } }, - "provisionaledits": null, - "parenttile_id": "726ccc5b-38de-4a05-b565-2192cea8fd12", "nodegroup_id": "6ead34dc-09cc-11e7-be0d-6c4008b05c4c", - "sortorder": 0, + "parenttile_id": "726ccc5b-38de-4a05-b565-2192cea8fd12", + "provisionaledits": null, "resourceinstance_id": "b546621a-9556-4955-8997-8be88dcabf2b", + "sortorder": 0, "tileid": "73914b7b-41fb-4543-a777-47b4cba852e3" }, { "data": { - "ab74af95-fa0e-11e6-9e3e-026d961c88e6": null, - "ab74afe2-fa0e-11e6-9e3e-026d961c88e6": null, - "ab74affd-fa0e-11e6-9e3e-026d961c88e6": "9fa56006-6828-480f-8395-ad5c5a84726b" + "677f1d5e-09cc-11e7-8716-6c4008b05c4c": null, + "677f3457-09cc-11e7-aef4-6c4008b05c4c": null, + "677f35b5-09cc-11e7-8978-6c4008b05c4c": "fad6f17d-f7c8-4fa1-b358-e8626571599e", + "677f36cf-09cc-11e7-98fd-6c4008b05c4c": "fad6f17d-f7c8-4fa1-b358-e8626571599e", + "677f384a-09cc-11e7-868f-6c4008b05c4c": "1964-02-02", + "677f3a97-09cc-11e7-813b-6c4008b05c4c": "1965-02-02", + "677f3b75-09cc-11e7-ac1e-6c4008b05c4c": null, + "677f3c4f-09cc-11e7-92a9-6c4008b05c4c": null, + "677f3d2b-09cc-11e7-ba91-6c4008b05c4c": null }, + "nodegroup_id": "677f1d5e-09cc-11e7-8716-6c4008b05c4c", + "parenttile_id": "074c5da1-9aad-495a-a5f7-043679f44de6", "provisionaledits": null, - "parenttile_id": null, - "nodegroup_id": "ab74af95-fa0e-11e6-9e3e-026d961c88e6", - "sortorder": 0, "resourceinstance_id": "b546621a-9556-4955-8997-8be88dcabf2b", - "tileid": "229e13a5-becc-4a01-9e41-4bc6777e3751" + "sortorder": 0, + "tileid": "8f0f6d90-a132-47cb-bd51-b2e644bd127c" }, { "data": { - "ab74affa-fa0e-11e6-9e3e-026d961c88e6": "7949d7b5-6e57-469a-8f38-87aac08e1788", - "ab74afc8-fa0e-11e6-9e3e-026d961c88e6": {"en": {"value": "2", "direction": "ltr"}}, - "ab74b016-fa0e-11e6-9e3e-026d961c88e6": null + "677f303d-09cc-11e7-9aa6-6c4008b05c4c": { + "en": { + "direction": "ltr", + "value": "Prehistoric building, 181-183 High Street" + } + }, + "677f39a8-09cc-11e7-834a-6c4008b05c4c": "7949d7b5-6e57-469a-8f38-87aac08e1788" }, + "nodegroup_id": "677f303d-09cc-11e7-9aa6-6c4008b05c4c", + "parenttile_id": "074c5da1-9aad-495a-a5f7-043679f44de6", "provisionaledits": null, - "parenttile_id": null, - "nodegroup_id": "ab74afc8-fa0e-11e6-9e3e-026d961c88e6", - "sortorder": 0, "resourceinstance_id": "b546621a-9556-4955-8997-8be88dcabf2b", - "tileid": "18249662-5ad3-4c9c-bcc3-39033bfb2a3e" - } - ], - "resourceinstance": { - "descriptors": { - "ar": {"description": "Undefined", "map_popup": "Undefined", "name": "Undefined"}, "de": {"description": "Undefined", "map_popup": "Undefined", "name": "Undefined"}, "en": {"description": "Undefined", "map_popup": "Undefined", "name": "Prehistoric building, 181-183 High Street"}, "en-gb": {"description": "Undefined", "map_popup": "Undefined", "name": "Undefined"}, "es": {"description": "Undefined", "map_popup": "Undefined", "name": "Undefined"} - }, - "publication_id": "f546e4de-aa0d-11ec-99a5-8e0f431ffc6d", - "graph_id": "ab74af76-fa0e-11e6-9e3e-026d961c88e6", - "resourceinstanceid": "b546621a-9556-4955-8997-8be88dcabf2b", - "legacyid": "b546621a-9556-4955-8997-8be88dcabf2b", - "name": "Prehistoric building, 181-183 High Street" - } - }, - { - "tiles": [ - { - "data": {}, - "provisionaledits": null, - "parenttile_id": null, - "nodegroup_id": "677f236e-09cc-11e7-8ff7-6c4008b05c4c", "sortorder": 0, - "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", - "tileid": "691178e1-1588-49f6-9dea-c4b69d04d3f7" + "tileid": "ce97c18b-c8da-4e77-b542-d18dea1a7ab0" }, { "data": { - "677f39a8-09cc-11e7-834a-6c4008b05c4c": "9fa56006-6828-480f-8395-ad5c5a84726b", - "677f303d-09cc-11e7-9aa6-6c4008b05c4c": {"en": {"value": "Prehistoric building, 181-183 High Street", "direction": "ltr"}} + "e4560237-fa36-11e6-9e3e-026d961c88e6": null, + "e4560243-fa36-11e6-9e3e-026d961c88e6": null, + "e4560245-fa36-11e6-9e3e-026d961c88e6": null, + "e4560246-fa36-11e6-9e3e-026d961c88e6": "9fa56006-6828-480f-8395-ad5c5a84726b", + "e4560249-fa36-11e6-9e3e-026d961c88e6": null, + "e456024f-fa36-11e6-9e3e-026d961c88e6": null }, + "nodegroup_id": "e4560237-fa36-11e6-9e3e-026d961c88e6", + "parenttile_id": "fcc6272f-06fd-49ad-869b-ae0ea284a0cb", "provisionaledits": null, - "parenttile_id": "691178e1-1588-49f6-9dea-c4b69d04d3f7", - "nodegroup_id": "677f303d-09cc-11e7-9aa6-6c4008b05c4c", + "resourceinstance_id": "b546621a-9556-4955-8997-8be88dcabf2b", "sortorder": 0, - "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", - "tileid": "e9343014-d4f7-4dc7-82ab-5b3301d1953e" + "tileid": "e0085315-dd30-4f6c-ace8-d4f78f4ebb99" }, { "data": { - "677f3c4f-09cc-11e7-92a9-6c4008b05c4c": null, - "677f3a97-09cc-11e7-813b-6c4008b05c4c": "2018-01-01", - "677f384a-09cc-11e7-868f-6c4008b05c4c": "2017-01-01", - "677f1d5e-09cc-11e7-8716-6c4008b05c4c": null, - "677f3457-09cc-11e7-aef4-6c4008b05c4c": null, - "677f36cf-09cc-11e7-98fd-6c4008b05c4c": "7949d7b5-6e57-469a-8f38-87aac08e1788", - "677f3b75-09cc-11e7-ac1e-6c4008b05c4c": null, - "677f3d2b-09cc-11e7-ba91-6c4008b05c4c": null, - "677f35b5-09cc-11e7-8978-6c4008b05c4c": "7949d7b5-6e57-469a-8f38-87aac08e1788" + "e456023d-fa36-11e6-9e3e-026d961c88e6": null }, + "nodegroup_id": "e456023d-fa36-11e6-9e3e-026d961c88e6", + "parenttile_id": null, "provisionaledits": null, - "parenttile_id": "691178e1-1588-49f6-9dea-c4b69d04d3f7", - "nodegroup_id": "677f1d5e-09cc-11e7-8716-6c4008b05c4c", + "resourceinstance_id": "b546621a-9556-4955-8997-8be88dcabf2b", "sortorder": 0, - "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", - "tileid": "97b14ee9-db22-47fe-abc1-a47c970de721" + "tileid": "fcc6272f-06fd-49ad-869b-ae0ea284a0cb" + } + ] + }, + { + "resourceinstance": { + "descriptors": { + "ar": { + "description": "Undefined", + "map_popup": "Undefined", + "name": "Undefined" + }, + "de": { + "description": "Undefined", + "map_popup": "Undefined", + "name": "Undefined" + }, + "en": { + "description": "Undefined", + "map_popup": "Undefined", + "name": "Prehistoric building, 181-183 High Street" + }, + "en-gb": { + "description": "Undefined", + "map_popup": "Undefined", + "name": "Undefined" + }, + "es": { + "description": "Undefined", + "map_popup": "Undefined", + "name": "Undefined" + } }, + "graph_id": "ab74af76-fa0e-11e6-9e3e-026d961c88e6", + "legacyid": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", + "name": "Prehistoric building, 181-183 High Street", + "publication_id": "f546e4de-aa0d-11ec-99a5-8e0f431ffc6d", + "resourceinstanceid": "75226cdb-6ecb-4dab-8f83-3537fa298ca3" + }, + "tiles": [ { - "data": {}, - "provisionaledits": null, + "data": { + "6ead3ef0-09cc-11e7-a37d-6c4008b05c4c": null + }, + "nodegroup_id": "6ead3ef0-09cc-11e7-a37d-6c4008b05c4c", "parenttile_id": null, - "nodegroup_id": "e456023d-fa36-11e6-9e3e-026d961c88e6", - "sortorder": 0, + "provisionaledits": null, "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", - "tileid": "a30e1f76-0468-4004-9a9e-7d77fb5b0c72" + "sortorder": 0, + "tileid": "007e78ea-b2e4-45e8-ac15-ac756fb106e4" }, { "data": { - "e456024f-fa36-11e6-9e3e-026d961c88e6": null, + "e4560237-fa36-11e6-9e3e-026d961c88e6": null, "e4560243-fa36-11e6-9e3e-026d961c88e6": null, "e4560245-fa36-11e6-9e3e-026d961c88e6": null, "e4560246-fa36-11e6-9e3e-026d961c88e6": "fad6f17d-f7c8-4fa1-b358-e8626571599e", - "e4560237-fa36-11e6-9e3e-026d961c88e6": null, - "e4560249-fa36-11e6-9e3e-026d961c88e6": null + "e4560249-fa36-11e6-9e3e-026d961c88e6": null, + "e456024f-fa36-11e6-9e3e-026d961c88e6": null }, - "provisionaledits": null, - "parenttile_id": "a30e1f76-0468-4004-9a9e-7d77fb5b0c72", "nodegroup_id": "e4560237-fa36-11e6-9e3e-026d961c88e6", - "sortorder": 0, - "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", - "tileid": "04fa42e8-d491-4182-8b60-ca5f6ef08e48" - }, - { - "data": {}, + "parenttile_id": "a30e1f76-0468-4004-9a9e-7d77fb5b0c72", "provisionaledits": null, - "parenttile_id": null, - "nodegroup_id": "6ead3ef0-09cc-11e7-a37d-6c4008b05c4c", - "sortorder": 0, "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", - "tileid": "007e78ea-b2e4-45e8-ac15-ac756fb106e4" + "sortorder": 0, + "tileid": "04fa42e8-d491-4182-8b60-ca5f6ef08e48" }, { "data": { "6ead34dc-09cc-11e7-be0d-6c4008b05c4c": { - "type": "FeatureCollection", "features": [ { "geometry": { - "type": "Point", "coordinates": [ -0.541732301953314, 53.2267920320966 - ] + ], + "type": "Point" }, - "type": "Feature", "id": "b9140d4c-220c-45c3-9800-23c37490c90c", - "properties": {} + "properties": {}, + "type": "Feature" } - ] + ], + "type": "FeatureCollection" } }, - "provisionaledits": null, - "parenttile_id": "007e78ea-b2e4-45e8-ac15-ac756fb106e4", "nodegroup_id": "6ead34dc-09cc-11e7-be0d-6c4008b05c4c", - "sortorder": 0, + "parenttile_id": "007e78ea-b2e4-45e8-ac15-ac756fb106e4", + "provisionaledits": null, "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", + "sortorder": 0, "tileid": "13147097-bf8a-430c-8b19-0b5c025f7bb0" }, + { + "data": { + "ab74afc8-fa0e-11e6-9e3e-026d961c88e6": { + "en": { + "direction": "ltr", + "value": "1" + } + }, + "ab74affa-fa0e-11e6-9e3e-026d961c88e6": "9fa56006-6828-480f-8395-ad5c5a84726b", + "ab74b016-fa0e-11e6-9e3e-026d961c88e6": null + }, + "nodegroup_id": "ab74afc8-fa0e-11e6-9e3e-026d961c88e6", + "parenttile_id": null, + "provisionaledits": null, + "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", + "sortorder": 0, + "tileid": "6553453c-d044-4d81-96fa-0a5f37e28812" + }, + { + "data": { + "677f236e-09cc-11e7-8ff7-6c4008b05c4c": null + }, + "nodegroup_id": "677f236e-09cc-11e7-8ff7-6c4008b05c4c", + "parenttile_id": null, + "provisionaledits": null, + "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", + "sortorder": 0, + "tileid": "691178e1-1588-49f6-9dea-c4b69d04d3f7" + }, { "data": { "ab74af95-fa0e-11e6-9e3e-026d961c88e6": null, "ab74afe2-fa0e-11e6-9e3e-026d961c88e6": null, "ab74affd-fa0e-11e6-9e3e-026d961c88e6": "fad6f17d-f7c8-4fa1-b358-e8626571599e" }, - "provisionaledits": null, - "parenttile_id": null, "nodegroup_id": "ab74af95-fa0e-11e6-9e3e-026d961c88e6", - "sortorder": 0, + "parenttile_id": null, + "provisionaledits": null, "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", + "sortorder": 0, "tileid": "7cc77141-2b3c-4118-93a6-51481118f292" }, { "data": { - "ab74affa-fa0e-11e6-9e3e-026d961c88e6": "9fa56006-6828-480f-8395-ad5c5a84726b", - "ab74afc8-fa0e-11e6-9e3e-026d961c88e6": {"en": {"value": "1", "direction": "ltr"}}, - "ab74b016-fa0e-11e6-9e3e-026d961c88e6": null + "677f1d5e-09cc-11e7-8716-6c4008b05c4c": null, + "677f3457-09cc-11e7-aef4-6c4008b05c4c": null, + "677f35b5-09cc-11e7-8978-6c4008b05c4c": "7949d7b5-6e57-469a-8f38-87aac08e1788", + "677f36cf-09cc-11e7-98fd-6c4008b05c4c": "7949d7b5-6e57-469a-8f38-87aac08e1788", + "677f384a-09cc-11e7-868f-6c4008b05c4c": "2017-01-01", + "677f3a97-09cc-11e7-813b-6c4008b05c4c": "2018-01-01", + "677f3b75-09cc-11e7-ac1e-6c4008b05c4c": null, + "677f3c4f-09cc-11e7-92a9-6c4008b05c4c": null, + "677f3d2b-09cc-11e7-ba91-6c4008b05c4c": null }, + "nodegroup_id": "677f1d5e-09cc-11e7-8716-6c4008b05c4c", + "parenttile_id": "691178e1-1588-49f6-9dea-c4b69d04d3f7", "provisionaledits": null, + "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", + "sortorder": 0, + "tileid": "97b14ee9-db22-47fe-abc1-a47c970de721" + }, + { + "data": { + "e456023d-fa36-11e6-9e3e-026d961c88e6": null + }, + "nodegroup_id": "e456023d-fa36-11e6-9e3e-026d961c88e6", "parenttile_id": null, - "nodegroup_id": "ab74afc8-fa0e-11e6-9e3e-026d961c88e6", + "provisionaledits": null, + "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", "sortorder": 0, + "tileid": "a30e1f76-0468-4004-9a9e-7d77fb5b0c72" + }, + { + "data": { + "677f303d-09cc-11e7-9aa6-6c4008b05c4c": { + "en": { + "direction": "ltr", + "value": "Prehistoric building, 181-183 High Street" + } + }, + "677f39a8-09cc-11e7-834a-6c4008b05c4c": "9fa56006-6828-480f-8395-ad5c5a84726b" + }, + "nodegroup_id": "677f303d-09cc-11e7-9aa6-6c4008b05c4c", + "parenttile_id": "691178e1-1588-49f6-9dea-c4b69d04d3f7", + "provisionaledits": null, "resourceinstance_id": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", - "tileid": "6553453c-d044-4d81-96fa-0a5f37e28812" + "sortorder": 0, + "tileid": "e9343014-d4f7-4dc7-82ab-5b3301d1953e" } - ], - "resourceinstance": { - "descriptors": { - "ar": {"description": "Undefined", "map_popup": "Undefined", "name": "Undefined"}, "de": {"description": "Undefined", "map_popup": "Undefined", "name": "Undefined"}, "en": {"description": "Undefined", "map_popup": "Undefined", "name": "Prehistoric building, 181-183 High Street"}, "en-gb": {"description": "Undefined", "map_popup": "Undefined", "name": "Undefined"}, "es": {"description": "Undefined", "map_popup": "Undefined", "name": "Undefined"} - }, - "name": "Prehistoric building, 181-183 High Street", - "publication_id": "f546e4de-aa0d-11ec-99a5-8e0f431ffc6d", - "graph_id": "ab74af76-fa0e-11e6-9e3e-026d961c88e6", - "resourceinstanceid": "75226cdb-6ecb-4dab-8f83-3537fa298ca3", - "legacyid": "75226cdb-6ecb-4dab-8f83-3537fa298ca3" - } + ] } ] } -} +} \ No newline at end of file diff --git a/tests/importer/jsonld_import_tests.py b/tests/importer/jsonld_import_tests.py index cbb2c2bad92..1a9978282fe 100644 --- a/tests/importer/jsonld_import_tests.py +++ b/tests/importer/jsonld_import_tests.py @@ -1426,17 +1426,19 @@ def test_8181_import_bug(self): actual_tiledata.append(tile.data) expected_tiledata = [ - {"f13ffd9c-3e76-11ec-9a49-faffc210b420": {"en": {"value": "ALLAN, DAVID", "direction": "ltr"}}}, + {'f13ffd9c-3e76-11ec-9a49-faffc210b420': {'en': {'value': 'ALLAN, DAVID', 'direction': 'ltr'}}}, { - "f1420740-3e76-11ec-9a49-faffc210b420": None, - "f1420f92-3e76-11ec-9a49-faffc210b420": None, - "f1417e1a-3e76-11ec-9a49-faffc210b420": ["b83cab06-1cfe-4aeb-9653-cb9f0cc45595"], - }, + 'f1403424-3e76-11ec-9a49-faffc210b420': None, + 'f1417e1a-3e76-11ec-9a49-faffc210b420': ['b83cab06-1cfe-4aeb-9653-cb9f0cc45595'], + 'f1420740-3e76-11ec-9a49-faffc210b420': None, + 'f1420f92-3e76-11ec-9a49-faffc210b420': None + }, { - "f1420740-3e76-11ec-9a49-faffc210b420": None, - "f1420f92-3e76-11ec-9a49-faffc210b420": None, - "f1417e1a-3e76-11ec-9a49-faffc210b420": ["fc3559c4-03c4-428c-a48e-0f480c8a3751"], - }, + 'f1403424-3e76-11ec-9a49-faffc210b420': None, + 'f1417e1a-3e76-11ec-9a49-faffc210b420': ['fc3559c4-03c4-428c-a48e-0f480c8a3751'], + 'f1420740-3e76-11ec-9a49-faffc210b420': None, + 'f1420f92-3e76-11ec-9a49-faffc210b420': None + } ] self.assertCountEqual(actual_tiledata, expected_tiledata) From 89dc9b1c02ab057f54471b478a4527da8b875003 Mon Sep 17 00:00:00 2001 From: njkim Date: Mon, 3 Jul 2023 15:59:33 -0700 Subject: [PATCH 09/44] Update trigger function and add index, #9671 --- .../9670_improve_bulk_load_performance.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 arches/app/models/migrations/9670_improve_bulk_load_performance.py diff --git a/arches/app/models/migrations/9670_improve_bulk_load_performance.py b/arches/app/models/migrations/9670_improve_bulk_load_performance.py new file mode 100644 index 00000000000..6139258088b --- /dev/null +++ b/arches/app/models/migrations/9670_improve_bulk_load_performance.py @@ -0,0 +1,96 @@ +from django.db import migrations + +class Migration(migrations.Migration): + + dependencies = [ + ("models", "8010_export_permissions"), + ] + + update_check_excess_tiles_trigger = """ + create or replace procedure __arches_complete_bulk_load() AS + $$ + DECLARE + cardinality_violations bigint; + BEGIN + alter table tiles enable trigger __arches_check_excess_tiles_trigger; + alter table tiles enable trigger __arches_trg_update_spatial_attributes; + + if (not __arches_refresh_spatial_views()) then + Raise EXCEPTION 'Unable to refresh spatial views'; + end if; + + with cardinality_violations as (SELECT t.resourceinstanceid, + t.nodegroupid, + COALESCE(t.parenttileid::text, '') parent_tileid, + count(*) + FROM tiles t, + node_groups ng + WHERE t.nodegroupid = ng.nodegroupid + AND ng.cardinality = '1' + group by t.resourceinstanceid, t.nodegroupid, parent_tileid + having count(*) > 1 + LIMIT 1) + select count(*) + into cardinality_violations + from cardinality_violations; + + if (cardinality_violations > 0) then + Raise Exception 'Cardinality violations found. Run `%` to list violations', + 'select * from __arches_get_tile_cardinality_violations()'; + else + Raise Notice 'No cardinality violations found'; + end if; + END + $$ + language plpgsql; + """ + + restore_check_excess_tiles_trigger = """ + create or replace procedure __arches_complete_bulk_load() as + $$ + DECLARE + cardinality_violations bigint; + BEGIN + alter table tiles enable trigger __arches_check_excess_tiles_trigger; + alter table tiles enable trigger __arches_trg_update_spatial_attributes; + + if (not __arches_refresh_spatial_views()) then + Raise EXCEPTION 'Unable to refresh spatial views'; + end if; + + with cardinality_violations as (SELECT t.resourceinstanceid, + t.nodegroupid, + COALESCE(t.parenttileid::text, '') parent_tileid, + count(*) + FROM tiles t, + node_groups ng + WHERE t.nodegroupid = ng.nodegroupid + AND ng.cardinality = '1' + group by t.resourceinstanceid, t.nodegroupid, parent_tileid + having count(*) > 1) + select count(*) + into cardinality_violations + from cardinality_violations; + + if (cardinality_violations > 0) then + Raise Exception 'Cardinality violations found. Run `%` to list violations', + 'select * from __arches_get_tile_cardinality_violations()'; + else + Raise Notice 'No cardinality violations found'; + end if; + END $$ + language plpgsql; + """ + + create_tileid_index_on_load_staging = """ + CREATE INDEX IF NOT EXISTS load_staging_tileid ON load_staging; + """ + + drop_tileid_index_on_load_staging = """ + DROP INDEX IF EXISTS load_staging_tileid; + """ + + operations = [ + migrations.RunSQL(update_check_excess_tiles_trigger, restore_check_excess_tiles_trigger), + migrations.RunSQL(create_tileid_index_on_load_staging, drop_tileid_index_on_load_staging), + ] From ceaea6c23cc21ce20028297561db31f5675c26e0 Mon Sep 17 00:00:00 2001 From: njkim Date: Mon, 3 Jul 2023 16:02:40 -0700 Subject: [PATCH 10/44] Skip check cardinality for editor, #9669 --- arches/app/etl_modules/base_data_editor.py | 2 +- arches/app/etl_modules/base_import_module.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/arches/app/etl_modules/base_data_editor.py b/arches/app/etl_modules/base_data_editor.py index 7aa8b118575..8389bd68436 100644 --- a/arches/app/etl_modules/base_data_editor.py +++ b/arches/app/etl_modules/base_data_editor.py @@ -367,7 +367,7 @@ def run_load_task(self, loadid, graph_id, node_id, operation, language_code, old return {"success": False, "data": {"title": _("Error"), "message": data_staged["message"]}} if data_updated["success"]: - data_updated = self.save_to_tiles(loadid) + data_updated = self.save_to_tiles(loadid, finalize_import=False) return {"success": True, "data": "done"} else: with connection.cursor() as cursor: diff --git a/arches/app/etl_modules/base_import_module.py b/arches/app/etl_modules/base_import_module.py index a95bed8d76c..b80ebd75a24 100644 --- a/arches/app/etl_modules/base_import_module.py +++ b/arches/app/etl_modules/base_import_module.py @@ -46,13 +46,13 @@ def reverse(self, request, **kwargs): logger.warn(response) return response - def save_to_tiles(self, loadid, multiprocessing=True): + def save_to_tiles(self, loadid, finalize_import=True, multiprocessing=True): self.loadid = loadid with connection.cursor() as cursor: try: cursor.execute("""CALL __arches_prepare_bulk_load();""") cursor.execute("""SELECT * FROM __arches_staging_to_tile(%s)""", [self.loadid]) - row = cursor.fetchall() + saved = cursor.fetchone()[0] except (IntegrityError, ProgrammingError) as e: logger.error(e) cursor.execute( @@ -66,9 +66,16 @@ def save_to_tiles(self, loadid, multiprocessing=True): "message": _("Unable to insert record into staging table"), } finally: - cursor.execute("""CALL __arches_complete_bulk_load();""") + cursor.execute("""alter table tiles enable trigger __arches_check_excess_tiles_trigger;""") + cursor.execute("""alter table tiles enable trigger __arches_trg_update_spatial_attributes;""") - if row[0][0]: + if finalize_import: + cursor.execute("""SELECT __arches_refresh_spatial_views();""") + refresh_successful = cursor.fetchone()[0] + if not refresh_successful: + raise Exception('Unable to refresh spatial views') + + if saved: cursor.execute( """UPDATE load_event SET (status, load_end_time) = (%s, %s) WHERE loadid = %s""", ("completed", datetime.now(), loadid), From e387d6834a3ba5b989e32633f26c68753ecb1e84 Mon Sep 17 00:00:00 2001 From: njkim Date: Mon, 3 Jul 2023 16:03:52 -0700 Subject: [PATCH 11/44] Try catching error after load before index, #9671 --- arches/app/etl_modules/base_import_module.py | 41 +++++++++++++------ arches/app/templates/javascript.htm | 1 + .../views/components/plugins/etl-manager.htm | 6 +++ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/arches/app/etl_modules/base_import_module.py b/arches/app/etl_modules/base_import_module.py index b80ebd75a24..95e06441068 100644 --- a/arches/app/etl_modules/base_import_module.py +++ b/arches/app/etl_modules/base_import_module.py @@ -66,26 +66,41 @@ def save_to_tiles(self, loadid, finalize_import=True, multiprocessing=True): "message": _("Unable to insert record into staging table"), } finally: - cursor.execute("""alter table tiles enable trigger __arches_check_excess_tiles_trigger;""") - cursor.execute("""alter table tiles enable trigger __arches_trg_update_spatial_attributes;""") + try: + cursor.execute("""alter table tiles enable trigger __arches_check_excess_tiles_trigger;""") + cursor.execute("""alter table tiles enable trigger __arches_trg_update_spatial_attributes;""") - if finalize_import: - cursor.execute("""SELECT __arches_refresh_spatial_views();""") - refresh_successful = cursor.fetchone()[0] - if not refresh_successful: - raise Exception('Unable to refresh spatial views') + if finalize_import: + cursor.execute("""SELECT __arches_refresh_spatial_views();""") + refresh_successful = cursor.fetchone()[0] + if not refresh_successful: + raise Exception('Unable to refresh spatial views') + except Exception as e: + logger.exception(e) + cursor.execute( + """UPDATE load_event SET (status, indexed_time, complete, successful) = (%s, %s, %s, %s) WHERE loadid = %s""", + ("unindexed", datetime.now(), True, True, loadid), + ) if saved: cursor.execute( """UPDATE load_event SET (status, load_end_time) = (%s, %s) WHERE loadid = %s""", ("completed", datetime.now(), loadid), ) - index_resources_by_transaction(loadid, quiet=True, use_multiprocessing=False, recalculate_descriptors=True) - cursor.execute( - """UPDATE load_event SET (status, indexed_time, complete, successful) = (%s, %s, %s, %s) WHERE loadid = %s""", - ("indexed", datetime.now(), True, True, loadid), - ) - return {"success": True, "data": "success"} + try: + index_resources_by_transaction(loadid, quiet=True, use_multiprocessing=False, recalculate_descriptors=True) + cursor.execute( + """UPDATE load_event SET (status, indexed_time, complete, successful) = (%s, %s, %s, %s) WHERE loadid = %s""", + ("indexed", datetime.now(), True, True, loadid), + ) + return {"success": True, "data": "indexed"} + except Exception as e: + logger.exception(e) + cursor.execute( + """UPDATE load_event SET (status, load_end_time) = (%s, %s) WHERE loadid = %s""", + ("unindexed", datetime.now(), loadid), + ) + return {"success": False, "data": "saved"} else: cursor.execute( """UPDATE load_event SET status = %s, load_end_time = %s WHERE loadid = %s""", diff --git a/arches/app/templates/javascript.htm b/arches/app/templates/javascript.htm index 6e223c6adf9..079ef31755e 100644 --- a/arches/app/templates/javascript.htm +++ b/arches/app/templates/javascript.htm @@ -735,6 +735,7 @@ remove-from-history='{% trans "remove from history" as removeFromHistory %} "{{ removeFromHistory|escapejs }}"' number-of-resources-updated='{% trans "Number of Resources Updated" as numberOfResourcesUpdated %} "{{ numberOfResourcesUpdated|escapejs }}"' indexing='{% trans "indexing" as indexing %} "{{ indexing|escapejs }}"' + loaded-but-unindexed='{% trans "loaded but unindexed" as loadedButUnindexed %} "{{ loadedButUnindexed|escapejs }}"' validating='{% trans "validating" as validating %} "{{ validating|escapejs }}"' completed='{% trans "completed" as completed %} "{{ completed|escapejs }}"' failed='{% trans "failed" as failed %} "{{ failed|escapejs }}"' diff --git a/arches/app/templates/views/components/plugins/etl-manager.htm b/arches/app/templates/views/components/plugins/etl-manager.htm index 3bafd6f0a8a..17e25b09e06 100644 --- a/arches/app/templates/views/components/plugins/etl-manager.htm +++ b/arches/app/templates/views/components/plugins/etl-manager.htm @@ -143,6 +143,12 @@

+ + +