From 4484fa9171a479c3c653ab420d14bcfca55d0da0 Mon Sep 17 00:00:00 2001 From: Scott Davis Date: Tue, 26 Nov 2024 08:55:36 -0700 Subject: [PATCH] fix(forklift): handle new daq txt file data sources --- forklift/update_app_database.py | 42 +++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/forklift/update_app_database.py b/forklift/update_app_database.py index 0c7a7bf3..8521b84b 100644 --- a/forklift/update_app_database.py +++ b/forklift/update_app_database.py @@ -50,6 +50,33 @@ def get_crate_infos(destination_gdb, test_layer=None): def get_related_table_crate_infos(destination_gdb, test_layer=None): return _get_crate_infos(destination_gdb, test_layer, related_tables=True) +def get_source_name_and_workspace(source_data): + # feature services + if source_data.startswith('https'): + if '/services/' in source_data: + [source_workspace, source_name] = source_data.split(r'/services/') + source_workspace = fr'{source_workspace}/services/' + else: + #: this is for the web-hosted txt files for air quality + name = source_data.split('/')[-1].split('.')[0] + local_copy = path.join(arcpy.env.scratchGDB, name) + if arcpy.Exists(local_copy): + arcpy.management.Delete(local_copy) + print(f'copying {source_data} to {local_copy}') + arcpy.management.CopyRows(source_data, local_copy) + source_workspace = arcpy.env.scratchGDB + source_name = name + + #: prepend path to database folder for database sources + else: + if not source_data.startswith(r'\\'): + source_data = path.join(settings.dbConnects, source_data) + + source_workspace = path.dirname(source_data) + source_name = path.basename(source_data) + + return [source_workspace, source_name] + def _get_crate_infos(destination_gdb, test_layer=None, temp=False, related_tables=False): if related_tables: @@ -79,17 +106,7 @@ def _get_crate_infos(destination_gdb, test_layer=None, temp=False, related_table if source_data.startswith(''): continue - # feature services - if source_data.startswith('https'): - [source_workspace, source_name] = source_data.split(r'/services/') - source_workspace = fr'{source_workspace}/services/' - #: prepend path to database folder for database sources - else: - if not source_data.startswith(r'\\'): - source_data = path.join(settings.dbConnects, source_data) - - source_workspace = path.dirname(source_data) - source_name = path.basename(source_data) + [source_workspace, source_name] = get_source_name_and_workspace(source_data) if not related_tables: is_table = arcpy.da.Describe(source_data)['datasetType'] == 'Table' @@ -135,7 +152,8 @@ def get_was_updated(crate): def get_spreadsheet_config_from_crate(crate): for config in spreadsheet.get_query_layers(): source_data = config[settings.fieldnames.sourceData] - if len(source_data) > 0 and get_source_from_crate(crate).endswith(source_data.split('.')[-1]): + source_name = get_source_name_and_workspace(source_data)[1] + if len(source_data) > 0 and get_source_from_crate(crate).endswith(source_name): return config raise Exception('{} not found in spreadsheet!'.format(get_source_from_crate(crate)))