diff --git a/osm_fieldwork/form_components/__init__.py b/osm_fieldwork/form_components/__init__.py new file mode 100644 index 00000000..3478f5b4 --- /dev/null +++ b/osm_fieldwork/form_components/__init__.py @@ -0,0 +1,5 @@ +"""OSM Fieldwork Form Builder Package.""" + +import os + +package_root = os.path.dirname(os.path.abspath(__file__)) diff --git a/osm_fieldwork/form_components/choice_fields.py b/osm_fieldwork/form_components/choice_fields.py new file mode 100644 index 00000000..24b87431 --- /dev/null +++ b/osm_fieldwork/form_components/choice_fields.py @@ -0,0 +1,69 @@ +#!/usr/bin/python3 + +# Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team +# +# This file is part of OSM-Fieldwork. +# +# This is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OSM-Fieldwork. If not, see . +# + +"""This creates DataFrames for choice lists used in the survey form. + +These include predefined options for fields such as yes/no responses +and issues related to digitization problems. Each choice contains +multilingual labels to support various languages. + +Returns: + tuple: Two pandas DataFrames containing the `choices` data for yes/no +and digitisation problems respectively. +""" + +import pandas as pd + +# Define the choices sheet +choices_data = [ + {"list_name": "yes_no", "name": "yes", "label::english(en)": "Yes"}, + {"list_name": "yes_no", "name": "no", "label::english(en)": "No"}, +] + +choices_df = pd.DataFrame(choices_data) + +digitisation_choices = [ + { + "list_name": "digitisation_problem", + "name": "lumped", + "label::english(en)": "Lumped - one polygon (more than one building digitized as one)", + "label::swahili(sw)": "Lumped - poligoni moja (zaidi ya jengo moja limewekwa dijiti kuwa moja)", + "label::french(fr)": "Lumped - un polygone (plus d'un bâtiment numérisé en un seul)", + "label::spanish(es)": "Agrupado - un polígono (más de un edificio digitalizado como uno)", + }, + { + "list_name": "digitisation_problem", + "name": "split", + "label::english(en)": "Split - one building (one building digitized as more than one polygon)", + "label::swahili(sw)": "Mgawanyiko - jengo moja (jengo moja limebadilishwa kuwa zaidi ya poligoni moja)", + "label::french(fr)": "Fractionnement - un bâtiment (un bâtiment numérisé sous la forme de plusieurs polygones)", + "label::spanish(es)": "Split - un edificio (un edificio digitalizado como más de un polígono)", + }, + { + "list_name": "digitisation_problem", + "name": "other", + "label::english(en)": "OTHER", + "label::swahili(sw)": "MENGINEYO", + "label::french(fr)": "AUTRES", + "label::spanish(es)": "OTROS", + }, +] + +digitisation_choices_df = pd.DataFrame(digitisation_choices) diff --git a/osm_fieldwork/form_components/digitisation_fields.py b/osm_fieldwork/form_components/digitisation_fields.py new file mode 100644 index 00000000..b0334814 --- /dev/null +++ b/osm_fieldwork/form_components/digitisation_fields.py @@ -0,0 +1,78 @@ +#!/usr/bin/python3 + +# Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team +# +# This file is part of OSM-Fieldwork. +# +# This is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OSM-Fieldwork. If not, see . +# + +"""This script creates a DataFrame for digitization-related fields in the survey form. + +These fields focus on verifying the accuracy of digitized locations, +capturing any issues with digitization, and adding additional notes or +images if required. The fields include logic for relevance, mandatory +requirements, and conditions based on user responses. + +Returns: + pd.DataFrame: A DataFrame containing the digitization-related fields. +""" + +import pandas as pd + +digitisation_fields = [ + { + "type": "begin group", + "name": "verification", + "label::english(en)": "Verification", + "relevant": "(${new_feature} != '') or (${building_exists} = 'yes')", + }, + { + "type": "select_one yes_no", + "name": "digitisation_correct", + "label::english(en)": "Is the digitized location for this feature correct?", + "relevant": "(${new_feature} != '') or (${building_exists} = 'yes')", + "calculation": "once(if(${new_feature} != '', 'yes', ''))", + "read_only": "${new_feature} != ''", + "required": "yes", + }, + { + "type": "select_one digitisation_problem", + "name": "digitisation_problem", + "label::english(en)": "What is wrong with the digitization?", + "relevant": "${digitisation_correct}='no'", + }, + { + "type": "text", + "name": "digitisation_problem_other", + "label::english(en)": "You said “Other.” Please tell us what went wrong with the digitization!", + "relevant": "${digitisation_problem}='other' ", + }, + {"type": "end group"}, + { + "type": "image", + "name": "image", + "label::english(en)": "Take a Picture", + "apperance": "minimal", + "parameters": "max-pixels=1000", + }, + { + "type": "note", + "name": "end_note", + "label::english(en)": "You can't proceed with data acquisition, if the building doesn't exist.", + "relevant": "${building_exists} = 'no'", + }, +] + +digitisation_df = pd.DataFrame(digitisation_fields) diff --git a/osm_fieldwork/form_components/mandatory_fields.py b/osm_fieldwork/form_components/mandatory_fields.py new file mode 100644 index 00000000..486fafab --- /dev/null +++ b/osm_fieldwork/form_components/mandatory_fields.py @@ -0,0 +1,175 @@ +#!/usr/bin/python3 + +# Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team +# +# This file is part of OSM-Fieldwork. +# +# This is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OSM-Fieldwork. If not, see . +# + +"""This script generates an XLS form with mandatory fields. + +For use in data collection and mapping tasks. +The generated form includes metadata, survey questions, and settings +required for compatibility with HOT's FMTM tools. +It programmatically organizes form sections into metadata, +mandatory fields, and entities, and outputs them in a structured format. + +Modules and functionalities: +- **Metadata Sheet**: Includes default metadata fields + such as `start`, `end`, `username`, and `deviceid`. +- **Survey Sheet**: Combines metadata with mandatory fields required for FMTM workflows. + - `warmup` for collecting initial location. + - `feature` for selecting map geometry from predefined options. + - `new_feature` for capturing GPS coordinates of new map features. + - Calculated fields such as `form_category`, `xid`, `xlocation`, `status`, and others. +- **Entities Sheet**: Defines entity management rules to handle mapping tasks dynamically. + - Includes rules for entity creation and updates with user-friendly labels. +- **Settings Sheet**: Sets the form ID, version, and configuration options. +""" + +from datetime import datetime + +import pandas as pd + +current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + +meta_data = [ + {"type": "start", "name": "start"}, + {"type": "end", "name": "end"}, + {"type": "today", "name": "today"}, + {"type": "phonenumber", "name": "phonenumber"}, + {"type": "deviceid", "name": "deviceid"}, + {"type": "username", "name": "username"}, + { + "type": "email", + "name": "email", + }, +] + +meta_df = pd.DataFrame(meta_data) + +mandatory_data = [ + { + "type": "note", + "name": "instructions", + "label::english(en)": """Welcome ${username}. This survey form was generated + by HOT's FMTM to record ${form_category} map features.""", + }, + {"notes": "Fields essential to FMTM"}, + {"type": "start-geopoint", "name": "warmup", "notes": "collects location on form start"}, + {"type": "select_one_from_file features.csv", "name": "feature", "label::english(en)": "Geometry", "appearance": "map"}, + { + "type": "geopoint", + "name": "new_feature", + "label::english(en)": "Alternatively, take a gps coordinates of a new feature", + "appearance": "placement-map", + "relevant": "${feature}= ''", + "required": "yes", + }, + { + "type": "calculate", + "name": "form_category", + "label::english(en)": "FMTM form category", + "appearance": "minimal", + "calculation": "once('Unkown')", + }, + { + "type": "calculate", + "name": "xid", + "notes": "e.g. OSM ID", + "label::english(en)": "Feature ID", + "appearance": "minimal", + "calculation": "if(${feature} != '', instance('features')/root/item[name=${feature}]/osm_id, '')", + }, + { + "type": "calculate", + "name": "xlocation", + "notes": "e.g. OSM Geometry", + "label::english(en)": "Feature Geometry", + "appearance": "minimal", + "calculation": "if(${feature} != '', instance('features')/root/item[name=${feature}]/geometry, ${new_feature})", + "save_to": "geometry", + }, + { + "type": "calculate", + "name": "task_id", + "notes": "e.g. FMTM Task ID", + "label::english(en)": "Task ID", + "appearance": "minimal", + "calculation": "if(${feature} != '', instance('features')/root/item[name=${feature}]/task_id, '')", + "save_to": "task_id", + }, + { + "type": "calculate", + "name": "status", + "notes": "Update the Entity 'status' field", + "label::english(en)": "Mapping Status", + "appearance": "minimal", + "calculation": """if(${new_feature} != '', 2, + if(${building_exists} = 'no', 5, + if(${digitisation_correct} = 'no', 6, + ${status})))""", + "default": "2", + "trigger": "${new_feature}", + "save_to": "status", + }, + { + "type": "select_one yes_no", + "name": "building_exists", + "label::english(en)": "Does this feature exist?", + "relevant": "${feature} != '' ", + }, + { + "type": "calculate", + "name": "submission_id", + "notes": "Update the submission id", + "label::english(en)": "Submission id", + "appearance": "minimal", + "calculation": "once(${instanceID})", + "save_to": "submission_id", + }, +] + +mandatory_df = pd.DataFrame(mandatory_data) + +# Define the survey sheet +survey_df = pd.concat([meta_df, mandatory_df]) + +# Define entities sheet +entities_data = [ + { + "list_name": "features", + "entity_id": "coalesce(${feature}, uuid())", + "create_if": "if(${new_feature}, true(), false())", + "update_if": "if(${new_feature}, false(), true())", + "label": """concat(if(${status} = '1', "🔒 ", + if(${status} = '2', "✅ ", if(${status} = '5', "❌ ", + if(${status} = '6', "❌ ", '')))), "Task ", ${task_id}, + " Feature ", if(${xid} != ' ', ${xid}, ' '))""", + } +] +entities_df = pd.DataFrame(entities_data) + +# Define the settings sheet +settings_data = [ + { + "form_id": "mandatory_fields", + "version": current_datetime, + "form_title": "Mandatory Fields Form", + "allow_choice_duplicates": "yes", + } +] + +settings_df = pd.DataFrame(settings_data) diff --git a/osm_fieldwork/update_xlsform.py b/osm_fieldwork/update_xlsform.py index fd249153..17fa3fb7 100644 --- a/osm_fieldwork/update_xlsform.py +++ b/osm_fieldwork/update_xlsform.py @@ -1,6 +1,7 @@ """Update an existing XLSForm with additional fields useful for field mapping.""" import logging +import re from datetime import datetime from io import BytesIO from uuid import uuid4 @@ -8,7 +9,9 @@ import pandas as pd from python_calamine.pandas import pandas_monkeypatch -from osm_fieldwork.xlsforms import xlsforms_path +from osm_fieldwork.form_components.choice_fields import choices_df, digitisation_choices_df +from osm_fieldwork.form_components.digitisation_fields import digitisation_df +from osm_fieldwork.form_components.mandatory_fields import entities_df, meta_df, settings_df, survey_df log = logging.getLogger(__name__) @@ -18,7 +21,48 @@ # Constants FEATURE_COLUMN = "feature" NAME_COLUMN = "name" +TYPE_COLUMN = "type" SURVEY_GROUP_NAME = "survey_questions" +DEFAULT_LANGUAGES = { + "english": "en", + "french": "fr", + "spanish": "es", + "swahili": "sw", + "nepali": "ne", +} + +# def handle_translations( +# mandatory_df: pd.DataFrame, user_question_df: pd.DataFrame, digitisation_df: pd.DataFrame, fields: list[str] +# ): +# """Handle translations, defaulting to English if no translations are present. + +# Handles all field types that can be translated, such as +# 'label', 'hint', 'required_message'. +# """ +# for field in fields: +# # Identify translation columns for this field in the user_question_df +# translation_columns = [col for col in user_question_df.columns if col.startswith(f"{field}::")] + +# if field in user_question_df.columns and not translation_columns: +# # If user_question_df has only the base field (e.g., 'label'), map English translation from mandatory and digitisation +# mandatory_df[field] = mandatory_df.get(f"{field}::english(en)", mandatory_df.get(field)) +# digitisation_df[field] = digitisation_df.get(f"{field}::english(en)", digitisation_df.get(field)) + +# # Then drop translation columns +# mandatory_df = mandatory_df.loc[:, ~mandatory_df.columns.str.startswith("label::")] +# digitisation_df = digitisation_df.loc[:, ~digitisation_df.columns.str.startswith("label::")] + +# else: +# # If translation columns exist, match them for mandatory and digitisation dataframes +# for col in translation_columns: +# mandatory_col = mandatory_df.get(col) +# digitisation_col = digitisation_df.get(col) +# if mandatory_col is not None: +# mandatory_df[col] = mandatory_col +# if digitisation_col is not None: +# digitisation_df[col] = digitisation_col + +# return mandatory_df, user_question_df, digitisation_df def standardize_xlsform_sheets(xlsform: dict) -> dict: @@ -34,36 +78,93 @@ def standardize_xlsform_sheets(xlsform: dict) -> dict: dict: The updated XLSForm dictionary with standardized column headers. """ - def clean_column_name(col_name): - if col_name == "label": - return "label::english(en)" - if "::" in col_name: - # Handle '::' columns (e.g., 'label::english (en)') - parts = col_name.split("::") - language_part = parts[1].replace(" ", "").lower() # Remove spaces and lowercase - return f"{parts[0]}::{language_part}" - return col_name.strip().lower() # General cleanup - - # Apply cleaning to each sheet - for _sheet_name, sheet_df in xlsform.items(): - sheet_df.columns = [clean_column_name(col) for col in sheet_df.columns] + def standardize_language_columns(df): + """Standardize existing language columns. + + :param df: Original DataFrame with existing translations. + :param DEFAULT_LANGAUGES: List of DEFAULT_LANGAUGES with their short codes, e.g., {"english": "en", "french": "fr"}. + :param base_columns: List of base columns to check (e.g., 'label', 'hint', 'required_message'). + :return: Updated DataFrame with standardized and complete language columns. + """ + base_columns = ["label", "hint", "required_message"] + df.columns = df.columns.str.lower() + existing_columns = df.columns.tolist() + + # Map existing columns and standardize their names + for col in existing_columns: + standardized_col = col + for base_col in base_columns: + if col.startswith(f"{base_col}::"): + match = re.match(rf"{base_col}::(\w+)", col) + if match: + lang_name = match.group(1) + if lang_name in DEFAULT_LANGUAGES: + standardized_col = f"{base_col}::{lang_name}({DEFAULT_LANGUAGES[lang_name]})" + + elif col == base_col: # if only label,hint or required_message then add '::english(en)' + standardized_col = f"{base_col}::english(en)" + + if col != standardized_col: + df.rename(columns={col: standardized_col}, inplace=True) + return df + + def filter_df_empty_rows(df: pd.DataFrame, column: str = NAME_COLUMN): + """Remove rows with None values in the specified column. + + NOTE We retain 'end group' and 'end group' rows even if they have no name. + NOTE A generic df.dropna(how="all") would not catch accidental spaces etc. + """ + if column in df.columns: + # Only retain 'begin group' and 'end group' if 'type' column exists + if "type" in df.columns: + return df[(df[column].notna()) | (df["type"].isin(["begin group", "end group", "begin_group", "end_group"]))] + else: + return df[df[column].notna()] + return df + + for sheet_name, sheet_df in xlsform.items(): + if sheet_df.empty: + continue + # standardize the language columns + sheet_df = standardize_language_columns(sheet_df) + sheet_df = filter_df_empty_rows(sheet_df) + xlsform[sheet_name] = sheet_df return xlsform -def merge_dataframes(mandatory_df: pd.DataFrame, user_question_df: pd.DataFrame, digitisation_df: pd.DataFrame): - """Merge multiple Pandas dataframes together, removing duplicate fields.""" - # Remove empty rows from dataframes - mandatory_df = filter_df_empty_rows(mandatory_df) - user_question_df = filter_df_empty_rows(user_question_df) - digitisation_df = filter_df_empty_rows(digitisation_df) +def create_survey_group(name: str) -> dict[str, pd.DataFrame]: + """Helper function to create a begin and end group for XLSForm.""" + begin_group = pd.DataFrame( + { + "type": ["begin group"], + "name": [name], + "label::english(en)": [name], + "label::swahili(sw)": [name], + "label::french(fr)": [name], + "label::spanish(es)": [name], + "relevant": "(${new_feature} != '') or (${building_exists} = 'yes')", + } + ) + end_group = pd.DataFrame( + { + "type": ["end group"], + } + ) + return {"begin": begin_group, "end": end_group} + - # Handle matching translation fields for label, hint, required_message, etc. - # FIXME this isn't working properly yet - # mandatory_df, user_question_df, digitisation_df = handle_translations( - # mandatory_df, user_question_df, digitisation_df, fields=["label", "hint", "required_message"] - # ) +def normalize_with_meta(row, meta_df): + """Replace metadata in user_question_df with metadata from meta_df of mandatory fields if exists.""" + matching_meta = meta_df[meta_df["type"] == row[TYPE_COLUMN]] + if not matching_meta.empty: + for col in matching_meta.columns: + row[col] = matching_meta.iloc[0][col] + return row + +def merge_dataframes(mandatory_df: pd.DataFrame, user_question_df: pd.DataFrame, digitisation_df: pd.DataFrame) -> pd.DataFrame: + """Merge multiple Pandas dataframes together, removing duplicate fields.""" if "list_name" in user_question_df.columns: merged_df = pd.concat( [ @@ -78,7 +179,7 @@ def merge_dataframes(mandatory_df: pd.DataFrame, user_question_df: pd.DataFrame, # to have duplicate NAME_COLUMN entries, if they are in different a `list_name`. return merged_df.drop_duplicates(subset=["list_name", NAME_COLUMN], ignore_index=True) - # Else we are processing the survey sheet, continue + user_question_df = user_question_df.apply(normalize_with_meta, axis=1, meta_df=meta_df) # Find common fields between user_question_df and mandatory_df or digitisation_df duplicate_fields = set(user_question_df[NAME_COLUMN]).intersection( @@ -108,81 +209,10 @@ def merge_dataframes(mandatory_df: pd.DataFrame, user_question_df: pd.DataFrame, ) -def handle_translations( - mandatory_df: pd.DataFrame, user_question_df: pd.DataFrame, digitisation_df: pd.DataFrame, fields: list[str] -): - """Handle translations, defaulting to English if no translations are present. - - Handles all field types that can be translated, such as - 'label', 'hint', 'required_message'. - """ - for field in fields: - # Identify translation columns for this field in the user_question_df - translation_columns = [col for col in user_question_df.columns if col.startswith(f"{field}::")] - - if field in user_question_df.columns and not translation_columns: - # If user_question_df has only the base field (e.g., 'label'), map English translation from mandatory and digitisation - mandatory_df[field] = mandatory_df.get(f"{field}::english(en)", mandatory_df.get(field)) - digitisation_df[field] = digitisation_df.get(f"{field}::english(en)", digitisation_df.get(field)) - - # Then drop translation columns - mandatory_df = mandatory_df.loc[:, ~mandatory_df.columns.str.startswith("label::")] - digitisation_df = digitisation_df.loc[:, ~digitisation_df.columns.str.startswith("label::")] - - else: - # If translation columns exist, match them for mandatory and digitisation dataframes - for col in translation_columns: - mandatory_col = mandatory_df.get(col) - digitisation_col = digitisation_df.get(col) - if mandatory_col is not None: - mandatory_df[col] = mandatory_col - if digitisation_col is not None: - digitisation_df[col] = digitisation_col - - return mandatory_df, user_question_df, digitisation_df - - -def filter_df_empty_rows(df: pd.DataFrame, column: str = NAME_COLUMN): - """Remove rows with None values in the specified column. - - NOTE We retain 'end group' and 'end group' rows even if they have no name. - NOTE A generic df.dropna(how="all") would not catch accidental spaces etc. - """ - if column in df.columns: - # Only retain 'begin group' and 'end group' if 'type' column exists - if "type" in df.columns: - return df[(df[column].notna()) | (df["type"].isin(["begin group", "end group", "begin_group", "end_group"]))] - else: - return df[df[column].notna()] - return df - - -def create_survey_group(name: str) -> dict[str, pd.DataFrame]: - """Helper function to create a begin and end group for XLSForm.""" - begin_group = pd.DataFrame( - { - "type": ["begin group"], - "name": [name], - "label::english(en)": [name], - "label::swahili(sw)": [name], - "label::french(fr)": [name], - "label::spanish(es)": [name], - "relevant": "(${new_feature} != '') or (${building_exists} = 'yes')", - } - ) - end_group = pd.DataFrame( - { - "type": ["end group"], - } - ) - return {"begin": begin_group, "end": end_group} - - def append_select_one_from_file_row(df: pd.DataFrame, entity_name: str) -> pd.DataFrame: """Add a new select_one_from_file question to reference an Entity.""" # Find the row index where name column = 'feature' select_one_from_file_index = df.index[df[NAME_COLUMN] == FEATURE_COLUMN].tolist() - if not select_one_from_file_index: raise ValueError(f"Row with '{NAME_COLUMN}' == '{FEATURE_COLUMN}' not found in survey sheet.") @@ -206,11 +236,11 @@ def append_select_one_from_file_row(df: pd.DataFrame, entity_name: str) -> pd.Da { "type": ["calculate"], "name": ["additional_geometry"], - "calculation": [f"instance('{entity_name}')/root/item[name=${entity_name}]/geometry"], - "label::English(en)": ["additional_geometry"], - "label::Swahili(sw)": ["additional_geometry"], - "label::French(fr)": ["additional_geometry"], - "label::Spanish(es)": ["additional_geometry"], + "calculation": [f"instance('{entity_name}')/root/item[name=${{{entity_name}}}]/geometry"], + "label::english(en)": ["additional_geometry"], + "label::swahili(sw)": ["additional_geometry"], + "label::french(fr)": ["additional_geometry"], + "label::spanish(es)": ["additional_geometry"], } ) # Insert the new row into the DataFrame @@ -241,20 +271,16 @@ async def append_mandatory_fields( """ log.info("Appending field mapping questions to XLSForm") custom_sheets = pd.read_excel(custom_form, sheet_name=None, engine="calamine") - mandatory_sheets = pd.read_excel(f"{xlsforms_path}/common/mandatory_fields.xls", sheet_name=None, engine="calamine") - digitisation_sheets = pd.read_excel(f"{xlsforms_path}/common/digitisation_fields.xls", sheet_name=None, engine="calamine") - custom_sheets = standardize_xlsform_sheets(custom_sheets) - # Merge 'survey' and 'choices' sheets if "survey" not in custom_sheets: msg = "Survey sheet is required in XLSForm!" log.error(msg) raise ValueError(msg) + custom_sheets = standardize_xlsform_sheets(custom_sheets) + log.debug("Merging survey sheet XLSForm data") - custom_sheets["survey"] = merge_dataframes( - mandatory_sheets.get("survey"), custom_sheets.get("survey"), digitisation_sheets.get("survey") - ) + custom_sheets["survey"] = merge_dataframes(survey_df, custom_sheets.get("survey"), digitisation_df) # Hardcode the form_category value for the start instructions if form_category.endswith("s"): # Plural to singular @@ -268,13 +294,12 @@ async def append_mandatory_fields( custom_sheets["choices"] = pd.DataFrame(columns=["list_name", "name", "label::english(en)"]) log.debug("Merging choices sheet XLSForm data") - custom_sheets["choices"] = merge_dataframes( - mandatory_sheets.get("choices"), custom_sheets.get("choices"), digitisation_sheets.get("choices") - ) + custom_sheets["choices"] = merge_dataframes(choices_df, custom_sheets.get("choices"), digitisation_choices_df) # Append or overwrite 'entities' and 'settings' sheets log.debug("Overwriting entities and settings XLSForm sheets") - custom_sheets.update({key: mandatory_sheets[key] for key in ["entities", "settings"] if key in mandatory_sheets}) + custom_sheets["entities"] = entities_df + custom_sheets["settings"] = settings_df if "entities" not in custom_sheets: msg = "Entities sheet is required in XLSForm!" log.error(msg) @@ -303,6 +328,5 @@ async def append_mandatory_fields( with pd.ExcelWriter(output, engine="openpyxl") as writer: for sheet_name, df in custom_sheets.items(): df.to_excel(writer, sheet_name=sheet_name, index=False) - output.seek(0) return (xform_id, output)