diff --git a/osm_fieldwork/form_components/__init__.py b/osm_fieldwork/form_components/__init__.py
new file mode 100644
index 00000000..3478f5b4
--- /dev/null
+++ b/osm_fieldwork/form_components/__init__.py
@@ -0,0 +1,5 @@
+"""OSM Fieldwork Form Builder Package."""
+
+import os
+
+package_root = os.path.dirname(os.path.abspath(__file__))
diff --git a/osm_fieldwork/form_components/choice_fields.py b/osm_fieldwork/form_components/choice_fields.py
new file mode 100644
index 00000000..24b87431
--- /dev/null
+++ b/osm_fieldwork/form_components/choice_fields.py
@@ -0,0 +1,69 @@
+#!/usr/bin/python3
+
+# Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team
+#
+# This file is part of OSM-Fieldwork.
+#
+# This is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OSM-Fieldwork. If not, see .
+#
+
+"""This creates DataFrames for choice lists used in the survey form.
+
+These include predefined options for fields such as yes/no responses
+and issues related to digitization problems. Each choice contains
+multilingual labels to support various languages.
+
+Returns:
+ tuple: Two pandas DataFrames containing the `choices` data for yes/no
+and digitisation problems respectively.
+"""
+
+import pandas as pd
+
+# Define the choices sheet
+choices_data = [
+ {"list_name": "yes_no", "name": "yes", "label::english(en)": "Yes"},
+ {"list_name": "yes_no", "name": "no", "label::english(en)": "No"},
+]
+
+choices_df = pd.DataFrame(choices_data)
+
+digitisation_choices = [
+ {
+ "list_name": "digitisation_problem",
+ "name": "lumped",
+ "label::english(en)": "Lumped - one polygon (more than one building digitized as one)",
+ "label::swahili(sw)": "Lumped - poligoni moja (zaidi ya jengo moja limewekwa dijiti kuwa moja)",
+ "label::french(fr)": "Lumped - un polygone (plus d'un bâtiment numérisé en un seul)",
+ "label::spanish(es)": "Agrupado - un polígono (más de un edificio digitalizado como uno)",
+ },
+ {
+ "list_name": "digitisation_problem",
+ "name": "split",
+ "label::english(en)": "Split - one building (one building digitized as more than one polygon)",
+ "label::swahili(sw)": "Mgawanyiko - jengo moja (jengo moja limebadilishwa kuwa zaidi ya poligoni moja)",
+ "label::french(fr)": "Fractionnement - un bâtiment (un bâtiment numérisé sous la forme de plusieurs polygones)",
+ "label::spanish(es)": "Split - un edificio (un edificio digitalizado como más de un polígono)",
+ },
+ {
+ "list_name": "digitisation_problem",
+ "name": "other",
+ "label::english(en)": "OTHER",
+ "label::swahili(sw)": "MENGINEYO",
+ "label::french(fr)": "AUTRES",
+ "label::spanish(es)": "OTROS",
+ },
+]
+
+digitisation_choices_df = pd.DataFrame(digitisation_choices)
diff --git a/osm_fieldwork/form_components/digitisation_fields.py b/osm_fieldwork/form_components/digitisation_fields.py
new file mode 100644
index 00000000..b0334814
--- /dev/null
+++ b/osm_fieldwork/form_components/digitisation_fields.py
@@ -0,0 +1,78 @@
+#!/usr/bin/python3
+
+# Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team
+#
+# This file is part of OSM-Fieldwork.
+#
+# This is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OSM-Fieldwork. If not, see .
+#
+
+"""This script creates a DataFrame for digitization-related fields in the survey form.
+
+These fields focus on verifying the accuracy of digitized locations,
+capturing any issues with digitization, and adding additional notes or
+images if required. The fields include logic for relevance, mandatory
+requirements, and conditions based on user responses.
+
+Returns:
+ pd.DataFrame: A DataFrame containing the digitization-related fields.
+"""
+
+import pandas as pd
+
+digitisation_fields = [
+ {
+ "type": "begin group",
+ "name": "verification",
+ "label::english(en)": "Verification",
+ "relevant": "(${new_feature} != '') or (${building_exists} = 'yes')",
+ },
+ {
+ "type": "select_one yes_no",
+ "name": "digitisation_correct",
+ "label::english(en)": "Is the digitized location for this feature correct?",
+ "relevant": "(${new_feature} != '') or (${building_exists} = 'yes')",
+ "calculation": "once(if(${new_feature} != '', 'yes', ''))",
+ "read_only": "${new_feature} != ''",
+ "required": "yes",
+ },
+ {
+ "type": "select_one digitisation_problem",
+ "name": "digitisation_problem",
+ "label::english(en)": "What is wrong with the digitization?",
+ "relevant": "${digitisation_correct}='no'",
+ },
+ {
+ "type": "text",
+ "name": "digitisation_problem_other",
+ "label::english(en)": "You said “Other.” Please tell us what went wrong with the digitization!",
+ "relevant": "${digitisation_problem}='other' ",
+ },
+ {"type": "end group"},
+ {
+ "type": "image",
+ "name": "image",
+ "label::english(en)": "Take a Picture",
+ "apperance": "minimal",
+ "parameters": "max-pixels=1000",
+ },
+ {
+ "type": "note",
+ "name": "end_note",
+ "label::english(en)": "You can't proceed with data acquisition, if the building doesn't exist.",
+ "relevant": "${building_exists} = 'no'",
+ },
+]
+
+digitisation_df = pd.DataFrame(digitisation_fields)
diff --git a/osm_fieldwork/form_components/mandatory_fields.py b/osm_fieldwork/form_components/mandatory_fields.py
new file mode 100644
index 00000000..486fafab
--- /dev/null
+++ b/osm_fieldwork/form_components/mandatory_fields.py
@@ -0,0 +1,175 @@
+#!/usr/bin/python3
+
+# Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team
+#
+# This file is part of OSM-Fieldwork.
+#
+# This is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OSM-Fieldwork. If not, see .
+#
+
+"""This script generates an XLS form with mandatory fields.
+
+For use in data collection and mapping tasks.
+The generated form includes metadata, survey questions, and settings
+required for compatibility with HOT's FMTM tools.
+It programmatically organizes form sections into metadata,
+mandatory fields, and entities, and outputs them in a structured format.
+
+Modules and functionalities:
+- **Metadata Sheet**: Includes default metadata fields
+ such as `start`, `end`, `username`, and `deviceid`.
+- **Survey Sheet**: Combines metadata with mandatory fields required for FMTM workflows.
+ - `warmup` for collecting initial location.
+ - `feature` for selecting map geometry from predefined options.
+ - `new_feature` for capturing GPS coordinates of new map features.
+ - Calculated fields such as `form_category`, `xid`, `xlocation`, `status`, and others.
+- **Entities Sheet**: Defines entity management rules to handle mapping tasks dynamically.
+ - Includes rules for entity creation and updates with user-friendly labels.
+- **Settings Sheet**: Sets the form ID, version, and configuration options.
+"""
+
+from datetime import datetime
+
+import pandas as pd
+
+current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+meta_data = [
+ {"type": "start", "name": "start"},
+ {"type": "end", "name": "end"},
+ {"type": "today", "name": "today"},
+ {"type": "phonenumber", "name": "phonenumber"},
+ {"type": "deviceid", "name": "deviceid"},
+ {"type": "username", "name": "username"},
+ {
+ "type": "email",
+ "name": "email",
+ },
+]
+
+meta_df = pd.DataFrame(meta_data)
+
+mandatory_data = [
+ {
+ "type": "note",
+ "name": "instructions",
+ "label::english(en)": """Welcome ${username}. This survey form was generated
+ by HOT's FMTM to record ${form_category} map features.""",
+ },
+ {"notes": "Fields essential to FMTM"},
+ {"type": "start-geopoint", "name": "warmup", "notes": "collects location on form start"},
+ {"type": "select_one_from_file features.csv", "name": "feature", "label::english(en)": "Geometry", "appearance": "map"},
+ {
+ "type": "geopoint",
+ "name": "new_feature",
+ "label::english(en)": "Alternatively, take a gps coordinates of a new feature",
+ "appearance": "placement-map",
+ "relevant": "${feature}= ''",
+ "required": "yes",
+ },
+ {
+ "type": "calculate",
+ "name": "form_category",
+ "label::english(en)": "FMTM form category",
+ "appearance": "minimal",
+ "calculation": "once('Unkown')",
+ },
+ {
+ "type": "calculate",
+ "name": "xid",
+ "notes": "e.g. OSM ID",
+ "label::english(en)": "Feature ID",
+ "appearance": "minimal",
+ "calculation": "if(${feature} != '', instance('features')/root/item[name=${feature}]/osm_id, '')",
+ },
+ {
+ "type": "calculate",
+ "name": "xlocation",
+ "notes": "e.g. OSM Geometry",
+ "label::english(en)": "Feature Geometry",
+ "appearance": "minimal",
+ "calculation": "if(${feature} != '', instance('features')/root/item[name=${feature}]/geometry, ${new_feature})",
+ "save_to": "geometry",
+ },
+ {
+ "type": "calculate",
+ "name": "task_id",
+ "notes": "e.g. FMTM Task ID",
+ "label::english(en)": "Task ID",
+ "appearance": "minimal",
+ "calculation": "if(${feature} != '', instance('features')/root/item[name=${feature}]/task_id, '')",
+ "save_to": "task_id",
+ },
+ {
+ "type": "calculate",
+ "name": "status",
+ "notes": "Update the Entity 'status' field",
+ "label::english(en)": "Mapping Status",
+ "appearance": "minimal",
+ "calculation": """if(${new_feature} != '', 2,
+ if(${building_exists} = 'no', 5,
+ if(${digitisation_correct} = 'no', 6,
+ ${status})))""",
+ "default": "2",
+ "trigger": "${new_feature}",
+ "save_to": "status",
+ },
+ {
+ "type": "select_one yes_no",
+ "name": "building_exists",
+ "label::english(en)": "Does this feature exist?",
+ "relevant": "${feature} != '' ",
+ },
+ {
+ "type": "calculate",
+ "name": "submission_id",
+ "notes": "Update the submission id",
+ "label::english(en)": "Submission id",
+ "appearance": "minimal",
+ "calculation": "once(${instanceID})",
+ "save_to": "submission_id",
+ },
+]
+
+mandatory_df = pd.DataFrame(mandatory_data)
+
+# Define the survey sheet
+survey_df = pd.concat([meta_df, mandatory_df])
+
+# Define entities sheet
+entities_data = [
+ {
+ "list_name": "features",
+ "entity_id": "coalesce(${feature}, uuid())",
+ "create_if": "if(${new_feature}, true(), false())",
+ "update_if": "if(${new_feature}, false(), true())",
+ "label": """concat(if(${status} = '1', "🔒 ",
+ if(${status} = '2', "✅ ", if(${status} = '5', "❌ ",
+ if(${status} = '6', "❌ ", '')))), "Task ", ${task_id},
+ " Feature ", if(${xid} != ' ', ${xid}, ' '))""",
+ }
+]
+entities_df = pd.DataFrame(entities_data)
+
+# Define the settings sheet
+settings_data = [
+ {
+ "form_id": "mandatory_fields",
+ "version": current_datetime,
+ "form_title": "Mandatory Fields Form",
+ "allow_choice_duplicates": "yes",
+ }
+]
+
+settings_df = pd.DataFrame(settings_data)
diff --git a/osm_fieldwork/update_xlsform.py b/osm_fieldwork/update_xlsform.py
index fd249153..17fa3fb7 100644
--- a/osm_fieldwork/update_xlsform.py
+++ b/osm_fieldwork/update_xlsform.py
@@ -1,6 +1,7 @@
"""Update an existing XLSForm with additional fields useful for field mapping."""
import logging
+import re
from datetime import datetime
from io import BytesIO
from uuid import uuid4
@@ -8,7 +9,9 @@
import pandas as pd
from python_calamine.pandas import pandas_monkeypatch
-from osm_fieldwork.xlsforms import xlsforms_path
+from osm_fieldwork.form_components.choice_fields import choices_df, digitisation_choices_df
+from osm_fieldwork.form_components.digitisation_fields import digitisation_df
+from osm_fieldwork.form_components.mandatory_fields import entities_df, meta_df, settings_df, survey_df
log = logging.getLogger(__name__)
@@ -18,7 +21,48 @@
# Constants
FEATURE_COLUMN = "feature"
NAME_COLUMN = "name"
+TYPE_COLUMN = "type"
SURVEY_GROUP_NAME = "survey_questions"
+DEFAULT_LANGUAGES = {
+ "english": "en",
+ "french": "fr",
+ "spanish": "es",
+ "swahili": "sw",
+ "nepali": "ne",
+}
+
+# def handle_translations(
+# mandatory_df: pd.DataFrame, user_question_df: pd.DataFrame, digitisation_df: pd.DataFrame, fields: list[str]
+# ):
+# """Handle translations, defaulting to English if no translations are present.
+
+# Handles all field types that can be translated, such as
+# 'label', 'hint', 'required_message'.
+# """
+# for field in fields:
+# # Identify translation columns for this field in the user_question_df
+# translation_columns = [col for col in user_question_df.columns if col.startswith(f"{field}::")]
+
+# if field in user_question_df.columns and not translation_columns:
+# # If user_question_df has only the base field (e.g., 'label'), map English translation from mandatory and digitisation
+# mandatory_df[field] = mandatory_df.get(f"{field}::english(en)", mandatory_df.get(field))
+# digitisation_df[field] = digitisation_df.get(f"{field}::english(en)", digitisation_df.get(field))
+
+# # Then drop translation columns
+# mandatory_df = mandatory_df.loc[:, ~mandatory_df.columns.str.startswith("label::")]
+# digitisation_df = digitisation_df.loc[:, ~digitisation_df.columns.str.startswith("label::")]
+
+# else:
+# # If translation columns exist, match them for mandatory and digitisation dataframes
+# for col in translation_columns:
+# mandatory_col = mandatory_df.get(col)
+# digitisation_col = digitisation_df.get(col)
+# if mandatory_col is not None:
+# mandatory_df[col] = mandatory_col
+# if digitisation_col is not None:
+# digitisation_df[col] = digitisation_col
+
+# return mandatory_df, user_question_df, digitisation_df
def standardize_xlsform_sheets(xlsform: dict) -> dict:
@@ -34,36 +78,93 @@ def standardize_xlsform_sheets(xlsform: dict) -> dict:
dict: The updated XLSForm dictionary with standardized column headers.
"""
- def clean_column_name(col_name):
- if col_name == "label":
- return "label::english(en)"
- if "::" in col_name:
- # Handle '::' columns (e.g., 'label::english (en)')
- parts = col_name.split("::")
- language_part = parts[1].replace(" ", "").lower() # Remove spaces and lowercase
- return f"{parts[0]}::{language_part}"
- return col_name.strip().lower() # General cleanup
-
- # Apply cleaning to each sheet
- for _sheet_name, sheet_df in xlsform.items():
- sheet_df.columns = [clean_column_name(col) for col in sheet_df.columns]
+ def standardize_language_columns(df):
+ """Standardize existing language columns.
+
+ :param df: Original DataFrame with existing translations.
+ :param DEFAULT_LANGAUGES: List of DEFAULT_LANGAUGES with their short codes, e.g., {"english": "en", "french": "fr"}.
+ :param base_columns: List of base columns to check (e.g., 'label', 'hint', 'required_message').
+ :return: Updated DataFrame with standardized and complete language columns.
+ """
+ base_columns = ["label", "hint", "required_message"]
+ df.columns = df.columns.str.lower()
+ existing_columns = df.columns.tolist()
+
+ # Map existing columns and standardize their names
+ for col in existing_columns:
+ standardized_col = col
+ for base_col in base_columns:
+ if col.startswith(f"{base_col}::"):
+ match = re.match(rf"{base_col}::(\w+)", col)
+ if match:
+ lang_name = match.group(1)
+ if lang_name in DEFAULT_LANGUAGES:
+ standardized_col = f"{base_col}::{lang_name}({DEFAULT_LANGUAGES[lang_name]})"
+
+ elif col == base_col: # if only label,hint or required_message then add '::english(en)'
+ standardized_col = f"{base_col}::english(en)"
+
+ if col != standardized_col:
+ df.rename(columns={col: standardized_col}, inplace=True)
+ return df
+
+ def filter_df_empty_rows(df: pd.DataFrame, column: str = NAME_COLUMN):
+ """Remove rows with None values in the specified column.
+
+ NOTE We retain 'end group' and 'end group' rows even if they have no name.
+ NOTE A generic df.dropna(how="all") would not catch accidental spaces etc.
+ """
+ if column in df.columns:
+ # Only retain 'begin group' and 'end group' if 'type' column exists
+ if "type" in df.columns:
+ return df[(df[column].notna()) | (df["type"].isin(["begin group", "end group", "begin_group", "end_group"]))]
+ else:
+ return df[df[column].notna()]
+ return df
+
+ for sheet_name, sheet_df in xlsform.items():
+ if sheet_df.empty:
+ continue
+ # standardize the language columns
+ sheet_df = standardize_language_columns(sheet_df)
+ sheet_df = filter_df_empty_rows(sheet_df)
+ xlsform[sheet_name] = sheet_df
return xlsform
-def merge_dataframes(mandatory_df: pd.DataFrame, user_question_df: pd.DataFrame, digitisation_df: pd.DataFrame):
- """Merge multiple Pandas dataframes together, removing duplicate fields."""
- # Remove empty rows from dataframes
- mandatory_df = filter_df_empty_rows(mandatory_df)
- user_question_df = filter_df_empty_rows(user_question_df)
- digitisation_df = filter_df_empty_rows(digitisation_df)
+def create_survey_group(name: str) -> dict[str, pd.DataFrame]:
+ """Helper function to create a begin and end group for XLSForm."""
+ begin_group = pd.DataFrame(
+ {
+ "type": ["begin group"],
+ "name": [name],
+ "label::english(en)": [name],
+ "label::swahili(sw)": [name],
+ "label::french(fr)": [name],
+ "label::spanish(es)": [name],
+ "relevant": "(${new_feature} != '') or (${building_exists} = 'yes')",
+ }
+ )
+ end_group = pd.DataFrame(
+ {
+ "type": ["end group"],
+ }
+ )
+ return {"begin": begin_group, "end": end_group}
+
- # Handle matching translation fields for label, hint, required_message, etc.
- # FIXME this isn't working properly yet
- # mandatory_df, user_question_df, digitisation_df = handle_translations(
- # mandatory_df, user_question_df, digitisation_df, fields=["label", "hint", "required_message"]
- # )
+def normalize_with_meta(row, meta_df):
+ """Replace metadata in user_question_df with metadata from meta_df of mandatory fields if exists."""
+ matching_meta = meta_df[meta_df["type"] == row[TYPE_COLUMN]]
+ if not matching_meta.empty:
+ for col in matching_meta.columns:
+ row[col] = matching_meta.iloc[0][col]
+ return row
+
+def merge_dataframes(mandatory_df: pd.DataFrame, user_question_df: pd.DataFrame, digitisation_df: pd.DataFrame) -> pd.DataFrame:
+ """Merge multiple Pandas dataframes together, removing duplicate fields."""
if "list_name" in user_question_df.columns:
merged_df = pd.concat(
[
@@ -78,7 +179,7 @@ def merge_dataframes(mandatory_df: pd.DataFrame, user_question_df: pd.DataFrame,
# to have duplicate NAME_COLUMN entries, if they are in different a `list_name`.
return merged_df.drop_duplicates(subset=["list_name", NAME_COLUMN], ignore_index=True)
- # Else we are processing the survey sheet, continue
+ user_question_df = user_question_df.apply(normalize_with_meta, axis=1, meta_df=meta_df)
# Find common fields between user_question_df and mandatory_df or digitisation_df
duplicate_fields = set(user_question_df[NAME_COLUMN]).intersection(
@@ -108,81 +209,10 @@ def merge_dataframes(mandatory_df: pd.DataFrame, user_question_df: pd.DataFrame,
)
-def handle_translations(
- mandatory_df: pd.DataFrame, user_question_df: pd.DataFrame, digitisation_df: pd.DataFrame, fields: list[str]
-):
- """Handle translations, defaulting to English if no translations are present.
-
- Handles all field types that can be translated, such as
- 'label', 'hint', 'required_message'.
- """
- for field in fields:
- # Identify translation columns for this field in the user_question_df
- translation_columns = [col for col in user_question_df.columns if col.startswith(f"{field}::")]
-
- if field in user_question_df.columns and not translation_columns:
- # If user_question_df has only the base field (e.g., 'label'), map English translation from mandatory and digitisation
- mandatory_df[field] = mandatory_df.get(f"{field}::english(en)", mandatory_df.get(field))
- digitisation_df[field] = digitisation_df.get(f"{field}::english(en)", digitisation_df.get(field))
-
- # Then drop translation columns
- mandatory_df = mandatory_df.loc[:, ~mandatory_df.columns.str.startswith("label::")]
- digitisation_df = digitisation_df.loc[:, ~digitisation_df.columns.str.startswith("label::")]
-
- else:
- # If translation columns exist, match them for mandatory and digitisation dataframes
- for col in translation_columns:
- mandatory_col = mandatory_df.get(col)
- digitisation_col = digitisation_df.get(col)
- if mandatory_col is not None:
- mandatory_df[col] = mandatory_col
- if digitisation_col is not None:
- digitisation_df[col] = digitisation_col
-
- return mandatory_df, user_question_df, digitisation_df
-
-
-def filter_df_empty_rows(df: pd.DataFrame, column: str = NAME_COLUMN):
- """Remove rows with None values in the specified column.
-
- NOTE We retain 'end group' and 'end group' rows even if they have no name.
- NOTE A generic df.dropna(how="all") would not catch accidental spaces etc.
- """
- if column in df.columns:
- # Only retain 'begin group' and 'end group' if 'type' column exists
- if "type" in df.columns:
- return df[(df[column].notna()) | (df["type"].isin(["begin group", "end group", "begin_group", "end_group"]))]
- else:
- return df[df[column].notna()]
- return df
-
-
-def create_survey_group(name: str) -> dict[str, pd.DataFrame]:
- """Helper function to create a begin and end group for XLSForm."""
- begin_group = pd.DataFrame(
- {
- "type": ["begin group"],
- "name": [name],
- "label::english(en)": [name],
- "label::swahili(sw)": [name],
- "label::french(fr)": [name],
- "label::spanish(es)": [name],
- "relevant": "(${new_feature} != '') or (${building_exists} = 'yes')",
- }
- )
- end_group = pd.DataFrame(
- {
- "type": ["end group"],
- }
- )
- return {"begin": begin_group, "end": end_group}
-
-
def append_select_one_from_file_row(df: pd.DataFrame, entity_name: str) -> pd.DataFrame:
"""Add a new select_one_from_file question to reference an Entity."""
# Find the row index where name column = 'feature'
select_one_from_file_index = df.index[df[NAME_COLUMN] == FEATURE_COLUMN].tolist()
-
if not select_one_from_file_index:
raise ValueError(f"Row with '{NAME_COLUMN}' == '{FEATURE_COLUMN}' not found in survey sheet.")
@@ -206,11 +236,11 @@ def append_select_one_from_file_row(df: pd.DataFrame, entity_name: str) -> pd.Da
{
"type": ["calculate"],
"name": ["additional_geometry"],
- "calculation": [f"instance('{entity_name}')/root/item[name=${entity_name}]/geometry"],
- "label::English(en)": ["additional_geometry"],
- "label::Swahili(sw)": ["additional_geometry"],
- "label::French(fr)": ["additional_geometry"],
- "label::Spanish(es)": ["additional_geometry"],
+ "calculation": [f"instance('{entity_name}')/root/item[name=${{{entity_name}}}]/geometry"],
+ "label::english(en)": ["additional_geometry"],
+ "label::swahili(sw)": ["additional_geometry"],
+ "label::french(fr)": ["additional_geometry"],
+ "label::spanish(es)": ["additional_geometry"],
}
)
# Insert the new row into the DataFrame
@@ -241,20 +271,16 @@ async def append_mandatory_fields(
"""
log.info("Appending field mapping questions to XLSForm")
custom_sheets = pd.read_excel(custom_form, sheet_name=None, engine="calamine")
- mandatory_sheets = pd.read_excel(f"{xlsforms_path}/common/mandatory_fields.xls", sheet_name=None, engine="calamine")
- digitisation_sheets = pd.read_excel(f"{xlsforms_path}/common/digitisation_fields.xls", sheet_name=None, engine="calamine")
- custom_sheets = standardize_xlsform_sheets(custom_sheets)
- # Merge 'survey' and 'choices' sheets
if "survey" not in custom_sheets:
msg = "Survey sheet is required in XLSForm!"
log.error(msg)
raise ValueError(msg)
+ custom_sheets = standardize_xlsform_sheets(custom_sheets)
+
log.debug("Merging survey sheet XLSForm data")
- custom_sheets["survey"] = merge_dataframes(
- mandatory_sheets.get("survey"), custom_sheets.get("survey"), digitisation_sheets.get("survey")
- )
+ custom_sheets["survey"] = merge_dataframes(survey_df, custom_sheets.get("survey"), digitisation_df)
# Hardcode the form_category value for the start instructions
if form_category.endswith("s"):
# Plural to singular
@@ -268,13 +294,12 @@ async def append_mandatory_fields(
custom_sheets["choices"] = pd.DataFrame(columns=["list_name", "name", "label::english(en)"])
log.debug("Merging choices sheet XLSForm data")
- custom_sheets["choices"] = merge_dataframes(
- mandatory_sheets.get("choices"), custom_sheets.get("choices"), digitisation_sheets.get("choices")
- )
+ custom_sheets["choices"] = merge_dataframes(choices_df, custom_sheets.get("choices"), digitisation_choices_df)
# Append or overwrite 'entities' and 'settings' sheets
log.debug("Overwriting entities and settings XLSForm sheets")
- custom_sheets.update({key: mandatory_sheets[key] for key in ["entities", "settings"] if key in mandatory_sheets})
+ custom_sheets["entities"] = entities_df
+ custom_sheets["settings"] = settings_df
if "entities" not in custom_sheets:
msg = "Entities sheet is required in XLSForm!"
log.error(msg)
@@ -303,6 +328,5 @@ async def append_mandatory_fields(
with pd.ExcelWriter(output, engine="openpyxl") as writer:
for sheet_name, df in custom_sheets.items():
df.to_excel(writer, sheet_name=sheet_name, index=False)
-
output.seek(0)
return (xform_id, output)