Add new Geography definitions for International and no-Geography (#116)

* Add new Geography definitions for International and no-Geography

* remove old International from data

* get tests working and make consts

* add geo region of 'other'

* use slugify

* fixup tests after geo re-structure

* also test international

app/core/ingestion/processor.py

@@ -34,6 +34,7 @@
     validate_unfccc_document_row,
 )
 from app.db.models.app.users import Organisation
+from app.db.models.law_policy.geography import GEO_INTERNATIONAL, GEO_NONE
 
 
 _LOGGER = logging.getLogger(__name__)
@@ -43,6 +44,16 @@
 ProcessFunc = Callable[[IngestContext, _RowType], None]
 
 
+def parse_csv_geography(csv_geo: str) -> str:
+    if csv_geo == "":
+        return GEO_NONE
+
+    if csv_geo == "INT":
+        return GEO_INTERNATIONAL  # Support old style
+
+    return csv_geo
+
+
 def build_params_from_cclw(row: CCLWDocumentIngestRow) -> IngestParameters:
     def add_metadata(db: Session, import_id: str, taxonomy: Taxonomy, taxonomy_id: int):
         add_cclw_metadata(db, import_id, taxonomy, taxonomy_id, row)
@@ -59,7 +70,7 @@ def add_metadata(db: Session, import_id: str, taxonomy: Taxonomy, taxonomy_id: i
         family_summary=row.family_summary,
         document_role=row.document_role,
         document_variant=row.document_variant,
-        geography_iso=row.geography_iso,
+        geography_iso=parse_csv_geography(row.geography_iso),
         documents=row.documents,
         category=row.category,
         document_type=row.document_type,
@@ -89,7 +100,7 @@ def add_metadata(db: Session, import_id: str, taxonomy: Taxonomy, taxonomy_id: i
         family_summary=row.family_summary,
         document_role=row.document_role,
         document_variant=row.document_variant,
-        geography_iso=row.geography_iso,
+        geography_iso=parse_csv_geography(row.geography_iso),
         documents=row.documents,
         category=row.category,
         document_type=row.submission_type,

app/core/ingestion/validator.py

@@ -22,6 +22,7 @@
     Variant,
     Geography,
 )
+from app.db.models.law_policy.geography import GEO_NONE
 from app.db.session import Base
 
 DbTable = Base
@@ -104,9 +105,12 @@ def validate_unfccc_document_row(
         errors.append(result)
 
     # validate: geography_iso: str
-    result = _check_geo_in_db(n, db, row.geography_iso)
-    if result.type != ResultType.OK:
-        errors.append(result)
+    if row.geography_iso != "":
+        result = _check_geo_in_db(n, db, row.geography_iso)
+        if result.type != ResultType.OK:
+            errors.append(result)
+    else:
+        row.geography_iso = GEO_NONE
 
     # validate: Submission type as document type
     result = _check_value_in_db(

app/data_migrations/data/geo_stats_data.json

@@ -995,18 +995,6 @@
     "visibility_status": "published",
     "iso": "IDN"
   },
-  {
-    "name": "International",
-    "legislative_process": "",
-    "federal": false,
-    "federal_details": "",
-    "political_groups": "",
-    "global_emissions_percent": null,
-    "climate_risk_index": null,
-    "worldbank_income_group": "",
-    "visibility_status": "draft",
-    "iso": "INT"
-  },
   {
     "name": "Iran",
     "legislative_process": "<p style=\"text-align: justify;\">The Islamic Republic of Iran has a semi-democratic political system established after the Islamic Revolution of 1979. The political system is based upon governance by an Islamic jurist. The system is comprised of a Supreme Leader, as well as the Executive, Legislative and Judicial branches of power. Although the Constitution affirms the independence of each of the three branches from each other, it states that all three branches are under the direction of the Supreme Leader.</p>\n<p style=\"text-align: justify;\">The Supreme Leader is the spiritual leader, commander-in-chief of the armed forces and controls intelligence and security operations. He is elected by the Assembly of Experts on the basis of his personality (leadership abilities, his religious qualifications and his popular esteem). The Supreme Leader appoints the head of the judicial branch, the head of state radio and television, and the supreme commander of the Islamic Revolutionary Guard Corps. He also appoints six of the 12 members of the Council of Guardians, and he designates the members of the Expediency Council.</p>\n<p style=\"text-align: justify;\">The head of the Executive branch is the President, who is directly elected by the popular vote to a four-year term, for a maximum of two terms. He is the second highest-ranking official and is responsible for the implementation of the Constitution and for the exercise of executive powers, except for matters directly related to the Supreme Leader. The President appoints and supervises the Council of Ministers. The Government is comprised of the ministers and vice-presidents in charge of different government ministries or organisations. Presently there are 21 ministers and 10 vice-presidents, including the head of the Environmental Protection Organisation. The ministers are accountable to the President as well as to Parliament. The last presidential elections in Iran took place in 2013, and the next elections should be in 2017.</p>\n<p style=\"text-align: justify;\">The Parliament has 290 members elected directly for four-year terms and is the main legislative body. It ratifies all laws and international treaties and approves the national Budget. Elections are direct and by secret popular ballot. Each Member of Parliament represents a particular geographic area. The last parliamentary election took place in February 2016, and the next election is expected for 2020.</p>\n<p style=\"text-align: justify;\">The upper house, the Council of Guardians, approves all laws ratified by Parliament. Its responsibility is to verify that the law is not in contradiction with Islamic law and the Constitution. They are 12 members; six Islamic Jurists who are Ayatollahs appointed by the Leader, and a further six high level attorneys elected by Parliament. The Council can approve or veto legislation passed by Parliament on the grounds that it is inconsistent with the Constitution and/or Islamic law. Following a veto, Parliament can amend the legislation in order to address the Council\u2019s concerns. If Parliament and the Council of Guardians fail to resolve their differences, the Expediency Council is empowered to make the final decision. The Expediency Council also serves as the Leader\u2019s advisory body to formulate the \u2018general policies\u2019 of the country. These range from economic to environmental, social or judicial matters. The Council\u2019s recommendations become law when they are ratified by the Leader.</p>",

app/data_migrations/data/geography_data.json

@@ -1443,14 +1443,6 @@
         },
         "children": []
       },
-      {
-        "node": {
-          "display_value": "International",
-          "value": "INT",
-          "type": "ISO-3166"
-        },
-        "children": []
-      },
       {
         "node": {
           "display_value": "Japan",

app/data_migrations/data/source/geography-iso-3166.csv

@@ -80,7 +80,6 @@ Hungary,HUN,National,Europe & Central Asia,FALSE,,OECD;EU,0.13%,69,High income
 Iceland,ISL,National,Europe & Central Asia,FALSE,,OECD,0.01%,170.33,High income
 India,IND,National,South Asia,TRUE,"29 states, 7 Union Territories, including a National Capital Territory",G77;G20,6.81%,38.67,Lower middle income
 Indonesia,IDN,National,East Asia & Pacific,FALSE,,G77;G20,4.69%,76.83,Lower middle income
-International,INT,Supranational,East Asia & Pacific,FALSE,,,-,,
 Iran,IRN,National,Middle East & North Africa,FALSE,,G77,1.83%,79,Upper middle income
 Iraq,IRQ,National,Middle East & North Africa,TRUE,18 provinces,G77;The Arab Group,0.40%,141.33,Upper middle income
 Ireland,IRL,National,Europe & Central Asia,FALSE,,OECD;EU,0.14%,119.17,High income

app/data_migrations/populate_geography.py

@@ -5,6 +5,11 @@
 from sqlalchemy.orm import Session
 
 from app.db.models.law_policy import Geography
+from app.db.models.law_policy.geography import (
+    CPR_DEFINED_GEOS,
+    GEO_OTHER,
+    GeoStatistics,
+)
 
 from .utils import has_rows, load_tree
 
@@ -19,10 +24,57 @@ def _add_geo_slugs(geo_tree: list[dict[str, dict]]):
             _add_geo_slugs(child_nodes)
 
 
+def remove_old_international_geo(db: Session) -> None:
+    db_international = (
+        db.query(Geography).filter(Geography.value == "INT").one_or_none()
+    )
+    if db_international is not None:
+        db_stats = (
+            db.query(GeoStatistics)
+            .filter(GeoStatistics.geography_id == db_international.id)
+            .one_or_none()
+        )
+        if db_stats is not None:
+            db.delete(db_stats)
+            db.flush()
+        db.delete(db_international)
+        db.flush()
+
+
 def populate_geography(db: Session) -> None:
     """Populates the geography table with pre-defined data."""
 
-    if has_rows(db, Geography):
+    geo_populated = has_rows(db, Geography)
+    # First ensure our defined entries are present
+    remove_old_international_geo(db)
+
+    # Add the Other region
+    other = db.query(Geography).filter(Geography.value == GEO_OTHER).one_or_none()
+    if other is None:
+        other = Geography(
+            display_value=GEO_OTHER,
+            slug=slugify(GEO_OTHER),
+            value=GEO_OTHER,
+            type="ISO-3166 CPR Extension",
+        )
+        db.add(other)
+        db.flush()
+
+    # Add the CPR geo definitions in Other
+    for value, description in CPR_DEFINED_GEOS.items():
+        db_geo = db.query(Geography).filter(Geography.value == value).one_or_none()
+        if db_geo is None:
+            db.add(
+                Geography(
+                    display_value=description,
+                    slug=slugify(value),
+                    value=value,
+                    type="ISO-3166 CPR Extension",
+                    parent_id=other.id,
+                )
+            )
+
+    if geo_populated:
         return
 
     with open("app/data_migrations/data/geography_data.json") as geo_data_file:

app/db/models/law_policy/geography.py

@@ -3,6 +3,13 @@
 from app.db.session import Base
 
 
+GEO_NONE = "XAA"
+GEO_INTERNATIONAL = "XAB"
+
+GEO_OTHER = "Other"
+CPR_DEFINED_GEOS = {GEO_NONE: "No Geography", GEO_INTERNATIONAL: "International"}
+
+
 class Geography(Base):  # noqa: D101
     """Database model for Geography"""