Merge pull request #142 from BritishGeologicalSurvey/geojson-api

Geojson api and gui

.github/workflows/lint_and_test.yml

@@ -15,10 +15,10 @@ jobs:
 
     steps:
     - name: Checkout source repository
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v4
 
     - name: Setup Python
-      uses: actions/setup-python@v4.5.0
+      uses: actions/setup-python@v5
       with:
         python-version: 3.11
         architecture: x64

.github/workflows/main.yml

@@ -30,7 +30,7 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4
 
       - name: Build image
         run: docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"

.github/workflows/pages.yml

@@ -14,11 +14,11 @@ jobs:
         run: |
           git config user.name github-actions[bot]
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.x
       - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           key: mkdocs-material-${{ env.cache_id }}
           path: .cache

app/borehole_map.py

@@ -22,7 +22,7 @@ def extract_geojson(filepath: Path) -> dict:
     Read an AGS4 file and extract geojson represenation of LOCA table and
     metadata.
     """
-    logger.info("Extracting geojson from  %s", filepath.name)
+    logger.info("Extracting geojson from %s", filepath.name)
 
     # Read data file
     tables, load_error, _ = load_tables_reporting_errors(filepath)

app/checkers.py

@@ -43,11 +43,11 @@ def check_ags(filename: Path, standard_AGS4_dictionary: Optional[str] = None) ->
         errors = {'File read error': [{'line': line_no, 'group': '', 'desc': description}]}
         dictionary = ''
 
-    # Discard unecessary summary from errors dictionary
-    errors.pop('Summary of data', None)
+    # Use summary from errors dictionary is available
+    summary = errors.pop('Summary of data', [])
 
     return dict(checker=f'python_ags4 v{python_ags4.__version__}',
-                errors=errors, dictionary=dictionary)
+                errors=errors, dictionary=dictionary, summary=summary)
 
 
 def check_bgs(filename: Path, **kwargs) -> dict:

app/routes.py

@@ -16,6 +16,7 @@
 from requests.exceptions import Timeout, ConnectionError, HTTPError
 
 from app import conversion, validation
+from app.borehole_map import extract_geojson
 from app.checkers import check_ags, check_bgs
 from app.errors import error_responses, InvalidPayloadError
 from app.schemas import ValidationResponse, BoreholeCountResponse
@@ -72,6 +73,14 @@ class Checker(StrEnum):
     bgs = "bgs"
 
 
+# Enum for sorting strategy logic
+class SortingStrategy(StrEnum):
+    default = "default"
+    alphabetical = "alphabetical"
+    hierarchy = "hierarchy"
+    dictionary = "dictionary"
+
+
 # Enum for pdf response type logic
 class ResponseType(StrEnum):
     attachment = "attachment"
@@ -89,6 +98,13 @@ class ResponseType(StrEnum):
     description='Response format: json or text',
 )
 
+geometry_form = Form(
+    default=False,
+    title='GeoJSON Option',
+    description=('Return GeoJSON if possible, otherwise return an error message '
+                 ' Option: True or False'),
+)
+
 dictionary_form = Form(
     default=None,
     title='Validation Dictionary',
@@ -114,10 +130,10 @@ class ResponseType(StrEnum):
 )
 
 sort_tables_form = Form(
-    default='default',
+    default=SortingStrategy.default,
     title='Sort worksheets',
-    description=('Sort the worksheets into alphabetical order '
-                 'or leave in the order found in the AGS file. '
+    description=('Sort the worksheets into alphabetical, hierarchical '
+                 'dictionary or default order, that found in the AGS file. '
                  'This option is ignored when converting to AGS.'),
 )
 
@@ -167,6 +183,7 @@ async def validate(background_tasks: BackgroundTasks,
                    std_dictionary: Dictionary = dictionary_form,
                    checkers: List[Checker] = validate_form,
                    fmt: Format = format_form,
+                   return_geometry: bool = geometry_form,
                    request: Request = None):
     """
     Validate an AGS4 file to the AGS File Format v4.x rules and the NGDC data submission requirements.
@@ -182,6 +199,8 @@ async def validate(background_tasks: BackgroundTasks,
 
     :param fmt: The format to return the validation results in. Options are "text" or "json".
     :type fmt: Format
+    :param return_geometry: Include GeoJSON in validation response. Options are True or False.
+    :type return_geometry: bool
     :param request: The request object.
     :type request: Request
     :return: A response with the validation results in either plain text or JSON format.
@@ -207,6 +226,13 @@ async def validate(background_tasks: BackgroundTasks,
         local_ags_file.write_bytes(contents)
         result = validation.validate(
             local_ags_file, checkers=checkers, standard_AGS4_dictionary=dictionary)
+        if return_geometry:
+            try:
+                geojson = extract_geojson(local_ags_file)
+                result['geojson'] = geojson
+            except ValueError as ve:
+                result['geojson'] = {}
+                result['geojson_error'] = str(ve)
         data.append(result)
 
     if fmt == Format.TEXT:
@@ -262,7 +288,7 @@ async def convert(background_tasks: BackgroundTasks,
     :raises Exception: If the conversion fails or an unexpected error occurs.
     """
 
-    if sort_tables == 'default':
+    if sort_tables == SortingStrategy.default:
         sort_tables = None
     if not files[0].filename:
         raise InvalidPayloadError(request)

app/schemas.py

@@ -47,7 +47,10 @@ class Validation(BaseModel):
     message: str = Field(None, example="7 error(s) found in file!")
     errors: Dict[str, List[LineError]] = Field(..., example="Rule 1a")
     valid: bool = Field(..., example='false')
+    summary: List[dict] = list()
     additional_metadata: dict = Field(...)
+    geojson: dict = dict()
+    geojson_error: str = None
 
     @validator('errors')
     def errors_keys_must_be_known_rules(cls, errors):

app/templates/landing_page.html

@@ -123,12 +123,20 @@ <h4>Future data validation rules: (Coming Soon)</h4>
         <br>
       <fieldset>
           <legend>Select response format:</legend>
-          <input type="radio" id="text" name="fmt" value="text">
-          <label for="text">Plain Text</label>
           <input type="radio" id="json" name="fmt" value="json">
-          <label for="json">JSON</label><br>
+          <label for="json">JSON</label>
+          <input type="radio" id="text" name="fmt" value="text">
+          <label for="text">Plain Text</label><br>
       </fieldset>
         <br>
+        <fieldset>
+          <legend>If HTML show LOCA features on a map / If JSON include GeoJSON</legend>
+          <input type="radio" id="return_geometry" name="return_geometry" value="true" checked="checked">
+          <label for="true">Yes</label>
+          <input type="radio" id="return_geometry" name="return_geometry" value="false">
+          <label for="false">No</label><br>
+      </fieldset>  
+        <br>
       <fieldset>
           <legend>Select .ags / .AGS file(s) for validation (v4.x only) <b>(50 Mb Maximum)</b></legend>
           <input name="files" type="file" multiple>
@@ -165,15 +173,15 @@ <h2>AGS Converter</h2>
   <br>
   <form action="/convert/" enctype="multipart/form-data" method="post" id="convertForm">
     <fieldset>
-      <legend>Sort worksheets in .xlsx file using sorting strategy<strong>(Warning: .ags to .xlsx only. The original group order will be lost)</strong></legend>
+      <legend>Sort worksheets in .xlsx file using a sorting strategy <strong>(Warning: .ags to .xlsx only. The original group order will be lost)</strong></legend>
+      <input type="radio" id="default" name="sort_tables" value="default" checked="checked">
+      <label for="default">None (Maintain Input File Order)</label><br>
       <input type="radio" id="dictionary" name="sort_tables" value="dictionary">
-      <label for="dictionary">Dictionary</label>
+      <label for="dictionary">File Dictionary</label><br>
       <input type="radio" id="alphabetical" name="sort_tables" value="alphabetical">
-      <label for="alphabetical">Alphabetical</label>
+      <label for="alphabetical">Alphabetical</label><br>
       <input type="radio" id="hierarchical" name="sort_tables" value="hierarchical">
-      <label for="hierarchical">Hierarchical</label>
-      <input type="radio" id="default" name="sort_tables" value="default" checked>
-      <label for="default">None (Maintain Input File Order)</label><br>
+      <label for="hierarchical">AGS Standard Hierarchy</label><br>
     </fieldset>
     <br>
     <fieldset>

app/validation.py

@@ -47,6 +47,8 @@ def validate(filename: Path,
 
     all_errors = {}
     all_checkers = []
+    bgs_additional_metadata = {}
+    ags_summary = []
     # Don't process if file is not .ags format
     if filename.suffix.lower() != '.ags':
         all_errors.update(
@@ -57,19 +59,32 @@ def validate(filename: Path,
         # Run checkers to extract errors and other metadata
         for checker in checkers:
             # result is a dictionary with 'errors', 'checker' and other keys
-            result = checker(filename, standard_AGS4_dictionary=dictionary_file)
+            result: dict = checker(filename, standard_AGS4_dictionary=dictionary_file)
+
             # Pull 'errors' out to add to running total
             all_errors.update(result.pop('errors'))
             all_checkers.append(result.pop('checker'))
-            # Handle additional metadata
+
+            # Extract checker-dependent additional metadata
             try:
-                response['additional_metadata'].update(result.pop('additional_metadata'))
+                bgs_additional_metadata = result.pop('additional_metadata')
             except KeyError:
-                # No additional metadata
                 pass
+
+            try:
+                ags_summary = result.pop('summary')
+            except KeyError:
+                pass
+
             # Add remaining keys to response
             response.update(result)
 
+    # We only want one checker-dependent metadata; use BGS if available.
+    if bgs_additional_metadata:
+        response['additional_metadata'] = bgs_additional_metadata
+    else:
+        response['summary'] = ags_summary
+
     error_count = len(reduce(lambda total, current: total + current, all_errors.values(), []))
     if error_count > 0:
         message = f'{error_count} error(s) found in file!'