Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add GeoJSON select & update github actions #139

Closed
wants to merge 25 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
1e05446
Extract load_tables_reporting_errors method
volcan01010 Mar 6, 2024
dd24ca1
Extract create_location_gpd function
volcan01010 Mar 6, 2024
91d2e89
Add extract_geojson function
volcan01010 Mar 6, 2024
da4f468
Refine output columns
volcan01010 Mar 6, 2024
fb1bb86
Add tests for exception cases
volcan01010 Mar 6, 2024
04899d1
Add concatenate_feature_collection function
volcan01010 Mar 6, 2024
7cfe4f4
Flake8 fixes
volcan01010 Mar 6, 2024
eed8922
Add geojson response option
KoalaGeo Mar 7, 2024
d58a8cc
Update GitHub Actions Versions
KoalaGeo Mar 7, 2024
9e6ba29
Python version 3.11 in actions
KoalaGeo Mar 7, 2024
4c7a57e
Add default selection for sorting strategy
KoalaGeo Mar 7, 2024
66bc41e
Update test file paths
ximenesuk Mar 7, 2024
fb4b411
Format Text
KoalaGeo Mar 7, 2024
8f84a7d
Remove nexus mirror configuration
ximenesuk Mar 7, 2024
4ba8e73
Merge pull request #137 from BritishGeologicalSurvey/32-geojson-extract
ximenesuk Mar 7, 2024
a0004e4
Add two missing rules
ximenesuk Mar 6, 2024
033d7d8
Add test case for rules 3, 4 & 5
ximenesuk Mar 7, 2024
09a6ca3
Exception no longer thrown
ximenesuk Mar 7, 2024
f2c4ef3
Merge pull request #138 from BritishGeologicalSurvey/missing-rules
KoalaGeo Mar 7, 2024
91b016b
Add geojson response option
KoalaGeo Mar 7, 2024
eddb960
Update GitHub Actions Versions
KoalaGeo Mar 7, 2024
27a070a
Python version 3.11 in actions
KoalaGeo Mar 7, 2024
ce6eab8
Add default selection for sorting strategy
KoalaGeo Mar 7, 2024
4e062d9
Format Text
KoalaGeo Mar 7, 2024
b12d63f
Merge branch 'add-geojson-gui' of https://github.com/BritishGeologica…
KoalaGeo Mar 7, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Extract create_location_gpd function
volcan01010 authored and ximenesuk committed Mar 7, 2024
commit dd24ca19c997f62434c92fa318c366a9aa065cf2
17 changes: 11 additions & 6 deletions app/bgs_rules.py
Original file line number Diff line number Diff line change
@@ -184,16 +184,11 @@ def check_loca_within_great_britain(tables: dict) -> List[dict]:

# Read data into geodataframe
try:
location = tables['LOCA'].set_index('LOCA_ID')
location['geometry'] = list(zip(location['LOCA_NATE'], location['LOCA_NATN']))
location = create_location_gpd(tables)
except KeyError:
# LOCA not present, already checked in earlier rule
return errors

location['geometry'] = location['geometry'].apply(Point)
location = gpd.GeoDataFrame(location, geometry='geometry', crs='EPSG:27700')
location['line_no'] = range(1, len(location) + 1)

inside_uk_eea_mask = location.intersects(uk_eea_outline)
inside_gb_mask = location.intersects(gb_outline)
as_irish_grid = location.to_crs("EPSG:29903")
@@ -225,6 +220,16 @@ def check_loca_within_great_britain(tables: dict) -> List[dict]:
return errors


def create_location_gpd(tables: dict[pd.DataFrame]) -> gpd.GeoDataFrame:
location: pd.DataFrame = tables['LOCA'].set_index('LOCA_ID')
location['geometry'] = list(zip(location['LOCA_NATE'], location['LOCA_NATN']))
location['geometry'] = location['geometry'].apply(Point)
location = gpd.GeoDataFrame(location, geometry='geometry', crs='EPSG:27700')
location['line_no'] = range(1, len(location) + 1)

return location


def check_locx_is_not_duplicate_of_other_column(tables: dict) -> List[dict]:
"""LOCA_LOCX and LOCA_LOCY are not duplicates of other columns"""

4 changes: 2 additions & 2 deletions test/integration/test_api.py
Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@
from python_ags4 import AGS4

from app.main import app
from app.checkers import load_AGS4_as_numeric
from app.checkers import load_ags4_as_numeric
import app.routes as app_routes
from test.fixtures import (BAD_FILE_DATA, DICTIONARIES, FROZEN_TIME,
GOOD_FILE_DATA)
@@ -545,7 +545,7 @@ def test_get_ags_export_single_id(client, tmp_path):
unzipped_ags_file = tmp_path / 'test.ags'
with open(unzipped_ags_file, 'wb') as f:
f.write(ags_file.read())
tables, _, _ = load_AGS4_as_numeric(unzipped_ags_file)
tables, _, _ = load_ags4_as_numeric(unzipped_ags_file)
assert tables['PROJ']['BGS_PROJ_ID'][0] == bgs_proj_id
# Confirm the metadata file is correct
with ags_zip.open(ags_metadata_file_name) as metadata_file:
26 changes: 13 additions & 13 deletions test/unit/test_bgs_rules.py
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@
import pytest

from app.bgs_rules import BGS_RULES
from app.checkers import load_AGS4_as_numeric
from app.checkers import load_ags4_as_numeric
from test.fixtures import BGS_RULES_ERRORS

TEST_FILE_DIR = Path(__file__).parent.parent / 'files'
@@ -16,7 +16,7 @@ def test_required_groups():
expected = {'line': '-',
'group': '',
'desc': 'Required groups not present: ABBR, TYPE, UNIT, (LOCA or HOLE)'}
tables, _, _ = load_AGS4_as_numeric(filename)
tables, _, _ = load_ags4_as_numeric(filename)

errors = BGS_RULES['BGS data validation: Required Groups'](tables)

@@ -29,7 +29,7 @@ def test_required_bgs_groups():
expected = {'line': '-',
'group': '',
'desc': 'Required BGS groups not present: GEOL'}
tables, _, _ = load_AGS4_as_numeric(filename)
tables, _, _ = load_ags4_as_numeric(filename)

errors = BGS_RULES['BGS data validation: Required BGS Groups'](tables)

@@ -42,7 +42,7 @@ def test_spatial_referencing():
expected = {'line': '-',
'group': 'LOCA',
'desc': 'Spatial referencing system not in LOCA_GREF, LOCA_LREF or LOCA_LLZ!'}
tables, _, _ = load_AGS4_as_numeric(filename)
tables, _, _ = load_ags4_as_numeric(filename)

errors = BGS_RULES['BGS data validation: Spatial Referencing'](tables)

@@ -60,7 +60,7 @@ def test_eastings_northings_present():
'group': 'LOCA',
'desc': 'LOCA_NATN contains zeros or null values'}
]
tables, _, _ = load_AGS4_as_numeric(filename)
tables, _, _ = load_ags4_as_numeric(filename)

errors = BGS_RULES['BGS data validation: Eastings/Northings Present'](tables)

@@ -78,7 +78,7 @@ def test_eastings_northings_range():
'group': 'LOCA',
'desc': 'LOCA_NATN values outside 100,000 to 1,400,000 range'},
]
tables, _, _ = load_AGS4_as_numeric(filename)
tables, _, _ = load_ags4_as_numeric(filename)

errors = BGS_RULES['BGS data validation: Eastings/Northings Range'](tables)

@@ -96,7 +96,7 @@ def test_drill_depth_present():
'group': 'HDPH',
'desc': 'HDPH_BASE contains zero or null values'},
]
tables, _, _ = load_AGS4_as_numeric(filename)
tables, _, _ = load_ags4_as_numeric(filename)

errors = BGS_RULES['BGS data validation: Drill Depth Present'](tables)

@@ -112,7 +112,7 @@ def test_drill_depth_geol_record():
{'line': '-', 'group': 'HDPH',
'desc': "GEOL LOCA_IDs not in HDPH group ({'BH109'})"},
]
tables, _, _ = load_AGS4_as_numeric(filename)
tables, _, _ = load_ags4_as_numeric(filename)

errors = BGS_RULES['BGS data validation: Drill Depth GEOL Record'](tables)

@@ -150,7 +150,7 @@ def test_loca_within_great_britain():
'group': 'LOCA',
'line': '6'}]

tables, _, _ = load_AGS4_as_numeric(filename)
tables, _, _ = load_ags4_as_numeric(filename)

errors = BGS_RULES['BGS data validation: LOCA within Great Britain'](tables)

@@ -168,7 +168,7 @@ def test_loca_locx_is_not_duplicate_of_other_column():
'group': 'LOCA',
'line': '-'},
]
tables, _, _ = load_AGS4_as_numeric(filename)
tables, _, _ = load_ags4_as_numeric(filename)

errors = BGS_RULES['BGS data validation: LOCA_LOCX is not duplicate of other column'](tables)

@@ -186,7 +186,7 @@ def test_loca_references_are_valid():
'group': 'SAMP',
'line': '-'},
]
tables, _, _ = load_AGS4_as_numeric(filename)
tables, _, _ = load_ags4_as_numeric(filename)

errors = BGS_RULES['BGS data validation: LOCA_ID references'](tables)

@@ -202,7 +202,7 @@ def test_non_numeric_coord_types():
"line": "-"}
]}

_, _, errors = load_AGS4_as_numeric(filename)
_, _, errors = load_ags4_as_numeric(filename)

assert errors == expected

@@ -216,7 +216,7 @@ def test_non_numeric_coord_types():
def test_sample_referential_integrity(filename, expected):
# Arrange
filename = TEST_FILE_DIR / 'bgs_rules' / filename
tables, _, _ = load_AGS4_as_numeric(filename)
tables, _, _ = load_ags4_as_numeric(filename)

errors = BGS_RULES['BGS data validation: Sample Referencing'](tables)