Skip to content

Commit

Permalink
Merge pull request #138 from BritishGeologicalSurvey/missing-rules
Browse files Browse the repository at this point in the history 
Missing rules
  • Loading branch information
@KoalaGeo
KoalaGeo authored Mar 7, 2024
2 parents 4ba8e73 + 09a6ca3 commit f2c4ef3
Show file tree
Hide file tree
Showing 6 changed files with 198 additions and 8 deletions.
4 changes: 0 additions & 4 deletions app/checkers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,6 @@ def check_ags(filename: Path, standard_AGS4_dictionary: Optional[str] = None) ->
description = f"UnicodeDecodeError: {err.reason}"
errors = {'File read error': [{'line': line_no, 'group': '', 'desc': description}]}
dictionary = ''
except AGS4.AGS4Error as err:
description = f"AGS4Error: {err}"
errors = {'File read error': [{'line': '-', 'group': '', 'desc': description}]}
dictionary = ''

# Discard unecessary summary from errors dictionary
errors.pop('Summary of data', None)
Expand Down
4 changes: 2 additions & 2 deletions app/schemas.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,13 @@
VALID_KEYS = [
# AGS schema rules
'AGS Format Rule 1', 'AGS Format Rule 2', 'AGS Format Rule 2a', 'AGS Format Rule 2b',
'AGS Format Rule 2c', 'AGS Format Rule 3', 'AGS Format Rule 4a', 'AGS Format Rule 4b',
'AGS Format Rule 2c', 'AGS Format Rule 3', 'AGS Format Rule 4', 'AGS Format Rule 4a', 'AGS Format Rule 4b',
'AGS Format Rule 5', 'AGS Format Rule 6', 'AGS Format Rule 7', 'AGS Format Rule 8',
'AGS Format Rule 9', 'AGS Format Rule 10a', 'AGS Format Rule 10b', 'AGS Format Rule 10c',
'AGS Format Rule 11a', 'AGS Format Rule 11b', 'AGS Format Rule 11c', 'AGS Format Rule 12',
'AGS Format Rule 13', 'AGS Format Rule 14', 'AGS Format Rule 15', 'AGS Format Rule 16',
'AGS Format Rule 17', 'AGS Format Rule 18', 'AGS Format Rule 19', 'AGS Format Rule 19a',
'AGS Format Rule 19b', 'AGS Format Rule 20', 'General',
'AGS Format Rule 19b', 'AGS Format Rule 20', 'AGS Format Rule ?', 'General',
# Warnings and FYIs
'Warning (Related to Rule 16)', 'FYI (Related to Rule 1)',
# Errors
Expand Down
68 changes: 68 additions & 0 deletions test/files/example_broken_ags.ags
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
"GROUP","PROJ"
"HEADING","PROJ_ID","PROJ_NAME","PROJ_LOC","PROJ_CLNT","PROJ_CONT","PROJ_ENG","PROJ_MEMO","FILE_FSET"
"UNIT","","","","","","","",""
"TYPE","X","X","X","X","X","X","X","X"
"DATA","121415","ACME Gas Works Redevelopment","Anytown","ACME Enterprises","ACME Drilling Ltd","","",""

"GROUP","ABBR"
"HEADING","ABBR_HDNG","ABBR_CODE","ABBR_DESC","ABBR_LIST","ABBR_REM","FILE_FSET"
"UNIT","","","","","",""
"TYPE","X","X","X","X","X","X"
"DATA","DICT_TYPE","GROUP","Flag to indicate definition is a GROUP","","",""
"DATA","DICT_TYPE","HEADING","Flag to indicate definition is a HEADING","","",""
"DATA","DICT_STAT","OTHER","Other Field","","",""
"DATA","DICT_STAT","KEY","Key Field","","",""
"DATA","SAMP_TYPE","U","Undisturbed sample - open drive","","",""

"GROUP","TRAN"
"HEADING","TRAN_ISNO","TRAN_DATE","TRAN_PROD","TRAN_STAT","TRAN_DESC","TRAN_AGS","TRAN_RECV","TRAN_DLIM","TRAN_RCON","TRAN_REM","FILE_FSET"
"UNIT","","yyyy-mm-dd","","","","","","","","",""
"TYPE","X","DT","X","X","X","X","X","X","X","X","X"
"DATA","1","2021-01-18","ACME Drilling Ltd","Preliminary","Draft Logs only","4.1","ACME Consulting","|","+","",""

"GROUP","TYPE"
"HEADING","TYPE_TYPE","TYPE_DESC","FILE_FSET"
"UNIT","","",""
"TYPE","X","X","X"
"DATA","U","Undefined",""
"DATA","X","Text",""
"DATA","ID","Unique identifier",""
"DATA","PA","ABBR pick list",""
"DATA","2DP","Value; required
number of decimal places, 2",""
"DATA","DT","Date Time (ISO 8601:2004)",""
"DATA","0DP","Value; required
number of decimal places, 0",""
"DATA","1DP","Value; required
number of decimal places, 1",""
"DATA","PT","TYPE pick list",""
"DATA","PU","UNIT pick list",""
"DATA","DMS","Degrees:Minutes:Seconds",""
"DATA","T","Elapsed time",""
"DATA","RL","Record link",""

"GROUP","UNIT"
"HEADING","UNIT_UNIT","UNIT_DESC","UNIT_REM","FILE_FSET"
"UNIT","","","",""
"TYPE","X","X","X","X"
"DATA","-","No unit","",""
"DATA","m","metre","",""
"DATA","yyyy-mm-dd","Date (ISO8601)","",""
"DATA","yyyy-mm-ddThh:mm:ss.sss","Date Time (ISO8601)","",""
"DATA","mm","millimetre","",""
"DATA","%","percentage","",""
"DATA","l/min","litres per minute","",""
"DATA","hh:mm:ss","hh:mm:ss","",""

"GROUP","LOCA"
"HEADING","LOCA_ID","LOCA_TYPE","LOCA_STAT","LOCA_NATE","LOCA_NATN","LOCA_GREF","LOCA_GL","LOCA_REM","LOCA_FDEP","LOCA_STAR","LOCA_PURP","LOCA_TERM","LOCA_ENDD","LOCA_LETT","LOCA_LOCX","LOCA_LOCY","LOCA_LOCZ","LOCA_LREF","LOCA_DATM","LOCA_ETRV","LOCA_NTRV","LOCA_LTRV","LOCA_XTRL","LOCA_YTRL","LOCA_ZTRL","LOCA_LAT","LOCA_LON","LOCA_ELAT","LOCA_ELON","LOCA_LLZ","LOCA_LOCM","LOCA_LOCA","LOCA_CLST","LOCA_ALID","LOCA_OFFS","LOCA_CNGE","LOCA_TRAN","FILE_FSET"
"UNIT","","","","m","m","","m","","m","yyyy-mm-dd","","","yyyy-mm-dd","","m","m","m","","","m","m","m","m","m","m","","","","","","","","","","","","",""
"TYPE","ID","PA","PA","2DP","2DP","PA","2DP","X","2DP","DT","X","X","DT","X","2DP","2DP","2DP","X","X","2DP","2DP","2DP","2DP","2DP","2DP","DMS","DMS","DMS","DMS","X","X","X","X","X","2DP","X","X","X"
"DATA","327-16A","","","523145.00","178456.12","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","",""

"GROUP","SAMP"
"HEADING","LOCA_ID","SAMP_TOP","SAMP_REF","SAMP_TYPE","SAMP_ID","SAMP_BASE","SAMP_DTIM","SAMP_UBLO","SAMP_CONT","SAMP_PREP","SAMP_SDIA","SAMP_WDEP","SAMP_RECV","SAMP_TECH","SAMP_MATX","SAMP_TYPC","SAMP_WHO","SAMP_WHY","SAMP_REM","SAMP_DESC","SAMP_DESD","SAMP_LOG","SAMP_COND","SAMP_CLSS","SAMP_BAR","SAMP_TEMP","SAMP_PRES","SAMP_FLOW","SAMP_ETIM","SAMP_DURN","SAMP_CAPT","SAMP_LINK","GEOL_STAT","FILE_FSET"
"UNIT","","m","","","","m","yyyy-mm-ddThh:mm:ss.sss","","","","mm","m","%","","","","","","","","yyyy-mm-dd","","","","","","","l/min","yyyy-mm-ddThh:mm:ss.sss","hh:mm:ss","","","",""
"TYPE","ID","2DP","X","PA","ID","2DP","DT","0DP","X","X","0DP","X","0DP","X","X","X","X","X","X","X","DT","X","X","X","1DP","0DP","1DP","1DP","DT","T","X","X","X","X"
"DATA","327-16A","24.55","24","U","ABC121415010","25.00","","","","","","","","","","","","","","","","","","","","","","","","","","","",""

85 changes: 85 additions & 0 deletions test/fixtures_json.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,91 @@
'valid': True,
'additional_metadata': {}
},
'example_broken_ags.ags': {
"filename": "example_broken_ags.ags",
"filesize": 4111,
"checkers": ["python_ags4 v0.5.0"],
'dictionary': 'Standard_dictionary_v4_1_1.ags',
'time': dt.datetime(2021, 8, 23, 14, 25, 43, tzinfo=dt.timezone.utc),
"message": "13 error(s) found in file!",
"errors": {
"AGS Format Rule 4": [
{
"line": 31,
"group": "TYPE",
"desc": "Number of fields does not match the HEADING row."
},
{
"line": 34,
"group": "TYPE",
"desc": "Number of fields does not match the HEADING row."
},
{
"line": 36,
"group": "TYPE",
"desc": "Number of fields does not match the HEADING row."
}
],
"AGS Format Rule 5": [
{
"line": 31,
"group": "",
"desc": "Contains fields that are not enclosed in double quotes."
},
{
"line": 32,
"group": "",
"desc": "Contains fields that are not enclosed in double quotes."
},
{
"line": 34,
"group": "",
"desc": "Contains fields that are not enclosed in double quotes."
},
{
"line": 35,
"group": "",
"desc": "Contains fields that are not enclosed in double quotes."
},
{
"line": 36,
"group": "",
"desc": "Contains fields that are not enclosed in double quotes."
},
{
"line": 37,
"group": "",
"desc": "Contains fields that are not enclosed in double quotes."
}
],
"AGS Format Rule 3": [
{
"line": 32,
"group": "",
"desc": "Does not start with a valid data descriptor."
},
{
"line": 35,
"group": "",
"desc": "Does not start with a valid data descriptor."
},
{
"line": 37,
"group": "",
"desc": "Does not start with a valid data descriptor."
}
],
"AGS Format Rule ?": [
{
"line": "-",
"group": "",
"desc": "Line 31 does not have the same number of entries as the HEADING row in TYPE."
}
],
},
"valid": False,
'additional_metadata': {}
},
'nonsense.AGS': {
'filename': 'nonsense.AGS',
'filesize': 9,
Expand Down
41 changes: 41 additions & 0 deletions test/fixtures_plain_text.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,47 @@
Time: 2021-08-23 14:25:43+00:00
================================================================================
""",
'example_broken_ags.ags': """
================================================================================
example_broken_ags.ags: 13 error(s) found in file!
# Metadata
File size: 4111 bytes
Checkers: ['python_ags4 v0.5.0']
Dictionary: Standard_dictionary_v4_1_1.ags
Time: 2021-08-23 14:25:43+00:00
# Errors
## AGS Format Rule 4
Line: 31 - Number of fields does not match the HEADING row.
Line: 34 - Number of fields does not match the HEADING row.
Line: 36 - Number of fields does not match the HEADING row.
## AGS Format Rule 5
Line: 31 - Contains fields that are not enclosed in double quotes.
Line: 32 - Contains fields that are not enclosed in double quotes.
Line: 34 - Contains fields that are not enclosed in double quotes.
Line: 35 - Contains fields that are not enclosed in double quotes.
Line: 36 - Contains fields that are not enclosed in double quotes.
Line: 37 - Contains fields that are not enclosed in double quotes.
## AGS Format Rule 3
Line: 32 - Does not start with a valid data descriptor.
Line: 35 - Does not start with a valid data descriptor.
Line: 37 - Does not start with a valid data descriptor.
## AGS Format Rule ?
Group: - Line 31 does not have the same number of entries as the HEADING row in TYPE.
================================================================================
""",
'nonsense.AGS': """
Expand Down
4 changes: 2 additions & 2 deletions test/unit/test_validation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,8 +164,8 @@ def test_validate_custom_dictionary_bad_file():


@pytest.mark.parametrize('filename', [
'example_ags.ags', 'nonsense.AGS', 'random_binary.ags',
'real/Blackburn Southern Bypass.ags'])
'example_ags.ags', 'example_broken_ags.ags', 'nonsense.AGS',
'random_binary.ags', 'real/Blackburn Southern Bypass.ags'])
def test_to_plain_text(filename):
# Arrange
response = JSON_RESPONSES[filename]
Expand Down

0 comments on commit f2c4ef3

Please sign in to comment.