From a0004e4a8ae68ea5da9d57175fed4d66b902023b Mon Sep 17 00:00:00 2001 From: Colin Blackburn Date: Wed, 6 Mar 2024 16:19:46 +0000 Subject: [PATCH 1/3] Add two missing rules --- app/schemas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/schemas.py b/app/schemas.py index b12af8d..c78477d 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -8,13 +8,13 @@ VALID_KEYS = [ # AGS schema rules 'AGS Format Rule 1', 'AGS Format Rule 2', 'AGS Format Rule 2a', 'AGS Format Rule 2b', - 'AGS Format Rule 2c', 'AGS Format Rule 3', 'AGS Format Rule 4a', 'AGS Format Rule 4b', + 'AGS Format Rule 2c', 'AGS Format Rule 3', 'AGS Format Rule 4', 'AGS Format Rule 4a', 'AGS Format Rule 4b', 'AGS Format Rule 5', 'AGS Format Rule 6', 'AGS Format Rule 7', 'AGS Format Rule 8', 'AGS Format Rule 9', 'AGS Format Rule 10a', 'AGS Format Rule 10b', 'AGS Format Rule 10c', 'AGS Format Rule 11a', 'AGS Format Rule 11b', 'AGS Format Rule 11c', 'AGS Format Rule 12', 'AGS Format Rule 13', 'AGS Format Rule 14', 'AGS Format Rule 15', 'AGS Format Rule 16', 'AGS Format Rule 17', 'AGS Format Rule 18', 'AGS Format Rule 19', 'AGS Format Rule 19a', - 'AGS Format Rule 19b', 'AGS Format Rule 20', 'General', + 'AGS Format Rule 19b', 'AGS Format Rule 20', 'AGS Format Rule ?', 'General', # Warnings and FYIs 'Warning (Related to Rule 16)', 'FYI (Related to Rule 1)', # Errors From 033d7d8deb3b7e43be980b28191525a9b102bf0e Mon Sep 17 00:00:00 2001 From: Colin Blackburn Date: Thu, 7 Mar 2024 08:57:37 +0000 Subject: [PATCH 2/3] Add test case for rules 3, 4 & 5 --- test/files/example_broken_ags.ags | 68 +++++++++++++++++++++++++ test/fixtures_json.py | 85 +++++++++++++++++++++++++++++++ test/fixtures_plain_text.py | 41 +++++++++++++++ test/unit/test_validation.py | 4 +- 4 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 test/files/example_broken_ags.ags diff --git a/test/files/example_broken_ags.ags b/test/files/example_broken_ags.ags new file mode 100644 index 0000000..1f0d9ed --- /dev/null +++ b/test/files/example_broken_ags.ags @@ -0,0 +1,68 @@ +"GROUP","PROJ" +"HEADING","PROJ_ID","PROJ_NAME","PROJ_LOC","PROJ_CLNT","PROJ_CONT","PROJ_ENG","PROJ_MEMO","FILE_FSET" +"UNIT","","","","","","","","" +"TYPE","X","X","X","X","X","X","X","X" +"DATA","121415","ACME Gas Works Redevelopment","Anytown","ACME Enterprises","ACME Drilling Ltd","","","" + +"GROUP","ABBR" +"HEADING","ABBR_HDNG","ABBR_CODE","ABBR_DESC","ABBR_LIST","ABBR_REM","FILE_FSET" +"UNIT","","","","","","" +"TYPE","X","X","X","X","X","X" +"DATA","DICT_TYPE","GROUP","Flag to indicate definition is a GROUP","","","" +"DATA","DICT_TYPE","HEADING","Flag to indicate definition is a HEADING","","","" +"DATA","DICT_STAT","OTHER","Other Field","","","" +"DATA","DICT_STAT","KEY","Key Field","","","" +"DATA","SAMP_TYPE","U","Undisturbed sample - open drive","","","" + +"GROUP","TRAN" +"HEADING","TRAN_ISNO","TRAN_DATE","TRAN_PROD","TRAN_STAT","TRAN_DESC","TRAN_AGS","TRAN_RECV","TRAN_DLIM","TRAN_RCON","TRAN_REM","FILE_FSET" +"UNIT","","yyyy-mm-dd","","","","","","","","","" +"TYPE","X","DT","X","X","X","X","X","X","X","X","X" +"DATA","1","2021-01-18","ACME Drilling Ltd","Preliminary","Draft Logs only","4.1","ACME Consulting","|","+","","" + +"GROUP","TYPE" +"HEADING","TYPE_TYPE","TYPE_DESC","FILE_FSET" +"UNIT","","","" +"TYPE","X","X","X" +"DATA","U","Undefined","" +"DATA","X","Text","" +"DATA","ID","Unique identifier","" +"DATA","PA","ABBR pick list","" +"DATA","2DP","Value; required +number of decimal places, 2","" +"DATA","DT","Date Time (ISO 8601:2004)","" +"DATA","0DP","Value; required +number of decimal places, 0","" +"DATA","1DP","Value; required +number of decimal places, 1","" +"DATA","PT","TYPE pick list","" +"DATA","PU","UNIT pick list","" +"DATA","DMS","Degrees:Minutes:Seconds","" +"DATA","T","Elapsed time","" +"DATA","RL","Record link","" + +"GROUP","UNIT" +"HEADING","UNIT_UNIT","UNIT_DESC","UNIT_REM","FILE_FSET" +"UNIT","","","","" +"TYPE","X","X","X","X" +"DATA","-","No unit","","" +"DATA","m","metre","","" +"DATA","yyyy-mm-dd","Date (ISO8601)","","" +"DATA","yyyy-mm-ddThh:mm:ss.sss","Date Time (ISO8601)","","" +"DATA","mm","millimetre","","" +"DATA","%","percentage","","" +"DATA","l/min","litres per minute","","" +"DATA","hh:mm:ss","hh:mm:ss","","" + +"GROUP","LOCA" +"HEADING","LOCA_ID","LOCA_TYPE","LOCA_STAT","LOCA_NATE","LOCA_NATN","LOCA_GREF","LOCA_GL","LOCA_REM","LOCA_FDEP","LOCA_STAR","LOCA_PURP","LOCA_TERM","LOCA_ENDD","LOCA_LETT","LOCA_LOCX","LOCA_LOCY","LOCA_LOCZ","LOCA_LREF","LOCA_DATM","LOCA_ETRV","LOCA_NTRV","LOCA_LTRV","LOCA_XTRL","LOCA_YTRL","LOCA_ZTRL","LOCA_LAT","LOCA_LON","LOCA_ELAT","LOCA_ELON","LOCA_LLZ","LOCA_LOCM","LOCA_LOCA","LOCA_CLST","LOCA_ALID","LOCA_OFFS","LOCA_CNGE","LOCA_TRAN","FILE_FSET" +"UNIT","","","","m","m","","m","","m","yyyy-mm-dd","","","yyyy-mm-dd","","m","m","m","","","m","m","m","m","m","m","","","","","","","","","","","","","" +"TYPE","ID","PA","PA","2DP","2DP","PA","2DP","X","2DP","DT","X","X","DT","X","2DP","2DP","2DP","X","X","2DP","2DP","2DP","2DP","2DP","2DP","DMS","DMS","DMS","DMS","X","X","X","X","X","2DP","X","X","X" +"DATA","327-16A","","","523145.00","178456.12","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","" + +"GROUP","SAMP" +"HEADING","LOCA_ID","SAMP_TOP","SAMP_REF","SAMP_TYPE","SAMP_ID","SAMP_BASE","SAMP_DTIM","SAMP_UBLO","SAMP_CONT","SAMP_PREP","SAMP_SDIA","SAMP_WDEP","SAMP_RECV","SAMP_TECH","SAMP_MATX","SAMP_TYPC","SAMP_WHO","SAMP_WHY","SAMP_REM","SAMP_DESC","SAMP_DESD","SAMP_LOG","SAMP_COND","SAMP_CLSS","SAMP_BAR","SAMP_TEMP","SAMP_PRES","SAMP_FLOW","SAMP_ETIM","SAMP_DURN","SAMP_CAPT","SAMP_LINK","GEOL_STAT","FILE_FSET" +"UNIT","","m","","","","m","yyyy-mm-ddThh:mm:ss.sss","","","","mm","m","%","","","","","","","","yyyy-mm-dd","","","","","","","l/min","yyyy-mm-ddThh:mm:ss.sss","hh:mm:ss","","","","" +"TYPE","ID","2DP","X","PA","ID","2DP","DT","0DP","X","X","0DP","X","0DP","X","X","X","X","X","X","X","DT","X","X","X","1DP","0DP","1DP","1DP","DT","T","X","X","X","X" +"DATA","327-16A","24.55","24","U","ABC121415010","25.00","","","","","","","","","","","","","","","","","","","","","","","","","","","","" + diff --git a/test/fixtures_json.py b/test/fixtures_json.py index 596d9ae..309ddeb 100644 --- a/test/fixtures_json.py +++ b/test/fixtures_json.py @@ -12,6 +12,91 @@ 'valid': True, 'additional_metadata': {} }, + 'example_broken_ags.ags': { + "filename": "example_broken_ags.ags", + "filesize": 4111, + "checkers": ["python_ags4 v0.5.0"], + 'dictionary': 'Standard_dictionary_v4_1_1.ags', + 'time': dt.datetime(2021, 8, 23, 14, 25, 43, tzinfo=dt.timezone.utc), + "message": "13 error(s) found in file!", + "errors": { + "AGS Format Rule 4": [ + { + "line": 31, + "group": "TYPE", + "desc": "Number of fields does not match the HEADING row." + }, + { + "line": 34, + "group": "TYPE", + "desc": "Number of fields does not match the HEADING row." + }, + { + "line": 36, + "group": "TYPE", + "desc": "Number of fields does not match the HEADING row." + } + ], + "AGS Format Rule 5": [ + { + "line": 31, + "group": "", + "desc": "Contains fields that are not enclosed in double quotes." + }, + { + "line": 32, + "group": "", + "desc": "Contains fields that are not enclosed in double quotes." + }, + { + "line": 34, + "group": "", + "desc": "Contains fields that are not enclosed in double quotes." + }, + { + "line": 35, + "group": "", + "desc": "Contains fields that are not enclosed in double quotes." + }, + { + "line": 36, + "group": "", + "desc": "Contains fields that are not enclosed in double quotes." + }, + { + "line": 37, + "group": "", + "desc": "Contains fields that are not enclosed in double quotes." + } + ], + "AGS Format Rule 3": [ + { + "line": 32, + "group": "", + "desc": "Does not start with a valid data descriptor." + }, + { + "line": 35, + "group": "", + "desc": "Does not start with a valid data descriptor." + }, + { + "line": 37, + "group": "", + "desc": "Does not start with a valid data descriptor." + } + ], + "AGS Format Rule ?": [ + { + "line": "-", + "group": "", + "desc": "Line 31 does not have the same number of entries as the HEADING row in TYPE." + } + ], + }, + "valid": False, + 'additional_metadata': {} + }, 'nonsense.AGS': { 'filename': 'nonsense.AGS', 'filesize': 9, diff --git a/test/fixtures_plain_text.py b/test/fixtures_plain_text.py index becc188..119a185 100644 --- a/test/fixtures_plain_text.py +++ b/test/fixtures_plain_text.py @@ -12,6 +12,47 @@ Time: 2021-08-23 14:25:43+00:00 +================================================================================ +""", + 'example_broken_ags.ags': """ +================================================================================ +example_broken_ags.ags: 13 error(s) found in file! + +# Metadata + +File size: 4111 bytes +Checkers: ['python_ags4 v0.5.0'] +Dictionary: Standard_dictionary_v4_1_1.ags +Time: 2021-08-23 14:25:43+00:00 + + +# Errors + +## AGS Format Rule 4 + +Line: 31 - Number of fields does not match the HEADING row. +Line: 34 - Number of fields does not match the HEADING row. +Line: 36 - Number of fields does not match the HEADING row. + +## AGS Format Rule 5 + +Line: 31 - Contains fields that are not enclosed in double quotes. +Line: 32 - Contains fields that are not enclosed in double quotes. +Line: 34 - Contains fields that are not enclosed in double quotes. +Line: 35 - Contains fields that are not enclosed in double quotes. +Line: 36 - Contains fields that are not enclosed in double quotes. +Line: 37 - Contains fields that are not enclosed in double quotes. + +## AGS Format Rule 3 + +Line: 32 - Does not start with a valid data descriptor. +Line: 35 - Does not start with a valid data descriptor. +Line: 37 - Does not start with a valid data descriptor. + +## AGS Format Rule ? + +Group: - Line 31 does not have the same number of entries as the HEADING row in TYPE. + ================================================================================ """, 'nonsense.AGS': """ diff --git a/test/unit/test_validation.py b/test/unit/test_validation.py index 4854ec2..2716409 100644 --- a/test/unit/test_validation.py +++ b/test/unit/test_validation.py @@ -164,8 +164,8 @@ def test_validate_custom_dictionary_bad_file(): @pytest.mark.parametrize('filename', [ - 'example_ags.ags', 'nonsense.AGS', 'random_binary.ags', - 'real/Blackburn Southern Bypass.ags']) + 'example_ags.ags', 'example_broken_ags.ags', 'nonsense.AGS', + 'random_binary.ags', 'real/Blackburn Southern Bypass.ags']) def test_to_plain_text(filename): # Arrange response = JSON_RESPONSES[filename] From 09a6ca35255110ef47c42d34820c5c7143e68f16 Mon Sep 17 00:00:00 2001 From: Colin Blackburn Date: Thu, 7 Mar 2024 08:58:20 +0000 Subject: [PATCH 3/3] Exception no longer thrown --- app/checkers.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/checkers.py b/app/checkers.py index ed858a5..a473626 100644 --- a/app/checkers.py +++ b/app/checkers.py @@ -42,10 +42,6 @@ def check_ags(filename: Path, standard_AGS4_dictionary: Optional[str] = None) -> description = f"UnicodeDecodeError: {err.reason}" errors = {'File read error': [{'line': line_no, 'group': '', 'desc': description}]} dictionary = '' - except AGS4.AGS4Error as err: - description = f"AGS4Error: {err}" - errors = {'File read error': [{'line': '-', 'group': '', 'desc': description}]} - dictionary = '' # Discard unecessary summary from errors dictionary errors.pop('Summary of data', None)