From 3543ececc06731e95ce5c545c995e1fe7a5409c3 Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Thu, 25 Jan 2024 10:24:26 +0000 Subject: [PATCH] Fix parsing of doubles to conform to xsd:double spec --- .../resources/test-inputs/revision/invalid-double-column.csv | 2 +- .../test-inputs/revision/one-row-invalid-double.csv | 2 +- .../src/tpximpact/datahost/ldapi/util/data/validation.clj | 5 ++++- .../tpximpact/datahost/ldapi/util/data/validation_test.clj | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/datahost-ld-openapi/env/test/resources/test-inputs/revision/invalid-double-column.csv b/datahost-ld-openapi/env/test/resources/test-inputs/revision/invalid-double-column.csv index a0fede77..58ba6668 100644 --- a/datahost-ld-openapi/env/test/resources/test-inputs/revision/invalid-double-column.csv +++ b/datahost-ld-openapi/env/test/resources/test-inputs/revision/invalid-double-column.csv @@ -1,3 +1,3 @@ Year,Double,Other Double,Comment 1980,100.10,11.0,this row is ok -1980,101,12.0,this row has a number without a decimal in 2nd column \ No newline at end of file +1980,nan,12.0,this row has an invalid xsd:double NaN literal in it \ No newline at end of file diff --git a/datahost-ld-openapi/env/test/resources/test-inputs/revision/one-row-invalid-double.csv b/datahost-ld-openapi/env/test/resources/test-inputs/revision/one-row-invalid-double.csv index 42728bd8..b33e906f 100644 --- a/datahost-ld-openapi/env/test/resources/test-inputs/revision/one-row-invalid-double.csv +++ b/datahost-ld-openapi/env/test/resources/test-inputs/revision/one-row-invalid-double.csv @@ -1,2 +1,2 @@ Year,Double,Other Double,Comment -1999,100,10.0,this row has a double without a decimal point \ No newline at end of file +1999,nan,10.0,this row has an incorrectly formatted xsd:double NaN value in it \ No newline at end of file diff --git a/datahost-ld-openapi/src/tpximpact/datahost/ldapi/util/data/validation.clj b/datahost-ld-openapi/src/tpximpact/datahost/ldapi/util/data/validation.clj index d145a80d..cbabbfdc 100644 --- a/datahost-ld-openapi/src/tpximpact/datahost/ldapi/util/data/validation.clj +++ b/datahost-ld-openapi/src/tpximpact/datahost/ldapi/util/data/validation.clj @@ -238,7 +238,10 @@ (defn parse-double* "Returns a double or nil." [v] - (XMLDatatypeUtil/parseDouble v)) + (try + (XMLDatatypeUtil/parseDouble v) + (catch NumberFormatException ex + nil))) (defn- datatype+parse-fn "Returns a tuple of [datatype parse-fn] as required diff --git a/datahost-ld-openapi/test/tpximpact/datahost/ldapi/util/data/validation_test.clj b/datahost-ld-openapi/test/tpximpact/datahost/ldapi/util/data/validation_test.clj index 8fa300c5..a738b217 100644 --- a/datahost-ld-openapi/test/tpximpact/datahost/ldapi/util/data/validation_test.clj +++ b/datahost-ld-openapi/test/tpximpact/datahost/ldapi/util/data/validation_test.clj @@ -128,4 +128,4 @@ (let [ex (try (data.validation/as-dataset csv {:enforce-schema row-schema}) (catch clojure.lang.ExceptionInfo ex ex))] - (is (= {"Double" ["100"]} (:error-samples (ex-data ex)))))))))) + (is (= {"Double" ["nan"]} (:error-samples (ex-data ex))))))))))