From e1f2dce17a7e787a8489cb65c1c7edfaa6907290 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 3 Aug 2023 09:10:39 -0700 Subject: [PATCH 1/4] add try/catch The general request is to support bad data in the database so add try/catch blocks around the conversions of the document such that processing continues. Catch all exceptions, log them, and press forward. --- .../model/Pds4ProductBusinessObject.java | 23 +++++++-- .../model/PdsProductBusinessObject.java | 49 ++++++++++++------- 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java index 8d95a601..5803980e 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java @@ -8,6 +8,8 @@ import java.util.TreeSet; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import gov.nasa.pds.api.registry.search.HitIterator; import gov.nasa.pds.model.Pds4Product; @@ -16,6 +18,7 @@ public class Pds4ProductBusinessObject extends ProductBusinessLogicImpl { + private static final Logger log = LoggerFactory.getLogger(Pds4ProductBusinessObject.class); @SuppressWarnings("unused") private ObjectMapper objectMapper; private Pds4Product product = null; @@ -87,8 +90,16 @@ public int setResponse(HitIterator hits, Summary summary, List fields) { uniqueProperties .addAll(ProductBusinessObject.getFilteredProperties(kvp, fields, null).keySet()); - Pds4Product prod = Pds4ProductFactory.createProduct(hits.getCurrentId(), kvp, this.isJSON); - list.add(prod); + try { + Pds4Product prod = Pds4ProductFactory.createProduct(hits.getCurrentId(), kvp, this.isJSON); + list.add(prod); + } catch (Throwable t) { + String lidvid = "unknown"; + if (kvp.containsKey("lidvid")) { + lidvid = kvp.get("lidvid").toString(); + } + log.error ("CRITICAL: could not convert opensearch document to Pds4Product for lidvid: " + lidvid); + } } products.setData(list); @@ -112,8 +123,12 @@ public int setResponse(SearchHits hits, Summary summary, List fields) { uniqueProperties .addAll(ProductBusinessObject.getFilteredProperties(fieldMap, fields, null).keySet()); - Pds4Product prod = Pds4ProductFactory.createProduct(id, fieldMap, this.isJSON); - list.add(prod); + try { + Pds4Product prod = Pds4ProductFactory.createProduct(id, fieldMap, this.isJSON); + list.add(prod); + } catch (Throwable t) { + log.error ("CRITICAL: could not convert opensearch document to Pds4Product for lidvid: " + hit.getId()); + } } products.setData(list); products.setSummary(summary); diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java b/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java index 6b6b5a2e..4b1fa03e 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java @@ -7,6 +7,8 @@ import java.util.TreeSet; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import gov.nasa.pds.api.registry.search.HitIterator; import gov.nasa.pds.model.PdsProduct; @@ -15,6 +17,7 @@ public class PdsProductBusinessObject extends ProductBusinessLogicImpl { + private static final Logger log = LoggerFactory.getLogger(ProductBusinessObject.class); private ObjectMapper objectMapper; private PdsProduct product = null; private PdsProducts products = null; @@ -62,14 +65,22 @@ public int setResponse(HitIterator hits, Summary summary, List fields) { Set uniqueProperties = new TreeSet(); for (Map kvp : hits) { - uniqueProperties - .addAll(ProductBusinessObject.getFilteredProperties(kvp, fields, null).keySet()); - - products.addDataItem(SearchUtil.entityProductToAPIProduct( - objectMapper.convertValue(kvp, EntityProduct.class), this.baseURL)); - products.getData().get(products.getData().size() - 1) - .setProperties((Map>) (Map) ProductBusinessObject - .getFilteredProperties(kvp, null, null)); + try { + uniqueProperties + .addAll(ProductBusinessObject.getFilteredProperties(kvp, fields, null).keySet()); + + products.addDataItem(SearchUtil.entityProductToAPIProduct( + objectMapper.convertValue(kvp, EntityProduct.class), this.baseURL)); + products.getData().get(products.getData().size() - 1) + .setProperties((Map>) (Map) ProductBusinessObject + .getFilteredProperties(kvp, null, null)); + } catch (Throwable t) { + String lidvid = "unknown"; + if (kvp.containsKey("lidvid")) { + lidvid = kvp.get("lidvid").toString(); + } + log.error ("CRITICAL: could not convert opensearch document to EntityProduct for lidvid: " + lidvid); + } } count = products.getData().size(); @@ -87,15 +98,19 @@ public int setResponse(SearchHits hits, Summary summary, List fields) { Set uniqueProperties = new TreeSet(); for (SearchHit hit : hits) { - kvp = hit.getSourceAsMap(); - uniqueProperties - .addAll(ProductBusinessObject.getFilteredProperties(kvp, fields, null).keySet()); - - products.addDataItem(SearchUtil.entityProductToAPIProduct( - objectMapper.convertValue(kvp, EntityProduct.class), this.baseURL)); - products.getData().get(products.getData().size() - 1) - .setProperties((Map>) (Map) ProductBusinessObject - .getFilteredProperties(kvp, null, null)); + try { + kvp = hit.getSourceAsMap(); + uniqueProperties + .addAll(ProductBusinessObject.getFilteredProperties(kvp, fields, null).keySet()); + + products.addDataItem(SearchUtil.entityProductToAPIProduct( + objectMapper.convertValue(kvp, EntityProduct.class), this.baseURL)); + products.getData().get(products.getData().size() - 1) + .setProperties((Map>) (Map) ProductBusinessObject + .getFilteredProperties(kvp, null, null)); + } catch (Throwable t) { + log.error("CRITICAL: could not convert opensearch document to EntityProduct for lidvid: " + hit.getId(), t); + } } summary.setProperties(new ArrayList(uniqueProperties)); From d7de3d9eb41eeab3d40dd71b205ce628b1fecfb0 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 3 Aug 2023 09:12:08 -0700 Subject: [PATCH 2/4] show stack trace as well --- .../pds/api/registry/model/Pds4ProductBusinessObject.java | 4 ++-- .../nasa/pds/api/registry/model/PdsProductBusinessObject.java | 2 +- service/src/main/resources/application.properties | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java index 5803980e..b7592232 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java @@ -98,7 +98,7 @@ public int setResponse(HitIterator hits, Summary summary, List fields) { if (kvp.containsKey("lidvid")) { lidvid = kvp.get("lidvid").toString(); } - log.error ("CRITICAL: could not convert opensearch document to Pds4Product for lidvid: " + lidvid); + log.error ("CRITICAL: could not convert opensearch document to Pds4Product for lidvid: " + lidvid, t); } } @@ -127,7 +127,7 @@ public int setResponse(SearchHits hits, Summary summary, List fields) { Pds4Product prod = Pds4ProductFactory.createProduct(id, fieldMap, this.isJSON); list.add(prod); } catch (Throwable t) { - log.error ("CRITICAL: could not convert opensearch document to Pds4Product for lidvid: " + hit.getId()); + log.error ("CRITICAL: could not convert opensearch document to Pds4Product for lidvid: " + hit.getId(), t); } } products.setData(list); diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java b/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java index 4b1fa03e..1b9d4116 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java @@ -79,7 +79,7 @@ public int setResponse(HitIterator hits, Summary summary, List fields) { if (kvp.containsKey("lidvid")) { lidvid = kvp.get("lidvid").toString(); } - log.error ("CRITICAL: could not convert opensearch document to EntityProduct for lidvid: " + lidvid); + log.error ("CRITICAL: could not convert opensearch document to EntityProduct for lidvid: " + lidvid, t); } } count = products.getData().size(); diff --git a/service/src/main/resources/application.properties b/service/src/main/resources/application.properties index f2e92257..729c1a52 100644 --- a/service/src/main/resources/application.properties +++ b/service/src/main/resources/application.properties @@ -36,7 +36,7 @@ openSearch.username=admin openSearch.password=admin openSearch.ssl=true # use only for development purpose, left it to true otherwise -openSearch.sslCertificateCNVerification=true +openSearch.sslCertificateCNVerification=false # Only show products with following archive statuses filter.archiveStatus=archived,certified From ad5c49611685429b7a2f89ee2f8a0fd861b8e658 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 3 Aug 2023 10:25:13 -0700 Subject: [PATCH 3/4] undo mistake --- service/src/main/resources/application.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/resources/application.properties b/service/src/main/resources/application.properties index 729c1a52..f2e92257 100644 --- a/service/src/main/resources/application.properties +++ b/service/src/main/resources/application.properties @@ -36,7 +36,7 @@ openSearch.username=admin openSearch.password=admin openSearch.ssl=true # use only for development purpose, left it to true otherwise -openSearch.sslCertificateCNVerification=false +openSearch.sslCertificateCNVerification=true # Only show products with following archive statuses filter.archiveStatus=archived,certified From 0f89a7be85c4cd64bee1aaf1279e425dc2f40957 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 3 Aug 2023 10:56:59 -0700 Subject: [PATCH 4/4] change critical to data error --- .../pds/api/registry/model/Pds4ProductBusinessObject.java | 4 ++-- .../nasa/pds/api/registry/model/PdsProductBusinessObject.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java index b7592232..edc16c42 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java @@ -98,7 +98,7 @@ public int setResponse(HitIterator hits, Summary summary, List fields) { if (kvp.containsKey("lidvid")) { lidvid = kvp.get("lidvid").toString(); } - log.error ("CRITICAL: could not convert opensearch document to Pds4Product for lidvid: " + lidvid, t); + log.error ("DATA ERROR: could not convert opensearch document to Pds4Product for lidvid: " + lidvid, t); } } @@ -127,7 +127,7 @@ public int setResponse(SearchHits hits, Summary summary, List fields) { Pds4Product prod = Pds4ProductFactory.createProduct(id, fieldMap, this.isJSON); list.add(prod); } catch (Throwable t) { - log.error ("CRITICAL: could not convert opensearch document to Pds4Product for lidvid: " + hit.getId(), t); + log.error ("DATA ERROR: could not convert opensearch document to Pds4Product for lidvid: " + hit.getId(), t); } } products.setData(list); diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java b/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java index 1b9d4116..b6fd3c96 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java @@ -79,7 +79,7 @@ public int setResponse(HitIterator hits, Summary summary, List fields) { if (kvp.containsKey("lidvid")) { lidvid = kvp.get("lidvid").toString(); } - log.error ("CRITICAL: could not convert opensearch document to EntityProduct for lidvid: " + lidvid, t); + log.error ("DATA ERROR: could not convert opensearch document to EntityProduct for lidvid: " + lidvid, t); } } count = products.getData().size(); @@ -109,7 +109,7 @@ public int setResponse(SearchHits hits, Summary summary, List fields) { .setProperties((Map>) (Map) ProductBusinessObject .getFilteredProperties(kvp, null, null)); } catch (Throwable t) { - log.error("CRITICAL: could not convert opensearch document to EntityProduct for lidvid: " + hit.getId(), t); + log.error("DATA ERROR: could not convert opensearch document to EntityProduct for lidvid: " + hit.getId(), t); } }