Issue 361: ignore data conversion errors (#368)

* add try/catch The general request is to support bad data in the database so add try/catch blocks around the conversions of the document such that processing continues. Catch all exceptions, log them, and press forward. * show stack trace as well * undo mistake * change critical to data error --------- Co-authored-by: Al Niessner <[email protected]>
NASA-PDS · Aug 14, 2023 · abf291d · abf291d
1 parent 0098bea
commit abf291d
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 21 deletions.
diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Pds4ProductBusinessObject.java
@@ -8,6 +8,8 @@
 import java.util.TreeSet;
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.SearchHits;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import gov.nasa.pds.api.registry.search.HitIterator;
 import gov.nasa.pds.model.Pds4Product;
@@ -16,6 +18,7 @@
 
 
 public class Pds4ProductBusinessObject extends ProductBusinessLogicImpl {
+  private static final Logger log = LoggerFactory.getLogger(Pds4ProductBusinessObject.class);
   @SuppressWarnings("unused")
   private ObjectMapper objectMapper;
   private Pds4Product product = null;
@@ -87,8 +90,16 @@ public int setResponse(HitIterator hits, Summary summary, List<String> fields) {
       uniqueProperties
           .addAll(ProductBusinessObject.getFilteredProperties(kvp, fields, null).keySet());
 
-      Pds4Product prod = Pds4ProductFactory.createProduct(hits.getCurrentId(), kvp, this.isJSON);
-      list.add(prod);
+      try {
+        Pds4Product prod = Pds4ProductFactory.createProduct(hits.getCurrentId(), kvp, this.isJSON);
+        list.add(prod);
+      } catch (Throwable t) {
+        String lidvid = "unknown";
+        if (kvp.containsKey("lidvid")) {
+          lidvid = kvp.get("lidvid").toString();
+        }
+        log.error ("DATA ERROR: could not convert opensearch document to Pds4Product for lidvid: " + lidvid, t);
+      }
     }
 
     products.setData(list);
@@ -112,8 +123,12 @@ public int setResponse(SearchHits hits, Summary summary, List<String> fields) {
       uniqueProperties
           .addAll(ProductBusinessObject.getFilteredProperties(fieldMap, fields, null).keySet());
 
-      Pds4Product prod = Pds4ProductFactory.createProduct(id, fieldMap, this.isJSON);
-      list.add(prod);
+      try {
+        Pds4Product prod = Pds4ProductFactory.createProduct(id, fieldMap, this.isJSON);
+        list.add(prod);
+      } catch (Throwable t) {
+        log.error ("DATA ERROR: could not convert opensearch document to Pds4Product for lidvid: " + hit.getId(), t);
+      }
     }
     products.setData(list);
     products.setSummary(summary);

diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java b/service/src/main/java/gov/nasa/pds/api/registry/model/PdsProductBusinessObject.java
@@ -7,6 +7,8 @@
 import java.util.TreeSet;
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.SearchHits;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import gov.nasa.pds.api.registry.search.HitIterator;
 import gov.nasa.pds.model.PdsProduct;
@@ -15,6 +17,7 @@
 
 
 public class PdsProductBusinessObject extends ProductBusinessLogicImpl {
+  private static final Logger log = LoggerFactory.getLogger(ProductBusinessObject.class);
   private ObjectMapper objectMapper;
   private PdsProduct product = null;
   private PdsProducts products = null;
@@ -62,14 +65,22 @@ public int setResponse(HitIterator hits, Summary summary, List<String> fields) {
     Set<String> uniqueProperties = new TreeSet<String>();
 
     for (Map<String, Object> kvp : hits) {
-      uniqueProperties
-          .addAll(ProductBusinessObject.getFilteredProperties(kvp, fields, null).keySet());
-
-      products.addDataItem(SearchUtil.entityProductToAPIProduct(
-          objectMapper.convertValue(kvp, EntityProduct.class), this.baseURL));
-      products.getData().get(products.getData().size() - 1)
-          .setProperties((Map<String, List<String>>) (Map<String, ?>) ProductBusinessObject
-              .getFilteredProperties(kvp, null, null));
+      try {
+        uniqueProperties
+            .addAll(ProductBusinessObject.getFilteredProperties(kvp, fields, null).keySet());
+
+        products.addDataItem(SearchUtil.entityProductToAPIProduct(
+            objectMapper.convertValue(kvp, EntityProduct.class), this.baseURL));
+        products.getData().get(products.getData().size() - 1)
+            .setProperties((Map<String, List<String>>) (Map<String, ?>) ProductBusinessObject
+                .getFilteredProperties(kvp, null, null));
+      } catch (Throwable t) {
+        String lidvid = "unknown";
+        if (kvp.containsKey("lidvid")) {
+          lidvid = kvp.get("lidvid").toString();
+        }
+        log.error ("DATA ERROR: could not convert opensearch document to EntityProduct for lidvid: " + lidvid, t);
+      }
     }
     count = products.getData().size();
 
@@ -87,15 +98,19 @@ public int setResponse(SearchHits hits, Summary summary, List<String> fields) {
     Set<String> uniqueProperties = new TreeSet<String>();
 
     for (SearchHit hit : hits) {
-      kvp = hit.getSourceAsMap();
-      uniqueProperties
-          .addAll(ProductBusinessObject.getFilteredProperties(kvp, fields, null).keySet());
-
-      products.addDataItem(SearchUtil.entityProductToAPIProduct(
-          objectMapper.convertValue(kvp, EntityProduct.class), this.baseURL));
-      products.getData().get(products.getData().size() - 1)
-          .setProperties((Map<String, List<String>>) (Map<String, ?>) ProductBusinessObject
-              .getFilteredProperties(kvp, null, null));
+      try {
+        kvp = hit.getSourceAsMap();
+        uniqueProperties
+            .addAll(ProductBusinessObject.getFilteredProperties(kvp, fields, null).keySet());
+
+        products.addDataItem(SearchUtil.entityProductToAPIProduct(
+            objectMapper.convertValue(kvp, EntityProduct.class), this.baseURL));
+        products.getData().get(products.getData().size() - 1)
+            .setProperties((Map<String, List<String>>) (Map<String, ?>) ProductBusinessObject
+                .getFilteredProperties(kvp, null, null));
+        } catch (Throwable t) {
+          log.error("DATA ERROR: could not convert opensearch document to EntityProduct for lidvid: " + hit.getId(), t);
+        }
     }
 
     summary.setProperties(new ArrayList<String>(uniqueProperties));