Remove blob (#513)

* remove blob from PdsProduct responses * fix the get credential scheduled configuration. wip on terraform * remove blob fields from kvp responses --------- Co-authored-by: thomas loubrieu <[email protected]>
NASA-PDS · Jul 24, 2024 · d2a2895 · d2a2895
1 parent acc790f
commit d2a2895
Show file tree

Hide file tree

Showing 7 changed files with 70 additions and 41 deletions.
diff --git a/service/src/main/java/gov/nasa/pds/api/registry/SpringBootMain.java b/service/src/main/java/gov/nasa/pds/api/registry/SpringBootMain.java
@@ -7,6 +7,7 @@
 import org.springframework.boot.ExitCodeGenerator;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.context.annotation.AnnotationConfigApplicationContext;
 import org.springframework.context.annotation.ComponentScan;
 import io.swagger.v3.oas.annotations.OpenAPIDefinition;
@@ -17,6 +18,7 @@
 
 @SpringBootApplication
 @OpenAPIDefinition
+@EnableScheduling
 @ComponentScan(basePackages = {"gov.nasa.pds.api.registry.configuration",
     "gov.nasa.pds.api.registry.controllers", "gov.nasa.pds.api.registry.model",
     "gov.nasa.pds.api.registry.search", "javax.servlet.http"})

diff --git a/...rc/main/java/gov/nasa/pds/api/registry/model/api_responses/Pds4ProductBusinessObject.java b/...rc/main/java/gov/nasa/pds/api/registry/model/api_responses/Pds4ProductBusinessObject.java
@@ -95,6 +95,7 @@ public int setResponse(HitIterator hits, Summary summary, List<String> fields) {
     Set<String> uniqueProperties = new TreeSet<String>();
 
     for (Map<String, Object> kvp : hits) {
+
       uniqueProperties.addAll(getFilteredProperties(kvp, fields, null).keySet());
 
       try {

diff --git a/...src/main/java/gov/nasa/pds/api/registry/model/api_responses/PdsProductBusinessObject.java b/...src/main/java/gov/nasa/pds/api/registry/model/api_responses/PdsProductBusinessObject.java
@@ -1,6 +1,7 @@
 package gov.nasa.pds.api.registry.model.api_responses;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -56,7 +57,9 @@ public void setResponse(Map<String, Object> kvp, List<String> fields) {
 
     // TODO: findout why the getFilteredProperties method is used here. Should we add fields as a
     // second argument instead of null ?
-    product.setProperties((Map<String, List<String>>) getFilteredProperties(kvp, null, null));
+
+    product.setProperties(
+        (Map<String, List<String>>) getFilteredProperties(kvp, fields, excludedProperties));
     this.product = product;
   }
 
@@ -79,12 +82,13 @@ public void setResponse(List<Map<String, Object>> hits, Summary summary, List<St
 
     for (Map<String, Object> kvp : hits) {
       try {
-        uniqueProperties.addAll(getFilteredProperties(kvp, fields, null).keySet());
+        uniqueProperties.addAll(getFilteredProperties(kvp, fields, excludedProperties).keySet());
 
         products.addDataItem(SearchUtil.entityProductToAPIProduct(
             objectMapper.convertValue(kvp, EntityProduct.class), this.baseURL));
-        products.getData().get(products.getData().size() - 1).setProperties(
-            (Map<String, List<String>>) (Map<String, ?>) getFilteredProperties(kvp, null, null));
+        products.getData().get(products.getData().size() - 1)
+            .setProperties((Map<String, List<String>>) (Map<String, ?>) getFilteredProperties(kvp,
+                fields, excludedProperties));
       } catch (Throwable t) {
         String lidvid = "unknown";
         if (kvp.containsKey("lidvid")) {
@@ -111,12 +115,13 @@ public int setResponse(HitIterator hits, Summary summary, List<String> fields) {
 
     for (Map<String, Object> kvp : hits) {
       try {
-        uniqueProperties.addAll(getFilteredProperties(kvp, fields, null).keySet());
+        uniqueProperties.addAll(getFilteredProperties(kvp, fields, excludedProperties).keySet());
 
         products.addDataItem(SearchUtil.entityProductToAPIProduct(
             objectMapper.convertValue(kvp, EntityProduct.class), this.baseURL));
-        products.getData().get(products.getData().size() - 1).setProperties(
-            (Map<String, List<String>>) (Map<String, ?>) getFilteredProperties(kvp, null, null));
+        products.getData().get(products.getData().size() - 1)
+            .setProperties((Map<String, List<String>>) (Map<String, ?>) getFilteredProperties(kvp,
+                fields, excludedProperties));
       } catch (Throwable t) {
         String lidvid = "unknown";
         if (kvp.containsKey("lidvid")) {
@@ -144,14 +149,15 @@ public int setResponse(SearchHits hits, Summary summary, List<String> fields) {
     for (SearchHit hit : hits) {
       try {
         kvp = hit.getSourceAsMap();
-        uniqueProperties.addAll(getFilteredProperties(kvp, fields, null).keySet());
+        uniqueProperties.addAll(getFilteredProperties(kvp, fields, excludedProperties).keySet());
 
         products.addDataItem(SearchUtil.entityProductToAPIProduct(
             objectMapper.convertValue(kvp, EntityProduct.class), this.baseURL));
         // TODO: understand why the getFilteredProperties method is called with null arguments,
         // should we add fields or not call the method at all ?
-        products.getData().get(products.getData().size() - 1).setProperties(
-            (Map<String, List<String>>) (Map<String, ?>) getFilteredProperties(kvp, null, null));
+        products.getData().get(products.getData().size() - 1)
+            .setProperties((Map<String, List<String>>) (Map<String, ?>) getFilteredProperties(kvp,
+                null, excludedProperties));
       } catch (Throwable t) {
         log.error("DATA ERROR: could not convert opensearch document to EntityProduct for lidvid: "
             + hit.getId(), t);

diff --git a/...src/main/java/gov/nasa/pds/api/registry/model/api_responses/ProductBusinessLogicImpl.java b/...src/main/java/gov/nasa/pds/api/registry/model/api_responses/ProductBusinessLogicImpl.java
@@ -3,6 +3,7 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -14,6 +15,7 @@
 import org.springframework.web.context.request.RequestContextHolder;
 import org.springframework.web.context.request.ServletRequestAttributes;
 import gov.nasa.pds.api.registry.exceptions.UnsupportedSearchProperty;
+import gov.nasa.pds.api.registry.model.BlobUtil;
 import gov.nasa.pds.api.registry.model.SearchUtil;
 
 @Component
@@ -25,6 +27,19 @@ public abstract class ProductBusinessLogicImpl implements ProductBusinessLogic {
 
   protected URL baseURL;
 
+  protected List<String> excludedProperties = getExcludedProperties();
+
+  private static List<String> getExcludedProperties() {
+    List<String> excludedProperties = new ArrayList<String>();
+    try {
+      excludedProperties.add(SearchUtil.openPropertyToJsonProperty(BlobUtil.JSON_BLOB_PROPERTY));
+      excludedProperties.add(SearchUtil.openPropertyToJsonProperty(BlobUtil.XML_BLOB_PROPERTY));
+    } catch (UnsupportedSearchProperty e) {
+      log.error("That should not happen, unless there is an error in the code");
+    }
+    log.info("The following properties will not be sent in the API response " + excludedProperties);
+    return excludedProperties;
+  }
 
 
   public ProductBusinessLogicImpl() {
@@ -93,11 +108,13 @@ public static Map<String, List<String>> getFilteredProperties(Map<String, Object
 
     if ((included_fields == null) || (included_fields.size() == 0)) {
       String apiProperty;
+      log.debug("Excluded fields are " + excluded_fields);
       for (Map.Entry<String, Object> entry : sourceAsMap.entrySet()) {
         try {
           apiProperty = SearchUtil.openPropertyToJsonProperty(entry.getKey());
-          if ((excluded_fields == null) || !excluded_fields.contains(apiProperty))
+          if ((excluded_fields == null) || !excluded_fields.contains(apiProperty)) {
             filteredMapJsonProperties.put(apiProperty, object2PropertyValue(entry.getValue()));
+          }
         } catch (UnsupportedSearchProperty e) {
           log.warn("openSearch property " + entry.getKey() + " is not supported, ignored");
         }

diff --git a/...ce/src/main/java/gov/nasa/pds/api/registry/model/api_responses/WyriwygBusinessObject.java b/...ce/src/main/java/gov/nasa/pds/api/registry/model/api_responses/WyriwygBusinessObject.java
@@ -55,19 +55,9 @@ public void setObjectMapper(ObjectMapper om) {
   @Override
   public void setResponse(Map<String, Object> kvps, List<String> fields) {
     // TODO: to be implemented
-    WyriwygProduct product = new WyriwygProduct();
-    for (Entry<String, Object> pair : kvps.entrySet()) {
-      WyriwygProductKeyValuePair kvp = new WyriwygProductKeyValuePair();
-      try {
-        kvp.setKey(SearchUtil.openPropertyToJsonProperty(pair.getKey()));
-        kvp.setValue(getStringValueOf(pair.getValue()));
-        product.addKeyValuePairsItem(kvp);
-      } catch (UnsupportedSearchProperty e) {
-        log.warn("openSearch property " + pair.getKey() + " is not supported, ignored");
-      }
-    }
 
-    this.product = product;
+
+    this.product = generateWyriwygProduct(kvps, fields);
   }
 
   @Override
@@ -142,25 +132,39 @@ public void setResponse(List<Map<String, Object>> hits, Summary summary, List<St
 
     for (Map<String, Object> hit : hits) {
 
-      uniqueProperties.addAll(getFilteredProperties(hit, fields, null).keySet());
+      uniqueProperties.addAll(getFilteredProperties(hit, fields, excludedProperties).keySet());
+      WyriwygProduct product = generateWyriwygProduct(hit, fields);
+      products.addDataItem(product);
+    }
 
-      WyriwygProduct product = new WyriwygProduct();
-      for (Entry<String, Object> pair : hit.entrySet()) {
-        WyriwygProductKeyValuePair kvp = new WyriwygProductKeyValuePair();
-        try {
-          kvp.setKey(SearchUtil.openPropertyToJsonProperty(pair.getKey()));
+    summary.setProperties(new ArrayList<String>(uniqueProperties));
+    products.setSummary(summary);
+    this.products = products;
+  }
+
+
+  private WyriwygProduct generateWyriwygProduct(Map<String, Object> hit,
+      List<String> included_fields) {
+    WyriwygProduct product = new WyriwygProduct();
+    String jsonProperty;
+    for (Entry<String, Object> pair : hit.entrySet()) {
+      WyriwygProductKeyValuePair kvp = new WyriwygProductKeyValuePair();
+      try {
+        jsonProperty = SearchUtil.openPropertyToJsonProperty(pair.getKey());
+        if (!excludedProperties.contains(jsonProperty)) {
+          kvp.setKey(jsonProperty);
           kvp.setValue(getStringValueOf(pair.getValue()));
           product.addKeyValuePairsItem(kvp);
-        } catch (UnsupportedSearchProperty e) {
-          log.warn("openSearch property " + pair.getKey() + " is not supported, ignored");
         }
+      } catch (UnsupportedSearchProperty e) {
+        log.warn("openSearch property " + pair.getKey() + " is not supported, ignored");
       }
-      products.addDataItem(product);
+
     }
 
-    summary.setProperties(new ArrayList<String>(uniqueProperties));
-    products.setSummary(summary);
-    this.products = products;
+    return product;
+
+
   }
 
 

diff --git a/service/src/main/java/gov/nasa/pds/api/registry/util/AWSCredentialsFetcher.java b/service/src/main/java/gov/nasa/pds/api/registry/util/AWSCredentialsFetcher.java
@@ -8,8 +8,7 @@
 import java.util.concurrent.TimeUnit;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.stereotype.Component;
 import org.springframework.scheduling.annotation.Scheduled;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -55,8 +54,7 @@ public String getExpiration() {
 }
 
 
-@Configuration
-@EnableScheduling
+@Component
 public class AWSCredentialsFetcher {
 
   private static final Logger log = LoggerFactory.getLogger(AWSCredentialsFetcher.class);
@@ -86,6 +84,7 @@ public void fetchCredentials() {
         awsCredProperties.setProperty("aws.secretAccessKey", awsCredentials.getSecretAccessKey());
         awsCredProperties.setProperty("aws.sessionToken", awsCredentials.getToken());
         System.setProperties(awsCredProperties);
+        log.info("Expiration of the AWS token is scheduled in " + awsCredentials.expiration);
       } catch (IOException e) {
         log.error("Unable to get or renew AWS Credentials", e);
       }

diff --git a/terraform/ecs.tf b/terraform/ecs.tf
@@ -116,12 +116,12 @@ EOF
 
 resource "aws_lb" "registry-api-lb" {
   name               = "registry-api-lb-new"
-  internal           = true
+  internal           = false
   load_balancer_type = "application"
   security_groups    = var.aws_fg_security_groups
   subnets            = var.aws_fg_subnets
 
-  enable_deletion_protection = true
+  enable_deletion_protection = false
 
   access_logs {
     bucket  = var.aws_s3_bucket_logs_id
-Original file line number
+Diff line change
@@ Expand Up @@
         Set<String> uniqueProperties = new TreeSet<String>();
         for (Map<String, Object> kvp : hits) {
           uniqueProperties.addAll(getFilteredProperties(kvp, fields, null).keySet());
           try {
@@ Expand Down @@