Address search request timeouts as transient error (#561) (#641)

Signed-off-by: jowg-amazon <[email protected]> (cherry picked from commit 3aa2844) Co-authored-by: Joanne Wang <[email protected]>
opensearch-project · Oct 5, 2023 · 1b8ec5d · 1b8ec5d
1 parent f733685
commit 1b8ec5d
Show file tree

Hide file tree

Showing 13 changed files with 26 additions and 26 deletions.
diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java b/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java
@@ -228,7 +228,7 @@ private void onAutoCorrelations(Detector detector, Finding finding, Map<String,
             @Override
             public void onResponse(SearchResponse response) {
                 if (response.isTimedOut()) {
-                    correlateFindingAction.onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT));
+                    correlateFindingAction.onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                 }
 
                 Iterator<SearchHit> hits = response.getHits().iterator();

diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java b/src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java
@@ -81,7 +81,7 @@ public void insertCorrelatedFindings(String detectorType, Finding finding, Strin
             @Override
             public void onResponse(SearchResponse response) {
                 if (response.isTimedOut()) {
-                    correlateFindingAction.onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT));
+                    correlateFindingAction.onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                 }
 
                 Map<String, Object> hitSource = response.getHits().getHits()[0].getSourceAsMap();
@@ -249,7 +249,7 @@ public void insertOrphanFindings(String detectorType, Finding finding, float tim
             @Override
             public void onResponse(SearchResponse response) {
                 if (response.isTimedOut()) {
-                    correlateFindingAction.onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT));
+                    correlateFindingAction.onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                 }
 
                 try {
@@ -425,7 +425,7 @@ public void onFailure(Exception e) {
                                 @Override
                                 public void onResponse(SearchResponse response) {
                                     if (response.isTimedOut()) {
-                                        correlateFindingAction.onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT));
+                                        correlateFindingAction.onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                                     }
 
                                     long totalHits = response.getHits().getTotalHits().value;

diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
@@ -279,7 +279,7 @@ private void doIndexLogTypeMetadata(ActionListener<Void> listener) {
             @Override
             public void onResponse(SearchResponse response) {
                 if (response.isTimedOut()) {
-                    listener.onFailure(new OpenSearchStatusException("Unknown error", RestStatus.INTERNAL_SERVER_ERROR));
+                    listener.onFailure(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                 }
                 if (response.getHits().getTotalHits().value > 0) {
                     listener.onResponse(null);

diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestSearchRuleAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestSearchRuleAction.java
@@ -81,7 +81,7 @@ private RestResponseListener<SearchResponse> searchRuleResponse(RestChannel chan
             @Override
             public RestResponse buildResponse(SearchResponse response) throws Exception {
                 if (response.isTimedOut()) {
-                    return new BytesRestResponse(RestStatus.REQUEST_TIMEOUT, response.toString());
+                    return new BytesRestResponse(RestStatus.REQUEST_TIMEOUT, "Search request timed out");
                 }
 
                 try {

diff --git a/...main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java b/...main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java
@@ -223,7 +223,7 @@ void start() {
                     @Override
                     public void onResponse(SearchResponse response) {
                         if (response.isTimedOut()) {
-                            onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT));
+                            onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                         }
 
                         SearchHits hits = response.getHits();
@@ -336,7 +336,7 @@ public void onResponse(IndexResponse response) {
                                         @Override
                                         public void onResponse(SearchResponse response) {
                                             if (response.isTimedOut()) {
-                                                onFailures(new OpenSearchStatusException(response.toString(), RestStatus.INTERNAL_SERVER_ERROR));
+                                                onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                                             }
 
                                             SearchHit[] hits = response.getHits().getHits();
@@ -392,7 +392,7 @@ public void onFailure(Exception e) {
                             @Override
                             public void onResponse(SearchResponse response) {
                                 if (response.isTimedOut()) {
-                                    onFailures(new OpenSearchStatusException(response.toString(), RestStatus.INTERNAL_SERVER_ERROR));
+                                    onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                                 }
 
                                 SearchHit[] hits = response.getHits().getHits();

diff --git a/...java/org/opensearch/securityanalytics/transport/TransportDeleteCorrelationRuleAction.java b/...java/org/opensearch/securityanalytics/transport/TransportDeleteCorrelationRuleAction.java
@@ -65,10 +65,10 @@ public void onResponse(BulkByScrollResponse response) {
                                     new OpenSearchStatusException(
                                         String.format(
                                                 Locale.getDefault(),
-                                                "Correlation Rule with id %s cannot be deleted",
+                                                "Request timed out. Correlation Rule with id %s cannot be deleted",
                                                 correlationRuleId
                                         ),
-                                        RestStatus.INTERNAL_SERVER_ERROR)
+                                        RestStatus.REQUEST_TIMEOUT)
                             );
                             return;
                         }

diff --git a/...n/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java b/...n/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java
@@ -174,7 +174,7 @@ private void onGetResponse(CustomLogType logType) {
                     @Override
                     public void onResponse(SearchResponse response) {
                         if (response.isTimedOut()) {
-                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR));
+                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be deleted", logType.getId()), RestStatus.REQUEST_TIMEOUT));
                             return;
                         }
 
@@ -187,7 +187,7 @@ public void onResponse(SearchResponse response) {
                             @Override
                             public void onResponse(SearchResponse response) {
                                 if (response.isTimedOut()) {
-                                    onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR));
+                                    onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be deleted", logType.getId()), RestStatus.REQUEST_TIMEOUT));
                                     return;
                                 }
 

diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteRuleAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteRuleAction.java
@@ -152,7 +152,7 @@ private void onGetResponse(Rule rule) {
                     @Override
                     public void onResponse(SearchResponse response) {
                         if (response.isTimedOut()) {
-                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Rule with id %s cannot be deleted", rule.getId()), RestStatus.INTERNAL_SERVER_ERROR));
+                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Rule with id %s cannot be deleted", rule.getId()), RestStatus.REQUEST_TIMEOUT));
                             return;
                         }
 
@@ -231,7 +231,7 @@ private void deleteRule(String ruleId) {
                     @Override
                     public void onResponse(BulkByScrollResponse response) {
                         if (response.isTimedOut()) {
-                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Rule with id %s cannot be deleted", ruleId), RestStatus.INTERNAL_SERVER_ERROR));
+                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Request timed out. Rule with id %s cannot be deleted", ruleId), RestStatus.REQUEST_TIMEOUT));
                             return;
                         }
 

diff --git a/...in/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java b/...in/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java
@@ -218,7 +218,7 @@ private void prepareCustomLogTypeIndexing() throws IOException {
                     @Override
                     public void onResponse(SearchResponse response) {
                         if (response.isTimedOut()) {
-                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR));
+                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be updated", logTypeId), RestStatus.REQUEST_TIMEOUT));
                             return;
                         }
 
@@ -243,7 +243,7 @@ public void onResponse(SearchResponse response) {
                                         @Override
                                         public void onResponse(SearchResponse response) {
                                             if (response.isTimedOut()) {
-                                                onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR));
+                                                onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be updated", logTypeId), RestStatus.REQUEST_TIMEOUT));
                                                 return;
                                             }
 
@@ -256,7 +256,7 @@ public void onResponse(SearchResponse response) {
                                                 @Override
                                                 public void onResponse(SearchResponse response) {
                                                     if (response.isTimedOut()) {
-                                                        onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR));
+                                                        onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be updated", logTypeId), RestStatus.REQUEST_TIMEOUT));
                                                         return;
                                                     }
 
@@ -379,7 +379,7 @@ public void onResponse(Void unused) {
                             @Override
                             public void onResponse(SearchResponse response) {
                                 if (response.isTimedOut()) {
-                                    onFailures(new OpenSearchStatusException(response.toString(), RestStatus.INTERNAL_SERVER_ERROR));
+                                    onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                                     return;
                                 }
 
@@ -399,7 +399,7 @@ public void onResponse(SearchResponse response) {
                                     @Override
                                     public void onResponse(SearchResponse response) {
                                         if (response.isTimedOut()) {
-                                            onFailures(new OpenSearchStatusException(response.toString(), RestStatus.INTERNAL_SERVER_ERROR));
+                                            onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                                             return;
                                         }
 

diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java
@@ -1237,7 +1237,7 @@ public void onFailure(Exception e) {
                     @Override
                     public void onResponse(SearchResponse response) {
                         if (response.isTimedOut()) {
-                            onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT));
+                            onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                         }
 
                         long count = response.getHits().getTotalHits().value;
@@ -1302,7 +1302,7 @@ public void importRules(IndexDetectorRequest request, ActionListener<List<IndexM
                 @Override
                 public void onResponse(SearchResponse response) {
                     if (response.isTimedOut()) {
-                        onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT));
+                        onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                     }
 
                     SearchHits hits = response.getHits();
@@ -1362,7 +1362,7 @@ public void importCustomRules(Detector detector, DetectorInput detectorInput, Li
                 @Override
                 public void onResponse(SearchResponse response) {
                     if (response.isTimedOut()) {
-                        onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT));
+                        onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                     }
 
                     SearchHits hits = response.getHits();

diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java
@@ -237,7 +237,7 @@ void indexRule(Rule rule, Map<String, String> ruleFieldMappings) throws IOExcept
                         @Override
                         public void onResponse(SearchResponse response) {
                             if (response.isTimedOut()) {
-                                onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Rule with id %s cannot be updated", rule.getId()), RestStatus.INTERNAL_SERVER_ERROR));
+                                onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Rule with id %s cannot be updated", rule.getId()), RestStatus.REQUEST_TIMEOUT));
                                 return;
                             }
 

diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportListCorrelationAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportListCorrelationAction.java
@@ -119,7 +119,7 @@ void start() {
                 @Override
                 public void onResponse(SearchResponse response) {
                     if (response.isTimedOut()) {
-                        onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT));
+                        onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                     }
 
                     Map<String, CorrelatedFinding> correlatedFindings = new HashMap<>();

diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java
@@ -327,7 +327,7 @@ public void onResponse(Void unused) {
                     @Override
                     public void onResponse(SearchResponse response) {
                         if (response.isTimedOut()) {
-                            listener.onFailure(new OpenSearchStatusException(response.toString(), RestStatus.INTERNAL_SERVER_ERROR));
+                            listener.onFailure(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
                         }
                         try {
                             SearchHit[] hits = response.getHits().getHits();