[Backport 2.16] adds toggling refresh disable/enable for deactivate/activate operation while updating URL_DOWNLOAD type configs

src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java

@@ -63,25 +63,7 @@
         builder.startObject()
                 .field(_ID, id)
                 .field(_VERSION, version);
-        builder.startObject("source_config")
-                .field(SATIFSourceConfigDto.NAME_FIELD, saTifSourceConfigDto.getName())
-                .field(SATIFSourceConfigDto.FORMAT_FIELD, saTifSourceConfigDto.getFormat())
-                .field(SATIFSourceConfigDto.TYPE_FIELD, saTifSourceConfigDto.getType())
-                .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTifSourceConfigDto.getIocTypes())
-                .field(SATIFSourceConfigDto.DESCRIPTION_FIELD, saTifSourceConfigDto.getDescription())
-                .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTifSourceConfigDto.getCreatedByUser())
-                .field(SATIFSourceConfigDto.CREATED_AT_FIELD, saTifSourceConfigDto.getCreatedAt())
-                .field(SATIFSourceConfigDto.SOURCE_FIELD, saTifSourceConfigDto.getSource())
-                .field(SATIFSourceConfigDto.ENABLED_FIELD, saTifSourceConfigDto.isEnabled())
-                .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTifSourceConfigDto.getEnabledTime())
-                .field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, saTifSourceConfigDto.getLastUpdateTime())
-                .field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTifSourceConfigDto.getSchedule())
-                .field(SATIFSourceConfigDto.STATE_FIELD, saTifSourceConfigDto.getState())
-                .field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, saTifSourceConfigDto.getRefreshType())
-                .field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, saTifSourceConfigDto.getLastRefreshedUser())
-                .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTifSourceConfigDto.getLastRefreshedTime());
-
-        builder.endObject();
+        saTifSourceConfigDto.innerXcontent(builder);
         return builder.endObject();
     }
 

src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java

@@ -17,6 +17,7 @@
 
 import java.io.IOException;
 
+import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto.SOURCE_CONFIG_FIELD;
 import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID;
 import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION;
 
@@ -56,40 +57,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject()
                 .field(_ID, id)
                 .field(_VERSION, version);
-
-        builder.startObject("source_config")
-                .field(SATIFSourceConfigDto.NAME_FIELD, saTifSourceConfigDto.getName())
-                .field(SATIFSourceConfigDto.FORMAT_FIELD, saTifSourceConfigDto.getFormat())
-                .field(SATIFSourceConfigDto.TYPE_FIELD, saTifSourceConfigDto.getType())
-                .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTifSourceConfigDto.getIocTypes())
-                .field(SATIFSourceConfigDto.DESCRIPTION_FIELD, saTifSourceConfigDto.getDescription())
-                .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTifSourceConfigDto.getCreatedByUser())
-                .field(SATIFSourceConfigDto.CREATED_AT_FIELD, saTifSourceConfigDto.getCreatedAt())
-                .field(SATIFSourceConfigDto.SOURCE_FIELD, saTifSourceConfigDto.getSource())
-                .field(SATIFSourceConfigDto.ENABLED_FIELD, saTifSourceConfigDto.isEnabled())
-                .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTifSourceConfigDto.getEnabledTime())
-                .field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, saTifSourceConfigDto.getLastUpdateTime())
-                .field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTifSourceConfigDto.getSchedule())
-                .field(SATIFSourceConfigDto.STATE_FIELD, saTifSourceConfigDto.getState())
-                .field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, saTifSourceConfigDto.getRefreshType())
-                .field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, saTifSourceConfigDto.getLastRefreshedUser())
-                .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTifSourceConfigDto.getLastRefreshedTime());
-
-        builder.endObject();
+        saTifSourceConfigDto.innerXcontent(builder);
         return builder.endObject();
     }
+
     @Override
     public String getTIFConfigId() {
         return id;
     }
+
     @Override
     public Long getVersion() {
         return version;
     }
+
     @Override
     public TIFSourceConfigDto getTIFConfigDto() {
         return saTifSourceConfigDto;
     }
+
     public RestStatus getStatus() {
         return status;
     }

src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java

@@ -46,7 +46,7 @@ public TIFLockService(final ClusterService clusterService, final Client client)
     }
 
     /**
-     * Synchronous method of #acquireLock
+     * Event-driven method of #acquireLock
      *
      * @param tifJobName tifJobName to acquire lock on
      * @param lockDurationSeconds the lock duration in seconds
@@ -81,6 +81,19 @@ public void releaseLock(final LockModel lockModel) {
         );
     }
 
+    /**
+     * Wrapper method of LockService#release
+     *
+     * @param lockModel the lock model
+     */
+    public void releaseLockEventDriven(final LockModel lockModel, final ActionListener<Boolean> listener) {
+        log.debug("Releasing lock with id [{}]", lockModel.getLockId());
+        lockService.release(
+                lockModel,
+                ActionListener.wrap(listener::onResponse, exception -> log.error("Failed to release the lock", exception))
+        );
+    }
+
     /**
      * Synchronous method of LockService#renewLock
      *

src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java

@@ -215,12 +215,18 @@ public void writeTo(final StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
-        builder.startObject()
-                .startObject(SOURCE_CONFIG_FIELD)
-                .field(NAME_FIELD, name)
-                .field(FORMAT_FIELD, format)
-                .field(TYPE_FIELD, type.name())
-                .field(DESCRIPTION_FIELD, description);
+        builder.startObject();
+        innerXcontent(builder);
+        builder.endObject();
+        return builder;
+    }
+
+    public XContentBuilder innerXcontent(XContentBuilder builder) throws IOException {
+        builder.startObject(SOURCE_CONFIG_FIELD);
+        builder.field(NAME_FIELD, name)
+        .field(FORMAT_FIELD, format)
+        .field(TYPE_FIELD, type.name())
+        .field(DESCRIPTION_FIELD, description);
         if (createdByUser == null) {
             builder.nullField(CREATED_BY_USER_FIELD);
         } else {
@@ -274,7 +280,6 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         builder.field(ENABLED_FOR_SCAN_FIELD, enabledForScan);
         builder.field(IOC_TYPES_FIELD, iocTypes);
         builder.endObject();
-        builder.endObject();
         return builder;
     }
 

src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java

@@ -280,6 +280,10 @@ public void updateIocAndTIFSourceConfig(
                         // Due to the lack of a different API to do activate/deactivate we will check if enabled_for_scan variable is changed between model and request.
                         // If yes, we will ONLY update enabled_for_scan field and ignore any updates to the rest of the fields to simulate a dedicated activate/deactivate API.
                         if (retrievedSaTifSourceConfig.isEnabledForScan() != saTifSourceConfigDto.isEnabledForScan()) {
+                            // FIXME add a disable_refresh api independent of update api so that it can be supported for default configs also
+                            boolean isEnabled = URL_DOWNLOAD.equals(retrievedSaTifSourceConfig.getType()) ?
+                                    saTifSourceConfigDto.isEnabledForScan() :
+                                    retrievedSaTifSourceConfig.isEnabled();
                             SATIFSourceConfig config = new SATIFSourceConfig(
                                     retrievedSaTifSourceConfig.getId(),
                                     retrievedSaTifSourceConfig.getVersion(),
@@ -297,7 +301,7 @@ public void updateIocAndTIFSourceConfig(
                                     retrievedSaTifSourceConfig.getRefreshType(),
                                     Instant.now(),
                                     updatedByUser,
-                                    retrievedSaTifSourceConfig.isEnabled(),
+                                    isEnabled,
                                     retrievedSaTifSourceConfig.getIocStoreConfig(),
                                     retrievedSaTifSourceConfig.getIocTypes(),
                                     saTifSourceConfigDto.isEnabledForScan() // update only enabled_for_scan

src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java

@@ -91,10 +91,10 @@
         try {
             lockService.acquireLock(request.getTIFConfigDto().getId(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> {
                 if (lock == null) {
+                    log.error("another processor is a lock, BAD_REQUEST error", RestStatus.BAD_REQUEST);
                     listener.onFailure(
                             new ConcurrentModificationException("another processor is holding a lock on the resource. Try again later")
                     );
-                    log.error("another processor is a lock, BAD_REQUEST error", RestStatus.BAD_REQUEST);
                     return;
                 }
                 try {
@@ -106,29 +106,59 @@
                             user,
                             ActionListener.wrap(
                                     saTifSourceConfigDtoResponse -> {
-                                        lockService.releaseLock(lock);
-                                        listener.onResponse(new SAIndexTIFSourceConfigResponse(
-                                                saTifSourceConfigDtoResponse.getId(),
-                                                saTifSourceConfigDtoResponse.getVersion(),
-                                                RestStatus.OK,
-                                                saTifSourceConfigDtoResponse
+                                        lockService.releaseLockEventDriven(lock, ActionListener.wrap(
+                                                r -> listener.onResponse(new SAIndexTIFSourceConfigResponse(
+                                                        saTifSourceConfigDtoResponse.getId(),
+                                                        saTifSourceConfigDtoResponse.getVersion(),
+                                                        RestStatus.OK,
+                                                        saTifSourceConfigDtoResponse
+                                                )),
+                                                e -> {
+                                                    log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config [%s].", lock.getLockId(), saTifSourceConfigDto.getId()), e);
+                                                    listener.onResponse(new SAIndexTIFSourceConfigResponse(
+                                                            saTifSourceConfigDtoResponse.getId(),
+                                                            saTifSourceConfigDtoResponse.getVersion(),
+                                                            RestStatus.OK,
+                                                            saTifSourceConfigDtoResponse
+                                                    ));
+                                                }
                                         ));
                                     }, e -> {
-                                        lockService.releaseLock(lock);
-                                        log.error("Failed to create IOCs and threat intel source config");
-                                        listener.onFailure(e);
+                                        lockService.releaseLockEventDriven(lock, ActionListener.wrap(
+                                                r -> {
+                                                    log.error("Failed to create IOCs and threat intel source config", e);
+                                                    listener.onFailure(e);
+                                                },
+                                                ex -> {
+                                                    String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create";
+                                                    log.error(String.format("Failed to %s IOCs and threat intel source config", action), e);
+                                                    log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config.", lock.getLockId()), e);
+                                                    listener.onFailure(e);
+                                                }
+                                        ));
                                     }
 
                             )
                     );
                 } catch (Exception e) {
-                    lockService.releaseLock(lock);
-                    listener.onFailure(e);
                     log.error("listener failed when executing", e);
+                    lockService.releaseLockEventDriven(lock, ActionListener.wrap(
+                            r -> {
+                                log.error("Failed to create IOCs and threat intel source config", e);
+                                listener.onFailure(e);
+                            },
+                            ex -> {
+                                String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create";
+                                log.error(String.format("Failed to %s IOCs and threat intel source config", action), e);
+                                log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config.", lock.getLockId()), e);
+                                listener.onFailure(e);
+                            }
+                    ));
                 }
             }, exception -> {
+                String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create";
+                log.error(String.format("Failed to acquire lock while trying to %s tif source config", action), exception);
                 listener.onFailure(exception);
-                log.error("execution failed", exception);
             }));
         } catch (Exception e) {
             log.error("Failed to acquire lock for job", e);