Skip to content

Commit

Permalink
adds toggling refresh disable/enable for deactivate/activate operatio…
Browse files Browse the repository at this point in the history 
…n while updating URL_DOWNLOAD type configs (#1240)

* adds toggling refresh disable/enable for deactivate/activate operation while updating URL_DOWNLOAD type configs

Signed-off-by: Surya Sashank Nistala <[email protected]>

* add sleep for lock issue

Signed-off-by: Surya Sashank Nistala <[email protected]>

* wait on release lock operation completion before returning update tif source config response

Signed-off-by: Surya Sashank Nistala <[email protected]>

* reset flag after integ tests for SourceConfigWithoutS3RestApiIT

Signed-off-by: Surya Sashank Nistala <[email protected]>

---------

Signed-off-by: Surya Sashank Nistala <[email protected]>
  • Loading branch information
@eirsep
eirsep authored Aug 10, 2024
1 parent 6916f8c commit 3e1f59d
Show file tree
Hide file tree
Showing 7 changed files with 169 additions and 68 deletions.
20 changes: 1 addition & 19 deletions ...ava/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,25 +63,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject()
.field(_ID, id)
.field(_VERSION, version);
builder.startObject("source_config")
.field(SATIFSourceConfigDto.NAME_FIELD, saTifSourceConfigDto.getName())
.field(SATIFSourceConfigDto.FORMAT_FIELD, saTifSourceConfigDto.getFormat())
.field(SATIFSourceConfigDto.TYPE_FIELD, saTifSourceConfigDto.getType())
.field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTifSourceConfigDto.getIocTypes())
.field(SATIFSourceConfigDto.DESCRIPTION_FIELD, saTifSourceConfigDto.getDescription())
.field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTifSourceConfigDto.getCreatedByUser())
.field(SATIFSourceConfigDto.CREATED_AT_FIELD, saTifSourceConfigDto.getCreatedAt())
.field(SATIFSourceConfigDto.SOURCE_FIELD, saTifSourceConfigDto.getSource())
.field(SATIFSourceConfigDto.ENABLED_FIELD, saTifSourceConfigDto.isEnabled())
.field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTifSourceConfigDto.getEnabledTime())
.field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, saTifSourceConfigDto.getLastUpdateTime())
.field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTifSourceConfigDto.getSchedule())
.field(SATIFSourceConfigDto.STATE_FIELD, saTifSourceConfigDto.getState())
.field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, saTifSourceConfigDto.getRefreshType())
.field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, saTifSourceConfigDto.getLastRefreshedUser())
.field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTifSourceConfigDto.getLastRefreshedTime());

builder.endObject();
saTifSourceConfigDto.innerXcontent(builder);
return builder.endObject();
}

Expand Down
26 changes: 6 additions & 20 deletions ...a/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import java.io.IOException;

import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto.SOURCE_CONFIG_FIELD;
import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID;
import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION;

Expand Down Expand Up @@ -56,40 +57,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject()
.field(_ID, id)
.field(_VERSION, version);

builder.startObject("source_config")
.field(SATIFSourceConfigDto.NAME_FIELD, saTifSourceConfigDto.getName())
.field(SATIFSourceConfigDto.FORMAT_FIELD, saTifSourceConfigDto.getFormat())
.field(SATIFSourceConfigDto.TYPE_FIELD, saTifSourceConfigDto.getType())
.field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTifSourceConfigDto.getIocTypes())
.field(SATIFSourceConfigDto.DESCRIPTION_FIELD, saTifSourceConfigDto.getDescription())
.field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTifSourceConfigDto.getCreatedByUser())
.field(SATIFSourceConfigDto.CREATED_AT_FIELD, saTifSourceConfigDto.getCreatedAt())
.field(SATIFSourceConfigDto.SOURCE_FIELD, saTifSourceConfigDto.getSource())
.field(SATIFSourceConfigDto.ENABLED_FIELD, saTifSourceConfigDto.isEnabled())
.field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTifSourceConfigDto.getEnabledTime())
.field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, saTifSourceConfigDto.getLastUpdateTime())
.field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTifSourceConfigDto.getSchedule())
.field(SATIFSourceConfigDto.STATE_FIELD, saTifSourceConfigDto.getState())
.field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, saTifSourceConfigDto.getRefreshType())
.field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, saTifSourceConfigDto.getLastRefreshedUser())
.field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTifSourceConfigDto.getLastRefreshedTime());

builder.endObject();
saTifSourceConfigDto.innerXcontent(builder);
return builder.endObject();
}

@Override
public String getTIFConfigId() {
return id;
}

@Override
public Long getVersion() {
return version;
}

@Override
public TIFSourceConfigDto getTIFConfigDto() {
return saTifSourceConfigDto;
}

public RestStatus getStatus() {
return status;
}
Expand Down
15 changes: 14 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public TIFLockService(final ClusterService clusterService, final Client client)
}

/**
* Synchronous method of #acquireLock
* Event-driven method of #acquireLock
*
* @param tifJobName tifJobName to acquire lock on
* @param lockDurationSeconds the lock duration in seconds
Expand Down Expand Up @@ -81,6 +81,19 @@ public void releaseLock(final LockModel lockModel) {
);
}

/**
* Wrapper method of LockService#release
*
* @param lockModel the lock model
*/
public void releaseLockEventDriven(final LockModel lockModel, final ActionListener<Boolean> listener) {
log.debug("Releasing lock with id [{}]", lockModel.getLockId());
lockService.release(
lockModel,
ActionListener.wrap(listener::onResponse, exception -> log.error("Failed to release the lock", exception))
);
}

/**
* Synchronous method of LockService#renewLock
*
Expand Down
19 changes: 12 additions & 7 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,12 +215,18 @@ public void writeTo(final StreamOutput out) throws IOException {

@Override
public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
builder.startObject()
.startObject(SOURCE_CONFIG_FIELD)
.field(NAME_FIELD, name)
.field(FORMAT_FIELD, format)
.field(TYPE_FIELD, type.name())
.field(DESCRIPTION_FIELD, description);
builder.startObject();
innerXcontent(builder);
builder.endObject();
return builder;
}

public XContentBuilder innerXcontent(XContentBuilder builder) throws IOException {
builder.startObject(SOURCE_CONFIG_FIELD);
builder.field(NAME_FIELD, name)
.field(FORMAT_FIELD, format)
.field(TYPE_FIELD, type.name())
.field(DESCRIPTION_FIELD, description);
if (createdByUser == null) {
builder.nullField(CREATED_BY_USER_FIELD);
} else {
Expand Down Expand Up @@ -274,7 +280,6 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
builder.field(ENABLED_FOR_SCAN_FIELD, enabledForScan);
builder.field(IOC_TYPES_FIELD, iocTypes);
builder.endObject();
builder.endObject();
return builder;
}

Expand Down
6 changes: 5 additions & 1 deletion .../opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,10 @@ public void updateIocAndTIFSourceConfig(
// Due to the lack of a different API to do activate/deactivate we will check if enabled_for_scan variable is changed between model and request.
// If yes, we will ONLY update enabled_for_scan field and ignore any updates to the rest of the fields to simulate a dedicated activate/deactivate API.
if (retrievedSaTifSourceConfig.isEnabledForScan() != saTifSourceConfigDto.isEnabledForScan()) {
// FIXME add a disable_refresh api independent of update api so that it can be supported for default configs also
boolean isEnabled = URL_DOWNLOAD.equals(retrievedSaTifSourceConfig.getType()) ?
saTifSourceConfigDto.isEnabledForScan() :
retrievedSaTifSourceConfig.isEnabled();
SATIFSourceConfig config = new SATIFSourceConfig(
retrievedSaTifSourceConfig.getId(),
retrievedSaTifSourceConfig.getVersion(),
Expand All @@ -297,7 +301,7 @@ public void updateIocAndTIFSourceConfig(
retrievedSaTifSourceConfig.getRefreshType(),
Instant.now(),
updatedByUser,
retrievedSaTifSourceConfig.isEnabled(),
isEnabled,
retrievedSaTifSourceConfig.getIocStoreConfig(),
retrievedSaTifSourceConfig.getIocTypes(),
saTifSourceConfigDto.isEnabledForScan() // update only enabled_for_scan
Expand Down
56 changes: 43 additions & 13 deletions ...ensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,10 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques
try {
lockService.acquireLock(request.getTIFConfigDto().getId(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> {
if (lock == null) {
log.error("another processor is a lock, BAD_REQUEST error", RestStatus.BAD_REQUEST);
listener.onFailure(
new ConcurrentModificationException("another processor is holding a lock on the resource. Try again later")
);
log.error("another processor is a lock, BAD_REQUEST error", RestStatus.BAD_REQUEST);
return;
}
try {
Expand All @@ -106,29 +106,59 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques
user,
ActionListener.wrap(
saTifSourceConfigDtoResponse -> {
lockService.releaseLock(lock);
listener.onResponse(new SAIndexTIFSourceConfigResponse(
saTifSourceConfigDtoResponse.getId(),
saTifSourceConfigDtoResponse.getVersion(),
RestStatus.OK,
saTifSourceConfigDtoResponse
lockService.releaseLockEventDriven(lock, ActionListener.wrap(
r -> listener.onResponse(new SAIndexTIFSourceConfigResponse(
saTifSourceConfigDtoResponse.getId(),
saTifSourceConfigDtoResponse.getVersion(),
RestStatus.OK,
saTifSourceConfigDtoResponse
)),
e -> {
log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config [%s].", lock.getLockId(), saTifSourceConfigDto.getId()), e);
listener.onResponse(new SAIndexTIFSourceConfigResponse(
saTifSourceConfigDtoResponse.getId(),
saTifSourceConfigDtoResponse.getVersion(),
RestStatus.OK,
saTifSourceConfigDtoResponse
));
}
));
}, e -> {
lockService.releaseLock(lock);
log.error("Failed to create IOCs and threat intel source config");
listener.onFailure(e);
lockService.releaseLockEventDriven(lock, ActionListener.wrap(
r -> {
log.error("Failed to create IOCs and threat intel source config", e);
listener.onFailure(e);
},
ex -> {
String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create";
log.error(String.format("Failed to %s IOCs and threat intel source config", action), e);
log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config.", lock.getLockId()), e);
listener.onFailure(e);
}
));
}

)
);
} catch (Exception e) {
lockService.releaseLock(lock);
listener.onFailure(e);
log.error("listener failed when executing", e);
lockService.releaseLockEventDriven(lock, ActionListener.wrap(
r -> {
log.error("Failed to create IOCs and threat intel source config", e);
listener.onFailure(e);
},
ex -> {
String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create";
log.error(String.format("Failed to %s IOCs and threat intel source config", action), e);
log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config.", lock.getLockId()), e);
listener.onFailure(e);
}
));
}
}, exception -> {
String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create";
log.error(String.format("Failed to acquire lock while trying to %s tif source config", action), exception);
listener.onFailure(exception);
log.error("execution failed", exception);
}));
} catch (Exception e) {
log.error("Failed to acquire lock for job", e);
Expand Down
Loading

0 comments on commit 3e1f59d

Please sign in to comment.