Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

adds toggling refresh disable/enable for deactivate/activate operation while updating URL_DOWNLOAD type configs #1240

Merged
merged 4 commits into from
Aug 10, 2024
Merged

adds toggling refresh disable/enable for deactivate/activate operation while updating URL_DOWNLOAD type configs #1240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8099f09
adds toggling refresh disable/enable for deactivate/activate operatio…
eirsep Aug 9, 2024
f72865c
add sleep for lock issue
eirsep Aug 10, 2024
8555fa4
wait on release lock operation completion before returning update tif…
eirsep Aug 10, 2024
9db4e6d
reset flag after integ tests for SourceConfigWithoutS3RestApiIT
eirsep Aug 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 1 addition & 19 deletions ...ava/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,25 +63,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject()
.field(_ID, id)
.field(_VERSION, version);
builder.startObject("source_config")
.field(SATIFSourceConfigDto.NAME_FIELD, saTifSourceConfigDto.getName())
.field(SATIFSourceConfigDto.FORMAT_FIELD, saTifSourceConfigDto.getFormat())
.field(SATIFSourceConfigDto.TYPE_FIELD, saTifSourceConfigDto.getType())
.field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTifSourceConfigDto.getIocTypes())
.field(SATIFSourceConfigDto.DESCRIPTION_FIELD, saTifSourceConfigDto.getDescription())
.field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTifSourceConfigDto.getCreatedByUser())
.field(SATIFSourceConfigDto.CREATED_AT_FIELD, saTifSourceConfigDto.getCreatedAt())
.field(SATIFSourceConfigDto.SOURCE_FIELD, saTifSourceConfigDto.getSource())
.field(SATIFSourceConfigDto.ENABLED_FIELD, saTifSourceConfigDto.isEnabled())
.field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTifSourceConfigDto.getEnabledTime())
.field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, saTifSourceConfigDto.getLastUpdateTime())
.field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTifSourceConfigDto.getSchedule())
.field(SATIFSourceConfigDto.STATE_FIELD, saTifSourceConfigDto.getState())
.field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, saTifSourceConfigDto.getRefreshType())
.field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, saTifSourceConfigDto.getLastRefreshedUser())
.field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTifSourceConfigDto.getLastRefreshedTime());

builder.endObject();
saTifSourceConfigDto.innerXcontent(builder);
return builder.endObject();
}

Expand Down
26 changes: 6 additions & 20 deletions ...a/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import java.io.IOException;

import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto.SOURCE_CONFIG_FIELD;
import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID;
import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION;

Expand Down Expand Up @@ -56,40 +57,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject()
.field(_ID, id)
.field(_VERSION, version);

builder.startObject("source_config")
.field(SATIFSourceConfigDto.NAME_FIELD, saTifSourceConfigDto.getName())
.field(SATIFSourceConfigDto.FORMAT_FIELD, saTifSourceConfigDto.getFormat())
.field(SATIFSourceConfigDto.TYPE_FIELD, saTifSourceConfigDto.getType())
.field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTifSourceConfigDto.getIocTypes())
.field(SATIFSourceConfigDto.DESCRIPTION_FIELD, saTifSourceConfigDto.getDescription())
.field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTifSourceConfigDto.getCreatedByUser())
.field(SATIFSourceConfigDto.CREATED_AT_FIELD, saTifSourceConfigDto.getCreatedAt())
.field(SATIFSourceConfigDto.SOURCE_FIELD, saTifSourceConfigDto.getSource())
.field(SATIFSourceConfigDto.ENABLED_FIELD, saTifSourceConfigDto.isEnabled())
.field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTifSourceConfigDto.getEnabledTime())
.field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, saTifSourceConfigDto.getLastUpdateTime())
.field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTifSourceConfigDto.getSchedule())
.field(SATIFSourceConfigDto.STATE_FIELD, saTifSourceConfigDto.getState())
.field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, saTifSourceConfigDto.getRefreshType())
.field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, saTifSourceConfigDto.getLastRefreshedUser())
.field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTifSourceConfigDto.getLastRefreshedTime());

builder.endObject();
saTifSourceConfigDto.innerXcontent(builder);
return builder.endObject();
}

@Override
public String getTIFConfigId() {
return id;
}

@Override
public Long getVersion() {
return version;
}

@Override
public TIFSourceConfigDto getTIFConfigDto() {
return saTifSourceConfigDto;
}

public RestStatus getStatus() {
return status;
}
Expand Down
15 changes: 14 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public TIFLockService(final ClusterService clusterService, final Client client)
}

/**
* Synchronous method of #acquireLock
* Event-driven method of #acquireLock
*
* @param tifJobName tifJobName to acquire lock on
* @param lockDurationSeconds the lock duration in seconds
Expand Down Expand Up @@ -81,6 +81,19 @@ public void releaseLock(final LockModel lockModel) {
);
}

/**
* Wrapper method of LockService#release
*
* @param lockModel the lock model
*/
public void releaseLockEventDriven(final LockModel lockModel, final ActionListener<Boolean> listener) {
log.debug("Releasing lock with id [{}]", lockModel.getLockId());
lockService.release(
lockModel,
ActionListener.wrap(listener::onResponse, exception -> log.error("Failed to release the lock", exception))
);
}

/**
* Synchronous method of LockService#renewLock
*
Expand Down
19 changes: 12 additions & 7 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,12 +215,18 @@ public void writeTo(final StreamOutput out) throws IOException {

@Override
public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
builder.startObject()
.startObject(SOURCE_CONFIG_FIELD)
.field(NAME_FIELD, name)
.field(FORMAT_FIELD, format)
.field(TYPE_FIELD, type.name())
.field(DESCRIPTION_FIELD, description);
builder.startObject();
innerXcontent(builder);
builder.endObject();
return builder;
}

public XContentBuilder innerXcontent(XContentBuilder builder) throws IOException {
builder.startObject(SOURCE_CONFIG_FIELD);
builder.field(NAME_FIELD, name)
.field(FORMAT_FIELD, format)
.field(TYPE_FIELD, type.name())
.field(DESCRIPTION_FIELD, description);
if (createdByUser == null) {
builder.nullField(CREATED_BY_USER_FIELD);
} else {
Expand Down Expand Up @@ -274,7 +280,6 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
builder.field(ENABLED_FOR_SCAN_FIELD, enabledForScan);
builder.field(IOC_TYPES_FIELD, iocTypes);
builder.endObject();
builder.endObject();
return builder;
}

Expand Down
6 changes: 5 additions & 1 deletion .../opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,10 @@ public void updateIocAndTIFSourceConfig(
// Due to the lack of a different API to do activate/deactivate we will check if enabled_for_scan variable is changed between model and request.
// If yes, we will ONLY update enabled_for_scan field and ignore any updates to the rest of the fields to simulate a dedicated activate/deactivate API.
if (retrievedSaTifSourceConfig.isEnabledForScan() != saTifSourceConfigDto.isEnabledForScan()) {
// FIXME add a disable_refresh api independent of update api so that it can be supported for default configs also
boolean isEnabled = URL_DOWNLOAD.equals(retrievedSaTifSourceConfig.getType()) ?
saTifSourceConfigDto.isEnabledForScan() :
retrievedSaTifSourceConfig.isEnabled();
SATIFSourceConfig config = new SATIFSourceConfig(
retrievedSaTifSourceConfig.getId(),
retrievedSaTifSourceConfig.getVersion(),
Expand All @@ -297,7 +301,7 @@ public void updateIocAndTIFSourceConfig(
retrievedSaTifSourceConfig.getRefreshType(),
Instant.now(),
updatedByUser,
retrievedSaTifSourceConfig.isEnabled(),
isEnabled,
retrievedSaTifSourceConfig.getIocStoreConfig(),
retrievedSaTifSourceConfig.getIocTypes(),
saTifSourceConfigDto.isEnabledForScan() // update only enabled_for_scan
Expand Down
56 changes: 43 additions & 13 deletions ...ensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,10 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques
try {
lockService.acquireLock(request.getTIFConfigDto().getId(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> {
if (lock == null) {
log.error("another processor is a lock, BAD_REQUEST error", RestStatus.BAD_REQUEST);
listener.onFailure(
new ConcurrentModificationException("another processor is holding a lock on the resource. Try again later")
);
log.error("another processor is a lock, BAD_REQUEST error", RestStatus.BAD_REQUEST);
return;
}
try {
Expand All @@ -106,29 +106,59 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques
user,
ActionListener.wrap(
saTifSourceConfigDtoResponse -> {
lockService.releaseLock(lock);
listener.onResponse(new SAIndexTIFSourceConfigResponse(
saTifSourceConfigDtoResponse.getId(),
saTifSourceConfigDtoResponse.getVersion(),
RestStatus.OK,
saTifSourceConfigDtoResponse
lockService.releaseLockEventDriven(lock, ActionListener.wrap(
r -> listener.onResponse(new SAIndexTIFSourceConfigResponse(
saTifSourceConfigDtoResponse.getId(),
saTifSourceConfigDtoResponse.getVersion(),
RestStatus.OK,
saTifSourceConfigDtoResponse
)),
e -> {
log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config [%s].", lock.getLockId(), saTifSourceConfigDto.getId()), e);
listener.onResponse(new SAIndexTIFSourceConfigResponse(
saTifSourceConfigDtoResponse.getId(),
saTifSourceConfigDtoResponse.getVersion(),
RestStatus.OK,
saTifSourceConfigDtoResponse
));
}
));
}, e -> {
lockService.releaseLock(lock);
log.error("Failed to create IOCs and threat intel source config");
listener.onFailure(e);
lockService.releaseLockEventDriven(lock, ActionListener.wrap(
r -> {
log.error("Failed to create IOCs and threat intel source config", e);
listener.onFailure(e);
},
ex -> {
String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create";
log.error(String.format("Failed to %s IOCs and threat intel source config", action), e);
log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config.", lock.getLockId()), e);
listener.onFailure(e);
}
));
}

)
);
} catch (Exception e) {
lockService.releaseLock(lock);
listener.onFailure(e);
log.error("listener failed when executing", e);
lockService.releaseLockEventDriven(lock, ActionListener.wrap(
r -> {
log.error("Failed to create IOCs and threat intel source config", e);
listener.onFailure(e);
},
ex -> {
String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create";
log.error(String.format("Failed to %s IOCs and threat intel source config", action), e);
log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config.", lock.getLockId()), e);
listener.onFailure(e);
}
));
}
}, exception -> {
String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create";
log.error(String.format("Failed to acquire lock while trying to %s tif source config", action), exception);
listener.onFailure(exception);
log.error("execution failed", exception);
}));
} catch (Exception e) {
log.error("Failed to acquire lock for job", e);
Expand Down
Loading
Loading