Skip to content

Commit

Permalink
address comments
Browse files Browse the repository at this point in the history 
Signed-off-by: Subhobrata Dey <[email protected]>
  • Loading branch information
@sbcd90
sbcd90 committed Jun 17, 2024
1 parent 2976be0 commit b09afc1
Show file tree
Hide file tree
Showing 15 changed files with 150 additions and 209 deletions.
2 changes: 0 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,6 @@
import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService;
import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService;
import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction;
Expand All @@ -117,7 +116,6 @@
import org.opensearch.securityanalytics.model.Rule;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorInput;
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.util.CorrelationIndices;
import org.opensearch.securityanalytics.util.CorrelationRuleIndices;
import org.opensearch.securityanalytics.util.CustomLogTypeIndices;
Expand Down
16 changes: 8 additions & 8 deletions ...analytics/model/threatintel/IocMatch.java → ...alytics/model/threatintel/IocFinding.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
* IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job
* It's the inverse of an IoC finding which maps a document to list of IoC's
*/
public class IocMatch implements Writeable, ToXContent {
public class IocFinding implements Writeable, ToXContent {
//TODO implement IoC_Match interface from security-analytics-commons
public static final String ID_FIELD = "id";
public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids";
Expand All @@ -42,8 +42,8 @@ public class IocMatch implements Writeable, ToXContent {
private final Instant timestamp;
private final String executionId;

public IocMatch(String id, List<String> relatedDocIds, List<String> feedIds, String iocScanJobId,
String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) {
public IocFinding(String id, List<String> relatedDocIds, List<String> feedIds, String iocScanJobId,
String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) {
validateIoCMatch(id, iocScanJobId, iocScanJobName, iocValue, timestamp, executionId, relatedDocIds);
this.id = id;
this.relatedDocIds = relatedDocIds;
Expand All @@ -56,7 +56,7 @@ public IocMatch(String id, List<String> relatedDocIds, List<String> feedIds, Str
this.executionId = executionId;
}

public IocMatch(StreamInput in) throws IOException {
public IocFinding(StreamInput in) throws IOException {
id = in.readString();
relatedDocIds = in.readStringList();
feedIds = in.readStringList();
Expand Down Expand Up @@ -133,7 +133,7 @@ public String getExecutionId() {
return executionId;
}

public static IocMatch parse(XContentParser xcp) throws IOException {
public static IocFinding parse(XContentParser xcp) throws IOException {
String id = null;
List<String> relatedDocIds = new ArrayList<>();
List<String> feedIds = new ArrayList<>();
Expand Down Expand Up @@ -197,11 +197,11 @@ public static IocMatch parse(XContentParser xcp) throws IOException {
}
}

return new IocMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId);
return new IocFinding(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId);
}

public static IocMatch readFrom(StreamInput in) throws IOException {
return new IocMatch(in);
public static IocFinding readFrom(StreamInput in) throws IOException {
return new IocFinding(in);
}


Expand Down
86 changes: 0 additions & 86 deletions src/main/java/org/opensearch/securityanalytics/model/threatintel/IocMatchWithDocs.java
View file

This file was deleted.

22 changes: 1 addition & 21 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,46 +24,31 @@ public class GetIocFindingsRequest extends ActionRequest {

private Instant endTime;

private String threatIntelMonitorId;

private Table table;

public static final String THREAT_INTEL_MONITOR_ID = "monitor_id";

public GetIocFindingsRequest(String threatIntelMonitorId) {
super();
this.threatIntelMonitorId = threatIntelMonitorId;
}

public GetIocFindingsRequest(StreamInput sin) throws IOException {
this(
sin.readOptionalStringList(),
sin.readOptionalInstant(),
sin.readOptionalInstant(),
sin.readOptionalString(),
Table.readFrom(sin)
);
}

public GetIocFindingsRequest(List<String> findingIds,
Instant startTime,
Instant endTime,
String threatIntelMonitorId,
Table table) {
this.findingIds = findingIds;
this.startTime = startTime;
this.endTime = endTime;
this.threatIntelMonitorId = threatIntelMonitorId;
this.table = table;
}

@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if (threatIntelMonitorId != null && threatIntelMonitorId.isEmpty()) {
validationException = ValidateActions.addValidationError(String.format(Locale.getDefault(),
"threat intel monitor id is missing"), validationException);
} else if (startTime != null && endTime != null && startTime.isAfter(endTime)) {
if (startTime != null && endTime != null && startTime.isAfter(endTime)) {
validationException = ValidateActions.addValidationError(String.format(Locale.getDefault(),
"startTime should be less than endTime"), validationException);
}
Expand All @@ -75,7 +60,6 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalStringCollection(findingIds);
out.writeOptionalInstant(startTime);
out.writeOptionalInstant(endTime);
out.writeOptionalString(threatIntelMonitorId);
table.writeTo(out);
}

Expand All @@ -91,10 +75,6 @@ public Instant getEndTime() {
return endTime;
}

public String getThreatIntelMonitorId() {
return threatIntelMonitorId;
}

public Table getTable() {
return table;
}
Expand Down
11 changes: 5 additions & 6 deletions ...main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.securityanalytics.model.threatintel.IocMatch;
import org.opensearch.securityanalytics.model.threatintel.IocMatchWithDocs;
import org.opensearch.securityanalytics.model.threatintel.IocFinding;

import java.io.IOException;
import java.util.Collections;
Expand All @@ -24,9 +23,9 @@ public class GetIocFindingsResponse extends ActionResponse implements ToXContent

private Integer totalFindings;

private List<IocMatchWithDocs> iocFindings;
private List<IocFinding> iocFindings;

public GetIocFindingsResponse(Integer totalFindings, List<IocMatchWithDocs> iocFindings) {
public GetIocFindingsResponse(Integer totalFindings, List<IocFinding> iocFindings) {
super();
this.totalFindings = totalFindings;
this.iocFindings = iocFindings;
Expand All @@ -35,7 +34,7 @@ public GetIocFindingsResponse(Integer totalFindings, List<IocMatchWithDocs> iocF
public GetIocFindingsResponse(StreamInput sin) throws IOException {
this(
sin.readInt(),
Collections.unmodifiableList(sin.readList(IocMatchWithDocs::new))
Collections.unmodifiableList(sin.readList(IocFinding::new))
);
}

Expand All @@ -57,7 +56,7 @@ public Integer getTotalFindings() {
return totalFindings;
}

public List<IocMatchWithDocs> getIocFindings() {
public List<IocFinding> getIocFindings() {
return iocFindings;
}
}
44 changes: 19 additions & 25 deletions ...eatIntel/iocscan/dao/IocMatchService.java → ...tIntel/iocscan/dao/IocFindingService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,14 @@
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.search.SearchHit;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.securityanalytics.SecurityAnalyticsPlugin;
import org.opensearch.securityanalytics.model.threatintel.IocMatch;
import org.opensearch.securityanalytics.model.threatintel.IocMatchWithDocs;
import org.opensearch.securityanalytics.model.threatintel.IocFinding;
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse;
import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
import org.opensearch.threadpool.ThreadPool;

import java.io.BufferedReader;
import java.io.IOException;
Expand All @@ -47,39 +45,39 @@
/**
* Data layer to perform CRUD operations for threat intel ioc match : store in system index.
*/
public class IocMatchService {
public class IocFindingService {
//TODO manage index rollover
public static final String INDEX_NAME = ".opensearch-sap-iocmatch";
private static final Logger log = LogManager.getLogger(IocMatchService.class);
private static final Logger log = LogManager.getLogger(IocFindingService.class);
private final Client client;
private final ClusterService clusterService;

private final NamedXContentRegistry xContentRegistry;

public IocMatchService(final Client client, final ClusterService clusterService, final NamedXContentRegistry xContentRegistry) {
public IocFindingService(final Client client, final ClusterService clusterService, final NamedXContentRegistry xContentRegistry) {
this.client = client;
this.clusterService = clusterService;
this.xContentRegistry = xContentRegistry;
}

public void indexIocMatches(List<IocMatch> iocMatches,
public void indexIocMatches(List<IocFinding> iocFindings,
final ActionListener<Void> actionListener) {
try {
Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE);
createIndexIfNotExists(ActionListener.wrap(
r -> {
List<BulkRequest> bulkRequestList = new ArrayList<>();
BulkRequest bulkRequest = new BulkRequest(INDEX_NAME);
for (int i = 0; i < iocMatches.size(); i++) {
IocMatch iocMatch = iocMatches.get(i);
for (int i = 0; i < iocFindings.size(); i++) {
IocFinding iocFinding = iocFindings.get(i);
try {
IndexRequest indexRequest = new IndexRequest(INDEX_NAME)
.source(iocMatch.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS))
.source(iocFinding.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS))
.opType(DocWriteRequest.OpType.CREATE);
bulkRequest.add(indexRequest);
if (
bulkRequest.requests().size() == batchSize
&& i != iocMatches.size() - 1 // final bulk request will be added outside for loop with refresh policy none
&& i != iocFindings.size() - 1 // final bulk request will be added outside for loop with refresh policy none
) {
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
bulkRequestList.add(bulkRequest);
Expand Down Expand Up @@ -122,7 +120,7 @@ public void indexIocMatches(List<IocMatch> iocMatches,

private String getIndexMapping() {
try {
try (InputStream is = IocMatchService.class.getResourceAsStream("/mappings/ioc_match_mapping.json")) {
try (InputStream is = IocFindingService.class.getResourceAsStream("/mappings/ioc_match_mapping.json")) {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
return reader.lines().map(String::trim).collect(Collectors.joining());
}
Expand Down Expand Up @@ -178,31 +176,27 @@ public void searchIocMatches(SearchSourceBuilder searchSourceBuilder, final Acti
public void onResponse(SearchResponse searchResponse) {
try {
long totalIocFindingsCount = searchResponse.getHits().getTotalHits().value;
MultiGetRequest mGetRequest = new MultiGetRequest();
List<IocMatchWithDocs> iocMatchWithDocs = new ArrayList<>();
List<IocMatch> iocMatches = new ArrayList<>();
List<IocFinding> iocFindings = new ArrayList<>();

for (SearchHit hit: searchResponse.getHits()) {
XContentParser xcp = XContentType.JSON.xContent()
.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString());
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp);
IocMatch iocMatch = IocMatch.parse(xcp);
iocMatches.add(iocMatch);

List<String> documentIds = iocMatch.getRelatedDocIds();
}

for (IocMatch iocMatch: iocMatches) {
iocMatchWithDocs.add(new IocMatchWithDocs(iocMatch, List.of()));
IocFinding iocFinding = IocFinding.parse(xcp);
iocFindings.add(iocFinding);
}
actionListener.onResponse(new GetIocFindingsResponse((int) totalIocFindingsCount, iocMatchWithDocs));
actionListener.onResponse(new GetIocFindingsResponse((int) totalIocFindingsCount, iocFindings));
} catch (Exception ex) {
this.onFailure(ex);
}
}

@Override
public void onFailure(Exception e) {
if (e instanceof IndexNotFoundException) {
actionListener.onResponse(new GetIocFindingsResponse(0, List.of()));
return;
}
actionListener.onFailure(e);
}
});
Expand Down
Loading

0 comments on commit b09afc1

Please sign in to comment.