diff --git a/docs/changelog/104802.yaml b/docs/changelog/104802.yaml new file mode 100644 index 0000000000000..d535318043ca2 --- /dev/null +++ b/docs/changelog/104802.yaml @@ -0,0 +1,5 @@ +pr: 104802 +summary: "[Connectors API] Fix bug when triggering a sync job via API" +area: Application +type: bug +issues: [] diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/400_connector_sync_job_post.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/400_connector_sync_job_post.yml index 7a484ccca652a..8c280d3541f24 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/400_connector_sync_job_post.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/400_connector_sync_job_post.yml @@ -27,7 +27,7 @@ setup: connector_sync_job.get: connector_sync_job_id: $id - - match: { connector.id: test-connector} + - match: { connector.id: test-connector } - match: { job_type: full } - match: { trigger_method: on_demand } - match: { status: pending } @@ -39,6 +39,106 @@ setup: - exists: created_at - exists: last_seen +--- +'Create connector sync job with filtering': + - do: + connector.update_filtering: + connector_id: test-connector + body: + filtering: + - active: + advanced_snippet: + created_at: "2023-05-25T12:30:00.000Z" + updated_at: "2023-05-25T12:30:00.000Z" + value: { } + rules: + - created_at: "2023-05-25T12:30:00.000Z" + field: _ + id: RULE-ACTIVE-SYNC-JOB-TEST + order: 0 + policy: include + rule: regex + updated_at: "2023-05-25T12:30:00.000Z" + value: ".*" + validation: + errors: [ ] + state: valid + domain: DEFAULT + draft: + advanced_snippet: + created_at: "2023-05-25T12:30:00.000Z" + updated_at: "2023-05-25T12:30:00.000Z" + value: { } + rules: + - created_at: "2023-05-25T12:30:00.000Z" + field: _ + id: RULE-DRAFT-0 + order: 0 + policy: include + rule: regex + updated_at: "2023-05-25T12:30:00.000Z" + value: ".*" + validation: + errors: [ ] + state: valid + - active: + advanced_snippet: + created_at: "2021-05-25T12:30:00.000Z" + updated_at: "2021-05-25T12:30:00.000Z" + value: { } + rules: + - created_at: "2021-05-25T12:30:00.000Z" + field: _ + id: RULE-ACTIVE-1 + order: 0 + policy: include + rule: regex + updated_at: "2021-05-25T12:30:00.000Z" + value: ".*" + validation: + errors: [ ] + state: valid + domain: TEST + draft: + advanced_snippet: + created_at: "2021-05-25T12:30:00.000Z" + updated_at: "2021-05-25T12:30:00.000Z" + value: { } + rules: + - created_at: "2021-05-25T12:30:00.000Z" + field: _ + id: RULE-DRAFT-1 + order: 0 + policy: exclude + rule: regex + updated_at: "2021-05-25T12:30:00.000Z" + value: ".*" + validation: + errors: [ ] + state: valid + + - match: { result: updated } + + - do: + connector_sync_job.post: + body: + id: test-connector + job_type: full + trigger_method: on_demand + + - set: { id: id } + + - match: { id: $id } + + - do: + connector_sync_job.get: + connector_sync_job_id: $id + + - match: { connector.filtering.rules.0.id: RULE-ACTIVE-SYNC-JOB-TEST } + - match: { connector.filtering.rules.0.rule: regex } + - match: { connector.filtering.validation.state: valid } + - match: { connector.filtering.advanced_snippet.created_at: "2023-05-25T12:30:00.000Z" } + --- 'Create connector sync job with complex connector document': diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java index 74d9be8db0fac..cc7ea5e0bf988 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java @@ -22,6 +22,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.connector.filtering.FilteringRules; import java.io.IOException; import java.time.Instant; @@ -47,6 +48,7 @@ *
  • An error string capturing the latest error encountered during the connector's operation, if any.
    • *
  • A {@link ConnectorFeatures} object encapsulating the set of features enabled for this connector.
    • *
  • A list of {@link ConnectorFiltering} objects for applying filtering rules to the data processed by the connector.
    • + *
  • An optional {@link FilteringRules} object that represents active filtering rules applied to a sync job.
    • *
  • The name of the Elasticsearch index where the synchronized data is stored or managed.
    • *
  • A boolean flag 'isNative' indicating whether the connector is a native Elasticsearch connector.
    • *
  • The language associated with the connector.
    • @@ -79,6 +81,8 @@ public class Connector implements NamedWriteable, ToXContentObject { private final ConnectorFeatures features; private final List filtering; @Nullable + private final FilteringRules syncJobFiltering; + @Nullable private final String indexName; private final boolean isNative; @Nullable @@ -110,6 +114,7 @@ public class Connector implements NamedWriteable, ToXContentObject { * @param error Information about the last error encountered by the connector, if any. * @param features Features enabled for the connector. * @param filtering Filtering settings applied by the connector. + * @param syncJobFiltering Filtering settings used by a sync job, it contains subset of data from 'filtering'. * @param indexName Name of the index associated with the connector. * @param isNative Flag indicating whether the connector is a native type. * @param language The language supported by the connector. @@ -132,6 +137,7 @@ private Connector( String error, ConnectorFeatures features, List filtering, + FilteringRules syncJobFiltering, String indexName, boolean isNative, String language, @@ -152,7 +158,8 @@ private Connector( this.description = description; this.error = error; this.features = features; - this.filtering = Objects.requireNonNull(filtering, "[filtering] cannot be null"); + this.filtering = filtering; + this.syncJobFiltering = syncJobFiltering; this.indexName = indexName; this.isNative = isNative; this.language = language; @@ -176,6 +183,7 @@ public Connector(StreamInput in) throws IOException { this.error = in.readOptionalString(); this.features = in.readOptionalWriteable(ConnectorFeatures::new); this.filtering = in.readOptionalCollectionAsList(ConnectorFiltering::new); + this.syncJobFiltering = in.readOptionalWriteable(FilteringRules::new); this.indexName = in.readOptionalString(); this.isNative = in.readBoolean(); this.language = in.readOptionalString(); @@ -388,6 +396,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(error); out.writeOptionalWriteable(features); out.writeOptionalCollection(filtering); + out.writeOptionalWriteable(syncJobFiltering); out.writeOptionalString(indexName); out.writeBoolean(isNative); out.writeOptionalString(language); @@ -434,6 +443,10 @@ public List getFiltering() { return filtering; } + public FilteringRules getSyncJobFiltering() { + return syncJobFiltering; + } + public String getIndexName() { return indexName; } @@ -497,6 +510,7 @@ public boolean equals(Object o) { && Objects.equals(error, connector.error) && Objects.equals(features, connector.features) && Objects.equals(filtering, connector.filtering) + && Objects.equals(syncJobFiltering, connector.syncJobFiltering) && Objects.equals(indexName, connector.indexName) && Objects.equals(language, connector.language) && Objects.equals(lastSeen, connector.lastSeen) @@ -520,6 +534,7 @@ public int hashCode() { error, features, filtering, + syncJobFiltering, indexName, isNative, language, @@ -550,6 +565,7 @@ public static class Builder { private String error; private ConnectorFeatures features; private List filtering = List.of(ConnectorFiltering.getDefaultConnectorFilteringConfig()); + private FilteringRules syncJobFiltering; private String indexName; private boolean isNative = false; private String language; @@ -603,6 +619,11 @@ public Builder setFiltering(List filtering) { return this; } + public Builder setSyncJobFiltering(FilteringRules syncJobFiltering) { + this.syncJobFiltering = syncJobFiltering; + return this; + } + public Builder setIndexName(String indexName) { this.indexName = indexName; return this; @@ -676,6 +697,7 @@ public Connector build() { error, features, filtering, + syncJobFiltering, indexName, isNative, language, diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFiltering.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFiltering.java index 8ade6cdbcc0b1..62a8a68cea5ca 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFiltering.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFiltering.java @@ -66,6 +66,18 @@ public ConnectorFiltering(StreamInput in) throws IOException { this.draft = new FilteringRules(in); } + public FilteringRules getActive() { + return active; + } + + public String getDomain() { + return domain; + } + + public FilteringRules getDraft() { + return draft; + } + private static final ParseField ACTIVE_FIELD = new ParseField("active"); private static final ParseField DOMAIN_FIELD = new ParseField("domain"); private static final ParseField DRAFT_FIELD = new ParseField("draft"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/filtering/FilteringRules.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/filtering/FilteringRules.java index dc96006f40349..fb4e25131449d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/filtering/FilteringRules.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/filtering/FilteringRules.java @@ -57,6 +57,18 @@ public FilteringRules(StreamInput in) throws IOException { this.filteringValidationInfo = new FilteringValidationInfo(in); } + public FilteringAdvancedSnippet getAdvancedSnippet() { + return advancedSnippet; + } + + public List getRules() { + return rules; + } + + public FilteringValidationInfo getFilteringValidationInfo() { + return filteringValidationInfo; + } + private static final ParseField ADVANCED_SNIPPET_FIELD = new ParseField("advanced_snippet"); private static final ParseField RULES_FIELD = new ParseField("rules"); private static final ParseField VALIDATION_FIELD = new ParseField("validation"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java index 84d91b7fe0f08..d480c975c76f9 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java @@ -25,15 +25,14 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.ConnectorConfiguration; -import org.elasticsearch.xpack.application.connector.ConnectorFiltering; import org.elasticsearch.xpack.application.connector.ConnectorIngestPipeline; import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus; +import org.elasticsearch.xpack.application.connector.filtering.FilteringRules; import java.io.IOException; import java.time.Instant; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Objects; @@ -343,21 +342,23 @@ private static Instant parseNullableInstant(XContentParser p) throws IOException String syncJobConnectorId = Strings.isNullOrEmpty(connectorId) ? parsedConnectorId : connectorId; return new Connector.Builder().setConnectorId(syncJobConnectorId) - .setFiltering((List) args[i++]) + .setFiltering(null) + .setSyncJobFiltering((FilteringRules) args[i++]) .setIndexName((String) args[i++]) .setLanguage((String) args[i++]) .setPipeline((ConnectorIngestPipeline) args[i++]) .setServiceType((String) args[i++]) - .setConfiguration((Map) args[i++]) + .setConfiguration((Map) args[i]) .build(); } ); static { SYNC_JOB_CONNECTOR_PARSER.declareString(optionalConstructorArg(), Connector.ID_FIELD); - SYNC_JOB_CONNECTOR_PARSER.declareObjectArray( + SYNC_JOB_CONNECTOR_PARSER.declareObjectOrNull( optionalConstructorArg(), - (p, c) -> ConnectorFiltering.fromXContent(p), + (p, c) -> FilteringRules.fromXContent(p), + null, Connector.FILTERING_FIELD ); SYNC_JOB_CONNECTOR_PARSER.declareStringOrNull(optionalConstructorArg(), Connector.INDEX_NAME_FIELD); @@ -491,8 +492,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (connector.getConnectorId() != null) { builder.field(Connector.ID_FIELD.getPreferredName(), connector.getConnectorId()); } - if (connector.getFiltering() != null) { - builder.field(Connector.FILTERING_FIELD.getPreferredName(), connector.getFiltering()); + if (connector.getSyncJobFiltering() != null) { + builder.field(Connector.FILTERING_FIELD.getPreferredName(), connector.getSyncJobFiltering()); } if (connector.getIndexName() != null) { builder.field(Connector.INDEX_NAME_FIELD.getPreferredName(), connector.getIndexName()); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java index ee35d8fb6372c..f549008b26d0b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java @@ -38,9 +38,11 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.application.connector.Connector; +import org.elasticsearch.xpack.application.connector.ConnectorFiltering; import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus; import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; +import org.elasticsearch.xpack.application.connector.filtering.FilteringRules; import org.elasticsearch.xpack.application.connector.syncjob.action.PostConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.UpdateConnectorSyncJobIngestionStatsAction; @@ -52,6 +54,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.function.BiConsumer; import java.util.stream.Stream; @@ -427,12 +430,24 @@ public void onResponse(GetResponse response) { return; } try { - final Connector syncJobConnectorInfo = ConnectorSyncJob.syncJobConnectorFromXContentBytes( + final Connector connector = Connector.fromXContentBytes( response.getSourceAsBytesRef(), connectorId, XContentType.JSON ); - listener.onResponse(syncJobConnectorInfo); + + // Build the connector representation for sync job + final Connector syncJobConnector = new Connector.Builder().setConnectorId(connector.getConnectorId()) + .setFiltering(null) + .setSyncJobFiltering(transformConnectorFilteringToSyncJobRepresentation(connector.getFiltering())) + .setIndexName(connector.getIndexName()) + .setLanguage(connector.getLanguage()) + .setPipeline(connector.getPipeline()) + .setServiceType(connector.getServiceType()) + .setConfiguration(connector.getConfiguration()) + .build(); + + listener.onResponse(syncJobConnector); } catch (Exception e) { listener.onFailure(e); } @@ -448,6 +463,20 @@ public void onFailure(Exception e) { } } + /** + * Transforms the first {@link ConnectorFiltering} object from a list into a {@link FilteringRules} representation for a sync job. + * This method specifically extracts the 'active' filtering rules from the first {@link ConnectorFiltering} object in the list, + * if the list is neither null nor empty. + * + * @param connectorFiltering The list of {@link ConnectorFiltering} objects to be transformed. Can be null or empty. + */ + FilteringRules transformConnectorFilteringToSyncJobRepresentation(List connectorFiltering) { + return Optional.ofNullable(connectorFiltering) + .filter(list -> list.isEmpty() == false) + .map(list -> list.get(0).getActive()) + .orElse(null); + } + /** * Sets the error for the {@link ConnectorSyncJob} in the underlying index. * This also sets the {@link ConnectorSyncStatus} to 'ERROR'. diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java index 6a16e6f183383..d9c53c8aaf0ea 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java @@ -187,8 +187,10 @@ public static ConnectorFiltering getRandomConnectorFiltering() { } public static Connector getRandomSyncJobConnectorInfo() { + ConnectorFiltering randomFiltering = getRandomConnectorFiltering(); return new Connector.Builder().setConnectorId(randomAlphaOfLength(10)) - .setFiltering(List.of(getRandomConnectorFiltering())) + .setSyncJobFiltering(randomFiltering.getActive()) + .setFiltering(List.of(randomFiltering)) .setIndexName(randomAlphaOfLength(10)) .setLanguage(randomAlphaOfLength(10)) .setServiceType(randomAlphaOfLength(10)) diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java index 85d8826b98683..170ed25c0b302 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xpack.application.connector.Connector; +import org.elasticsearch.xpack.application.connector.ConnectorFiltering; import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; @@ -35,6 +36,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -62,34 +64,35 @@ public class ConnectorSyncJobIndexServiceTests extends ESSingleNodeTestCase { private static final int ONE_SECOND_IN_MILLIS = 1000; private ConnectorSyncJobIndexService connectorSyncJobIndexService; - private Connector connectorOne; - private Connector connectorTwo; + + private String connectorOneId; + private String connectorTwoId; @Before public void setup() throws Exception { - connectorOne = ConnectorTestUtils.getRandomSyncJobConnectorInfo(); - connectorTwo = ConnectorTestUtils.getRandomSyncJobConnectorInfo(); - createConnector(connectorOne); - createConnector(connectorTwo); + connectorOneId = createConnector(); + connectorTwoId = createConnector(); this.connectorSyncJobIndexService = new ConnectorSyncJobIndexService(client()); } - private void createConnector(Connector connector) throws IOException, InterruptedException, ExecutionException, TimeoutException { + private String createConnector() throws IOException, InterruptedException, ExecutionException, TimeoutException { + + Connector connector = ConnectorTestUtils.getRandomConnector(); + final IndexRequest indexRequest = new IndexRequest(ConnectorIndexService.CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(connector.getConnectorId()) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(connector.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); ActionFuture index = client().index(indexRequest); // wait 10 seconds for connector creation - index.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + return index.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).getId(); } public void testCreateConnectorSyncJob() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( - connectorOne.getConnectorId() + connectorOneId ); ConnectorSyncJobType requestJobType = syncJobRequest.getJobType(); ConnectorSyncJobTriggerMethod requestTriggerMethod = syncJobRequest.getTriggerMethod(); @@ -110,7 +113,7 @@ public void testCreateConnectorSyncJob() throws Exception { public void testCreateConnectorSyncJob_WithMissingJobType_ExpectDefaultJobTypeToBeSet() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = new PostConnectorSyncJobAction.Request( - connectorOne.getConnectorId(), + connectorOneId, null, ConnectorSyncJobTriggerMethod.ON_DEMAND ); @@ -123,7 +126,7 @@ public void testCreateConnectorSyncJob_WithMissingJobType_ExpectDefaultJobTypeTo public void testCreateConnectorSyncJob_WithMissingTriggerMethod_ExpectDefaultTriggerMethodToBeSet() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = new PostConnectorSyncJobAction.Request( - connectorOne.getConnectorId(), + connectorOneId, ConnectorSyncJobType.FULL, null ); @@ -148,7 +151,7 @@ public void testCreateConnectorSyncJob_WithMissingConnectorId_ExpectException() public void testDeleteConnectorSyncJob() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( - connectorOne.getConnectorId() + connectorOneId ); PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); String syncJobId = response.getId(); @@ -166,7 +169,7 @@ public void testDeleteConnectorSyncJob_WithMissingSyncJobId_ExpectException() { public void testGetConnectorSyncJob() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( - connectorOne.getConnectorId() + connectorOneId ); ConnectorSyncJobType jobType = syncJobRequest.getJobType(); ConnectorSyncJobTriggerMethod triggerMethod = syncJobRequest.getTriggerMethod(); @@ -179,7 +182,7 @@ public void testGetConnectorSyncJob() throws Exception { assertThat(syncJob.getId(), equalTo(syncJobId)); assertThat(syncJob.getJobType(), equalTo(jobType)); assertThat(syncJob.getTriggerMethod(), equalTo(triggerMethod)); - assertThat(syncJob.getConnector().getConnectorId(), equalTo(connectorOne.getConnectorId())); + assertThat(syncJob.getConnector().getConnectorId(), equalTo(connectorOneId)); } public void testGetConnectorSyncJob_WithMissingSyncJobId_ExpectException() { @@ -188,7 +191,7 @@ public void testGetConnectorSyncJob_WithMissingSyncJobId_ExpectException() { public void testCheckInConnectorSyncJob() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( - connectorOne.getConnectorId() + connectorOneId ); PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); String syncJobId = response.getId(); @@ -227,7 +230,7 @@ public void testCheckInConnectorSyncJob_WithMissingSyncJobId_ExpectException() { public void testCancelConnectorSyncJob() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( - connectorOne.getConnectorId() + connectorOneId ); PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); String syncJobId = response.getId(); @@ -269,7 +272,7 @@ public void testListConnectorSyncJobs() throws Exception { for (int i = 0; i < numberOfSyncJobs; i++) { PostConnectorSyncJobAction.Request request = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( - connectorOne.getConnectorId() + connectorOneId ); PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(request); ConnectorSyncJob syncJob = awaitGetConnectorSyncJob(response.getId()); @@ -309,7 +312,7 @@ public void testListConnectorSyncJobs() throws Exception { } public void testListConnectorSyncJobs_WithStatusPending_GivenOnePendingTwoCancelled_ExpectOnePending() throws Exception { - String connectorId = connectorOne.getConnectorId(); + String connectorId = connectorOneId; PostConnectorSyncJobAction.Request requestOne = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest(connectorId); PostConnectorSyncJobAction.Request requestTwo = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest(connectorId); @@ -342,9 +345,6 @@ public void testListConnectorSyncJobs_WithStatusPending_GivenOnePendingTwoCancel @AwaitsFix(bugUrl = "https://github.com/elastic/enterprise-search-team/issues/6351") public void testListConnectorSyncJobs_WithConnectorOneId_GivenTwoOverallOneFromConnectorOne_ExpectOne() throws Exception { - String connectorOneId = connectorOne.getConnectorId(); - String connectorTwoId = connectorTwo.getConnectorId(); - PostConnectorSyncJobAction.Request requestOne = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( connectorOneId ); @@ -378,7 +378,7 @@ public void testListConnectorSyncJobs_WithNoSyncJobs_ReturnEmptyResult() throws public void testUpdateConnectorSyncJobError() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( - connectorOne.getConnectorId() + connectorOneId ); PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); String syncJobId = response.getId(); @@ -407,7 +407,7 @@ public void testUpdateConnectorSyncJobError_WithMissingSyncJobId_ExceptException public void testUpdateConnectorSyncJobIngestionStats() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( - connectorOne.getConnectorId() + connectorOneId ); PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); String syncJobId = response.getId(); @@ -451,7 +451,7 @@ public void testUpdateConnectorSyncJobIngestionStats() throws Exception { public void testUpdateConnectorSyncJobIngestionStats_WithoutLastSeen_ExpectUpdateOfLastSeen() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( - connectorOne.getConnectorId() + connectorOneId ); PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); String syncJobId = response.getId(); @@ -492,6 +492,23 @@ public void testUpdateConnectorSyncJobIngestionStats_WithMissingSyncJobId_Expect ); } + public void testTransformConnectorFilteringToSyncJobRepresentation_WithFilteringEqualNull() { + List filtering = null; + assertNull(connectorSyncJobIndexService.transformConnectorFilteringToSyncJobRepresentation(filtering)); + } + + public void testTransformConnectorFilteringToSyncJobRepresentation_WithFilteringEmpty() { + List filtering = Collections.emptyList(); + assertNull(connectorSyncJobIndexService.transformConnectorFilteringToSyncJobRepresentation(filtering)); + } + + public void testTransformConnectorFilteringToSyncJobRepresentation_WithFilteringRules() { + ConnectorFiltering filtering1 = ConnectorTestUtils.getRandomConnectorFiltering(); + + List filtering = List.of(filtering1, ConnectorTestUtils.getRandomConnectorFiltering()); + assertEquals(connectorSyncJobIndexService.transformConnectorFilteringToSyncJobRepresentation(filtering), filtering1.getActive()); + } + private UpdateResponse awaitUpdateConnectorSyncJobIngestionStats(UpdateConnectorSyncJobIngestionStatsAction.Request request) throws Exception { CountDownLatch latch = new CountDownLatch(1); diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java index b82db8d04d3a9..64f11923ce164 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java @@ -21,7 +21,6 @@ import java.util.List; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.notNullValue; @@ -46,88 +45,60 @@ public final void testRandomSerialization() throws IOException { public void testFromXContent_WithAllFields_AllSet() throws IOException { String content = XContentHelper.stripWhitespace(""" { - "cancelation_requested_at": "2023-12-01T14:19:39.394194Z", - "canceled_at": "2023-12-01T14:19:39.394194Z", - "completed_at": "2023-12-01T14:19:39.394194Z", - "connector": { - "id": "connector-id", - "filtering": [ - { - "active": { - "advanced_snippet": { - "created_at": "2023-12-01T14:18:37.397819Z", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": {} - }, - "rules": [ - { - "created_at": "2023-12-01T14:18:37.397819Z", - "field": "_", - "id": "DEFAULT", - "order": 0, - "policy": "include", - "rule": "regex", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": ".*" - } - ], - "validation": { - "errors": [], - "state": "valid" - } - }, - "domain": "DEFAULT", - "draft": { - "advanced_snippet": { - "created_at": "2023-12-01T14:18:37.397819Z", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": {} - }, - "rules": [ - { - "created_at": "2023-12-01T14:18:37.397819Z", - "field": "_", - "id": "DEFAULT", - "order": 0, - "policy": "include", - "rule": "regex", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": ".*" - } - ], - "validation": { - "errors": [], - "state": "valid" - } - } - } - ], - "index_name": "search-connector", - "language": "english", - "pipeline": { - "extract_binary_content": true, - "name": "ent-search-generic-ingestion", - "reduce_whitespace": true, - "run_ml_inference": false - }, - "service_type": "service type", - "configuration": {} - }, - "created_at": "2023-12-01T14:18:43.07693Z", - "deleted_document_count": 10, - "error": "some-error", - "id": "HIC-JYwB9RqKhB7x_hIE", - "indexed_document_count": 10, - "indexed_document_volume": 10, - "job_type": "full", - "last_seen": "2023-12-01T14:18:43.07693Z", - "metadata": {}, - "started_at": "2023-12-01T14:18:43.07693Z", - "status": "canceling", - "total_document_count": 0, - "trigger_method": "scheduled", - "worker_hostname": "worker-hostname" - } + "cancelation_requested_at": "2023-12-01T14:19:39.394194Z", + "canceled_at": "2023-12-01T14:19:39.394194Z", + "completed_at": "2023-12-01T14:19:39.394194Z", + "connector": { + "id": "connector-id", + "filtering": { + "advanced_snippet": { + "created_at": "2023-12-01T14:18:37.397819Z", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": {} + }, + "rules": [ + { + "created_at": "2023-12-01T14:18:37.397819Z", + "field": "_", + "id": "DEFAULT", + "order": 0, + "policy": "include", + "rule": "regex", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": ".*" + } + ], + "validation": { + "errors": [], + "state": "valid" + } + }, + "index_name": "search-connector", + "language": "english", + "pipeline": { + "extract_binary_content": true, + "name": "ent-search-generic-ingestion", + "reduce_whitespace": true, + "run_ml_inference": false + }, + "service_type": "service type", + "configuration": {} + }, + "created_at": "2023-12-01T14:18:43.07693Z", + "deleted_document_count": 10, + "error": "some-error", + "id": "HIC-JYwB9RqKhB7x_hIE", + "indexed_document_count": 10, + "indexed_document_volume": 10, + "job_type": "full", + "last_seen": "2023-12-01T14:18:43.07693Z", + "metadata": {}, + "started_at": "2023-12-01T14:18:43.07693Z", + "status": "canceling", + "total_document_count": 0, + "trigger_method": "scheduled", + "worker_hostname": "worker-hostname" + } """); ConnectorSyncJob syncJob = ConnectorSyncJob.fromXContentBytes(new BytesArray(content), XContentType.JSON); @@ -137,7 +108,7 @@ public void testFromXContent_WithAllFields_AllSet() throws IOException { assertThat(syncJob.getCompletedAt(), equalTo(Instant.parse("2023-12-01T14:19:39.394194Z"))); assertThat(syncJob.getConnector().getConnectorId(), equalTo("connector-id")); - assertThat(syncJob.getConnector().getFiltering(), hasSize(greaterThan(0))); + assertThat(syncJob.getConnector().getSyncJobFiltering().getRules(), hasSize(1)); assertThat(syncJob.getConnector().getIndexName(), equalTo("search-connector")); assertThat(syncJob.getConnector().getLanguage(), equalTo("english")); assertThat(syncJob.getConnector().getPipeline(), notNullValue()); @@ -161,82 +132,54 @@ public void testFromXContent_WithAllFields_AllSet() throws IOException { public void testFromXContent_WithOnlyNonNullableFieldsSet_DoesNotThrow() throws IOException { String content = XContentHelper.stripWhitespace(""" { - "connector": { - "id": "connector-id", - "filtering": [ - { - "active": { - "advanced_snippet": { - "created_at": "2023-12-01T14:18:37.397819Z", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": {} - }, - "rules": [ - { - "created_at": "2023-12-01T14:18:37.397819Z", - "field": "_", - "id": "DEFAULT", - "order": 0, - "policy": "include", - "rule": "regex", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": ".*" - } - ], - "validation": { - "errors": [], - "state": "valid" - } - }, - "domain": "DEFAULT", - "draft": { - "advanced_snippet": { - "created_at": "2023-12-01T14:18:37.397819Z", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": {} - }, - "rules": [ - { - "created_at": "2023-12-01T14:18:37.397819Z", - "field": "_", - "id": "DEFAULT", - "order": 0, - "policy": "include", - "rule": "regex", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": ".*" - } - ], - "validation": { - "errors": [], - "state": "valid" - } - } - } - ], - "index_name": "search-connector", - "language": "english", - "pipeline": { - "extract_binary_content": true, - "name": "ent-search-generic-ingestion", - "reduce_whitespace": true, - "run_ml_inference": false - }, - "service_type": "service type", - "configuration": {} - }, - "created_at": "2023-12-01T14:18:43.07693Z", - "deleted_document_count": 10, - "id": "HIC-JYwB9RqKhB7x_hIE", - "indexed_document_count": 10, - "indexed_document_volume": 10, - "job_type": "full", - "last_seen": "2023-12-01T14:18:43.07693Z", - "metadata": {}, - "status": "canceling", - "total_document_count": 0, - "trigger_method": "scheduled" - } + "connector": { + "id": "connector-id", + "filtering": { + "advanced_snippet": { + "created_at": "2023-12-01T14:18:37.397819Z", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": {} + }, + "rules": [ + { + "created_at": "2023-12-01T14:18:37.397819Z", + "field": "_", + "id": "DEFAULT", + "order": 0, + "policy": "include", + "rule": "regex", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": ".*" + } + ], + "validation": { + "errors": [], + "state": "valid" + } + }, + "index_name": "search-connector", + "language": "english", + "pipeline": { + "extract_binary_content": true, + "name": "ent-search-generic-ingestion", + "reduce_whitespace": true, + "run_ml_inference": false + }, + "service_type": "service type", + "configuration": {} + }, + "created_at": "2023-12-01T14:18:43.07693Z", + "deleted_document_count": 10, + "id": "HIC-JYwB9RqKhB7x_hIE", + "indexed_document_count": 10, + "indexed_document_volume": 10, + "job_type": "full", + "last_seen": "2023-12-01T14:18:43.07693Z", + "metadata": {}, + "status": "canceling", + "total_document_count": 0, + "trigger_method": "scheduled" + } """); ConnectorSyncJob.fromXContentBytes(new BytesArray(content), XContentType.JSON); @@ -245,88 +188,60 @@ public void testFromXContent_WithOnlyNonNullableFieldsSet_DoesNotThrow() throws public void testFromXContent_WithAllNullableFieldsSetToNull_DoesNotThrow() throws IOException { String content = XContentHelper.stripWhitespace(""" { - "cancelation_requested_at": null, - "canceled_at": null, - "completed_at": null, - "connector": { - "id": "connector-id", - "filtering": [ - { - "active": { - "advanced_snippet": { - "created_at": "2023-12-01T14:18:37.397819Z", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": {} - }, - "rules": [ - { - "created_at": "2023-12-01T14:18:37.397819Z", - "field": "_", - "id": "DEFAULT", - "order": 0, - "policy": "include", - "rule": "regex", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": ".*" - } - ], - "validation": { - "errors": [], - "state": "valid" - } - }, - "domain": "DEFAULT", - "draft": { - "advanced_snippet": { - "created_at": "2023-12-01T14:18:37.397819Z", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": {} - }, - "rules": [ - { - "created_at": "2023-12-01T14:18:37.397819Z", - "field": "_", - "id": "DEFAULT", - "order": 0, - "policy": "include", - "rule": "regex", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": ".*" - } - ], - "validation": { - "errors": [], - "state": "valid" - } - } - } - ], - "index_name": "search-connector", - "language": "english", - "pipeline": { - "extract_binary_content": true, - "name": "ent-search-generic-ingestion", - "reduce_whitespace": true, - "run_ml_inference": false - }, - "service_type": "service type", - "configuration": {} - }, - "created_at": "2023-12-01T14:18:43.07693Z", - "deleted_document_count": 10, - "error": null, - "id": "HIC-JYwB9RqKhB7x_hIE", - "indexed_document_count": 10, - "indexed_document_volume": 10, - "job_type": "full", - "last_seen": null, - "metadata": {}, - "started_at": null, - "status": "canceling", - "total_document_count": 0, - "trigger_method": "scheduled", - "worker_hostname": null - } + "cancelation_requested_at": null, + "canceled_at": null, + "completed_at": null, + "connector": { + "id": "connector-id", + "filtering": { + "advanced_snippet": { + "created_at": "2023-12-01T14:18:37.397819Z", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": {} + }, + "rules": [ + { + "created_at": "2023-12-01T14:18:37.397819Z", + "field": "_", + "id": "DEFAULT", + "order": 0, + "policy": "include", + "rule": "regex", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": ".*" + } + ], + "validation": { + "errors": [], + "state": "valid" + } + }, + "index_name": "search-connector", + "language": "english", + "pipeline": { + "extract_binary_content": true, + "name": "ent-search-generic-ingestion", + "reduce_whitespace": true, + "run_ml_inference": false + }, + "service_type": "service type", + "configuration": {} + }, + "created_at": "2023-12-01T14:18:43.07693Z", + "deleted_document_count": 10, + "error": null, + "id": "HIC-JYwB9RqKhB7x_hIE", + "indexed_document_count": 10, + "indexed_document_volume": 10, + "job_type": "full", + "last_seen": null, + "metadata": {}, + "started_at": null, + "status": "canceling", + "total_document_count": 0, + "trigger_method": "scheduled", + "worker_hostname": null + } """); ConnectorSyncJob.fromXContentBytes(new BytesArray(content), XContentType.JSON); @@ -336,57 +251,29 @@ public void testSyncJobConnectorFromXContent_WithAllFieldsSet() throws IOExcepti String content = XContentHelper.stripWhitespace(""" { "id": "connector-id", - "filtering": [ - { - "active": { - "advanced_snippet": { - "created_at": "2023-12-01T14:18:37.397819Z", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": {} - }, - "rules": [ - { - "created_at": "2023-12-01T14:18:37.397819Z", - "field": "_", - "id": "DEFAULT", - "order": 0, - "policy": "include", - "rule": "regex", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": ".*" - } - ], - "validation": { - "errors": [], - "state": "valid" - } - }, - "domain": "DEFAULT", - "draft": { - "advanced_snippet": { - "created_at": "2023-12-01T14:18:37.397819Z", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": {} - }, - "rules": [ - { - "created_at": "2023-12-01T14:18:37.397819Z", - "field": "_", - "id": "DEFAULT", - "order": 0, - "policy": "include", - "rule": "regex", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": ".*" - } - ], - "validation": { - "errors": [], - "state": "valid" - } + "filtering": { + "advanced_snippet": { + "created_at": "2023-12-01T14:18:37.397819Z", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": {} + }, + "rules": [ + { + "created_at": "2023-12-01T14:18:37.397819Z", + "field": "_", + "id": "DEFAULT", + "order": 0, + "policy": "include", + "rule": "regex", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": ".*" } + ], + "validation": { + "errors": [], + "state": "valid" } - ], + }, "index_name": "search-connector", "language": "english", "pipeline": { @@ -403,7 +290,7 @@ public void testSyncJobConnectorFromXContent_WithAllFieldsSet() throws IOExcepti Connector connector = ConnectorSyncJob.syncJobConnectorFromXContentBytes(new BytesArray(content), null, XContentType.JSON); assertThat(connector.getConnectorId(), equalTo("connector-id")); - assertThat(connector.getFiltering().size(), equalTo(1)); + assertThat(connector.getSyncJobFiltering().getRules(), hasSize(1)); assertThat(connector.getIndexName(), equalTo("search-connector")); assertThat(connector.getLanguage(), equalTo("english")); assertThat(connector.getPipeline(), notNullValue()); @@ -415,57 +302,29 @@ public void testSyncJobConnectorFromXContent_WithAllNonOptionalFieldsSet_DoesNot String content = XContentHelper.stripWhitespace(""" { "id": "connector-id", - "filtering": [ - { - "active": { - "advanced_snippet": { - "created_at": "2023-12-01T14:18:37.397819Z", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": {} - }, - "rules": [ - { - "created_at": "2023-12-01T14:18:37.397819Z", - "field": "_", - "id": "DEFAULT", - "order": 0, - "policy": "include", - "rule": "regex", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": ".*" - } - ], - "validation": { - "errors": [], - "state": "valid" - } - }, - "domain": "DEFAULT", - "draft": { - "advanced_snippet": { - "created_at": "2023-12-01T14:18:37.397819Z", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": {} - }, - "rules": [ - { - "created_at": "2023-12-01T14:18:37.397819Z", - "field": "_", - "id": "DEFAULT", - "order": 0, - "policy": "include", - "rule": "regex", - "updated_at": "2023-12-01T14:18:37.397819Z", - "value": ".*" - } - ], - "validation": { - "errors": [], - "state": "valid" - } + "filtering": { + "advanced_snippet": { + "created_at": "2023-12-01T14:18:37.397819Z", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": {} + }, + "rules": [ + { + "created_at": "2023-12-01T14:18:37.397819Z", + "field": "_", + "id": "DEFAULT", + "order": 0, + "policy": "include", + "rule": "regex", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": ".*" } + ], + "validation": { + "errors": [], + "state": "valid" } - ], + }, "index_name": null, "language": null, "pipeline": null,