[Connectors API] Relax strict response parsing for get/list operations

docs/changelog/104909.yaml

@@ -0,0 +1,5 @@
+pr: 104909
+summary: "[Connectors API] Relax strict response parsing for get/list operations"
+area: Application
+type: enhancement
+issues: []

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java

@@ -356,34 +356,38 @@ public static Connector fromXContent(XContentParser parser, String docId) throws
         return PARSER.parse(parser, docId);
     }
 
+    public void toInnerXContent(XContentBuilder builder, Params params) throws IOException {
+        // The "id": connectorId is included in GET and LIST responses to provide the connector's docID.
+        // Note: This ID is not written to the Elasticsearch index; it's only for API response purposes.
+        if (connectorId != null) {
+            builder.field(ID_FIELD.getPreferredName(), connectorId);
+        }
+        builder.field(API_KEY_ID_FIELD.getPreferredName(), apiKeyId);
+        builder.xContentValuesMap(CONFIGURATION_FIELD.getPreferredName(), configuration);
+        builder.xContentValuesMap(CUSTOM_SCHEDULING_FIELD.getPreferredName(), customScheduling);
+        builder.field(DESCRIPTION_FIELD.getPreferredName(), description);
+        builder.field(ERROR_FIELD.getPreferredName(), error);
+        builder.field(FEATURES_FIELD.getPreferredName(), features);
+        builder.xContentList(FILTERING_FIELD.getPreferredName(), filtering);
+        builder.field(INDEX_NAME_FIELD.getPreferredName(), indexName);
+        builder.field(IS_NATIVE_FIELD.getPreferredName(), isNative);
+        builder.field(LANGUAGE_FIELD.getPreferredName(), language);
+        builder.field(LAST_SEEN_FIELD.getPreferredName(), lastSeen);
+        syncInfo.toXContent(builder, params);
+        builder.field(NAME_FIELD.getPreferredName(), name);
+        builder.field(PIPELINE_FIELD.getPreferredName(), pipeline);
+        builder.field(SCHEDULING_FIELD.getPreferredName(), scheduling);
+        builder.field(SERVICE_TYPE_FIELD.getPreferredName(), serviceType);
+        builder.field(SYNC_CURSOR_FIELD.getPreferredName(), syncCursor);
+        builder.field(STATUS_FIELD.getPreferredName(), status.toString());
+        builder.field(SYNC_NOW_FIELD.getPreferredName(), syncNow);
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         {
-            // The "id": connectorId is included in GET and LIST responses to provide the connector's docID.
-            // Note: This ID is not written to the Elasticsearch index; it's only for API response purposes.
-            if (connectorId != null) {
-                builder.field(ID_FIELD.getPreferredName(), connectorId);
-            }
-            builder.field(API_KEY_ID_FIELD.getPreferredName(), apiKeyId);
-            builder.xContentValuesMap(CONFIGURATION_FIELD.getPreferredName(), configuration);
-            builder.xContentValuesMap(CUSTOM_SCHEDULING_FIELD.getPreferredName(), customScheduling);
-            builder.field(DESCRIPTION_FIELD.getPreferredName(), description);
-            builder.field(ERROR_FIELD.getPreferredName(), error);
-            builder.field(FEATURES_FIELD.getPreferredName(), features);
-            builder.xContentList(FILTERING_FIELD.getPreferredName(), filtering);
-            builder.field(INDEX_NAME_FIELD.getPreferredName(), indexName);
-            builder.field(IS_NATIVE_FIELD.getPreferredName(), isNative);
-            builder.field(LANGUAGE_FIELD.getPreferredName(), language);
-            builder.field(LAST_SEEN_FIELD.getPreferredName(), lastSeen);
-            syncInfo.toXContent(builder, params);
-            builder.field(NAME_FIELD.getPreferredName(), name);
-            builder.field(PIPELINE_FIELD.getPreferredName(), pipeline);
-            builder.field(SCHEDULING_FIELD.getPreferredName(), scheduling);
-            builder.field(SERVICE_TYPE_FIELD.getPreferredName(), serviceType);
-            builder.field(SYNC_CURSOR_FIELD.getPreferredName(), syncCursor);
-            builder.field(STATUS_FIELD.getPreferredName(), status.toString());
-            builder.field(SYNC_NOW_FIELD.getPreferredName(), syncNow);
+            toInnerXContent(builder, params);
         }
         builder.endObject();
         return builder;

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java

@@ -30,7 +30,6 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.application.connector.action.PostConnectorAction;
 import org.elasticsearch.xpack.application.connector.action.PutConnectorAction;
 import org.elasticsearch.xpack.application.connector.action.UpdateConnectorConfigurationAction;
@@ -175,7 +174,7 @@ private Connector createConnectorWithDefaultValues(
      * @param connectorId The id of the connector object.
      * @param listener    The action listener to invoke on response/failure.
      */
-    public void getConnector(String connectorId, ActionListener<Connector> listener) {
+    public void getConnector(String connectorId, ActionListener<ConnectorSearchResult> listener) {
         try {
             final GetRequest getRequest = new GetRequest(CONNECTOR_INDEX_NAME).id(connectorId).realtime(true);
 
@@ -185,11 +184,11 @@ public void getConnector(String connectorId, ActionListener<Connector> listener)
                     return;
                 }
                 try {
-                    final Connector connector = Connector.fromXContentBytes(
-                        getResponse.getSourceAsBytesRef(),
-                        connectorId,
-                        XContentType.JSON
-                    );
+                    final ConnectorSearchResult connector = new ConnectorSearchResult.Builder().setId(connectorId)
+                        .setResultBytes(getResponse.getSourceAsBytesRef())
+                        .setResultMap(getResponse.getSourceAsMap())
+                        .build();
+
                     l.onResponse(connector);
                 } catch (Exception e) {
                     listener.onFailure(e);
@@ -567,7 +566,9 @@ public void updateConnectorServiceType(UpdateConnectorServiceTypeAction.Request
             String connectorId = request.getConnectorId();
             getConnector(connectorId, listener.delegateFailure((l, connector) -> {
 
-                ConnectorStatus prevStatus = connector.getStatus();
+                ConnectorStatus prevStatus = ConnectorStatus.connectorStatus(
+                    (String) connector.getResultMap().get(Connector.STATUS_FIELD.getPreferredName())
+                );
                 ConnectorStatus newStatus = prevStatus == ConnectorStatus.CREATED
                     ? ConnectorStatus.CREATED
                     : ConnectorStatus.NEEDS_CONFIGURATION;
@@ -603,20 +604,23 @@ public void updateConnectorServiceType(UpdateConnectorServiceTypeAction.Request
     }
 
     private static ConnectorIndexService.ConnectorResult mapSearchResponseToConnectorList(SearchResponse response) {
-        final List<Connector> connectorResults = Arrays.stream(response.getHits().getHits())
+        final List<ConnectorSearchResult> connectorResults = Arrays.stream(response.getHits().getHits())
             .map(ConnectorIndexService::hitToConnector)
             .toList();
         return new ConnectorIndexService.ConnectorResult(connectorResults, (int) response.getHits().getTotalHits().value);
     }
 
-    private static Connector hitToConnector(SearchHit searchHit) {
+    private static ConnectorSearchResult hitToConnector(SearchHit searchHit) {
 
         // todo: don't return sensitive data from configuration in list endpoint
 
-        return Connector.fromXContentBytes(searchHit.getSourceRef(), searchHit.getId(), XContentType.JSON);
+        return new ConnectorSearchResult.Builder().setId(searchHit.getId())
+            .setResultBytes(searchHit.getSourceRef())
+            .setResultMap(searchHit.getSourceAsMap())
+            .build();
     }
 
-    public record ConnectorResult(List<Connector> connectors, long totalResults) {}
+    public record ConnectorResult(List<ConnectorSearchResult> connectors, long totalResults) {}
 
     /**
      * Listeners that checks failures for IndexNotFoundException, and transforms them in ResourceNotFoundException,

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorSearchResult.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.connector;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class ConnectorSearchResult extends ConnectorsAPISearchResult {
+
+    public ConnectorSearchResult(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    private ConnectorSearchResult(BytesReference resultBytes, Map<String, Object> resultMap, String id) {
+        super(resultBytes, resultMap, id);
+    }
+
+    public static class Builder {
+
+        private BytesReference resultBytes;
+        private Map<String, Object> resultMap;
+        private String id;
+
+        public Builder setResultBytes(BytesReference resultBytes) {
+            this.resultBytes = resultBytes;
+            return this;
+        }
+
+        public Builder setResultMap(Map<String, Object> resultMap) {
+            this.resultMap = resultMap;
+            return this;
+        }
+
+        public Builder setId(String id) {
+            this.id = id;
+            return this;
+        }
+
+        public ConnectorSearchResult build() {
+            return new ConnectorSearchResult(resultBytes, resultMap, id);
+        }
+    }
+}

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorsAPISearchResult.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.connector;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Represents the outcome of a search query in the connectors and sync job index, encapsulating the search result.
+ * It includes a raw byte reference to the result which can be deserialized into a {@link Connector} or {@link ConnectorSyncJob} object,
+ * and a result map for returning the data without strict deserialization.
+ */
+public class ConnectorsAPISearchResult implements Writeable, ToXContentObject {
+
+    private final BytesReference resultBytes;
+    private final Map<String, Object> resultMap;
+    private final String docId;
+
+    protected ConnectorsAPISearchResult(BytesReference resultBytes, Map<String, Object> resultMap, String id) {
+        this.resultBytes = resultBytes;
+        this.resultMap = resultMap;
+        this.docId = id;
+    }
+
+    public ConnectorsAPISearchResult(StreamInput in) throws IOException {
+        this.resultBytes = in.readBytesReference();
+        this.resultMap = in.readGenericMap();
+        this.docId = in.readString();
+    }
+
+    public BytesReference getSourceRef() {
+        return resultBytes;
+    }
+
+    public Map<String, Object> getResultMap() {
+        return resultMap;
+    }
+
+    public String getDocId() {
+        return docId;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        {
+            builder.field("id", docId);
+            builder.mapContents(resultMap);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeBytesReference(resultBytes);
+        out.writeGenericMap(resultMap);
+        out.writeString(docId);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ConnectorsAPISearchResult that = (ConnectorsAPISearchResult) o;
+        return Objects.equals(resultBytes, that.resultBytes)
+            && Objects.equals(resultMap, that.resultMap)
+            && Objects.equals(docId, that.docId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(resultBytes, resultMap, docId);
+    }
+}

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java

@@ -19,7 +19,7 @@
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.application.connector.Connector;
+import org.elasticsearch.xpack.application.connector.ConnectorSearchResult;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -110,15 +110,15 @@ public static Request parse(XContentParser parser) {
 
     public static class Response extends ActionResponse implements ToXContentObject {
 
-        private final Connector connector;
+        private final ConnectorSearchResult connector;
 
-        public Response(Connector connector) {
+        public Response(ConnectorSearchResult connector) {
             this.connector = connector;
         }
 
         public Response(StreamInput in) throws IOException {
             super(in);
-            this.connector = new Connector(in);
+            this.connector = new ConnectorSearchResult(in);
         }
 
         @Override
@@ -131,10 +131,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             return connector.toXContent(builder, params);
         }
 
-        public static GetConnectorAction.Response fromXContent(XContentParser parser, String docId) throws IOException {
-            return new GetConnectorAction.Response(Connector.fromXContent(parser, docId));
-        }
-
         @Override
         public boolean equals(Object o) {
             if (this == o) return true;

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java

@@ -18,7 +18,7 @@
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.application.connector.Connector;
+import org.elasticsearch.xpack.application.connector.ConnectorSearchResult;
 import org.elasticsearch.xpack.core.action.util.PageParams;
 import org.elasticsearch.xpack.core.action.util.QueryPage;
 
@@ -105,14 +105,14 @@ public static class Response extends ActionResponse implements ToXContentObject
 
         public static final ParseField RESULT_FIELD = new ParseField("results");
 
-        final QueryPage<Connector> queryPage;
+        final QueryPage<ConnectorSearchResult> queryPage;
 
         public Response(StreamInput in) throws IOException {
             super(in);
-            this.queryPage = new QueryPage<>(in, Connector::new);
+            this.queryPage = new QueryPage<>(in, ConnectorSearchResult::new);
         }
 
-        public Response(List<Connector> items, Long totalResults) {
+        public Response(List<ConnectorSearchResult> items, Long totalResults) {
             this.queryPage = new QueryPage<>(items, totalResults, RESULT_FIELD);
         }
 
@@ -126,7 +126,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             return queryPage.toXContent(builder, params);
         }
 
-        public QueryPage<Connector> queryPage() {
+        public QueryPage<ConnectorSearchResult> queryPage() {
             return queryPage;
         }