Return a 410 (Gone) status code for unavailable API endpoints (#97397)

Updates RestController to return a 410 (Gone) status code for known REST
handlers that are unavailable in the current environment (typically
serverless). Also uses ElasticsearchException to return the detailed format
by default, that contains more useful information for caller-side
diagnostics and logs (in particular the "type" property).

docs/changelog/97397.yaml

@@ -0,0 +1,5 @@
+pr: 97397
+summary: Return a 410 (Gone) status code for unavailable API endpoints
+area: Infra/REST API
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/ElasticsearchException.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.mapper.DocumentParsingException;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.rest.ApiNotAvailableException;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchException;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
@@ -1844,7 +1845,8 @@ private enum ElasticsearchExceptionHandle {
             ElasticsearchRoleRestrictionException::new,
             170,
             TransportVersion.V_8_500_016
-        );
+        ),
+        API_NOT_AVAILABLE_EXCEPTION(ApiNotAvailableException.class, ApiNotAvailableException::new, 171, TransportVersion.V_8_500_065);
 
         final Class<? extends ElasticsearchException> exceptionClass;
         final CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException> constructor;

server/src/main/java/org/elasticsearch/TransportVersion.java

@@ -178,6 +178,7 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId
     public static final TransportVersion V_8_500_062 = registerTransportVersion(8_500_062, "09CD9C9B-3207-4B40-8756-B7A12001A885");
     public static final TransportVersion V_8_500_063 = registerTransportVersion(8_500_063, "31dedced-0055-4f34-b952-2f6919be7488");
     public static final TransportVersion V_8_500_064 = registerTransportVersion(8_500_064, "3a795175-5e6f-40ff-90fe-5571ea8ab04e");
+    public static final TransportVersion V_8_500_065 = registerTransportVersion(8_500_065, "4e253c58-1b3d-11ee-be56-0242ac120002");
 
     /*
      * STOP! READ THIS FIRST! No, really,
@@ -201,7 +202,7 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId
      */
 
     private static class CurrentHolder {
-        private static final TransportVersion CURRENT = findCurrent(V_8_500_064);
+        private static final TransportVersion CURRENT = findCurrent(V_8_500_065);
 
         // finds the pluggable current version, or uses the given fallback
         private static TransportVersion findCurrent(TransportVersion fallback) {

server/src/main/java/org/elasticsearch/rest/ApiNotAvailableException.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.rest;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.rest.RestStatus.GONE;
+
+/**
+ * Thrown when an API is not available in the current environment.
+ */
+public class ApiNotAvailableException extends ElasticsearchException {
+
+    public ApiNotAvailableException(String msg, Object... args) {
+        super(msg, args);
+    }
+
+    public ApiNotAvailableException(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public RestStatus status() {
+        return GONE;
+    }
+}

server/src/main/java/org/elasticsearch/rest/RestController.java

@@ -57,7 +57,6 @@
 import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
 import static org.elasticsearch.rest.RestStatus.METHOD_NOT_ALLOWED;
 import static org.elasticsearch.rest.RestStatus.NOT_ACCEPTABLE;
-import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestController implements HttpServerTransport.Dispatcher {
@@ -664,17 +663,8 @@ public static void handleBadRequest(String uri, RestRequest.Method method, RestC
 
     public static void handleServerlessRequestToProtectedResource(String uri, RestRequest.Method method, RestChannel channel)
         throws IOException {
-        try (XContentBuilder builder = channel.newErrorBuilder()) {
-            builder.startObject();
-            {
-                builder.field(
-                    "error",
-                    "uri [" + uri + "] with method [" + method + "] exists but is not available when running in " + "serverless mode"
-                );
-            }
-            builder.endObject();
-            channel.sendResponse(new RestResponse(NOT_FOUND, builder));
-        }
+        String msg = "uri [" + uri + "] with method [" + method + "] exists but is not available when running in serverless mode";
+        channel.sendResponse(new RestResponse(channel, new ApiNotAvailableException(msg)));
     }
 
     /**

server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java

@@ -67,6 +67,7 @@
 import org.elasticsearch.ingest.IngestProcessorException;
 import org.elasticsearch.repositories.RepositoryConflictException;
 import org.elasticsearch.repositories.RepositoryException;
+import org.elasticsearch.rest.ApiNotAvailableException;
 import org.elasticsearch.rest.RestResponseTests;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException;
@@ -830,6 +831,7 @@ public void testIds() {
         ids.put(168, DocumentParsingException.class);
         ids.put(169, HttpHeadersValidationException.class);
         ids.put(170, ElasticsearchRoleRestrictionException.class);
+        ids.put(171, ApiNotAvailableException.class);
 
         Map<Class<? extends ElasticsearchException>, Integer> reverse = new HashMap<>();
         for (Map.Entry<Integer, Class<? extends ElasticsearchException>> entry : ids.entrySet()) {

server/src/test/java/org/elasticsearch/rest/RestControllerTests.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.http.HttpHeadersValidationException;
@@ -939,8 +940,17 @@ public void testApiProtectionWithServerlessEnabledAsEndUser() {
         });
         final Consumer<List<String>> checkProtected = paths -> paths.forEach(path -> {
             RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath(path).build();
-            AssertingChannel channel = new AssertingChannel(request, false, RestStatus.NOT_FOUND);
+            AssertingChannel channel = new AssertingChannel(request, true, RestStatus.GONE);
             restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+
+            RestResponse restResponse = channel.getRestResponse();
+            Map<String, Object> map = XContentHelper.convertToMap(restResponse.content(), false, XContentType.JSON).v2();
+            assertEquals(410, map.get("status"));
+            @SuppressWarnings("unchecked")
+            Map<String, Object> error = (Map<String, Object>) map.get("error");
+            assertEquals("api_not_available_exception", error.get("type"));
+            assertTrue(error.get("reason").toString().contains("not available when running in serverless mode"));
+
         });
 
         List<String> accessiblePaths = List.of("/public", "/internal");