Update REST API specs to v7.4.0 (#4117)

* Transform REST API spec to existing format

This commit changes the in memory structure of the ApiEndpoint in 7.4.0 to the old structure expected by the ApiGenerator.

* Update specs to v7.4.0

This commit updates the REST API specs to those in the v7.4.0 branch, and re-runs the ApiGenerator. New 7.4.0 APIs are excluded from code generation for now.

* Patch 7.4.0 breaking changes

- delete_by_query
- msearch_template
- search_template

(cherry picked from commit 6da223b)

src/CodeGeneration/ApiGenerator/Configuration/CodeConfiguration.cs

@@ -26,6 +26,7 @@ public static class CodeConfiguration
 			"data_frame.put_data_frame_transform.json",
 			"data_frame.start_data_frame_transform.json",
 			"data_frame.stop_data_frame_transform.json",
+			"data_frame.update_data_frame_transform.json",
 
 			"ml.evaluate_data_frame.json",
 			"ml.delete_data_frame_analytics.json",
@@ -41,7 +42,15 @@ public static class CodeConfiguration
 			// these APIs are new and need to be mapped
 			"ml.set_upgrade_mode.json",
 			"ml.find_file_structure.json",
-			"monitoring.bulk.json"
+			"monitoring.bulk.json",
+			"snapshot.cleanup_repository.json",
+			"ml.estimate_memory_usage.json",
+			"indices.clone.json",
+
+			"slm.delete_lifecycle.json",
+			"slm.execute_lifecycle.json",
+			"slm.get_lifecycle.json",
+			"slm.put_lifecycle.json",
 		};
 
 

src/CodeGeneration/ApiGenerator/Domain/Specification/ApiEndpoint.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using ApiGenerator.Configuration.Overrides;
@@ -7,6 +8,7 @@
 using ApiGenerator.Domain.Code.LowLevel;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
+using Newtonsoft.Json.Linq;
 
 namespace ApiGenerator.Domain.Specification
 {
@@ -32,7 +34,7 @@ public class ApiEndpoint
 		public Stability Stability { get; set; }
 
 		[JsonProperty("documentation")]
-		public string OfficialDocumentationLink { get; set; }
+		public Documentation OfficialDocumentationLink { get; set; }
 
 		public UrlInformation Url { get; set; }
 
@@ -48,13 +50,13 @@ public class ApiEndpoint
 			CsharpNames = CsharpNames,
 			UrlParts = Url.Parts,
 			PartialParameters = Body == null ? Enumerable.Empty<QueryParameters>().ToList() : Url.Params.Values.Where(p=>p.RenderPartial && !p.Skip).ToList(),
-			OfficialDocumentationLink = OfficialDocumentationLink
+			OfficialDocumentationLink = OfficialDocumentationLink.Url
 		};
 
 		public RequestPartialImplementation RequestPartialImplementation => new RequestPartialImplementation
 		{
 			CsharpNames = CsharpNames,
-			OfficialDocumentationLink = OfficialDocumentationLink,
+			OfficialDocumentationLink = OfficialDocumentationLink.Url,
 			Stability = Stability,
 			Paths = Url.Paths,
 			Parts = Url.Parts,
@@ -67,7 +69,7 @@ public class ApiEndpoint
 		public DescriptorPartialImplementation DescriptorPartialImplementation => new DescriptorPartialImplementation
 		{
 			CsharpNames = CsharpNames,
-			OfficialDocumentationLink = OfficialDocumentationLink,
+			OfficialDocumentationLink = OfficialDocumentationLink.Url,
 			Constructors = Constructor.DescriptorConstructors(CsharpNames, Url).ToList(),
 			Paths = Url.Paths,
 			Parts = Url.Parts,
@@ -78,7 +80,7 @@ public class ApiEndpoint
 		public RequestParameterImplementation RequestParameterImplementation => new RequestParameterImplementation
 		{
 			CsharpNames = CsharpNames,
-			OfficialDocumentationLink = OfficialDocumentationLink,
+			OfficialDocumentationLink = OfficialDocumentationLink.Url,
 			Params = Url.Params.Values.Where(p=>!p.Skip).ToList(),
 			HttpMethod = PreferredHttpMethod
 		};
@@ -101,16 +103,16 @@ public string PreferredHttpMethod
 			CsharpNames = CsharpNames,
 			Fluent = new FluentMethod(CsharpNames, Url.Parts,
 				selectorIsOptional: Body == null || !Body.Required || HttpMethods.Contains("GET"),
-				link: OfficialDocumentationLink,
+				link: OfficialDocumentationLink.Url,
 				summary: HighLevelMethodXmlDocDescription
 			),
 			FluentBound = !CsharpNames.DescriptorBindsOverMultipleDocuments ? null : new BoundFluentMethod(CsharpNames, Url.Parts,
 				selectorIsOptional: Body == null || !Body.Required || HttpMethods.Contains("GET"),
-				link: OfficialDocumentationLink,
+				link: OfficialDocumentationLink.Url,
 				summary: HighLevelMethodXmlDocDescription
 			),
 			Initializer = new InitializerMethod(CsharpNames,
-				link: OfficialDocumentationLink,
+				link: OfficialDocumentationLink.Url,
 				summary: HighLevelMethodXmlDocDescription
 			)
 		};
@@ -145,7 +147,7 @@ public IReadOnlyCollection<LowLevelClientMethod> LowLevelClientMethods
 						CsharpNames = CsharpNames,
 						PerPathMethodName = methodName,
 						HttpMethod = httpMethod,
-						OfficialDocumentationLink = OfficialDocumentationLink,
+						OfficialDocumentationLink = OfficialDocumentationLink.Url,
 						Stability = Stability,
 						DeprecatedPath = path.Deprecation,
 						Path = path.Path,

src/CodeGeneration/ApiGenerator/Domain/Specification/Documentation.cs

@@ -0,0 +1,54 @@
+using System;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace ApiGenerator.Domain.Specification
+{
+	[JsonConverter(typeof(DocumentationConverter))]
+	public class Documentation
+	{
+		public string Description { get; set; }
+		public string Url { get; set; }
+	}
+
+	public class DocumentationConverter : JsonConverter
+	{
+		public override bool CanWrite { get; } = false;
+
+		public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) => throw new NotSupportedException();
+
+		public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+		{
+
+			var documentation = new Documentation();
+
+			if (reader.TokenType == JsonToken.String)
+			{
+				documentation.Url = (string)reader.Value;
+				return documentation;
+			}
+
+			while (reader.Read())
+			{
+				if (reader.TokenType == JsonToken.EndObject)
+					break;
+
+				var prop = (string)reader.Value;
+				switch (prop)
+				{
+					case "url":
+						documentation.Url = reader.ReadAsString();
+						break;
+					case "description":
+						documentation.Description = reader.ReadAsString();
+						break;
+					default:
+						throw new Exception($"Property '{prop}' unexpected in documentation object");
+				}
+			}
+			return documentation;
+		}
+
+		public override bool CanConvert(Type objectType) => true;
+	}
+}

src/CodeGeneration/ApiGenerator/Generator/ApiEndpointFactory.cs

@@ -7,35 +7,37 @@
 using ApiGenerator.Domain;
 using ApiGenerator.Domain.Code;
 using ApiGenerator.Domain.Specification;
+using Microsoft.Extensions.DependencyModel;
 using Newtonsoft.Json.Linq;
 
-namespace ApiGenerator.Generator 
+namespace ApiGenerator.Generator
 {
 	public static class ApiEndpointFactory
 	{
 		public static ApiEndpoint FromFile(string jsonFile)
 		{
 			var officialJsonSpec = JObject.Parse(File.ReadAllText(jsonFile));
+			TransformNewSpecStructureToOld(officialJsonSpec);
 			PatchOfficialSpec(officialJsonSpec, jsonFile);
 			var (name, endpoint) = officialJsonSpec.ToObject<Dictionary<string, ApiEndpoint>>().First();
-			
+
 			endpoint.FileName = Path.GetFileName(jsonFile);
 			endpoint.Name = name;
 			var tokens = name.Split(".");
-			
+
 			endpoint.MethodName = tokens.Last();
 			if (tokens.Length > 1)
 				endpoint.Namespace = tokens[0];
 			//todo side effect
 			endpoint.CsharpNames = new CsharpNames(name, endpoint.MethodName, endpoint.Namespace);
-			
+
 			LoadOverridesOnEndpoint(endpoint);
 			PatchRequestParameters(endpoint);
 
 			EnforceRequiredOnParts(jsonFile, endpoint.Url);
 			return endpoint;
 		}
-		
+
 		/// <summary>
 		/// This makes sure required is configured correctly by inspecting the paths.
 		/// Will emit a warning if the spec file got this wrong
@@ -70,7 +72,7 @@ private static void PatchRequestParameters(ApiEndpoint endpoint)
 			var newParams = ApiQueryParametersPatcher.Patch(endpoint.Name, endpoint.Url.Params, endpoint.Overrides);
 			endpoint.Url.Params = newParams;
 		}
-		
+
 		/// <summary>
 		/// Finds a patch file in patches and union merges this with the official spec.
 		/// This allows us to check in tweaks should breaking changes occur in the spec before we catch them
@@ -102,5 +104,76 @@ void ReplaceOptions(string path)
 			ReplaceOptions("*.url.parts.metric.options");
 			ReplaceOptions("*.url.parts.index_metric.options");
 		}
+
+		/// <summary>
+		/// Changes the structure of new REST API spec in 7.4.0 to one that matches prior spec structure.
+		/// </summary>
+		private static void TransformNewSpecStructureToOld(JObject original)
+		{
+			var name = (JProperty)original.First;
+			var spec = (JObject)name.Value;
+
+			// old spec structure, nothing to change
+			if (spec.ContainsKey("methods"))
+				return;
+
+			var methods = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
+			JObject parts = null;
+			var paths = new List<string>();
+			var deprecatedPaths = new List<JObject>();
+
+			foreach (var path in spec["url"]["paths"].Cast<JObject>())
+			{
+				if (path.ContainsKey("deprecated"))
+				{
+					var deprecated = new JObject
+					{
+						["version"] = path["deprecated"]["version"].Value<string>(),
+						["path"] = path["path"].Value<string>(),
+						["description"] = path["deprecated"]["description"].Value<string>()
+					};
+
+					deprecatedPaths.Add(deprecated);
+				}
+				else
+					paths.Add(path["path"].Value<string>());
+
+				if (path.ContainsKey("parts"))
+				{
+					if (parts == null)
+						parts = path["parts"].Value<JObject>();
+					else
+						parts.Merge(path["parts"].Value<JObject>(), new JsonMergeSettings
+						{
+							MergeArrayHandling = MergeArrayHandling.Union
+						});
+				}
+
+				foreach (var method in path["methods"].Cast<JValue>())
+					methods.Add(method.Value<string>());
+			}
+
+
+
+			var newUrl = new JObject
+			{
+				["paths"] = new JArray(paths.ToArray()),
+			};
+
+			if (spec.ContainsKey("params"))
+			{
+				newUrl["params"] = spec["params"];
+				spec.Remove("params");
+			}
+
+			if (parts != null)
+				newUrl["parts"] = parts;
+
+			if (deprecatedPaths.Any())
+				newUrl["deprecated_paths"] = new JArray(deprecatedPaths.ToArray());
+
+			spec["url"] = newUrl;
+			spec["methods"] = new JArray(methods.ToArray());
+		}
 	}
 }