Commands Info/Docs Update (#913)

* Some small changes to CommandInfoUpdater tool + updating CommandsInfo/Docs JSON files

* format

* test fix

* fix

* Added command metadata coverage tests

* added comment

playground/CommandInfoUpdater/CommandDocsUpdater.cs

@@ -11,6 +11,7 @@ namespace CommandInfoUpdater
 {
     public class CommandDocsUpdater
     {
+        const int QUERY_CMD_BATCH_SIZE = 25;
         private static readonly string CommandDocsFileName = "RespCommandsDocs.json";
         private static readonly string GarnetCommandDocsJsonPath = "GarnetCommandsDocs.json";
 
@@ -59,36 +60,43 @@ public static bool TryUpdateCommandDocs(string outputDir, int respServerPort, IP
             IDictionary<string, RespCommandDocs> queriedCommandsDocs = new Dictionary<string, RespCommandDocs>();
             var commandsToQuery = commandsToAdd.Keys.Select(k => k.Command)
                 .Where(c => updatedCommandsInfo.ContainsKey(c) && !updatedCommandsInfo[c].IsInternal).ToArray();
-            if (commandsToQuery.Length > 0 && !TryGetCommandsDocs(commandsToQuery, respServerPort, respServerHost,
-                    logger, out queriedCommandsDocs))
+            if (commandsToQuery.Length > 0)
             {
-                logger.LogError("Unable to get RESP command docs from local RESP server.");
-                return false;
+                for (var i = 0; i < commandsToQuery.Length; i += QUERY_CMD_BATCH_SIZE)
+                {
+                    var batchToQuery = commandsToQuery.Skip(i).Take(QUERY_CMD_BATCH_SIZE).ToArray();
+                    if (!TryGetCommandsDocs(batchToQuery, respServerPort, respServerHost,
+                            logger, ref queriedCommandsDocs))
+                    {
+                        logger.LogError("Unable to get RESP command info from local RESP server.");
+                        return false;
+                    }
+                }
             }
 
             var additionalCommandsDocs = new Dictionary<string, RespCommandDocs>();
             foreach (var cmd in garnetCommandsDocs.Keys.Union(queriedCommandsDocs.Keys))
             {
                 if (!additionalCommandsDocs.ContainsKey(cmd))
                 {
-                    var baseCommandDocs = queriedCommandsDocs.TryGetValue(cmd, out var doc)
-                        ? doc
-                        : garnetCommandsDocs[cmd];
+                    var inQueried = queriedCommandsDocs.TryGetValue(cmd, out var queriedCommandDocs);
+                    var inGarnet = garnetCommandsDocs.TryGetValue(cmd, out var garnetCommandDocs);
+                    var baseCommandDocs = inGarnet ? garnetCommandDocs : queriedCommandDocs;
 
                     RespCommandDocs[] subCommandsDocs;
-                    if (garnetCommandsDocs.ContainsKey(cmd) && queriedCommandsDocs.ContainsKey(cmd))
+                    if (inQueried && inGarnet)
                     {
                         var subCommandsInfoMap = new Dictionary<string, RespCommandDocs>();
 
-                        if (garnetCommandsDocs.TryGetValue(cmd, out var garnetCmdDocs) && garnetCmdDocs.SubCommands != null)
+                        if (garnetCommandDocs.SubCommands != null)
                         {
-                            foreach (var sc in garnetCmdDocs.SubCommands)
+                            foreach (var sc in garnetCommandDocs.SubCommands)
                                 subCommandsInfoMap.Add(sc.Name, sc);
                         }
 
-                        if (queriedCommandsDocs.TryGetValue(cmd, out var queriedCmdDocs) && queriedCmdDocs.SubCommands != null)
+                        if (queriedCommandDocs.SubCommands != null)
                         {
-                            foreach (var sc in queriedCmdDocs.SubCommands)
+                            foreach (var sc in queriedCommandDocs.SubCommands)
                             {
                                 subCommandsInfoMap.TryAdd(sc.Name, sc);
                             }
@@ -134,15 +142,11 @@ public static bool TryUpdateCommandDocs(string outputDir, int respServerPort, IP
         /// <param name="commandDocs">Queried commands docs</param>
         /// <returns>True if succeeded</returns>
         private static unsafe bool TryGetCommandsDocs(string[] commandsToQuery, int respServerPort,
-            IPAddress respServerHost, ILogger logger, out IDictionary<string, RespCommandDocs> commandDocs)
+            IPAddress respServerHost, ILogger logger, ref IDictionary<string, RespCommandDocs> commandDocs)
         {
-            commandDocs = default;
-
             // If there are no commands to query, return
             if (commandsToQuery.Length == 0) return true;
 
-            var tmpCommandsDocs = new Dictionary<string, RespCommandDocs>();
-
             // Get a map of supported commands to Garnet's RespCommand & ArrayCommand for the parser
             var supportedCommands = new ReadOnlyDictionary<string, RespCommand>(
                 SupportedCommand.SupportedCommandsFlattenedMap.ToDictionary(kvp => kvp.Key,
@@ -171,10 +175,9 @@ private static unsafe bool TryGetCommandsDocs(string[] commandsToQuery, int resp
                     return false;
                 }
 
-                tmpCommandsDocs.Add(cmdName, cmdDocs);
+                commandDocs.Add(cmdName, cmdDocs);
             }
 
-            commandDocs = tmpCommandsDocs;
             return true;
         }
 
@@ -212,7 +215,7 @@ private static IReadOnlyDictionary<string, RespCommandDocs> GetUpdatedCommandsDo
                     : existingCommandsDocs[command.Command].SubCommands.Select(sc => sc.Name).ToArray();
                 var remainingSubCommands = existingSubCommands == null ? null :
                     command.SubCommands == null ? existingSubCommands :
-                    existingSubCommands.Except(command.SubCommands.Keys).ToArray();
+                    [.. existingSubCommands.Except(command.SubCommands.Keys)];
 
                 // Create updated command docs based on existing command
                 var existingCommandDoc = existingCommandsDocs[command.Command];
@@ -227,7 +230,7 @@ private static IReadOnlyDictionary<string, RespCommandDocs> GetUpdatedCommandsDo
                     existingCommandDoc.Arguments,
                     remainingSubCommands == null || remainingSubCommands.Length == 0
                         ? null
-                        : existingCommandDoc.SubCommands.Where(sc => remainingSubCommands.Contains(sc.Name)).ToArray());
+                        : [.. existingCommandDoc.SubCommands.Where(sc => remainingSubCommands.Contains(sc.Name))]);
 
                 updatedCommandsDocs.Add(updatedCommandDoc.Name, updatedCommandDoc);
             }
@@ -267,7 +270,7 @@ private static IReadOnlyDictionary<string, RespCommandDocs> GetUpdatedCommandsDo
                     // Update sub-commands to contain supported sub-commands only
                     updatedSubCommandsDocs = command.SubCommands == null
                         ? null
-                        : baseCommandDocs.SubCommands.Where(sc => command.SubCommands.Keys.Contains(sc.Name)).ToList();
+                        : [.. baseCommandDocs.SubCommands.Where(sc => command.SubCommands.Keys.Contains(sc.Name))];
                 }
 
                 // Create updated command docs based on base command & updated sub-commands
@@ -280,7 +283,7 @@ private static IReadOnlyDictionary<string, RespCommandDocs> GetUpdatedCommandsDo
                     baseCommandDocs.DocFlags,
                     baseCommandDocs.ReplacedBy,
                     baseCommandDocs.Arguments,
-                    updatedSubCommandsDocs?.ToArray());
+                    updatedSubCommandsDocs?.OrderBy(sc => sc.Name).ToArray());
 
                 updatedCommandsDocs.Add(updatedCommandDocs.Name, updatedCommandDocs);
             }

playground/CommandInfoUpdater/CommandInfoUpdater.cs

@@ -15,6 +15,7 @@ namespace CommandInfoUpdater
     /// </summary>
     public class CommandInfoUpdater
     {
+        const int QUERY_CMD_BATCH_SIZE = 25;
         private static readonly string CommandInfoFileName = "RespCommandsInfo.json";
         private static readonly string GarnetCommandInfoJsonPath = "GarnetCommandsInfo.json";
 
@@ -63,35 +64,43 @@ public static bool TryUpdateCommandInfo(string outputDir, int respServerPort, IP
             var commandsToQuery = commandsToAdd.Keys.Select(k => k.Command)
                 .Where(c => !garnetCommandsInfo.ContainsKey(c) || !garnetCommandsInfo[c].IsInternal).ToArray();
 
-            if (commandsToQuery.Length > 0 && !TryGetCommandsInfo(commandsToQuery, respServerPort, respServerHost,
-                    logger, out queriedCommandsInfo))
+            if (commandsToQuery.Length > 0)
             {
-                logger.LogError("Unable to get RESP command info from local RESP server.");
-                return false;
+                for (var i = 0; i < commandsToQuery.Length; i += QUERY_CMD_BATCH_SIZE)
+                {
+                    var batchToQuery = commandsToQuery.Skip(i).Take(QUERY_CMD_BATCH_SIZE).ToArray();
+                    if (!TryGetCommandsInfo(batchToQuery, respServerPort, respServerHost,
+                            logger, ref queriedCommandsInfo))
+                    {
+                        logger.LogError("Unable to get RESP command info from local RESP server.");
+                        return false;
+                    }
+                }
             }
 
             var additionalCommandsInfo = new Dictionary<string, RespCommandsInfo>();
             foreach (var cmd in garnetCommandsInfo.Keys.Union(queriedCommandsInfo.Keys))
             {
                 if (!additionalCommandsInfo.ContainsKey(cmd))
                 {
-                    var baseCommandInfo = queriedCommandsInfo.TryGetValue(cmd, out var cmdInfo)
-                        ? cmdInfo : garnetCommandsInfo[cmd];
+                    var inQueried = queriedCommandsInfo.TryGetValue(cmd, out var queriedCommandInfo);
+                    var inGarnet = garnetCommandsInfo.TryGetValue(cmd, out var garnetCommandInfo);
+                    var baseCommandInfo = inGarnet ? garnetCommandInfo : queriedCommandInfo;
 
                     RespCommandsInfo[] subCommandsInfo;
-                    if (garnetCommandsInfo.ContainsKey(cmd) && queriedCommandsInfo.ContainsKey(cmd))
+                    if (inQueried && inGarnet)
                     {
                         var subCommandsInfoMap = new Dictionary<string, RespCommandsInfo>();
 
-                        if (garnetCommandsInfo.TryGetValue(cmd, out var garnetCmdInfo) && garnetCmdInfo.SubCommands != null)
+                        if (garnetCommandInfo.SubCommands != null)
                         {
-                            foreach (var sc in garnetCmdInfo.SubCommands)
+                            foreach (var sc in garnetCommandInfo.SubCommands)
                                 subCommandsInfoMap.Add(sc.Name, sc);
                         }
 
-                        if (queriedCommandsInfo.TryGetValue(cmd, out var queriedCmdInfo) && queriedCmdInfo.SubCommands != null)
+                        if (queriedCommandInfo.SubCommands != null)
                         {
-                            foreach (var sc in queriedCmdInfo.SubCommands)
+                            foreach (var sc in queriedCommandInfo.SubCommands)
                             {
                                 subCommandsInfoMap.TryAdd(sc.Name, sc);
                             }
@@ -104,7 +113,7 @@ public static bool TryUpdateCommandInfo(string outputDir, int respServerPort, IP
                         subCommandsInfo = baseCommandInfo.SubCommands;
                     }
 
-                    additionalCommandsInfo.Add(cmd, new RespCommandsInfo()
+                    additionalCommandsInfo.Add(cmd, new RespCommandsInfo
                     {
                         Command = baseCommandInfo.Command,
                         Name = baseCommandInfo.Name,
@@ -117,7 +126,7 @@ public static bool TryUpdateCommandInfo(string outputDir, int respServerPort, IP
                         AclCategories = baseCommandInfo.AclCategories,
                         Tips = baseCommandInfo.Tips,
                         KeySpecifications = baseCommandInfo.KeySpecifications,
-                        SubCommands = subCommandsInfo
+                        SubCommands = subCommandsInfo?.OrderBy(sc => sc.Name).ToArray()
                     });
                 }
             }
@@ -147,10 +156,8 @@ public static bool TryUpdateCommandInfo(string outputDir, int respServerPort, IP
         /// <param name="commandsInfo">Queried commands info</param>
         /// <returns>True if succeeded</returns>
         private static unsafe bool TryGetCommandsInfo(string[] commandsToQuery, int respServerPort,
-            IPAddress respServerHost, ILogger logger, out IDictionary<string, RespCommandsInfo> commandsInfo)
+            IPAddress respServerHost, ILogger logger, ref IDictionary<string, RespCommandsInfo> commandsInfo)
         {
-            commandsInfo = default;
-
             // If there are no commands to query, return
             if (commandsToQuery.Length == 0) return true;
 
@@ -167,8 +174,6 @@ private static unsafe bool TryGetCommandsInfo(string[] commandsToQuery, int resp
                 return false;
             }
 
-            var tmpCommandsInfo = new Dictionary<string, RespCommandsInfo>();
-
             // Get a map of supported commands to Garnet's RespCommand & ArrayCommand for the parser
             var supportedCommands = new ReadOnlyDictionary<string, RespCommand>(
                 SupportedCommand.SupportedCommandsFlattenedMap.ToDictionary(kvp => kvp.Key,
@@ -197,11 +202,10 @@ private static unsafe bool TryGetCommandsInfo(string[] commandsToQuery, int resp
                     }
 
                     if (command != null)
-                        tmpCommandsInfo.Add(command.Name, command);
+                        commandsInfo.Add(command.Name, command);
                 }
             }
 
-            commandsInfo = tmpCommandsInfo;
             return true;
         }
 
@@ -238,7 +242,7 @@ private static IReadOnlyDictionary<string, RespCommandsInfo> GetUpdatedCommandsI
                     : existingCommandsInfo[command.Command].SubCommands.Select(sc => sc.Name).ToArray();
                 var remainingSubCommands = existingSubCommands == null ? null :
                     command.SubCommands == null ? existingSubCommands :
-                    existingSubCommands.Except(command.SubCommands.Keys).ToArray();
+                    [.. existingSubCommands.Except(command.SubCommands.Keys)];
 
                 // Create updated command info based on existing command
                 var existingCommand = existingCommandsInfo[command.Command];
@@ -257,7 +261,7 @@ private static IReadOnlyDictionary<string, RespCommandsInfo> GetUpdatedCommandsI
                     KeySpecifications = existingCommand.KeySpecifications,
                     SubCommands = remainingSubCommands == null || remainingSubCommands.Length == 0
                         ? null
-                        : existingCommand.SubCommands.Where(sc => remainingSubCommands.Contains(sc.Name)).ToArray()
+                        : [.. existingCommand.SubCommands.Where(sc => remainingSubCommands.Contains(sc.Name))]
                 };
 
                 updatedCommandsInfo.Add(updatedCommand.Name, updatedCommand);
@@ -294,7 +298,7 @@ private static IReadOnlyDictionary<string, RespCommandsInfo> GetUpdatedCommandsI
                     // Update sub-commands to contain supported sub-commands only
                     updatedSubCommands = command.SubCommands == null
                         ? null
-                        : baseCommand.SubCommands.Where(sc => command.SubCommands.ContainsKey(sc.Name)).ToList();
+                        : [.. baseCommand.SubCommands.Where(sc => command.SubCommands.ContainsKey(sc.Name))];
                 }
 
                 // Create updated command info based on base command & updated sub-commands

playground/CommandInfoUpdater/CommonUtils.cs

@@ -105,8 +105,8 @@ internal static (IDictionary<SupportedCommand, bool>, IDictionary<SupportedComma
                     var existingSubCommands = new HashSet<string>(existingCommandsInfo[supportedCommand.Command]
                         .SubCommands
                         .Select(sc => sc.Name));
-                    subCommandsToAdd = supportedCommand.SubCommands
-                        .Where(subCommand => !existingSubCommands.Contains(subCommand.Key)).Select(sc => sc.Value).ToArray();
+                    subCommandsToAdd = [.. supportedCommand.SubCommands
+                        .Where(subCommand => !existingSubCommands.Contains(subCommand.Key)).Select(sc => sc.Value)];
                 }
 
                 // If there are sub-commands to add, add a new supported command with the sub-commands to add