fix: Provide paging through discovery groups to discover 1000+ resources

changelog/content/experimental/unreleased.md

@@ -6,9 +6,11 @@ version:
 
 #### Scraper
 
-- {{% tag fixed %}} Fix "Try it out" capability for end-users in Swagger UI when using reverse proxies ([#1898](https://github.com/tomkerkhove/promitor/pull/1898))
+- {{% tag fixed %}} Discovered resources are no longer capped at 1000 resources ([#1828](https://github.com/tomkerkhove/promitor/pull/1828))
 - {{% tag changed %}} Reduce amount of API calls to scrape metrics for Azure services ([#1914](https://github.com/tomkerkhove/promitor/issues/1914))
+- {{% tag fixed %}} Fix "Try it out" capability for end-users in Swagger UI when using reverse proxies ([#1898](https://github.com/tomkerkhove/promitor/pull/1898))
 
 #### Resource Discovery
 
+- {{% tag fixed %}} Discovered resources are no longer capped at 1000 resources ([#1828](https://github.com/tomkerkhove/promitor/pull/1828))
 - {{% tag fixed %}} Fix "Try it out" capability for end-users in Swagger UI when using reverse proxies ([#1898](https://github.com/tomkerkhove/promitor/pull/1898))

src/Promitor.Agents.Core/Extensions/IServiceCollectionExtensions.cs

@@ -1,9 +1,12 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text.Json.Serialization;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.ApiExplorer;
+using Microsoft.AspNetCore.Mvc.Controllers;
 using Microsoft.AspNetCore.Mvc.Formatters;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.OpenApi.Models;
@@ -100,11 +103,45 @@ public static IServiceCollection UseOpenApiSpecifications(this IServiceCollectio
                 {
                     swaggerGenerationOptions.IncludeXmlComments(xmlDocumentationPath);
                 }
+
+                swaggerGenerationOptions.TagActionsBy(CustomizeActionTagging);
             });
 
             return services;
         }
 
+        private static IList<string> CustomizeActionTagging(ApiDescription apiDescription)
+        {
+            if (apiDescription.ActionDescriptor is ControllerActionDescriptor == false)
+            {
+                return new List<string>();
+            }
+
+            // Determine name of the controller
+            var controllerActionDescriptor = apiDescription.ActionDescriptor as ControllerActionDescriptor;
+            var controllerName = controllerActionDescriptor?.ControllerName;
+            if (string.IsNullOrWhiteSpace(controllerName))
+            {
+                throw new Exception("No controller name was found to tag actions with");
+            }
+
+            var tagName = controllerName;
+
+            // If the controller contains a version name, remove the suffix
+            if (controllerName.Contains("V", StringComparison.InvariantCulture))
+            {
+                var lastIndex = controllerName.LastIndexOf("V", StringComparison.InvariantCulture);
+                var suffix = controllerName.Substring(lastIndex + 1);
+
+                if (int.TryParse(suffix, out var _))
+                {
+                    tagName = controllerName.Substring(0, lastIndex);
+                }
+            }
+
+            return new List<string> { tagName };
+        }
+
         private static void RestrictToJsonContentType(MvcOptions options)
         {
             var allButJsonInputFormatters = options.InputFormatters.Where(formatter => !(formatter is SystemTextJsonInputFormatter));

src/Promitor.Agents.ResourceDiscovery/Controllers/v1/DiscoveryV1Controller.cs

@@ -0,0 +1,90 @@
+using System.Collections.Generic;
+using System.Net;
+using System.Threading.Tasks;
+using GuardNet;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+using Promitor.Agents.ResourceDiscovery.Graph.Exceptions;
+using Promitor.Agents.ResourceDiscovery.Graph.Repositories.Interfaces;
+using Promitor.Core.Contracts;
+
+namespace Promitor.Agents.ResourceDiscovery.Controllers.v1
+{
+    /// <summary>
+    /// API endpoint to discover Azure resources
+    /// </summary>
+    [ApiController]
+    [Route("api/v1/resources/groups")]
+    public class DiscoveryV1Controller : ControllerBase
+    {
+        private readonly JsonSerializerSettings _serializerSettings;
+        private readonly IAzureResourceRepository _azureResourceRepository;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="DiscoveryV1Controller"/> class.
+        /// </summary>
+        public DiscoveryV1Controller(IAzureResourceRepository azureResourceRepository)
+        {
+            Guard.NotNull(azureResourceRepository, nameof(azureResourceRepository));
+
+            _azureResourceRepository = azureResourceRepository;
+            _serializerSettings = new JsonSerializerSettings
+            {
+                NullValueHandling = NullValueHandling.Ignore,
+                TypeNameHandling = TypeNameHandling.Objects
+            };
+            _serializerSettings.Converters.Add(new StringEnumConverter());
+        }
+
+        /// <summary>
+        ///     Discover Resources
+        /// </summary>
+        /// <remarks>Discovers Azure resources matching the criteria.</remarks>
+        [HttpGet("{resourceDiscoveryGroup}/discover", Name = "Discovery_Get")]
+        [ProducesResponseType(typeof(List<AzureResourceDefinition>), StatusCodes.Status200OK)]
+        [ProducesResponseType(typeof(ValidationProblemDetails), StatusCodes.Status400BadRequest)]
+        public async Task<IActionResult> Get(string resourceDiscoveryGroup, int pageSize = 1000, int currentPage = 1)
+        {
+            if (currentPage < 1) 
+            {
+                return ValidationProblem(detail: "Current page has to be 1 or more");
+            }
+
+            if (pageSize < 1)
+            {
+                return ValidationProblem(detail: "Page size cannot be less than 1");
+            }
+
+            if (pageSize > 1000)
+            {
+                return ValidationProblem(detail: "Page size cannot be higher than 1000");
+            }
+
+            try
+            {
+                var pagedDiscoveredResources = await _azureResourceRepository.GetResourcesAsync(resourceDiscoveryGroup, pageSize, currentPage);
+                if (pagedDiscoveredResources == null)
+                {
+                    return NotFound(new {Information = "No resource discovery group was found with specified name"});
+                }
+
+                Response.Headers.Add("X-Paging-Page-Size", pagedDiscoveredResources.PageSize.ToString());
+                Response.Headers.Add("X-Paging-Current-Page", pagedDiscoveredResources.CurrentPage.ToString());
+                Response.Headers.Add("X-Paging-Total", pagedDiscoveredResources.TotalRecords.ToString());
+
+                var serializedResources = JsonConvert.SerializeObject(pagedDiscoveredResources.Result, _serializerSettings);
+
+                var response = Content(serializedResources, "application/json");
+                response.StatusCode = (int) HttpStatusCode.OK;
+
+                return response;
+            }
+            catch (ResourceTypeNotSupportedException resourceTypeNotSupportedException)
+            {
+                return StatusCode((int) HttpStatusCode.NotImplemented, new ResourceDiscoveryFailedDetails {Details = $"Resource type '{resourceTypeNotSupportedException.ResourceType}' for discovery group '{resourceDiscoveryGroup}' is not supported"});
+            }
+        }
+    }
+}

src/Promitor.Agents.ResourceDiscovery/Controllers/HealthController.cs → src/Promitor.Agents.ResourceDiscovery/Controllers/v1/HealthV1Controller.cs

@@ -5,20 +5,20 @@
 using Promitor.Agents.Core.Controllers;
 using Swashbuckle.AspNetCore.Filters;
 
-namespace Promitor.Agents.ResourceDiscovery.Controllers
+namespace Promitor.Agents.ResourceDiscovery.Controllers.v1
 {
     /// <summary>
     /// API endpoint to check the health of the application.
     /// </summary>
     [ApiController]
     [Route("api/v1/health")]
-    public class HealthController : OperationsController
+    public class HealthV1Controller : OperationsController
     {
         /// <summary>
-        /// Initializes a new instance of the <see cref="HealthController"/> class.
+        /// Initializes a new instance of the <see cref="HealthV1Controller"/> class.
         /// </summary>
         /// <param name="healthCheckService">The service to provide the health of the API application.</param>
-        public HealthController(HealthCheckService healthCheckService)
+        public HealthV1Controller(HealthCheckService healthCheckService)
             : base(healthCheckService)
         {
         }

src/Promitor.Agents.ResourceDiscovery/Controllers/ResourceDiscoveryGroupsController.cs → src/Promitor.Agents.ResourceDiscovery/Controllers/v1/ResourceDiscoveryGroupsV1Controller.cs

@@ -1,23 +1,23 @@
-using Microsoft.AspNetCore.Mvc;
-using GuardNet;
+using GuardNet;
+using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
 using Promitor.Agents.ResourceDiscovery.Configuration;
 
-namespace Promitor.Agents.ResourceDiscovery.Controllers
+namespace Promitor.Agents.ResourceDiscovery.Controllers.v1
 {
     /// <summary>
     /// API endpoint to interact with resource discovery groups
     /// </summary>
     [ApiController]
     [Route("api/v1/resources/groups")]
-    public class ResourceDiscoveryGroupsController : ControllerBase
+    public class ResourceDiscoveryGroupsV1Controller : ControllerBase
     {
         private readonly IOptionsMonitor<ResourceDeclaration> _resourceDeclarationMonitor;
 
         /// <summary>
-        ///     Initializes a new instance of the <see cref="ResourceDiscoveryGroupsController" /> class.
+        ///     Initializes a new instance of the <see cref="ResourceDiscoveryGroupsV1Controller" /> class.
         /// </summary>
-        public ResourceDiscoveryGroupsController(IOptionsMonitor<ResourceDeclaration> resourceDeclarationMonitor)
+        public ResourceDiscoveryGroupsV1Controller(IOptionsMonitor<ResourceDeclaration> resourceDeclarationMonitor)
         {
             Guard.NotNull(resourceDeclarationMonitor, nameof(resourceDeclarationMonitor));
 

src/Promitor.Agents.ResourceDiscovery/Controllers/SystemController.cs → src/Promitor.Agents.ResourceDiscovery/Controllers/v1/SystemV1Controller.cs

@@ -4,10 +4,10 @@
 using Promitor.Core;
 using Swashbuckle.AspNetCore.Annotations;
 
-namespace Promitor.Agents.ResourceDiscovery.Controllers
+namespace Promitor.Agents.ResourceDiscovery.Controllers.v1
 {
     [Route("api/v1/system")]
-    public class SystemController : Controller
+    public class SystemV1Controller : Controller
     {
         /// <summary>
         ///     Get System Info

src/Promitor.Agents.ResourceDiscovery/Controllers/DiscoveryController.cs → src/Promitor.Agents.ResourceDiscovery/Controllers/v2/DiscoveryV2Controller.cs

@@ -1,29 +1,31 @@
-using System.Net;
+using System.Collections.Generic;
+using System.Net;
 using System.Threading.Tasks;
-using Microsoft.AspNetCore.Mvc;
 using GuardNet;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
 using Newtonsoft.Json;
-using Promitor.Agents.ResourceDiscovery.Graph.Exceptions;
-using Promitor.Core.Contracts;
 using Newtonsoft.Json.Converters;
+using Promitor.Agents.ResourceDiscovery.Graph.Exceptions;
 using Promitor.Agents.ResourceDiscovery.Graph.Repositories.Interfaces;
+using Promitor.Core.Contracts;
 
-namespace Promitor.Agents.ResourceDiscovery.Controllers
+namespace Promitor.Agents.ResourceDiscovery.Controllers.v2
 {
     /// <summary>
     /// API endpoint to discover Azure resources
     /// </summary>
     [ApiController]
-    [Route("api/v1/resources/groups")]
-    public class DiscoveryController : ControllerBase
+    [Route("api/v2/resources/groups")]
+    public class DiscoveryV2Controller : ControllerBase
     {
         private readonly JsonSerializerSettings _serializerSettings;
         private readonly IAzureResourceRepository _azureResourceRepository;
 
         /// <summary>
-        /// Initializes a new instance of the <see cref="DiscoveryController"/> class.
+        /// Initializes a new instance of the <see cref="DiscoveryV2Controller"/> class.
         /// </summary>
-        public DiscoveryController(IAzureResourceRepository azureResourceRepository)
+        public DiscoveryV2Controller(IAzureResourceRepository azureResourceRepository)
         {
             Guard.NotNull(azureResourceRepository, nameof(azureResourceRepository));
 
@@ -40,26 +42,44 @@ public DiscoveryController(IAzureResourceRepository azureResourceRepository)
         ///     Discover Resources
         /// </summary>
         /// <remarks>Discovers Azure resources matching the criteria.</remarks>
-        [HttpGet("{resourceDiscoveryGroup}/discover", Name = "Discovery_Get")]
-        public async Task<IActionResult> Get(string resourceDiscoveryGroup)
+        [HttpGet("{resourceDiscoveryGroup}/discover", Name = "DiscoveryV2_Get")]
+        [ProducesResponseType(typeof(PagedResult<List<AzureResourceDefinition>>), StatusCodes.Status200OK)]
+        [ProducesResponseType(typeof(ValidationProblemDetails), StatusCodes.Status400BadRequest)]
+        public async Task<IActionResult> Get(string resourceDiscoveryGroup, int pageSize = 1000, int currentPage = 1)
         {
+            if (currentPage < 1)
+            {
+                return ValidationProblem(detail: "Current page has to be 1 or more");
+            }
+
+            if (pageSize < 1)
+            {
+                return ValidationProblem(detail: "Page size cannot be less than 1");
+            }
+
+            if (pageSize > 1000)
+            {
+                return ValidationProblem(detail: "Page size cannot be higher than 1000");
+            }
+
             try
             {
-                var foundResources = await _azureResourceRepository.GetResourcesAsync(resourceDiscoveryGroup);
-                if (foundResources == null)
+                var pagedDiscoveredResources = await _azureResourceRepository.GetResourcesAsync(resourceDiscoveryGroup, pageSize, currentPage);
+                if (pagedDiscoveredResources == null)
                 {
                     return NotFound(new {Information = "No resource discovery group was found with specified name"});
                 }
 
-                var serializedResources = JsonConvert.SerializeObject(foundResources, _serializerSettings);
-                
-                var response= Content(serializedResources, "application/json");
+                var serializedResources = JsonConvert.SerializeObject(pagedDiscoveredResources, _serializerSettings);
+
+                var response = Content(serializedResources, "application/json");
                 response.StatusCode = (int) HttpStatusCode.OK;
+
                 return response;
             }
             catch (ResourceTypeNotSupportedException resourceTypeNotSupportedException)
             {
-                return StatusCode((int)HttpStatusCode.NotImplemented, new ResourceDiscoveryFailedDetails { Details=$"Resource type '{resourceTypeNotSupportedException.ResourceType}' for discovery group '{resourceDiscoveryGroup}' is not supported"});
+                return StatusCode((int) HttpStatusCode.NotImplemented, new ResourceDiscoveryFailedDetails {Details = $"Resource type '{resourceTypeNotSupportedException.ResourceType}' for discovery group '{resourceDiscoveryGroup}' is not supported"});
             }
         }
     }