OFFI-109: Delete already existing elasticsearch indexes related to the tenant

Lombiq.Hosting.Tenants.Maintenance/Constants/FeatureNames.cs

@@ -9,4 +9,6 @@ public static class FeatureNames
     public const string AddSiteOwnerPermissionToRole = Maintenance + "." + nameof(AddSiteOwnerPermissionToRole);
     public const string RemoveUsers = Maintenance + "." + nameof(RemoveUsers);
     public const string ChangeUserSensitiveContent = Maintenance + "." + nameof(ChangeUserSensitiveContent);
+    public const string DeleteOrRebuildElasticsearchIndices = Maintenance + "." + nameof(DeleteOrRebuildElasticsearchIndices);
+    public const string DeleteElasticsearchIndicesBeforeSetup = Maintenance + "." + nameof(DeleteElasticsearchIndicesBeforeSetup);
 }

Lombiq.Hosting.Tenants.Maintenance/Lombiq.Hosting.Tenants.Maintenance.csproj

@@ -27,6 +27,7 @@
         <PackageReference Include="OrchardCore.ContentManagement" Version="1.8.0" />
         <PackageReference Include="OrchardCore.ContentTypes.Abstractions" Version="1.8.0" />
         <PackageReference Include="OrchardCore.DisplayManagement" Version="1.8.0" />
+        <PackageReference Include="OrchardCore.Search.Elasticsearch.Core" Version="1.8.0" />
         <PackageReference Include="OrchardCore.Settings" Version="1.8.0" />
         <PackageReference Include="OrchardCore.Users.Abstractions" Version="1.8.0" />
         <PackageReference Include="OrchardCore.Users.Core" Version="1.8.0" />

Lombiq.Hosting.Tenants.Maintenance/Maintenance/AddSiteOwnerPermissionToRole/Startup.cs

@@ -1,6 +1,6 @@
+using Lombiq.HelpfulLibraries.OrchardCore.Mvc;
 using Lombiq.Hosting.Tenants.Maintenance.Constants;
 using Lombiq.Hosting.Tenants.Maintenance.Services;
-using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using OrchardCore.Environment.Shell.Configuration;
 using OrchardCore.Modules;
@@ -17,10 +17,9 @@ public Startup(IShellConfiguration shellConfiguration) =>
 
     public override void ConfigureServices(IServiceCollection services)
     {
-        var options = new AddSiteOwnerPermissionToRoleMaintenanceOptions();
-        var configSection = _shellConfiguration.GetSection("Lombiq_Hosting_Tenants_Maintenance:AddSiteOwnerPermissionToRole");
-        configSection.Bind(options);
-        services.Configure<AddSiteOwnerPermissionToRoleMaintenanceOptions>(configSection);
+        services.BindAndConfigureSection<AddSiteOwnerPermissionToRoleMaintenanceOptions>(
+            _shellConfiguration,
+            "Lombiq_Hosting_Tenants_Maintenance:AddSiteOwnerPermissionToRole");
 
         services.AddScoped<IMaintenanceProvider, AddSiteOwnerPermissionToRoleMaintenanceProvider>();
     }

Lombiq.Hosting.Tenants.Maintenance/Maintenance/ChangeUserSensitiveContent/Startup.cs

@@ -1,6 +1,6 @@
+using Lombiq.HelpfulLibraries.OrchardCore.Mvc;
 using Lombiq.Hosting.Tenants.Maintenance.Constants;
 using Lombiq.Hosting.Tenants.Maintenance.Services;
-using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using OrchardCore.Environment.Shell.Configuration;
 using OrchardCore.Modules;
@@ -17,11 +17,9 @@ public Startup(IShellConfiguration shellConfiguration) =>
 
     public override void ConfigureServices(IServiceCollection services)
     {
-        var options = new ChangeUserSensitiveContentMaintenanceOptions();
-        var configSection = _shellConfiguration
-            .GetSection("Lombiq_Hosting_Tenants_Maintenance:ChangeUserSensitiveContent");
-        configSection.Bind(options);
-        services.Configure<ChangeUserSensitiveContentMaintenanceOptions>(configSection);
+        services.BindAndConfigureSection<ChangeUserSensitiveContentMaintenanceOptions>(
+            _shellConfiguration,
+            "Lombiq_Hosting_Tenants_Maintenance:ChangeUserSensitiveContent");
 
         services.AddScoped<IMaintenanceProvider, ChangeUserSensitiveContentMaintenanceProvider>();
     }

Lombiq.Hosting.Tenants.Maintenance/Maintenance/ElasticsearchIndices/DeleteElasticsearchIndicesMaintenanceProvider.cs

@@ -0,0 +1,31 @@
+using Lombiq.Hosting.Tenants.Maintenance.Extensions;
+using Lombiq.Hosting.Tenants.Maintenance.Models;
+using Lombiq.Hosting.Tenants.Maintenance.Services;
+using Microsoft.Extensions.Options;
+using OrchardCore.Search.Elasticsearch.Core.Services;
+using System.Threading.Tasks;
+
+namespace Lombiq.Hosting.Tenants.Maintenance.Maintenance.ElasticsearchIndices;
+
+public class DeleteElasticsearchIndicesMaintenanceProvider : MaintenanceProviderBase
+{
+    private readonly IOptions<ElasticsearchIndicesMaintenanceOptions> _options;
+    private readonly ElasticIndexManager _elasticIndexManager;
+
+    public DeleteElasticsearchIndicesMaintenanceProvider(
+        IOptions<ElasticsearchIndicesMaintenanceOptions> options,
+        ElasticIndexManager elasticIndexManager)
+    {
+        _options = options;
+        _elasticIndexManager = elasticIndexManager;
+    }
+
+    public override Task<bool> ShouldExecuteAsync(MaintenanceTaskExecutionContext context) =>
+        Task.FromResult(
+            _options.Value.DeleteMaintenanceIsEnabled &&
+            !context.WasLatestExecutionSuccessful());
+
+    public override Task ExecuteAsync(MaintenanceTaskExecutionContext context) =>
+        // Delete all tenant specific indexes in Elasticsearch.
+        _elasticIndexManager.DeleteIndex("*");
+}

Lombiq.Hosting.Tenants.Maintenance/Maintenance/ElasticsearchIndices/DeleteElasticsearchIndicesMiddleware.cs

@@ -0,0 +1,83 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.DependencyInjection;
+using OrchardCore.Environment.Shell;
+using OrchardCore.Locking.Distributed;
+using OrchardCore.Search.Elasticsearch.Core.Services;
+using System;
+using System.Net;
+using System.Threading.Tasks;
+
+namespace Lombiq.Hosting.Tenants.Maintenance.Maintenance.ElasticsearchIndices;
+
+public class DeleteElasticsearchIndicesMiddleware
+{
+    private readonly RequestDelegate _next;
+
+    private readonly ShellSettings _shellSettings;
+
+    private readonly IShellSettingsManager _shellSettingsManager;
+
+    private readonly IDistributedLock _distributedLock;
+
+    public DeleteElasticsearchIndicesMiddleware(
+        RequestDelegate next,
+        ShellSettings shellSettings,
+        IShellSettingsManager shellSettingsManager,
+        IDistributedLock distributedLock)
+    {
+        _next = next;
+        _shellSettings = shellSettings;
+        _shellSettingsManager = shellSettingsManager;
+        _distributedLock = distributedLock;
+    }
+
+    public async Task InvokeAsync(HttpContext httpContext)
+    {
+        if (httpContext.Request.Method != WebRequestMethods.Http.Get)
+        {
+            await _next.Invoke(httpContext);
+            return;
+        }
+
+        if (await InvokeNextIfUninitializedAsync(_shellSettings, httpContext)) return;
+
+        // Try to acquire a lock before starting installation
+        var (locker, locked) = await _distributedLock.TryAcquireLockAsync(
+            "ELASTICSEARCH_INDICES_DELETION",
+            TimeSpan.FromMinutes(1),
+            TimeSpan.FromMinutes(1));
+
+        if (!locked)
+        {
+            throw new TimeoutException($"Fails to acquire an elasticsearch indices deletion lock for the tenant: {_shellSettings.Name}");
+        }
+
+        await using var acquiredLock = locker;
+
+        // Check if the tenant was installed by another instance.
+        if (await InvokeNextIfUninitializedAsync(_shellSettings, httpContext)) return;
+
+        using var settings = (await _shellSettingsManager.LoadSettingsAsync(_shellSettings.Name)).AsDisposable();
+
+        // If the tenant was initialized by another instance, then skip again.
+        if (await InvokeNextIfUninitializedAsync(settings, httpContext)) return;
+
+        var elasticIndexManager = httpContext.RequestServices.GetRequiredService<ElasticIndexManager>();
+
+        // Delete all tenant specific indexes in Elasticsearch.
+        await elasticIndexManager.DeleteIndex("*");
+
+        await _next.Invoke(httpContext);
+    }
+
+    private async Task<bool> InvokeNextIfUninitializedAsync(ShellSettings shellSettings, HttpContext httpContext)
+    {
+        if (shellSettings.IsUninitialized())
+        {
+            return false;
+        }
+
+        await _next.Invoke(httpContext);
+        return true;
+    }
+}

Lombiq.Hosting.Tenants.Maintenance/Maintenance/ElasticsearchIndices/ElasticsearchIndicesMaintenanceOptions.cs

@@ -0,0 +1,8 @@
+namespace Lombiq.Hosting.Tenants.Maintenance.Maintenance.ElasticsearchIndices;
+
+public class ElasticsearchIndicesMaintenanceOptions
+{
+    public bool DeleteMaintenanceIsEnabled { get; set; }
+    public bool RebuildMaintenanceIsEnabled { get; set; }
+    public bool BeforeSetupMiddlewareIsEnabled { get; set; }
+}

Lombiq.Hosting.Tenants.Maintenance/Maintenance/ElasticsearchIndices/RebuildElasticsearchIndicesMaintenanceProvider.cs

@@ -0,0 +1,50 @@
+using Lombiq.Hosting.Tenants.Maintenance.Extensions;
+using Lombiq.Hosting.Tenants.Maintenance.Models;
+using Lombiq.Hosting.Tenants.Maintenance.Services;
+using Microsoft.Extensions.Options;
+using OrchardCore.Search.Elasticsearch.Core.Services;
+using System.Threading.Tasks;
+
+namespace Lombiq.Hosting.Tenants.Maintenance.Maintenance.ElasticsearchIndices;
+
+public class RebuildElasticsearchIndicesMaintenanceProvider : MaintenanceProviderBase
+{
+    private readonly IOptions<ElasticsearchIndicesMaintenanceOptions> _options;
+    private readonly ElasticIndexingService _elasticIndexingService;
+    private readonly ElasticIndexSettingsService _elasticIndexSettingsService;
+
+    public RebuildElasticsearchIndicesMaintenanceProvider(
+        IOptions<ElasticsearchIndicesMaintenanceOptions> options,
+        ElasticIndexingService elasticIndexingService,
+        ElasticIndexSettingsService elasticIndexSettingsService)
+    {
+        _options = options;
+        _elasticIndexingService = elasticIndexingService;
+        _elasticIndexSettingsService = elasticIndexSettingsService;
+    }
+
+    public override Task<bool> ShouldExecuteAsync(MaintenanceTaskExecutionContext context) =>
+        Task.FromResult(
+            _options.Value.RebuildMaintenanceIsEnabled &&
+            !context.WasLatestExecutionSuccessful());
+
+    public override async Task ExecuteAsync(MaintenanceTaskExecutionContext context)
+    {
+        var settings = await _elasticIndexSettingsService.GetSettingsAsync();
+        foreach (var setting in settings)
+        {
+            await _elasticIndexingService.RebuildIndexAsync(setting);
+
+            if (setting.QueryAnalyzerName != setting.AnalyzerName)
+            {
+                // Query Analyzer may be different until the index in rebuilt.
+                // Since the index is rebuilt, lets make sure we query using the same analyzer.
+                setting.QueryAnalyzerName = setting.AnalyzerName;
+
+                await _elasticIndexSettingsService.UpdateIndexAsync(setting);
+            }
+
+            await _elasticIndexingService.ProcessContentItemsAsync(setting.IndexName);
+        }
+    }
+}

Lombiq.Hosting.Tenants.Maintenance/Maintenance/ElasticsearchIndices/Startup.cs

@@ -0,0 +1,162 @@
+using Elasticsearch.Net;
+using Lombiq.HelpfulLibraries.OrchardCore.Mvc;
+using Lombiq.Hosting.Tenants.Maintenance.Constants;
+using Lombiq.Hosting.Tenants.Maintenance.Services;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Routing;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
+using Nest;
+using OrchardCore.Environment.Shell;
+using OrchardCore.Environment.Shell.Configuration;
+using OrchardCore.Modules;
+using OrchardCore.Search.Elasticsearch.Core.Models;
+using OrchardCore.Search.Elasticsearch.Core.Services;
+using System;
+using System.Linq;
+
+namespace Lombiq.Hosting.Tenants.Maintenance.Maintenance.ElasticsearchIndices;
+
+[Feature(FeatureNames.DeleteOrRebuildElasticsearchIndices)]
+public class DeleteElasticsearchIndicesStartup : StartupBase
+{
+    private readonly IShellConfiguration _shellConfiguration;
+
+    public DeleteElasticsearchIndicesStartup(IShellConfiguration shellConfiguration) => _shellConfiguration = shellConfiguration;
+
+    public override void ConfigureServices(IServiceCollection services)
+    {
+        services.BindAndConfigureSection<ElasticsearchIndicesMaintenanceOptions>(
+            _shellConfiguration,
+            "Lombiq_Hosting_Tenants_Maintenance:ElasticsearchIndicesOptions");
+
+        services.AddScoped<IMaintenanceProvider, DeleteElasticsearchIndicesMaintenanceProvider>();
+        services.AddScoped<IMaintenanceProvider, RebuildElasticsearchIndicesMaintenanceProvider>();
+    }
+}
+
+[Feature(FeatureNames.DeleteElasticsearchIndicesBeforeSetup)]
+public class DeleteElasticsearchIndicesBeforeSetupStartup : StartupBase
+{
+    private readonly IShellConfiguration _shellConfiguration;
+    private readonly ShellSettings _shellSettings;
+
+    public DeleteElasticsearchIndicesBeforeSetupStartup(IShellConfiguration shellConfiguration, ShellSettings shellSettings)
+    {
+        _shellConfiguration = shellConfiguration;
+        _shellSettings = shellSettings;
+    }
+
+    public override void ConfigureServices(IServiceCollection services)
+    {
+        services.BindAndConfigureSection<ElasticsearchIndicesMaintenanceOptions>(
+            _shellConfiguration,
+            "Lombiq_Hosting_Tenants_Maintenance:ElasticsearchIndicesOptions");
+
+        // After the setup, the Elasticsearch module can be loaded the regular way.
+        if (!_shellSettings.IsUninitialized()) return;
+
+        // This is necessary because the Elasticsearch module can't be enabled the regular way if this module is added
+        // as a setup feature, otherwise you get a ContentsAdminList shape missing exception on the admin dashboard. For
+        // more info see:
+        var configuration = _shellConfiguration.GetSection("OrchardCore_Elasticsearch");
+        var elasticConfiguration = configuration.Get<ElasticConnectionOptions>();
+        services.Configure<ElasticConnectionOptions>(o => o.ConfigurationExists = true);
+        // Otherwise the ElasticClient won't work. Copied all this from the OC Elasticsearch module.
+#pragma warning disable CA2000 // Call System. IDisposable. Dispose on object created by
+        // 'GetConnectionSettings(elasticConfiguration)' before all references to it are out of scope
+        var settings = GetConnectionSettings(elasticConfiguration);
+#pragma warning restore CA2000
+        services.AddSingleton<IElasticClient>(new ElasticClient(settings));
+        services.AddSingleton<ElasticIndexManager>();
+    }
+
+    public override void Configure(IApplicationBuilder app, IEndpointRouteBuilder routes, IServiceProvider serviceProvider)
+    {
+        if (!_shellSettings.IsUninitialized()) return;
+
+        var options = serviceProvider.GetRequiredService<IOptions<ElasticsearchIndicesMaintenanceOptions>>().Value;
+        if (options.BeforeSetupMiddlewareIsEnabled)
+        {
+            app.UseMiddleware<DeleteElasticsearchIndicesMiddleware>();
+        }
+    }
+
+    private static ConnectionSettings GetConnectionSettings(ElasticConnectionOptions elasticConfiguration)
+    {
+        // This is a copy of the OC Elasticsearch module's OrchardCore.Search.Elasticsearch.Startup.GetConnectionSettings method.
+#pragma warning disable CA2000 // Call System. IDisposable. Dispose on object created by
+        // 'GetConnectionPool(elasticConfiguration)' before all references to it are out of scope
+        var pool = GetConnectionPool(elasticConfiguration);
+#pragma warning restore CA2000
+
+        var settings = new ConnectionSettings(pool);
+
+        if (elasticConfiguration.ConnectionType != "CloudConnectionPool" &&
+            !string.IsNullOrWhiteSpace(elasticConfiguration.Username) &&
+            !string.IsNullOrWhiteSpace(elasticConfiguration.Password))
+        {
+            settings.BasicAuthentication(elasticConfiguration.Username, elasticConfiguration.Password);
+        }
+
+        if (!string.IsNullOrWhiteSpace(elasticConfiguration.CertificateFingerprint))
+        {
+            settings.CertificateFingerprint(elasticConfiguration.CertificateFingerprint);
+        }
+
+        if (elasticConfiguration.EnableApiVersioningHeader)
+        {
+            settings.EnableApiVersioningHeader();
+        }
+
+        return settings;
+    }
+
+    private static IConnectionPool GetConnectionPool(ElasticConnectionOptions elasticConfiguration)
+    {
+        var uris = elasticConfiguration.Ports.Select(port => new Uri($"{elasticConfiguration.Url}:{port.ToTechnicalString()}")).Distinct();
+        IConnectionPool pool = null;
+        switch (elasticConfiguration.ConnectionType)
+        {
+            case "SingleNodeConnectionPool":
+                pool = new SingleNodeConnectionPool(uris.First());
+                break;
+
+            case "CloudConnectionPool":
+                if (!string.IsNullOrWhiteSpace(elasticConfiguration.Username) &&
+                    !string.IsNullOrWhiteSpace(elasticConfiguration.Password) &&
+                    !string.IsNullOrWhiteSpace(elasticConfiguration.CloudId))
+                {
+                    // This is a copy of the OC Elasticsearch module's OrchardCore.Search.Elasticsearch.Startup.GetConnectionPool method.
+#pragma warning disable CA2000 // CA2000: Call System. IDisposable. Dispose on object created by
+                    // 'new BasicAuthenticationCredentials(' before all references to it are out of scope
+                    var credentials = new BasicAuthenticationCredentials(
+                        elasticConfiguration.Username,
+                        elasticConfiguration.Password);
+#pragma warning restore CA2000
+                    pool = new CloudConnectionPool(elasticConfiguration.CloudId, credentials);
+                }
+
+                break;
+
+            case "StaticConnectionPool":
+                pool = new StaticConnectionPool(uris);
+                break;
+
+            case "SniffingConnectionPool":
+                pool = new SniffingConnectionPool(uris);
+                break;
+
+            case "StickyConnectionPool":
+                pool = new StickyConnectionPool(uris);
+                break;
+
+            default:
+                pool = new SingleNodeConnectionPool(uris.First());
+                break;
+        }
+
+        return pool;
+    }
+}