Merge pull request #1003 from IgorAlymov/SITKO-CORE-T-24

feat: added Highlighting to opensearch

src/Sitko.Core.Repository.Search/BaseRepositorySearchProvider.cs

@@ -34,11 +34,22 @@ await repository.GetAllAsync(q => q.Paginate(page, batchSize).OrderBy(e => e.Id)
         }
     }
 
-    protected override Task<TEntity[]> GetEntitiesAsync(TSearchModel[] searchModels,
+    protected override async Task<SearchResult<TEntity, TSearchModel>[]> GetEntitiesAsync(TSearchModel[] searchModels,
         CancellationToken cancellationToken = default)
     {
         var ids = searchModels.Select(s => ParseId(s.Id)).Distinct().ToArray();
-        return repository.GetByIdsAsync(ids, cancellationToken);
+        var entities = await repository.GetByIdsAsync(ids, cancellationToken);
+        List<SearchResult<TEntity, TSearchModel>> result = [];
+        foreach (var entity in entities)
+        {
+            var searchModel = searchModels.ToList().FirstOrDefault(model => model.Id == entity.Id.ToString());
+            if (searchModel != null)
+            {
+                result.Add(new SearchResult<TEntity, TSearchModel> { Entity = entity, ResultModel = searchModel });
+            }
+        }
+
+        return result.ToArray();
     }
 
     protected override string GetId(TEntity entity) =>

src/Sitko.Core.Search.ElasticSearch/ElasticSearcher.cs

@@ -101,7 +101,7 @@ public async Task<long> CountAsync(string indexName, string term, CancellationTo
         return resultsCount.Count;
     }
 
-    public async Task<TSearchModel[]> SearchAsync(string indexName, string term, int limit, SearchType searchType,
+    public async Task<TSearchModel[]> SearchAsync(string indexName, string term, int limit, SearchType searchType, bool withHighlight = false,
         CancellationToken cancellationToken = default)
     {
         indexName = $"{Options.Prefix}_{indexName}";

src/Sitko.Core.Search.OpenSearch/OpenSearchModuleOptions.cs

@@ -11,6 +11,8 @@ public class OpenSearchModuleOptions : SearchModuleOptions
     public bool EnableClientLogging { get; set; }
     public bool DisableCertificatesValidation { get; set; }
     public string CustomStemmer { get; set; } = "";
+    public string PreTags { get; set; } = "";
+    public string PostTags { get; set; } = "";
 }
 
 public class OpenSearchModuleOptionsValidator : AbstractValidator<OpenSearchModuleOptions>

src/Sitko.Core.Search.OpenSearch/OpenSearchSearcher.cs

@@ -10,7 +10,7 @@ public class OpenSearchSearcher<TSearchModel>(
     IOptionsMonitor<OpenSearchModuleOptions> optionsMonitor,
     ILogger<OpenSearchSearcher<TSearchModel>> logger)
     : ISearcher<TSearchModel>
-    where TSearchModel : BaseSearchModel
+    where TSearchModel : BaseSearchModel, new()
 {
     private OpenSearchModuleOptions Options => optionsMonitor.CurrentValue;
     private OpenSearchClient? client;
@@ -100,17 +100,29 @@ public async Task<long> CountAsync(string indexName, string term, CancellationTo
     }
 
     public async Task<TSearchModel[]> SearchAsync(string indexName, string term, int limit,
-        SearchType searchType, CancellationToken cancellationToken = default)
+        SearchType searchType, bool withHighlight = false, CancellationToken cancellationToken = default)
     {
         indexName = $"{Options.Prefix}_{indexName}";
-        var results = await GetClient()
-            .SearchAsync<TSearchModel>(x => GetSearchRequest(x, indexName, term, searchType, limit), cancellationToken);
-        if (results.ServerError != null)
+        var searchResponse = await GetClient()
+            .SearchAsync<TSearchModel>(x => GetSearchRequest(x, indexName, term, searchType, limit, withHighlight),
+                cancellationToken);
+        if (searchResponse.ServerError != null)
         {
-            logger.LogError("Error while searching in {IndexName}: {ErrorText}", indexName, results.ServerError);
+            logger.LogError("Error while searching in {IndexName}: {ErrorText}", indexName, searchResponse.ServerError);
         }
 
-        return results.Documents.ToArray();
+        var result = searchResponse.Hits.Select(h =>
+            new TSearchModel
+            {
+                Id = h.Source.Id,
+                Content = h.Source.Content,
+                Date = h.Source.Date,
+                Title = h.Source.Title,
+                Url = h.Source.Url,
+                Highlight = h.Highlight
+            }
+        ).ToArray();
+        return result;
     }
 
     public async Task<TSearchModel[]> GetSimilarAsync(string indexName, string id, int limit,
@@ -226,6 +238,7 @@ private OpenSearchClient GetClient()
             settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll)
                 .ServerCertificateValidationCallback((_, _, _, _) => true);
         }
+
         client = new OpenSearchClient(settings);
         return client;
     }
@@ -241,8 +254,8 @@ private static string GetSearchText(string? term)
         return names;
     }
 
-    private static SearchDescriptor<TSearchModel> GetSearchRequest(SearchDescriptor<TSearchModel> descriptor,
-        string indexName, string term, SearchType searchType, int limit = 0)
+    private SearchDescriptor<TSearchModel> GetSearchRequest(SearchDescriptor<TSearchModel> descriptor,
+        string indexName, string term, SearchType searchType, int limit = 0, bool withHighlight = false)
     {
         var names = GetSearchText(term);
         switch (searchType)
@@ -260,7 +273,17 @@ private static SearchDescriptor<TSearchModel> GetSearchRequest(SearchDescriptor<
                 break;
         }
 
-        return descriptor.Sort(s => s.Descending(SortSpecialField.Score).Descending(model => model.Date))
+        if (withHighlight)
+        {
+            descriptor.Highlight(h =>
+                h.Fields(fs => fs
+                    .Field(p => p.Title)
+                    .PreTags(Options.PreTags)
+                    .PostTags(Options.PostTags)));
+        }
+
+        return descriptor
+            .Sort(s => s.Descending(SortSpecialField.Score).Descending(model => model.Date))
             .Size(limit > 0 ? limit : 20)
             .Index(indexName.ToLowerInvariant());
     }

src/Sitko.Core.Search/BaseSearchProvider.cs

@@ -4,7 +4,7 @@
 namespace Sitko.Core.Search;
 
 [PublicAPI]
-public abstract class BaseSearchProvider<T, TEntityPk, TSearchModel> : ISearchProvider<T, TEntityPk>
+public abstract class BaseSearchProvider<T, TEntityPk, TSearchModel> : ISearchProvider<T, TEntityPk, TSearchModel>
     where T : class where TSearchModel : BaseSearchModel
 {
     private readonly ISearcher<TSearchModel> searcher;
@@ -31,20 +31,23 @@ public Task<long> CountAsync(string term, CancellationToken cancellationToken =
     public Task InitAsync(CancellationToken cancellationToken = default) =>
         searcher.InitAsync(IndexName, cancellationToken);
 
-    public async Task<T[]> SearchAsync(string term, int limit, SearchType searchType, CancellationToken cancellationToken = default)
+    public async Task<SearchResult<T, TSearchModel>[]> SearchAsync(string term, int limit, SearchType searchType,
+        bool withHighlight = false, CancellationToken cancellationToken = default)
     {
-        var result = await searcher.SearchAsync(IndexName, term, limit, searchType, cancellationToken);
+        var result = await searcher.SearchAsync(IndexName, term, limit, searchType, withHighlight, cancellationToken);
         return await LoadEntities(result, cancellationToken);
     }
 
     public async Task<TEntityPk[]> GetIdsAsync(string term, int limit, SearchType searchType,
+        bool withHighlight = false,
         CancellationToken cancellationToken = default)
     {
-        var result = await searcher.SearchAsync(IndexName, term, limit, searchType, cancellationToken);
+        var result = await searcher.SearchAsync(IndexName, term, limit, searchType, withHighlight, cancellationToken);
         return result.Select(m => ParseId(m.Id)).ToArray();
     }
 
-    public async Task<T[]> GetSimilarAsync(string id, int limit, CancellationToken cancellationToken = default)
+    public async Task<SearchResult<T, TSearchModel>[]> GetSimilarAsync(string id, int limit,
+        CancellationToken cancellationToken = default)
     {
         var result = await searcher.GetSimilarAsync(IndexName, id, limit, cancellationToken);
         return await LoadEntities(result, cancellationToken);
@@ -74,19 +77,18 @@ await searcher.DeleteAsync(IndexName, await GetSearchModelsAsync(entities, cance
 
     protected abstract TEntityPk ParseId(string id);
 
-    protected virtual async Task<T[]> LoadEntities(TSearchModel[] searchModels,
+    protected virtual async Task<SearchResult<T, TSearchModel>[]> LoadEntities(TSearchModel[] searchModels,
         CancellationToken cancellationToken = default)
     {
         var entities = await GetEntitiesAsync(searchModels, cancellationToken);
-        return entities.OrderBy(e => Array.FindIndex(searchModels, model => model.Id == GetId(e))).ToArray();
+        return entities.OrderBy(e => Array.FindIndex(searchModels, model => model.Id == GetId(e.Entity))).ToArray();
     }
 
     protected abstract Task<TSearchModel[]> GetSearchModelsAsync(T[] entities,
         CancellationToken cancellationToken = default);
 
-    protected abstract Task<T[]> GetEntitiesAsync(TSearchModel[] searchModels,
+    protected abstract Task<SearchResult<T, TSearchModel>[]> GetEntitiesAsync(TSearchModel[] searchModels,
         CancellationToken cancellationToken = default);
 
     protected abstract string GetId(T entity);
 }
-

src/Sitko.Core.Search/ISearchProvider.cs

@@ -8,18 +8,40 @@ public interface ISearchProvider
     Task InitAsync(CancellationToken cancellationToken = default);
 }
 
-public interface ISearchProvider<TEntity, TEntityPk> : ISearchProvider where TEntity : class
+public interface ISearchProvider<TEntity> : ISearchProvider where TEntity : class
 {
-    Task<TEntity[]> SearchAsync(string term, int limit, SearchType searchType, CancellationToken cancellationToken = default);
-    Task<TEntityPk[]> GetIdsAsync(string term, int limit, SearchType searchType, CancellationToken cancellationToken = default);
-    Task<TEntity[]> GetSimilarAsync(string id, int limit, CancellationToken cancellationToken = default);
-
-    Task<TEntityPk[]> GetSimilarIdsAsync(string id, int limit,
-        CancellationToken cancellationToken = default);
-
     Task AddOrUpdateEntityAsync(TEntity entity, CancellationToken cancellationToken = default);
     Task<bool> AddOrUpdateEntitiesAsync(TEntity[] entities, CancellationToken cancellationToken = default);
     Task<bool> DeleteEntityAsync(TEntity entity, CancellationToken cancellationToken = default);
     Task<bool> DeleteEntitiesAsync(TEntity[] entities, CancellationToken cancellationToken = default);
 }
 
+public interface ISearchProvider<TEntity, TEntityPk> : ISearchProvider<TEntity> where TEntity : class
+{
+    Task<TEntity[]> SearchAsync(string term, int limit, SearchType searchType,
+        CancellationToken cancellationToken = default);
+
+    Task<TEntityPk[]> GetIdsAsync(string term, int limit, SearchType searchType,
+        CancellationToken cancellationToken = default);
+
+    Task<TEntity[]> GetSimilarAsync(string id, int limit, CancellationToken cancellationToken = default);
+
+    Task<TEntityPk[]> GetSimilarIdsAsync(string id, int limit,
+        CancellationToken cancellationToken = default);
+}
+
+public interface ISearchProvider<TEntity, TEntityPk, TSearchModel> : ISearchProvider<TEntity>
+    where TEntity : class where TSearchModel : BaseSearchModel
+{
+    Task<SearchResult<TEntity, TSearchModel>[]> SearchAsync(string term, int limit, SearchType searchType,
+        bool withHighlight = false, CancellationToken cancellationToken = default);
+
+    Task<TEntityPk[]> GetIdsAsync(string term, int limit, SearchType searchType, bool withHighlight = false,
+        CancellationToken cancellationToken = default);
+
+    Task<SearchResult<TEntity, TSearchModel>[]> GetSimilarAsync(string id, int limit,
+        CancellationToken cancellationToken = default);
+
+    Task<TEntityPk[]> GetSimilarIdsAsync(string id, int limit,
+        CancellationToken cancellationToken = default);
+}

src/Sitko.Core.Search/ISearcher.cs

@@ -10,11 +10,12 @@ Task<bool> DeleteAsync(string indexName, IEnumerable<T> searchModels,
 
     Task<bool> DeleteAsync(string indexName, CancellationToken cancellationToken = default);
     Task<long> CountAsync(string indexName, string term, CancellationToken cancellationToken = default);
-    Task<T[]> SearchAsync(string indexName, string term, int limit, SearchType searchType, CancellationToken cancellationToken = default);
+
+    Task<T[]> SearchAsync(string indexName, string term, int limit, SearchType searchType, bool withHighlight = false,
+        CancellationToken cancellationToken = default);
 
     Task<T[]> GetSimilarAsync(string indexName, string id, int limit,
         CancellationToken cancellationToken = default);
 
     Task InitAsync(string indexName, CancellationToken cancellationToken = default);
 }
-

src/Sitko.Core.Search/SearchModel.cs

@@ -2,19 +2,14 @@ namespace Sitko.Core.Search;
 
 public class BaseSearchModel
 {
-    public BaseSearchModel(string id, string title, string url, string content, DateTimeOffset date)
+    public BaseSearchModel()
     {
-        Id = id;
-        Title = title;
-        Url = url;
-        Content = content;
-        Date = date;
     }
 
     public string Id { get; set; }
     public string Title { get; set; }
     public string Url { get; set; }
     public DateTimeOffset Date { get; set; }
     public string Content { get; set; }
+    public IReadOnlyDictionary<string, IReadOnlyCollection<string>> Highlight { get; set; }
 }
-

src/Sitko.Core.Search/SearchModule.cs

@@ -49,12 +49,12 @@ public override async Task InitAsync(IApplicationContext applicationContext, ISe
 
 public static class SearchModuleExtensions
 {
-    public static IServiceCollection RegisterSearchProvider<TSearchProvider, TEntity, TEntityPk>(
+    public static IServiceCollection RegisterSearchProvider<TSearchProvider, TEntity, TEntityPk, TSearchModel>(
         this IServiceCollection serviceCollection)
-        where TSearchProvider : class, ISearchProvider<TEntity, TEntityPk>
-        where TEntity : class =>
+        where TSearchProvider : class, ISearchProvider<TEntity, TEntityPk, TSearchModel>
+        where TEntity : class
+        where TSearchModel : BaseSearchModel =>
         serviceCollection.Scan(a => a.FromType<TSearchProvider>().AsSelfWithInterfaces());
 }
 
 public abstract class SearchModuleOptions : BaseModuleOptions;
-

src/Sitko.Core.Search/SearchResult.cs

@@ -0,0 +1,7 @@
+namespace Sitko.Core.Search;
+
+public record SearchResult<TEntity, TSearchModel>
+{
+    public TEntity Entity { get; set; }
+    public TSearchModel ResultModel { get; set; }
+}

tests/Sitko.Core.Search.ElasticSearch.Tests/ElasticSearchTests.cs

@@ -61,7 +61,7 @@ protected override IHostApplicationBuilder ConfigureApplication(IHostApplication
         });
 
         hostBuilder.Services.AddSingleton<TestModelProvider>();
-        hostBuilder.Services.RegisterSearchProvider<TestSearchProvider, TestModel, Guid>();
+        hostBuilder.Services.RegisterSearchProvider<TestSearchProvider, TestModel, Guid, TestSearchModel>();
         return hostBuilder;
     }
 }
@@ -75,27 +75,52 @@ public class TestModel
     public DateTimeOffset Date { get; set; } = DateTimeOffset.UtcNow;
 }
 
-public class TestSearchProvider : BaseSearchProvider<TestModel, Guid, BaseSearchModel>
+public class TestSearchModel : BaseSearchModel
+{
+    public TestSearchModel()
+    {
+    }
+}
+
+public class TestSearchProvider : BaseSearchProvider<TestModel, Guid, TestSearchModel>
 {
     private readonly TestModelProvider testModelProvider;
 
     public TestSearchProvider(ILogger<TestSearchProvider> logger,
         TestModelProvider testModelProvider,
-        ISearcher<BaseSearchModel>? searcher = null) : base(logger, searcher) =>
+        ISearcher<TestSearchModel>? searcher = null) : base(logger, searcher) =>
         this.testModelProvider = testModelProvider;
 
     protected override Guid ParseId(string id) => Guid.Parse(id);
 
-    protected override Task<BaseSearchModel[]> GetSearchModelsAsync(TestModel[] entities,
+    protected override Task<TestSearchModel[]> GetSearchModelsAsync(TestModel[] entities,
         CancellationToken cancellationToken = default) =>
         Task.FromResult(entities
-            .Select(e => new BaseSearchModel(e.Id.ToString(), e.Title, e.Url, e.Description, e.Date)).ToArray());
-
-    protected override Task<TestModel[]> GetEntitiesAsync(BaseSearchModel[] searchModels,
+            .Select(e => new TestSearchModel
+                {
+                    Id = e.Id.ToString(),
+                    Title = e.Title,
+                    Url = e.Url,
+                    Date = e.Date,
+                    Content = e.Description
+                }).ToArray());
+
+    protected override Task<SearchResult<TestModel, TestSearchModel>[]> GetEntitiesAsync(TestSearchModel[] searchModels,
         CancellationToken cancellationToken = default)
     {
         var ids = searchModels.Select(m => Guid.Parse(m.Id));
-        return Task.FromResult(testModelProvider.Models.Where(m => ids.Contains(m.Id)).ToArray());
+        var entities = testModelProvider.Models.Where(m => ids.Contains(m.Id));
+        List<SearchResult<TestModel, TestSearchModel>> result = [];
+        foreach (var entity in entities)
+        {
+            var searchModel = searchModels.ToList().FirstOrDefault(model => model.Id == entity.Id.ToString());
+            if (searchModel != null)
+            {
+                result.Add(new SearchResult<TestModel, TestSearchModel> { Entity = entity, ResultModel = searchModel });
+            }
+        }
+
+        return Task.FromResult(result.ToArray());
     }
 
     protected override string GetId(TestModel entity) => entity.Id.ToString();