.Net: ADR for Text Search Abstractions

docs/decisions/diagrams/text-search-abstraction.mmd

@@ -0,0 +1,81 @@
+---
+title: ITextSearch
+---
+classDiagram
+    %% Use https://mermaid.live/ to preview this diagram. The VS Code extension does not handle namespaces.
+    direction TB
+
+    namespace Connectors_Memory_VectorStoreSearch {
+        class VectorStoreSearch~T~ {
+            SearchAsync~string~(query, searchSettings, cancellationToken) Task~KernelSearchResults~string~~
+            SearchAsync~TextSearchResult~(query, searchSettings, cancellationToken) Task~KernelSearchResults~TextSearchResult~~
+            SearchAsync~T~(query, searchSettings, cancellationToken) Task~KernelSearchResults~T~~
+        }
+    }
+
+    namespace Plugins_Web {
+        class BingTextSearch {
+            SearchAsync~string~(query, searchSettings, cancellationToken) Task~KernelSearchResults~string~~
+            SearchAsync~TextSearchResult~(query, searchSettings, cancellationToken) Task~KernelSearchResults~TextSearchResult~~
+            SearchAsync~BingWebPage~(query, searchSettings, cancellationToken) Task~KernelSearchResults~BingWebPage~~
+        }
+
+        class GoogleTextSearch {
+            SearchAsync~string~(query, searchSettings, cancellationToken) Task~KernelSearchResults~string~~
+            SearchAsync~TextSearchResult~(query, searchSettings, cancellationToken) Task~KernelSearchResults~TextSearchResult~~
+            SearchAsync~Result~(query, searchSettings, cancellationToken) Task~KernelSearchResults~Result~~
+        }
+    }
+
+    namespace Search {
+        class KernelSearchResults~T~ {
+            +long? TotalCount
+            +object? InnerContent
+            +IReadOnlyDictionary? Metadata
+            +IAsyncEnumerable~T~ Results
+        }
+
+        class ITextSearch~T~ {
+            <<interface>>
+            SearchAsync~T~(query, searchSettings, cancellationToken) Task~KernelSearchResults~T~~
+        }
+
+        class SearchOptions {
+            +string Index
+            +int Count
+            +int Offset
+            +BasicFilterOptions BasicFilter
+        }
+
+        class BasicFilterOptions {
+            +IEnumerable~FilterClause~ FilterClauses
+            Equality(field, value) BasicFilterOptions
+        }
+
+        class FilterClause {
+            +FilterClauseType Type
+        }
+
+        class FilterClauseType {
+            Equality
+        }
+
+        class TextSearchResult {
+            +string? Name
+            +string? Value
+            +string? Link
+        }
+    }
+
+    ITextSearch ..> SearchOptions
+    ITextSearch ..> KernelSearchResults
+    SearchOptions ..> BasicFilterOptions
+    BasicFilterOptions ..> FilterClause
+    BingTextSearch --|> ITextSearch
+    GoogleTextSearch --|> ITextSearch
+    AzureAISearch --|> ITextSearch
+
+    BingTextSearch ..> TextSearchResult
+    GoogleTextSearch ..> TextSearchResult
+    AzureAISearch ..> TextSearchResult
+

docs/decisions/diagrams/text-search-imemorystore.mmd

@@ -0,0 +1,35 @@
+block-beta
+columns 1
+  block:Plugin
+    PluginAbstraction[["TextMemoryPlugin"]]
+    PluginDescription["Provides a plugin to save or recall information from the long or short term memory."]
+  end
+  space
+  block:SemanticMemory
+    SemanticMemoryAbstraction[["ISemanticTextMemory"]]
+    SemanticMemoryDescription["An interface for semantic memory that creates and recalls memories associated with text."]
+  end
+  space
+  block:Memory
+    MemoryAbstraction[["IMemoryStore"]]
+    MemoryDescription["An interface for storing and retrieving indexed MemoryRecord objects in a data store."]
+  end
+  space
+  block:MemoryClient
+    MemoryClientAbstraction[["Memory Client"]]
+    MemoryClientDescription["A database specific service client."]
+  end
+  space
+  block:VectorDatabaseService
+    VectorDatabase[["Vector Database"]]
+    VectorDatabaseDescription["A vector database e.g. Redis, Milvus, Pinecone, Qdrant."]
+  end
+  Plugin-- "Uses the provided Semantic Memory implementation to recall text memories." -->SemanticMemory
+  SemanticMemory-- "Uses the provided Memory Store to query the database." -->Memory
+  Memory-- "Uses a Memory Client (if available) to interact with the database." -->MemoryClient
+  MemoryClient-- "Invokes the Vector Database Service." -->VectorDatabaseService
+  style PluginDescription fill:#FFF,stroke-width:0px
+  style SemanticMemoryDescription fill:#FFF,stroke-width:0px
+  style MemoryDescription fill:#FFF,stroke-width:0px
+  style MemoryClientDescription fill:#FFF,stroke-width:0px
+  style VectorDatabaseDescription fill:#FFF,stroke-width:0px

docs/decisions/diagrams/text-search-iwebsearchengineconnector.mmd

@@ -0,0 +1,23 @@
+block-beta
+columns 1
+  block:Plugin
+    PluginAbstraction[["WebSearchEnginePlugin"]]
+    PluginDescription["Provides a plugin to perform a web search."]
+  end
+  space
+  block:Connector
+    ConnectorAbstraction[["IWebSearchEngineConnector"]]
+    ConnectorDescription["An interface for a web search engine connector.<br/>Returns either search result summaries or a normalised <code>WebPage</code> instance."]
+  end
+  space
+  block:Service
+    ServiceType[["Web Search Service"]]
+    ServiceDescription["A web search service e.g. Bing, Google, ..."]
+  end
+
+  Plugin-- "Uses the provided IWebSearchEngineConnector implementation to invoke a web search engine." -->Connector
+  Connector-- "Invokes the web search engine using REST or a service client." -->Service
+
+  style PluginDescription fill:#FFF,stroke-width:0px
+  style ConnectorDescription fill:#FFF,stroke-width:0px
+  style ServiceDescription fill:#FFF,stroke-width:0px

dotnet/src/SemanticKernel.Abstractions/Search/BasicFilterOptions.cs

@@ -0,0 +1,39 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.SemanticKernel.Search;
+
+/// <summary>
+/// Used to provide basic filtering when using <see cref="ITextSearch"/>.
+/// </summary>
+/// <remarks>
+/// A basic filter has a collection of <see cref="FilterClause"/>s that can be used by
+/// an <see cref="ITextSearch"/> implementation to request that the underlying search
+/// service filter the search results.
+/// </remarks>
+[Experimental("SKEXP0001")]
+public sealed class BasicFilterOptions
+{
+    /// <summary>
+    /// The clauses to apply to the <see cref="BasicFilterOptions" />
+    /// </summary>
+    public IEnumerable<FilterClause> FilterClauses => this._filterClauses;
+
+    /// <summary>
+    /// Add a equality clause to the filter options.
+    /// </summary>
+    /// <param name="fieldName">Name of the field.</param>
+    /// <param name="value">Value of the field.</param>
+    /// <returns>FilterOptions instance to allow fluent configuration.</returns>
+    public BasicFilterOptions Equality(string fieldName, object value)
+    {
+        this._filterClauses.Add(new EqualityFilterClause(fieldName, value));
+        return this;
+    }
+
+    #region private
+    private readonly List<FilterClause> _filterClauses = [];
+    #endregion
+}

dotnet/src/SemanticKernel.Abstractions/Search/EqualityFilterClause.cs

@@ -0,0 +1,28 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.SemanticKernel.Search;
+
+/// <summary>
+/// A <see cref="FilterClause"/> which filters using equality of a field value with the specified field value.
+/// </summary>
+/// <remarks>
+/// The <see cref="EqualityFilterClause"/> is used to request that the underlying search service should
+/// filter search results based on the equality of a field value with the specified field value.
+/// </remarks>
+/// <param name="fieldName">Field name.</param>
+/// <param name="value">Field value.</param>
+[Experimental("SKEXP0001")]
+public sealed class EqualityFilterClause(string fieldName, object value) : FilterClause
+{
+    /// <summary>
+    /// Field name to match.
+    /// </summary>
+    public string FieldName { get; private set; } = fieldName;
+
+    /// <summary>
+    /// Field value to match.
+    /// </summary>
+    public object Value { get; private set; } = value;
+}

dotnet/src/SemanticKernel.Abstractions/Search/FilterClause.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.SemanticKernel.Search;
+
+/// <summary>
+/// Base class for filter clauses.
+/// </summary>
+/// <remarks>
+/// The <see cref="FilterClause"/> is used to request that the underlying search service should
+/// filter search results based on the specified criteria.
+/// </remarks>
+[Experimental("SKEXP0001")]
+public abstract class FilterClause;

dotnet/src/SemanticKernel.Abstractions/Search/ITextSearch.cs

@@ -0,0 +1,47 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Diagnostics.CodeAnalysis;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.SemanticKernel.Search;
+
+/// <summary>
+/// Interface for text based search queries for use with Semantic Kernel prompts and automatic function calling.
+/// </summary>
+[Experimental("SKEXP0001")]
+public interface ITextSearch
+{
+    /// <summary>
+    /// Perform a search for content related to the specified query and return <see cref="string"/> values representing the search results.
+    /// </summary>
+    /// <param name="query">What to search for.</param>
+    /// <param name="searchOptions">Options used when executing a text search.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    public Task<KernelSearchResults<string>> SearchAsync(
+        string query,
+        TextSearchOptions? searchOptions = null,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Perform a search for content related to the specified query and return <see cref="TextSearchResult"/> values representing the search results.
+    /// </summary>
+    /// <param name="query">What to search for.</param>
+    /// <param name="searchOptions">Options used when executing a text search.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    public Task<KernelSearchResults<TextSearchResult>> GetTextSearchResultsAsync(
+        string query,
+        TextSearchOptions? searchOptions = null,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Perform a search for content related to the specified query and return <see cref="object"/> values representing the search results.
+    /// </summary>
+    /// <param name="query">What to search for.</param>
+    /// <param name="searchOptions">Options used when executing a text search.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    public Task<KernelSearchResults<object>> GetSearchResultsAsync(
+        string query,
+        TextSearchOptions? searchOptions = null,
+        CancellationToken cancellationToken = default);
+}

dotnet/src/SemanticKernel.Abstractions/Search/KernelSearchResults.cs

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft. All rights reserved.
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.SemanticKernel.Search;
+
+/// <summary>
+/// Represents the search results returned from a <see cref="ITextSearch" /> service.
+/// </summary>
+/// <remarks>
+/// Initializes a new instance of the <see cref="KernelSearchResults{T}"/> class.
+/// </remarks>
+/// <param name="results">The search results.</param>
+/// <param name="totalCount">The total count of results found by the search operation, or null if the count was not requested.</param>
+/// <param name="metadata">Metadata associated with the search results.</param>
+[Experimental("SKEXP0001")]
+public sealed class KernelSearchResults<T>(IAsyncEnumerable<T> results, long? totalCount = null, IReadOnlyDictionary<string, object?>? metadata = null)
+{
+    /// <summary>
+    /// The total count of results found by the search operation, or null
+    /// if the count was not requested or cannot be computed.
+    /// </summary>
+    /// <remarks>
+    /// This value represents the total number of results that are available for the current query and not the number of results being returned.
+    /// </remarks>
+    public long? TotalCount { get; internal set; } = totalCount;
+
+    /// <summary>
+    /// The metadata associated with the content.
+    /// </summary>
+    public IReadOnlyDictionary<string, object?>? Metadata { get; } = metadata;
+
+    /// <summary>
+    /// The search results.
+    /// </summary>
+    public IAsyncEnumerable<T> Results { get; } = results;
+}

dotnet/src/SemanticKernel.Abstractions/Search/TextSearchOptions.cs

@@ -0,0 +1,40 @@
+// Copyright (c) Microsoft. All rights reserved.
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.SemanticKernel.Search;
+
+/// <summary>
+/// Options which can be applied when using <see cref="ITextSearch"/>.
+/// </summary>
+[Experimental("SKEXP0001")]
+public sealed class TextSearchOptions
+{
+    /// <summary>
+    /// Default number of search results to return.
+    /// </summary>
+    public static readonly int DefaultCount = 5;
+
+    /// <summary>
+    /// Flag indicating the total count should be included in the results.
+    /// </summary>
+    /// <remarks>
+    /// Default value is false.
+    /// Not all text search implementations will support this option.
+    /// </remarks>
+    public bool IncludeTotalCount { get; set; } = false;
+
+    /// <summary>
+    /// The basic filter expression to apply to the search query.
+    /// </summary>
+    public BasicFilterOptions? BasicFilter { get; set; }
+
+    /// <summary>
+    /// Number of search results to return.
+    /// </summary>
+    public int Count { get; set; } = DefaultCount;
+
+    /// <summary>
+    /// The index of the first result to return.
+    /// </summary>
+    public int Offset { get; set; } = 0;
+}