Merge #379

379: Add text separators r=curquiza a=ahmednfwela # Pull Request ## Related issue Fixes #359 ## What does this PR do? - Adds support to text separators - Fixes an extra bracket in the `geosearch_guide_filter_usage_3` sample ## PR checklist Please check if your PR fulfills the following requirements: - [x] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)? - [x] Have you read the contributing guidelines? - [x] Have you made sure that the title is accurate and descriptive of the changes? Thank you so much for contributing to Meilisearch! Co-authored-by: Ahmed Fwela <[email protected]>
meilisearch · Feb 19, 2024 · 077eb32 · 077eb32
2 parents b8ef30d + ad6043f
commit 077eb32
Show file tree

Hide file tree

Showing 5 changed files with 216 additions and 5 deletions.
diff --git a/.code-samples.meilisearch.yaml b/.code-samples.meilisearch.yaml
@@ -288,5 +288,11 @@ security_guide_delete_key_1: |-
   await client.deleteKey('ac5cd97d-5a4b-4226-a868-2d0eb6d197ab');
 search_parameter_guide_crop_marker_1: "await client.index('movies').search(\n      'shifu',\n      SearchQuery(\n        attributesToCrop: ['overview'],\n        cropMarker: '[…]',\n      ),\n    );"
 search_parameter_guide_highlight_tag_1: "await client.index('movies').search(\n      'winter feast',\n      SearchQuery(\n        attributesToHighlight: ['overview'],\n        highlightPreTag: '<span class=\"highlight\">',\n        highlightPostTag: '<\/span>',\n      ),\n    );"
-geosearch_guide_filter_usage_3: "await client.index('restaurants').search(\n    '',\n    SearchQuery(\n        filter:\n            '_geoBoundingBox([45.494181, 9.214024], [45.449484, 9.179175])'));\n});"
+geosearch_guide_filter_usage_3: "await client.index('restaurants').search(\n      '',\n      SearchQuery(\n        filter:\n            '_geoBoundingBox([45.494181, 9.214024], [45.449484, 9.179175])',\n      ),\n    );"
 search_get_1: await client.index('movies').search('American ninja');
+get_separator_tokens_1: await client.index('articles').getSeparatorTokens();
+update_separator_tokens_1: "await client.index('articles').updateSeparatorTokens([\"|\", \"&hellip;\"]);"
+reset_separator_tokens_1: await client.index('articles').resetSeparatorTokens();
+get_non_separator_tokens_1: await client.index('articles').getNonSeparatorTokens();
+update_non_separator_tokens_1: "await client.index('articles').updateNonSeparatorTokens([\"@\", \"#\"]);"
+reset_non_separator_tokens_1: await client.index('articles').resetNonSeparatorTokens();
diff --git a/lib/src/index.dart b/lib/src/index.dart
@@ -608,6 +608,56 @@ class MeiliSearchIndex {
     );
   }
 
+  /// Get separator tokens of the index.
+  Future<List<String>> getSeparatorTokens() async {
+    final response = await http
+        .getMethod<List<Object?>>('/indexes/$uid/settings/separator-tokens');
+
+    return response.data!.cast<String>();
+  }
+
+  /// Reset separator tokens of the index.
+  Future<Task> resetSeparatorTokens() async {
+    return await _getTask(
+      http.deleteMethod('/indexes/$uid/settings/separator-tokens'),
+    );
+  }
+
+  /// Update separator tokens of the index.
+  Future<Task> updateSeparatorTokens(List<String> separatorTokens) async {
+    return await _getTask(
+      http.putMethod(
+        '/indexes/$uid/settings/separator-tokens',
+        data: separatorTokens,
+      ),
+    );
+  }
+
+  /// Get non separator tokens of the index.
+  Future<List<String>> getNonSeparatorTokens() async {
+    final response = await http.getMethod<List<Object?>>(
+        '/indexes/$uid/settings/non-separator-tokens');
+
+    return response.data!.cast<String>();
+  }
+
+  /// Reset separator tokens of the index.
+  Future<Task> resetNonSeparatorTokens() async {
+    return await _getTask(
+      http.deleteMethod('/indexes/$uid/settings/non-separator-tokens'),
+    );
+  }
+
+  /// Update separator tokens of the index.
+  Future<Task> updateNonSeparatorTokens(List<String> nonSeparatorTokens) async {
+    return await _getTask(
+      http.putMethod(
+        '/indexes/$uid/settings/non-separator-tokens',
+        data: nonSeparatorTokens,
+      ),
+    );
+  }
+
   /// Get searchable attributes of the index.
   Future<List<String>> getSearchableAttributes() async {
     final response = await http.getMethod<List<Object?>>(

diff --git a/lib/src/settings/index_settings.dart b/lib/src/settings/index_settings.dart
@@ -15,6 +15,8 @@ class IndexSettings {
     this.typoTolerance,
     this.pagination,
     this.faceting,
+    this.separatorTokens,
+    this.nonSeparatorTokens,
   });
 
   static const allAttributes = <String>['*'];
@@ -28,6 +30,12 @@ class IndexSettings {
   /// List of ranking rules sorted by order of importance
   List<String>? rankingRules;
 
+  /// List of tokens that will be considered as word separators by Meilisearch.
+  List<String>? separatorTokens;
+
+  /// List of tokens that will not be considered as word separators by Meilisearch.
+  List<String>? nonSeparatorTokens;
+
   /// Attributes to use in [filters](https://www.meilisearch.com/docs/reference/api/search#filter)
   List<String>? filterableAttributes;
 
@@ -64,6 +72,8 @@ class IndexSettings {
         'typoTolerance': typoTolerance?.toMap(),
         'pagination': pagination?.toMap(),
         'faceting': faceting?.toMap(),
+        'separatorTokens': separatorTokens,
+        'nonSeparatorTokens': nonSeparatorTokens
       };
 
   factory IndexSettings.fromMap(Map<String, Object?> map) {
@@ -77,6 +87,8 @@ class IndexSettings {
     final searchableAttributes = map['searchableAttributes'];
     final displayedAttributes = map['displayedAttributes'];
     final sortableAttributes = map['sortableAttributes'];
+    final separatorTokens = map['separatorTokens'];
+    final nonSeparatorTokens = map['nonSeparatorTokens'];
 
     return IndexSettings(
       synonyms: synonyms is Map<String, Object?>
@@ -109,6 +121,12 @@ class IndexSettings {
           : null,
       faceting:
           faceting is Map<String, Object?> ? Faceting.fromMap(faceting) : null,
+      nonSeparatorTokens: nonSeparatorTokens is List<Object?>
+          ? nonSeparatorTokens.cast<String>()
+          : null,
+      separatorTokens: separatorTokens is List<Object?>
+          ? separatorTokens.cast<String>()
+          : null,
     );
   }
 }
diff --git a/test/code_samples.dart b/test/code_samples.dart
@@ -846,12 +846,33 @@ void main() {
 
       // #docregion geosearch_guide_filter_usage_3
       await client.index('restaurants').search(
-          '',
-          SearchQuery(
+            '',
+            SearchQuery(
               filter:
-                  '_geoBoundingBox([45.494181, 9.214024], [45.449484, 9.179175])'));
+                  '_geoBoundingBox([45.494181, 9.214024], [45.449484, 9.179175])',
+            ),
+          );
+      // #enddocregion
+      // #docregion get_separator_tokens_1
+      await client.index('articles').getSeparatorTokens();
+      // #enddocregion
+      // #docregion update_separator_tokens_1
+      await client.index('articles').updateSeparatorTokens(["|", "&hellip;"]);
+      // #enddocregion
+      // #docregion reset_separator_tokens_1
+      await client.index('articles').resetSeparatorTokens();
+      // #enddocregion
+      // #docregion get_non_separator_tokens_1
+      await client.index('articles').getNonSeparatorTokens();
+      // #enddocregion
+      // #docregion update_non_separator_tokens_1
+      await client.index('articles').updateNonSeparatorTokens(["@", "#"]);
+      // #enddocregion
+      // #docregion reset_non_separator_tokens_1
+      await client.index('articles').resetNonSeparatorTokens();
+      // #enddocregion
     });
-    // #enddocregion
+
     // skip this test, since it's only used for generating code samples
   }, skip: true);
 

diff --git a/test/settings_test.dart b/test/settings_test.dart
@@ -149,6 +149,122 @@ void main() {
       expect(resetRules, defaultRankingRules);
     });
 
+    group('separator tokens', () {
+      Future<List<String>> doUpdate() async {
+        final toUpdate = <String>['zz', 'ff'];
+        var response =
+            await index.updateSeparatorTokens(toUpdate).waitFor(client: client);
+
+        expect(response.status, "succeeded");
+        return toUpdate;
+      }
+
+      test("Get", () async {
+        final initial = await index.getSeparatorTokens();
+        final initialFromSettings =
+            await index.getSettings().then((value) => value.separatorTokens);
+
+        expect(
+          initial,
+          equals(initialFromSettings),
+        );
+      });
+
+      test("Update", () async {
+        final toUpdate = await doUpdate();
+
+        final afterUpdate = await index.getSeparatorTokens();
+        final afterUpdateFromSettings =
+            await index.getSettings().then((value) => value.separatorTokens);
+        expect(
+          afterUpdateFromSettings,
+          unorderedEquals(toUpdate),
+        );
+        expect(
+          afterUpdate,
+          unorderedEquals(toUpdate),
+        );
+      });
+
+      test("Reset", () async {
+        //first update, then reset
+        await doUpdate();
+        final response =
+            await index.resetSeparatorTokens().waitFor(client: client);
+
+        expect(response.status, 'succeeded');
+        final afterReset = await index.getSeparatorTokens();
+        final afterResetFromSettings =
+            await index.getSettings().then((value) => value.separatorTokens);
+        expect(
+          afterReset,
+          equals(<String>[]),
+        );
+        expect(
+          afterResetFromSettings,
+          equals(<String>[]),
+        );
+      });
+    });
+    group('Non separator tokens', () {
+      Future<List<String>> doUpdate() async {
+        final toUpdate = <String>['/'];
+        var response = await index
+            .updateNonSeparatorTokens(toUpdate)
+            .waitFor(client: client);
+
+        expect(response.status, "succeeded");
+        return toUpdate;
+      }
+
+      test("Get", () async {
+        final initial = await index.getNonSeparatorTokens();
+        final initialFromSettings =
+            await index.getSettings().then((value) => value.nonSeparatorTokens);
+
+        expect(
+          initial,
+          equals(initialFromSettings),
+        );
+      });
+
+      test("Update", () async {
+        final toUpdate = await doUpdate();
+
+        final afterUpdate = await index.getNonSeparatorTokens();
+        final afterUpdateFromSettings =
+            await index.getSettings().then((value) => value.nonSeparatorTokens);
+        expect(
+          afterUpdateFromSettings,
+          unorderedEquals(toUpdate),
+        );
+        expect(
+          afterUpdate,
+          unorderedEquals(toUpdate),
+        );
+      });
+
+      test("Reset", () async {
+        //first update, then reset
+        await doUpdate();
+        final response =
+            await index.resetNonSeparatorTokens().waitFor(client: client);
+
+        expect(response.status, 'succeeded');
+        final afterReset = await index.getNonSeparatorTokens();
+        final afterResetFromSettings =
+            await index.getSettings().then((value) => value.nonSeparatorTokens);
+        expect(
+          afterReset,
+          equals(<String>[]),
+        );
+        expect(
+          afterResetFromSettings,
+          equals(<String>[]),
+        );
+      });
+    });
+
     test('Getting, setting, and deleting searchable attributes', () async {
       final updatedSearchableAttributes = ['title', 'id'];
       await index