Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add text separators #379

Merged
merged 2 commits into from
Feb 19, 2024
Merged

Add text separators #379

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8a10439
support text separators
ahmednfwela Feb 16, 2024
ad6043f
fix linting
ahmednfwela Feb 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion .code-samples.meilisearch.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,5 +288,11 @@ security_guide_delete_key_1: |-
await client.deleteKey('ac5cd97d-5a4b-4226-a868-2d0eb6d197ab');
search_parameter_guide_crop_marker_1: "await client.index('movies').search(\n 'shifu',\n SearchQuery(\n attributesToCrop: ['overview'],\n cropMarker: '[…]',\n ),\n );"
search_parameter_guide_highlight_tag_1: "await client.index('movies').search(\n 'winter feast',\n SearchQuery(\n attributesToHighlight: ['overview'],\n highlightPreTag: '<span class=\"highlight\">',\n highlightPostTag: '<\/span>',\n ),\n );"
geosearch_guide_filter_usage_3: "await client.index('restaurants').search(\n '',\n SearchQuery(\n filter:\n '_geoBoundingBox([45.494181, 9.214024], [45.449484, 9.179175])'));\n});"
geosearch_guide_filter_usage_3: "await client.index('restaurants').search(\n '',\n SearchQuery(\n filter:\n '_geoBoundingBox([45.494181, 9.214024], [45.449484, 9.179175])',\n ),\n );"
search_get_1: await client.index('movies').search('American ninja');
get_separator_tokens_1: await client.index('articles').getSeparatorTokens();
update_separator_tokens_1: "await client.index('articles').updateSeparatorTokens([\"|\", \"&hellip;\"]);"
reset_separator_tokens_1: await client.index('articles').resetSeparatorTokens();
get_non_separator_tokens_1: await client.index('articles').getNonSeparatorTokens();
update_non_separator_tokens_1: "await client.index('articles').updateNonSeparatorTokens([\"@\", \"#\"]);"
reset_non_separator_tokens_1: await client.index('articles').resetNonSeparatorTokens();
50 changes: 50 additions & 0 deletions lib/src/index.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -608,6 +608,56 @@ class MeiliSearchIndex {
);
}

/// Get separator tokens of the index.
Future<List<String>> getSeparatorTokens() async {
final response = await http
.getMethod<List<Object?>>('/indexes/$uid/settings/separator-tokens');

return response.data!.cast<String>();
}

/// Reset separator tokens of the index.
Future<Task> resetSeparatorTokens() async {
return await _getTask(
http.deleteMethod('/indexes/$uid/settings/separator-tokens'),
);
}

/// Update separator tokens of the index.
Future<Task> updateSeparatorTokens(List<String> separatorTokens) async {
return await _getTask(
http.putMethod(
'/indexes/$uid/settings/separator-tokens',
data: separatorTokens,
),
);
}

/// Get non separator tokens of the index.
Future<List<String>> getNonSeparatorTokens() async {
final response = await http.getMethod<List<Object?>>(
'/indexes/$uid/settings/non-separator-tokens');

return response.data!.cast<String>();
}

/// Reset separator tokens of the index.
Future<Task> resetNonSeparatorTokens() async {
return await _getTask(
http.deleteMethod('/indexes/$uid/settings/non-separator-tokens'),
);
}

/// Update separator tokens of the index.
Future<Task> updateNonSeparatorTokens(List<String> nonSeparatorTokens) async {
return await _getTask(
http.putMethod(
'/indexes/$uid/settings/non-separator-tokens',
data: nonSeparatorTokens,
),
);
}

/// Get searchable attributes of the index.
Future<List<String>> getSearchableAttributes() async {
final response = await http.getMethod<List<Object?>>(
Expand Down
18 changes: 18 additions & 0 deletions lib/src/settings/index_settings.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ class IndexSettings {
this.typoTolerance,
this.pagination,
this.faceting,
this.separatorTokens,
this.nonSeparatorTokens,
});

static const allAttributes = <String>['*'];
Expand All @@ -28,6 +30,12 @@ class IndexSettings {
/// List of ranking rules sorted by order of importance
List<String>? rankingRules;

/// List of tokens that will be considered as word separators by Meilisearch.
List<String>? separatorTokens;

/// List of tokens that will not be considered as word separators by Meilisearch.
List<String>? nonSeparatorTokens;

/// Attributes to use in [filters](https://www.meilisearch.com/docs/reference/api/search#filter)
List<String>? filterableAttributes;

Expand Down Expand Up @@ -64,6 +72,8 @@ class IndexSettings {
'typoTolerance': typoTolerance?.toMap(),
'pagination': pagination?.toMap(),
'faceting': faceting?.toMap(),
'separatorTokens': separatorTokens,
'nonSeparatorTokens': nonSeparatorTokens
};

factory IndexSettings.fromMap(Map<String, Object?> map) {
Expand All @@ -77,6 +87,8 @@ class IndexSettings {
final searchableAttributes = map['searchableAttributes'];
final displayedAttributes = map['displayedAttributes'];
final sortableAttributes = map['sortableAttributes'];
final separatorTokens = map['separatorTokens'];
final nonSeparatorTokens = map['nonSeparatorTokens'];

return IndexSettings(
synonyms: synonyms is Map<String, Object?>
Expand Down Expand Up @@ -109,6 +121,12 @@ class IndexSettings {
: null,
faceting:
faceting is Map<String, Object?> ? Faceting.fromMap(faceting) : null,
nonSeparatorTokens: nonSeparatorTokens is List<Object?>
? nonSeparatorTokens.cast<String>()
: null,
separatorTokens: separatorTokens is List<Object?>
? separatorTokens.cast<String>()
: null,
);
}
}
29 changes: 25 additions & 4 deletions test/code_samples.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -846,12 +846,33 @@ void main() {

// #docregion geosearch_guide_filter_usage_3
await client.index('restaurants').search(
'',
SearchQuery(
'',
SearchQuery(
filter:
'_geoBoundingBox([45.494181, 9.214024], [45.449484, 9.179175])'));
'_geoBoundingBox([45.494181, 9.214024], [45.449484, 9.179175])',
),
);
// #enddocregion
// #docregion get_separator_tokens_1
await client.index('articles').getSeparatorTokens();
// #enddocregion
// #docregion update_separator_tokens_1
await client.index('articles').updateSeparatorTokens(["|", "&hellip;"]);
// #enddocregion
// #docregion reset_separator_tokens_1
await client.index('articles').resetSeparatorTokens();
// #enddocregion
// #docregion get_non_separator_tokens_1
await client.index('articles').getNonSeparatorTokens();
// #enddocregion
// #docregion update_non_separator_tokens_1
await client.index('articles').updateNonSeparatorTokens(["@", "#"]);
// #enddocregion
// #docregion reset_non_separator_tokens_1
await client.index('articles').resetNonSeparatorTokens();
// #enddocregion
});
// #enddocregion

// skip this test, since it's only used for generating code samples
}, skip: true);

Expand Down
116 changes: 116 additions & 0 deletions test/settings_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,122 @@ void main() {
expect(resetRules, defaultRankingRules);
});

group('separator tokens', () {
Future<List<String>> doUpdate() async {
final toUpdate = <String>['zz', 'ff'];
var response =
await index.updateSeparatorTokens(toUpdate).waitFor(client: client);

expect(response.status, "succeeded");
return toUpdate;
}

test("Get", () async {
final initial = await index.getSeparatorTokens();
final initialFromSettings =
await index.getSettings().then((value) => value.separatorTokens);

expect(
initial,
equals(initialFromSettings),
);
});

test("Update", () async {
final toUpdate = await doUpdate();

final afterUpdate = await index.getSeparatorTokens();
final afterUpdateFromSettings =
await index.getSettings().then((value) => value.separatorTokens);
expect(
afterUpdateFromSettings,
unorderedEquals(toUpdate),
);
expect(
afterUpdate,
unorderedEquals(toUpdate),
);
});

test("Reset", () async {
//first update, then reset
await doUpdate();
final response =
await index.resetSeparatorTokens().waitFor(client: client);

expect(response.status, 'succeeded');
final afterReset = await index.getSeparatorTokens();
final afterResetFromSettings =
await index.getSettings().then((value) => value.separatorTokens);
expect(
afterReset,
equals(<String>[]),
);
expect(
afterResetFromSettings,
equals(<String>[]),
);
});
});
group('Non separator tokens', () {
Future<List<String>> doUpdate() async {
final toUpdate = <String>['/'];
var response = await index
.updateNonSeparatorTokens(toUpdate)
.waitFor(client: client);

expect(response.status, "succeeded");
return toUpdate;
}

test("Get", () async {
final initial = await index.getNonSeparatorTokens();
final initialFromSettings =
await index.getSettings().then((value) => value.nonSeparatorTokens);

expect(
initial,
equals(initialFromSettings),
);
});

test("Update", () async {
final toUpdate = await doUpdate();

final afterUpdate = await index.getNonSeparatorTokens();
final afterUpdateFromSettings =
await index.getSettings().then((value) => value.nonSeparatorTokens);
expect(
afterUpdateFromSettings,
unorderedEquals(toUpdate),
);
expect(
afterUpdate,
unorderedEquals(toUpdate),
);
});

test("Reset", () async {
//first update, then reset
await doUpdate();
final response =
await index.resetNonSeparatorTokens().waitFor(client: client);

expect(response.status, 'succeeded');
final afterReset = await index.getNonSeparatorTokens();
final afterResetFromSettings =
await index.getSettings().then((value) => value.nonSeparatorTokens);
expect(
afterReset,
equals(<String>[]),
);
expect(
afterResetFromSettings,
equals(<String>[]),
);
});
});

test('Getting, setting, and deleting searchable attributes', () async {
final updatedSearchableAttributes = ['title', 'id'];
await index
Expand Down
Loading