Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Reporting] use point-in-time for paging search results #144201

Merged
merged 15 commits into from
Nov 7, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -903,6 +903,13 @@ describe('SearchSource', () => {
expect(Object.keys(JSON.parse(searchSourceJSON))).toEqual(['highlightAll', 'from', 'sort']);
});

test('should add pit', () => {
const pit = { id: 'flimflam', keep_alive: '1m' };
searchSource.setField('pit', pit);
const { searchSourceJSON } = searchSource.serialize();
expect(searchSourceJSON).toBe(JSON.stringify({ pit }));
});

test('should serialize filters', () => {
const filter = [
{
Original file line number Diff line number Diff line change
@@ -667,6 +667,8 @@ export class SearchSource {
getConfig(UI_SETTINGS.SORT_OPTIONS)
);
return addToBody(key, sort);
case 'pit':
return addToRoot(key, val);
case 'aggs':
if ((val as unknown) instanceof AggConfigs) {
return addToBody('aggs', val.toDsl());
@@ -768,7 +770,7 @@ export class SearchSource {
const { getConfig } = this.dependencies;
const searchRequest = this.mergeProps();
searchRequest.body = searchRequest.body || {};
const { body, index, query, filters, highlightAll } = searchRequest;
const { body, index, query, filters, highlightAll, pit } = searchRequest;
searchRequest.indexType = this.getIndexType(index);
const metaFields = getConfig(UI_SETTINGS.META_FIELDS) ?? [];

@@ -911,6 +913,10 @@ export class SearchSource {
delete searchRequest.highlightAll;
}

if (pit) {
body.pit = pit;
}

return searchRequest;
}

11 changes: 9 additions & 2 deletions src/plugins/data/common/search/search_source/types.ts
Original file line number Diff line number Diff line change
@@ -39,6 +39,9 @@ export interface ISearchStartSearchSource
createEmpty: () => ISearchSource;
}

/**
* @deprecated use {@link estypes.SortResults} instead.
*/
export type EsQuerySearchAfter = [string | number, string | number];

export enum SortDirection {
@@ -112,9 +115,13 @@ export interface SearchSourceFields {
* {@link IndexPatternService}
*/
index?: DataView;
searchAfter?: EsQuerySearchAfter;
timeout?: string;
terminate_after?: number;
searchAfter?: estypes.SortResults;
/**
* Allow querying to use a point-in-time ID for paging results
*/
pit?: estypes.SearchPointInTimeReference;

parent?: SearchSourceFields;
}
@@ -160,7 +167,7 @@ export type SerializedSearchSourceFields = {
* {@link IndexPatternService}
*/
index?: string | DataViewSpec;
searchAfter?: EsQuerySearchAfter;
searchAfter?: estypes.SortResults;
timeout?: string;
terminate_after?: number;

Original file line number Diff line number Diff line change
@@ -35,13 +35,17 @@ export const esSearchStrategyProvider = (
throw new KbnServerError(`Unsupported index pattern type ${request.indexType}`, 400);
}

const isPit = request.params?.body?.pit != null;

const search = async () => {
try {
const config = await firstValueFrom(config$);
// @ts-expect-error params fall back to any, but should be valid SearchRequest params
const { terminateAfter, ...requestParams } = request.params ?? {};
const defaults = await getDefaultSearchParams(uiSettingsClient, { isPit });

const params = {
...(await getDefaultSearchParams(uiSettingsClient)),
...defaults,
...getShardTimeout(config),
...(terminateAfter ? { terminate_after: terminateAfter } : {}),
...requestParams,
Original file line number Diff line number Diff line change
@@ -18,19 +18,29 @@ export function getShardTimeout(
}

export async function getDefaultSearchParams(
uiSettingsClient: Pick<IUiSettingsClient, 'get'>
uiSettingsClient: Pick<IUiSettingsClient, 'get'>,
options = { isPit: false }
): Promise<{
max_concurrent_shard_requests?: number;
ignore_unavailable: boolean;
ignore_unavailable?: boolean;
track_total_hits: boolean;
}> {
const maxConcurrentShardRequests = await uiSettingsClient.get<number>(
UI_SETTINGS.COURIER_MAX_CONCURRENT_SHARD_REQUESTS
);
return {

const defaults: Awaited<ReturnType<typeof getDefaultSearchParams>> = {
max_concurrent_shard_requests:
maxConcurrentShardRequests > 0 ? maxConcurrentShardRequests : undefined,
ignore_unavailable: true, // Don't fail if the index/indices don't exist
track_total_hits: true,
};

// If the request has a point-in-time ID attached, it can not include ignore_unavailable from {@link estypes.IndicesOptions}.
// ES will reject the request as that option was set when the point-in-time was created.
// Otherwise, this option allows search to not fail when the index/indices don't exist
if (!options.isPit) {
defaults.ignore_unavailable = true;
}

return defaults;
}

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading