Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Search Sessions] Save all sessions, with persisted flag #89570

Merged
merged 84 commits into from
Feb 3, 2021
Merged
Show file tree
Hide file tree
Changes from 44 commits
Commits
Show all changes
84 commits
Select commit Hold shift + click to select a range
c22024f
[data.search] Add search session methods to search service contract
lukasolson Jan 12, 2021
1046c54
Merge branch 'master' into search-session-api
lukasolson Jan 15, 2021
7040716
Fix types
lukasolson Jan 15, 2021
eeddf84
Fix tests and switch to cancel
lukasolson Jan 15, 2021
f09eb38
Update docs
lukasolson Jan 15, 2021
e6124af
Merge branch 'master' into search-session-api
lukasolson Jan 19, 2021
e160019
Fix types/tests
lukasolson Jan 19, 2021
3c08c3d
Fix tests
lukasolson Jan 19, 2021
0cd92ed
Update status of SO before cancelling search requests
lukasolson Jan 19, 2021
f1a1d0b
Add API integration test
lukasolson Jan 19, 2021
552e545
Merge branch 'master' into search-session-api
lukasolson Jan 20, 2021
ee6e337
Merge branch 'master' into search-session-api
lukasolson Jan 25, 2021
32315b7
Fix types
lukasolson Jan 25, 2021
542de95
Update expiration route to use config defaultExpiration
lukasolson Jan 25, 2021
5b22249
Fix test
lukasolson Jan 25, 2021
66420b4
Update docs
lukasolson Jan 25, 2021
8a5e3d3
New logic for extend
lukasolson Jan 26, 2021
71582c7
Remove declare module
lukasolson Jan 26, 2021
03f413a
Search Sessions: Unskip Flaky Functional Test
tsullivan Jan 26, 2021
65840b3
Review feedback
lukasolson Jan 26, 2021
fdb940b
Merge branch 'master' of github.com:elastic/kibana into pr/87966
Jan 27, 2021
b2bce4b
fix ts
Jan 27, 2021
2e09f56
Merge branch 'master' into flaky/search-sessions
tsullivan Jan 28, 2021
cb1de74
Save all search sessions and then manage them based on their persiste…
Jan 28, 2021
2b3b90e
Get default search session expiration from config
Jan 28, 2021
cd23cef
randomize sleep time
Jan 28, 2021
5f21181
fix test
Jan 28, 2021
50081b5
Merge branch 'master' into search-session-api
lukasolson Jan 28, 2021
e24181c
Remove test that is no longer valid
lukasolson Jan 28, 2021
08ed572
Merge remote-tracking branch 'origin/search-session-api' into search-…
lukasolson Jan 28, 2021
cdcf562
fix test
Jan 28, 2021
bdbc748
Merge branch 'master' into sessions/save-all-sessions
kibanamachine Jan 28, 2021
667c9d3
Make sure we poll, and dont persist, searches not in the context of a…
Jan 28, 2021
867f1f4
Merge branch 'sessions/save-all-sessions' of github.com:lizozom/kiban…
Jan 28, 2021
51545ae
Merge branch 'master' of github.com:elastic/kibana into sessions/save…
Jan 28, 2021
763746b
Added keepalive unit tests
Jan 28, 2021
44ef260
Merge branch 'master' into search-session-api
lukasolson Jan 28, 2021
bbf85c5
Merge branch 'master' into sessions/save-all-sessions
kibanamachine Jan 29, 2021
01f292d
fix ts
Dosant Jan 29, 2021
7a85729
Merge branch 'master' into flaky/search-sessions
kibanamachine Jan 29, 2021
46cdf7f
Merge branch 'master' of github.com:elastic/kibana into sessions/save…
Jan 31, 2021
393d800
Merge branch 'sessions/save-all-sessions' of github.com:lizozom/kiban…
Jan 31, 2021
2d75b96
code review @lukasolson
Jan 31, 2021
13691e6
Merge branch 'master' of github.com:elastic/kibana into sessions/save…
Jan 31, 2021
1cd21fd
ts
Jan 31, 2021
a9cc263
More tests, rename onScreenTimeout to completedTimeout
Jan 31, 2021
bfcbdb2
lint
Jan 31, 2021
9b914f6
lint
Jan 31, 2021
ddb86a7
Merge branch 'master' into flaky/search-sessions
kibanamachine Jan 31, 2021
7ab767d
Delete async seaches
Jan 31, 2021
d924604
Support saved object pagination
Feb 1, 2021
11f429e
Merge branch 'master' of github.com:elastic/kibana into sessions/save…
Feb 1, 2021
48a90fb
better PersistedSearchSessionSavedObjectAttributes ts
Feb 1, 2021
de99c73
test titles
Feb 1, 2021
1bad580
Merge branch 'master' of github.com:elastic/kibana into sessions/save…
Feb 1, 2021
53a820e
Merge branch 'master' into sessions/save-all-sessions
kibanamachine Feb 1, 2021
ab0ce8f
Merge branch 'master' into sessions/save-all-sessions
kibanamachine Feb 1, 2021
99b4ca0
Merge branch 'master' into search-session-api
lukasolson Feb 1, 2021
99dc347
Fix undefined bug
lukasolson Feb 1, 2021
89d0887
Merge branch 'master' into sessions/save-all-sessions
kibanamachine Feb 2, 2021
eb735ef
Merge branch 'master' of github.com:elastic/kibana into sessions/save…
Feb 2, 2021
f305245
Remove runAt from monitoring task
Feb 2, 2021
68c163d
Merge branch 'sessions/save-all-sessions' of github.com:lizozom/kiban…
Feb 2, 2021
d92996e
support workload histograms that take into account overdue tasks
gmmorris Feb 2, 2021
0587c27
Merge branch 'sessions/save-all-sessions' of https://github.com/lizoz…
gmmorris Feb 2, 2021
353f722
Update touched when changing session status to complete \ error
Feb 2, 2021
899d547
Merge branch 'sessions/save-all-sessions' of github.com:lizozom/kiban…
Feb 2, 2021
6bd93ac
removed test
Feb 2, 2021
a6491dd
Updated management test data
Feb 2, 2021
fc29c7f
Rename configs
Feb 2, 2021
d61fa45
Merge branch 'sessions/save-all-sessions' into pr/89370
Feb 2, 2021
e582910
Merge branch 'master' into search-session-api
lukasolson Feb 2, 2021
fbfc1b1
Fix types
lukasolson Feb 2, 2021
2bd7e2b
delete tap first
Feb 3, 2021
cf45bb3
Merge branch 'master' of github.com:elastic/kibana into sessions/save…
Feb 3, 2021
19fa7e1
Merge branch 'sessions/save-all-sessions' of github.com:lizozom/kiban…
Feb 3, 2021
5c564c2
Merge branch 'master' of github.com:elastic/kibana into pr/87966
Feb 3, 2021
2b5c2aa
Merge branch 'pr/87966' into sessions/save-all-sessions
Feb 3, 2021
3eb8f01
Use DataRequestHandlerContext in maps
Feb 3, 2021
6462da6
ts
Feb 3, 2021
4b89dbf
Merge branch 'pr/87966' into sessions/save-all-sessions
Feb 3, 2021
c890574
Merge branch 'master' of github.com:elastic/kibana into sessions/save…
Feb 3, 2021
592c81b
Fixed ts
Feb 3, 2021
07523f3
Merge branch 'master' of github.com:elastic/kibana into sessions/save…
Feb 3, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions x-pack/plugins/data_enhanced/common/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
*/

export {
SEARCH_SESSION_TYPE,
ENHANCED_ES_SEARCH_STRATEGY,
EQL_SEARCH_STRATEGY,
EqlRequestParams,
Expand Down
22 changes: 17 additions & 5 deletions x-pack/plugins/data_enhanced/common/search/session/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,25 @@

import { SearchSessionStatus } from './';

export const SEARCH_SESSION_TYPE = 'search-session';
export interface SearchSessionSavedObjectAttributes {
sessionId: string;
/**
* User-facing session name to be displayed in session management
*/
name: string;
name?: string;
/**
* App that created the session. e.g 'discover'
*/
appId: string;
appId?: string;
/**
* Creation time of the session
*/
created: string;
/**
* Last touch time of the session
*/
touched: string;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need an explicit touched property or can we just go by the updated_at which every saved object already has?

/**
* Expiration time of the session. Expiration itself is managed by Elasticsearch.
*/
Expand All @@ -30,22 +36,28 @@ export interface SearchSessionSavedObjectAttributes {
/**
* urlGeneratorId
*/
urlGeneratorId: string;
urlGeneratorId?: string;
/**
* The application state that was used to create the session.
* Should be used, for example, to re-load an expired search session.
*/
initialState: Record<string, unknown>;
initialState?: Record<string, unknown>;
/**
* Application state that should be used to restore the session.
* For example, relative dates are conveted to absolute ones.
*/
restoreState: Record<string, unknown>;
restoreState?: Record<string, unknown>;
/**
* Mapping of search request hashes to their corresponsing info (async search id, etc.)
*/
idMapping: Record<string, SearchSessionRequestInfo>;

/**
* This value is true if the session was actively stored by the user. If it is false, the session may be purged by the system.
*/
persisted: boolean;
lukasolson marked this conversation as resolved.
Show resolved Hide resolved
}

export interface SearchSessionRequestInfo {
/**
* ID of the async search request
Expand Down
3 changes: 2 additions & 1 deletion x-pack/plugins/data_enhanced/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ export const configSchema = schema.object({
enabled: schema.boolean({ defaultValue: false }),
pageSize: schema.number({ defaultValue: 10000 }),
trackingInterval: schema.duration({ defaultValue: '10s' }),
inMemTimeout: schema.duration({ defaultValue: '1m' }),
notTouchedTimeout: schema.duration({ defaultValue: '5m' }),
lizozom marked this conversation as resolved.
Show resolved Hide resolved
notTouchedInProgressTimeout: schema.duration({ defaultValue: '1m' }),
maxUpdateRetries: schema.number({ defaultValue: 3 }),
defaultExpiration: schema.duration({ defaultValue: '7d' }),
management: schema.object({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,11 @@ import moment from 'moment';
import { from, race, timer } from 'rxjs';
import { mapTo, tap } from 'rxjs/operators';
import type { SharePluginStart } from 'src/plugins/share/public';
import { SessionsConfigSchema } from '../';
import type { ISessionsClient } from '../../../../../../../src/plugins/data/public';
import type { SearchSessionSavedObjectAttributes } from '../../../../common';
import { ISessionsClient } from '../../../../../../../src/plugins/data/public';
import { SearchSessionStatus } from '../../../../common/search';
import { ACTION } from '../components/actions';
import { UISession } from '../types';
import { PersistedSearchSessionSavedObjectAttributes, UISession } from '../types';
import { SessionsConfigSchema } from '..';

type UrlGeneratorsStart = SharePluginStart['urlGenerators'];

Expand Down Expand Up @@ -48,7 +47,7 @@ async function getUrlFromState(

// Helper: factory for a function to map server objects to UI objects
const mapToUISession = (urls: UrlGeneratorsStart, config: SessionsConfigSchema) => async (
savedObject: SavedObject<SearchSessionSavedObjectAttributes>
savedObject: SavedObject<PersistedSearchSessionSavedObjectAttributes>
): Promise<UISession> => {
const {
name,
Expand Down Expand Up @@ -110,6 +109,8 @@ export class SearchSessionsMgmtAPI {
perPage: mgmtConfig.maxSessions,
sortField: 'created',
sortOrder: 'asc',
searchFields: ['persisted'],
search: 'true',
})
);
const timeout$ = timer(refreshTimeout.asMilliseconds()).pipe(
Expand All @@ -129,7 +130,7 @@ export class SearchSessionsMgmtAPI {
const result = await race<FetchResult | null>(fetch$, timeout$).toPromise();
if (result && result.saved_objects) {
const savedObjects = result.saved_objects as Array<
SavedObject<SearchSessionSavedObjectAttributes>
SavedObject<PersistedSearchSessionSavedObjectAttributes>
>;
return await Promise.all(savedObjects.map(mapToUISession(this.deps.urls, this.config)));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,23 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { SearchSessionStatus } from '../../../common';
import { SearchSessionSavedObjectAttributes, SearchSessionStatus } from '../../../common';
import { ACTION } from './components/actions';

export const DATE_STRING_FORMAT = 'D MMM, YYYY, HH:mm:ss';

/**
* Some properties are optional for a non-persisted Search Session.
* This interface makes them mandatory, because management only shows persisted search sessions.
*/
export type PersistedSearchSessionSavedObjectAttributes = SearchSessionSavedObjectAttributes &
Required<
Pick<
SearchSessionSavedObjectAttributes,
'name' | 'appId' | 'urlGeneratorId' | 'initialState' | 'restoreState'
>
>;

export interface UISession {
id: string;
name: string;
Expand Down
7 changes: 6 additions & 1 deletion x-pack/plugins/data_enhanced/server/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
*/

import { CoreSetup, CoreStart, Logger, Plugin, PluginInitializerContext } from 'kibana/server';
import { Observable } from 'rxjs';
import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server';
import {
PluginSetup as DataPluginSetup,
Expand All @@ -22,6 +23,7 @@ import {
} from './search';
import { getUiSettings } from './ui_settings';
import type { DataEnhancedRequestHandlerContext } from './type';
import { ConfigSchema } from '../config';

interface SetupDependencies {
data: DataPluginSetup;
Expand All @@ -37,9 +39,11 @@ export class EnhancedDataServerPlugin
implements Plugin<void, void, SetupDependencies, StartDependencies> {
private readonly logger: Logger;
private sessionService!: SearchSessionService;
private config$: Observable<ConfigSchema>;

constructor(private initializerContext: PluginInitializerContext) {
constructor(private initializerContext: PluginInitializerContext<ConfigSchema>) {
this.logger = initializerContext.logger.get('data_enhanced');
this.config$ = this.initializerContext.config.create();
}

public setup(core: CoreSetup<DataPluginStart>, deps: SetupDependencies) {
Expand All @@ -51,6 +55,7 @@ export class EnhancedDataServerPlugin
deps.data.search.registerSearchStrategy(
ENHANCED_ES_SEARCH_STRATEGY,
enhancedEsSearchStrategyProvider(
this.config$,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just FYI, config is available synchronously: https://github.com/elastic/kibana/pull/88981

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm going to make a small separate PR out of this change

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this.initializerContext.config.legacy.globalConfig$,
this.logger,
usage
Expand Down
6 changes: 5 additions & 1 deletion x-pack/plugins/data_enhanced/server/routes/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,18 +91,22 @@ export function registerSessionRoutes(router: DataEnhancedPluginRouter, logger:
sortField: schema.maybe(schema.string()),
sortOrder: schema.maybe(schema.string()),
filter: schema.maybe(schema.string()),
searchFields: schema.maybe(schema.arrayOf(schema.string())),
search: schema.maybe(schema.string()),
}),
},
},
async (context, request, res) => {
const { page, perPage, sortField, sortOrder, filter } = request.body;
const { page, perPage, sortField, sortOrder, filter, searchFields, search } = request.body;
try {
const response = await context.search!.session.find({
page,
perPage,
sortField,
sortOrder,
filter,
searchFields,
search,
});

return res.ok({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,17 @@
*/

import { SavedObjectsType } from 'kibana/server';

export const SEARCH_SESSION_TYPE = 'search-session';
import { SEARCH_SESSION_TYPE } from '../../common';

export const searchSessionMapping: SavedObjectsType = {
name: SEARCH_SESSION_TYPE,
namespaceType: 'single',
hidden: true,
mappings: {
properties: {
persisted: {
type: 'boolean',
},
sessionId: {
type: 'keyword',
},
Expand All @@ -26,6 +28,9 @@ export const searchSessionMapping: SavedObjectsType = {
expires: {
type: 'date',
},
touched: {
type: 'date',
},
status: {
type: 'keyword',
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,6 @@ describe('EQL search strategy', () => {
expect(request).toEqual(
expect.objectContaining({
wait_for_completion_timeout: '100ms',
keep_alive: '1m',
})
);
});
Expand Down Expand Up @@ -156,7 +155,6 @@ describe('EQL search strategy', () => {
expect(request).toEqual(
expect.objectContaining({
wait_for_completion_timeout: '5ms',
keep_alive: '1m',
keep_on_completion: false,
})
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ export const eqlSearchStrategyProvider = (
logger: Logger
): ISearchStrategy<EqlSearchStrategyRequest, EqlSearchStrategyResponse> => {
async function cancelAsyncSearch(id: string, esClient: IScopedClusterClient) {
await esClient.asCurrentUser.asyncSearch.delete({ id });
const client = esClient.asCurrentUser.eql;
await client.delete({ id });
}

return {
Expand All @@ -41,11 +42,11 @@ export const eqlSearchStrategyProvider = (
uiSettingsClient
);
const params = id
? getDefaultAsyncGetParams()
? getDefaultAsyncGetParams(options)
: {
...(await getIgnoreThrottled(uiSettingsClient)),
...defaultParams,
...getDefaultAsyncGetParams(),
...getDefaultAsyncGetParams(options),
...request.params,
};
const promise = id
Expand Down
Loading