Skip to content

Commit

Permalink
[8.10] Simplify test code, reenable skipped suite (elastic#163158) (e…
Browse files Browse the repository at this point in the history
…lastic#165333)

# Backport

This will backport the following commits from `main` to `8.10`:
- [Simplify test code, reenable skipped suite
(elastic#163158)](elastic#163158)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Gerard
Soldevila","email":"[email protected]"},"sourceCommit":{"committedDate":"2023-08-31T13:09:19Z","message":"Simplify
test code, reenable skipped suite (elastic#163158)\n\nAttempt at fixing
https://github.com/elastic/kibana/issues/149611\r\n\r\nI updated the
test code as follows:\r\n* Removed the RxJS logic and simply factorised
the reads to read only\r\nonce.\r\n* Got rid of the \"retry\" service.
There's already a mechanism in place\r\nto make sure the logs are
up-to-date.\r\n* Updated the `setCommonlyUsedTime` method to make sure
it awaits for\r\nthe popup to be ready before clicking.\r\n* Skipped 4
tests that seem outdated, the logs don't have the related\r\nentries
even after waiting for more than one minute and flushing (in\r\nfact,
they all seem to systematically fail on `main` too):\r\n *
lnsLegacyMetric\r\n * [Flights] Delays & Cancellations\r\n * [Flights]
Destination Weather\r\n * [Flights] Delay Buckets\r\n\r\nAttached is the
generated\r\n[kibana.log](https://github.com/elastic/kibana/files/12260144/kibana.log)\r\n(focussing
only the `browser.ts` tests).\r\n\r\nSo for the skipped tests, this does
not look like flakiness anymore, but\r\nrather outdated / incorrect
checks. I propose we review and update them\r\non a separate issue /
PR.\r\n\r\n50 runs
results\r\n[here](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3026).","sha":"a62d9a90f53c6ebec48d4982393c637cc73c26ac","branchLabelMapping":{"^v8.11.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Core","release_note:skip","test-failure-flaky","backport:prev-minor","v8.11.0"],"number":163158,"url":"https://github.com/elastic/kibana/pull/163158","mergeCommit":{"message":"Simplify
test code, reenable skipped suite (elastic#163158)\n\nAttempt at fixing
https://github.com/elastic/kibana/issues/149611\r\n\r\nI updated the
test code as follows:\r\n* Removed the RxJS logic and simply factorised
the reads to read only\r\nonce.\r\n* Got rid of the \"retry\" service.
There's already a mechanism in place\r\nto make sure the logs are
up-to-date.\r\n* Updated the `setCommonlyUsedTime` method to make sure
it awaits for\r\nthe popup to be ready before clicking.\r\n* Skipped 4
tests that seem outdated, the logs don't have the related\r\nentries
even after waiting for more than one minute and flushing (in\r\nfact,
they all seem to systematically fail on `main` too):\r\n *
lnsLegacyMetric\r\n * [Flights] Delays & Cancellations\r\n * [Flights]
Destination Weather\r\n * [Flights] Delay Buckets\r\n\r\nAttached is the
generated\r\n[kibana.log](https://github.com/elastic/kibana/files/12260144/kibana.log)\r\n(focussing
only the `browser.ts` tests).\r\n\r\nSo for the skipped tests, this does
not look like flakiness anymore, but\r\nrather outdated / incorrect
checks. I propose we review and update them\r\non a separate issue /
PR.\r\n\r\n50 runs
results\r\n[here](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3026).","sha":"a62d9a90f53c6ebec48d4982393c637cc73c26ac"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.11.0","labelRegex":"^v8.11.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/163158","number":163158,"mergeCommit":{"message":"Simplify
test code, reenable skipped suite (elastic#163158)\n\nAttempt at fixing
https://github.com/elastic/kibana/issues/149611\r\n\r\nI updated the
test code as follows:\r\n* Removed the RxJS logic and simply factorised
the reads to read only\r\nonce.\r\n* Got rid of the \"retry\" service.
There's already a mechanism in place\r\nto make sure the logs are
up-to-date.\r\n* Updated the `setCommonlyUsedTime` method to make sure
it awaits for\r\nthe popup to be ready before clicking.\r\n* Skipped 4
tests that seem outdated, the logs don't have the related\r\nentries
even after waiting for more than one minute and flushing (in\r\nfact,
they all seem to systematically fail on `main` too):\r\n *
lnsLegacyMetric\r\n * [Flights] Delays & Cancellations\r\n * [Flights]
Destination Weather\r\n * [Flights] Delay Buckets\r\n\r\nAttached is the
generated\r\n[kibana.log](https://github.com/elastic/kibana/files/12260144/kibana.log)\r\n(focussing
only the `browser.ts` tests).\r\n\r\nSo for the skipped tests, this does
not look like flakiness anymore, but\r\nrather outdated / incorrect
checks. I propose we review and update them\r\non a separate issue /
PR.\r\n\r\n50 runs
results\r\n[here](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3026).","sha":"a62d9a90f53c6ebec48d4982393c637cc73c26ac"}}]}]
BACKPORT-->

Co-authored-by: Gerard Soldevila <[email protected]>
  • Loading branch information
kibanamachine and gsoldevila authored Aug 31, 2023
1 parent 0323830 commit fe93fa7
Show file tree
Hide file tree
Showing 6 changed files with 71 additions and 126 deletions.
2 changes: 2 additions & 0 deletions test/functional/page_objects/time_picker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,9 @@ export class TimePickerPageObject extends FtrService {
* @param option 'Today' | 'This_week' | 'Last_15 minutes' | 'Last_24 hours' ...
*/
async setCommonlyUsedTime(option: CommonlyUsed | string) {
await this.testSubjects.exists('superDatePickerToggleQuickMenuButton', { timeout: 5000 });
await this.testSubjects.click('superDatePickerToggleQuickMenuButton');
await this.testSubjects.exists(`superDatePickerCommonlyUsed_${option}`, { timeout: 5000 });
await this.testSubjects.click(`superDatePickerCommonlyUsed_${option}`);
}

Expand Down
101 changes: 22 additions & 79 deletions x-pack/test/functional_execution_context/test_utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,15 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import Fs from 'fs/promises';
import Path from 'path';
import JSON5 from 'json5';
import Fs from 'fs/promises';
import { isEqualWith } from 'lodash';
import type { Ecs, KibanaExecutionContext } from '@kbn/core/server';
import type { RetryService } from '@kbn/ftr-common-functional-services';
import { concatMap, defer, filter, firstValueFrom, ReplaySubject, scan, timeout } from 'rxjs';

export const logFilePath = Path.resolve(__dirname, './kibana.log');
export const ANY = Symbol('any');

let logstream$: ReplaySubject<Ecs> | undefined;

export function getExecutionContextFromLogRecord(record: Ecs | undefined): KibanaExecutionContext {
if (record?.log?.logger !== 'execution_context' || !record?.message) {
throw new Error(`The record is not an entry of execution context`);
Expand All @@ -37,95 +34,41 @@ export function isExecutionContextLog(
}
}

// to avoid splitting log record containing \n symbol
const endOfLine = /(?<=})\s*\n/;
export async function assertLogContains({
description,
/**
* Checks the provided log records against the provided predicate
*/
export function assertLogContains({
logs,
predicate,
retry,
description,
}: {
description: string;
logs: Ecs[];
predicate: (record: Ecs) => boolean;
retry: RetryService;
}): Promise<void> {
// logs are written to disk asynchronously. I sacrificed performance to reduce flakiness.
await retry.waitFor(description, async () => {
if (!logstream$) {
logstream$ = getLogstream$();
}
try {
await firstValueFrom(logstream$.pipe(filter(predicate), timeout(5_000)));
return true;
} catch (err) {
return false;
}
});
description: string;
}) {
if (!logs.some(predicate)) {
throw new Error(`Unable to find log entries: ${description}`);
}
}

/**
* Creates an observable that continuously tails the log file.
* Reads the log file and parses the JSON objects that it contains.
*/
function getLogstream$(): ReplaySubject<Ecs> {
const stream$ = new ReplaySubject<Ecs>();

defer(async function* () {
const fd = await Fs.open(logFilePath, 'rs');
while (!stream$.isStopped) {
const { bytesRead, buffer } = await fd.read();
if (bytesRead) {
yield buffer.toString('utf8', 0, bytesRead);
}
}
await fd.close();
})
.pipe(
scan<string, { buffer: string; records: Ecs[] }>(
({ buffer }, chunk) => {
const logString = buffer.concat(chunk);
const lines = logString.split(endOfLine);
const lastLine = lines.pop();
const records = lines.map((s) => JSON.parse(s));

let leftover = '';
if (lastLine) {
try {
const validRecord = JSON.parse(lastLine);
records.push(validRecord);
} catch (err) {
leftover = lastLine;
}
}

return { buffer: leftover, records };
},
{
records: [], // The ECS entries in the logs
buffer: '', // Accumulated leftovers from the previous operation
}
),
concatMap(({ records }) => records)
)
.subscribe(stream$);

// let the content start flowing
stream$.subscribe();

return stream$;
}

export function closeLogstream() {
logstream$?.complete();
logstream$ = undefined;
export async function readLogFile(): Promise<Ecs[]> {
await forceSyncLogFile();
const logFileContent = await Fs.readFile(logFilePath, 'utf-8');
return logFileContent
.split('\n')
.filter(Boolean)
.map<Ecs>((str) => JSON5.parse(str));
}

/**
* Truncates the log file to avoid tests looking at the logs from previous executions.
*/
export async function clearLogFile() {
closeLogstream();
await Fs.writeFile(logFilePath, '', 'utf8');
await forceSyncLogFile();
logstream$ = getLogstream$();
}

/**
Expand Down
Loading

0 comments on commit fe93fa7

Please sign in to comment.