Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

unskipping functional tests to check their flakiness #95349

Closed
wants to merge 5 commits into from
Closed

unskipping functional tests to check their flakiness #95349

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
520c93b
unskipping functional tests to check their flakiness
rashmivkulkarni Mar 24, 2021
02d8329
Merge branch 'master' of github.com:elastic/kibana into unskip_functi…
rashmivkulkarni Mar 30, 2021
eae9aaf
Merge branch 'master' into unskip_functional_tests_master
kibanamachine Apr 2, 2021
237e32a
Merge branch 'master' into unskip_functional_tests_master
rashmivkulkarni Apr 9, 2021
ecb2d08
Merge branch 'master' into unskip_functional_tests_master
rashmivkulkarni Apr 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion test/functional/apps/dashboard/dashboard_save.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const testSubjects = getService('testSubjects');

// FLAKY: https://github.com/elastic/kibana/issues/89476
describe.skip('dashboard save', function describeIndexTests() {
describe('dashboard save', function describeIndexTests() {
this.tags('includeFirefox');
const dashboardName = 'Dashboard Save Test';
const dashboardNameEnterKey = 'Dashboard Save Test with Enter Key';
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/dashboard/dashboard_unsaved_state.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
let unsavedPanelCount = 0;

// FLAKY: https://github.com/elastic/kibana/issues/91191
describe.skip('dashboard unsaved panels', () => {
describe('dashboard unsaved panels', () => {
before(async () => {
await esArchiver.load('dashboard/current/kibana');
await kibanaServer.uiSettings.replace({
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/dashboard/embeddable_rendering.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
};

// Failing: See https://github.com/elastic/kibana/issues/76245
describe.skip('dashboard embeddable rendering', function describeIndexTests() {
describe('dashboard embeddable rendering', function describeIndexTests() {
before(async () => {
await security.testUser.setRoles(['kibana_admin', 'animals', 'test_logstash_reader']);
await esArchiver.load('dashboard/current/kibana');
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/dashboard/url_field_formatter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
};

// FLAKY: https://github.com/elastic/kibana/issues/79463
describe.skip('Changing field formatter to Url', () => {
describe('Changing field formatter to Url', () => {
before(async function () {
await esArchiver.load('dashboard/current/kibana');
await kibanaServer.uiSettings.replace({
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/discover/_data_grid_context.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const browser = getService('browser');

// FLAKY: https://github.com/elastic/kibana/issues/94545
describe.skip('discover data grid context tests', () => {
describe('discover data grid context tests', () => {
before(async () => {
await esArchiver.load('discover');
await esArchiver.loadIfNeeded('logstash_functional');
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/discover/_data_grid_doc_table.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
});

// flaky https://github.com/elastic/kibana/issues/94889
it.skip('should show popover with expanded cell content by click on expand button', async () => {
it('should show popover with expanded cell content by click on expand button', async () => {
log.debug('open popover with expanded cell content to get json from the editor');
const documentCell = await dataGrid.getCellElement(1, 3);
await documentCell.click();
Expand Down
4 changes: 2 additions & 2 deletions test/functional/apps/management/_scripted_fields.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ export default function ({ getService, getPageObjects }) {

//add a test to sort boolean
//existing bug: https://github.com/elastic/kibana/issues/75519 hence the issue is skipped.
it.skip('should sort scripted field value in Discover', async function () {
it('should sort scripted field value in Discover', async function () {
await testSubjects.click(`docTableHeaderFieldSort_${scriptedPainlessFieldName2}`);
// after the first click on the scripted field, it becomes secondary sort after time.
// click on the timestamp twice to make it be the secondary sort key.
Expand Down Expand Up @@ -424,7 +424,7 @@ export default function ({ getService, getPageObjects }) {

//add a test to sort date scripted field
//https://github.com/elastic/kibana/issues/75711
it.skip('should sort scripted field value in Discover', async function () {
it('should sort scripted field value in Discover', async function () {
await testSubjects.click(`docTableHeaderFieldSort_${scriptedPainlessFieldName2}`);
// after the first click on the scripted field, it becomes secondary sort after time.
// click on the timestamp twice to make it be the secondary sort key.
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/management/_scripted_fields_preview.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export default function ({ getService, getPageObjects }) {
const SCRIPTED_FIELD_NAME = 'myScriptedField';

// FLAKY: https://github.com/elastic/kibana/issues/89475
describe.skip('scripted fields preview', () => {
describe('scripted fields preview', () => {
before(async function () {
await browser.setWindowSize(1200, 800);
await PageObjects.settings.createIndexPattern();
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/management/exports/_import_too_big.ndjson
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion test/functional/apps/saved_objects_management/edit_saved_object.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
};

// Flaky: https://github.com/elastic/kibana/issues/68400
describe.skip('saved objects edition page', () => {
describe('saved objects edition page', () => {
beforeEach(async () => {
await esArchiver.load('saved_objects_management/edit_saved_object');
});
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/visualize/_visualize_listing.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const listingTable = getService('listingTable');

// FLAKY: https://github.com/elastic/kibana/issues/40912
describe.skip('visualize listing page', function describeIndexTests() {
describe('visualize listing page', function describeIndexTests() {
const vizName = 'Visualize Listing Test';

describe('create and delete', function () {
Expand Down
8 changes: 4 additions & 4 deletions x-pack/test/functional/apps/dashboard/feature_controls/dashboard_security.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
const savedQueryManagementComponent = getService('savedQueryManagementComponent');

// FLAKY: https://github.com/elastic/kibana/issues/86950
describe.skip('dashboard feature controls security', () => {
describe('dashboard feature controls security', () => {
before(async () => {
await esArchiver.load('dashboard/feature_controls/security');
await esArchiver.loadIfNeeded('logstash_functional');
Expand Down Expand Up @@ -109,7 +109,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
});

// Can't figure out how to get this test to pass
it.skip(`create new dashboard shows addNew button`, async () => {
it(`create new dashboard shows addNew button`, async () => {
await PageObjects.common.navigateToActualUrl(
'dashboard',
DashboardConstants.CREATE_NEW_DASHBOARD_URL,
Expand Down Expand Up @@ -321,7 +321,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
});

// Has this behavior changed?
it.skip(`create new dashboard redirects to the home page`, async () => {
it(`create new dashboard redirects to the home page`, async () => {
await PageObjects.common.navigateToActualUrl(
'dashboard',
DashboardConstants.CREATE_NEW_DASHBOARD_URL,
Expand Down Expand Up @@ -439,7 +439,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
});

// Has this behavior changed?
it.skip(`create new dashboard redirects to the home page`, async () => {
it(`create new dashboard redirects to the home page`, async () => {
await PageObjects.common.navigateToActualUrl(
'dashboard',
DashboardConstants.CREATE_NEW_DASHBOARD_URL,
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/dashboard/reporting/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@ Every now and then visual changes will be made that will require the snapshots t
3. **Download the reports, and save them into the `reports/baseline` folder.**
Change the names of the PNG/PDF files to overwrite the stored baselines.

The next time functional tests run, the generated reports will be compared to the latest image that you have saved :bowtie:
The next time functional tests run, the generated reports will be compared to the latest image that you have saved :bowtie:
4 changes: 2 additions & 2 deletions x-pack/test/functional/apps/grok_debugger/grok_debugger.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export default function ({ getService, getPageObjects }) {

const PageObjects = getPageObjects(['grokDebugger']);
// FLAKY: https://github.com/elastic/kibana/issues/84440
describe.skip('grok debugger app', function () {
describe('grok debugger app', function () {
this.tags('includeFirefox');
before(async () => {
await esArchiver.load('empty_kibana');
Expand Down Expand Up @@ -50,7 +50,7 @@ export default function ({ getService, getPageObjects }) {
});

describe('syntax highlighting', () => {
it.skip('applies the correct CSS classes', async () => {
it('applies the correct CSS classes', async () => {
const grokPattern = '\\[(?:-|%{NUMBER:bytes:int})\\]';

await grokDebugger.setPatternInput(grokPattern);
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/infra/home_page.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const supertest = getService('supertest');

// FLAKY: https://github.com/elastic/kibana/issues/75724
describe.skip('Home page', function () {
describe('Home page', function () {
this.tags('includeFirefox');
before(async () => {
await esArchiver.load('empty_kibana');
Expand Down
6 changes: 3 additions & 3 deletions x-pack/test/functional/apps/lens/chart_data.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,21 +61,21 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {

// Partition chart tests have to be skipped until
// https://github.com/elastic/elastic-charts/issues/917 gets fixed
it.skip('should render pie chart', async () => {
it('should render pie chart', async () => {
await PageObjects.lens.switchToVisualization('pie');
await PageObjects.lens.waitForVisualization();
const data = await PageObjects.lens.getCurrentChartDebugState();
assertMatchesExpectedData(data!);
});

it.skip('should render donut chart', async () => {
it('should render donut chart', async () => {
await PageObjects.lens.switchToVisualization('donut');
await PageObjects.lens.waitForVisualization();
const data = await PageObjects.lens.getCurrentChartDebugState();
assertMatchesExpectedData(data!);
});

it.skip('should render treemap chart', async () => {
it('should render treemap chart', async () => {
await PageObjects.lens.switchToVisualization('treemap');
await PageObjects.lens.waitForVisualization();
const data = await PageObjects.lens.getCurrentChartDebugState();
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/maps/feature_controls/maps_spaces.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
const appsMenu = getService('appsMenu');

// FLAKY: https://github.com/elastic/kibana/issues/38414
describe.skip('spaces feature controls', () => {
describe('spaces feature controls', () => {
before(async () => {
PageObjects.maps.setBasePath('/s/custom_space');
});
Expand Down
4 changes: 2 additions & 2 deletions x-pack/test/functional/apps/maps/layer_errors.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import expect from '@kbn/expect';
export default function ({ getPageObjects }) {
const PageObjects = getPageObjects(['maps', 'header']);

describe.skip('layer errors', () => {
describe('layer errors', () => {
before(async () => {
await PageObjects.maps.loadSavedMap('layer with errors');
});
Expand Down Expand Up @@ -90,7 +90,7 @@ export default function ({ getPageObjects }) {
// Flaky test on cloud and windows when run against a snapshot build of 7.11.
// https://github.com/elastic/kibana/issues/91043

it.skip('should diplay error message in layer panel', async () => {
it('should diplay error message in layer panel', async () => {
const errorMsg = await PageObjects.maps.getLayerErrorText(LAYER_NAME);
expect(errorMsg).to.equal(
`Unable to find EMS tile configuration for id: ${MISSING_EMS_ID}. Kibana is unable to access Elastic Maps Service. Contact your system administrator.`
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/ml/data_frame_analytics/feature_importance.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export default function ({ getService }: FtrProviderContext) {
const ml = getService('ml');

// FLAKY: https://github.com/elastic/kibana/issues/93188
describe.skip('total feature importance panel and decision path popover', function () {
describe('total feature importance panel and decision path popover', function () {
Comment on lines 17 to +18
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As you can see from the issues linked in the comment, this has just recently been skipped and we're investigating how to stabilize it. It's failing rather rarely but it's definitely flaky, so chances are that it would pass your PR checks but fail later after re-activation. Please don't remove this skip for now.

const testDataList: Array<{
suiteTitle: string;
archive: string;
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/monitoring/elasticsearch/indices.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ export default function ({ getService, getPageObjects }) {
});

// Revisit once https://github.com/elastic/eui/issues/1322 is resolved
it.skip('should show indices table with correct rows after sorting by Search Rate Desc', async () => {
it('should show indices table with correct rows after sorting by Search Rate Desc', async () => {
await indicesList.clickSearchCol();
await indicesList.clickSearchCol();

Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/monitoring/elasticsearch/shards.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export default function ({ getService, getPageObjects }) {
const retry = getService('retry');

// FLAKY: https://github.com/elastic/kibana/issues/47184
describe.skip('Elasticsearch shard legends', () => {
describe('Elasticsearch shard legends', () => {
const { setup, tearDown } = getLifecycleMethods(getService, getPageObjects);

before(async () => {
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/monitoring/setup/metricbeat_migration.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export default function ({ getService, getPageObjects }) {
const PageObjects = getPageObjects(['common', 'console']);

// FLAKY: https://github.com/elastic/kibana/issues/74327
describe.skip('Setup mode metricbeat migration', function () {
describe('Setup mode metricbeat migration', function () {
describe('setup mode btn', () => {
const { setup, tearDown } = getLifecycleMethods(getService, getPageObjects);

Expand Down
4 changes: 2 additions & 2 deletions x-pack/test/functional/apps/monitoring/time_filter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,15 @@ export default function ({ getService, getPageObjects }) {
});

// FLAKY: https://github.com/elastic/kibana/issues/48910
it.skip('should send another request when clicking Refresh', async () => {
it('should send another request when clicking Refresh', async () => {
await testSubjects.click('querySubmitButton');
const isLoading = await PageObjects.header.isGlobalLoadingIndicatorVisible();
expect(isLoading).to.be(true);
});

// TODO: [cr] I'm not sure this test is any better than the above one, we might need to rely solely on unit tests
// for this functionality
it.skip('should send another request when changing the time picker', async () => {
it('should send another request when changing the time picker', async () => {
await PageObjects.timePicker.setAbsoluteRange(
'Aug 15, 2016 @ 21:00:00.000',
'Aug 16, 2016 @ 00:00:00.000'
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/status_page/status_page.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export default function statusPageFunctonalTests({
const PageObjects = getPageObjects(['security', 'statusPage', 'home']);

// FLAKY: https://github.com/elastic/kibana/issues/50448
describe.skip('Status Page', function () {
describe('Status Page', function () {
this.tags(['skipCloud', 'includeFirefox']);
before(async () => await esArchiver.load('empty_kibana'));
after(async () => await esArchiver.unload('empty_kibana'));
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/upgrade_assistant/upgrade_assistant.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export default function upgradeAssistantFunctionalTests({
const retry = getService('retry');

// Failing: See https://github.com/elastic/kibana/issues/86546
describe.skip('Upgrade Checkup', function () {
describe('Upgrade Checkup', function () {
this.tags('includeFirefox');

before(async () => {
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/uptime/locations.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
};

// FLAKY: https://github.com/elastic/kibana/issues/85208
describe.skip('Observer location', () => {
describe('Observer location', () => {
const start = '~ 15 minutes ago';
const end = 'now';

Expand Down