From ea868c44f6cdd55012f58ae2086ffec80afdd617 Mon Sep 17 00:00:00 2001 From: Sumedha Pramod Date: Thu, 26 Apr 2018 13:31:56 -0400 Subject: [PATCH] Fix: Functional tests + cleanup annotations after Features (#186) * Fix: Functional tests + cleanup annotations after Features - Added jsdocs for methods * Chore: functional test running tweaks * Chore: Remove unstable reply functional tests - Remove mobile tests until separate UI tests are setup --- .eslintrc | 2 +- .travis.yml | 22 +---- codecept.conf.js | 17 +++- functional-tests/helpers/actions.js | 28 ++++-- functional-tests/helpers/cleanup.js | 6 +- functional-tests/helpers/validation.js | 65 +++++++++++--- functional-tests/run-all.js | 96 +++++++++++++++++++++ functional-tests/tests/draw.js | 5 ++ functional-tests/tests/highlight_comment.js | 21 ++--- functional-tests/tests/plain_highlight.js | 9 +- functional-tests/tests/point.js | 31 +++---- 11 files changed, 225 insertions(+), 77 deletions(-) create mode 100755 functional-tests/run-all.js diff --git a/.eslintrc b/.eslintrc index 803a34fbb..613cb8540 100644 --- a/.eslintrc +++ b/.eslintrc @@ -26,7 +26,7 @@ "Assert": false, "Feature": false, "Before": false, - "BeforeSuite": false, + "After": false, "Scenario": false }, "rules": { diff --git a/.travis.yml b/.travis.yml index 58dfa8334..bc4a781b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,16 +15,13 @@ aliases: - sleep 5 addons: sauce_connect: true - script: yarn run functional-tests-ci - - &sauce-labs-mobile - <<: *sauce-labs - script: yarn run functional-tests-ci --grep @mobile + script: travis_retry yarn run functional-tests-ci - &sauce-labs-desktop <<: *sauce-labs - script: yarn run functional-tests-ci --grep @desktop + script: travis_retry yarn run functional-tests-ci --grep @desktop - &sauce-labs-ie <<: *sauce-labs - script: yarn run functional-tests-ci --grep @ie + script: travis_retry yarn run functional-tests-ci --grep @ie jobs: include: - script: yarn run ci @@ -42,15 +39,4 @@ jobs: env: BROWSER_PLATFORM="Windows 10" BROWSER_NAME="MicrosoftEdge" FILE_ID="285567976309" FILE_VERSION_ID="300496707445" # Windows IE - <<: *sauce-labs-ie - env: BROWSER_PLATFORM="Windows 10" BROWSER_NAME="internet explorer" FILE_ID="285568624824" FILE_VERSION_ID="300497342136" - # iPhone - - <<: *sauce-labs-mobile - env: BROWSER_PLATFORM="iOS" DEVICE_NAME="iPhone 6 Simulator" PLATFORM_VERSION="11.2" BROWSER_NAME="Safari" FILE_ID="285569765346" FILE_VERSION_ID="300498497346" - # iPad - - <<: *sauce-labs-mobile - # Uses firefox file id - env: BROWSER_PLATFORM="iOS" DEVICE_NAME="iPad Simulator" PLATFORM_VERSION="11.2" BROWSER_NAME="Safari" FILE_ID="285568802145" FILE_VERSION_ID="300497533713" - # Android - - <<: *sauce-labs-mobile - # Uses chrome file id - env: BROWSER_PLATFORM="Android" DEVICE_NAME="Android GoogleAPI Emulator" PLATFORM_VERSION="7.1" BROWSER_NAME="Chrome" FILE_ID="285567874839" FILE_VERSION_ID="300496591287" + env: BROWSER_PLATFORM="Windows 10" BROWSER_NAME="internet explorer" FILE_ID="285568624824" FILE_VERSION_ID="300497342136" \ No newline at end of file diff --git a/codecept.conf.js b/codecept.conf.js index 8e8a44237..fce5810f0 100644 --- a/codecept.conf.js +++ b/codecept.conf.js @@ -8,9 +8,11 @@ const { BROWSER_PLATFORM, PLATFORM_VERSION, DEVICE_NAME, - DEFAULT_WAIT_TIME = 90000 + DEFAULT_WAIT_TIME = 90000, + RUN_LOCALLY=false } = process.env; const MOBILE_PLATFORMS = ['iOS', 'Android']; +const { cleanupAnnotations } = require('./functional-tests/helpers/cleanup'); // Local selenium config const commonConfigObj = { @@ -21,7 +23,7 @@ const commonConfigObj = { }; const helperObj = {}; -const isLocalBuild = typeof SAUCE_USERNAME === 'undefined'; +const isLocalBuild = typeof SAUCE_USERNAME === 'undefined' || RUN_LOCALLY; if (isLocalBuild) { helperObj.WebDriverIO = commonConfigObj; @@ -61,14 +63,21 @@ if (isLocalBuild) { } } +/** + * @return {void} +*/ +function cleanup() { + cleanupAnnotations() ; +} + exports.config = { tests: './functional-tests/tests/*.js', timeout: DEFAULT_WAIT_TIME, output: './functional-tests/output', helpers: helperObj, include: {}, - bootstrap: './functional-tests/helpers/cleanup.js', - teardown: './functional-tests/helpers/cleanup.js', + bootstrap: cleanup, + teardown: cleanup, mocha: {}, name: 'box-annotations', hooks: isLocalBuild ? [] : ['./functional-tests/helpers/eventHooks.js'] diff --git a/functional-tests/helpers/actions.js b/functional-tests/helpers/actions.js index 38048c37d..11bbc71f4 100644 --- a/functional-tests/helpers/actions.js +++ b/functional-tests/helpers/actions.js @@ -10,7 +10,15 @@ const { } = require('../helpers/constants'); const { validateReply, validateDeleteConfirmation } = require('./validation'); -function replyToThread(I) { +/** + * Replies to an annotation thread + * + * @param {Object} I - the codeceptjs I + * @param {string} selector - annotation CSS selector + * + * @return {void} + */ +function replyToThread(I, selector) { I.say('Reply to highlight comment annotation'); I.fillField(SELECTOR_REPLY_TEXTAREA, 'Sample reply'); I.click(`${SELECTOR_REPLY_CONTAINER} ${SELECTOR_ANNOTATION_BUTTON_POST}`); @@ -20,15 +28,25 @@ function replyToThread(I) { I.say('Cancel a reply to a highlight comment annotation'); I.fillField(SELECTOR_REPLY_TEXTAREA, 'Sample canceled reply'); I.click(`${SELECTOR_REPLY_CONTAINER} ${SELECTOR_ANNOTATION_BUTTON_CANCEL}`); - I.waitNumberOfVisibleElements(SELECTOR_ANNOTATION_COMMENT, 2); + I.waitNumberOfVisibleElements(SELECTOR_ANNOTATION_COMMENT, 1); } -function deleteAnnotation(I, annotationCount) { +/** + * Replies to an annotation thread + * + * @param {Object} I - the codeceptjs I + * @param {number} annotationCount - current number of annotations in threads + * @param {string} selector - the selector to use, defaults to .annotation-comment + * + * @return {void} + */ +function deleteAnnotation(I, annotationCount, selector = SELECTOR_ANNOTATION_COMMENT) { I.waitNumberOfVisibleElements(SELECTOR_ANNOTATION_COMMENT, annotationCount); I.say('Delete the annotation'); - I.click(SELECTOR_DELETE_COMMENT_BTN); - validateDeleteConfirmation(I); + I.waitForEnabled(`${selector} ${SELECTOR_DELETE_COMMENT_BTN}`, 9); + I.click(`${selector} ${SELECTOR_DELETE_COMMENT_BTN}`); + validateDeleteConfirmation(I, selector); I.say('Annotation should be deleted'); if (annotationCount > 1) { diff --git a/functional-tests/helpers/cleanup.js b/functional-tests/helpers/cleanup.js index 975edeb83..3a92e86e4 100644 --- a/functional-tests/helpers/cleanup.js +++ b/functional-tests/helpers/cleanup.js @@ -25,7 +25,7 @@ function deleteAnnotation(annotation) { }); } -module.exports = function() { +function cleanupAnnotations() { client.get(`/files/${FILE_ID}/annotations?version=${FILE_VERSION_ID}`, {}, function(err, response) { if (err) { // handle error @@ -41,4 +41,6 @@ module.exports = function() { console.log(`Deleting ${entries.length} annotations`); entries.forEach(deleteAnnotation); }); -} \ No newline at end of file +} + +exports.cleanupAnnotations = cleanupAnnotations; \ No newline at end of file diff --git a/functional-tests/helpers/validation.js b/functional-tests/helpers/validation.js index d11413662..90f571282 100644 --- a/functional-tests/helpers/validation.js +++ b/functional-tests/helpers/validation.js @@ -18,12 +18,32 @@ const { } = require('../helpers/constants'); const { expect } = require('chai'); + +/** + * Ensures the SVG icon is of the expected color + * + * @param {Object} I - the codeceptjs I + * @param {string} selector - the selector to use + * @param {string} color - rgb icon color + * + * @return {void} + */ async function validateIconColor(I, selector, color) { I.waitForElement(selector); const clr = await I.grabCssPropertyFrom(`${selector} svg`, 'fill'); expect(clr).to.equal(color); } + +/** + * Validates that the text area appears as expected + * + * @param {Object} I - the codeceptjs I + * @param {string} containerSel - the container selector to use + * @param {string} textareaSel - the textarea selector to use + * + * @return {void} + */ function* validateTextarea(I, containerSel, textareaSel) { I.say(`Validate ${containerSel} ${textareaSel}`); I.waitForVisible(`${containerSel} ${textareaSel}${SELECTOR_ACTIVE}`); @@ -39,19 +59,31 @@ function* validateTextarea(I, containerSel, textareaSel) { I.waitForVisible(SELECTOR_ANNOTATION_DIALOG); } +/** + * Validates that the annotation appears as expected + * + * @param {Object} I - the codeceptjs I + * + * @return {void} + */ function validateAnnotation(I) { I.say('Dialog should contain new annotation'); I.waitForVisible(SELECTOR_ANNOTATION_DIALOG); - I.see('Posting...', SELECTOR_USER_NAME); I.waitForVisible(SELECTOR_ANNOTATION_CONTAINER); I.waitNumberOfVisibleElements(SELECTOR_ANNOTATION_COMMENT, 1); I.waitForEnabled(SELECTOR_DELETE_COMMENT_BTN); I.waitForVisible(SELECTOR_PROFILE_IMG_CONTAINER); - I.waitForText('Kanye West', 10, SELECTOR_USER_NAME); - validateTextarea(I, SELECTOR_REPLY_CONTAINER, SELECTOR_REPLY_TEXTAREA); + I.waitForText('Kanye West', 15, `${SELECTOR_ANNOTATION_COMMENT} ${SELECTOR_USER_NAME}`); } +/** + * Validates that the annotation reply appears as expected + * + * @param {Object} I - the codeceptjs I + * + * @return {void} + */ function validateReply(I) { I.say('Reply should be added to dialog'); I.waitForVisible(SELECTOR_ANNOTATION_DIALOG); @@ -64,22 +96,31 @@ function validateReply(I) { validateTextarea(I, SELECTOR_REPLY_CONTAINER, SELECTOR_REPLY_TEXTAREA); } -function validateDeleteConfirmation(I) { +/** + * Validates that the delete confirmation message appears + * and acts as expected + * + * @param {Object} I - the codeceptjs I + * @param {string} selector - the selector to use + * + * @return {void} + */ +function validateDeleteConfirmation(I, selector = '') { I.say('Validate delete confirmation'); - I.waitForText('Delete this annotation?', 10, SELECTOR_DELETE_CONFIRM_MESSAGE); - I.waitForVisible(SELECTOR_CONFIRM_DELETE_BTN); - I.waitForVisible(SELECTOR_CANCEL_DELETE_BTN); + I.waitForText('Delete this annotation?', 10, `${selector} ${SELECTOR_DELETE_CONFIRM_MESSAGE}`); + I.waitForVisible(`${selector} ${SELECTOR_CONFIRM_DELETE_BTN}`); + I.waitForVisible(`${selector} ${SELECTOR_CANCEL_DELETE_BTN}`); // Cancel annotation delete - I.click(SELECTOR_CANCEL_DELETE_BTN); - I.waitForInvisible(SELECTOR_DELETE_CONFIRM_MESSAGE); + I.click(`${selector} ${SELECTOR_CANCEL_DELETE_BTN}`); + I.waitForInvisible(`${selector} ${SELECTOR_DELETE_CONFIRM_MESSAGE}`); // Delete the annotation - I.click(SELECTOR_DELETE_COMMENT_BTN); + I.click(`${selector} ${SELECTOR_DELETE_COMMENT_BTN}`); // Delete confirmation should appear - I.waitForVisible(SELECTOR_CONFIRM_DELETE_BTN); - I.click(SELECTOR_CONFIRM_DELETE_BTN); + I.waitForVisible(`${selector} ${SELECTOR_CONFIRM_DELETE_BTN}`); + I.click(`${selector} ${SELECTOR_CONFIRM_DELETE_BTN}`); } exports.validateIconColor = validateIconColor; diff --git a/functional-tests/run-all.js b/functional-tests/run-all.js new file mode 100755 index 000000000..8f71579f3 --- /dev/null +++ b/functional-tests/run-all.js @@ -0,0 +1,96 @@ +#!/usr/bin/env node + +/* eslint-disable no-console */ +const async = require('async'); +const util = require('util'); +const colors = require('colors'); +const exec = util.promisify(require('child_process').exec); + +const { SAUCE_USERNAME, SAUCE_ACCESS_KEY, TRAVIS_JOB_NUMBER } = process.env; + +// browsers +const CHROME = 'chrome'; +const FIREFOX = 'firefox'; +const EDGE = 'MicrosoftEdge'; +const IE = 'internet explorer'; + +// platforms +const SAFARI = 'Safari'; +const WINDOWS = 'Windows 10'; +const OSX = 'macOS 10.13'; +const ios = 'iOS'; +const android = 'Android'; + +// file information +const CHROME_FILE = { + id: '285567874839', + version: '300496591287' +}; +const SAFARI_FILE = { + id: '285569765346', + version: '300498497346' +}; +const FIREFOX_FILE = { + id: '285568802145', + version: '300497533713' +}; +const EDGE_FILE = { + id: '285567976309', + version: '300496707445' +}; +const IE_FILE = { + id: '285568624824', + version: '300497342136' +}; + +const envArr = [ + `BROWSER_PLATFORM="${OSX}" BROWSER_NAME="${CHROME}" FILE_ID="${CHROME_FILE.id}" FILE_VERSION_ID="${CHROME_FILE.version}"`, + `BROWSER_PLATFORM="${OSX}" BROWSER_NAME="safari" FILE_ID="${SAFARI_FILE.id}" FILE_VERSION_ID="${SAFARI_FILE.version}"`, + `BROWSER_PLATFORM="${OSX}" BROWSER_NAME="${FIREFOX}" FILE_ID="${FIREFOX_FILE.id}" FILE_VERSION_ID="${FIREFOX_FILE.version}"`, + `BROWSER_PLATFORM="${WINDOWS}" BROWSER_NAME="${EDGE}" FILE_ID="${EDGE_FILE.id}" FILE_VERSION_ID="${EDGE_FILE.version}"` +]; + +if (!TRAVIS_JOB_NUMBER || !SAUCE_USERNAME || !TRAVIS_JOB_NUMBER) { + throw new Error('missing TRAVIS_JOB_NUMBER, SAUCE_USERNAME, or TRAVIS_JOB_NUMBER'); +} + +const processArr = []; +async.eachLimit( + envArr, + 4, + async (envStr) => { + let grepStr = ''; + + const mobileRegex = /iOS|Android/; + + if (mobileRegex.test(envStr)) { + grepStr = '--grep "@mobile"'; + } else { + grepStr = '--grep "@desktop"'; + } + + const cmd = `cd .. && CI=true SAUCE_USERNAME=${SAUCE_USERNAME} SAUCE_ACCESS_KEY=${SAUCE_ACCESS_KEY} TRAVIS_JOB_NUMBER=${TRAVIS_JOB_NUMBER} ${envStr} node ./node_modules/codeceptjs/bin/codecept.js run --steps ${grepStr}`; + + console.log('Running cmd: ', cmd); + const process = exec(cmd); + processArr.push(process); + await process; + }, + (err) => { + if (err) { + console.log(colors.red.underline(err)); + console.log(colors.red(err.stdout)); + processArr.forEach((process) => { + if (process && process.kill) { + try { + process.kill(); + } catch (err2) { + console.error(err2); + } + } + }); + throw new Error(); + } + console.log('SUCCESS!'); + } +); diff --git a/functional-tests/tests/draw.js b/functional-tests/tests/draw.js index 5aa7e4f71..0ac9acff6 100644 --- a/functional-tests/tests/draw.js +++ b/functional-tests/tests/draw.js @@ -17,6 +17,7 @@ const { } = require('../helpers/constants'); const { draw, clickAtLocation } = require('../helpers/mouseEvents'); +const { cleanupAnnotations } = require('../helpers/cleanup'); Feature('Draw Annotation Sanity'); @@ -24,6 +25,10 @@ Before(function(I) { I.amOnPage('/'); }); +After(function() { + cleanupAnnotations(); +}); + Scenario('Create/Delete drawing @desktop', function(I) { /* * Can enter/exit drawing mode properly @desktop diff --git a/functional-tests/tests/highlight_comment.js b/functional-tests/tests/highlight_comment.js index 7edddf288..681330d6c 100644 --- a/functional-tests/tests/highlight_comment.js +++ b/functional-tests/tests/highlight_comment.js @@ -10,14 +10,13 @@ const { SELECTOR_ANNOTATION_TEXTAREA, SELECTOR_ANNOTATION_BUTTON_POST, SELECTOR_ANNOTATION_BUTTON_CANCEL, - SELECTOR_ANNOTATION_DIALOG, - SELECTOR_DELETE_COMMENT_BTN, SELECTOR_ANNOTATION_COMMENT } = require('../helpers/constants'); const { selectText } = require('../helpers/mouseEvents'); const { validateTextarea, validateAnnotation } = require('../helpers/validation'); -const { replyToThread, deleteAnnotation } = require('../helpers/actions'); +const { deleteAnnotation } = require('../helpers/actions'); +const { cleanupAnnotations } = require('../helpers/cleanup'); Feature('Highlight Comment Annotation Sanity'); @@ -25,7 +24,11 @@ Before(function(I) { I.amOnPage('/'); }); -Scenario('Create/Reply/Delete a new highlight comment annotation @desktop', function(I) { +After(function() { + cleanupAnnotations(); +}); + +Scenario('Create/Delete a new highlight comment annotation @desktop', function(I) { I.waitForVisible(SELECTOR_ANNOTATIONS_LOADED); I.say('Highlight dialog should appear after selecting text'); @@ -62,19 +65,11 @@ Scenario('Create/Reply/Delete a new highlight comment annotation @desktop', func I.say('Post highlight comment annotation'); I.fillField(SELECTOR_ANNOTATION_TEXTAREA, 'Sample comment'); I.click(SELECTOR_ANNOTATION_BUTTON_POST); - validateAnnotation(I); I.waitNumberOfVisibleElements(SELECTOR_ANNOTATION_COMMENT, 1); - - replyToThread(I); + validateAnnotation(I); /* * Delete the highlight comment annotation and reply */ - I.say('Highlight dialog should appear on click'); - I.click(`${SELECTOR_TEXT_LAYER} div`); - I.waitForVisible(SELECTOR_ANNOTATION_DIALOG); - I.waitForEnabled(SELECTOR_DELETE_COMMENT_BTN); - - deleteAnnotation(I, 2); deleteAnnotation(I, 1); }); diff --git a/functional-tests/tests/plain_highlight.js b/functional-tests/tests/plain_highlight.js index d0c12fcf3..1bd4702c5 100644 --- a/functional-tests/tests/plain_highlight.js +++ b/functional-tests/tests/plain_highlight.js @@ -11,6 +11,7 @@ const { const { selectText } = require('../helpers/mouseEvents'); const { validateIconColor } = require('../helpers/validation'); +const { cleanupAnnotations } = require('../helpers/cleanup'); Feature('Plain Highlight Annotation Sanity'); @@ -18,6 +19,10 @@ Before(function(I) { I.amOnPage('/'); }); +After(function() { + cleanupAnnotations(); +}); + Scenario('Create/Delete a new plain highlight annotation @desktop', function(I) { I.waitForVisible(SELECTOR_ANNOTATIONS_LOADED); @@ -41,10 +46,6 @@ Scenario('Create/Delete a new plain highlight annotation @desktop', function(I) /* * Delete plain highlight annotation */ - I.say('Highlight dialog should appear on click'); - I.click(`${SELECTOR_TEXT_LAYER} div`); - I.waitForVisible(SELECTOR_ANNOTATION_HIGHLIGHT_DIALOG); - I.say('Delete the highlight annotation'); I.click(`${SELECTOR_ADD_HIGHLIGHT_BTN}${SELECTOR_ACTIVE}`); diff --git a/functional-tests/tests/point.js b/functional-tests/tests/point.js index 3d641353d..e64aec7ae 100644 --- a/functional-tests/tests/point.js +++ b/functional-tests/tests/point.js @@ -11,13 +11,13 @@ const { SELECTOR_ANNOTATION_TEXTAREA, SELECTOR_ANNOTATION_BUTTON_POST, SELECTOR_ANNOTATION_BUTTON_CANCEL, - SELECTOR_ANNOTATION_COMMENT, - SELECTOR_DELETE_COMMENT_BTN + SELECTOR_ANNOTATION_COMMENT } = require('../helpers/constants'); const { clickAtLocation } = require('../helpers/mouseEvents'); const { validateTextarea, validateAnnotation } = require('../helpers/validation'); -const { replyToThread, deleteAnnotation } = require('../helpers/actions'); +const { deleteAnnotation } = require('../helpers/actions'); +const { cleanupAnnotations } = require('../helpers/cleanup'); Feature('Point Annotation Sanity'); @@ -25,7 +25,11 @@ Before(function(I) { I.amOnPage('/'); }); -Scenario('Create/Reply/Delete point annotation @desktop', function(I) { +After(function() { + cleanupAnnotations(); +}); + +Scenario('Create/Delete point annotation @desktop', function(I) { I.waitForVisible(SELECTOR_ANNOTATIONS_LOADED); I.waitForVisible(SELECTOR_ANNOTATION_BUTTON_POINT); @@ -65,23 +69,14 @@ Scenario('Create/Reply/Delete point annotation @desktop', function(I) { validateAnnotation(I); I.waitNumberOfVisibleElements(SELECTOR_ANNOTATION_COMMENT, 1); - replyToThread(I); - - I.say('Exit point annotation mode'); - I.click(SELECTOR_ANNOTATION_BUTTON_POINT_EXIT); - I.dontSeeElement(SELECTOR_ANNNOTATION_MODE_BACKGROUND); - I.waitForVisible(SELECTOR_ANNOTATION_BUTTON_POINT); - /* * Delete the point annotation */ - - I.say('Point dialog should appear on click'); - I.click(SELECTOR_ANNOTATION_POINT_MARKER); - I.waitForVisible(SELECTOR_ANNOTATION_DIALOG); - I.waitForEnabled(SELECTOR_DELETE_COMMENT_BTN); - - deleteAnnotation(I, 2); deleteAnnotation(I, 1); I.waitForDetached(SELECTOR_ANNOTATION_POINT_MARKER, 1); + + I.say('Exit point annotation mode'); + I.click(SELECTOR_ANNOTATION_BUTTON_POINT_EXIT); + I.dontSeeElement(SELECTOR_ANNNOTATION_MODE_BACKGROUND); + I.waitForVisible(SELECTOR_ANNOTATION_BUTTON_POINT); }); \ No newline at end of file