test: reenable ignored tests, reduce flakiness, run modified modules …

…on PR (#349)

Related-to: vaadin/vaadin-grid-flow#635 vaadin/vaadin-grid-flow#122 vaadin/flow#4268

pom.xml

@@ -350,6 +350,7 @@
           </executions>
           <configuration>
             <forkCount>${failsafe.forkCount}</forkCount>
+            <argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
             <reuseForks>true</reuseForks>
             <trimStackTrace>false</trimStackTrace>
             <systemPropertyVariables>

scripts/build.sh

@@ -10,8 +10,10 @@ then
         FORK_COUNT=`echo $i | cut -d = -f2`;;
       parallel=*)
         TESTS_IN_PARALLEL=`echo $i | cut -d = -f2`;;
-	  testMode=*)
-	    TEST_MODE=`echo $i | cut -d = -f2`;;
+      pr=*)
+        PR=`echo $i | cut -d = -f2`;;
+      testMode=*)
+	      TEST_MODE=`echo $i | cut -d = -f2`;;
       *)
         modules=vaadin-$i-flow-parent/vaadin-$i-flow-integration-tests,vaadin-$i-flow-parent/vaadin-$i-flow-integration-tests/pom-bower-mode.xml,$modules
         elements="$elements $i"
@@ -20,6 +22,21 @@ then
   done
 fi
 
+## compute modules that were modified in this PR
+if [ -z "$modules" -a -n "$PR" ]
+then
+  modified=`curl -s https://api.github.com/repos/vaadin/vaadin-flow-components/pulls/$PR/files \
+    | jq -r '.[] | .filename' | grep 'vaadin.*parent' | perl -pe 's,^vaadin-(.*)-flow-parent.*,$1,g' | sort -u`
+  if [ `echo "$modified" | wc -w` -lt 5 ]
+  then
+    for i in $modified
+    do
+      modules=vaadin-$i-flow-parent/vaadin-$i-flow-integration-tests,$modules
+      elements="$elements $i"
+    done
+  fi
+fi
+
 tcMsg() (
   echo "##teamcity[$1]"
 )
@@ -39,12 +56,12 @@ tcStatus() {
 
 saveFailedTests() {
   try=$1
-  testFolder=$2
-  failed=`egrep '<<< ERROR|<<< FAILURE' integration-tests/target/failsafe-reports/*txt | perl -pe 's,.*/(.*).txt:.*,$1,g' | sort -u`
+  failedMethods=`egrep '<<< ERROR!$|<<< FAILURE!$' integration-tests/target/failsafe-reports/*txt | perl -pe 's,.*:(.*)\((.*)\).*,$2.$1,g' | sort -u`
+  failed=`egrep '<<< ERROR!$|<<< FAILURE!$' integration-tests/target/failsafe-reports/*txt | perl -pe 's,.*:(.*)\((.*)\).*,$2,g' | sort -u`
   nfailed=`echo "$failed" | wc -w`
   ### collect tests numbers for TC status
-  ncompleted=`grep -Poh 'Tests run:\K[^,]*' integration-tests/target/failsafe-reports/*txt | awk '{SUM+=$1} END { print SUM }'`
-  nskipped=`grep -Poh 'Skipped:\K[^,]*' integration-tests/target/failsafe-reports/*txt | awk '{SUM+=$1} END { print SUM }'` 
+  ncompleted=`grep 'Tests run: ' vaadin*/*flow/target/surefire-reports/*.txt integration-tests/target/failsafe-reports/*txt | awk '{SUM+=$3} END { print SUM }'`
+  nskipped=`grep 'Tests run: ' vaadin*/*flow/target/surefire-reports/*.txt integration-tests/target/failsafe-reports/*txt | awk '{SUM+=$9} END { print SUM }'`
   if [ "$nfailed" -ge 1 ]
   then
     mkdir -p integration-tests/error-screenshots/$testFolder/$try
@@ -80,16 +97,23 @@ type npm && npm --version
 type pnpm && pnpm --version
 uname -a
 
+## Compile all java files including tests in ITs modules
+cmd="mvn clean test-compile -DskipFrontend -B -q $args"
+tcLog "Compiling flow components - $cmd"
+$cmd || tcStatus 1 "Compilation failed"
+
+## Notify TC that we are going to run maven tests
 tcLog "Running report watcher for Tests "
 tcMsg "importData type='surefire' path='**/*-reports/TEST*xml'";
 
-cmd="mvn install -Drelease -B -q -T $FORK_COUNT"
+## Compile and install all modules excluding ITs
+cmd="mvn install -Drelease -B -q -T $FORK_COUNT $args"
 tcLog "Unit-Testing and Installing flow components - $cmd"
 $cmd
 if [ $? != 0 ]
 then
   tcLog "Unit-Testing and Installing flow components (2nd try) - $cmd"
-  sleep 30
+  sleep 15
   $cmd || tcStatus 1 "Unit-Testing failed"
 fi
 
@@ -112,7 +136,7 @@ $cmd || tcStatus 1 "Merging ITs failed"
 [ -n "$TBHUB" ] && args="$args -Dtest.use.hub=true -Dcom.vaadin.testbench.Parameters.hubHostname=$TBHUB"
 if [ -n "$SAUCE_USER" ]
 then
-   test -n  "$SAUCE_ACCESS_KEY" || { echo "\$SAUCE_ACCESS_KEY needs to be defined to use Saucelabs" >&2 ; exit 1; }
+   test -n "$SAUCE_ACCESS_KEY" || { echo "\$SAUCE_ACCESS_KEY needs to be defined to use Saucelabs" >&2 ; exit 1; }
    args="$args -P saucelabs -Dtest.use.hub=true -Dsauce.user=$SAUCE_USER -Dsauce.sauceAccessKey=$SAUCE_ACCESS_KEY"
 fi
 echo "$args"
@@ -148,15 +172,15 @@ fi
 if [ -n "$modules" ] && [ -z "$USE_MERGED_MODULE" ]
 then
   ### Run IT's in original modules
-  cmd="mvn clean verify -Dfailsafe.forkCount=$FORK_COUNT $args -pl $modules $(reuse_browser $TESTBENCH_REUSE_BROWSER)"
-  tcLog "Running module ITs - mvn clean verify -pl ..."
+  cmd="mvn clean verify -Dfailsafe.forkCount=$FORK_COUNT $args -pl $modules -Dtest=none $(reuse_browser $TESTBENCH_REUSE_BROWSER)"
+  tcLog "Running module ITs ($elements) - mvn clean verify -pl ..."
   echo $cmd
   $cmd
 else
   mode="-Dfailsafe.forkCount=$FORK_COUNT -Dcom.vaadin.testbench.Parameters.testsInParallel=$TESTS_IN_PARALLEL"
   ### Run IT's in merged module
-  cmd="mvn verify -B -q -D$testMode -Drelease -Dvaadin.productionMode -Dfailsafe.rerunFailingTestsCount=2 $mode $args -pl integration-tests/$pomFile $(reuse_browser $TESTBENCH_REUSE_BROWSER)"
-  tcLog "Running merged ITs under $testMode - mvn verify -B -D$testMode -Drelease -pl integration-tests/$pomFile ..."
+  cmd="mvn verify -B -q -Drun-it -Drelease -Dvaadin.productionMode -Dfailsafe.rerunFailingTestsCount=2 $mode $args -pl integration-tests/$pomFile -Dtest=none $(reuse_browser $TESTBENCH_REUSE_BROWSER)"
+  tcLog "Running merged ITs - mvn verify -B -Drun-it -Drelease -pl integration-tests/$pomFile ..."
   echo $cmd
   $cmd
   error=$?
@@ -166,22 +190,26 @@ else
 
   if [ "$nfailed" -gt 0 ]
   then
-    tcLog "There were $nfailed Failed Tests: "
-    echo "$failed"
-    rerunFailed=$nfailed
-
-    if [ "$nfailed" -le 15 ]
-    then
-      failed=`echo "$failed" | tr '\n' ','`
-      mode="-Dfailsafe.forkCount=2 -Dcom.vaadin.testbench.Parameters.testsInParallel=3"
-      cmd="mvn verify -B -q -D$testMode -Drelease -Dvaadin.productionMode -DskipFrontend $mode $args -pl integration-tests/$pomFile -Dit.test=$failed $(reuse_browser false)"
-      tcLog "Re-Running $nfailed failed tests($testMode) ..."
-      echo $cmd
-      $cmd
-      error=$?
-      saveFailedTests run-2 $testMode
-      tcStatus $error "Test failed: $nfailed" "($testMode)Tests passed: $ncompleted, ignored: $nskipped (there were $rerunFailed tests failing on the 1st run, but passed on the 2nd try.)"
-    fi
+      ## Give a second try to failed tests
+      tcLog "There were $nfailed failed IT classes in first round."
+      echo "$failedMethods"
+
+      rerunFailed=$nfailed
+      if [ "$nfailed" -le 15 ]
+      then
+        failed=`echo "$failed" | tr '\n' ','`
+        mode="-Dfailsafe.forkCount=2 -Dcom.vaadin.testbench.Parameters.testsInParallel=3"
+        cmd="mvn verify -B -q -Drun-it -Drelease -Dvaadin.productionMode -DskipFrontend $mode $args -pl integration-tests/$pomFile -Dtest=none -Dit.test=$failed $(reuse_browser false)"
+        tcLog "Re-Running $nfailed failed IT classes ..."
+        echo $cmd
+        $cmd
+        error=$?
+        tcLog "Re-Run exited with code $error"
+        saveFailedTests run-2
+        tcStatus $error "(IT2)Test failed: $nfailed" "(IT2)Tests passed: $ncompleted ($rerunFailed retried, $nfailed failed), ignored: $nskipped"
+      else
+        tcStatus $error "(IT1)Test failed: $nfailed" "(IT1)Tests passed: $ncompleted (more than 15 failed), ignored: $nskipped"
+      fi
   fi
   exit $error
 fi

vaadin-charts-flow-parent/vaadin-charts-flow-integration-tests/src/test/java/com/vaadin/flow/component/charts/tests/ServerSideEventsIT.java

@@ -225,9 +225,9 @@ private void assertNthHistoryEventIsType(
     }
 
     private void resetHistory() {
+        waitUntil(e -> $(ButtonElement.class).exists());
         WebElement resetHistoryButton = $(ButtonElement.class).id("resetHistory");
         resetHistoryButton.click();
-
     }
 
     private SeriesCheckboxClickEvent readCheckboxEventDetails() {

vaadin-context-menu-flow-parent/vaadin-context-menu-flow-integration-tests/src/test/java/com/vaadin/flow/component/contextmenu/it/SubMenuIT.java

@@ -20,7 +20,6 @@
 
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.openqa.selenium.By;
 import org.openqa.selenium.WebElement;
@@ -58,7 +57,6 @@ public void addItemToSubMenu_subMenuRendered_clickListenerWorks() {
     }
 
     @Test
-    @Ignore("Flaky test - issue #67")
     public void openAndCloseSubMenu_addContent_contentUpdatedAndFunctional() {
         rightClickOn("target");
         openSubMenu(getMenuItems().get(0));
@@ -86,7 +84,6 @@ public void openAndCloseSubMenu_addContent_contentUpdatedAndFunctional() {
     }
 
     @Test
-    @Ignore("Flaky test - issue #67")
     public void openAndCloseSubMenu_addSubSubMenu_contentUpdatedAndFunctional() {
         rightClickOn("target");
         openSubMenu(getMenuItems().get(0));

vaadin-crud-flow-parent/vaadin-crud-flow-integration-tests/src/test/java/com/vaadin/flow/component/crud/test/BasicUseIT.java

@@ -1,16 +1,16 @@
 package com.vaadin.flow.component.crud.test;
 
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
 import com.vaadin.flow.component.button.testbench.ButtonElement;
 import com.vaadin.flow.component.crud.testbench.CrudElement;
 import com.vaadin.flow.component.grid.testbench.GridElement;
 import com.vaadin.flow.component.textfield.testbench.TextFieldElement;
 import com.vaadin.testbench.TestBenchElement;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.List;
 
 public class BasicUseIT extends AbstractParallelTest {
 
@@ -59,20 +59,20 @@ public void filterValueCorrect() {
     }
 
     @Test
-    @Ignore("Unable to access the sorting controls in Grid")
     public void sortEnabledInGrid() {
         GridElement grid = $(GridElement.class).waitForFirst();
-        Assert.assertTrue(grid.getHeaderCell(1).$("vaadin-grid-sorter").exists());
+        Assert.assertTrue(grid.getHeaderCellContent(0, 0).$("vaadin-grid-sorter").exists());
     }
 
     @Test
-    @Ignore("Unable to access the sorting controls in Grid")
     public void sortOrdersCorrect() {
         GridElement grid = $(GridElement.class).waitForFirst();
-        List<TestBenchElement> sorters = grid.getHeaderCell(1).$("vaadin-grid-sorter").all();
 
-        sorters.get(0).click(); // First name ascending
-        sorters.get(2).click(); sorters.get(2).click(); // Last name descending
+        TestBenchElement firstNameSorter = grid.getHeaderCellContent(0, 0).$("vaadin-grid-sorter").get(0);
+        TestBenchElement lasttNameSorter = grid.getHeaderCellContent(0, 2).$("vaadin-grid-sorter").get(0);
+
+        firstNameSorter.click(); // First name ascending
+        lasttNameSorter.click(); lasttNameSorter.click(); // Last name descending
 
         ButtonElement showFilterButton = getTestButton("showFilter");
         showFilterButton.click();

vaadin-custom-field-flow-parent/vaadin-custom-field-flow-integration-tests/src/test/java/com/vaadin/flow/component/customfield/test/ErrorIT.java

@@ -1,11 +1,11 @@
 package com.vaadin.flow.component.customfield.test;
 
-import com.vaadin.flow.component.customfield.testbench.CustomFieldElement;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
+import com.vaadin.flow.component.customfield.testbench.CustomFieldElement;
+
 public class ErrorIT extends AbstractParallelTest {
     @Before
     public void init() {
@@ -14,7 +14,6 @@ public void init() {
     }
 
     @Test
-    @Ignore
     public void checkProperties() {
         final CustomFieldElement customField = $(CustomFieldElement.class)
             .waitForFirst();

vaadin-flow-components-shared/src/main/java/com/vaadin/tests/TabbedComponentDemoTest.java

@@ -1,9 +1,10 @@
 package com.vaadin.tests;
 
+import java.util.List;
+
 import org.junit.AfterClass;
 import org.openqa.selenium.remote.DesiredCapabilities;
-
-import java.util.List;
+import org.openqa.selenium.support.ui.ExpectedCondition;
 
 public abstract class TabbedComponentDemoTest extends com.vaadin.flow.demo.TabbedComponentDemoTest {
 
@@ -27,4 +28,9 @@ protected List<DesiredCapabilities> getHubBrowsersToTest() {
     public static void runAfterTest() {
         browser.clear();
     }
+
+    @Override
+    protected <T> T waitUntil(ExpectedCondition<T> condition) {
+        return super.waitUntil(condition, 120);
+    }
 }

vaadin-form-layout-flow-parent/vaadin-form-layout-flow-demo/src/main/java/com/vaadin/flow/component/formlayout/demo/FormLayoutView.java

@@ -178,9 +178,9 @@ private void createResponsiveLayout() {
         nameLayout.add(titleField, firstNameField, lastNameField);
 
         nameLayout.setResponsiveSteps(
-                new ResponsiveStep("25em", 1),
-                new ResponsiveStep("32em", 2),
-                new ResponsiveStep("40em", 3));
+                new ResponsiveStep("1px", 1),
+                new ResponsiveStep("600px", 2),
+                new ResponsiveStep("700px", 3));
         add(nameLayout);
         // end-source-example
         // @formatter:on

vaadin-form-layout-flow-parent/vaadin-form-layout-flow-integration-tests/src/test/java/com/vaadin/flow/component/formlayout/tests/FormLayoutIT.java

@@ -16,10 +16,8 @@
 package com.vaadin.flow.component.formlayout.tests;
 
 import java.util.List;
-
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.openqa.selenium.By;
 import org.openqa.selenium.Dimension;
@@ -39,7 +37,6 @@ public void init() {
     }
 
     @Test
-    @Ignore
     /*
      * The test works locally but fails on TC. Disabling it for now.
      *
@@ -52,37 +49,32 @@ public void custom_responsive_layouting() {
         List<WebElement> textFields = firstLayout
                 .findElements(By.tagName("vaadin-text-field"));
         Assert.assertEquals(3, textFields.size());
-
+
+        // 3 columns, all should be horizontally aligned (tolerance of some pixels)
         getDriver().manage().window().setSize(new Dimension(1000, 1000));
+        int y2 = textFields.get(2).getLocation().getY();
+        int y1 = textFields.get(1).getLocation().getY();
+        Assert.assertTrue("All 3 columns should be horizontally aligned y1="
+                + y1 + " y2=" + y2, Math.abs(y2 - y1) < 2);
 
-        // 3 columns, all should be horizontally aligned (tolerance of 2 pixels
-        // given)
-        Assert.assertTrue("All 3 columns should be horizontally aligned",
-                Math.abs(textFields.get(2).getLocation().getY()
-                        - textFields.get(1).getLocation().getY()) < 2);
-        Assert.assertTrue(Math.abs(textFields.get(1).getLocation().getY()
-                - textFields.get(0).getLocation().getY()) < 2);
 
-        getDriver().manage().window().setSize(new Dimension(380, 620));
+        // window resized, should be in 2 column mode, two below one
+        getDriver().manage().window().setSize(new Dimension(620, 620));
 
-        // window resized, should be in 2 column mode, last textfield below
-        // other two
+        y2 = textFields.get(2).getLocation().getY();
+        y1 = textFields.get(1).getLocation().getY();
         Assert.assertTrue(
-                "Layout should be in 2 column mode, last field should be below the first two",
-                textFields.get(2).getLocation().getY() > textFields.get(1)
-                        .getLocation().getY());
-        Assert.assertTrue(textFields.get(2).getLocation().getY() > textFields
-                .get(0).getLocation().getY());
+                "Layout should be in 2 column mode, last field should be below the first two y1="
+                        + y1 + " y2=" + y2, y2 > y1 + 2);
 
-        getDriver().manage().window().setSize(new Dimension(300, 620));
 
         // resized to 1 column mode, fields should be arranged below one another
+        getDriver().manage().window().setSize(new Dimension(100, 620));
+        y1 = textFields.get(1).getLocation().getY();
+        int y0 = textFields.get(0).getLocation().getY();
         Assert.assertTrue(
-                "Layout should be in 1 column mode, all fields should be below one another",
-                textFields.get(2).getLocation().getY() > textFields.get(1)
-                        .getLocation().getY());
-        Assert.assertTrue(textFields.get(1).getLocation().getY() > textFields
-                .get(0).getLocation().getY());
+                "Layout should be in 1 column mode, all fields should be below one another y0="
+                        + y0 + " y1=" + y1, y1 > y0);
     }
 
     @Test

vaadin-grid-flow-parent/vaadin-grid-flow-integration-tests/src/main/java/com/vaadin/flow/component/grid/it/GridSingleSelectionPage.java

@@ -108,9 +108,11 @@ private Grid<String> setDeselectAllowedAndSetItems() {
                 ITEMS_GRID);
         Button text = new Button();
         grid.addSelectionListener( e -> {
-            text.setId("item"+ e.getFirstSelectedItem().get());
-            text.setText("The row "+ e.getFirstSelectedItem().get() + " is selected");
-            add(text);
+            if (e.getFirstSelectedItem().isPresent()) {
+                text.setId("item"+ e.getFirstSelectedItem().get());
+                text.setText("The row "+ e.getFirstSelectedItem().get() + " is selected");
+                add(text);
+            }
         });
         return grid;
     }