test: reenable ignored tests

pom.xml

@@ -278,6 +278,7 @@
           </executions>
           <configuration>
             <forkCount>${failsafe.forkCount}</forkCount>
+            <argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
             <reuseForks>true</reuseForks>
             <trimStackTrace>false</trimStackTrace>
             <systemPropertyVariables>

scripts/build.sh

@@ -10,6 +10,8 @@ then
         FORK_COUNT=`echo $i | cut -d = -f2`;;
       parallel=*)
         TESTS_IN_PARALLEL=`echo $i | cut -d = -f2`;;
+      pr=*)
+        PR=`echo $i | cut -d = -f2`;;
       *)
         modules=vaadin-$i-flow-parent/vaadin-$i-flow-integration-tests,$modules
         elements="$elements $i"
@@ -18,6 +20,21 @@ then
   done
 fi
 
+## compute modules that were modified in this PR
+if [ -z "$modules" -a -n "$PR" ]
+then
+  modified=`curl -s https://api.github.com/repos/vaadin/vaadin-flow-components/pulls/$PR/files \
+    | jq -r '.[] | .filename' | grep 'vaadin.*parent' | perl -pe 's,^vaadin-(.*)-flow-parent.*,$1,g' | sort -u`
+  if [ `echo "$modified" | wc -w` -lt 5 ]
+  then
+    for i in $modified
+    do
+      modules=vaadin-$i-flow-parent/vaadin-$i-flow-integration-tests,$modules
+      elements="$elements $i"
+    done
+  fi
+fi
+
 tcMsg() (
   echo "##teamcity[$1]"
 )
@@ -37,11 +54,12 @@ tcStatus() {
 
 saveFailedTests() {
   try=$1
-  failed=`egrep '<<< ERROR|<<< FAILURE' integration-tests/target/failsafe-reports/*txt | perl -pe 's,.*/(.*).txt:.*,$1,g' | sort -u`
+  failedMethods=`egrep '<<< ERROR!$|<<< FAILURE!$' integration-tests/target/failsafe-reports/*txt | perl -pe 's,.*:(.*)\((.*)\).*,$2.$1,g' | sort -u`
+  failed=`egrep '<<< ERROR!$|<<< FAILURE!$' integration-tests/target/failsafe-reports/*txt | perl -pe 's,.*:(.*)\((.*)\).*,$2,g' | sort -u`
   nfailed=`echo "$failed" | wc -w`
   ### collect tests numbers for TC status
-  ncompleted=`grep -Poh 'Tests run:\K[^,]*' integration-tests/target/failsafe-reports/*txt | awk '{SUM+=$1} END { print SUM }'`
-  nskipped=`grep -Poh 'Skipped:\K[^,]*' integration-tests/target/failsafe-reports/*txt | awk '{SUM+=$1} END { print SUM }'` 
+  ncompleted=`grep 'Tests run: ' vaadin*/*flow/target/surefire-reports/*.txt integration-tests/target/failsafe-reports/*txt | awk '{SUM+=$3} END { print SUM }'`
+  nskipped=`grep 'Tests run: ' vaadin*/*flow/target/surefire-reports/*.txt integration-tests/target/failsafe-reports/*txt | awk '{SUM+=$9} END { print SUM }'`
   if [ "$nfailed" -ge 1 ]
   then
     mkdir -p integration-tests/error-screenshots/$try
@@ -78,7 +96,7 @@ type pnpm && pnpm --version
 uname -a
 
 ## Compile all java files including tests in ITs modules
-cmd="mvn clean test-compile -DskipFrontend -B -q"
+cmd="mvn clean test-compile -DskipFrontend -B -q $args"
 tcLog "Compiling flow components - $cmd"
 $cmd || tcStatus 1 "Compilation failed"
 
@@ -87,15 +105,15 @@ tcLog "Running report watcher for Tests "
 tcMsg "importData type='surefire' path='**/*-reports/TEST*xml'";
 
 ## Compile and install all modules excluding ITs
-cmd="mvn install -Drelease -B -q -T $FORK_COUNT"
+cmd="mvn install -Drelease -B -q -T $FORK_COUNT $args"
 tcLog "Unit-Testing and Installing flow components - $cmd"
 $cmd
 if [ $? != 0 ]
 then
   ## Some times install fails because of maven multithread race condition
   ## running a second time it is mitigated
   tcLog "Unit-Testing and Installing flow components (2nd try) - $cmd"
-  sleep 30
+  sleep 15
   $cmd || tcStatus 1 "Unit-Testing failed"
 fi
 
@@ -123,7 +141,7 @@ $cmd || tcStatus 1 "Merging ITs failed"
 [ -n "$TBHUB" ] && args="$args -Dtest.use.hub=true -Dcom.vaadin.testbench.Parameters.hubHostname=$TBHUB"
 if [ -n "$SAUCE_USER" ]
 then
-   test -n  "$SAUCE_ACCESS_KEY" || { echo "\$SAUCE_ACCESS_KEY needs to be defined to use Saucelabs" >&2 ; exit 1; }
+   test -n "$SAUCE_ACCESS_KEY" || { echo "\$SAUCE_ACCESS_KEY needs to be defined to use Saucelabs" >&2 ; exit 1; }
    args="$args -P saucelabs -Dtest.use.hub=true -Dsauce.user=$SAUCE_USER -Dsauce.sauceAccessKey=$SAUCE_ACCESS_KEY"
 fi
 
@@ -147,17 +165,18 @@ reuse_browser() {
     [ -z "$1" ] || echo "-Dcom.vaadin.tests.SharedBrowser.reuseBrowser=$1"
 }
 
+
 if [ -n "$modules" ] && [ -z "$USE_MERGED_MODULE" ]
 then
   ### Run IT's in original modules
-  cmd="mvn clean verify -Dfailsafe.forkCount=$FORK_COUNT $args -pl $modules $(reuse_browser $TESTBENCH_REUSE_BROWSER)"
-  tcLog "Running module ITs - mvn clean verify -pl ..."
+  cmd="mvn clean verify -Dfailsafe.forkCount=$FORK_COUNT $args -pl $modules -Dtest=none $(reuse_browser $TESTBENCH_REUSE_BROWSER)"
+  tcLog "Running module ITs ($elements) - mvn clean verify -pl ..."
   echo $cmd
   $cmd
 else
   mode="-Dfailsafe.forkCount=$FORK_COUNT -Dcom.vaadin.testbench.Parameters.testsInParallel=$TESTS_IN_PARALLEL"
   ### Run IT's in merged module
-  cmd="mvn verify -B -q -Drun-it -Drelease -Dvaadin.productionMode -Dfailsafe.rerunFailingTestsCount=2 $mode $args -pl integration-tests $(reuse_browser $TESTBENCH_REUSE_BROWSER)"
+  cmd="mvn verify -B -q -Drun-it -Drelease -Dvaadin.productionMode -Dfailsafe.rerunFailingTestsCount=2 $mode $args -pl integration-tests -Dtest=none $(reuse_browser $TESTBENCH_REUSE_BROWSER)"
   tcLog "Running merged ITs - mvn verify -B -Drun-it -Drelease -pl integration-tests ..."
   echo $cmd
   $cmd
@@ -169,21 +188,22 @@ else
   if [ "$nfailed" -gt 0 ]
   then
       ## Give a second try to failed tests
-      tcLog "There were $nfailed IT classes: "
-      echo "$failed"
-      rerunFailed=$nfailed
+      tcLog "There were $nfailed failed IT classes in first round."
+      echo "$failedMethods"
 
+      rerunFailed=$nfailed
       if [ "$nfailed" -le 15 ]
       then
         failed=`echo "$failed" | tr '\n' ','`
         mode="-Dfailsafe.forkCount=2 -Dcom.vaadin.testbench.Parameters.testsInParallel=3"
-        cmd="mvn verify -B -q -Drun-it -Drelease -Dvaadin.productionMode -DskipFrontend $mode $args -pl integration-tests -Dit.test=$failed $(reuse_browser false)"
-        tcLog "Re-Running $nfailed IT classes ..."
+        cmd="mvn verify -B -q -Drun-it -Drelease -Dvaadin.productionMode -DskipFrontend $mode $args -pl integration-tests -Dtest=none -Dit.test=$failed $(reuse_browser false)"
+        tcLog "Re-Running $nfailed failed IT classes ..."
         echo $cmd
         $cmd
         error=$?
+        tcLog "Re-Run exited with code $error"
         saveFailedTests run-2
-        tcStatus $error "(IT2)Test failed: $nfailed" "(IT2)Tests passed: $ncompleted ($rerunFailed retried), ignored: $nskipped"
+        tcStatus $error "(IT2)Test failed: $nfailed" "(IT2)Tests passed: $ncompleted ($rerunFailed retried, $nfailed failed), ignored: $nskipped"
       else
         tcStatus $error "(IT1)Test failed: $nfailed" "(IT1)Tests passed: $ncompleted (more than 15 failed), ignored: $nskipped"
       fi

vaadin-charts-flow-parent/vaadin-charts-flow-integration-tests/src/test/java/com/vaadin/flow/component/charts/tests/ColumnWithLazyMultiLevelDrilldownCallbackTestsIT.java

@@ -24,6 +24,8 @@ protected Class<? extends AbstractChartExample> getTestView() {
     @Test
     public void test() throws AssertionError {
         ChartElement chart = $(ChartElement.class).first();
+        waitUntil(e -> getElementFromShadowRoot(chart,
+                By.cssSelector(".highcharts-drilldown-point")) != null);
         clickDrilldownPoint(chart, 0);
         // Can't drilldown with null callback
         assertEquals(0, getLogMessages().size());

vaadin-charts-flow-parent/vaadin-charts-flow-integration-tests/src/test/java/com/vaadin/flow/component/charts/tests/DynamicChangingChartIT.java

@@ -1,11 +1,12 @@
 package com.vaadin.flow.component.charts.tests;
 
+import org.junit.Test;
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebElement;
+
 import com.vaadin.flow.component.charts.AbstractChartExample;
 import com.vaadin.flow.component.charts.examples.other.DynamicChangingChart;
 import com.vaadin.flow.component.charts.testbench.ChartElement;
-import org.junit.Assert;
-import org.junit.Test;
-import org.openqa.selenium.By;
 
 public class DynamicChangingChartIT extends AbstractTBTest {
     @Override
@@ -26,8 +27,7 @@ public void setConfiguration_changes_chart() {
     }
 
     private void assertTitle(ChartElement chart, String expectedTitle) {
-        Assert.assertEquals(expectedTitle,
-            getElementFromShadowRoot(chart, By.className("highcharts-title"))
-                .getText());
+        WebElement title = getElementFromShadowRoot(chart, By.className("highcharts-title"));
+        waitUntil(e -> expectedTitle.equals(title.getText()), 2);
     }
 }

vaadin-charts-flow-parent/vaadin-charts-flow-integration-tests/src/test/java/com/vaadin/flow/component/charts/tests/ServerSideEventsIT.java

@@ -225,9 +225,9 @@ private void assertNthHistoryEventIsType(
     }
 
     private void resetHistory() {
+        waitUntil(e -> $(ButtonElement.class).exists());
         WebElement resetHistoryButton = $(ButtonElement.class).id("resetHistory");
         resetHistoryButton.click();
-
     }
 
     private SeriesCheckboxClickEvent readCheckboxEventDetails() {

vaadin-context-menu-flow-parent/vaadin-context-menu-flow-integration-tests/src/test/java/com/vaadin/flow/component/contextmenu/it/SubMenuIT.java

@@ -20,7 +20,6 @@
 
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.openqa.selenium.By;
 import org.openqa.selenium.WebElement;
@@ -58,7 +57,6 @@ public void addItemToSubMenu_subMenuRendered_clickListenerWorks() {
     }
 
     @Test
-    @Ignore("Flaky test - issue #67")
     public void openAndCloseSubMenu_addContent_contentUpdatedAndFunctional() {
         rightClickOn("target");
         openSubMenu(getMenuItems().get(0));
@@ -86,7 +84,6 @@ public void openAndCloseSubMenu_addContent_contentUpdatedAndFunctional() {
     }
 
     @Test
-    @Ignore("Flaky test - issue #67")
     public void openAndCloseSubMenu_addSubSubMenu_contentUpdatedAndFunctional() {
         rightClickOn("target");
         openSubMenu(getMenuItems().get(0));

vaadin-crud-flow-parent/vaadin-crud-flow-integration-tests/src/test/java/com/vaadin/flow/component/crud/test/BasicUseIT.java

@@ -1,16 +1,16 @@
 package com.vaadin.flow.component.crud.test;
 
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
 import com.vaadin.flow.component.button.testbench.ButtonElement;
 import com.vaadin.flow.component.crud.testbench.CrudElement;
 import com.vaadin.flow.component.grid.testbench.GridElement;
 import com.vaadin.flow.component.textfield.testbench.TextFieldElement;
 import com.vaadin.testbench.TestBenchElement;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.List;
 
 public class BasicUseIT extends AbstractParallelTest {
 
@@ -59,20 +59,20 @@ public void filterValueCorrect() {
     }
 
     @Test
-    @Ignore("Unable to access the sorting controls in Grid")
     public void sortEnabledInGrid() {
         GridElement grid = $(GridElement.class).waitForFirst();
-        Assert.assertTrue(grid.getHeaderCell(1).$("vaadin-grid-sorter").exists());
+        Assert.assertTrue(grid.getHeaderCellContent(0, 0).$("vaadin-grid-sorter").exists());
     }
 
     @Test
-    @Ignore("Unable to access the sorting controls in Grid")
     public void sortOrdersCorrect() {
         GridElement grid = $(GridElement.class).waitForFirst();
-        List<TestBenchElement> sorters = grid.getHeaderCell(1).$("vaadin-grid-sorter").all();
 
-        sorters.get(0).click(); // First name ascending
-        sorters.get(2).click(); sorters.get(2).click(); // Last name descending
+        TestBenchElement firstNameSorter = grid.getHeaderCellContent(0, 0).$("vaadin-grid-sorter").get(0);
+        TestBenchElement lasttNameSorter = grid.getHeaderCellContent(0, 2).$("vaadin-grid-sorter").get(0);
+
+        firstNameSorter.click(); // First name ascending
+        lasttNameSorter.click(); lasttNameSorter.click(); // Last name descending
 
         ButtonElement showFilterButton = getTestButton("showFilter");
         showFilterButton.click();

vaadin-crud-flow-parent/vaadin-crud-flow-integration-tests/src/test/java/com/vaadin/flow/component/crud/test/CustomGridIT.java

@@ -89,14 +89,14 @@ public void editorShouldHaveRightTitleWhenOpenedInExistingItemMode() {
         GridElement grid = $(GridElement.class).first();
 
         customGridClickToEditButton().click();
-        
+
         crud.getNewItemButton().ifPresent(button -> button.click());
         Assert.assertEquals("New item", getEditorHeaderText(crud));
-        
+
         crud.getEditorCancelButton().click();
         grid.getCell(0, 0).click();
-        String editorHeaderText = getEditorHeaderText(crud);
-        Assert.assertEquals("Edit item", editorHeaderText);
+
+        waitUntil((c) -> "Edit item".equals(getEditorHeaderText(crud)), 200);
     }
 
     @Test
@@ -110,7 +110,7 @@ public void editorShouldHaveRightTitleWhenOpenedInNewItemMode() {
 
         crud.$("vaadin-crud-edit").first().click();
         Assert.assertEquals("Edit item", getEditorHeaderText(crud));
-        
+
         crud.getEditorCancelButton().click();
 
         newItemButton().click();

vaadin-custom-field-flow-parent/vaadin-custom-field-flow-integration-tests/src/test/java/com/vaadin/flow/component/customfield/test/ErrorIT.java

@@ -1,11 +1,11 @@
 package com.vaadin.flow.component.customfield.test;
 
-import com.vaadin.flow.component.customfield.testbench.CustomFieldElement;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
+import com.vaadin.flow.component.customfield.testbench.CustomFieldElement;
+
 public class ErrorIT extends AbstractParallelTest {
     @Before
     public void init() {
@@ -14,7 +14,6 @@ public void init() {
     }
 
     @Test
-    @Ignore
     public void checkProperties() {
         final CustomFieldElement customField = $(CustomFieldElement.class)
             .waitForFirst();

vaadin-flow-components-shared/src/main/java/com/vaadin/tests/TabbedComponentDemoTest.java

@@ -1,9 +1,10 @@
 package com.vaadin.tests;
 
+import java.util.List;
+
 import org.junit.AfterClass;
 import org.openqa.selenium.remote.DesiredCapabilities;
-
-import java.util.List;
+import org.openqa.selenium.support.ui.ExpectedCondition;
 
 public abstract class TabbedComponentDemoTest extends com.vaadin.flow.demo.TabbedComponentDemoTest {
 
@@ -27,4 +28,9 @@ protected List<DesiredCapabilities> getHubBrowsersToTest() {
     public static void runAfterTest() {
         browser.clear();
     }
+
+    @Override
+    protected <T> T waitUntil(ExpectedCondition<T> condition) {
+        return super.waitUntil(condition, 120);
+    }
 }

vaadin-form-layout-flow-parent/vaadin-form-layout-flow-demo/src/main/java/com/vaadin/flow/component/formlayout/demo/FormLayoutView.java

@@ -181,9 +181,9 @@ private void createResponsiveLayout() {
         // the browser horizontally, you can notice that the number of the
         // columns in the FormLayout changes.
         nameLayout.setResponsiveSteps(
-                new ResponsiveStep("25em", 1),
-                new ResponsiveStep("32em", 2),
-                new ResponsiveStep("40em", 3));
+                new ResponsiveStep("1px", 1),
+                new ResponsiveStep("600px", 2),
+                new ResponsiveStep("700px", 3));
         // end-source-example
         // @formatter:on