Merge pull request #1001 from freedomofpress/oh_my_flaky

Fixes #956 Adds flaky for functinoal tests

dev-requirements.in

@@ -4,6 +4,7 @@ attrs==18.2.0
 Click==7.0
 coverage==4.5.1
 flake8==3.6.0
+flaky==3.6.1
 MarkupSafe>=1.1
 mccabe==0.6.1
 more-itertools==4.3.0

dev-requirements.txt

@@ -68,6 +68,9 @@ execnet==1.7.1 \
 flake8==3.6.0 \
     --hash=sha256:6a35f5b8761f45c5513e3405f110a86bea57982c3b75b766ce7b65217abe1670 \
     --hash=sha256:c01f8a3963b3571a8e6bd7a4063359aff90749e160778e03817cd9b71c9e07d2
+flaky==3.6.1 \
+    --hash=sha256:5471615b32b0f8086573de924475b1f0d31e0e8655a089eb9c38a0fbff3f11aa \
+    --hash=sha256:8cd5455bb00c677f787da424eaf8c4a58a922d0e97126d3085db5b279a98b698
 idna==2.7 \
     --hash=sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e \
     --hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16

tests/functional/test_delete_source.py

@@ -5,10 +5,12 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from .utils import get_safe_tempdir, get_logged_in_test_context
 
 
+@flaky
 @pytest.mark.vcr()
 def test_delete_source_and_their_docs(qtbot, mocker):
     """

tests/functional/test_download_file.py

@@ -5,11 +5,13 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from securedrop_client.gui.widgets import FileWidget
 from .utils import get_safe_tempdir, get_logged_in_test_context
 
 
+@flaky
 @pytest.mark.vcr()
 def test_download_file(qtbot, mocker):
     """

tests/functional/test_export_dialog.py

@@ -5,11 +5,13 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from securedrop_client.gui.widgets import FileWidget
 from .utils import get_safe_tempdir, get_logged_in_test_context
 
 
+@flaky
 @pytest.mark.vcr()
 def test_export_dialog(qtbot, mocker):
     """

tests/functional/test_login.py

@@ -5,6 +5,7 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from .utils import get_safe_tempdir, get_test_context, USERNAME, PASSWORD
 from securedrop_client.gui.main import Window
@@ -26,6 +27,7 @@ def test_login_ensure_errors_displayed(qtbot, mocker):
     assert actual == expected
 
 
+@flaky
 @pytest.mark.vcr()  # Ensure any API network traffic is recorded/replayed.
 def test_login_as_journalist(qtbot, mocker):
     """

tests/functional/test_login_from_offline.py

@@ -2,10 +2,12 @@
 Functional test for logging out and then login again from offline mode.
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from .utils import get_safe_tempdir, get_logged_in_test_context, USERNAME, PASSWORD
 
 
+@flaky
 @pytest.mark.vcr()
 def test_login_from_offline(qtbot, mocker):
     """

tests/functional/test_logout.py

@@ -5,9 +5,11 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from .utils import get_safe_tempdir, get_logged_in_test_context
 
 
+@flaky
 @pytest.mark.vcr()
 def test_logout_as_journalist(qtbot, mocker):
     """

tests/functional/test_offline_delete_source.py

@@ -5,10 +5,12 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from .utils import get_safe_tempdir, get_logged_in_test_context
 
 
+@flaky
 @pytest.mark.vcr()
 def test_offline_delete_source_and_their_docs(qtbot, mocker):
     """

tests/functional/test_offline_read_conversations.py

@@ -5,10 +5,12 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from .utils import get_safe_tempdir, get_logged_in_test_context
 
 
+@flaky
 @pytest.mark.vcr()
 def test_offline_read_conversations(qtbot, mocker):
     """
@@ -28,6 +30,10 @@ def check_for_sources():
     first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)
 
+    # Otherwise our test is running too fast to create all files/directories
+    # as received via API call.
+    qtbot.wait(1000)
+
     # Now logout.
     def check_login_button():
         assert gui.left_pane.user_profile.login_button.isVisible()

tests/functional/test_offline_send_reply.py

@@ -5,10 +5,12 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from .utils import get_safe_tempdir, get_logged_in_test_context
 
 
+@flaky
 @pytest.mark.vcr()
 def test_offline_send_reply_to_source(qtbot, mocker):
     """

tests/functional/test_offline_star_source.py

@@ -5,10 +5,12 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from .utils import get_safe_tempdir, get_logged_in_test_context
 
 
+@flaky
 @pytest.mark.vcr()
 def test_offline_star_source(qtbot, mocker):
     """

tests/functional/test_receive_message.py

@@ -5,11 +5,13 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from securedrop_client.gui.widgets import FileWidget
 from .utils import get_safe_tempdir, get_logged_in_test_context
 
 
+@flaky
 @pytest.mark.vcr()
 def test_receive_message_from_source(qtbot, mocker):
     """

tests/functional/test_send_reply.py

@@ -5,10 +5,12 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from .utils import get_safe_tempdir, get_logged_in_test_context
 
 
+@flaky
 @pytest.mark.vcr()
 def test_send_reply_to_source(qtbot, mocker):
     """

tests/functional/test_star_source.py

@@ -5,10 +5,12 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from .utils import get_safe_tempdir, get_logged_in_test_context
 
 
+@flaky
 @pytest.mark.vcr()
 def test_star_source(qtbot, mocker):
     """

tests/functional/test_unstar_source.py

@@ -5,10 +5,12 @@
 https://github.com/freedomofpress/securedrop-client/wiki/Test-plan#basic-client-testing
 """
 import pytest
+from flaky import flaky
 from PyQt5.QtCore import Qt
 from .utils import get_safe_tempdir, get_logged_in_test_context
 
 
+@flaky
 @pytest.mark.vcr()
 def test_unstar_source(qtbot, mocker):
     """