From c49cfdcc40a740f0bf31d4fbd5ddc84fdabfa394 Mon Sep 17 00:00:00 2001 From: Josef Date: Sat, 16 Jul 2022 09:46:29 +0200 Subject: [PATCH 1/5] Added known limitation section - added known limitations to documentation --- changelog/796.doc | Bin 0 -> 9216 bytes docs/index.rst | 27 +++++++++++++++++++++++++++ src/xdist/report.py | 3 ++- testing/test_dsession.py | 3 ++- 4 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 changelog/796.doc diff --git a/changelog/796.doc b/changelog/796.doc new file mode 100644 index 0000000000000000000000000000000000000000..80cf878b1fe1522d4c9640567439a539b9d7ee69 GIT binary patch literal 9216 zcmeI1U2Icj7=Yi?wWFP5bOq-Glrkzze#X!E8G_P{!RlZ?FccAlbn6I{wZ*LjFkTRh zi5HS!G|>waV}kKQW2WACVWKzcjYe-sB;H`+4T&a5ydj3w=l$B_O6k_KYLJnB&2zrf z@9%xT_j}Lze$J0S)Lwb-lli|$Uf3cPGM@HHWlp-q{T%zhU&O_|rj4i5X*-$Qa+pCD zxS7>e$I#6ndO*i~7I?u2Rd63v!)&MlEsqPBcm{R1tZJSG_DYgpN`kUO62v2NhEa7M z>hPG{CY{vGeP1{!U#v`7{)7dJ*-G#A6V9GqC0$YEUbkF-l; z=CW?yZzB?ys0{O8=draRSst=lakpk^TP{7F6|x7mch*Zh;&G3{u82!|JKd6qxVb>|K0~w2-X!TUNlL zYe?^-^ozXJqx4?QTq|xQc=7uNT5qtzq|U;NlUWT1 zUeGm&eXnZ~4O&)%OJ6z~G=T?@;+aKCN8JT{sd(7OV8l^)6mxbHpCZ;tP#q*e^$oP+ z#3TEdDC5#kO?9j0rOu?c;)+9AR~%A}_5E2S?50LPnvPH}#eEVzgJy8Gt)tW~)~8LC z&a~^MRkcnZ4yLx)=CIyw$`4R8hK8z{T2dXpPpx7P*c82-tcP|xxmN3Ns8bjE3}CY) znk2E3E^!W^Sp-WCY2QK@cyx+~$@ssFPyd%HHU5@?J)ai2{e5ow1qsifDqNRW-eeOD0ZDx;5MSG68ML^D-7)K7B#?@sm7 zC717%sIr{01FGd1w9Y=EBhbw;D6&S_P;A{eB4f&CofIUb!GE@bLVf{cvu=w(#hp#92qSDVP@^nLh(bDl^)0#O$cG!R|$wKZh z>yJ`H71H(FiS|ijC+(=d_DR874sWCMyi^@(vhr271}E}W*Pr$9gWxfTy>Uh$%3 None: " bbb\n" "+XXX\n" " ccc\n" - "-YYY" + "-YYY\n" + "To see why this happens see Known limitations in documentation" ) msg = report_collection_diff(from_collection, to_collection, "1", "2") From b52e51b54a21505efc6972382c690ce4265ff090 Mon Sep 17 00:00:00 2001 From: Josef Date: Fri, 22 Jul 2022 00:10:34 +0200 Subject: [PATCH 2/5] Improve known limitations documentation * moved doc file to rst file * moved know limitations section to its own file * added quick workarounds section that documents quick fixes --- changelog/796.doc | Bin 9216 -> 0 bytes changelog/796.rst | 37 ++++++++++++++++++++++++++++ docs/index.rst | 28 +--------------------- docs/known-limitations.rst | 48 +++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 27 deletions(-) delete mode 100644 changelog/796.doc create mode 100644 changelog/796.rst create mode 100644 docs/known-limitations.rst diff --git a/changelog/796.doc b/changelog/796.doc deleted file mode 100644 index 80cf878b1fe1522d4c9640567439a539b9d7ee69..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9216 zcmeI1U2Icj7=Yi?wWFP5bOq-Glrkzze#X!E8G_P{!RlZ?FccAlbn6I{wZ*LjFkTRh zi5HS!G|>waV}kKQW2WACVWKzcjYe-sB;H`+4T&a5ydj3w=l$B_O6k_KYLJnB&2zrf z@9%xT_j}Lze$J0S)Lwb-lli|$Uf3cPGM@HHWlp-q{T%zhU&O_|rj4i5X*-$Qa+pCD zxS7>e$I#6ndO*i~7I?u2Rd63v!)&MlEsqPBcm{R1tZJSG_DYgpN`kUO62v2NhEa7M z>hPG{CY{vGeP1{!U#v`7{)7dJ*-G#A6V9GqC0$YEUbkF-l; z=CW?yZzB?ys0{O8=draRSst=lakpk^TP{7F6|x7mch*Zh;&G3{u82!|JKd6qxVb>|K0~w2-X!TUNlL zYe?^-^ozXJqx4?QTq|xQc=7uNT5qtzq|U;NlUWT1 zUeGm&eXnZ~4O&)%OJ6z~G=T?@;+aKCN8JT{sd(7OV8l^)6mxbHpCZ;tP#q*e^$oP+ z#3TEdDC5#kO?9j0rOu?c;)+9AR~%A}_5E2S?50LPnvPH}#eEVzgJy8Gt)tW~)~8LC z&a~^MRkcnZ4yLx)=CIyw$`4R8hK8z{T2dXpPpx7P*c82-tcP|xxmN3Ns8bjE3}CY) znk2E3E^!W^Sp-WCY2QK@cyx+~$@ssFPyd%HHU5@?J)ai2{e5ow1qsifDqNRW-eeOD0ZDx;5MSG68ML^D-7)K7B#?@sm7 zC717%sIr{01FGd1w9Y=EBhbw;D6&S_P;A{eB4f&CofIUb!GE@bLVf{cvu=w(#hp#92qSDVP@^nLh(bDl^)0#O$cG!R|$wKZh z>yJ`H71H(FiS|ijC+(=d_DR874sWCMyi^@(vhr271}E}W*Pr$9gWxfTy>Uh$%3`_: {{ text }} + +{% endfor %} +{% else %} +- {{ sections[section][category]['']|sort|join(', ') }} + + +{% endif %} +{% if sections[section][category]|length == 0 %} + +Added known limitations to documentation. + + +{% else %} +{% endif %} +{% endfor %} +{% else %} +Added known limitation to documentation. + + +{% endif %} +{% endfor %} diff --git a/docs/index.rst b/docs/index.rst index 588176f3..6e8e6112 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,31 +60,5 @@ Features crash how-to how-it-works + known-limitations changelog - -Known limitations ------------------ -pytest-xdist have some limitation that may be supported in pytest but can't be supported in pytest-xdist - -Order and amount of test must be constant -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Is is not possible to have test that differs in order or their amount across threads/execution. - -This is especially true with `pytest.mark.parametrize`. -When parametrize is used with set or other unordered iterable-like/generator -pytest-xdist - - -Example -:: - import pytest - - @pytest.mark.parametrize("param",{"a","b"}) - def test_pytest_parametrize_unordered(param): - pass - -will cause issue despite pytest supports it. - -Truncated pytest signature: -:: - argvalues: Iterable[Union[ParameterSet, Sequence[object], object]] diff --git a/docs/known-limitations.rst b/docs/known-limitations.rst new file mode 100644 index 00000000..cdde796a --- /dev/null +++ b/docs/known-limitations.rst @@ -0,0 +1,48 @@ +Known limitations +----------------- +pytest-xdist has some limitations that may be supported in pytest but can't be supported in pytest-xdist. + +Order and amount of test must be consistent +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Is is not possible to have tests that differ in order or their amount across workers. + +This is especially true with ``pytest.mark.parametrize``. +When parametrize is used with set or other unordered iterable-like/generator pytest-xdist fails. + + +Example +.. code-block:: python + import pytest + + @pytest.mark.parametrize("param",{"a","b"}) + def test_pytest_parametrize_unordered(param): + pass + +In the example above, the fact that ``set``s are not necessarily ordered can cause different workers +to collect tests in different order, which will throw an error. + +Quick workarounds to ordering limitation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +A solution to this is to guarantee that the parametrized values have the same order. + +There are two simple solution: +# to convert your set to array. +# sort your set (your items must be sortable) + +Array approach +.. code-block:: python + + import pytest + + @pytest.mark.parametrize("param", ["a","b"]) + def test_pytest_parametrize_unordered(param): + pass + +Sorted approach +.. code-block:: python + + import pytest + + @pytest.mark.parametrize("param", sorted({"a","b"})) + def test_pytest_parametrize_unordered(param): + pass From f30656ffe89c43f7cf7bb9b260bc051e573912e0 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 22 Jul 2022 10:27:07 -0300 Subject: [PATCH 3/5] Update 796.rst --- changelog/796.rst | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/changelog/796.rst b/changelog/796.rst index 6404e9f5..28a10bc4 100644 --- a/changelog/796.rst +++ b/changelog/796.rst @@ -1,37 +1 @@ -{% for section in sections %} -{% set underline = "-" %} -{% if section %} -{{section}} -{{ underline * section|length }}{% set underline = "~" %} - -{% endif %} -{% if sections[section] %} -{% for category, val in definitions.items() if category in sections[section] %} - -{{ definitions[category]['name'] }} -{{ underline * definitions[category]['name']|length }} - -{% if definitions[category]['showcontent'] %} -{% for text, values in sections[section][category]|dictsort(by='value') %} -- `{{ values[0] }} `_: {{ text }} - -{% endfor %} -{% else %} -- {{ sections[section][category]['']|sort|join(', ') }} - - -{% endif %} -{% if sections[section][category]|length == 0 %} - -Added known limitations to documentation. - - -{% else %} -{% endif %} -{% endfor %} -{% else %} -Added known limitation to documentation. - - -{% endif %} -{% endfor %} +Added known limitations section to documentation. From 43f04d7ece0d4721bc1aa663d1c5717cafe9d789 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 22 Jul 2022 10:27:24 -0300 Subject: [PATCH 4/5] Rename 796.rst to 796.doc.rst --- changelog/{796.rst => 796.doc.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog/{796.rst => 796.doc.rst} (100%) diff --git a/changelog/796.rst b/changelog/796.doc.rst similarity index 100% rename from changelog/796.rst rename to changelog/796.doc.rst From 0c836be40afef8614c07f6e377795853d7840846 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 22 Jul 2022 10:33:01 -0300 Subject: [PATCH 5/5] Update known-limitations.rst --- docs/known-limitations.rst | 40 +++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/docs/known-limitations.rst b/docs/known-limitations.rst index cdde796a..04641a7b 100644 --- a/docs/known-limitations.rst +++ b/docs/known-limitations.rst @@ -1,48 +1,52 @@ Known limitations ------------------ +================= + pytest-xdist has some limitations that may be supported in pytest but can't be supported in pytest-xdist. Order and amount of test must be consistent -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------- + Is is not possible to have tests that differ in order or their amount across workers. -This is especially true with ``pytest.mark.parametrize``. -When parametrize is used with set or other unordered iterable-like/generator pytest-xdist fails. +This is especially true with ``pytest.mark.parametrize``, when values are produced with sets or other unordered iterables/generators. + +Example: -Example .. code-block:: python + import pytest - @pytest.mark.parametrize("param",{"a","b"}) + @pytest.mark.parametrize("param", {"a","b"}) def test_pytest_parametrize_unordered(param): pass -In the example above, the fact that ``set``s are not necessarily ordered can cause different workers +In the example above, the fact that ``set`` are not necessarily ordered can cause different workers to collect tests in different order, which will throw an error. -Quick workarounds to ordering limitation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Workarounds +~~~~~~~~~~~ + A solution to this is to guarantee that the parametrized values have the same order. -There are two simple solution: -# to convert your set to array. -# sort your set (your items must be sortable) +Some solutions: -Array approach -.. code-block:: python +* Convert your sequence to a ``list``. + + .. code-block:: python import pytest - @pytest.mark.parametrize("param", ["a","b"]) + @pytest.mark.parametrize("param", ["a", "b"]) def test_pytest_parametrize_unordered(param): pass -Sorted approach -.. code-block:: python +* Sort your sequence, guaranteeing order. + + .. code-block:: python import pytest - @pytest.mark.parametrize("param", sorted({"a","b"})) + @pytest.mark.parametrize("param", sorted({"a", "b"})) def test_pytest_parametrize_unordered(param): pass