Merge pull request #144679 from hercules-ci/fix-issue-144613-nixosTes…

…t-wait-stdout

nixosTest: document wait for stdout behavior, fix #144613

nixos/doc/manual/development/writing-nixos-tests.section.md

@@ -159,6 +159,10 @@ The following methods are available on machine objects:
 `execute`
 
 :   Execute a shell command, returning a list `(status, stdout)`.
+    If the command detaches, it must close stdout, as `execute` will wait
+    for this to consume all output reliably. This can be achieved by
+    redirecting stdout to stderr `>&2`, to `/dev/console`, `/dev/null` or
+    a file.
     Takes an optional parameter `check_return` that defaults to `True`.
     Setting this parameter to `False` will not check for the return code
     and return -1 instead. This can be used for commands that shut down
@@ -179,6 +183,8 @@ The following methods are available on machine objects:
 
     -   Dereferencing unset variables fail the command.
 
+    -   It will wait for stdout to be closed. See `execute`.
+
 `fail`
 
 :   Like `succeed`, but raising an exception if the command returns a zero

nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml

@@ -266,7 +266,12 @@ start_all()
       <listitem>
         <para>
           Execute a shell command, returning a list
-          <literal>(status, stdout)</literal>. Takes an optional
+          <literal>(status, stdout)</literal>. If the command detaches,
+          it must close stdout, as <literal>execute</literal> will wait
+          for this to consume all output reliably. This can be achieved
+          by redirecting stdout to stderr <literal>&gt;&amp;2</literal>,
+          to <literal>/dev/console</literal>,
+          <literal>/dev/null</literal> or a file. Takes an optional
           parameter <literal>check_return</literal> that defaults to
           <literal>True</literal>. Setting this parameter to
           <literal>False</literal> will not check for the return code
@@ -306,6 +311,12 @@ start_all()
               Dereferencing unset variables fail the command.
             </para>
           </listitem>
+          <listitem>
+            <para>
+              It will wait for stdout to be closed. See
+              <literal>execute</literal>.
+            </para>
+          </listitem>
         </itemizedlist>
       </listitem>
     </varlistentry>

nixos/doc/manual/from_md/release-notes/rl-2111.section.xml

@@ -423,6 +423,23 @@
   <section xml:id="sec-release-21.11-incompatibilities">
     <title>Backward Incompatibilities</title>
     <itemizedlist>
+      <listitem>
+        <para>
+          The NixOS VM test framework,
+          <literal>pkgs.nixosTest</literal>/<literal>make-test-python.nix</literal>,
+          now requires non-terminating commands such as
+          <literal>succeed(&quot;foo &amp;&quot;)</literal> to close
+          stdout. This can be done with a redirect such as
+          <literal>succeed(&quot;foo &gt;&amp;2 &amp;&quot;)</literal>.
+          This breaking change was necessitated by a race condition
+          causing tests to fail or hang. It applies to all methods that
+          invoke commands on the nodes, including
+          <literal>execute</literal>, <literal>succeed</literal>,
+          <literal>fail</literal>,
+          <literal>wait_until_succeeds</literal>,
+          <literal>wait_until_fails</literal>.
+        </para>
+      </listitem>
       <listitem>
         <para>
           The <literal>services.wakeonlan</literal> option was removed,

nixos/doc/manual/release-notes/rl-2111.section.md

@@ -128,6 +128,10 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 ## Backward Incompatibilities {#sec-release-21.11-incompatibilities}
 
+- The NixOS VM test framework, `pkgs.nixosTest`/`make-test-python.nix`, now requires non-terminating commands such as `succeed("foo &")` to close stdout.
+  This can be done with a redirect such as `succeed("foo >&2 &")`. This breaking change was necessitated by a race condition causing tests to fail or hang.
+  It applies to all methods that invoke commands on the nodes, including `execute`, `succeed`, `fail`, `wait_until_succeeds`, `wait_until_fails`.
+
 - The `services.wakeonlan` option was removed, and replaced with `networking.interfaces.<name>.wakeOnLan`.
 
 - The `security.wrappers` option now requires to always specify an owner, group and whether the setuid/setgid bit should be set.

nixos/tests/containers-imperative.nix

@@ -119,7 +119,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
 
       with subtest("Stop a container early"):
           machine.succeed(f"nixos-container stop {id1}")
-          machine.succeed(f"nixos-container start {id1} &")
+          machine.succeed(f"nixos-container start {id1} >&2 &")
           machine.wait_for_console_text("Stage 2")
           machine.succeed(f"nixos-container stop {id1}")
           machine.wait_for_console_text(f"Container {id1} exited successfully")

nixos/tests/croc.nix

@@ -38,7 +38,7 @@ in {
     sender.execute("echo Hello World > testfile01.txt")
     sender.execute("echo Hello Earth > testfile02.txt")
     sender.execute(
-        "croc --pass ${pass} --relay relay send --code topSecret testfile01.txt testfile02.txt &"
+        "croc --pass ${pass} --relay relay send --code topSecret testfile01.txt testfile02.txt >&2 &"
     )
 
     # receive the testfiles and check them

nixos/tests/emacs-daemon.nix

@@ -33,7 +33,7 @@ import ./make-test-python.nix ({ pkgs, ...} : {
       )
 
       # connects to the daemon
-      machine.succeed("emacsclient --create-frame $EDITOR &")
+      machine.succeed("emacsclient --create-frame $EDITOR >&2 &")
 
       # checks that Emacs shows the edited filename
       machine.wait_for_text("emacseditor")

nixos/tests/enlightenment.nix

@@ -88,7 +88,7 @@ import ./make-test-python.nix ({ pkgs, ...} :
         machine.screenshot("wizard12")
 
     with subtest("Run Terminology"):
-        machine.succeed("terminology &")
+        machine.succeed("terminology >&2 &")
         machine.sleep(5)
         machine.send_chars("ls --color -alF\n")
         machine.sleep(2)

nixos/tests/etesync-dav.nix

@@ -13,7 +13,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
     ''
       machine.wait_for_unit("multi-user.target")
       machine.succeed("etesync-dav --version")
-      machine.execute("etesync-dav &")
+      machine.execute("etesync-dav >&2 &")
       machine.wait_for_open_port(37358)
       with subtest("Check that the web interface is accessible"):
           assert "Add User" in machine.succeed("curl -s http://localhost:37358/.web/add/")

nixos/tests/firefox.nix

@@ -91,15 +91,15 @@ import ./make-test-python.nix ({ pkgs, firefoxPackage, ... }: {
 
       with subtest("Wait until Firefox has finished loading the Valgrind docs page"):
           machine.execute(
-              "xterm -e 'firefox file://${pkgs.valgrind.doc}/share/doc/valgrind/html/index.html' &"
+              "xterm -e 'firefox file://${pkgs.valgrind.doc}/share/doc/valgrind/html/index.html' >&2 &"
           )
           machine.wait_for_window("Valgrind")
           machine.sleep(40)
 
       with subtest("Check whether Firefox can play sound"):
           with audio_recording(machine):
               machine.succeed(
-                  "firefox file://${pkgs.sound-theme-freedesktop}/share/sounds/freedesktop/stereo/phone-incoming-call.oga &"
+                  "firefox file://${pkgs.sound-theme-freedesktop}/share/sounds/freedesktop/stereo/phone-incoming-call.oga >&2 &"
               )
               wait_for_sound(machine)
           machine.copy_from_vm("/tmp/record.wav")

nixos/tests/ft2-clone.nix

@@ -22,7 +22,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
       # Add a dummy sound card, or the program won't start
       machine.execute("modprobe snd-dummy")
 
-      machine.execute("ft2-clone &")
+      machine.execute("ft2-clone >&2 &")
 
       machine.wait_for_window(r"Fasttracker")
       machine.sleep(5)

nixos/tests/hibernate.nix

@@ -110,7 +110,7 @@ in makeTest {
       )
 
       # Hibernate machine
-      hibernate.execute("systemctl hibernate &", check_return=False)
+      hibernate.execute("systemctl hibernate >&2 &", check_return=False)
       hibernate.wait_for_shutdown()
 
       # Restore machine from hibernation, validate our ramfs file is there.

nixos/tests/keepassxc.nix

@@ -26,7 +26,7 @@ import ./make-test-python.nix ({ pkgs, ...} :
     machine.wait_for_x()
 
     # start KeePassXC window
-    machine.execute("su - alice -c keepassxc &")
+    machine.execute("su - alice -c keepassxc >&2 &")
 
     machine.wait_for_text("KeePassXC ${pkgs.keepassxc.version}")
     machine.screenshot("KeePassXC")

nixos/tests/kexec.nix

@@ -18,7 +18,7 @@ import ./make-test-python.nix ({ pkgs, lib, ...} : {
   testScript =
     ''
       machine.wait_for_unit("multi-user.target")
-      machine.execute("systemctl kexec &", check_return=False)
+      machine.execute("systemctl kexec >&2 &", check_return=False)
       machine.connected = False
       machine.wait_for_unit("multi-user.target")
     '';

nixos/tests/keymap.nix

@@ -46,7 +46,7 @@ let
 
               # set up process that expects all the keys to be entered
               machine.succeed(
-                  "{} {} {} {} &".format(
+                  "{} {} {} {} >&2 &".format(
                       cmd,
                       "${testReader}",
                       len(inputs),

nixos/tests/libreswan.nix

@@ -89,7 +89,7 @@ in
           """
           Sends a message as Alice to Bob
           """
-          bob.execute("nc -lu ::0 1234 >/tmp/msg &")
+          bob.execute("nc -lu ::0 1234 >/tmp/msg >&2 &")
           alice.sleep(1)
           alice.succeed(f"echo '{msg}' | nc -uw 0 bob 1234")
           bob.succeed(f"grep '{msg}' /tmp/msg")
@@ -100,7 +100,7 @@ in
           Starts eavesdropping on Alice and Bob
           """
           match = "src host alice and dst host bob"
-          eve.execute(f"tcpdump -i br0 -c 1 -Avv {match} >/tmp/log &")
+          eve.execute(f"tcpdump -i br0 -c 1 -Avv {match} >/tmp/log >&2 &")
 
 
       start_all()
@@ -120,7 +120,7 @@ in
           alice.succeed("ipsec verify 1>&2")
 
       with subtest("Alice and Bob can start the tunnel"):
-          alice.execute("ipsec auto --start tunnel &")
+          alice.execute("ipsec auto --start tunnel >&2 &")
           bob.succeed("ipsec auto --start tunnel")
           # apparently this is needed to "wake" the tunnel
           bob.execute("ping -c1 alice")

nixos/tests/lorri/default.nix

@@ -14,7 +14,7 @@ import ../make-test-python.nix {
     )
 
     # Start the daemon and wait until it is ready
-    machine.execute("lorri daemon > lorri.stdout 2> lorri.stderr &")
+    machine.execute("lorri daemon > lorri.stdout 2> lorri.stderr >&2 &")
     machine.wait_until_succeeds("grep --fixed-strings 'ready' lorri.stdout")
 
     # Ping the daemon

nixos/tests/magic-wormhole-mailbox-server.nix

@@ -29,7 +29,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
 
     # Create a secret file and send it to Bob
     client_alice.succeed("echo mysecret > secretfile")
-    client_alice.succeed("wormhole --relay-url=ws://server:4000/v1 send -0 secretfile &")
+    client_alice.succeed("wormhole --relay-url=ws://server:4000/v1 send -0 secretfile >&2 &")
 
     # Retrieve a secret file from Alice and check its content
     client_bob.succeed("wormhole --relay-url=ws://server:4000/v1 receive -0 --accept-file")

nixos/tests/minecraft.nix

@@ -20,7 +20,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
     let user = nodes.client.config.users.users.alice;
     in ''
       client.wait_for_x()
-      client.execute("su - alice -c minecraft-launcher &")
+      client.execute("su - alice -c minecraft-launcher >&2 &")
       client.wait_for_text("Create a new Microsoft account")
       client.sleep(10)
       client.screenshot("launcher")

nixos/tests/mpv.nix

@@ -21,7 +21,7 @@ in
     };
 
   testScript = ''
-    machine.execute("set -m; mpv --script-opts=webui-port=${port} --idle=yes &")
+    machine.execute("set -m; mpv --script-opts=webui-port=${port} --idle=yes >&2 &")
     machine.wait_for_open_port(${port})
     assert "<title>simple-mpv-webui" in machine.succeed("curl -s localhost:${port}")
   '';

nixos/tests/mumble.nix

@@ -38,8 +38,8 @@ in
     client1.wait_for_x()
     client2.wait_for_x()
 
-    client1.execute("mumble mumble://client1:testpassword\@server/test &")
-    client2.execute("mumble mumble://client2:testpassword\@server/test &")
+    client1.execute("mumble mumble://client1:testpassword\@server/test >&2 &")
+    client2.execute("mumble mumble://client2:testpassword\@server/test >&2 &")
 
     # cancel client audio configuration
     client1.wait_for_window(r"Audio Tuning Wizard")

nixos/tests/musescore.nix

@@ -44,7 +44,7 @@ in
     )
 
     # Start MuseScore window
-    machine.execute("DISPLAY=:0.0 mscore &")
+    machine.execute("DISPLAY=:0.0 mscore >&2 &")
 
     # Wait until MuseScore has launched
     machine.wait_for_window("MuseScore")

nixos/tests/nfs/simple.nix

@@ -66,7 +66,7 @@ in
           client2.succeed("time flock -n -s /data/lock true")
 
       with subtest("client 2 fails to acquire lock held by client 1"):
-          client1.succeed("flock -x /data/lock -c 'touch locked; sleep 100000' &")
+          client1.succeed("flock -x /data/lock -c 'touch locked; sleep 100000' >&2 &")
           client1.wait_for_file("locked")
           client2.fail("flock -n -s /data/lock true")
 

nixos/tests/nginx-etag.nix

@@ -76,7 +76,7 @@ import ./make-test-python.nix {
 
     server.wait_for_unit("nginx.service")
     client.wait_for_unit("multi-user.target")
-    client.execute("test-runner &")
+    client.execute("test-runner >&2 &")
     client.wait_for_file("/tmp/passed_stage1")
 
     server.succeed(

nixos/tests/nixops/default.nix

@@ -78,7 +78,7 @@ let
         # Put newlines on console, to flush the console reader's line buffer
         # in case nixops' last output did not end in a newline, as is the case
         # with a status line (if implemented?)
-        deployer.succeed("while sleep 60s; do echo [60s passed] >/dev/console; done &")
+        deployer.succeed("while sleep 60s; do echo [60s passed]; done >&2 &")
 
         deployer_do("cd ~/unicorn; ssh -oStrictHostKeyChecking=accept-new root@server echo hi")
 

nixos/tests/openarena.nix

@@ -38,8 +38,8 @@ in {
       client1.wait_for_x()
       client2.wait_for_x()
 
-      client1.execute("openarena +set r_fullscreen 0 +set name Foo +connect server &")
-      client2.execute("openarena +set r_fullscreen 0 +set name Bar +connect server &")
+      client1.execute("openarena +set r_fullscreen 0 +set name Foo +connect server >&2 &")
+      client2.execute("openarena +set r_fullscreen 0 +set name Bar +connect server >&2 &")
 
       server.wait_until_succeeds(
           "journalctl -u openarena -e | grep -q 'Foo.*entered the game'"

nixos/tests/plotinus.nix

@@ -14,7 +14,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
 
   testScript = ''
     machine.wait_for_x()
-    machine.succeed("gnome-calculator &")
+    machine.succeed("gnome-calculator >&2 &")
     machine.wait_for_window("gnome-calculator")
     machine.succeed(
         "xdotool search --sync --onlyvisible --class gnome-calculator "

nixos/tests/pt2-clone.nix

@@ -22,7 +22,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
       # Add a dummy sound card, or the program won't start
       machine.execute("modprobe snd-dummy")
 
-      machine.execute("pt2-clone &")
+      machine.execute("pt2-clone >&2 &")
 
       machine.wait_for_window(r"ProTracker")
       machine.sleep(5)

nixos/tests/shattered-pixel-dungeon.nix

@@ -19,7 +19,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
   testScript =
     ''
       machine.wait_for_x()
-      machine.execute("shattered-pixel-dungeon &")
+      machine.execute("shattered-pixel-dungeon >&2 &")
       machine.wait_for_window(r"Shattered Pixel Dungeon")
       machine.sleep(5)
       if "Enter" not in machine.get_screen_text():

nixos/tests/signal-desktop.nix

@@ -41,7 +41,7 @@ in {
     machine.wait_for_x()
 
     # start signal desktop
-    machine.execute("su - alice -c signal-desktop &")
+    machine.execute("su - alice -c signal-desktop >&2 &")
 
     # Wait for the Signal window to appear. Since usually the tests
     # are run sandboxed and therfore with no internet, we can not wait

nixos/tests/soapui.nix

@@ -16,7 +16,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
 
   testScript = ''
     machine.wait_for_x()
-    machine.succeed("soapui &")
+    machine.succeed("soapui >&2 &")
     machine.wait_for_window(r"SoapUI \d+\.\d+\.\d+")
     machine.sleep(1)
     machine.screenshot("soapui")

nixos/tests/tigervnc.nix

@@ -35,13 +35,13 @@ makeTest {
     for host in [server, client]:
         host.succeed("echo foobar | vncpasswd -f > vncpasswd")
 
-    server.succeed("Xvnc -geometry 720x576 :1 -PasswordFile vncpasswd &")
+    server.succeed("Xvnc -geometry 720x576 :1 -PasswordFile vncpasswd >&2 &")
     server.wait_until_succeeds("nc -z localhost 5901", timeout=10)
     server.succeed("DISPLAY=:1 xwininfo -root | grep 720x576")
-    server.execute("DISPLAY=:1 display -size 360x200 -font sans -gravity south label:'HELLO VNC WORLD' &")
+    server.execute("DISPLAY=:1 display -size 360x200 -font sans -gravity south label:'HELLO VNC WORLD' >&2 &")
 
     client.wait_for_x()
-    client.execute("vncviewer server:1 -PasswordFile vncpasswd &")
+    client.execute("vncviewer server:1 -PasswordFile vncpasswd >&2 &")
     client.wait_for_window(r"VNC")
     client.screenshot("screenshot")
     text = client.get_screen_text()

nixos/tests/turbovnc-headless-server.nix

@@ -97,7 +97,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
         )
         machine.execute(
             # Note trailing & for backgrounding.
-            f"({xvnc_command} | tee /tmp/Xvnc.stdout) 3>&1 1>&2 2>&3 | tee /tmp/Xvnc.stderr &",
+            f"({xvnc_command} | tee /tmp/Xvnc.stdout) 3>&1 1>&2 2>&3 | tee /tmp/Xvnc.stderr >&2 &",
         )
 
 
@@ -119,7 +119,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
     def test_glxgears_failing_with_bad_driver_path():
         machine.execute(
             # Note trailing & for backgrounding.
-            "(env DISPLAY=:0 LIBGL_DRIVERS_PATH=/nonexistent glxgears -info | tee /tmp/glxgears-should-fail.stdout) 3>&1 1>&2 2>&3 | tee /tmp/glxgears-should-fail.stderr &"
+            "(env DISPLAY=:0 LIBGL_DRIVERS_PATH=/nonexistent glxgears -info | tee /tmp/glxgears-should-fail.stdout) 3>&1 1>&2 2>&3 | tee /tmp/glxgears-should-fail.stderr >&2 &"
         )
         machine.wait_until_succeeds("test -f /tmp/glxgears-should-fail.stderr")
         wait_until_terminated_or_succeeds(
@@ -136,7 +136,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
     def test_glxgears_prints_renderer():
         machine.execute(
             # Note trailing & for backgrounding.
-            "(env DISPLAY=:0 glxgears -info | tee /tmp/glxgears.stdout) 3>&1 1>&2 2>&3 | tee /tmp/glxgears.stderr &"
+            "(env DISPLAY=:0 glxgears -info | tee /tmp/glxgears.stdout) 3>&1 1>&2 2>&3 | tee /tmp/glxgears.stderr >&2 &"
         )
         machine.wait_until_succeeds("test -f /tmp/glxgears.stderr")
         wait_until_terminated_or_succeeds(