WIP test macOS ci flakiness

.github/workflows/ci-macvim.yaml

@@ -55,9 +55,9 @@ jobs:
         include:
           # Oldest version of Xcode supported on GitHub Action to test source code backwards compatibility
           # This version of Xcode can run on macOS 11 where as 13.3 has to run on macOS 12.
-          - os: macos-12
-            xcode: '13.2.1'
-            extra: [vimtags, check-xcodeproj-compat]
+#          - os: macos-12
+#            xcode: '13.2.1'
+#            extra: [vimtags, check-xcodeproj-compat]
 
           # Older version of Xcode, and used to publish legacy builds (for macOS 10.9 - 10.12)
           - os: macos-12
@@ -66,9 +66,9 @@ jobs:
             legacy: true
             publish_postfix: '_10.9'
 
-          - os: macos-13
+          - os: macos-14
             xcode: '15.2'
-            optimized: true
+            optimized: false
 
           # Most up to date OS and Xcode. Used to publish release for the main build.
           # This runner uses Apple Silicon.
@@ -345,10 +345,10 @@ jobs:
             echo 'MacVim_xcode8.xcodeproj is outdated. Run "make -C src macvim-xcodeproj-compat" to re-generate it.'; false
           fi
 
-      - name: Test MacVim
-        timeout-minutes: 10
-        run: |
-          make ${MAKE_BUILD_ARGS} -C src macvim-tests
+#      - name: Test MacVim
+#        timeout-minutes: 10
+#        run: |
+#          make ${MAKE_BUILD_ARGS} -C src macvim-tests
 
       - name: Build Vim test binaries
         run: |
@@ -363,16 +363,21 @@ jobs:
       - name: Test Vim
         timeout-minutes: 25
         run: make ${MAKE_BUILD_ARGS} test
+        #run: (cd src; TEST_FILTER=Test_terminal_aucmd_on_close TEST_NO_RETRY=yes make test_terminal)
+
+      - name: Print timing info
+        if: ${{ !cancelled() }}
+        run: cat src/testdir/ycc*
 
       - name: Upload failed test files
         if: ${{ !cancelled() && failure() }}
         uses: ./.github/actions/screendump
 
-      - name: Test Vim (GUI)
-        timeout-minutes: 25
-        run: |
-          make ${MAKE_BUILD_ARGS} -C src/testdir clean
-          make ${MAKE_BUILD_ARGS} -C src testgui
+#      - name: Test Vim (GUI)
+#        timeout-minutes: 25
+#        run: |
+#          make ${MAKE_BUILD_ARGS} -C src/testdir clean
+#          make ${MAKE_BUILD_ARGS} -C src testgui
 
       # Creates a DMG package of MacVim. Note that this doesn't create a GitHub release for us, because we would prefer to
       # do it manually, for two reasons: 1) signing / notarization are currently done out of CI, 2) we want to manually

src/testdir/shared.vim

@@ -180,6 +180,9 @@ func s:WaitForCommon(expr, assert, timeout)
     let start = reltime()
   endif
 
+  let numtimes = 0
+  let sleepdurations=[]
+
   while 1
     if type(a:expr) == v:t_func
       let success = a:expr()
@@ -189,6 +192,8 @@ func s:WaitForCommon(expr, assert, timeout)
       let success = eval(a:expr)
     endif
     if success
+      "call writefile(['WaitForCommon: slept for ' . slept], 'ycc_waittime.txt')
+      let g:ycc_waittime=['WaitForCommon: slept for ' . slept]
       return slept
     endif
 
@@ -201,12 +206,15 @@ func s:WaitForCommon(expr, assert, timeout)
     endif
 
     sleep 10m
+    let numtimes+=1
     if exists('*reltimefloat')
       let slept = float2nr(reltimefloat(reltime(start)) * 1000)
+      let sleepdurations+=[slept]
     else
       let slept += 10
     endif
   endwhile
+  let g:ycc_waittime=['WaitForCommon: slept for ' . slept . ' for times: ' . numtimes, 'WaitForCommon: sleep durations ' . string(sleepdurations)]
 
   return -1  " timed out
 endfunc

src/testdir/test_short_sleep.py

@@ -8,4 +8,10 @@
 
 if __name__ == "__main__":
 
+    f = open('ycc_pythontime.txt', 'a')
+    t1 = time.time()
     time.sleep(0.1)  # sleep 100 msec
+    t2 = time.time()
+    f.write("test_short_sleep.py time elapsed: " + str(t2-t1) + "\n")
+    f.flush()
+    f.close()

src/testdir/test_terminal.vim

@@ -1203,13 +1203,15 @@ func Test_terminal_aucmd_on_close()
   aug END
 
   let [cmd, waittime] = s:get_sleep_cmd()
+  call writefile(['Test_terminal_aucmd_on_close: cmd, waittime: ' . cmd . ',' . waittime], 'ycc_aucmd.txt')
 
   call assert_equal(1, winnr('$'))
   new
   call setline(1, ['one', 'two'])
   exe 'term ++close ' . cmd
   wincmd p
   call WaitForAssert({-> assert_equal(2, winnr('$'))}, waittime)
+  call writefile(g:ycc_waittime, 'ycc_waittime.txt')
   call assert_equal(1, s:called)
   bwipe!