Create exclusion list of integration tests and cover easiest cases

.github/workflows/mononoke-integration_mac.yml

@@ -71,7 +71,7 @@ jobs:
         pip install click
     - name: Install Brew dependencies
       run: |
-        brew install curl-openssl jq
+        brew install bash coreutils curl-openssl gnu-sed grep jq
     - name: Check space
       run: df -h
     - name: Run Monononke integration tests

eden/mononoke/tests/integration/library.sh

@@ -25,6 +25,25 @@ REPONAME=repo
 COMMON_ARGS=(--skip-caching --mysql-master-only --tunables-config "file:$TESTTMP/mononoke_tunables.json")
 TEST_CERTDIR="${HGTEST_CERTDIR:-"$TEST_CERTS"}"
 
+case "$(uname -s)" in
+  # Workarounds for running tests on MacOS
+  Darwin*)
+    # Use the brew installed versions of GNU utils
+    PATH="/usr/local/opt/gnu-sed/libexec/gnubin:\
+/usr/local/opt/grep/libexec/gnubin:\
+/usr/local/opt/coreutils/libexec/gnubin:\
+$PATH"
+  ;;
+esac
+
+function killandwait {
+  # sends KILL to the given process and waits for it so that nothing is printed
+  # to the terminal on MacOS
+  { kill -9 $1 && wait $1; } > /dev/null 2>&1
+  # We don't care for wait exit code
+  true
+}
+
 function get_free_socket {
 
 # From https://unix.stackexchange.com/questions/55913/whats-the-easiest-way-to-find-an-unused-local-port

eden/mononoke/tests/integration/run_tests_getdeps.py

@@ -71,18 +71,12 @@
     tests = list(tests)
 else:
     excluded_tests = {
-        "test-alias-verify.t",  # "wc -l" problem
-        "test-backfill-derived-data.t",  # "wc -l" problem
-        "test-backfill-git-mapping.t",  # "${ENV^^}" problem
         "test-backsync-forever.t",  # Unknown issue
         "test-backsyncer-merges.t",  # Missing BACKSYNCER
-        "test-blobimport-derived-data.t",  # "|&" problem
-        "test-blobimport-inline.t",  # Illegal byte sequence and "sed -i"
         "test-blobimport-lfs.t",  # Timed out
         "test-blobimport.t",  # Case insensitivity of paths in MacOS
         "test-blobstore_healer.t",  # PANIC not implemented in sql_ext
         "test-bookmarks-filler.t",  # Probably missing binary
-        "test-bundle2-preservation.t",  # "wc -l" problem
         "test-cmd-manual-scrub.t",  # Just wrong outout
         "test-cross-repo-commit-sync-live.t",  # Unknown issue
         "test-cross-repo-commit-sync-merge.t",  # requires FB-specific command
@@ -93,15 +87,10 @@
         "test-edenapi-server-history.t",  # Missing eden/scm's commands
         "test-edenapi-server-trees.t",  # Missing eden/scm's commands
         "test-fastreplay-inline-args.t",  # Returns different data in OSS
-        "test-fastreplay-remote-args.t",  # "grep -A" problem
-        "test-gettreepack-designated-nodes.t",  # "wc -l" problem
-        "test-gettreepack.t",  # "wc -l" problem
         "test-gitimport-octopus.t",  # Missing MONONOKE_REGENERATE_HG_FILENODES
         "test-gitimport.t",  # Issue with hggit extension
         "test-hook-tailer.t",  # Issue with hggit extension
         "test-hooks.t",  # Hooks are not in OSS yet
-        "test-inconsistent-hash.t",  # Illegal byte sequence and "sed -i"
-        "test-infinitepush-hydrated.t",  # "|&" problem
         "test-infinitepush-lfs.t",  # Timed out
         "test-large-path-and-content.t",  # Complex bash issues
         "test-lfs-copytracing.t",  # Timed out
@@ -119,7 +108,6 @@
         "test-lfs-server-scuba-logging.t",  # Timed out
         "test-lfs-server.t",  # Timed out
         "test-lfs-to-mononoke.t",  # Timed out
-        "test-lfs-upload-alias-on-fetch.t",  # "wc -l" problem
         "test-lfs-wantslfspointers.t",  # Timed out
         "test-lfs.t",  # Timed out
         "test-megarepo-tool.t",  # Missing MONONOKE_HG_SYNC
@@ -133,16 +121,12 @@
         "test-mononoke-hg-sync-job-sync-globalrevs.t",  # Missing MONONOKE_HG_SYNC
         "test-mononoke-hg-sync-job-with-copies.t",  # Missing MONONOKE_HG_SYNC
         "test-mononoke-hg-sync-job.t",  # Missing MONONOKE_HG_SYNC
-        "test-multiplexed-chaosblob.t",  # "wc -l" problem
-        "test-multiplexed-packblob.t",  # "du --bytes" problem
         "test-push-protocol-lfs.t",  # Timed out
         "test-push-redirector-pushrebase-hooks.t",  # Hooks are not in OSS yet
         "test-push-redirector-pushrebase-onesided.t",  # Missing MONONOKE_X_REPO_SYNC
-        "test-push-redirector-pushrebase.t",  # "[[ -v ... ]]" in library-push-redirector.sh and "ls" problem
         "test-push-redirector-sync-job.t",  # Missing BACKSYNCER
         "test-pushrebase-block-casefolding.t",  # Most likely MacOS path case insensitivity
         "test-pushrebase-discovery.t",  # Hooks are not in OSS yet
-        "test-rechunker.t",  # "wc -l" problem
         "test-remotefilelog-lfs.t",  # Timed out
         "test-scs-blame.t",  # Missing SCS_SERVER
         "test-scs-common-base.t",  # Missing SCS_SERVER
@@ -154,20 +138,7 @@
         "test-scs.t",  # Missing SCS_SERVER
         "test-server.t",  # Returns different data in OSS
         "test-traffic-replay.t",  # Missing TRAFFIC_REPLAY
-        "test-unbundle-replay-hg-bookmark.t",  # "kill" printing problem
         "test-unbundle-replay-hg-recording.t",  # Returns different data in OSS
-        "test-walker-compression-benefit.t",  # strip_glog problem
-        "test-walker-corpus.t",  # "du --bytes" problem
-        "test-walker-count-objects.t",  # strip_glog problem
-        "test-walker-enable-derive.t",  # strip_glog problem
-        "test-walker-error-as-data.t",  # "sed -r" problem
-        "test-walker-sample-by-node.t",  # strip_glog problem
-        "test-walker-sample-by-path.t",  # strip_glog problem
-        "test-walker-scrub-blobstore.t",  # "sed -r" problem
-        "test-walker-scrub-queue.t",  # "sed -r" problem
-        "test-walker-throttle.t",  # "date" and arithmetic problem
-        "test-walker-validate-non-public.t",  # strip_glog problem
-        "test-walker-validate.t",  # strip_glog problem
     }
 
     tests = [

eden/mononoke/tests/integration/test-blobimport-inline.t

@@ -29,7 +29,7 @@
   $ for (( i=0; i < $lines_cnt; i++ ))
   > do
   >  LINE_LENGTH=$(random_int $max_line_length)
-  >  echo $(head -c 10000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $LINE_LENGTH 2>/dev/null | head -n 1) >> file
+  >  echo $(head -c 10000 /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w $LINE_LENGTH 2>/dev/null | head -n 1) >> file
   > done
 
   $ hg ci -Aqm "commit"$c
@@ -41,7 +41,7 @@
   >   do
   >     LINE_LENGTH=$(random_int $max_line_length)
   >     LINE_NUMBER=$(random_int $lines_cnt)
-  >     CONTENT=$(head -c 10000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $LINE_LENGTH 2>/dev/null | head -n 1)
+  >     CONTENT=$(head -c 10000 /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w $LINE_LENGTH 2>/dev/null | head -n 1)
   >     sed -i "$LINE_NUMBER""s/.*/$CONTENT/" file
   >   done
   >   hg ci -Aqm "commit"$c

eden/mononoke/tests/integration/test-bundle2-preservation.t

@@ -60,7 +60,7 @@ Do a push, while bundle preservation is disabled
   [1]
 
 Restart mononoke with enabled bundle2 preservation
-  $ kill $MONONOKE_PID
+  $ killandwait $MONONOKE_PID
   $ rm -rf $TESTTMP/mononoke-config
   $ export ENABLE_PRESERVE_BUNDLE2=1
   $ setup_common_config "blob_files"

eden/mononoke/tests/integration/test-cache-warmup-microwave.t

@@ -20,7 +20,7 @@ Check that Mononoke booted despite the lack of microwave snapshot
 
 Kill Mononoke
 
-  $ kill "$MONONOKE_PID"
+  $ killandwait "$MONONOKE_PID"
   $ truncate -s 0 "$TESTTMP/mononoke.out"
 
 Delete filenodes

eden/mononoke/tests/integration/test-inconsistent-hash.t

@@ -53,7 +53,7 @@
   $ PACK_TO_CORRUPT=".hg/store/packs/dee3d9750ad87ede865d69e20330c34e51ec83d5.datapack"
 # change access to file, as it is readonly
   $ chmod 666 "$PACK_TO_CORRUPT"
-  $ sed -i s/hello_world/aaaaaaaaaaa/ "$PACK_TO_CORRUPT"
+  $ LC_ALL=C sed -i s/hello_world/aaaaaaaaaaa/ "$PACK_TO_CORRUPT"
 
 Do a push, but disable cache verification on the client side, otherwise
 filenode won't be send at all

eden/mononoke/tests/integration/test-infinitepush-hydrated.t

@@ -178,7 +178,7 @@ check unhydrated infinitepush pulls
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
 stop mononoke before running it with a different config
-  $ kill "$MONONOKE_PID"
+  $ killandwait "$MONONOKE_PID"
   $ rm -rf "$TESTTMP/mononoke-config"
 
 setup a new config and restart mononoke

eden/mononoke/tests/integration/test-infinitepush-vs-push.t

@@ -58,7 +58,7 @@ Do infinitepush (aka commit cloud) push
   backing up stack rooted at * (glob)
   commitcloud: backed up 1 commit
 
-  $ kill $MONONOKE_PID && wait $MONONOKE_PID
+  $ killandwait $MONONOKE_PID
 
 At least once infinitepush was performed
   $ jq '.normal | contains({log_tag: "Unbundle resolved", msg: "infinitepush"})' < "$SCUBA_LOGGING_PATH" | grep true

eden/mononoke/tests/integration/test-lfs-server-proxy-skip-upstream.t

@@ -77,4 +77,4 @@
   $ truncate -s 0 "$scuba_proxy"
 
 # Kill nc, otherwise we don't exit properly :/
-  $ kill -KILL "$nc_pid"
+  $ killandwait "$nc_pid"

eden/mononoke/tests/integration/test-redaction-config.t

@@ -89,7 +89,7 @@ Censor the redacted blob (file 'b' in commit '14961831bd3af3a6331fef7e63367d61cb
   * changeset resolved as: * (glob)
 
 Restart mononoke
-  $ kill $MONONOKE_PID
+  $ killandwait $MONONOKE_PID
   $ rm -rf "$TESTTMP/mononoke-config"
   $ setup_common_config blob_files
   $ mononoke
@@ -110,7 +110,7 @@ Should gives us the tombstone file since it is redacted
   This version of the file is redacted and you are not allowed to access it. Update or rebase to a newer commit.
 
 Restart mononoke and disable redaction verification
-  $ kill $MONONOKE_PID
+  $ killandwait $MONONOKE_PID
   $ rm -rf "$TESTTMP/mononoke-config"
   $ export REDACTION_DISABLED=1
   $ setup_common_config blob_files

eden/mononoke/tests/integration/test-redaction.t

@@ -168,7 +168,7 @@ Censore the redacted blob (file 'c' in commit '064d994d0240f9738dba1ef7479f0a4ce
   * changeset resolved as: * (glob)
 
 Restart mononoke
-  $ kill $MONONOKE_PID
+  $ killandwait $MONONOKE_PID
   $ rm -rf "$TESTTMP/mononoke-config"
   $ setup_common_config blob_files
   $ mononoke

eden/mononoke/tests/integration/test-server.t

@@ -44,9 +44,7 @@ test TLS Session/Ticket resumption when using client certs
   SUCCESS
 
 test TLS Tickets use encryption keys from seeds - sessions should persist across restarts
-  $ kill -9 $MONONOKE_PID && wait $MONONOKE_PID
-  $TESTTMP.sh: * Killed * (glob)
-  [137]
+  $ killandwait $MONONOKE_PID
   $ mononoke
   $ wait_for_mononoke
   $ echo -e "hello\n" | s_client -sess_in $TMPFILE -state | grep -E "^SSL_connect"

eden/mononoke/tests/integration/test-unbundle-replay-hg-bookmark.t

@@ -66,7 +66,7 @@ Export the bundles so we can replay it as it if were coming from hg, through the
 
 Blow everything away: we're going to re-do the push from scratch, in a new repo.
 
-  $ kill -9 "$MONONOKE_PID"
+  $ killandwait "$MONONOKE_PID"
   $ rm -rf "$TESTTMP/mononoke-config" "$TESTTMP/monsql" "$TESTTMP/blobstore"
   $ ASSIGN_GLOBALREVS=1 BLOB_TYPE="blob_files" quiet default_setup
 

eden/mononoke/tests/integration/test-unbundle-replay-hg-recording.t

@@ -53,7 +53,7 @@ Export the bundle so we can replay it as it if were coming from hg, through the
 
 Blow everything away: we're going to re-do the push from scratch, in a new repo.
 
-  $ kill -9 "$MONONOKE_PID"
+  $ killandwait "$MONONOKE_PID"
   $ rm -rf "$TESTTMP/mononoke-config" "$TESTTMP/monsql" "$TESTTMP/blobstore"
   $ ASSIGN_GLOBALREVS=1 BLOB_TYPE="blob_files" quiet default_setup
 

eden/mononoke/tests/integration/test-walker-throttle.t

@@ -30,15 +30,15 @@ Base case, check can walk fine
   Walked* (glob)
 
 Check reads throttle
-  $ START_SECS=$(/usr/bin/date "+%s")
+  $ START_SECS=$(date "+%s")
   $ mononoke_walker --storage-id=blobstore --readonly-storage --cachelib-only-blobstore --blobstore-read-qps=5 scrub -I deep -q --bookmark master_bookmark 2>&1 | strip_glog
   Walking roots * (glob)
   Walking edge types * (glob)
   Walking node types * (glob)
   Final count: (40, 40)
   Bytes/s,* (glob)
   Walked* (glob)
-  $ END_SECS=$(/usr/bin/date "+%s")
+  $ END_SECS=$(date "+%s")
   $ ELAPSED_SECS=$(( "$END_SECS" - "$START_SECS" ))
   $ if [[ "$ELAPSED_SECS" -ge 4 ]]; then echo Took Long Enough Read; else echo "Too short: $ELAPSED_SECS"; fi
   Took Long Enough Read
@@ -49,7 +49,7 @@ Delete all data from one side of the multiplex
   $ rm blobstore/0/blobs/*
 
 Check writes throttle in Repair mode
-  $ START_SECS=$(/usr/bin/date "+%s")
+  $ START_SECS=$(date "+%s")
   $ mononoke_walker --storage-id=blobstore --readonly-storage --cachelib-only-blobstore --blobstore-write-qps=5 scrub --scrub-blobstore-action=Repair -I deep -q --bookmark master_bookmark 2>&1 | strip_glog | sed -re 's/^(scrub: blobstore_id BlobstoreId.0. repaired for repo0000.).*/\1/' | uniq -c | sed 's/^ *//'
   1 Walking roots * (glob)
   1 Walking edge types * (glob)
@@ -58,7 +58,7 @@ Check writes throttle in Repair mode
   1 Final count: (40, 40)
   1 Bytes/s,* (glob)
   1 Walked* (glob)
-  $ END_SECS=$(/usr/bin/date "+%s")
+  $ END_SECS=$(date "+%s")
   $ ELAPSED_SECS=$(( "$END_SECS" - "$START_SECS" ))
   $ if [[ "$ELAPSED_SECS" -ge 4 ]]; then echo Took Long Enough Repair; else echo "Too short: $ELAPSED_SECS"; fi
   Took Long Enough Repair