From e06975769833300d92453610a011ecc8441e52fa Mon Sep 17 00:00:00 2001 From: Alex Hornby Date: Fri, 20 Sep 2024 15:35:26 -0700 Subject: [PATCH] restore mononoke getdeps integration tests Summary: X-link: https://github.com/facebookincubator/zstrong/pull/995 Bring back mononoke getdeps integration tests. This is a refesh of the previously working export-D34186407 branch from 2022 Main changes since: * depend on restored sapling manifest from previous commit * bring back selected manifests needed by mononoke that were deleted in D51869247. I added the sqlite binary packages from this to the sqlite3 manifest rather than bring back the sqllite3-bin manifest * add manifests for new tools used: ripgrep, git-lfs, and zstd cli (we already had the zstd libs) * fix a few test expecations that where too closely tied to git cli or TLS version etc (ubuntu 22.04 is on older version) * getdeps MakefileBuilder.run_tests() improvements * fix error status reporting, was not failing if tests failed * pass of --num-jobs to tests to stop it OOMing my machine * pass of --filter to tests so can iterate on one test more easily Can to iterate local execution for one test with: ``` python3 ./build/fbcode_builder/getdeps.py test --num-jobs 4 --allow-system-packages --no-facebook-internal --src-dir=. mononoke_integration --retry 0 --filter server/test-gettreepack.t ``` X-link: https://github.com/facebook/sapling/pull/951 Reviewed By: quark-zju Differential Revision: D62978526 Pulled By: ahornby fbshipit-source-id: 0070a67d798bb23ee9e78e1a5149ba5364d548c9 --- build/fbcode_builder/getdeps.py | 6 ++- build/fbcode_builder/getdeps/builder.py | 17 +++++- build/fbcode_builder/manifests/git-lfs | 15 ++++++ build/fbcode_builder/manifests/jq | 25 +++++++++ .../manifests/mononoke_integration | 53 +++++++++++++++++++ build/fbcode_builder/manifests/nmap | 30 +++++++++++ build/fbcode_builder/manifests/python-click | 9 ++++ build/fbcode_builder/manifests/ripgrep | 15 ++++++ build/fbcode_builder/manifests/sqlite3 | 2 + build/fbcode_builder/manifests/tree | 37 +++++++++++++ build/fbcode_builder/manifests/zstd | 1 + 11 files changed, 207 insertions(+), 3 deletions(-) create mode 100644 build/fbcode_builder/manifests/git-lfs create mode 100644 build/fbcode_builder/manifests/jq create mode 100644 build/fbcode_builder/manifests/mononoke_integration create mode 100644 build/fbcode_builder/manifests/nmap create mode 100644 build/fbcode_builder/manifests/python-click create mode 100644 build/fbcode_builder/manifests/ripgrep create mode 100644 build/fbcode_builder/manifests/tree diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index c8b038024..037a695d2 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -1193,9 +1193,13 @@ def write_job_for_platform(self, platform, args): # noqa: C901 and manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off" ): + num_jobs_arg = "" + if args.num_jobs: + num_jobs_arg = f"--num-jobs {args.num_jobs} " + out.write(" - name: Test %s\n" % manifest.name) out.write( - f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" + f" run: {getdepscmd}{allow_sys_arg} test {num_jobs_arg}--src-dir=. {manifest.name} {project_prefix}\n" ) if build_opts.free_up_disk and not build_opts.is_windows(): out.write(" - name: Show disk space at end\n") diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 5a5ea2303..6ad9f2859 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -315,9 +315,22 @@ def run_tests(self, schedule_type, owner, test_filter, retry, no_testpilot) -> N return env = self._compute_env() + if test_filter: + env["GETDEPS_TEST_FILTER"] = test_filter + else: + env["GETDEPS_TEST_FILTER"] = "" - cmd = [self._make_binary] + self.test_args + self._get_prefix() - self._run_cmd(cmd, env=env) + if retry: + env["GETDEPS_TEST_RETRY"] = retry + else: + env["GETDEPS_TEST_RETRY"] = 0 + + cmd = ( + [self._make_binary, "-j%s" % self.num_jobs] + + self.test_args + + self._get_prefix() + ) + self._run_cmd(cmd, allow_fail=False, env=env) class CMakeBootStrapBuilder(MakeBuilder): diff --git a/build/fbcode_builder/manifests/git-lfs b/build/fbcode_builder/manifests/git-lfs new file mode 100644 index 000000000..19b24e247 --- /dev/null +++ b/build/fbcode_builder/manifests/git-lfs @@ -0,0 +1,15 @@ +[manifest] +name = git-lfs + +[rpms] +git-lfs + +[debs] +git-lfs + +[homebrew] +git-lfs + +# only used from system packages currently +[build] +builder = nop diff --git a/build/fbcode_builder/manifests/jq b/build/fbcode_builder/manifests/jq new file mode 100644 index 000000000..354854b2c --- /dev/null +++ b/build/fbcode_builder/manifests/jq @@ -0,0 +1,25 @@ +[manifest] +name = jq + +[rpms.distro=fedora] +jq + +[homebrew] +jq + +[download.not(os=windows)] +# we use jq-1.7+ to get fix for number truncation https://github.com/jqlang/jq/pull/1752 +url = https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-1.7.1.tar.gz +sha256 = 478c9ca129fd2e3443fe27314b455e211e0d8c60bc8ff7df703873deeee580c2 + +[build.not(os=windows)] +builder = autoconf +subdir = jq-1.7.1 + +[build.os=windows] +builder = nop + +[autoconf.args] +# This argument turns off some developers tool and it is recommended in jq's +# README +--disable-maintainer-mode diff --git a/build/fbcode_builder/manifests/mononoke_integration b/build/fbcode_builder/manifests/mononoke_integration new file mode 100644 index 000000000..e1f171403 --- /dev/null +++ b/build/fbcode_builder/manifests/mononoke_integration @@ -0,0 +1,53 @@ +[manifest] +name = mononoke_integration +fbsource_path = fbcode/eden +shipit_project = eden +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebook/sapling.git + +[build.not(os=windows)] +builder = make +subdir = eden/mononoke/tests/integration + +[build.os=windows] +# building Mononoke on windows is not supported +builder = nop + +[make.build_args] +build-getdeps + +[make.install_args] +install-getdeps + +[make.test_args] +test-getdeps + +[shipit.pathmap] +fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration + +[shipit.strip] +^.*/facebook/.*$ +^.*/fb/.*$ + +[dependencies] +git-lfs +jq +mononoke +nmap +python +python-click +ripgrep +sapling +tree +zstd + +[dependencies.os=linux] +sqlite3 + +[dependencies.os=darwin] +gnu-bash +gnu-coreutils +gnu-grep +gnu-sed diff --git a/build/fbcode_builder/manifests/nmap b/build/fbcode_builder/manifests/nmap new file mode 100644 index 000000000..3e935177b --- /dev/null +++ b/build/fbcode_builder/manifests/nmap @@ -0,0 +1,30 @@ +[manifest] +name = nmap + +[rpms] +nmap +nmap-ncat + +[debs] +nmap + +# 18.04 combines ncat into the nmap package, newer need the separate one +[debs.not(all(distro=ubuntu,distro_vers="18.04"))] +ncat + +[download.not(os=windows)] +url = https://api.github.com/repos/nmap/nmap/tarball/ef8213a36c2e89233c806753a57b5cd473605408 +sha256 = eda39e5a8ef4964fac7db16abf91cc11ff568eac0fa2d680b0bfa33b0ed71f4a + +[build.not(os=windows)] +builder = autoconf +subdir = nmap-nmap-ef8213a +build_in_src_dir = true + +[build.os=windows] +builder = nop + +[autoconf.args] +# Without this option the build was filing to find some third party libraries +# that we don't need +enable_rdma=no diff --git a/build/fbcode_builder/manifests/python-click b/build/fbcode_builder/manifests/python-click new file mode 100644 index 000000000..ea9a9d2d3 --- /dev/null +++ b/build/fbcode_builder/manifests/python-click @@ -0,0 +1,9 @@ +[manifest] +name = python-click + +[download] +url = https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl +sha256 = dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc + +[build] +builder = python-wheel diff --git a/build/fbcode_builder/manifests/ripgrep b/build/fbcode_builder/manifests/ripgrep new file mode 100644 index 000000000..140a4e8af --- /dev/null +++ b/build/fbcode_builder/manifests/ripgrep @@ -0,0 +1,15 @@ +[manifest] +name = ripgrep + +[rpms] +ripgrep + +[debs] +ripgrep + +[homebrew] +ripgrep + +# only used from system packages currently +[build] +builder = nop diff --git a/build/fbcode_builder/manifests/sqlite3 b/build/fbcode_builder/manifests/sqlite3 index 1966f0fab..5a983f8aa 100644 --- a/build/fbcode_builder/manifests/sqlite3 +++ b/build/fbcode_builder/manifests/sqlite3 @@ -3,6 +3,7 @@ name = sqlite3 [debs] libsqlite3-dev +sqlite3 [homebrew] sqlite @@ -10,6 +11,7 @@ sqlite [rpms] sqlite-devel sqlite-libs +sqlite [pps] sqlite3 diff --git a/build/fbcode_builder/manifests/tree b/build/fbcode_builder/manifests/tree new file mode 100644 index 000000000..ccd0180a7 --- /dev/null +++ b/build/fbcode_builder/manifests/tree @@ -0,0 +1,37 @@ +[manifest] +name = tree + +[debs] +tree + +[homebrew] +tree + +[rpms] +tree + +[download.os=linux] +url = https://salsa.debian.org/debian/tree-packaging/-/archive/debian/1.8.0-1/tree-packaging-debian-1.8.0-1.tar.gz +sha256 = a841eee1d52bfd64a48f54caab9937b9bd92935055c48885c4ab1ae4dab7fae5 + +[download.os=darwin] +# The official package of tree source requires users of non-Linux platform to +# comment/uncomment certain lines in the Makefile to build for their platform. +# Besauce getdeps.py doesn't have that functionality we just use this custom +# fork of tree which has proper lines uncommented for a OSX build +url = https://github.com/lukaspiatkowski/tree-command/archive/debian/1.8.0-1-macos.tar.gz +sha256 = 9cbe889553d95cf5a2791dd0743795d46a3c092c5bba691769c0e5c52e11229e + +[build.os=linux] +builder = make +subdir = tree-packaging-debian-1.8.0-1 + +[build.os=darwin] +builder = make +subdir = tree-command-debian-1.8.0-1-macos + +[build.os=windows] +builder = nop + +[make.install_args] +install diff --git a/build/fbcode_builder/manifests/zstd b/build/fbcode_builder/manifests/zstd index aac189fb8..1bd087538 100644 --- a/build/fbcode_builder/manifests/zstd +++ b/build/fbcode_builder/manifests/zstd @@ -7,6 +7,7 @@ zstd # 18.04 zstd is too old [debs.not(all(distro=ubuntu,distro_vers="18.04"))] libzstd-dev +zstd [rpms] libzstd-devel