From 95d23c757c0436d3a8d9146d60112f2eaf965b96 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 25 Jan 2023 23:40:04 +0100 Subject: [PATCH] feat: mechanism compile and embed tor (#1052) This diff introduces a mechanism to compile and embed tor. We plan on using this functionality on Android and iOS but we have not finished working on updating the build yet. In the meanwhile, here's an interim result. We're committing the code to run unit and integration testing of this new functionality on linux/amd64. The reference issue is https://github.com/ooni/probe/issues/2365. The Go based build was sketched out in https://github.com/ooni/probe/issues/2401. --- .github/workflows/libtorlinux.yml | 29 ++ CDEPS/README.md | 3 + CDEPS/libevent/000.patch | 16 + CDEPS/libevent/001.patch | 15 + CDEPS/libevent/002.patch | 15 + CDEPS/openssl/000.patch | 15 + CDEPS/openssl/001.patch | 14 + CDEPS/tor/000.patch | 15 + CDEPS/tor/001.patch | 15 + CDEPS/tor/002.patch | 15 + CDEPS/zlib/000.patch | 15 + MONOREPO/w/build-android-with-cli.bash | 3 +- internal/cmd/buildtool/cdeps.go | 155 +++++++ internal/cmd/buildtool/cdepslibevent.go | 69 ++++ internal/cmd/buildtool/cdepsopenssl.go | 59 +++ internal/cmd/buildtool/cdepstor.go | 66 +++ internal/cmd/buildtool/cdepszlib.go | 51 +++ .../internal/buildtooltest/buildtooltest.go | 2 +- internal/cmd/buildtool/linux.go | 1 + internal/cmd/buildtool/linuxcdeps.go | 64 +++ internal/cmd/buildtool/linuxcdeps_test.go | 383 ++++++++++++++++++ internal/libtor/android/386/.gitignore | 1 + internal/libtor/android/amd64/.gitignore | 1 + internal/libtor/android/arm/.gitignore | 1 + internal/libtor/android/arm64/.gitignore | 1 + internal/libtor/enabled.go | 277 +++++++++++++ internal/libtor/enabled_test.go | 255 ++++++++++++ internal/libtor/fallback.go | 10 + internal/libtor/linux/amd64/.gitignore | 1 + internal/libtor/testdata/.gitignore | 1 + internal/tunnel/tordesktop.go | 5 +- internal/tunnel/torembed.go | 30 ++ 32 files changed, 1598 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/libtorlinux.yml create mode 100644 CDEPS/README.md create mode 100644 CDEPS/libevent/000.patch create mode 100644 CDEPS/libevent/001.patch create mode 100644 CDEPS/libevent/002.patch create mode 100644 CDEPS/openssl/000.patch create mode 100644 CDEPS/openssl/001.patch create mode 100644 CDEPS/tor/000.patch create mode 100644 CDEPS/tor/001.patch create mode 100644 CDEPS/tor/002.patch create mode 100644 CDEPS/zlib/000.patch create mode 100644 internal/cmd/buildtool/cdeps.go create mode 100644 internal/cmd/buildtool/cdepslibevent.go create mode 100644 internal/cmd/buildtool/cdepsopenssl.go create mode 100644 internal/cmd/buildtool/cdepstor.go create mode 100644 internal/cmd/buildtool/cdepszlib.go create mode 100644 internal/cmd/buildtool/linuxcdeps.go create mode 100644 internal/cmd/buildtool/linuxcdeps_test.go create mode 100644 internal/libtor/android/386/.gitignore create mode 100644 internal/libtor/android/amd64/.gitignore create mode 100644 internal/libtor/android/arm/.gitignore create mode 100644 internal/libtor/android/arm64/.gitignore create mode 100644 internal/libtor/enabled.go create mode 100644 internal/libtor/enabled_test.go create mode 100644 internal/libtor/fallback.go create mode 100644 internal/libtor/linux/amd64/.gitignore create mode 100644 internal/libtor/testdata/.gitignore create mode 100644 internal/tunnel/torembed.go diff --git a/.github/workflows/libtorlinux.yml b/.github/workflows/libtorlinux.yml new file mode 100644 index 0000000000..9366c96f77 --- /dev/null +++ b/.github/workflows/libtorlinux.yml @@ -0,0 +1,29 @@ +# Runs tests for internal/libtor with -tags=ooni_libtor +name: libtorlinux +on: + pull_request: + push: + branches: + - "master" + - "release/**" + - "fullbuild" + +jobs: + test_ooni_libtor: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + - name: Get GOVERSION content + id: goversion + run: echo ::set-output name=version::$(cat GOVERSION) + + - uses: magnetikonline/action-golang-cache@v2 + with: + go-version: "${{ steps.goversion.outputs.version }}" + cache-key-suffix: "-libtorlinux-${{ steps.goversion.outputs.version }}" + + - run: go run ./internal/cmd/buildtool linux cdeps zlib openssl libevent tor + + - run: go test -count 1 -v -cover -tags ooni_libtor -race ./internal/libtor/... diff --git a/CDEPS/README.md b/CDEPS/README.md new file mode 100644 index 0000000000..e8286f9c5d --- /dev/null +++ b/CDEPS/README.md @@ -0,0 +1,3 @@ +# Code to build C dependencies + +Directory used to compile C dependencies. diff --git a/CDEPS/libevent/000.patch b/CDEPS/libevent/000.patch new file mode 100644 index 0000000000..752a9fdb13 --- /dev/null +++ b/CDEPS/libevent/000.patch @@ -0,0 +1,16 @@ +diff --git a/bufferevent_openssl.c b/bufferevent_openssl.c +index b51b834..06b219e 100644 +--- a/bufferevent_openssl.c ++++ b/bufferevent_openssl.c +@@ -67,6 +67,11 @@ + #include + #include "openssl-compat.h" + ++#include ++#ifndef OPENSSL_OONI ++#error "We're not including the correct openssl/opensslv.h file" ++#endif ++ + /* + * Define an OpenSSL bio that targets a bufferevent. + */ diff --git a/CDEPS/libevent/001.patch b/CDEPS/libevent/001.patch new file mode 100644 index 0000000000..b27287bb95 --- /dev/null +++ b/CDEPS/libevent/001.patch @@ -0,0 +1,15 @@ +diff --git a/test/regress_zlib.c b/test/regress_zlib.c +index 5fe7749..558155c 100644 +--- a/test/regress_zlib.c ++++ b/test/regress_zlib.c +@@ -80,6 +80,10 @@ + + #include + ++#ifndef ZLIB_OONI ++#error "We're not including the correct zlib.h file" ++#endif ++ + static int infilter_calls; + static int outfilter_calls; + static int readcb_finished; diff --git a/CDEPS/libevent/002.patch b/CDEPS/libevent/002.patch new file mode 100644 index 0000000000..29e17ff362 --- /dev/null +++ b/CDEPS/libevent/002.patch @@ -0,0 +1,15 @@ +diff --git a/include/event2/event.h b/include/event2/event.h +index a6b6144..6abb474 100644 +--- a/include/event2/event.h ++++ b/include/event2/event.h +@@ -1665,6 +1665,10 @@ int event_base_update_cache_time(struct event_base *base); + EVENT2_EXPORT_SYMBOL + void libevent_global_shutdown(void); + ++/* EVENT_OONI is used by dependencies to ensure they are using the ++ correct event.h header and not some other header. */ ++#define EVENT_OONI 1 ++ + #ifdef __cplusplus + } + #endif diff --git a/CDEPS/openssl/000.patch b/CDEPS/openssl/000.patch new file mode 100644 index 0000000000..e313ac356a --- /dev/null +++ b/CDEPS/openssl/000.patch @@ -0,0 +1,15 @@ +diff --git a/crypto/comp/c_zlib.c b/crypto/comp/c_zlib.c +index b819337..e479bd6 100644 +--- a/crypto/comp/c_zlib.c ++++ b/crypto/comp/c_zlib.c +@@ -34,6 +34,10 @@ static COMP_METHOD zlib_method_nozlib = { + + # include + ++#ifndef ZLIB_OONI ++# error "We're not including the correct zlib.h file" ++#endif ++ + static int zlib_stateful_init(COMP_CTX *ctx); + static void zlib_stateful_finish(COMP_CTX *ctx); + static int zlib_stateful_compress_block(COMP_CTX *ctx, unsigned char *out, diff --git a/CDEPS/openssl/001.patch b/CDEPS/openssl/001.patch new file mode 100644 index 0000000000..780faa59a1 --- /dev/null +++ b/CDEPS/openssl/001.patch @@ -0,0 +1,14 @@ +diff --git a/include/openssl/opensslv.h b/include/openssl/opensslv.h +index fd9400a..0d2c9dc 100644 +--- a/include/openssl/opensslv.h ++++ b/include/openssl/opensslv.h +@@ -94,6 +94,9 @@ extern "C" { + # define SHLIB_VERSION_HISTORY "" + # define SHLIB_VERSION_NUMBER "1.1" + ++/* OPENSSL_OONI is used by dependencies to ensure they are using the ++ correct OpenSSL headers and not some other headers. */ ++#define OPENSSL_OONI 1 + + #ifdef __cplusplus + } diff --git a/CDEPS/tor/000.patch b/CDEPS/tor/000.patch new file mode 100644 index 0000000000..a3fd2cd9dc --- /dev/null +++ b/CDEPS/tor/000.patch @@ -0,0 +1,15 @@ +diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c +index 77de2d6..ce46554 100644 +--- a/src/lib/tls/tortls_openssl.c ++++ b/src/lib/tls/tortls_openssl.c +@@ -45,6 +45,10 @@ DISABLE_GCC_WARNING("-Wredundant-decls") + #error "We require OpenSSL with ECC support" + #endif + ++#ifndef OPENSSL_OONI ++#error "We're not including the correct openssl/opensslv.h file" ++#endif ++ + #include + #include + #include diff --git a/CDEPS/tor/001.patch b/CDEPS/tor/001.patch new file mode 100644 index 0000000000..382f25440d --- /dev/null +++ b/CDEPS/tor/001.patch @@ -0,0 +1,15 @@ +diff --git a/src/lib/compress/compress_zlib.c b/src/lib/compress/compress_zlib.c +index 52f9509..fb7e39e 100644 +--- a/src/lib/compress/compress_zlib.c ++++ b/src/lib/compress/compress_zlib.c +@@ -45,6 +45,10 @@ + #error "We require zlib version 1.2 or later." + #endif + ++#ifndef ZLIB_OONI ++#error "We're not including the correct zlib.h file" ++#endif ++ + static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + diff --git a/CDEPS/tor/002.patch b/CDEPS/tor/002.patch new file mode 100644 index 0000000000..e0331e3356 --- /dev/null +++ b/CDEPS/tor/002.patch @@ -0,0 +1,15 @@ +diff --git a/src/lib/evloop/compat_libevent.c b/src/lib/evloop/compat_libevent.c +index fd840f8..2ec37e7 100644 +--- a/src/lib/evloop/compat_libevent.c ++++ b/src/lib/evloop/compat_libevent.c +@@ -19,6 +19,10 @@ + #include + #include + ++#ifndef EVENT_OONI ++#error "We're not including the correct event2/event.h file" ++#endif ++ + /** A string which, if it appears in a libevent log, should be ignored. */ + static const char *suppress_msg = NULL; + /** Callback function passed to event_set_log() so we can intercept diff --git a/CDEPS/zlib/000.patch b/CDEPS/zlib/000.patch new file mode 100644 index 0000000000..d8ddd3d7f7 --- /dev/null +++ b/CDEPS/zlib/000.patch @@ -0,0 +1,15 @@ +diff --git a/zlib.h b/zlib.h +index 953cb50..ec2e2f8 100644 +--- a/zlib.h ++++ b/zlib.h +@@ -1928,6 +1928,10 @@ ZEXTERN int ZEXPORTVA gzvprintf Z_ARG((gzFile file, + # endif + #endif + ++/* ZLIB_OONI is used by dependencies to ensure they are using the ++ correct zlib.h header and not some other header. */ ++#define ZLIB_OONI 1 ++ + #ifdef __cplusplus + } + #endif diff --git a/MONOREPO/w/build-android-with-cli.bash b/MONOREPO/w/build-android-with-cli.bash index cc51f646ea..a09d37e9e9 100755 --- a/MONOREPO/w/build-android-with-cli.bash +++ b/MONOREPO/w/build-android-with-cli.bash @@ -8,8 +8,7 @@ source $reporoot/MONOREPO/tools/libcore.bash ./MOBILE/android/newkeystore -./MOBILE/gomobile android ./pkg/oonimkall - +make ./MOBILE/android run cp -v MOBILE/android/oonimkall.aar ./MONOREPO/repo/probe-android/engine-experimental/ ( diff --git a/internal/cmd/buildtool/cdeps.go b/internal/cmd/buildtool/cdeps.go new file mode 100644 index 0000000000..8c25d11320 --- /dev/null +++ b/internal/cmd/buildtool/cdeps.go @@ -0,0 +1,155 @@ +package main + +// +// Building C dependencies: common code +// + +import ( + "os" + "path/filepath" + "sort" + "strings" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// cdepsEnv contains the environment for compiling a C dependency. +type cdepsEnv struct { + // cflags contains the CFLAGS to use when compiling. + cflags []string + + // configureHost is the value to pass to ./configure's --host option. + configureHost string + + // destdir is the directory where to install. + destdir string + + // lfdlags contains the LDFLAGS to use when compiling. + ldflags []string + + // openSSLAPIDefine is an extra define we need to add on Android. + openSSLAPIDefine string + + // openSSLCompiler is the compiler name for OpenSSL. + openSSLCompiler string +} + +// cdepsDependencies groups dependencies used when building cdeps. +type cdepsDependencies interface { + // mustChdir changes the current working directory and returns the + // function to return to the original working directory. + mustChdir(dirname string) func() + + // absoluteCurDir returns the absolute current directory. + absoluteCurDir() string + + // verifySHA256 verifies that the tarball has the given checksum. + verifySHA256(expectedSHA256, tarball string) +} + +// cdepsDependenciesStdlib are the [cdepsDependencies] used by default. +type cdepsDependenciesStdlib struct{} + +var _ cdepsDependencies = &cdepsDependenciesStdlib{} + +// absoluteCurDir implements cdepsDependencies +func (c *cdepsDependenciesStdlib) absoluteCurDir() string { + return cdepsMustAbsoluteCurdir() +} + +// verifySHA256 implements cdepsDependencies +func (c *cdepsDependenciesStdlib) verifySHA256(expectedSHA256, tarball string) { + cdepsMustVerifySHA256(expectedSHA256, tarball) +} + +// mustChdir implements cdepsDependencies +func (c *cdepsDependenciesStdlib) mustChdir(dirname string) func() { + return cdepsMustChdir(dirname) +} + +// cdepsAddCflags merges this struct's cflags with the extra cflags and +// then stores the merged cflags into the given envp. +func cdepsAddCflags(envp *shellx.Envp, c *cdepsEnv, extraCflags ...string) { + mergedCflags := append([]string{}, c.cflags...) + mergedCflags = append(mergedCflags, extraCflags...) + envp.Append("CFLAGS", strings.Join(mergedCflags, " ")) +} + +// cdepsAddLdflags merges this struct's ldflags with the extra ldflags and +// then stores the merged ldflags into the given envp. +func cdepsAddLdflags(envp *shellx.Envp, c *cdepsEnv, extraLdflags ...string) { + mergedLdflags := append([]string{}, c.ldflags...) + mergedLdflags = append(mergedLdflags, extraLdflags...) + envp.Append("LDFLAGS", strings.Join(mergedLdflags, " ")) +} + +// cdepsMustMkdirTemp creates a temporary directory. +func cdepsMustMkdirTemp() string { + return runtimex.Try1(os.MkdirTemp("", "")) +} + +// cdepsMustChdir changes the current directory to the given dir and +// returns a function to return to the original working dir. +func cdepsMustChdir(work string) func() { + prevdir := runtimex.Try1(os.Getwd()) + log.Infof("cd %s", work) + runtimex.Try0(os.Chdir(work)) + return func() { + runtimex.Try0(os.Chdir(prevdir)) + log.Infof("cd %s", prevdir) + } +} + +// cdepsMustFetch fetches the given URL using curl. +func cdepsMustFetch(URL string) { + must.Run(log.Log, "curl", "-fsSLO", URL) +} + +// cdepsMustVerifySHA256 verifies the SHA256 of the given tarball. +func cdepsMustVerifySHA256(expectedSHA256, tarball string) { + firstline := string(must.FirstLineBytes(must.RunOutput( + log.Log, "sha256sum", tarball, + ))) + sha256, _, good := strings.Cut(firstline, " ") + runtimex.Assert(good, "cannot obtain the first token") + runtimex.Assert(expectedSHA256 == sha256, "SHA256 mismatch") +} + +// cdepsMustAbsoluteCurdir returns the absolute path of the current dir. +func cdepsMustAbsoluteCurdir() string { + return runtimex.Try1(filepath.Abs(".")) +} + +// cdepsMustListPatches returns all the patches inside a dir. +func cdepsMustListPatches(dir string) (out []string) { + entries := runtimex.Try1(os.ReadDir(dir)) + for _, entry := range entries { + if !entry.Type().IsRegular() { + continue + } + if !strings.HasSuffix(entry.Name(), ".patch") { + continue + } + out = append(out, filepath.Join(dir, entry.Name())) + } + sort.Strings(out) + return +} + +// cdepsDefaultShellxConfig returns the default config used when calling shellx.RunEx. +func cdepsDefaultShellxConfig() *shellx.Config { + return &shellx.Config{ + Logger: log.Log, + Flags: shellx.FlagShowStdoutStderr, + } +} + +// cdepsMustRunWithDefaultConfig is a convenience wrapper +// around calling [shellx.RunEx] and checking the return value. +func cdepsMustRunWithDefaultConfig(envp *shellx.Envp, command string, args ...string) { + argv := runtimex.Try1(shellx.NewArgv(command, args...)) + runtimex.Try0(shellx.RunEx(cdepsDefaultShellxConfig(), argv, envp)) +} diff --git a/internal/cmd/buildtool/cdepslibevent.go b/internal/cmd/buildtool/cdepslibevent.go new file mode 100644 index 0000000000..a5424d8621 --- /dev/null +++ b/internal/cmd/buildtool/cdepslibevent.go @@ -0,0 +1,69 @@ +package main + +// +// Building C dependencies: libevent +// +// Adapted from https://github.com/guardianproject/tor-android +// SPDX-License-Identifier: BSD-3-Clause + +import ( + "path/filepath" + "runtime" + "strconv" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// cdepsLibeventBuildMain is the script that builds libevent. +func cdepsLibeventBuildMain(cdenv *cdepsEnv, deps cdepsDependencies) { + topdir := deps.absoluteCurDir() // must be mockable + work := cdepsMustMkdirTemp() + restore := cdepsMustChdir(work) + defer restore() + + // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/libevent.rb + cdepsMustFetch("https://github.com/libevent/libevent/archive/release-2.1.12-stable.tar.gz") + deps.verifySHA256( // must be mockable + "7180a979aaa7000e1264da484f712d403fcf7679b1e9212c4e3d09f5c93efc24", + "release-2.1.12-stable.tar.gz", + ) + must.Run(log.Log, "tar", "-xf", "release-2.1.12-stable.tar.gz") + _ = deps.mustChdir("libevent-release-2.1.12-stable") // must be mockable + + mydir := filepath.Join(topdir, "CDEPS", "libevent") + for _, patch := range cdepsMustListPatches(mydir) { + must.Run(log.Log, "git", "apply", patch) + } + + must.Run(log.Log, "./autogen.sh") + + envp := &shellx.Envp{} + cdepsAddCflags(envp, cdenv, "-I"+cdenv.destdir+"/include") + cdepsAddLdflags(envp, cdenv, "-L"+cdenv.destdir+"/lib") + + argv := runtimex.Try1(shellx.NewArgv("./configure")) + if cdenv.configureHost != "" { + argv.Append("--host=" + cdenv.configureHost) + } + argv.Append("--disable-libevent-regress", "--disable-samples", "--disable-shared", "--prefix=/") + runtimex.Try0(shellx.RunEx(cdepsDefaultShellxConfig(), argv, envp)) + + must.Run(log.Log, "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU())) + must.Run(log.Log, "make", "DESTDIR="+cdenv.destdir, "install") + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "bin")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "pkgconfig")) + + // we just need libevent.a + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent.la")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_core.a")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_core.la")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_extra.a")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_extra.la")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_openssl.a")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_openssl.la")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_pthreads.a")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_pthreads.la")) +} diff --git a/internal/cmd/buildtool/cdepsopenssl.go b/internal/cmd/buildtool/cdepsopenssl.go new file mode 100644 index 0000000000..ebb389e977 --- /dev/null +++ b/internal/cmd/buildtool/cdepsopenssl.go @@ -0,0 +1,59 @@ +package main + +// +// Building C dependencies: OpenSSL +// +// Adapted from https://github.com/guardianproject/tor-android +// SPDX-License-Identifier: BSD-3-Clause + +import ( + "path/filepath" + "runtime" + "strconv" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// cdepsOpenSSLBuildMain is the script that builds OpenSSL. +func cdepsOpenSSLBuildMain(cdenv *cdepsEnv, deps cdepsDependencies) { + topdir := deps.absoluteCurDir() // must be mockable + work := cdepsMustMkdirTemp() + restore := cdepsMustChdir(work) + defer restore() + + // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/openssl@1.1.rb + cdepsMustFetch("https://www.openssl.org/source/openssl-1.1.1s.tar.gz") + deps.verifySHA256( // must be mockable + "c5ac01e760ee6ff0dab61d6b2bbd30146724d063eb322180c6f18a6f74e4b6aa", + "openssl-1.1.1s.tar.gz", + ) + must.Run(log.Log, "tar", "-xf", "openssl-1.1.1s.tar.gz") + _ = deps.mustChdir("openssl-1.1.1s") // must be mockable + + mydir := filepath.Join(topdir, "CDEPS", "openssl") + for _, patch := range cdepsMustListPatches(mydir) { + must.Run(log.Log, "git", "apply", patch) + } + + envp := &shellx.Envp{} + cdepsAddCflags(envp, cdenv, "-Wno-macro-redefined") + argv := runtimex.Try1(shellx.NewArgv( + "./Configure", "no-comp", "no-dtls", "no-ec2m", "no-psk", "no-srp", + "no-ssl2", "no-ssl3", "no-camellia", "no-idea", "no-md2", "no-md4", + "no-mdc2", "no-rc2", "no-rc4", "no-rc5", "no-rmd160", "no-whirlpool", + "no-dso", "no-hw", "no-ui-console", "no-shared", "no-unit-test", + cdenv.openSSLCompiler, + )) + if cdenv.openSSLAPIDefine != "" { + argv.Append(cdenv.openSSLAPIDefine) + } + argv.Append("--libdir=lib", "--prefix=/", "--openssldir=/") + runtimex.Try0(shellx.RunEx(cdepsDefaultShellxConfig(), argv, envp)) + + must.Run(log.Log, "make", "-j", strconv.Itoa(runtime.NumCPU())) + must.Run(log.Log, "make", "DESTDIR="+cdenv.destdir, "install_dev") + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "pkgconfig")) +} diff --git a/internal/cmd/buildtool/cdepstor.go b/internal/cmd/buildtool/cdepstor.go new file mode 100644 index 0000000000..6608329cb2 --- /dev/null +++ b/internal/cmd/buildtool/cdepstor.go @@ -0,0 +1,66 @@ +package main + +// +// Building C dependencies: tor +// +// Adapted from https://github.com/guardianproject/tor-android +// SPDX-License-Identifier: BSD-3-Clause + +import ( + "path/filepath" + "runtime" + "strconv" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// cdepsTorBuildMain is the script that builds tor. +func cdepsTorBuildMain(cdenv *cdepsEnv, deps cdepsDependencies) { + topdir := deps.absoluteCurDir() // must be mockable + work := cdepsMustMkdirTemp() + restore := cdepsMustChdir(work) + defer restore() + + // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/tor.rb + cdepsMustFetch("https://www.torproject.org/dist/tor-0.4.7.12.tar.gz") + deps.verifySHA256( // must be mockable + "3b5d969712c467851bd028f314343ef15a97ea457191e93ffa97310b05b9e395", + "tor-0.4.7.12.tar.gz", + ) + must.Run(log.Log, "tar", "-xf", "tor-0.4.7.12.tar.gz") + _ = deps.mustChdir("tor-0.4.7.12") // must be mockable + + mydir := filepath.Join(topdir, "CDEPS", "tor") + for _, patch := range cdepsMustListPatches(mydir) { + must.Run(log.Log, "git", "apply", patch) + } + + envp := &shellx.Envp{} + cdepsAddCflags(envp, cdenv) + cdepsAddLdflags(envp, cdenv) + + argv := runtimex.Try1(shellx.NewArgv("./configure")) + if cdenv.configureHost != "" { + argv.Append("--host=" + cdenv.configureHost) + } + argv.Append( + "--enable-pic", + "--enable-static-libevent", "--with-libevent-dir="+cdenv.destdir, + "--enable-static-openssl", "--with-openssl-dir="+cdenv.destdir, + "--enable-static-zlib", "--with-zlib-dir="+cdenv.destdir, + "--disable-module-dirauth", + "--disable-zstd", "--disable-lzma", + "--disable-tool-name-check", + "--disable-systemd", + "--prefix=/", + ) + runtimex.Try0(shellx.RunEx(cdepsDefaultShellxConfig(), argv, envp)) + + must.Run(log.Log, "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU())) + + must.Run(log.Log, "install", "-m644", "src/feature/api/tor_api.h", cdenv.destdir+"/include") + must.Run(log.Log, "install", "-m644", "libtor.a", cdenv.destdir+"/lib") +} diff --git a/internal/cmd/buildtool/cdepszlib.go b/internal/cmd/buildtool/cdepszlib.go new file mode 100644 index 0000000000..3231f70aa4 --- /dev/null +++ b/internal/cmd/buildtool/cdepszlib.go @@ -0,0 +1,51 @@ +package main + +// +// Building C dependencies: zlib +// +// Adapted from https://github.com/guardianproject/tor-android +// SPDX-License-Identifier: BSD-3-Clause + +import ( + "path/filepath" + "runtime" + "strconv" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// cdepsZlibBuildMain is the script that builds zlib. +func cdepsZlibBuildMain(cdenv *cdepsEnv, deps cdepsDependencies) { + topdir := deps.absoluteCurDir() // must be mockable + work := cdepsMustMkdirTemp() + restore := cdepsMustChdir(work) + defer restore() + + // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/zlib.rb + cdepsMustFetch("https://zlib.net/zlib-1.2.13.tar.gz") + deps.verifySHA256( // must be mockable + "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", + "zlib-1.2.13.tar.gz", + ) + must.Run(log.Log, "tar", "-xf", "zlib-1.2.13.tar.gz") + _ = deps.mustChdir("zlib-1.2.13") // must be mockable + + mydir := filepath.Join(topdir, "CDEPS", "zlib") + for _, patch := range cdepsMustListPatches(mydir) { + must.Run(log.Log, "git", "apply", patch) + } + + envp := &shellx.Envp{} + if cdenv.configureHost != "" { + envp.Append("CHOST", cdenv.configureHost) // zlib's configure otherwise uses Apple's libtool + } + cdepsAddCflags(envp, cdenv) + cdepsMustRunWithDefaultConfig(envp, "./configure", "--prefix=/", "--static") + + must.Run(log.Log, "make", "-j", strconv.Itoa(runtime.NumCPU())) + must.Run(log.Log, "make", "DESTDIR="+cdenv.destdir, "install") + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "pkgconfig")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "share")) +} diff --git a/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go b/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go index 0037b178e1..a6e10f21d1 100644 --- a/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go +++ b/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go @@ -66,7 +66,7 @@ func CompareEnv(expected, got []string) error { case weExpected | weGot: // nothing case weGot: - issues = append(issues, fmt.Sprintf("* we got %s, which we don't expected", value)) + issues = append(issues, fmt.Sprintf("* we got %s, which we didn't expect", value)) case weExpected: issues = append(issues, fmt.Sprintf("* we expected but did not see %s", value)) } diff --git a/internal/cmd/buildtool/linux.go b/internal/cmd/buildtool/linux.go index 3e051a4bc3..6960faae18 100644 --- a/internal/cmd/buildtool/linux.go +++ b/internal/cmd/buildtool/linux.go @@ -12,6 +12,7 @@ func linuxSubcommand() *cobra.Command { Use: "linux", Short: "Builds ooniprobe and miniooni for linux", } + cmd.AddCommand(linuxCdepsSubcommand()) cmd.AddCommand(linuxDockerSubcommand()) cmd.AddCommand(linuxStaticSubcommand()) return cmd diff --git a/internal/cmd/buildtool/linuxcdeps.go b/internal/cmd/buildtool/linuxcdeps.go new file mode 100644 index 0000000000..89e48a1187 --- /dev/null +++ b/internal/cmd/buildtool/linuxcdeps.go @@ -0,0 +1,64 @@ +package main + +// +// Allows building C dependencies using Linux +// + +import ( + "fmt" + "path/filepath" + "runtime" + + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +// linuxCdepsSubcommand returns the linuxCdeps sucommand. +func linuxCdepsSubcommand() *cobra.Command { + return &cobra.Command{ + Use: "cdeps {zlib|openssl|libevent|tor} [zlib|openssl|libevent|tor...]", + Short: "Builds C dependencies on Linux systems (experimental)", + Run: func(cmd *cobra.Command, args []string) { + for _, arg := range args { + linuxCdepsBuildMain(arg, &cdepsDependenciesStdlib{}) + } + }, + Args: cobra.MinimumNArgs(1), + } +} + +// linuxCdepsBuildMain is the main of the linuxCdeps build. +func linuxCdepsBuildMain(name string, deps cdepsDependencies) { + runtimex.Assert( + runtime.GOOS == "linux" && runtime.GOARCH == "amd64", + "this command requires linux/amd64", + ) + cdenv := &cdepsEnv{ + cflags: []string{ + // See https://airbus-seclab.github.io/c-compiler-security/ + "-D_FORTIFY_SOURCE=2", + "-fstack-protector-strong", + "-fstack-clash-protection", + "-fPIC", // makes more sense than -fPIE given that we're building a library + "-fsanitize=bounds", + "-fsanitize-undefined-trap-on-error", + "-O2", + }, + destdir: runtimex.Try1(filepath.Abs(filepath.Join( // must be absolute + "internal", "libtor", "linux", runtime.GOARCH, + ))), + openSSLCompiler: "linux-x86_64", + } + switch name { + case "libevent": + cdepsLibeventBuildMain(cdenv, deps) + case "openssl": + cdepsOpenSSLBuildMain(cdenv, deps) + case "tor": + cdepsTorBuildMain(cdenv, deps) + case "zlib": + cdepsZlibBuildMain(cdenv, deps) + default: + panic(fmt.Errorf("unknown dependency: %s", name)) + } +} diff --git a/internal/cmd/buildtool/linuxcdeps_test.go b/internal/cmd/buildtool/linuxcdeps_test.go new file mode 100644 index 0000000000..12df7b585d --- /dev/null +++ b/internal/cmd/buildtool/linuxcdeps_test.go @@ -0,0 +1,383 @@ +package main + +import ( + "fmt" + "path/filepath" + "runtime" + "strconv" + "testing" + + "github.com/ooni/probe-cli/v3/internal/cmd/buildtool/internal/buildtooltest" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx/shellxtesting" +) + +// cdepsFakeDependencies implements fake [cdepsDependencies] for unit testing. +type cdepsFakeDependencies struct{} + +var _ cdepsDependencies = &cdepsFakeDependencies{} + +// absoluteCurDir implements cdepsDependencies +func (*cdepsFakeDependencies) absoluteCurDir() string { + return runtimex.Try1(filepath.Abs("../../../")) // pretend we're in the real topdir +} + +// check implements cdepsVerifier +func (*cdepsFakeDependencies) verifySHA256(expectedSHA256 string, tarball string) { + // nothing +} + +// mustChdir implements cdepsDependencies +func (*cdepsFakeDependencies) mustChdir(dirname string) func() { + return func() {} // nothing +} + +func TestLinuxCdepsBuildMain(t *testing.T) { + if runtime.GOOS != "linux" && runtime.GOARCH != "amd64" { + t.Skip("skip test for GOOS != linux and GOARCH != amd64") + } + + faketopdir := (&cdepsFakeDependencies{}).absoluteCurDir() + + // testspec specifies a test case for this test + type testspec struct { + // name is the name of the test case + name string + + // target is the target to build + target string + + // expectations contains the commands we expect to see + expect []buildtooltest.ExecExpectations + } + + var testcases = []testspec{{ + name: "we can build zlib", + target: "zlib", + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://zlib.net/zlib-1.2.13.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "zlib-1.2.13.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/zlib/000.patch", + }, + }, { + Env: []string{ + "CFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong -fstack-clash-protection -fPIC -fsanitize=bounds -fsanitize-undefined-trap-on-error -O2", + }, + Argv: []string{ + "./configure", "--prefix=/", "--static", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64", + "install", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/pkgconfig", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/share", + }, + }}, + }, { + name: "we can build openssl", + target: "openssl", + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-1.1.1s.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "openssl-1.1.1s.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/openssl/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/openssl/001.patch", + }, + }, { + Env: []string{ + "CFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong -fstack-clash-protection -fPIC -fsanitize=bounds -fsanitize-undefined-trap-on-error -O2 -Wno-macro-redefined", + }, + Argv: []string{ + "./Configure", "no-comp", "no-dtls", "no-ec2m", "no-psk", "no-srp", + "no-ssl2", "no-ssl3", "no-camellia", "no-idea", "no-md2", "no-md4", + "no-mdc2", "no-rc2", "no-rc4", "no-rc5", "no-rmd160", "no-whirlpool", + "no-dso", "no-hw", "no-ui-console", "no-shared", "no-unit-test", + "linux-x86_64", "--libdir=lib", "--prefix=/", "--openssldir=/", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64", + "install_dev", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/pkgconfig", + }, + }}, + }, { + name: "we can build libevent", + target: "libevent", + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{ + "curl", + "-fsSLO", + "https://github.com/libevent/libevent/archive/release-2.1.12-stable.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "release-2.1.12-stable.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/001.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/002.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "./autogen.sh", + }, + }, { + Env: []string{ + fmt.Sprintf( + "CFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong -fstack-clash-protection -fPIC -fsanitize=bounds -fsanitize-undefined-trap-on-error -O2 -I%s/internal/cmd/buildtool/internal/libtor/linux/amd64/include", + faketopdir, + ), + fmt.Sprintf( + "LDFLAGS=-L%s/internal/cmd/buildtool/internal/libtor/linux/amd64/lib", + faketopdir, + ), + }, + Argv: []string{ + "./configure", + "--disable-libevent-regress", + "--disable-samples", + "--disable-shared", + "--prefix=/", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64", + "install", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/bin", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/pkgconfig", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_core.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_core.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_extra.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_extra.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_openssl.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_openssl.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_pthreads.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_pthreads.la", + }, + }}, + }, { + name: "we can build tor", + target: "tor", + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.7.12.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "tor-0.4.7.12.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/001.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/002.patch", + }, + }, { + Env: []string{ + "CFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong -fstack-clash-protection -fPIC -fsanitize=bounds -fsanitize-undefined-trap-on-error -O2", + "LDFLAGS=", + }, + Argv: []string{ + "./configure", + "--enable-pic", + "--enable-static-libevent", + "--with-libevent-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64", + "--enable-static-openssl", + "--with-openssl-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64", + "--enable-static-zlib", + "--with-zlib-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64", + "--disable-module-dirauth", + "--disable-zstd", + "--disable-lzma", + "--disable-tool-name-check", + "--disable-systemd", + "--prefix=/", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "install", "-m644", "src/feature/api/tor_api.h", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/include", + }, + }, { + Env: []string{}, + Argv: []string{ + "install", "-m644", "libtor.a", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib", + }, + }}, + }} + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + cc := &buildtooltest.SimpleCommandCollector{} + + shellxtesting.WithCustomLibrary(cc, func() { + linuxCdepsBuildMain(testcase.target, &cdepsFakeDependencies{}) + }) + + if err := buildtooltest.CheckManyCommands(cc.Commands, testcase.expect); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/libtor/android/386/.gitignore b/internal/libtor/android/386/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/internal/libtor/android/386/.gitignore @@ -0,0 +1 @@ +* diff --git a/internal/libtor/android/amd64/.gitignore b/internal/libtor/android/amd64/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/internal/libtor/android/amd64/.gitignore @@ -0,0 +1 @@ +* diff --git a/internal/libtor/android/arm/.gitignore b/internal/libtor/android/arm/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/internal/libtor/android/arm/.gitignore @@ -0,0 +1 @@ +* diff --git a/internal/libtor/android/arm64/.gitignore b/internal/libtor/android/arm64/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/internal/libtor/android/arm64/.gitignore @@ -0,0 +1 @@ +* diff --git a/internal/libtor/enabled.go b/internal/libtor/enabled.go new file mode 100644 index 0000000000..27dd5ac1e7 --- /dev/null +++ b/internal/libtor/enabled.go @@ -0,0 +1,277 @@ +//go:build ooni_libtor + +package libtor + +// Adapted from https://github.com/cretz/bine +// SPDX-License-Identifier: MIT + +// +// #cgo linux,amd64 CFLAGS: -I${SRCDIR}/linux/amd64/include +// #cgo linux,amd64 LDFLAGS: -L${SRCDIR}/linux/amd64/lib -ltor -levent -lssl -lcrypto -lz -lm +// +// #cgo android,arm CFLAGS: -I${SRCDIR}/android/arm/include +// #cgo android,arm LDFLAGS: -L${SRCDIR}/android/arm/lib -ltor -levent -lssl -lcrypto -lz +// #cgo android,arm64 CFLAGS: -I${SRCDIR}/android/arm64/include +// #cgo android,arm64 LDFLAGS: -L${SRCDIR}/android/arm64/lib -ltor -levent -lssl -lcrypto -lz +// #cgo android,386 CFLAGS: -I${SRCDIR}/android/386/include +// #cgo android,386 LDFLAGS: -L${SRCDIR}/android/386/lib -ltor -levent -lssl -lcrypto -lz +// #cgo android,amd64 CFLAGS: -I${SRCDIR}/android/amd64/include +// #cgo android,amd64 LDFLAGS: -L${SRCDIR}/android/amd64/lib -ltor -levent -lssl -lcrypto -lz +// +// #include +// #include +// #include +// +// #include +// +// /* Note: we need to define inline helpers because we cannot index C arrays in Go. */ +// +// static char **cstringArrayNew(size_t size) { +// char **argv = calloc(size, sizeof(char *)); +// if (argv == NULL) { +// abort(); +// } +// return argv; +// } +// +// static void cstringArraySet(char **argv, size_t index, char *entry) { +// argv[index] = entry; +// } +// +// static void cstringArrayFree(char **argv, size_t size) { +// for (size_t idx = 0; idx < size; idx++) { +// free(argv[idx]); +// } +// free(argv); +// } +// +// static bool filedescIsGood(tor_control_socket_t fd) { +// return fd != INVALID_TOR_CONTROL_SOCKET; +// } +// +import "C" + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "sync" + + "github.com/cretz/bine/process" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// MaybeCreator returns a valid [process.Creator], if possible, otherwise false. +func MaybeCreator() (process.Creator, bool) { + return &torCreator{}, true +} + +// torCreator implements [process.Creator]. +type torCreator struct{} + +var _ process.Creator = &torCreator{} + +// New implements [process.Creator]. +func (c *torCreator) New(ctx context.Context, args ...string) (process.Process, error) { + left, right := net.Pipe() + proc := &torProcess{ + awaitStart: make(chan any, 1), // buffer + controlConn: left, + startErr: make(chan error, 1), // buffer + startOnce: sync.Once{}, + waitErr: make(chan error, 1), // buffer + waitOnce: sync.Once{}, + + closedWhenNotStarted: make(chan any, 1), // buffer + simulateBadControlSocket: false, + simulateFileConnFailure: false, + simulateNonzeroExitCode: false, + } + go proc.runtor(ctx, right, args...) + return proc, nil +} + +// torProcess implements [process.Process]. +type torProcess struct { + // ordinary state variables + awaitStart chan any + controlConn net.Conn + startErr chan error + startOnce sync.Once + waitErr chan error + waitOnce sync.Once + + // for testing + closedWhenNotStarted chan any + simulateBadControlSocket bool + simulateFileConnFailure bool + simulateNonzeroExitCode bool +} + +var _ process.Process = &torProcess{} + +// EmbeddedControlConn implements [process.Process]. +func (p *torProcess) EmbeddedControlConn() (net.Conn, error) { + // Implementation note: this function SHOULD only be called + // once and BEFORE Start is called 😬😬😬 + return p.controlConn, nil +} + +// Start implements [process.Process]. +func (p *torProcess) Start() (err error) { + p.startOnce.Do(func() { + p.awaitStart <- true + err = <-p.startErr + }) + return err +} + +// Wait implements [process.Process]. +func (p *torProcess) Wait() (err error) { + p.waitOnce.Do(func() { + err = <-p.waitErr + }) + return +} + +// ErrTooManyArguments indicates that p.args contains too many arguments +var ErrTooManyArguments = errors.New("libtor: too many arguments") + +// ErrCannotCreateControlSocket indicates that we cannot create a control socket. +var ErrCannotCreateControlSocket = errors.New("libtor: cannot create a control socket") + +// ErrNonzeroExitCode indicates that tor returned a nonzero exit code +var ErrNonzeroExitCode = errors.New("libtor: command completed with nonzero exit code") + +// runtor runs tor until completion and ensures that tor exits when +// the given ctx is cancelled or its deadline expires. +func (p *torProcess) runtor(ctx context.Context, cc net.Conn, args ...string) { + // wait for Start or context to expire + select { + case <-p.awaitStart: + case <-ctx.Done(): + p.startErr <- ctx.Err() // nonblocking chan + close(p.closedWhenNotStarted) + return + } + + // Note: when writing this code I was wondering whether I needed to + // use unsafe.Pointer to track pointers that matter to C code. Reading + // this message[1] has been useful to understand that the most likely + // answer to this question is "obviously, no". + // + // See https://groups.google.com/g/golang-nuts/c/yNis7bQG_rY/m/yaJFoSx1hgIJ + + // Create argc and argv for tor + argv := append([]string{"tor"}, args...) + const toomany = 256 // arbitrary low limit to make C.int and C.size_t casts always work + if len(argv) > toomany { + p.startErr <- ErrTooManyArguments // nonblocking channel + return + } + argc := C.size_t(len(argv)) + // Note: here we allocate argc + 1 because a "null pointer always follows + // the last element: argv[argc] is this null pointer." + // + // See https://www.gnu.org/software/libc/manual/html_node/Program-Arguments.html + allocSiz := argc + 1 + cargv := C.cstringArrayNew(allocSiz) + defer C.cstringArrayFree(cargv, argc) + for idx, entry := range argv { + C.cstringArraySet(cargv, C.size_t(idx), C.CString(entry)) + } + + // Add to config a WEAK REFERENCE to argc and argv + config := C.tor_main_configuration_new() + runtimex.PanicIfNil(config, "C.tor_main_configuration_new failed") + defer C.tor_main_configuration_free(config) + code := C.tor_main_configuration_set_command_line(config, C.int(argc), cargv) + runtimex.Assert(code == 0, "C.tor_main_configuration_set_command_line failed") + + // Create OWNING file descriptor + filedesc := C.tor_main_configuration_setup_control_socket(config) + if p.simulateBadControlSocket { + filedesc = C.INVALID_TOR_CONTROL_SOCKET + } + if !C.filedescIsGood(filedesc) { + p.startErr <- ErrCannotCreateControlSocket // nonblocking channel + return + } + + // Convert the OWNING file descriptor into a proper file. Because + // filedesc is good, os.NewFile shouldn't fail. + filep := os.NewFile(uintptr(filedesc), "") + runtimex.Assert(filep != nil, "os.NewFile should not fail") + conn, err := net.FileConn(filep) + if p.simulateFileConnFailure { + err = ErrCannotCreateControlSocket + } + if err != nil { + p.startErr <- err // nonblocking channel + return + } + + // In the following we're going to possibly call Close multiple + // times. Let's be very sure that this close is idempotent. + conn = withIdempotentClose(conn) + cc = withIdempotentClose(cc) + + // Make sure we close filep when the context is done. Because the + // socket is OWNING, this will also cause tor to return. + go func() { + defer conn.Close() + defer cc.Close() + <-ctx.Done() + }() + + // Route messages from and to the control connection. + go sendrecvThenClose(cc, conn) + go sendrecvThenClose(conn, cc) + + // Let the user know that startup was successful. + p.startErr <- nil // nonblocking channel + + // Run tor until completion. + if !p.simulateNonzeroExitCode { + code = C.tor_run_main(config) + } else { + code = 1 + } + if code != 0 { + p.waitErr <- fmt.Errorf("%w: %d", ErrNonzeroExitCode, code) // nonblocking channel + return + } + p.waitErr <- nil // nonblocking channel +} + +// sendrecvThenClose routes traffic between two connections and then +// closes both of them when done with routing traffic. +func sendrecvThenClose(left, right net.Conn) { + defer left.Close() + defer right.Close() + netxlite.CopyContext(context.Background(), left, right) +} + +// withIdempotentClose ensures that a connection has idempotent close. +func withIdempotentClose(c net.Conn) net.Conn { + return &idempotentClose{ + Conn: c, + once: sync.Once{}, + } +} + +// idempotentClose ensures close is idempotent for a net.Conn +type idempotentClose struct { + net.Conn + once sync.Once +} + +func (c *idempotentClose) Close() (err error) { + c.once.Do(func() { + err = c.Conn.Close() + }) + return +} diff --git a/internal/libtor/enabled_test.go b/internal/libtor/enabled_test.go new file mode 100644 index 0000000000..d749bc0d55 --- /dev/null +++ b/internal/libtor/enabled_test.go @@ -0,0 +1,255 @@ +//go:build ooni_libtor + +package libtor + +import ( + "context" + "errors" + "io" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/cretz/bine/tor" +) + +func TestNormalUsage(t *testing.T) { + ctx := context.Background() + + datadir, err := filepath.Abs(filepath.Join("testdata", "datadir")) + if err != nil { + t.Fatal(err) + } + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + config := &tor.StartConf{ + ProcessCreator: creator, + DataDir: datadir, + ExtraArgs: nil, + NoHush: true, + } + + instance, err := tor.Start(ctx, config) + if err != nil { + t.Fatal(err) + } + defer instance.Close() + + if err := instance.EnableNetwork(context.Background(), true); err != nil { + t.Fatal(err) + } +} + +func TestContextAlreadyExpired(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // fail immediately + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + process, err := creator.New(ctx) + if err != nil { + t.Fatal(err) + } + + // Make sure that start happens after the background + // goroutine noticing that ctx is cancelled. + <-process.(*torProcess).closedWhenNotStarted + + if err := process.Start(); !errors.Is(err, context.Canceled) { + t.Fatal("unexpected err", err) + } +} + +func TestTooManyCommandLineArguments(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + argv := make([]string, 4096) + process, err := creator.New(ctx, argv...) + if err != nil { + t.Fatal(err) + } + + if err := process.Start(); !errors.Is(err, ErrTooManyArguments) { + t.Fatal("unexpected err", err) + } +} + +func TestSetupControlSocketFails(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + process, err := creator.New(ctx) + if err != nil { + t.Fatal(err) + } + process.(*torProcess).simulateBadControlSocket = true + + if err := process.Start(); !errors.Is(err, ErrCannotCreateControlSocket) { + t.Fatal("unexpected err", err) + } +} + +func TestFileConnFails(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + process, err := creator.New(ctx) + if err != nil { + t.Fatal(err) + } + process.(*torProcess).simulateFileConnFailure = true + + if err := process.Start(); !errors.Is(err, ErrCannotCreateControlSocket) { + t.Fatal("unexpected err", err) + } +} + +func TestNonzeroExitCode(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + process, err := creator.New(ctx) + if err != nil { + t.Fatal(err) + } + process.(*torProcess).simulateNonzeroExitCode = true + + if err := process.Start(); err != nil { + t.Fatal(err) + } + + if err := process.Wait(); !errors.Is(err, ErrNonzeroExitCode) { + t.Fatal("unexpected err", err) + } +} + +func TestContextCanceledWhileTorIsRunning(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + process, err := creator.New(ctx) + if err != nil { + t.Fatal(err) + } + + cconn, err := process.EmbeddedControlConn() + if err != nil { + t.Fatal(err) + } + + if err := process.Start(); err != nil { + t.Fatal(err) + } + + message := []byte("SETEVENTS STATUS_CLIENT\r\n") + if _, err := cconn.Write(message); err != nil { + t.Fatal(err) + } + + for { + message := make([]byte, 1<<20) + count, err := cconn.Read(message) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatal(err) + } + message = message[:count] + t.Log(strings.Trim(string(message), "\r\n")) + } + + if err := process.Wait(); err != nil { + t.Fatal(err) + } +} + +// In theory this case SHOULD NOT happen judging from the description of +// the EmbeddedControlConn method, which reads: +// +// Note, this should only be called once per process before +// Start, and the connection does not need to be closed. +// +// However, it MIGHT happen. So, let's also cover this possibility. +func TestControlConnectionExplicitlyClosed(t *testing.T) { + ctx := context.Background() + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + process, err := creator.New(ctx) + if err != nil { + t.Fatal(err) + } + + cconn, err := process.EmbeddedControlConn() + if err != nil { + t.Fatal(err) + } + + go func() { + <-time.After(2 * time.Second) + cconn.Close() + }() + + if err := process.Start(); err != nil { + t.Fatal(err) + } + + message := []byte("SETEVENTS STATUS_CLIENT\r\n") + if _, err := cconn.Write(message); err != nil { + t.Fatal(err) + } + + for { + message := make([]byte, 1<<20) + count, err := cconn.Read(message) + if errors.Is(err, io.ErrClosedPipe) { + break + } + if err != nil { + t.Fatal(err) + } + message = message[:count] + t.Log(strings.Trim(string(message), "\r\n")) + } + + if err := process.Wait(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/libtor/fallback.go b/internal/libtor/fallback.go new file mode 100644 index 0000000000..069d303bef --- /dev/null +++ b/internal/libtor/fallback.go @@ -0,0 +1,10 @@ +//go:build !ooni_libtor + +package libtor + +import "github.com/cretz/bine/process" + +// MaybeCreator returns a valid [process.Creator], if possible, otherwise false. +func MaybeCreator() (process.Creator, bool) { + return nil, false +} diff --git a/internal/libtor/linux/amd64/.gitignore b/internal/libtor/linux/amd64/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/internal/libtor/linux/amd64/.gitignore @@ -0,0 +1 @@ +* diff --git a/internal/libtor/testdata/.gitignore b/internal/libtor/testdata/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/internal/libtor/testdata/.gitignore @@ -0,0 +1 @@ +* diff --git a/internal/tunnel/tordesktop.go b/internal/tunnel/tordesktop.go index bf1a7a08cb..7a0b55544e 100644 --- a/internal/tunnel/tordesktop.go +++ b/internal/tunnel/tordesktop.go @@ -1,8 +1,9 @@ -//go:build !android && !ios +//go:build !android && !ios && !ooni_libtor package tunnel -// This file implements our strategy for running tor on desktop. +// This file implements our strategy for running tor on desktop in most +// configurations except for experimental ones. import ( "strings" diff --git a/internal/tunnel/torembed.go b/internal/tunnel/torembed.go new file mode 100644 index 0000000000..770190608a --- /dev/null +++ b/internal/tunnel/torembed.go @@ -0,0 +1,30 @@ +//go:build ooni_libtor + +package tunnel + +// This file implements an experimental strategy for running tor. + +import ( + "errors" + "strings" + + "github.com/cretz/bine/tor" + "github.com/ooni/probe-cli/v3/internal/libtor" +) + +// getTorStartConf in this configuration returns a tor.StartConf +// configured to run the version of tor we embed as a library. +func getTorStartConf(config *Config, dataDir string, extraArgs []string) (*tor.StartConf, error) { + creator, good := libtor.MaybeCreator() + if !good { + return nil, errors.New("no embedded tor") + } + config.logger().Infof("tunnel: tor: exec: %s %s", + dataDir, strings.Join(extraArgs, " ")) + return &tor.StartConf{ + ProcessCreator: creator, + DataDir: dataDir, + ExtraArgs: extraArgs, + NoHush: true, + }, nil +}