diff --git a/.github/workflows/libtorlinux.yml b/.github/workflows/libtorlinux.yml new file mode 100644 index 0000000000..9366c96f77 --- /dev/null +++ b/.github/workflows/libtorlinux.yml @@ -0,0 +1,29 @@ +# Runs tests for internal/libtor with -tags=ooni_libtor +name: libtorlinux +on: + pull_request: + push: + branches: + - "master" + - "release/**" + - "fullbuild" + +jobs: + test_ooni_libtor: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + - name: Get GOVERSION content + id: goversion + run: echo ::set-output name=version::$(cat GOVERSION) + + - uses: magnetikonline/action-golang-cache@v2 + with: + go-version: "${{ steps.goversion.outputs.version }}" + cache-key-suffix: "-libtorlinux-${{ steps.goversion.outputs.version }}" + + - run: go run ./internal/cmd/buildtool linux cdeps zlib openssl libevent tor + + - run: go test -count 1 -v -cover -tags ooni_libtor -race ./internal/libtor/... diff --git a/CDEPS/README.md b/CDEPS/README.md new file mode 100644 index 0000000000..e8286f9c5d --- /dev/null +++ b/CDEPS/README.md @@ -0,0 +1,3 @@ +# Code to build C dependencies + +Directory used to compile C dependencies. diff --git a/CDEPS/libevent/000.patch b/CDEPS/libevent/000.patch new file mode 100644 index 0000000000..752a9fdb13 --- /dev/null +++ b/CDEPS/libevent/000.patch @@ -0,0 +1,16 @@ +diff --git a/bufferevent_openssl.c b/bufferevent_openssl.c +index b51b834..06b219e 100644 +--- a/bufferevent_openssl.c ++++ b/bufferevent_openssl.c +@@ -67,6 +67,11 @@ + #include + #include "openssl-compat.h" + ++#include ++#ifndef OPENSSL_OONI ++#error "We're not including the correct openssl/opensslv.h file" ++#endif ++ + /* + * Define an OpenSSL bio that targets a bufferevent. + */ diff --git a/CDEPS/libevent/001.patch b/CDEPS/libevent/001.patch new file mode 100644 index 0000000000..b27287bb95 --- /dev/null +++ b/CDEPS/libevent/001.patch @@ -0,0 +1,15 @@ +diff --git a/test/regress_zlib.c b/test/regress_zlib.c +index 5fe7749..558155c 100644 +--- a/test/regress_zlib.c ++++ b/test/regress_zlib.c +@@ -80,6 +80,10 @@ + + #include + ++#ifndef ZLIB_OONI ++#error "We're not including the correct zlib.h file" ++#endif ++ + static int infilter_calls; + static int outfilter_calls; + static int readcb_finished; diff --git a/CDEPS/libevent/002.patch b/CDEPS/libevent/002.patch new file mode 100644 index 0000000000..29e17ff362 --- /dev/null +++ b/CDEPS/libevent/002.patch @@ -0,0 +1,15 @@ +diff --git a/include/event2/event.h b/include/event2/event.h +index a6b6144..6abb474 100644 +--- a/include/event2/event.h ++++ b/include/event2/event.h +@@ -1665,6 +1665,10 @@ int event_base_update_cache_time(struct event_base *base); + EVENT2_EXPORT_SYMBOL + void libevent_global_shutdown(void); + ++/* EVENT_OONI is used by dependencies to ensure they are using the ++ correct event.h header and not some other header. */ ++#define EVENT_OONI 1 ++ + #ifdef __cplusplus + } + #endif diff --git a/CDEPS/openssl/000.patch b/CDEPS/openssl/000.patch new file mode 100644 index 0000000000..e313ac356a --- /dev/null +++ b/CDEPS/openssl/000.patch @@ -0,0 +1,15 @@ +diff --git a/crypto/comp/c_zlib.c b/crypto/comp/c_zlib.c +index b819337..e479bd6 100644 +--- a/crypto/comp/c_zlib.c ++++ b/crypto/comp/c_zlib.c +@@ -34,6 +34,10 @@ static COMP_METHOD zlib_method_nozlib = { + + # include + ++#ifndef ZLIB_OONI ++# error "We're not including the correct zlib.h file" ++#endif ++ + static int zlib_stateful_init(COMP_CTX *ctx); + static void zlib_stateful_finish(COMP_CTX *ctx); + static int zlib_stateful_compress_block(COMP_CTX *ctx, unsigned char *out, diff --git a/CDEPS/openssl/001.patch b/CDEPS/openssl/001.patch new file mode 100644 index 0000000000..780faa59a1 --- /dev/null +++ b/CDEPS/openssl/001.patch @@ -0,0 +1,14 @@ +diff --git a/include/openssl/opensslv.h b/include/openssl/opensslv.h +index fd9400a..0d2c9dc 100644 +--- a/include/openssl/opensslv.h ++++ b/include/openssl/opensslv.h +@@ -94,6 +94,9 @@ extern "C" { + # define SHLIB_VERSION_HISTORY "" + # define SHLIB_VERSION_NUMBER "1.1" + ++/* OPENSSL_OONI is used by dependencies to ensure they are using the ++ correct OpenSSL headers and not some other headers. */ ++#define OPENSSL_OONI 1 + + #ifdef __cplusplus + } diff --git a/CDEPS/tor/000.patch b/CDEPS/tor/000.patch new file mode 100644 index 0000000000..a3fd2cd9dc --- /dev/null +++ b/CDEPS/tor/000.patch @@ -0,0 +1,15 @@ +diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c +index 77de2d6..ce46554 100644 +--- a/src/lib/tls/tortls_openssl.c ++++ b/src/lib/tls/tortls_openssl.c +@@ -45,6 +45,10 @@ DISABLE_GCC_WARNING("-Wredundant-decls") + #error "We require OpenSSL with ECC support" + #endif + ++#ifndef OPENSSL_OONI ++#error "We're not including the correct openssl/opensslv.h file" ++#endif ++ + #include + #include + #include diff --git a/CDEPS/tor/001.patch b/CDEPS/tor/001.patch new file mode 100644 index 0000000000..382f25440d --- /dev/null +++ b/CDEPS/tor/001.patch @@ -0,0 +1,15 @@ +diff --git a/src/lib/compress/compress_zlib.c b/src/lib/compress/compress_zlib.c +index 52f9509..fb7e39e 100644 +--- a/src/lib/compress/compress_zlib.c ++++ b/src/lib/compress/compress_zlib.c +@@ -45,6 +45,10 @@ + #error "We require zlib version 1.2 or later." + #endif + ++#ifndef ZLIB_OONI ++#error "We're not including the correct zlib.h file" ++#endif ++ + static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + diff --git a/CDEPS/tor/002.patch b/CDEPS/tor/002.patch new file mode 100644 index 0000000000..e0331e3356 --- /dev/null +++ b/CDEPS/tor/002.patch @@ -0,0 +1,15 @@ +diff --git a/src/lib/evloop/compat_libevent.c b/src/lib/evloop/compat_libevent.c +index fd840f8..2ec37e7 100644 +--- a/src/lib/evloop/compat_libevent.c ++++ b/src/lib/evloop/compat_libevent.c +@@ -19,6 +19,10 @@ + #include + #include + ++#ifndef EVENT_OONI ++#error "We're not including the correct event2/event.h file" ++#endif ++ + /** A string which, if it appears in a libevent log, should be ignored. */ + static const char *suppress_msg = NULL; + /** Callback function passed to event_set_log() so we can intercept diff --git a/CDEPS/zlib/000.patch b/CDEPS/zlib/000.patch new file mode 100644 index 0000000000..d8ddd3d7f7 --- /dev/null +++ b/CDEPS/zlib/000.patch @@ -0,0 +1,15 @@ +diff --git a/zlib.h b/zlib.h +index 953cb50..ec2e2f8 100644 +--- a/zlib.h ++++ b/zlib.h +@@ -1928,6 +1928,10 @@ ZEXTERN int ZEXPORTVA gzvprintf Z_ARG((gzFile file, + # endif + #endif + ++/* ZLIB_OONI is used by dependencies to ensure they are using the ++ correct zlib.h header and not some other header. */ ++#define ZLIB_OONI 1 ++ + #ifdef __cplusplus + } + #endif diff --git a/MONOREPO/w/build-android-with-cli.bash b/MONOREPO/w/build-android-with-cli.bash index cc51f646ea..a09d37e9e9 100755 --- a/MONOREPO/w/build-android-with-cli.bash +++ b/MONOREPO/w/build-android-with-cli.bash @@ -8,8 +8,7 @@ source $reporoot/MONOREPO/tools/libcore.bash ./MOBILE/android/newkeystore -./MOBILE/gomobile android ./pkg/oonimkall - +make ./MOBILE/android run cp -v MOBILE/android/oonimkall.aar ./MONOREPO/repo/probe-android/engine-experimental/ ( diff --git a/internal/cmd/buildtool/cdeps.go b/internal/cmd/buildtool/cdeps.go new file mode 100644 index 0000000000..8c25d11320 --- /dev/null +++ b/internal/cmd/buildtool/cdeps.go @@ -0,0 +1,155 @@ +package main + +// +// Building C dependencies: common code +// + +import ( + "os" + "path/filepath" + "sort" + "strings" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// cdepsEnv contains the environment for compiling a C dependency. +type cdepsEnv struct { + // cflags contains the CFLAGS to use when compiling. + cflags []string + + // configureHost is the value to pass to ./configure's --host option. + configureHost string + + // destdir is the directory where to install. + destdir string + + // lfdlags contains the LDFLAGS to use when compiling. + ldflags []string + + // openSSLAPIDefine is an extra define we need to add on Android. + openSSLAPIDefine string + + // openSSLCompiler is the compiler name for OpenSSL. + openSSLCompiler string +} + +// cdepsDependencies groups dependencies used when building cdeps. +type cdepsDependencies interface { + // mustChdir changes the current working directory and returns the + // function to return to the original working directory. + mustChdir(dirname string) func() + + // absoluteCurDir returns the absolute current directory. + absoluteCurDir() string + + // verifySHA256 verifies that the tarball has the given checksum. + verifySHA256(expectedSHA256, tarball string) +} + +// cdepsDependenciesStdlib are the [cdepsDependencies] used by default. +type cdepsDependenciesStdlib struct{} + +var _ cdepsDependencies = &cdepsDependenciesStdlib{} + +// absoluteCurDir implements cdepsDependencies +func (c *cdepsDependenciesStdlib) absoluteCurDir() string { + return cdepsMustAbsoluteCurdir() +} + +// verifySHA256 implements cdepsDependencies +func (c *cdepsDependenciesStdlib) verifySHA256(expectedSHA256, tarball string) { + cdepsMustVerifySHA256(expectedSHA256, tarball) +} + +// mustChdir implements cdepsDependencies +func (c *cdepsDependenciesStdlib) mustChdir(dirname string) func() { + return cdepsMustChdir(dirname) +} + +// cdepsAddCflags merges this struct's cflags with the extra cflags and +// then stores the merged cflags into the given envp. +func cdepsAddCflags(envp *shellx.Envp, c *cdepsEnv, extraCflags ...string) { + mergedCflags := append([]string{}, c.cflags...) + mergedCflags = append(mergedCflags, extraCflags...) + envp.Append("CFLAGS", strings.Join(mergedCflags, " ")) +} + +// cdepsAddLdflags merges this struct's ldflags with the extra ldflags and +// then stores the merged ldflags into the given envp. +func cdepsAddLdflags(envp *shellx.Envp, c *cdepsEnv, extraLdflags ...string) { + mergedLdflags := append([]string{}, c.ldflags...) + mergedLdflags = append(mergedLdflags, extraLdflags...) + envp.Append("LDFLAGS", strings.Join(mergedLdflags, " ")) +} + +// cdepsMustMkdirTemp creates a temporary directory. +func cdepsMustMkdirTemp() string { + return runtimex.Try1(os.MkdirTemp("", "")) +} + +// cdepsMustChdir changes the current directory to the given dir and +// returns a function to return to the original working dir. +func cdepsMustChdir(work string) func() { + prevdir := runtimex.Try1(os.Getwd()) + log.Infof("cd %s", work) + runtimex.Try0(os.Chdir(work)) + return func() { + runtimex.Try0(os.Chdir(prevdir)) + log.Infof("cd %s", prevdir) + } +} + +// cdepsMustFetch fetches the given URL using curl. +func cdepsMustFetch(URL string) { + must.Run(log.Log, "curl", "-fsSLO", URL) +} + +// cdepsMustVerifySHA256 verifies the SHA256 of the given tarball. +func cdepsMustVerifySHA256(expectedSHA256, tarball string) { + firstline := string(must.FirstLineBytes(must.RunOutput( + log.Log, "sha256sum", tarball, + ))) + sha256, _, good := strings.Cut(firstline, " ") + runtimex.Assert(good, "cannot obtain the first token") + runtimex.Assert(expectedSHA256 == sha256, "SHA256 mismatch") +} + +// cdepsMustAbsoluteCurdir returns the absolute path of the current dir. +func cdepsMustAbsoluteCurdir() string { + return runtimex.Try1(filepath.Abs(".")) +} + +// cdepsMustListPatches returns all the patches inside a dir. +func cdepsMustListPatches(dir string) (out []string) { + entries := runtimex.Try1(os.ReadDir(dir)) + for _, entry := range entries { + if !entry.Type().IsRegular() { + continue + } + if !strings.HasSuffix(entry.Name(), ".patch") { + continue + } + out = append(out, filepath.Join(dir, entry.Name())) + } + sort.Strings(out) + return +} + +// cdepsDefaultShellxConfig returns the default config used when calling shellx.RunEx. +func cdepsDefaultShellxConfig() *shellx.Config { + return &shellx.Config{ + Logger: log.Log, + Flags: shellx.FlagShowStdoutStderr, + } +} + +// cdepsMustRunWithDefaultConfig is a convenience wrapper +// around calling [shellx.RunEx] and checking the return value. +func cdepsMustRunWithDefaultConfig(envp *shellx.Envp, command string, args ...string) { + argv := runtimex.Try1(shellx.NewArgv(command, args...)) + runtimex.Try0(shellx.RunEx(cdepsDefaultShellxConfig(), argv, envp)) +} diff --git a/internal/cmd/buildtool/cdepslibevent.go b/internal/cmd/buildtool/cdepslibevent.go new file mode 100644 index 0000000000..a5424d8621 --- /dev/null +++ b/internal/cmd/buildtool/cdepslibevent.go @@ -0,0 +1,69 @@ +package main + +// +// Building C dependencies: libevent +// +// Adapted from https://github.com/guardianproject/tor-android +// SPDX-License-Identifier: BSD-3-Clause + +import ( + "path/filepath" + "runtime" + "strconv" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// cdepsLibeventBuildMain is the script that builds libevent. +func cdepsLibeventBuildMain(cdenv *cdepsEnv, deps cdepsDependencies) { + topdir := deps.absoluteCurDir() // must be mockable + work := cdepsMustMkdirTemp() + restore := cdepsMustChdir(work) + defer restore() + + // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/libevent.rb + cdepsMustFetch("https://github.com/libevent/libevent/archive/release-2.1.12-stable.tar.gz") + deps.verifySHA256( // must be mockable + "7180a979aaa7000e1264da484f712d403fcf7679b1e9212c4e3d09f5c93efc24", + "release-2.1.12-stable.tar.gz", + ) + must.Run(log.Log, "tar", "-xf", "release-2.1.12-stable.tar.gz") + _ = deps.mustChdir("libevent-release-2.1.12-stable") // must be mockable + + mydir := filepath.Join(topdir, "CDEPS", "libevent") + for _, patch := range cdepsMustListPatches(mydir) { + must.Run(log.Log, "git", "apply", patch) + } + + must.Run(log.Log, "./autogen.sh") + + envp := &shellx.Envp{} + cdepsAddCflags(envp, cdenv, "-I"+cdenv.destdir+"/include") + cdepsAddLdflags(envp, cdenv, "-L"+cdenv.destdir+"/lib") + + argv := runtimex.Try1(shellx.NewArgv("./configure")) + if cdenv.configureHost != "" { + argv.Append("--host=" + cdenv.configureHost) + } + argv.Append("--disable-libevent-regress", "--disable-samples", "--disable-shared", "--prefix=/") + runtimex.Try0(shellx.RunEx(cdepsDefaultShellxConfig(), argv, envp)) + + must.Run(log.Log, "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU())) + must.Run(log.Log, "make", "DESTDIR="+cdenv.destdir, "install") + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "bin")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "pkgconfig")) + + // we just need libevent.a + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent.la")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_core.a")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_core.la")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_extra.a")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_extra.la")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_openssl.a")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_openssl.la")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_pthreads.a")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_pthreads.la")) +} diff --git a/internal/cmd/buildtool/cdepsopenssl.go b/internal/cmd/buildtool/cdepsopenssl.go new file mode 100644 index 0000000000..ebb389e977 --- /dev/null +++ b/internal/cmd/buildtool/cdepsopenssl.go @@ -0,0 +1,59 @@ +package main + +// +// Building C dependencies: OpenSSL +// +// Adapted from https://github.com/guardianproject/tor-android +// SPDX-License-Identifier: BSD-3-Clause + +import ( + "path/filepath" + "runtime" + "strconv" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// cdepsOpenSSLBuildMain is the script that builds OpenSSL. +func cdepsOpenSSLBuildMain(cdenv *cdepsEnv, deps cdepsDependencies) { + topdir := deps.absoluteCurDir() // must be mockable + work := cdepsMustMkdirTemp() + restore := cdepsMustChdir(work) + defer restore() + + // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/openssl@1.1.rb + cdepsMustFetch("https://www.openssl.org/source/openssl-1.1.1s.tar.gz") + deps.verifySHA256( // must be mockable + "c5ac01e760ee6ff0dab61d6b2bbd30146724d063eb322180c6f18a6f74e4b6aa", + "openssl-1.1.1s.tar.gz", + ) + must.Run(log.Log, "tar", "-xf", "openssl-1.1.1s.tar.gz") + _ = deps.mustChdir("openssl-1.1.1s") // must be mockable + + mydir := filepath.Join(topdir, "CDEPS", "openssl") + for _, patch := range cdepsMustListPatches(mydir) { + must.Run(log.Log, "git", "apply", patch) + } + + envp := &shellx.Envp{} + cdepsAddCflags(envp, cdenv, "-Wno-macro-redefined") + argv := runtimex.Try1(shellx.NewArgv( + "./Configure", "no-comp", "no-dtls", "no-ec2m", "no-psk", "no-srp", + "no-ssl2", "no-ssl3", "no-camellia", "no-idea", "no-md2", "no-md4", + "no-mdc2", "no-rc2", "no-rc4", "no-rc5", "no-rmd160", "no-whirlpool", + "no-dso", "no-hw", "no-ui-console", "no-shared", "no-unit-test", + cdenv.openSSLCompiler, + )) + if cdenv.openSSLAPIDefine != "" { + argv.Append(cdenv.openSSLAPIDefine) + } + argv.Append("--libdir=lib", "--prefix=/", "--openssldir=/") + runtimex.Try0(shellx.RunEx(cdepsDefaultShellxConfig(), argv, envp)) + + must.Run(log.Log, "make", "-j", strconv.Itoa(runtime.NumCPU())) + must.Run(log.Log, "make", "DESTDIR="+cdenv.destdir, "install_dev") + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "pkgconfig")) +} diff --git a/internal/cmd/buildtool/cdepstor.go b/internal/cmd/buildtool/cdepstor.go new file mode 100644 index 0000000000..6608329cb2 --- /dev/null +++ b/internal/cmd/buildtool/cdepstor.go @@ -0,0 +1,66 @@ +package main + +// +// Building C dependencies: tor +// +// Adapted from https://github.com/guardianproject/tor-android +// SPDX-License-Identifier: BSD-3-Clause + +import ( + "path/filepath" + "runtime" + "strconv" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// cdepsTorBuildMain is the script that builds tor. +func cdepsTorBuildMain(cdenv *cdepsEnv, deps cdepsDependencies) { + topdir := deps.absoluteCurDir() // must be mockable + work := cdepsMustMkdirTemp() + restore := cdepsMustChdir(work) + defer restore() + + // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/tor.rb + cdepsMustFetch("https://www.torproject.org/dist/tor-0.4.7.12.tar.gz") + deps.verifySHA256( // must be mockable + "3b5d969712c467851bd028f314343ef15a97ea457191e93ffa97310b05b9e395", + "tor-0.4.7.12.tar.gz", + ) + must.Run(log.Log, "tar", "-xf", "tor-0.4.7.12.tar.gz") + _ = deps.mustChdir("tor-0.4.7.12") // must be mockable + + mydir := filepath.Join(topdir, "CDEPS", "tor") + for _, patch := range cdepsMustListPatches(mydir) { + must.Run(log.Log, "git", "apply", patch) + } + + envp := &shellx.Envp{} + cdepsAddCflags(envp, cdenv) + cdepsAddLdflags(envp, cdenv) + + argv := runtimex.Try1(shellx.NewArgv("./configure")) + if cdenv.configureHost != "" { + argv.Append("--host=" + cdenv.configureHost) + } + argv.Append( + "--enable-pic", + "--enable-static-libevent", "--with-libevent-dir="+cdenv.destdir, + "--enable-static-openssl", "--with-openssl-dir="+cdenv.destdir, + "--enable-static-zlib", "--with-zlib-dir="+cdenv.destdir, + "--disable-module-dirauth", + "--disable-zstd", "--disable-lzma", + "--disable-tool-name-check", + "--disable-systemd", + "--prefix=/", + ) + runtimex.Try0(shellx.RunEx(cdepsDefaultShellxConfig(), argv, envp)) + + must.Run(log.Log, "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU())) + + must.Run(log.Log, "install", "-m644", "src/feature/api/tor_api.h", cdenv.destdir+"/include") + must.Run(log.Log, "install", "-m644", "libtor.a", cdenv.destdir+"/lib") +} diff --git a/internal/cmd/buildtool/cdepszlib.go b/internal/cmd/buildtool/cdepszlib.go new file mode 100644 index 0000000000..3231f70aa4 --- /dev/null +++ b/internal/cmd/buildtool/cdepszlib.go @@ -0,0 +1,51 @@ +package main + +// +// Building C dependencies: zlib +// +// Adapted from https://github.com/guardianproject/tor-android +// SPDX-License-Identifier: BSD-3-Clause + +import ( + "path/filepath" + "runtime" + "strconv" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// cdepsZlibBuildMain is the script that builds zlib. +func cdepsZlibBuildMain(cdenv *cdepsEnv, deps cdepsDependencies) { + topdir := deps.absoluteCurDir() // must be mockable + work := cdepsMustMkdirTemp() + restore := cdepsMustChdir(work) + defer restore() + + // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/zlib.rb + cdepsMustFetch("https://zlib.net/zlib-1.2.13.tar.gz") + deps.verifySHA256( // must be mockable + "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", + "zlib-1.2.13.tar.gz", + ) + must.Run(log.Log, "tar", "-xf", "zlib-1.2.13.tar.gz") + _ = deps.mustChdir("zlib-1.2.13") // must be mockable + + mydir := filepath.Join(topdir, "CDEPS", "zlib") + for _, patch := range cdepsMustListPatches(mydir) { + must.Run(log.Log, "git", "apply", patch) + } + + envp := &shellx.Envp{} + if cdenv.configureHost != "" { + envp.Append("CHOST", cdenv.configureHost) // zlib's configure otherwise uses Apple's libtool + } + cdepsAddCflags(envp, cdenv) + cdepsMustRunWithDefaultConfig(envp, "./configure", "--prefix=/", "--static") + + must.Run(log.Log, "make", "-j", strconv.Itoa(runtime.NumCPU())) + must.Run(log.Log, "make", "DESTDIR="+cdenv.destdir, "install") + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "pkgconfig")) + must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "share")) +} diff --git a/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go b/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go index 0037b178e1..a6e10f21d1 100644 --- a/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go +++ b/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go @@ -66,7 +66,7 @@ func CompareEnv(expected, got []string) error { case weExpected | weGot: // nothing case weGot: - issues = append(issues, fmt.Sprintf("* we got %s, which we don't expected", value)) + issues = append(issues, fmt.Sprintf("* we got %s, which we didn't expect", value)) case weExpected: issues = append(issues, fmt.Sprintf("* we expected but did not see %s", value)) } diff --git a/internal/cmd/buildtool/linux.go b/internal/cmd/buildtool/linux.go index 3e051a4bc3..6960faae18 100644 --- a/internal/cmd/buildtool/linux.go +++ b/internal/cmd/buildtool/linux.go @@ -12,6 +12,7 @@ func linuxSubcommand() *cobra.Command { Use: "linux", Short: "Builds ooniprobe and miniooni for linux", } + cmd.AddCommand(linuxCdepsSubcommand()) cmd.AddCommand(linuxDockerSubcommand()) cmd.AddCommand(linuxStaticSubcommand()) return cmd diff --git a/internal/cmd/buildtool/linuxcdeps.go b/internal/cmd/buildtool/linuxcdeps.go new file mode 100644 index 0000000000..89e48a1187 --- /dev/null +++ b/internal/cmd/buildtool/linuxcdeps.go @@ -0,0 +1,64 @@ +package main + +// +// Allows building C dependencies using Linux +// + +import ( + "fmt" + "path/filepath" + "runtime" + + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +// linuxCdepsSubcommand returns the linuxCdeps sucommand. +func linuxCdepsSubcommand() *cobra.Command { + return &cobra.Command{ + Use: "cdeps {zlib|openssl|libevent|tor} [zlib|openssl|libevent|tor...]", + Short: "Builds C dependencies on Linux systems (experimental)", + Run: func(cmd *cobra.Command, args []string) { + for _, arg := range args { + linuxCdepsBuildMain(arg, &cdepsDependenciesStdlib{}) + } + }, + Args: cobra.MinimumNArgs(1), + } +} + +// linuxCdepsBuildMain is the main of the linuxCdeps build. +func linuxCdepsBuildMain(name string, deps cdepsDependencies) { + runtimex.Assert( + runtime.GOOS == "linux" && runtime.GOARCH == "amd64", + "this command requires linux/amd64", + ) + cdenv := &cdepsEnv{ + cflags: []string{ + // See https://airbus-seclab.github.io/c-compiler-security/ + "-D_FORTIFY_SOURCE=2", + "-fstack-protector-strong", + "-fstack-clash-protection", + "-fPIC", // makes more sense than -fPIE given that we're building a library + "-fsanitize=bounds", + "-fsanitize-undefined-trap-on-error", + "-O2", + }, + destdir: runtimex.Try1(filepath.Abs(filepath.Join( // must be absolute + "internal", "libtor", "linux", runtime.GOARCH, + ))), + openSSLCompiler: "linux-x86_64", + } + switch name { + case "libevent": + cdepsLibeventBuildMain(cdenv, deps) + case "openssl": + cdepsOpenSSLBuildMain(cdenv, deps) + case "tor": + cdepsTorBuildMain(cdenv, deps) + case "zlib": + cdepsZlibBuildMain(cdenv, deps) + default: + panic(fmt.Errorf("unknown dependency: %s", name)) + } +} diff --git a/internal/cmd/buildtool/linuxcdeps_test.go b/internal/cmd/buildtool/linuxcdeps_test.go new file mode 100644 index 0000000000..12df7b585d --- /dev/null +++ b/internal/cmd/buildtool/linuxcdeps_test.go @@ -0,0 +1,383 @@ +package main + +import ( + "fmt" + "path/filepath" + "runtime" + "strconv" + "testing" + + "github.com/ooni/probe-cli/v3/internal/cmd/buildtool/internal/buildtooltest" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx/shellxtesting" +) + +// cdepsFakeDependencies implements fake [cdepsDependencies] for unit testing. +type cdepsFakeDependencies struct{} + +var _ cdepsDependencies = &cdepsFakeDependencies{} + +// absoluteCurDir implements cdepsDependencies +func (*cdepsFakeDependencies) absoluteCurDir() string { + return runtimex.Try1(filepath.Abs("../../../")) // pretend we're in the real topdir +} + +// check implements cdepsVerifier +func (*cdepsFakeDependencies) verifySHA256(expectedSHA256 string, tarball string) { + // nothing +} + +// mustChdir implements cdepsDependencies +func (*cdepsFakeDependencies) mustChdir(dirname string) func() { + return func() {} // nothing +} + +func TestLinuxCdepsBuildMain(t *testing.T) { + if runtime.GOOS != "linux" && runtime.GOARCH != "amd64" { + t.Skip("skip test for GOOS != linux and GOARCH != amd64") + } + + faketopdir := (&cdepsFakeDependencies{}).absoluteCurDir() + + // testspec specifies a test case for this test + type testspec struct { + // name is the name of the test case + name string + + // target is the target to build + target string + + // expectations contains the commands we expect to see + expect []buildtooltest.ExecExpectations + } + + var testcases = []testspec{{ + name: "we can build zlib", + target: "zlib", + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://zlib.net/zlib-1.2.13.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "zlib-1.2.13.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/zlib/000.patch", + }, + }, { + Env: []string{ + "CFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong -fstack-clash-protection -fPIC -fsanitize=bounds -fsanitize-undefined-trap-on-error -O2", + }, + Argv: []string{ + "./configure", "--prefix=/", "--static", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64", + "install", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/pkgconfig", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/share", + }, + }}, + }, { + name: "we can build openssl", + target: "openssl", + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-1.1.1s.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "openssl-1.1.1s.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/openssl/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/openssl/001.patch", + }, + }, { + Env: []string{ + "CFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong -fstack-clash-protection -fPIC -fsanitize=bounds -fsanitize-undefined-trap-on-error -O2 -Wno-macro-redefined", + }, + Argv: []string{ + "./Configure", "no-comp", "no-dtls", "no-ec2m", "no-psk", "no-srp", + "no-ssl2", "no-ssl3", "no-camellia", "no-idea", "no-md2", "no-md4", + "no-mdc2", "no-rc2", "no-rc4", "no-rc5", "no-rmd160", "no-whirlpool", + "no-dso", "no-hw", "no-ui-console", "no-shared", "no-unit-test", + "linux-x86_64", "--libdir=lib", "--prefix=/", "--openssldir=/", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64", + "install_dev", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/pkgconfig", + }, + }}, + }, { + name: "we can build libevent", + target: "libevent", + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{ + "curl", + "-fsSLO", + "https://github.com/libevent/libevent/archive/release-2.1.12-stable.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "release-2.1.12-stable.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/001.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/002.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "./autogen.sh", + }, + }, { + Env: []string{ + fmt.Sprintf( + "CFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong -fstack-clash-protection -fPIC -fsanitize=bounds -fsanitize-undefined-trap-on-error -O2 -I%s/internal/cmd/buildtool/internal/libtor/linux/amd64/include", + faketopdir, + ), + fmt.Sprintf( + "LDFLAGS=-L%s/internal/cmd/buildtool/internal/libtor/linux/amd64/lib", + faketopdir, + ), + }, + Argv: []string{ + "./configure", + "--disable-libevent-regress", + "--disable-samples", + "--disable-shared", + "--prefix=/", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64", + "install", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/bin", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/pkgconfig", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_core.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_core.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_extra.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_extra.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_openssl.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_openssl.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_pthreads.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib/libevent_pthreads.la", + }, + }}, + }, { + name: "we can build tor", + target: "tor", + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.7.12.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "tor-0.4.7.12.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/001.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/002.patch", + }, + }, { + Env: []string{ + "CFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong -fstack-clash-protection -fPIC -fsanitize=bounds -fsanitize-undefined-trap-on-error -O2", + "LDFLAGS=", + }, + Argv: []string{ + "./configure", + "--enable-pic", + "--enable-static-libevent", + "--with-libevent-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64", + "--enable-static-openssl", + "--with-openssl-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64", + "--enable-static-zlib", + "--with-zlib-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64", + "--disable-module-dirauth", + "--disable-zstd", + "--disable-lzma", + "--disable-tool-name-check", + "--disable-systemd", + "--prefix=/", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "install", "-m644", "src/feature/api/tor_api.h", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/include", + }, + }, { + Env: []string{}, + Argv: []string{ + "install", "-m644", "libtor.a", + faketopdir + "/internal/cmd/buildtool/internal/libtor/linux/amd64/lib", + }, + }}, + }} + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + cc := &buildtooltest.SimpleCommandCollector{} + + shellxtesting.WithCustomLibrary(cc, func() { + linuxCdepsBuildMain(testcase.target, &cdepsFakeDependencies{}) + }) + + if err := buildtooltest.CheckManyCommands(cc.Commands, testcase.expect); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/libtor/android/386/.gitignore b/internal/libtor/android/386/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/internal/libtor/android/386/.gitignore @@ -0,0 +1 @@ +* diff --git a/internal/libtor/android/amd64/.gitignore b/internal/libtor/android/amd64/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/internal/libtor/android/amd64/.gitignore @@ -0,0 +1 @@ +* diff --git a/internal/libtor/android/arm/.gitignore b/internal/libtor/android/arm/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/internal/libtor/android/arm/.gitignore @@ -0,0 +1 @@ +* diff --git a/internal/libtor/android/arm64/.gitignore b/internal/libtor/android/arm64/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/internal/libtor/android/arm64/.gitignore @@ -0,0 +1 @@ +* diff --git a/internal/libtor/enabled.go b/internal/libtor/enabled.go new file mode 100644 index 0000000000..27dd5ac1e7 --- /dev/null +++ b/internal/libtor/enabled.go @@ -0,0 +1,277 @@ +//go:build ooni_libtor + +package libtor + +// Adapted from https://github.com/cretz/bine +// SPDX-License-Identifier: MIT + +// +// #cgo linux,amd64 CFLAGS: -I${SRCDIR}/linux/amd64/include +// #cgo linux,amd64 LDFLAGS: -L${SRCDIR}/linux/amd64/lib -ltor -levent -lssl -lcrypto -lz -lm +// +// #cgo android,arm CFLAGS: -I${SRCDIR}/android/arm/include +// #cgo android,arm LDFLAGS: -L${SRCDIR}/android/arm/lib -ltor -levent -lssl -lcrypto -lz +// #cgo android,arm64 CFLAGS: -I${SRCDIR}/android/arm64/include +// #cgo android,arm64 LDFLAGS: -L${SRCDIR}/android/arm64/lib -ltor -levent -lssl -lcrypto -lz +// #cgo android,386 CFLAGS: -I${SRCDIR}/android/386/include +// #cgo android,386 LDFLAGS: -L${SRCDIR}/android/386/lib -ltor -levent -lssl -lcrypto -lz +// #cgo android,amd64 CFLAGS: -I${SRCDIR}/android/amd64/include +// #cgo android,amd64 LDFLAGS: -L${SRCDIR}/android/amd64/lib -ltor -levent -lssl -lcrypto -lz +// +// #include +// #include +// #include +// +// #include +// +// /* Note: we need to define inline helpers because we cannot index C arrays in Go. */ +// +// static char **cstringArrayNew(size_t size) { +// char **argv = calloc(size, sizeof(char *)); +// if (argv == NULL) { +// abort(); +// } +// return argv; +// } +// +// static void cstringArraySet(char **argv, size_t index, char *entry) { +// argv[index] = entry; +// } +// +// static void cstringArrayFree(char **argv, size_t size) { +// for (size_t idx = 0; idx < size; idx++) { +// free(argv[idx]); +// } +// free(argv); +// } +// +// static bool filedescIsGood(tor_control_socket_t fd) { +// return fd != INVALID_TOR_CONTROL_SOCKET; +// } +// +import "C" + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "sync" + + "github.com/cretz/bine/process" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// MaybeCreator returns a valid [process.Creator], if possible, otherwise false. +func MaybeCreator() (process.Creator, bool) { + return &torCreator{}, true +} + +// torCreator implements [process.Creator]. +type torCreator struct{} + +var _ process.Creator = &torCreator{} + +// New implements [process.Creator]. +func (c *torCreator) New(ctx context.Context, args ...string) (process.Process, error) { + left, right := net.Pipe() + proc := &torProcess{ + awaitStart: make(chan any, 1), // buffer + controlConn: left, + startErr: make(chan error, 1), // buffer + startOnce: sync.Once{}, + waitErr: make(chan error, 1), // buffer + waitOnce: sync.Once{}, + + closedWhenNotStarted: make(chan any, 1), // buffer + simulateBadControlSocket: false, + simulateFileConnFailure: false, + simulateNonzeroExitCode: false, + } + go proc.runtor(ctx, right, args...) + return proc, nil +} + +// torProcess implements [process.Process]. +type torProcess struct { + // ordinary state variables + awaitStart chan any + controlConn net.Conn + startErr chan error + startOnce sync.Once + waitErr chan error + waitOnce sync.Once + + // for testing + closedWhenNotStarted chan any + simulateBadControlSocket bool + simulateFileConnFailure bool + simulateNonzeroExitCode bool +} + +var _ process.Process = &torProcess{} + +// EmbeddedControlConn implements [process.Process]. +func (p *torProcess) EmbeddedControlConn() (net.Conn, error) { + // Implementation note: this function SHOULD only be called + // once and BEFORE Start is called 😬😬😬 + return p.controlConn, nil +} + +// Start implements [process.Process]. +func (p *torProcess) Start() (err error) { + p.startOnce.Do(func() { + p.awaitStart <- true + err = <-p.startErr + }) + return err +} + +// Wait implements [process.Process]. +func (p *torProcess) Wait() (err error) { + p.waitOnce.Do(func() { + err = <-p.waitErr + }) + return +} + +// ErrTooManyArguments indicates that p.args contains too many arguments +var ErrTooManyArguments = errors.New("libtor: too many arguments") + +// ErrCannotCreateControlSocket indicates that we cannot create a control socket. +var ErrCannotCreateControlSocket = errors.New("libtor: cannot create a control socket") + +// ErrNonzeroExitCode indicates that tor returned a nonzero exit code +var ErrNonzeroExitCode = errors.New("libtor: command completed with nonzero exit code") + +// runtor runs tor until completion and ensures that tor exits when +// the given ctx is cancelled or its deadline expires. +func (p *torProcess) runtor(ctx context.Context, cc net.Conn, args ...string) { + // wait for Start or context to expire + select { + case <-p.awaitStart: + case <-ctx.Done(): + p.startErr <- ctx.Err() // nonblocking chan + close(p.closedWhenNotStarted) + return + } + + // Note: when writing this code I was wondering whether I needed to + // use unsafe.Pointer to track pointers that matter to C code. Reading + // this message[1] has been useful to understand that the most likely + // answer to this question is "obviously, no". + // + // See https://groups.google.com/g/golang-nuts/c/yNis7bQG_rY/m/yaJFoSx1hgIJ + + // Create argc and argv for tor + argv := append([]string{"tor"}, args...) + const toomany = 256 // arbitrary low limit to make C.int and C.size_t casts always work + if len(argv) > toomany { + p.startErr <- ErrTooManyArguments // nonblocking channel + return + } + argc := C.size_t(len(argv)) + // Note: here we allocate argc + 1 because a "null pointer always follows + // the last element: argv[argc] is this null pointer." + // + // See https://www.gnu.org/software/libc/manual/html_node/Program-Arguments.html + allocSiz := argc + 1 + cargv := C.cstringArrayNew(allocSiz) + defer C.cstringArrayFree(cargv, argc) + for idx, entry := range argv { + C.cstringArraySet(cargv, C.size_t(idx), C.CString(entry)) + } + + // Add to config a WEAK REFERENCE to argc and argv + config := C.tor_main_configuration_new() + runtimex.PanicIfNil(config, "C.tor_main_configuration_new failed") + defer C.tor_main_configuration_free(config) + code := C.tor_main_configuration_set_command_line(config, C.int(argc), cargv) + runtimex.Assert(code == 0, "C.tor_main_configuration_set_command_line failed") + + // Create OWNING file descriptor + filedesc := C.tor_main_configuration_setup_control_socket(config) + if p.simulateBadControlSocket { + filedesc = C.INVALID_TOR_CONTROL_SOCKET + } + if !C.filedescIsGood(filedesc) { + p.startErr <- ErrCannotCreateControlSocket // nonblocking channel + return + } + + // Convert the OWNING file descriptor into a proper file. Because + // filedesc is good, os.NewFile shouldn't fail. + filep := os.NewFile(uintptr(filedesc), "") + runtimex.Assert(filep != nil, "os.NewFile should not fail") + conn, err := net.FileConn(filep) + if p.simulateFileConnFailure { + err = ErrCannotCreateControlSocket + } + if err != nil { + p.startErr <- err // nonblocking channel + return + } + + // In the following we're going to possibly call Close multiple + // times. Let's be very sure that this close is idempotent. + conn = withIdempotentClose(conn) + cc = withIdempotentClose(cc) + + // Make sure we close filep when the context is done. Because the + // socket is OWNING, this will also cause tor to return. + go func() { + defer conn.Close() + defer cc.Close() + <-ctx.Done() + }() + + // Route messages from and to the control connection. + go sendrecvThenClose(cc, conn) + go sendrecvThenClose(conn, cc) + + // Let the user know that startup was successful. + p.startErr <- nil // nonblocking channel + + // Run tor until completion. + if !p.simulateNonzeroExitCode { + code = C.tor_run_main(config) + } else { + code = 1 + } + if code != 0 { + p.waitErr <- fmt.Errorf("%w: %d", ErrNonzeroExitCode, code) // nonblocking channel + return + } + p.waitErr <- nil // nonblocking channel +} + +// sendrecvThenClose routes traffic between two connections and then +// closes both of them when done with routing traffic. +func sendrecvThenClose(left, right net.Conn) { + defer left.Close() + defer right.Close() + netxlite.CopyContext(context.Background(), left, right) +} + +// withIdempotentClose ensures that a connection has idempotent close. +func withIdempotentClose(c net.Conn) net.Conn { + return &idempotentClose{ + Conn: c, + once: sync.Once{}, + } +} + +// idempotentClose ensures close is idempotent for a net.Conn +type idempotentClose struct { + net.Conn + once sync.Once +} + +func (c *idempotentClose) Close() (err error) { + c.once.Do(func() { + err = c.Conn.Close() + }) + return +} diff --git a/internal/libtor/enabled_test.go b/internal/libtor/enabled_test.go new file mode 100644 index 0000000000..d749bc0d55 --- /dev/null +++ b/internal/libtor/enabled_test.go @@ -0,0 +1,255 @@ +//go:build ooni_libtor + +package libtor + +import ( + "context" + "errors" + "io" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/cretz/bine/tor" +) + +func TestNormalUsage(t *testing.T) { + ctx := context.Background() + + datadir, err := filepath.Abs(filepath.Join("testdata", "datadir")) + if err != nil { + t.Fatal(err) + } + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + config := &tor.StartConf{ + ProcessCreator: creator, + DataDir: datadir, + ExtraArgs: nil, + NoHush: true, + } + + instance, err := tor.Start(ctx, config) + if err != nil { + t.Fatal(err) + } + defer instance.Close() + + if err := instance.EnableNetwork(context.Background(), true); err != nil { + t.Fatal(err) + } +} + +func TestContextAlreadyExpired(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // fail immediately + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + process, err := creator.New(ctx) + if err != nil { + t.Fatal(err) + } + + // Make sure that start happens after the background + // goroutine noticing that ctx is cancelled. + <-process.(*torProcess).closedWhenNotStarted + + if err := process.Start(); !errors.Is(err, context.Canceled) { + t.Fatal("unexpected err", err) + } +} + +func TestTooManyCommandLineArguments(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + argv := make([]string, 4096) + process, err := creator.New(ctx, argv...) + if err != nil { + t.Fatal(err) + } + + if err := process.Start(); !errors.Is(err, ErrTooManyArguments) { + t.Fatal("unexpected err", err) + } +} + +func TestSetupControlSocketFails(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + process, err := creator.New(ctx) + if err != nil { + t.Fatal(err) + } + process.(*torProcess).simulateBadControlSocket = true + + if err := process.Start(); !errors.Is(err, ErrCannotCreateControlSocket) { + t.Fatal("unexpected err", err) + } +} + +func TestFileConnFails(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + process, err := creator.New(ctx) + if err != nil { + t.Fatal(err) + } + process.(*torProcess).simulateFileConnFailure = true + + if err := process.Start(); !errors.Is(err, ErrCannotCreateControlSocket) { + t.Fatal("unexpected err", err) + } +} + +func TestNonzeroExitCode(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + process, err := creator.New(ctx) + if err != nil { + t.Fatal(err) + } + process.(*torProcess).simulateNonzeroExitCode = true + + if err := process.Start(); err != nil { + t.Fatal(err) + } + + if err := process.Wait(); !errors.Is(err, ErrNonzeroExitCode) { + t.Fatal("unexpected err", err) + } +} + +func TestContextCanceledWhileTorIsRunning(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + process, err := creator.New(ctx) + if err != nil { + t.Fatal(err) + } + + cconn, err := process.EmbeddedControlConn() + if err != nil { + t.Fatal(err) + } + + if err := process.Start(); err != nil { + t.Fatal(err) + } + + message := []byte("SETEVENTS STATUS_CLIENT\r\n") + if _, err := cconn.Write(message); err != nil { + t.Fatal(err) + } + + for { + message := make([]byte, 1<<20) + count, err := cconn.Read(message) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatal(err) + } + message = message[:count] + t.Log(strings.Trim(string(message), "\r\n")) + } + + if err := process.Wait(); err != nil { + t.Fatal(err) + } +} + +// In theory this case SHOULD NOT happen judging from the description of +// the EmbeddedControlConn method, which reads: +// +// Note, this should only be called once per process before +// Start, and the connection does not need to be closed. +// +// However, it MIGHT happen. So, let's also cover this possibility. +func TestControlConnectionExplicitlyClosed(t *testing.T) { + ctx := context.Background() + + creator, good := MaybeCreator() + if !good { + t.Fatal("expected to see true here") + } + + process, err := creator.New(ctx) + if err != nil { + t.Fatal(err) + } + + cconn, err := process.EmbeddedControlConn() + if err != nil { + t.Fatal(err) + } + + go func() { + <-time.After(2 * time.Second) + cconn.Close() + }() + + if err := process.Start(); err != nil { + t.Fatal(err) + } + + message := []byte("SETEVENTS STATUS_CLIENT\r\n") + if _, err := cconn.Write(message); err != nil { + t.Fatal(err) + } + + for { + message := make([]byte, 1<<20) + count, err := cconn.Read(message) + if errors.Is(err, io.ErrClosedPipe) { + break + } + if err != nil { + t.Fatal(err) + } + message = message[:count] + t.Log(strings.Trim(string(message), "\r\n")) + } + + if err := process.Wait(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/libtor/fallback.go b/internal/libtor/fallback.go new file mode 100644 index 0000000000..069d303bef --- /dev/null +++ b/internal/libtor/fallback.go @@ -0,0 +1,10 @@ +//go:build !ooni_libtor + +package libtor + +import "github.com/cretz/bine/process" + +// MaybeCreator returns a valid [process.Creator], if possible, otherwise false. +func MaybeCreator() (process.Creator, bool) { + return nil, false +} diff --git a/internal/libtor/linux/amd64/.gitignore b/internal/libtor/linux/amd64/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/internal/libtor/linux/amd64/.gitignore @@ -0,0 +1 @@ +* diff --git a/internal/libtor/testdata/.gitignore b/internal/libtor/testdata/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/internal/libtor/testdata/.gitignore @@ -0,0 +1 @@ +* diff --git a/internal/tunnel/tordesktop.go b/internal/tunnel/tordesktop.go index bf1a7a08cb..7a0b55544e 100644 --- a/internal/tunnel/tordesktop.go +++ b/internal/tunnel/tordesktop.go @@ -1,8 +1,9 @@ -//go:build !android && !ios +//go:build !android && !ios && !ooni_libtor package tunnel -// This file implements our strategy for running tor on desktop. +// This file implements our strategy for running tor on desktop in most +// configurations except for experimental ones. import ( "strings" diff --git a/internal/tunnel/torembed.go b/internal/tunnel/torembed.go new file mode 100644 index 0000000000..770190608a --- /dev/null +++ b/internal/tunnel/torembed.go @@ -0,0 +1,30 @@ +//go:build ooni_libtor + +package tunnel + +// This file implements an experimental strategy for running tor. + +import ( + "errors" + "strings" + + "github.com/cretz/bine/tor" + "github.com/ooni/probe-cli/v3/internal/libtor" +) + +// getTorStartConf in this configuration returns a tor.StartConf +// configured to run the version of tor we embed as a library. +func getTorStartConf(config *Config, dataDir string, extraArgs []string) (*tor.StartConf, error) { + creator, good := libtor.MaybeCreator() + if !good { + return nil, errors.New("no embedded tor") + } + config.logger().Infof("tunnel: tor: exec: %s %s", + dataDir, strings.Join(extraArgs, " ")) + return &tor.StartConf{ + ProcessCreator: creator, + DataDir: dataDir, + ExtraArgs: extraArgs, + NoHush: true, + }, nil +}