From 6790cdc5ee386208316c71add375bb6fc53b6881 Mon Sep 17 00:00:00 2001 From: ain ghazal Date: Tue, 8 Oct 2024 17:07:12 +0200 Subject: [PATCH] feat: add fallback domain names while working on this, I also gave more priority to possible oonirun descriptors passed in the command line. - Related: #2805 --- internal/experiment/openvpn/endpoint.go | 8 ++ internal/experiment/openvpn/openvpn.go | 2 +- internal/experiment/openvpn/openvpn_test.go | 2 +- internal/experiment/openvpn/richerinput.go | 78 ++++++++---- internal/experiment/openvpn/targets.go | 129 +++++++++++++++++--- internal/experiment/openvpn/targets_test.go | 26 ++-- 6 files changed, 182 insertions(+), 63 deletions(-) diff --git a/internal/experiment/openvpn/endpoint.go b/internal/experiment/openvpn/endpoint.go index df255527d..9c41a62c4 100644 --- a/internal/experiment/openvpn/endpoint.go +++ b/internal/experiment/openvpn/endpoint.go @@ -25,6 +25,10 @@ type endpoint struct { // IPAddr is the IP Address for this endpoint. IPAddr string + // DomainName is an optional domain name that we use internally to get the IP address. + // This is just a convenience field, the experiments should always be done against a canonical IPAddr. + DomainName string + // Obfuscation is any obfuscation method use to connect to this endpoint. // Valid values are: obfs4, none. Obfuscation string @@ -32,6 +36,10 @@ type endpoint struct { // Port is the Port for this endpoint. Port string + // PreferredCountries is an optional array of country codes. Probes in these countries have preference on this + // endpoint. + PreferredCountries []string + // Protocol is the tunneling protocol (openvpn, openvpn+obfs4). Protocol string diff --git a/internal/experiment/openvpn/openvpn.go b/internal/experiment/openvpn/openvpn.go index 17faa139c..f49396877 100644 --- a/internal/experiment/openvpn/openvpn.go +++ b/internal/experiment/openvpn/openvpn.go @@ -17,7 +17,7 @@ import ( const ( testName = "openvpn" - testVersion = "0.1.5" + testVersion = "0.1.6" openVPNProtocol = "openvpn" ) diff --git a/internal/experiment/openvpn/openvpn_test.go b/internal/experiment/openvpn/openvpn_test.go index 73467ddc5..b7d6a4b69 100644 --- a/internal/experiment/openvpn/openvpn_test.go +++ b/internal/experiment/openvpn/openvpn_test.go @@ -41,7 +41,7 @@ func TestNewExperimentMeasurer(t *testing.T) { if m.ExperimentName() != "openvpn" { t.Fatal("invalid ExperimentName") } - if m.ExperimentVersion() != "0.1.5" { + if m.ExperimentVersion() != "0.1.6" { t.Fatal("invalid ExperimentVersion") } } diff --git a/internal/experiment/openvpn/richerinput.go b/internal/experiment/openvpn/richerinput.go index 1439add60..f3c88fc14 100644 --- a/internal/experiment/openvpn/richerinput.go +++ b/internal/experiment/openvpn/richerinput.go @@ -84,6 +84,13 @@ type targetLoader struct { // Load implements model.ExperimentTargetLoader. func (tl *targetLoader) Load(ctx context.Context) ([]model.ExperimentTarget, error) { + // First, attempt to load the static inputs from CLI and files + inputs, err := targetloading.LoadStatic(tl.loader) + // Handle the case where we couldn't load from CLI or files (fallthru) + if err != nil { + tl.loader.Logger.Warnf("Error loading OpenVPN targets from cli") + } + // If inputs and files are all empty and there are no options, let's use the backend if len(tl.loader.StaticInputs) <= 0 && len(tl.loader.SourceFiles) <= 0 && reflectx.StructOrStructPtrIsZero(tl.options) { @@ -91,16 +98,7 @@ func (tl *targetLoader) Load(ctx context.Context) ([]model.ExperimentTarget, err if err == nil { return targets, nil } - } - - tl.loader.Logger.Warnf("Error fetching OpenVPN targets from backend") - - // Otherwise, attempt to load the static inputs from CLI and files - inputs, err := targetloading.LoadStatic(tl.loader) - - // Handle the case where we couldn't load from CLI or files: - if err != nil { - return nil, err + tl.loader.Logger.Warnf("Error fetching OpenVPN targets from backend") } // Build the list of targets that we should measure. @@ -119,22 +117,52 @@ func (tl *targetLoader) Load(ctx context.Context) ([]model.ExperimentTarget, err return tl.loadFromDefaultEndpoints() } -func (tl *targetLoader) loadFromDefaultEndpoints() ([]model.ExperimentTarget, error) { - tl.loader.Logger.Warnf("Using default OpenVPN endpoints") +func makeTargetListPerProtocol(cc string, num int) []model.ExperimentTarget { targets := []model.ExperimentTarget{} - if udp, err := defaultOONIOpenVPNTargetUDP(); err == nil { - targets = append(targets, - &Target{ - Config: pickFromDefaultOONIOpenVPNConfig(), - URL: udp, - }) + var reverse bool + switch num { + case 1, 2: + // for single or few picks, we start the list in the natural order + reverse = false + default: + // for multiple picks, we start the list from the bottom, so that we can lookup + // custom country campaigns first. + reverse = true + } + if inputsUDP, err := pickOONIOpenVPNTargets("udp", cc, num, reverse); err == nil { + for _, t := range inputsUDP { + targets = append(targets, + &Target{ + Config: pickFromDefaultOONIOpenVPNConfig(), + URL: t, + }) + } } - if tcp, err := defaultOONIOpenVPNTargetTCP(); err == nil { - targets = append(targets, - &Target{ - Config: pickFromDefaultOONIOpenVPNConfig(), - URL: tcp, - }) + if inputsTCP, err := pickOONIOpenVPNTargets("tcp", cc, num, reverse); err == nil { + for _, t := range inputsTCP { + targets = append(targets, + &Target{ + Config: pickFromDefaultOONIOpenVPNConfig(), + URL: t, + }) + } + } + return targets +} + +func (tl *targetLoader) loadFromDefaultEndpoints() ([]model.ExperimentTarget, error) { + cc := tl.session.ProbeCC() + + tl.loader.Logger.Warnf("Using default OpenVPN endpoints") + tl.loader.Logger.Warnf("Picking endpoints for %s", cc) + + var targets []model.ExperimentTarget + switch cc { + case "RU", "CN", "IR", "EG", "NL": + // we want to cover all of our bases for a few interest countries + targets = makeTargetListPerProtocol(cc, 20) + default: + targets = makeTargetListPerProtocol(cc, 1) } return targets, nil } @@ -160,8 +188,6 @@ func (tl *targetLoader) loadFromBackend(ctx context.Context) ([]model.Experiment for _, input := range apiConfig.Inputs { config := &Config{ - // TODO(ainghazal): Auth and Cipher are hardcoded for now. - // Backend should provide them as richer input; and if empty we can use these as defaults. Auth: "SHA512", Cipher: "AES-256-GCM", } diff --git a/internal/experiment/openvpn/targets.go b/internal/experiment/openvpn/targets.go index 6a85b48ca..e797760a4 100644 --- a/internal/experiment/openvpn/targets.go +++ b/internal/experiment/openvpn/targets.go @@ -4,9 +4,87 @@ import ( "fmt" "math/rand" "net" + "slices" ) -const defaultOpenVPNEndpoint = "openvpn-server1.ooni.io" +// defaultOpenVPNEndpoints contain a list of all default endpoints +// to be tried, in the order that we want the name resolution to happen. +var defaultOpenVPNEndpoints = []endpoint{ + // default domain. this should work fine for most places. + { + IPAddr: "", + DomainName: "openvpn-server1.ooni.io", + Obfuscation: "none", + Port: "1194", + Protocol: "openvpn", + Provider: "oonivpn", + Transport: "tcp", + }, + { + IPAddr: "", + DomainName: "openvpn-server1.ooni.io", + Obfuscation: "none", + Port: "1194", + Protocol: "openvpn", + Provider: "oonivpn", + Transport: "udp", + }, + // alt domain 1. still same endpoint ports, one udp and one tcp. + // TODO(ain): update to real domain names + { + IPAddr: "", + DomainName: "alt-domain1.example.org", + Obfuscation: "none", + Port: "1194", + Protocol: "openvpn", + Provider: "oonivpn", + Transport: "udp", + }, + { + IPAddr: "", + DomainName: "alt-domain1.example.org", + Obfuscation: "none", + Port: "1194", + Protocol: "openvpn", + Provider: "oonivpn", + Transport: "tcp", + }, + // alt domain 2. still same endpoint ports, one udp and one tcp. + // TODO(ain): update to real domain names + { + IPAddr: "", + DomainName: "alt-domain2.example.org", + Obfuscation: "none", + Port: "53", + Protocol: "openvpn", + Provider: "oonivpn", + Transport: "udp", + }, + { + IPAddr: "", + DomainName: "alt-domain2.example.org", + Obfuscation: "none", + Port: "443", + Protocol: "openvpn", + Provider: "oonivpn", + Transport: "tcp", + }, + // alt domain 3. this is reserved. + // TODO(ain): update to real domain names + { + IPAddr: "", + DomainName: "alt-domain3.example.org", + Obfuscation: "none", + Port: "443", + Protocol: "openvpn", + Provider: "oonivpn", + Transport: "tcp", + PreferredCountries: []string{ + "AM", "AZ", "BY", "GE", "KZ", "KG", "LT", "MD", "RU", "TJ", "TM", "UA", "UZ", + "IR", "CN", "EG"}, + }, + // TODO: add more backup domains here +} // this is a safety toggle: it's on purpose that the experiment will receive no // input if the resolution fails. This also implies that we have no way of knowing if this @@ -15,6 +93,7 @@ const defaultOpenVPNEndpoint = "openvpn-server1.ooni.io" // and perhaps also transform DNS failure into a specific failure of the experiment, not // a skip. // TODO(ain): update the openvpn spec to reflect the CURRENT state of delivering the targets. +// If the probe services ever gets deployed, this step will not be needed anymore. func resolveTarget(domain string) (string, error) { ips, err := net.LookupIP(domain) if err != nil { @@ -23,27 +102,43 @@ func resolveTarget(domain string) (string, error) { if len(ips) > 0 { return ips[0].String(), nil } - return "", fmt.Errorf("cannot resolve %v", defaultOpenVPNEndpoint) -} - -func defaultOONITargetURL(ip string) string { - return "openvpn://oonivpn.corp/?address=" + ip + ":1194" + return "", fmt.Errorf("cannot resolve %v", domain) } -func defaultOONIOpenVPNTargetUDP() (string, error) { - ip, err := resolveTarget(defaultOpenVPNEndpoint) - if err != nil { - return "", err +// pickOONIOpenVPNTargets returns an array of input URIs from the list of available endpoints, up to max, +// for the given transport. By default, we use the first endpoint that resolves to an IP. If reverseOrder +// is specified, we reverse the list before attempting resolution. +func pickOONIOpenVPNTargets(transport string, cc string, max int, reverseOrder bool) ([]string, error) { + endpoints := slices.Clone(defaultOpenVPNEndpoints)[:] + if reverseOrder { + slices.Reverse(endpoints) } - return defaultOONITargetURL(ip) + "&transport=udp", nil -} + targets := make([]string, 0) + for _, endpoint := range endpoints { + if endpoint.Transport != transport { + continue + } + if len(endpoint.PreferredCountries) > 0 && !slices.Contains(endpoint.PreferredCountries, cc) { + // not for us + continue + } + // Do note that this will get the wrong result if we got DNS poisoning. + // When analyzing this data, you should be careful about bogus IPs. + ip, err := resolveTarget(endpoint.DomainName) + if err != nil { + continue + } + endpoint.IPAddr = ip -func defaultOONIOpenVPNTargetTCP() (string, error) { - ip, err := resolveTarget(defaultOpenVPNEndpoint) - if err != nil { - return "", err + targets = append(targets, endpoint.AsInputURI()) + if len(targets) == max { + return targets, nil + } + } + if len(targets) > 0 { + return targets, nil } - return defaultOONITargetURL(ip) + "&transport=tcp", nil + return nil, fmt.Errorf("cannot find any endpoint for %s", transport) } func pickFromDefaultOONIOpenVPNConfig() *Config { diff --git a/internal/experiment/openvpn/targets_test.go b/internal/experiment/openvpn/targets_test.go index 480be65f5..704750bb8 100644 --- a/internal/experiment/openvpn/targets_test.go +++ b/internal/experiment/openvpn/targets_test.go @@ -1,6 +1,7 @@ package openvpn import ( + "net/url" "testing" "github.com/google/go-cmp/cmp" @@ -35,8 +36,8 @@ func Test_resolveTarget(t *testing.T) { } } -func Test_defaultOONIOpenVPNTargetUDP(t *testing.T) { - url, err := defaultOONIOpenVPNTargetUDP() +func Test_pickOpenVPNTargets(t *testing.T) { + urls, err := pickOONIOpenVPNTargets("udp", "IT", 1, false) if err != nil { if err.Error() == "connection_refused" { // connection_refused is raised when running this test @@ -46,25 +47,14 @@ func Test_defaultOONIOpenVPNTargetUDP(t *testing.T) { } t.Fatal("unexpected error") } - expected := "openvpn://oonivpn.corp/?address=37.218.243.98:1194&transport=udp" - if diff := cmp.Diff(url, expected); diff != "" { - t.Fatal(diff) - } -} + expected := "openvpn://oonivpn.corp?address=37.218.243.98:1194&transport=udp" -func Test_defaultOONIOpenVPNTargetTCP(t *testing.T) { - url, err := defaultOONIOpenVPNTargetTCP() + got, err := url.QueryUnescape(urls[0]) if err != nil { - if err.Error() == "connection_refused" { - // connection_refused is raised when running this test - // on the restricted network for coverage tests. - // so we bail out - return - } - t.Fatal("unexpected error") + t.Fatal(err) } - expected := "openvpn://oonivpn.corp/?address=37.218.243.98:1194&transport=tcp" - if diff := cmp.Diff(url, expected); diff != "" { + + if diff := cmp.Diff(got, expected); diff != "" { t.Fatal(diff) } }