poc: running experiments bypassing internal/engine

This PoC investigates whether it would be possible to run
experiments directly without using the internal/engine
abstraction as the middle man.

The PoC is in the context of ooni/ooni.org#1295

internal/cmd/dismantle/backendclient/backendclient.go

@@ -0,0 +1,67 @@
+package backendclient
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/httpapi"
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/ooapi"
+)
+
+type Config struct {
+	KVStore    model.KeyValueStore
+	HTTPClient model.HTTPClient
+	Logger     model.Logger
+	UserAgent  string
+
+	// optional fields
+	BaseURL  *url.URL
+	ProxyURL *url.URL
+}
+
+type Client struct {
+	endpoint *httpapi.Endpoint
+}
+
+func New(config *Config) *Client {
+	baseURL := "https://api.ooni.io/"
+	if config.BaseURL != nil {
+		baseURL = config.BaseURL.String()
+	}
+	endpoint := &httpapi.Endpoint{
+		BaseURL:    baseURL,
+		HTTPClient: config.HTTPClient,
+		Host:       "",
+		Logger:     config.Logger,
+		UserAgent:  config.UserAgent,
+	}
+	backendClient := &Client{
+		endpoint: endpoint,
+	}
+	return backendClient
+}
+
+func (c *Client) CheckIn(
+	ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) {
+	return httpapi.Call(ctx, ooapi.NewDescriptorCheckIn(config), c.endpoint)
+}
+
+func (c *Client) FetchPsiphonConfig(ctx context.Context) ([]byte, error) {
+	panic("not implemented")
+}
+
+func (c *Client) FetchTorTargets(
+	ctx context.Context, cc string) (result map[string]model.OOAPITorTarget, err error) {
+	panic("not implemented")
+}
+
+func (c *Client) Submit(ctx context.Context, m *model.Measurement) error {
+	req := &model.OOAPICollectorUpdateRequest{
+		Format:  "json",
+		Content: m,
+	}
+	descriptor := newSubmitDescriptor(req, m.ReportID)
+	_, err := httpapi.Call(ctx, descriptor, c.endpoint)
+	return err
+}

internal/cmd/dismantle/backendclient/measurement.go

@@ -0,0 +1,57 @@
+package backendclient
+
+import (
+	"fmt"
+	"runtime"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/geolocate"
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/platform"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/version"
+)
+
+const dateFormat = "2006-01-02 15:04:05"
+
+func NewMeasurement(
+	location *geolocate.Results,
+	testName string,
+	testVersion string,
+	testStartTime time.Time,
+	reportID string,
+	softwareName string,
+	softwareVersion string,
+	input string,
+) *model.Measurement {
+	utctimenow := time.Now().UTC()
+	m := &model.Measurement{
+		DataFormatVersion:         model.OOAPIReportDefaultDataFormatVersion,
+		Input:                     model.MeasurementTarget(input),
+		MeasurementStartTime:      utctimenow.Format(dateFormat),
+		MeasurementStartTimeSaved: utctimenow,
+		ProbeIP:                   model.DefaultProbeIP,
+		ProbeASN:                  location.ASNString(),
+		ProbeCC:                   location.CountryCode,
+		ProbeNetworkName:          location.NetworkName,
+		ReportID:                  reportID,
+		ResolverASN:               fmt.Sprintf("AS%d", location.ResolverASN), // XXX
+		ResolverIP:                location.ResolverIP,
+		ResolverNetworkName:       location.ResolverNetworkName,
+		SoftwareName:              softwareName,
+		SoftwareVersion:           softwareVersion,
+		TestName:                  testName,
+		TestStartTime:             testStartTime.Format(dateFormat),
+		TestVersion:               testVersion,
+	}
+	m.AddAnnotation("architecture", runtime.GOARCH)
+	m.AddAnnotation("engine_name", "ooniprobe-engine")
+	m.AddAnnotation("engine_version", version.Version)
+	m.AddAnnotation("go_version", runtimex.BuildInfo.GoVersion)
+	m.AddAnnotation("platform", platform.Name())
+	m.AddAnnotation("vcs_modified", runtimex.BuildInfo.VcsModified)
+	m.AddAnnotation("vcs_revision", runtimex.BuildInfo.VcsRevision)
+	m.AddAnnotation("vcs_time", runtimex.BuildInfo.VcsTime)
+	m.AddAnnotation("vcs_tool", runtimex.BuildInfo.VcsTool)
+	return m
+}

internal/cmd/dismantle/backendclient/submitter.go

@@ -0,0 +1,34 @@
+package backendclient
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/ooni/probe-cli/v3/internal/httpapi"
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
+)
+
+func newSubmitDescriptor(
+	req *model.OOAPICollectorUpdateRequest, reportID string) *httpapi.Descriptor[
+	*model.OOAPICollectorUpdateRequest, *model.OOAPICollectorUpdateResponse] {
+	rawBody, err := json.Marshal(req)
+	runtimex.PanicOnError(err, "json.Marshal failed")
+	return &httpapi.Descriptor[*model.OOAPICollectorUpdateRequest, *model.OOAPICollectorUpdateResponse]{
+		Accept:             httpapi.ApplicationJSON,
+		Authorization:      "",
+		AcceptEncodingGzip: false,
+		ContentType:        httpapi.ApplicationJSON,
+		LogBody:            true,
+		MaxBodySize:        0,
+		Method:             http.MethodPost,
+		Request: &httpapi.RequestDescriptor[*model.OOAPICollectorUpdateRequest]{
+			Body: rawBody,
+		},
+		Response: &httpapi.JSONResponseDescriptor[model.OOAPICollectorUpdateResponse]{},
+		Timeout:  0,
+		URLPath:  fmt.Sprintf("/report/%s", reportID),
+		URLQuery: nil,
+	}
+}

internal/cmd/dismantle/main.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/bytecounter"
+	"github.com/ooni/probe-cli/v3/internal/cmd/dismantle/backendclient"
+	"github.com/ooni/probe-cli/v3/internal/cmd/dismantle/sessionhttpclient"
+	"github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte"
+	"github.com/ooni/probe-cli/v3/internal/geolocate"
+	"github.com/ooni/probe-cli/v3/internal/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/logx"
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/platform"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/sessionresolver"
+	"github.com/ooni/probe-cli/v3/internal/tunnel"
+	"github.com/ooni/probe-cli/v3/internal/version"
+)
+
+func main() {
+	const softwareName = "dismantle"
+	const softwareVersion = "0.1.0-dev"
+	userAgent := fmt.Sprintf(
+		"%s/%s ooniprobe-engine/%s",
+		softwareName, softwareVersion,
+		version.Version,
+	)
+
+	logHandler := logx.NewHandlerWithDefaultSettings()
+	logHandler.Emoji = true
+	logger := &log.Logger{Level: log.InfoLevel, Handler: logHandler}
+	progressBar := model.NewPrinterCallbacks(logger)
+	counter := bytecounter.New()
+	home := filepath.Join(os.Getenv("HOME"), ".miniooni")
+	statedir := filepath.Join(home, "kvstore2")
+	ctx := context.Background()
+	tunnelDir := filepath.Join(home, "tunnel")
+	runtimex.Try0(os.MkdirAll(tunnelDir, 0700))
+
+	kvstore := runtimex.Try1(kvstore.NewFS(statedir))
+
+	tunnelConfig := &tunnel.Config{
+		Name:      "tor",
+		TunnelDir: tunnelDir,
+		Logger:    logger,
+	}
+	tunnel, _ := runtimex.Try2(tunnel.Start(ctx, tunnelConfig))
+	defer tunnel.Stop()
+	proxyURL := tunnel.SOCKS5ProxyURL()
+
+	sessionResolver := &sessionresolver.Resolver{
+		ByteCounter: counter,
+		KVStore:     kvstore,
+		Logger:      logger,
+		ProxyURL:    proxyURL,
+	}
+	defer sessionResolver.CloseIdleConnections()
+
+	geolocateConfig := &geolocate.Config{
+		Resolver:  sessionResolver,
+		Logger:    logger,
+		UserAgent: model.HTTPHeaderUserAgent,
+	}
+	geolocateTask := geolocate.NewTask(*geolocateConfig) // XXX
+	location := runtimex.Try1(geolocateTask.Run(ctx))
+	logger.Infof("%+v", location)
+
+	sessionHTTPClientConfig := &sessionhttpclient.Config{
+		ByteCounter: counter,
+		Logger:      logger,
+		Resolver:    sessionResolver,
+		ProxyURL:    proxyURL,
+	}
+	sessionHTTPClient := sessionhttpclient.New(sessionHTTPClientConfig)
+	defer sessionHTTPClient.CloseIdleConnections()
+
+	backendClientConfig := &backendclient.Config{
+		KVStore:    kvstore,
+		HTTPClient: sessionHTTPClient,
+		Logger:     logger,
+		UserAgent:  userAgent,
+		BaseURL:    nil,
+		ProxyURL:   proxyURL,
+	}
+	backendClient := backendclient.New(backendClientConfig)
+
+	checkInConfig := &model.OOAPICheckInConfig{
+		Charging:        false,
+		OnWiFi:          false,
+		Platform:        platform.Name(),
+		ProbeASN:        location.ASNString(),
+		ProbeCC:         location.CountryCode,
+		RunType:         "manual",
+		SoftwareName:    softwareName,
+		SoftwareVersion: softwareName,
+		WebConnectivity: model.OOAPICheckInConfigWebConnectivity{
+			CategoryCodes: []string{},
+		},
+	}
+	checkInResult := runtimex.Try1(backendClient.CheckIn(ctx, checkInConfig))
+	logger.Infof("%+v", checkInResult)
+
+	runtimex.Assert(checkInResult.Tests.WebConnectivity != nil, "no web connectivity info")
+	reportID := checkInResult.Tests.WebConnectivity.ReportID
+
+	experimentSession := &experimentSession{
+		httpClient:  sessionHTTPClient,
+		location:    location,
+		logger:      logger,
+		testHelpers: checkInResult.Conf.TestHelpers,
+		userAgent:   userAgent,
+	}
+
+	testStartTime := time.Now()
+	for _, input := range checkInResult.Tests.WebConnectivity.URLs {
+		cfg := &webconnectivitylte.Config{}
+		runner := webconnectivitylte.NewExperimentMeasurer(cfg)
+		measurement := backendclient.NewMeasurement(
+			location, runner.ExperimentName(), runner.ExperimentVersion(),
+			testStartTime, reportID, softwareName, softwareVersion, input.URL,
+		)
+		args := &model.ExperimentArgs{
+			Callbacks:   progressBar,
+			Measurement: measurement,
+			Session:     experimentSession,
+		}
+		if err := runner.Run(ctx, args); err != nil {
+			logger.Warnf("runner.Run failed: %s", err.Error())
+		}
+		if err := backendClient.Submit(ctx, measurement); err != nil {
+			logger.Warnf("backendClient.Submit failed: %s", err.Error())
+		}
+		log.Infof("measurement URL: %s", makeExplorerURL(reportID, input.URL))
+	}
+}
+
+func makeExplorerURL(reportID, input string) string {
+	query := url.Values{}
+	query.Add("input", input)
+	explorerURL := &url.URL{
+		Scheme:      "https",
+		Host:        "explorer.ooni.org",
+		Path:        fmt.Sprintf("/measurement/%s", reportID),
+		RawQuery:    query.Encode(),
+		Fragment:    "",
+		RawFragment: "",
+	}
+	return explorerURL.String()
+}

internal/cmd/dismantle/session.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"context"
+
+	"github.com/ooni/probe-cli/v3/internal/geolocate"
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+type experimentSession struct {
+	httpClient  model.HTTPClient
+	location    *geolocate.Results
+	logger      model.Logger
+	testHelpers map[string][]model.OOAPIService
+	userAgent   string
+}
+
+var _ model.ExperimentSession = &experimentSession{}
+
+// DefaultHTTPClient implements model.ExperimentSession
+func (es *experimentSession) DefaultHTTPClient() model.HTTPClient {
+	return es.httpClient
+}
+
+// FetchPsiphonConfig implements model.ExperimentSession
+func (es *experimentSession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) {
+	// FIXME: we need to call the backend API for this I think?
+	panic("unimplemented")
+}
+
+// FetchTorTargets implements model.ExperimentSession
+func (es *experimentSession) FetchTorTargets(ctx context.Context, cc string) (map[string]model.OOAPITorTarget, error) {
+	// FIXME: we need to call the backend API for this I think?
+	panic("unimplemented")
+}
+
+// GetTestHelpersByName implements model.ExperimentSession
+func (es *experimentSession) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) {
+	value, found := es.testHelpers[name]
+	return value, found
+}
+
+// Logger implements model.ExperimentSession
+func (es *experimentSession) Logger() model.Logger {
+	return es.logger
+}
+
+// ProbeCC implements model.ExperimentSession
+func (es *experimentSession) ProbeCC() string {
+	return es.location.CountryCode
+}
+
+// ResolverIP implements model.ExperimentSession
+func (es *experimentSession) ResolverIP() string {
+	return es.location.ResolverIP
+}
+
+// TempDir implements model.ExperimentSession
+func (es *experimentSession) TempDir() string {
+	panic("unimplemented") // FIXME
+}
+
+// TorArgs implements model.ExperimentSession
+func (es *experimentSession) TorArgs() []string {
+	panic("unimplemented") // FIXME
+}
+
+// TorBinary implements model.ExperimentSession
+func (es *experimentSession) TorBinary() string {
+	panic("unimplemented") // FIXME
+}
+
+// TunnelDir implements model.ExperimentSession
+func (es *experimentSession) TunnelDir() string {
+	panic("unimplemented") // FIXME
+}
+
+// UserAgent implements model.ExperimentSession
+func (es *experimentSession) UserAgent() string {
+	return es.userAgent
+}