NS API with directory v2 (#5068)

* Use unstable explorer client

* Clean up stale testruns & logging
- log gw identity key
- better agent testrun logging
- log responses
- change response code for agents

* Better logging on agent

* Testrun stores gw identity key instead of gw pk

* Agent 0.1.3

* Agent 0.1.4

* Sqlx offline query data + clippy

* Compatible with directory v2

* Point to internal deps + rebase + v0.1.5

* self described field not null

* Fix build.rs typo

common/models/src/ns_api.rs

@@ -0,0 +1,7 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct TestrunAssignment {
+    pub testrun_id: i64,
+    pub gateway_identity_key: String,
+}

nym-node-status-agent/Cargo.toml

@@ -0,0 +1,27 @@
+# Copyright 2024 - Nym Technologies SA <[email protected]>
+# SPDX-License-Identifier: Apache-2.0
+
+
+[package]
+name = "nym-node-status-agent"
+version = "0.1.4"
+authors.workspace = true
+repository.workspace = true
+homepage.workspace = true
+documentation.workspace = true
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+readme.workspace = true
+
+[dependencies]
+anyhow = { workspace = true}
+clap = { workspace = true, features = ["derive", "env"] }
+nym-bin-common = { path = "../common/bin-common", features = ["models"]}
+nym-common-models = { path = "../common/models" }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process"] }
+tokio-util = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
+reqwest = { workspace = true, features = ["json"] }
+serde_json = { workspace = true }

nym-node-status-agent/run.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -eu
+
+environment="qa"
+
+source ../envs/${environment}.env
+
+export RUST_LOG="debug"
+
+crate_root=$(dirname $(realpath "$0"))
+gateway_probe_src=$(dirname $(dirname "$crate_root"))/nym-vpn-client/nym-vpn-core
+echo "gateway_probe_src=$gateway_probe_src"
+echo "crate_root=$crate_root"
+
+export NODE_STATUS_AGENT_PROBE_PATH="$crate_root/nym-gateway-probe"
+
+# build & copy over GW probe
+function copy_gw_probe() {
+    pushd $gateway_probe_src
+    git switch main
+    git pull
+    cargo build --release --package nym-gateway-probe
+    cp target/release/nym-gateway-probe "$crate_root"
+    $crate_root/nym-gateway-probe --version
+    popd
+}
+
+function build_agent() {
+    cargo build --package nym-node-status-agent --release
+}
+
+function swarm() {
+    local workers=$1
+    echo "Running $workers in parallel"
+
+    build_agent
+
+    for ((i = 1; i <= $workers; i++)); do
+        ../target/release/nym-node-status-agent run-probe &
+    done
+
+    wait
+
+    echo "All agents completed"
+}
+
+export NODE_STATUS_AGENT_SERVER_ADDRESS="http://127.0.0.1"
+export NODE_STATUS_AGENT_SERVER_PORT="8000"
+
+copy_gw_probe
+
+swarm 8
+
+# cargo run -- run-probe

nym-node-status-agent/src/cli.rs

@@ -0,0 +1,105 @@
+use clap::{Parser, Subcommand};
+use nym_bin_common::bin_info;
+use nym_common_models::ns_api::TestrunAssignment;
+use std::sync::OnceLock;
+use tracing::instrument;
+
+use crate::probe::GwProbe;
+
+// Helper for passing LONG_VERSION to clap
+fn pretty_build_info_static() -> &'static str {
+    static PRETTY_BUILD_INFORMATION: OnceLock<String> = OnceLock::new();
+    PRETTY_BUILD_INFORMATION.get_or_init(|| bin_info!().pretty_print())
+}
+
+#[derive(Parser, Debug)]
+#[clap(author = "Nymtech", version, long_version = pretty_build_info_static(), about)]
+pub(crate) struct Args {
+    #[command(subcommand)]
+    pub(crate) command: Command,
+    #[arg(short, long, env = "NODE_STATUS_AGENT_SERVER_ADDRESS")]
+    pub(crate) server_address: String,
+
+    #[arg(short = 'p', long, env = "NODE_STATUS_AGENT_SERVER_PORT")]
+    pub(crate) server_port: u16,
+    // TODO dz accept keypair for identification / auth
+}
+
+#[derive(Subcommand, Debug)]
+pub(crate) enum Command {
+    RunProbe {
+        /// path of binary to run
+        #[arg(long, env = "NODE_STATUS_AGENT_PROBE_PATH")]
+        probe_path: String,
+    },
+}
+
+impl Args {
+    pub(crate) async fn execute(&self) -> anyhow::Result<()> {
+        match &self.command {
+            Command::RunProbe { probe_path } => self.run_probe(probe_path).await?,
+        }
+
+        Ok(())
+    }
+
+    async fn run_probe(&self, probe_path: &str) -> anyhow::Result<()> {
+        let server_address = format!("{}:{}", &self.server_address, self.server_port);
+
+        let probe = GwProbe::new(probe_path.to_string());
+
+        let version = probe.version().await;
+        tracing::info!("Probe version:\n{}", version);
+
+        let testrun = request_testrun(&server_address).await?;
+
+        let log = probe.run_and_get_log(&Some(testrun.gateway_identity_key));
+
+        submit_results(&server_address, testrun.testrun_id, log).await?;
+
+        Ok(())
+    }
+}
+
+const URL_BASE: &str = "internal/testruns";
+
+#[instrument(level = "debug", skip_all)]
+async fn request_testrun(server_addr: &str) -> anyhow::Result<TestrunAssignment> {
+    let target_url = format!("{}/{}", server_addr, URL_BASE);
+    let client = reqwest::Client::new();
+    let res = client
+        .get(target_url)
+        .send()
+        .await
+        .and_then(|response| response.error_for_status())?;
+    res.json()
+        .await
+        .map(|testrun| {
+            tracing::info!("Received testrun assignment: {:?}", testrun);
+            testrun
+        })
+        .map_err(|err| {
+            tracing::error!("err");
+            err.into()
+        })
+}
+
+#[instrument(level = "debug", skip(probe_outcome))]
+async fn submit_results(
+    server_addr: &str,
+    testrun_id: i64,
+    probe_outcome: String,
+) -> anyhow::Result<()> {
+    let target_url = format!("{}/{}/{}", server_addr, URL_BASE, testrun_id);
+    let client = reqwest::Client::new();
+
+    let res = client
+        .post(target_url)
+        .body(probe_outcome)
+        .send()
+        .await
+        .and_then(|response| response.error_for_status())?;
+
+    tracing::debug!("Submitted results: {})", res.status());
+    Ok(())
+}

nym-node-status-agent/src/probe.rs

@@ -0,0 +1,60 @@
+use tracing::error;
+
+pub(crate) struct GwProbe {
+    path: String,
+}
+
+impl GwProbe {
+    pub(crate) fn new(probe_path: String) -> Self {
+        Self { path: probe_path }
+    }
+
+    pub(crate) async fn version(&self) -> String {
+        let mut command = tokio::process::Command::new(&self.path);
+        command.stdout(std::process::Stdio::piped());
+        command.arg("--version");
+
+        match command.spawn() {
+            Ok(child) => {
+                if let Ok(output) = child.wait_with_output().await {
+                    return String::from_utf8(output.stdout)
+                        .unwrap_or("Unable to get log from test run".to_string());
+                }
+                "Unable to get probe version".to_string()
+            }
+            Err(e) => {
+                error!("Failed to get probe version: {}", e);
+                "Failed to get probe version".to_string()
+            }
+        }
+    }
+
+    pub(crate) fn run_and_get_log(&self, gateway_key: &Option<String>) -> String {
+        let mut command = std::process::Command::new(&self.path);
+        command.stdout(std::process::Stdio::piped());
+
+        if let Some(gateway_id) = gateway_key {
+            command.arg("--gateway").arg(gateway_id);
+        }
+
+        match command.spawn() {
+            Ok(child) => {
+                if let Ok(output) = child.wait_with_output() {
+                    if !output.status.success() {
+                        let out = String::from_utf8_lossy(&output.stdout);
+                        let err = String::from_utf8_lossy(&output.stderr);
+                        tracing::error!("Probe exited with {:?}:\n{}\n{}", output.status, out, err);
+                    }
+
+                    return String::from_utf8(output.stdout)
+                        .unwrap_or("Unable to get log from test run".to_string());
+                }
+                "Unable to get log from test run".to_string()
+            }
+            Err(e) => {
+                error!("Failed to spawn test: {}", e);
+                "Failed to spawn test run task".to_string()
+            }
+        }
+    }
+}