From f263a27825c2aa11673c6821760712221a957826 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sun, 3 Nov 2024 09:49:14 +1300 Subject: [PATCH 01/54] chore(build): fix autopublish of Debian packages --- .github/workflows/release.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 84c174d..cd5b4ce 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,6 @@ on: env: CARGO_TERM_COLOR: always - WORKFLOW_TEST: ${{ github.event_name == 'workflow_dispatch' }} permissions: contents: write @@ -58,7 +57,7 @@ jobs: leading-dir: true tar: unix zip: windows - dry_run: ${{ env.WORKFLOW_TEST }} + dry_run: ${{ github.event_name != 'release' }} - name: Make deb package if: ${{ matrix.build_deb }} run: scripts/make-debian-package --release ${{ matrix.target }} @@ -74,7 +73,7 @@ jobs: name: qcp-deb-${{ matrix.target }} path: target/**/debian/qcp*.deb - name: Publish deb package to release - if: ${{ ! env.WORKFLOW_TEST }} + if: ${{ github.event_name == 'release' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: gh release upload ${{ github.ref }} ${{ env.BUILT_DEB_FILE }} From 98eb49cfa455ab795696c63e8b8001a3aa82f64f Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sun, 10 Nov 2024 11:44:08 +1300 Subject: [PATCH 02/54] chore(skip,build): update changelog config --- release-plz.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/release-plz.toml b/release-plz.toml index 9e0eb3f..c1cd03b 100644 --- a/release-plz.toml +++ b/release-plz.toml @@ -50,7 +50,8 @@ body = """ """ commit_parsers = [ - { message = "^feat", group = "⛰️ Features" }, + { body = ".*security", group = "πŸ›‘οΈ Security" }, + { message = "^feat", group = "⛰️ Features" }, { message = "^fix", group = "πŸ› Bug Fixes" }, { message = "^doc", group = "πŸ“š Documentation" }, { message = "^perf", group = "⚑ Performance" }, @@ -63,11 +64,10 @@ commit_parsers = [ { message = "^chore\\(deps.*\\)", skip = true }, { message = "^chore\\(pr\\)", skip = true }, { message = "^chore\\(pull\\)", skip = true }, - { message = "^chore\\(npm\\).*yarn\\.lock", skip = true }, - { message = "^build", group = "πŸ—οΈ Build & CI" }, - { message = "^chore|^ci|^misc|^tidyup", group = "βš™οΈ Miscellaneous Tasks" }, - { body = ".*security", group = "πŸ›‘οΈ Security" }, - { message = "^revert", group = "◀️ Revert" }, + { message = "^chore\\(skip", skip = true }, + { message = "^build|^ci", group = "πŸ—οΈ Build & CI" }, + { message = "^chore|^misc|^tidyup", group = "βš™οΈ Miscellaneous Tasks" }, + { message = "^revert", group = "◀️ Revert" }, ] link_parsers = [ From 828f063b534eff534c9908f5624184ad64d22ab3 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sun, 10 Nov 2024 11:44:24 +1300 Subject: [PATCH 03/54] chore(skip,ci): make better use of BUILT_DEB_FILE, set RUST_BACKTRACE at build time - always set BUILT_DEB_FILE to silence a warning - make better use of BUILT_DEB_FILE - set RUST_BACKTRACE at build time - don't upload a deb in release workflow unless the matrix is configured to build one --- .github/workflows/ci.yml | 2 ++ .github/workflows/release.yml | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2dc0e2..d07f18c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,9 @@ on: workflow_dispatch: env: + BUILT_DEB_FILE: "invalid.deb" # updated by make-debian-package script CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 jobs: # Build the app on all supported platforms, at least for the time being diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cd5b4ce..ea25dfc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,9 @@ on: workflow_dispatch: # for testing env: + BUILT_DEB_FILE: "invalid.deb" # updated by make-debian-package script CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 permissions: contents: write @@ -71,9 +73,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: qcp-deb-${{ matrix.target }} - path: target/**/debian/qcp*.deb + path: ${{ env.BUILT_DEB_FILE }} - name: Publish deb package to release - if: ${{ github.event_name == 'release' }} + if: ${{ matrix.build_deb }} && ${{ github.event_name == 'release' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: gh release upload ${{ github.ref }} ${{ env.BUILT_DEB_FILE }} From c7e9b8f591866bbc8d2b44b25ae0d0980959fadf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Nov 2024 22:21:47 +1300 Subject: [PATCH 04/54] fix(deps): bump rustls from 0.23.16 to 0.23.18 (#15) Bumps [rustls](https://github.com/rustls/rustls) from 0.23.16 to 0.23.18. - [Release notes](https://github.com/rustls/rustls/releases) - [Changelog](https://github.com/rustls/rustls/blob/main/CHANGELOG.md) - [Commits](https://github.com/rustls/rustls/compare/v/0.23.16...v/0.23.18) --- updated-dependencies: - dependency-name: rustls dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00c024a..e35733e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -866,9 +866,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.16" +version = "0.23.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" +checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" dependencies = [ "once_cell", "ring", From 38de5194b4b17c75e71c961da8742b09053ff22e Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Mon, 18 Nov 2024 21:04:16 +1300 Subject: [PATCH 05/54] feat!: configuration file system (#17) - initial file format is TOML - lots of refactoring, most notably the CLI argument structures - added CLI options `--show-config` and `--config-files` - renamed CLI options --ipv4, --ipv6 to --address-family 4|6 (short args `-4` and `-6` unchanged) --- .vscode/launch.json | 16 +- .vscode/tasks.json | 2 +- Cargo.lock | 481 +++++++++++++++++++++++++++++-- Cargo.toml | 14 +- README.md | 6 + src/cli/args.rs | 97 +++++-- src/cli/cli_main.rs | 83 ++++-- src/cli/mod.rs | 2 +- src/cli/styles.rs | 12 +- src/client/args.rs | 122 -------- src/client/control.rs | 59 ++-- src/client/job.rs | 50 ++-- src/client/main_loop.rs | 92 +++--- src/client/mod.rs | 4 +- src/client/options.rs | 93 ++++++ src/config/manager.rs | 572 +++++++++++++++++++++++++++++++++++++ src/config/mod.rs | 38 +++ src/config/structure.rs | 252 ++++++++++++++++ src/lib.rs | 12 + src/protocol/mod.rs | 4 +- src/server.rs | 48 ++-- src/transport.rs | 191 ++----------- src/util/address_family.rs | 125 ++++++++ src/util/cli.rs | 148 +++------- src/util/dns.rs | 12 +- src/util/humanu64.rs | 122 ++++++++ src/util/mod.rs | 13 +- src/util/optionalify.rs | 158 ++++++++++ src/util/port_range.rs | 134 +++++++++ src/util/stats.rs | 4 +- 30 files changed, 2357 insertions(+), 609 deletions(-) delete mode 100644 src/client/args.rs create mode 100644 src/client/options.rs create mode 100644 src/config/manager.rs create mode 100644 src/config/mod.rs create mode 100644 src/config/structure.rs create mode 100644 src/util/address_family.rs create mode 100644 src/util/humanu64.rs create mode 100644 src/util/optionalify.rs create mode 100644 src/util/port_range.rs diff --git a/.vscode/launch.json b/.vscode/launch.json index 5915432..c0172a8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,15 +7,15 @@ { "type": "lldb", "request": "launch", - "name": "Debug executable 'qcpt'", + "name": "Debug executable 'qcp'", "cargo": { "args": [ "build", - "--bin=qcpt", - "--package=qcpt" + "--bin=qcp", + "--package=qcp" ], "filter": { - "name": "qcpt", + "name": "qcp", "kind": "bin" } }, @@ -25,16 +25,16 @@ { "type": "lldb", "request": "launch", - "name": "Debug unit tests in executable 'qcpt'", + "name": "Debug unit tests in executable 'qcp'", "cargo": { "args": [ "test", "--no-run", - "--bin=qcpt", - "--package=qcpt" + "--bin=qcp", + "--package=qcp" ], "filter": { - "name": "qcpt", + "name": "qcp", "kind": "bin" } }, diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 9a6a5f7..af22b87 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -4,7 +4,7 @@ { "type": "cargo", "command": "doc", - "args": ["--no-deps"], + "args": ["--no-deps", "--locked"], "problemMatcher": [ "$rustc" ], diff --git a/Cargo.lock b/Cargo.lock index 42fee71..15202ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,6 +87,15 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "atomic" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +dependencies = [ + "bytemuck", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -120,6 +129,27 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytecount" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" + +[[package]] +name = "bytemuck" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" + [[package]] name = "byteorder" version = "1.5.0" @@ -210,10 +240,10 @@ version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -237,10 +267,29 @@ dependencies = [ "encode_unicode", "lazy_static", "libc", - "unicode-width", + "unicode-width 0.1.14", "windows-sys 0.52.0", ] +[[package]] +name = "cpufeatures" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "deranged" version = "0.3.11" @@ -250,6 +299,44 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive-deftly" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f9bc3564f74be6c35d49a7efee54380d7946ccc631323067f33fabb9246027" +dependencies = [ + "derive-deftly-macros", + "heck 0.5.0", +] + +[[package]] +name = "derive-deftly-macros" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b84d32b18d9a256d81e4fec2e4cfd0ab6dde5e5ff49be1713ae0adbd0060c2" +dependencies = [ + "heck 0.5.0", + "indexmap", + "itertools", + "proc-macro-crate", + "proc-macro2", + "quote", + "sha3", + "strum", + "syn 2.0.85", + "void", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dns-lookup" version = "2.0.4" @@ -262,6 +349,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "embedded-io" version = "0.6.1" @@ -274,6 +367,12 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.9" @@ -284,12 +383,42 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "fastrand" version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic", + "serde", + "toml", + "uncased", + "version_check", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "futures" version = "0.3.31" @@ -355,6 +484,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "gethostname" version = "0.5.0" @@ -382,6 +521,18 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "hashbrown" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -394,6 +545,15 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "human-repr" version = "1.1.0" @@ -406,6 +566,16 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "016b02deb8b0c415d8d56a6f0ab265e50c22df61194e37f9be75ed3a722de8a6" +[[package]] +name = "indexmap" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "indicatif" version = "0.17.8" @@ -417,7 +587,7 @@ dependencies = [ "number_prefix", "portable-atomic", "tokio", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -435,6 +605,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -461,6 +640,21 @@ dependencies = [ "libc", ] +[[package]] +name = "json" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" + +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -596,6 +790,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "papergrid" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b0f8def1f117e13c895f3eda65a7b5650688da29d6ad04635f61bc7b92eebd" +dependencies = [ + "bytecount", + "fnv", + "unicode-width 0.2.0", +] + [[package]] name = "pem" version = "3.0.4" @@ -639,6 +844,37 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "proc-macro2" version = "1.0.89" @@ -659,23 +895,36 @@ dependencies = [ "capnpc", "clap", "console", + "derive-deftly", "dns-lookup", + "etcetera", "fastrand", + "figment", "futures-util", "gethostname", + "heck 0.5.0", "human-repr", "humanize-rs", "indicatif", "jemallocator", + "json", "nix", "num-format", "quinn", + "rand", "rcgen", "rustls-pki-types", + "serde", + "serde_json", + "serde_test", "static_assertions", + "struct-field-names-as-array", "strum_macros", + "tabled", + "tempfile", "tokio", "tokio-util", + "toml", "tracing", "tracing-subscriber", ] @@ -853,9 +1102,9 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustix" -version = "0.38.37" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" dependencies = [ "bitflags", "errno", @@ -901,24 +1150,70 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + [[package]] name = "serde" -version = "1.0.213" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.213" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", +] + +[[package]] +name = "serde_json" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_test" +version = "1.0.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f901ee573cab6b3060453d2d5f0bae4e6d628c23c0a962ff9b5f1d7c8d4f1ed" +dependencies = [ + "serde", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", ] [[package]] @@ -988,17 +1283,46 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "struct-field-names-as-array" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ba4bae771f9cc992c4f403636c54d2ef13acde6367583e99d06bb336674dd9" +dependencies = [ + "struct-field-names-as-array-derive", +] + +[[package]] +name = "struct-field-names-as-array-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2dbf8b57f3ce20e4bb171a11822b283bdfab6c4bb0fe64fa729f045f23a0938" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + [[package]] name = "strum_macros" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.85", ] [[package]] @@ -1007,6 +1331,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.85" @@ -1018,6 +1353,42 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tabled" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6709222f3973137427ce50559cd564dc187a95b9cfe01613d2f4e93610e510a" +dependencies = [ + "papergrid", + "tabled_derive", +] + +[[package]] +name = "tabled_derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "931be476627d4c54070a1f3a9739ccbfec9b36b39815106a20cce2243bbcefe1" +dependencies = [ + "heck 0.4.1", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tempfile" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "terminal_size" version = "0.4.0" @@ -1045,7 +1416,7 @@ checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -1117,7 +1488,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -1134,6 +1505,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tracing" version = "0.1.40" @@ -1153,7 +1558,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -1195,6 +1600,21 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-ident" version = "1.0.13" @@ -1207,6 +1627,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "untrusted" version = "0.9.0" @@ -1225,6 +1651,18 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1401,6 +1839,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +dependencies = [ + "memchr", +] + [[package]] name = "yasna" version = "0.5.2" @@ -1428,7 +1875,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cec2857..a701da5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,9 +23,13 @@ capnp = "0.20.1" capnp-futures = "0.20.0" clap = { version = "4.5.19", features = ["wrap_help", "derive", "cargo", "help", "string"] } console = "0.15.8" +derive-deftly = "0.14.2" dns-lookup = "2.0.4" +etcetera = "0.8.0" +figment = { version = "0.10.19", features = ["toml"] } futures-util = { version = "0.3.31", default-features = false } gethostname = "0.5.0" +heck = "0.5.0" human-repr = "1.1.0" humanize-rs = "0.1.5" indicatif = { version = "0.17.8", features = ["tokio"] } @@ -33,8 +37,11 @@ num-format = { version = "0.4.4" } quinn = { version = "0.11.5", default-features = false, features = ["runtime-tokio", "rustls", "ring"] } rcgen = { version = "0.13.1" } rustls-pki-types = "1.9.0" +serde = { version = "1.0.215", features = ["derive"] } static_assertions = "1.1.0" +struct-field-names-as-array = "0.3.0" strum_macros = "0.26.4" +tabled = "0.17.0" tokio = { version = "1.40.0", default-features = true, features = ["fs", "io-std", "macros", "process", "rt", "time", "sync"] } tokio-util = { version = "0.7.12", features = ["compat"] } tracing = "0.1.40" @@ -51,7 +58,12 @@ capnpc = "0.20.0" [dev-dependencies] fastrand = "2.1.1" - +json = "0.12.4" +rand = "0.8.5" +serde_json = "1.0.132" +serde_test = "1.0.177" +tempfile = "3.14.0" +toml = "0.8.19" [lints.rust] dead_code = "warn" diff --git a/README.md b/README.md index a4fb3c2..83f1111 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,11 @@ qcp my-server:/tmp/testfile /tmp/ --rx 37M --tx 12M Performance tuning can be a tricky subject. See the [performance] documentation. +#### Persistent configuration + +The useful options -- those you might want to use regularly including `rx`, `tx` and `rtt` -- can be specified +in a configuration file. See [config] for details. + ## πŸ“– How qcp works The brief version: @@ -141,6 +146,7 @@ Some ideas for the future, in no particular order: [rfc9000]: https://www.rfc-editor.org/rfc/rfc9000.html [buying me a coffee]: https://buymeacoffee.com/rossyounger [ko-fi]: https://ko-fi.com/rossyounger +[config]: https://docs.rs/qcp/latest/qcp/doc/config/index.html [protocol]: https://docs.rs/qcp/latest/qcp/protocol/index.html [performance]: https://docs.rs/qcp/latest/qcp/doc/performance/index.html [Github sponsorship]: https://github.com/sponsors/crazyscot?frequency=recurring&sponsor=crazyscot diff --git a/src/cli/args.rs b/src/cli/args.rs index 8066d83..2fbbb33 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -1,10 +1,12 @@ // QCP top-level command-line arguments // (c) 2024 Ross Younger -use clap::{Args as _, FromArgMatches as _, Parser}; +use clap::{ArgAction::SetTrue, Args as _, FromArgMatches as _, Parser}; + +use crate::{client::CopyJobSpec, config::Manager, util::AddressFamily}; /// Options that switch us into another mode i.e. which don't require source/destination arguments -pub(crate) const MODE_OPTIONS: &[&str] = &["server", "help_buffers"]; +pub(crate) const MODE_OPTIONS: &[&str] = &["server", "help_buffers", "show_config", "config_files"]; #[derive(Debug, Parser, Clone)] #[command( @@ -24,6 +26,7 @@ pub(crate) const MODE_OPTIONS: &[&str] = &["server", "help_buffers"]; " ))] #[command(styles=super::styles::get())] +#[allow(clippy::struct_excessive_bools)] pub(crate) struct CliArgs { // MODE SELECTION ====================================================================== /// Operates in server mode. @@ -32,48 +35,86 @@ pub(crate) struct CliArgs { /// intended for interactive use. #[arg( long, help_heading("Modes"), hide = true, - conflicts_with_all(["help_buffers", "quiet", "statistics", "ipv4", "ipv6", "remote_debug", "profile", "source", "destination", "ssh", "ssh_opt", "remote_port"]) + conflicts_with_all([ + "help_buffers", "show_config", "config_files", + "quiet", "statistics", "remote_debug", "profile", + "ssh", "ssh_opt", "remote_port", + "source", "destination", + ]) )] pub server: bool, + /// Outputs the configuration, then exits + #[arg(long, help_heading("Configuration"))] + pub show_config: bool, + /// Outputs the paths to configuration file(s), then exits + #[arg(long, help_heading("Configuration"))] + pub config_files: bool, + /// Outputs additional information about kernel UDP buffer sizes and platform-specific tips #[arg(long, action, help_heading("Network tuning"), display_order(50))] pub help_buffers: bool, - // CLIENT-ONLY OPTIONS ================================================================= + // CONFIGURABLE OPTIONS ================================================================ #[command(flatten)] - pub client: crate::client::Options, + pub config: crate::config::Configuration_Optional, - // NETWORK OPTIONS ===================================================================== + // CLIENT-SIDE NON-CONFIGURABLE OPTIONS ================================================ + // (including positional arguments!) #[command(flatten)] - pub bandwidth: crate::transport::BandwidthParams, + pub client_params: crate::client::Parameters, - #[command(flatten)] - pub quic: crate::transport::QuicParams, - // DEBUG OPTIONS ======================================================================= - /// Enable detailed debug output - /// - /// This has the same effect as setting `RUST_LOG=qcp=debug` in the environment. - /// If present, `RUST_LOG` overrides this option. - #[arg(short, long, action, help_heading("Debug"))] - pub debug: bool, - /// Log to a file - /// - /// By default the log receives everything printed to stderr. - /// To override this behaviour, set the environment variable `RUST_LOG_FILE_DETAIL` (same semantics as `RUST_LOG`). - #[arg(short('l'), long, action, help_heading("Debug"), value_name("FILE"))] - pub log_file: Option, - // - // ====================================================================================== - // - // N.B. ClientOptions has positional arguments! + /// Convenience alias for `--address-family 4` + // this is actioned by our custom parser + #[arg( + short = '4', + help_heading("Connection"), + group("ip address"), + action(SetTrue) + )] + pub ipv4_alias__: bool, + /// Convenience alias for `--address-family 6` + // this is actioned by our custom parser + #[arg( + short = '6', + help_heading("Connection"), + group("ip address"), + action(SetTrue) + )] + pub ipv6_alias__: bool, } impl CliArgs { - /// Sets up and executes ou + /// Sets up and executes our parser pub(crate) fn custom_parse() -> Self { let cli = clap::Command::new(clap::crate_name!()); let cli = CliArgs::augment_args(cli).version(crate::version::short()); - CliArgs::from_arg_matches(&cli.get_matches_from(std::env::args_os())).unwrap() + let mut args = + CliArgs::from_arg_matches(&cli.get_matches_from(std::env::args_os())).unwrap(); + // Custom logic: '-4' and '-6' convenience aliases + if args.ipv4_alias__ { + args.config.address_family = Some(Some(AddressFamily::V4)); + } else if args.ipv6_alias__ { + args.config.address_family = Some(Some(AddressFamily::V6)); + } + args + } +} + +impl From<&CliArgs> for Manager { + /// Merge options from the CLI into the structure. + /// Any new option packs (_Optional structs) need to be added here. + fn from(value: &CliArgs) -> Self { + let mut mgr = Manager::new(); + mgr.merge_provider(&value.config); + mgr + } +} + +impl TryFrom<&CliArgs> for CopyJobSpec { + type Error = anyhow::Error; + + fn try_from(args: &CliArgs) -> Result { + CopyJobSpec::try_from(&args.client_params) } } diff --git a/src/cli/cli_main.rs b/src/cli/cli_main.rs index eac3b90..dfb6ee2 100644 --- a/src/cli/cli_main.rs +++ b/src/cli/cli_main.rs @@ -3,10 +3,11 @@ use std::process::ExitCode; -use super::args::CliArgs; +use super::{args::CliArgs, styles}; use crate::{ - client::{client_main, MAX_UPDATE_FPS}, + client::{client_main, Parameters as ClientParameters, MAX_UPDATE_FPS}, + config::{Configuration, Manager}, os, server::server_main, util::setup_tracing, @@ -14,6 +15,16 @@ use crate::{ use indicatif::{MultiProgress, ProgressDrawTarget}; use tracing::error_span; +fn trace_level(args: &ClientParameters) -> &str { + if args.debug { + "debug" + } else if args.quiet { + "error" + } else { + "info" + } +} + /// Main CLI entrypoint /// /// Call this from `main`. It reads argv. @@ -25,36 +36,64 @@ pub async fn cli() -> anyhow::Result { let args = CliArgs::custom_parse(); if args.help_buffers { os::print_udp_buffer_size_help_message( - args.bandwidth.recv_buffer(), - args.bandwidth.send_buffer(), + Configuration::recv_buffer(), + Configuration::send_buffer(), ); return Ok(ExitCode::SUCCESS); } - let trace_level = if args.debug { - "debug" - } else if args.client.quiet { - "error" - } else { - "info" - }; - let progress = if args.server { - None - } else { - Some(MultiProgress::with_draw_target( - ProgressDrawTarget::stderr_with_hz(MAX_UPDATE_FPS), - )) + + let progress = (!args.server).then(|| { + MultiProgress::with_draw_target(ProgressDrawTarget::stderr_with_hz(MAX_UPDATE_FPS)) + }); + setup_tracing( + trace_level(&args.client_params), + progress.as_ref(), + &args.client_params.log_file, + ) + .inspect_err(|e| eprintln!("{e:?}"))?; + + if args.config_files { + // do this before attempting to read config, in case it fails + println!("{:?}", Manager::config_files()); + return Ok(ExitCode::SUCCESS); + } + + // Now fold the arguments in with the CLI config (which may fail) + let config_manager = Manager::from(&args); + + let config = match config_manager.get::() { + Ok(c) => c, + Err(err) => { + println!( + "{}: Failed to parse configuration", + styles::error().apply_to("ERROR") + ); + if err.count() == 1 { + println!("{err}"); + } else { + let inf = styles::info(); + for (i, e) in err.into_iter().enumerate() { + println!("{}: {e}", inf.apply_to(i + 1)); + } + } + return Ok(ExitCode::FAILURE); + } }; - setup_tracing(trace_level, progress.as_ref(), &args.log_file) - .inspect_err(|e| eprintln!("{e:?}"))?; - if args.server { + if args.show_config { + println!( + "{}", + config_manager.to_display_adapter::(true) + ); + Ok(ExitCode::SUCCESS) + } else if args.server { let _span = error_span!("REMOTE").entered(); - server_main(args.bandwidth, args.quic) + server_main(&config) .await .map(|()| ExitCode::SUCCESS) .inspect_err(|e| tracing::error!("{e}")) } else { - client_main(args.client, args.bandwidth, args.quic, progress.unwrap()) + client_main(&config, progress.unwrap(), args.client_params) .await .inspect_err(|e| tracing::error!("{e}")) .or_else(|_| Ok(false)) diff --git a/src/cli/mod.rs b/src/cli/mod.rs index fe5d677..447d74e 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -2,6 +2,6 @@ /// (c) 2024 Ross Younger mod args; mod cli_main; -mod styles; +pub(crate) mod styles; pub(crate) use args::MODE_OPTIONS; pub use cli_main::cli; diff --git a/src/cli/styles.rs b/src/cli/styles.rs index dd8c5d1..724a100 100644 --- a/src/cli/styles.rs +++ b/src/cli/styles.rs @@ -1,4 +1,4 @@ -// Default styling for qcp's help output +// Default styling for CLI output #[must_use] /// Styling factory @@ -37,3 +37,13 @@ pub(crate) fn get() -> clap::builder::Styles { anstyle::Style::new().fg_color(Some(anstyle::Color::Ansi(anstyle::AnsiColor::Cyan))), ) } + +pub(crate) fn error() -> console::Style { + console::Style::new().red() +} +pub(crate) fn warning() -> console::Style { + console::Style::new().yellow() +} +pub(crate) fn info() -> console::Style { + console::Style::new().cyan() +} diff --git a/src/client/args.rs b/src/client/args.rs deleted file mode 100644 index 6b14ccd..0000000 --- a/src/client/args.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! Options specific to qcp client-mode -// (c) 2024 Ross Younger - -use clap::Parser; - -use crate::{protocol::control::ConnectionType, util::PortRange}; - -use super::job::FileSpec; - -/// Options specific to qcp client mode -#[derive(Debug, Parser, Clone)] -#[allow(clippy::struct_excessive_bools)] -pub struct Options { - /// Quiet mode - /// - /// Switches off progress display and statistics; reports only errors - #[arg(short, long, action, conflicts_with("debug"))] - pub quiet: bool, - - /// Outputs additional transfer statistics - #[arg(short = 's', long, alias("stats"), action, conflicts_with("quiet"))] - pub statistics: bool, - - /// Forces IPv4 connection [default: autodetect] - #[arg(short = '4', long, action, help_heading("Connection"))] - pub ipv4: bool, - /// Forces IPv6 connection [default: autodetect] - #[arg( - short = '6', - long, - action, - conflicts_with("ipv4"), - help_heading("Connection") - )] - pub ipv6: bool, - - /// Specifies the ssh client program to use - #[arg(long, default_value("ssh"), help_heading("Connection"))] - pub ssh: String, - - /// Provides an additional option or argument to pass to the ssh client. - /// - /// Note that you must repeat `-S` for each. - /// For example, to pass `-i /dev/null` to ssh, specify: `-S -i -S /dev/null` - #[arg( - short = 'S', - action, - value_name("ssh-option"), - allow_hyphen_values(true), - help_heading("Connection") - )] - pub ssh_opt: Vec, - - /// Uses the given UDP port or range on the remote endpoint. - /// - /// This can be useful when there is a firewall between the endpoints. - #[arg(short = 'P', long, value_name("M-N"), help_heading("Connection"))] - pub remote_port: Option, - - // CLIENT DEBUG ---------------------------- - /// Enables detailed debug output from the remote endpoint - #[arg(long, action, help_heading("Debug"))] - pub remote_debug: bool, - /// Prints timing profile data after completion - #[arg(long, action, help_heading("Debug"))] - pub profile: bool, - - // POSITIONAL ARGUMENTS ================================================================ - /// The source file. This may be a local filename, or remote specified as HOST:FILE or USER@HOST:FILE. - /// - /// Exactly one of source and destination must be remote. - #[arg( - conflicts_with_all(crate::cli::MODE_OPTIONS), - required = true, - value_name = "SOURCE" - )] - pub source: Option, - - /// Destination. This may be a file or directory. It may be local or remote. - /// - /// If remote, specify as HOST:DESTINATION or USER@HOST:DESTINATION; or simply HOST: or USER@HOST: to copy to your home directory there. - /// - /// Exactly one of source and destination must be remote. - #[arg( - conflicts_with_all(crate::cli::MODE_OPTIONS), - required = true, - value_name = "DESTINATION" - )] - pub destination: Option, -} - -impl Options { - pub(crate) fn address_family(&self) -> Option { - if self.ipv4 { - Some(ConnectionType::Ipv4) - } else if self.ipv6 { - Some(ConnectionType::Ipv6) - } else { - None - } - } - - pub(crate) fn remote_user_host(&self) -> anyhow::Result<&str> { - let src = self.source.as_ref().ok_or(anyhow::anyhow!( - "both source and destination must be specified" - ))?; - let dest = self.destination.as_ref().ok_or(anyhow::anyhow!( - "both source and destination must be specified" - ))?; - Ok(src - .host - .as_ref() - .unwrap_or_else(|| dest.host.as_ref().unwrap())) - } - - pub(crate) fn remote_host(&self) -> anyhow::Result<&str> { - let user_host = self.remote_user_host()?; - // It might be user@host, or it might be just the hostname or IP. - let (_, host) = user_host.split_once('@').unwrap_or(("", user_host)); - Ok(host) - } -} diff --git a/src/client/control.rs b/src/client/control.rs index 4081051..5df7f1d 100644 --- a/src/client/control.rs +++ b/src/client/control.rs @@ -1,7 +1,7 @@ //! Control channel management for the qcp client // (c) 2024 Ross Younger -use std::{net::IpAddr, process::Stdio, time::Duration}; +use std::{process::Stdio, time::Duration}; use anyhow::{anyhow, Context as _, Result}; use indicatif::MultiProgress; @@ -12,12 +12,12 @@ use tokio::{ use tracing::{debug, trace, warn}; use crate::{ + config::Configuration, protocol::control::{ClientMessage, ClosedownReport, ConnectionType, ServerMessage, BANNER}, - transport::{BandwidthParams, QuicParams}, - util::Credentials, + util::{AddressFamily, Credentials}, }; -use super::args::Options; +use super::Parameters; /// Control channel abstraction #[derive(Debug)] @@ -35,16 +35,17 @@ impl Channel { } /// Opens the control channel, checks the banner, sends the Client Message, reads the Server Message. + #[allow(clippy::too_many_arguments)] pub async fn transact( credentials: &Credentials, - server_address: IpAddr, + remote_host: &str, + connection_type: ConnectionType, display: &MultiProgress, - client: &Options, - bandwidth: BandwidthParams, - quic: QuicParams, + config: &Configuration, + parameters: &Parameters, ) -> Result<(Channel, ServerMessage)> { trace!("opening control channel"); - let mut new1 = Self::launch(display, client, bandwidth, quic)?; + let mut new1 = Self::launch(display, config, remote_host, parameters)?; new1.wait_for_banner().await?; let mut pipe = new1 @@ -52,7 +53,7 @@ impl Channel { .stdin .as_mut() .ok_or(anyhow!("could not access process stdin (can't happen?)"))?; - ClientMessage::write(&mut pipe, &credentials.certificate, server_address.into()) + ClientMessage::write(&mut pipe, &credentials.certificate, connection_type) .await .with_context(|| "writing client message")?; @@ -78,49 +79,49 @@ impl Channel { /// This is effectively a constructor. At present, it launches a subprocess. fn launch( display: &MultiProgress, - client: &Options, - bandwidth: BandwidthParams, - quic: QuicParams, + config: &Configuration, + remote_host: &str, + parameters: &Parameters, ) -> Result { - let mut server = tokio::process::Command::new(&client.ssh); + let mut server = tokio::process::Command::new(&config.ssh); let _ = server.kill_on_drop(true); - let _ = match client.address_family() { + let _ = match config.address_family { None => &mut server, - Some(ConnectionType::Ipv4) => server.arg("-4"), - Some(ConnectionType::Ipv6) => server.arg("-6"), + Some(AddressFamily::V4) => server.arg("-4"), + Some(AddressFamily::V6) => server.arg("-6"), }; - let _ = server.args(&client.ssh_opt); + let _ = server.args(&config.ssh_opt); let _ = server.args([ - client.remote_user_host()?, + remote_host, "qcp", "--server", // Remote receive bandwidth = our transmit bandwidth "-b", - &bandwidth.tx().to_string(), + &config.tx().to_string(), // Remote transmit bandwidth = our receive bandwidth "-B", - &bandwidth.rx().to_string(), + &config.rx().to_string(), "--rtt", - &bandwidth.rtt.to_string(), + &config.rtt.to_string(), "--congestion", - &bandwidth.congestion.to_string(), + &config.congestion.to_string(), "--timeout", - &quic.timeout.as_secs().to_string(), + &config.timeout.to_string(), ]); - if client.remote_debug { + if parameters.remote_debug { let _ = server.arg("--debug"); } - if let Some(w) = bandwidth.initial_congestion_window { + if let Some(w) = config.initial_congestion_window { let _ = server.args(["--initial-congestion-window", &w.to_string()]); } - if let Some(pr) = client.remote_port { + if let Some(pr) = config.remote_port { let _ = server.args(["--port", &pr.to_string()]); } let _ = server .stdin(Stdio::piped()) .stdout(Stdio::piped()) .kill_on_drop(true); - if !client.quiet { + if !parameters.quiet { let _ = server.stderr(Stdio::piped()); } // else inherit debug!("spawning command: {:?}", server); @@ -129,7 +130,7 @@ impl Channel { .context("Could not launch control connection to remote server")?; // Whatever the remote outputs, send it to our output in a way that doesn't mess things up. - if !client.quiet { + if !parameters.quiet { let stderr = process.stderr.take(); let Some(stderr) = stderr else { anyhow::bail!("could not get stderr of remote process"); diff --git a/src/client/job.rs b/src/client/job.rs index 089be1b..03e4f63 100644 --- a/src/client/job.rs +++ b/src/client/job.rs @@ -5,10 +5,8 @@ use std::str::FromStr; use crate::transport::ThroughputMode; -use super::args::Options; - /// A file source or destination specified by the user -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct FileSpec { /// The remote host for the file. /// @@ -54,7 +52,6 @@ impl FromStr for FileSpec { } /// Details of a file copy job. -/// (This is a helper struct for the contents of `CliArgs` .) #[derive(Debug, Clone)] pub struct CopyJobSpec { pub(crate) source: FileSpec, @@ -70,31 +67,34 @@ impl CopyJobSpec { ThroughputMode::Tx } } -} -impl TryFrom<&Options> for CopyJobSpec { - type Error = anyhow::Error; - - fn try_from(args: &Options) -> Result { - let source = args - .source - .as_ref() - .ok_or_else(|| anyhow::anyhow!("source and destination are required"))? - .clone(); - let destination = args - .destination + /* + pub(crate) fn remote_user_host(&self) -> anyhow::Result<&str> { + let src = self.source.as_ref().ok_or(anyhow::anyhow!( + "both source and destination must be specified" + ))?; + let dest = self.destination.as_ref().ok_or(anyhow::anyhow!( + "both source and destination must be specified" + ))?; + Ok(src + .host .as_ref() - .ok_or_else(|| anyhow::anyhow!("source and destination are required"))? - .clone(); + .unwrap_or_else(|| dest.host.as_ref().unwrap())) + } + */ - if !(source.host.is_none() ^ destination.host.is_none()) { - anyhow::bail!("One file argument must be remote"); - } + pub(crate) fn remote_user_host(&self) -> &str { + self.source + .host + .as_ref() + .unwrap_or_else(|| self.destination.host.as_ref().unwrap()) + } - Ok(Self { - source, - destination, - }) + pub(crate) fn remote_host(&self) -> &str { + let user_host = self.remote_user_host(); + // It might be user@host, or it might be just the hostname or IP. + let (_, host) = user_host.split_once('@').unwrap_or(("", user_host)); + host } } diff --git a/src/client/main_loop.rs b/src/client/main_loop.rs index ec3538a..a9b1a01 100644 --- a/src/client/main_loop.rs +++ b/src/client/main_loop.rs @@ -1,14 +1,15 @@ // qcp client event loop // (c) 2024 Ross Younger -use crate::client::control::Channel; -use crate::client::progress::spinner_style; -use crate::protocol::session::Status; -use crate::protocol::session::{FileHeader, FileTrailer, Response}; -use crate::protocol::{RawStreamPair, StreamPair}; -use crate::transport::{BandwidthParams, QuicParams, ThroughputMode}; -use crate::util::{ - self, lookup_host_by_family, time::Stopwatch, time::StopwatchChain, Credentials, PortRange, +use crate::{ + client::{control::Channel, progress::spinner_style}, + config::Configuration, + protocol::{ + session::{FileHeader, FileTrailer, Response, Status}, + RawStreamPair, StreamPair, + }, + transport::ThroughputMode, + util::{self, lookup_host_by_family, time::Stopwatch, time::StopwatchChain, Credentials}, }; use anyhow::{Context, Result}; @@ -26,8 +27,8 @@ use tokio::time::Instant; use tokio::{self, io::AsyncReadExt, time::timeout, time::Duration}; use tracing::{debug, error, info, span, trace, trace_span, warn, Instrument as _, Level}; -use super::args::Options; use super::job::CopyJobSpec; +use super::Parameters as ClientParameters; const SHOW_TIME: &str = "file transfer"; @@ -35,40 +36,40 @@ const SHOW_TIME: &str = "file transfer"; // Caution: As we are using ProgressBar, anything to be printed to console should use progress.println() ! #[allow(clippy::module_name_repetitions)] pub async fn client_main( - options: Options, - bandwidth: BandwidthParams, - quic: QuicParams, + config: &Configuration, display: MultiProgress, + parameters: ClientParameters, ) -> anyhow::Result { // N.B. While we have a MultiProgress we do not set up any `ProgressBar` within it yet... // not until the control channel is in place, in case ssh wants to ask for a password or passphrase. let _guard = trace_span!("CLIENT").entered(); - let job_spec = CopyJobSpec::try_from(&options)?; let mut timers = StopwatchChain::new_running("setup"); // Prep -------------------------- + let job_spec = crate::client::CopyJobSpec::try_from(¶meters)?; let credentials = Credentials::generate()?; - let server_address = lookup_host_by_family(options.remote_host()?, options.address_family())?; + let remote_host = job_spec.remote_host(); + let remote_address = lookup_host_by_family(remote_host, config.address_family)?; // Control channel --------------- timers.next("control channel"); let (mut control, server_message) = Channel::transact( &credentials, - server_address, + remote_host, + remote_address.into(), &display, - &options, - bandwidth, - quic, + config, + ¶meters, ) .await?; // Data channel ------------------ - let server_address_port = match server_address { + let server_address_port = match remote_address { std::net::IpAddr::V4(ip) => SocketAddrV4::new(ip, server_message.port).into(), std::net::IpAddr::V6(ip) => SocketAddrV6::new(ip, server_message.port, 0, 0).into(), }; - let spinner = if options.quiet { + let spinner = if parameters.quiet { ProgressBar::hidden() } else { display.add(ProgressBar::new_spinner().with_style(spinner_style()?)) @@ -80,15 +81,14 @@ pub async fn client_main( &credentials, server_message.cert.into(), &server_address_port, - quic.port, - bandwidth, + config, job_spec.throughput_mode(), )?; debug!("Opening QUIC connection to {server_address_port:?}"); debug!("Local endpoint address is {:?}", endpoint.local_addr()?); let connection = timeout( - quic.timeout, + config.timeout_duration(), endpoint.connect(server_address_port, &server_message.name)?, ) .await @@ -102,8 +102,8 @@ pub async fn client_main( job_spec, display.clone(), spinner.clone(), - bandwidth, - options.quiet, + config, + parameters.quiet, ) .await; let total_bytes = match result { @@ -118,11 +118,11 @@ pub async fn client_main( let remote_stats = control.read_closedown_report().await?; let control_fut = control.close(); - let _ = timeout(quic.timeout, endpoint.wait_idle()) + let _ = timeout(config.timeout_duration(), endpoint.wait_idle()) .await .inspect_err(|_| warn!("QUIC shutdown timed out")); // otherwise ignore errors trace!("QUIC closed; waiting for control channel"); - let _ = timeout(quic.timeout, control_fut) + let _ = timeout(config.timeout_duration(), control_fut) .await .inspect_err(|_| warn!("control channel timed out")); // Ignore errors. If the control channel closedown times out, we expect its drop handler will do the Right Thing. @@ -130,19 +130,19 @@ pub async fn client_main( timers.stop(); // Post-transfer chatter ----------- - if !options.quiet { + if !parameters.quiet { let transport_time = timers.find(SHOW_TIME).and_then(Stopwatch::elapsed); crate::util::stats::process_statistics( &connection.stats(), total_bytes, transport_time, remote_stats, - bandwidth, - options.statistics, + config, + parameters.statistics, ); } - if options.profile { + if parameters.profile { info!("Elapsed time by phase:\n{timers}"); } display.clear()?; @@ -157,11 +157,12 @@ async fn manage_request( copy_spec: CopyJobSpec, display: MultiProgress, spinner: ProgressBar, - bandwidth: BandwidthParams, + config: &Configuration, quiet: bool, ) -> Result { let mut tasks = tokio::task::JoinSet::new(); let connection = connection.clone(); + let config = config.clone(); let _jh = tasks.spawn(async move { // This async block returns a Result let sp = connection.open_bi().map_err(|e| anyhow::anyhow!(e)).await?; @@ -169,12 +170,12 @@ async fn manage_request( // This async block reports on errors. if copy_spec.source.host.is_some() { // This is a Get - do_get(sp, ©_spec, display, spinner, bandwidth, quiet) + do_get(sp, ©_spec, display, spinner, &config, quiet) .instrument(trace_span!("GET", filename = copy_spec.source.filename)) .await } else { // This is a Put - do_put(sp, ©_spec, display, spinner, bandwidth, quiet) + do_put(sp, ©_spec, display, spinner, &config, quiet) .instrument(trace_span!("PUT", filename = copy_spec.source.filename)) .await } @@ -256,8 +257,7 @@ pub(crate) fn create_endpoint( credentials: &Credentials, server_cert: CertificateDer<'_>, server_addr: &SocketAddr, - port: Option, - bandwidth: BandwidthParams, + options: &Configuration, mode: ThroughputMode, ) -> Result { let _ = span!(Level::TRACE, "create_endpoint").entered(); @@ -271,16 +271,16 @@ pub(crate) fn create_endpoint( ); let mut config = quinn::ClientConfig::new(Arc::new(QuicClientConfig::try_from(tls_config)?)); - let _ = config.transport_config(crate::transport::create_config(bandwidth, mode)?); + let _ = config.transport_config(crate::transport::create_config(options, mode)?); - trace!("bind & configure socket, port={port:?}", port = port); - let mut socket = util::socket::bind_range_for_peer(server_addr, port)?; + trace!("bind & configure socket, port={:?}", options.port); + let mut socket = util::socket::bind_range_for_peer(server_addr, options.port)?; let wanted_send = match mode { - ThroughputMode::Both | ThroughputMode::Tx => Some(bandwidth.send_buffer().try_into()?), + ThroughputMode::Both | ThroughputMode::Tx => Some(Configuration::send_buffer().try_into()?), ThroughputMode::Rx => None, }; let wanted_recv = match mode { - ThroughputMode::Both | ThroughputMode::Rx => Some(bandwidth.recv_buffer().try_into()?), + ThroughputMode::Both | ThroughputMode::Rx => Some(Configuration::recv_buffer().try_into()?), ThroughputMode::Tx => None, }; @@ -301,7 +301,7 @@ async fn do_get( job: &CopyJobSpec, display: MultiProgress, spinner: ProgressBar, - bandwidth: BandwidthParams, + config: &Configuration, quiet: bool, ) -> Result { let filename = &job.source.filename; @@ -338,7 +338,7 @@ async fn do_get( .with_elapsed(Instant::now().duration_since(real_start)); let mut meter = - crate::client::meter::InstaMeterRunner::new(&progress_bar, spinner, bandwidth.rx()); + crate::client::meter::InstaMeterRunner::new(&progress_bar, spinner, config.rx()); meter.start().await; let inbound = progress_bar.wrap_async_read(stream.recv); @@ -366,7 +366,7 @@ async fn do_put( job: &CopyJobSpec, display: MultiProgress, spinner: ProgressBar, - bandwidth: BandwidthParams, + config: &Configuration, quiet: bool, ) -> Result { let mut stream: StreamPair = sp.into(); @@ -393,11 +393,11 @@ async fn do_put( let progress_bar = progress_bar_for(&display, job, steps, quiet)?; let mut outbound = progress_bar.wrap_async_write(stream.send); let mut meter = - crate::client::meter::InstaMeterRunner::new(&progress_bar, spinner, bandwidth.tx()); + crate::client::meter::InstaMeterRunner::new(&progress_bar, spinner, config.tx()); meter.start().await; trace!("sending command"); - let mut file = BufReader::with_capacity(bandwidth.send_buffer().try_into()?, file); + let mut file = BufReader::with_capacity(Configuration::send_buffer().try_into()?, file); outbound .write_all(&crate::protocol::session::Command::new_put(dest_filename).serialize()) diff --git a/src/client/mod.rs b/src/client/mod.rs index 577e3b9..21978a4 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,7 +1,7 @@ //! client-side (_initiator_) main loop and supporting structures -mod args; -pub use args::Options; +mod options; +pub use options::Parameters; mod control; pub use control::Channel; diff --git a/src/client/options.rs b/src/client/options.rs new file mode 100644 index 0000000..f82050d --- /dev/null +++ b/src/client/options.rs @@ -0,0 +1,93 @@ +//! Options specific to qcp client-mode +// (c) 2024 Ross Younger + +use super::FileSpec; +use clap::Parser; + +#[derive(Debug, Parser, Clone, Default)] +#[allow(clippy::struct_excessive_bools)] +/// Client-side options which may be provided on the command line, but are not persistent configuration options. +pub struct Parameters { + /// Enable detailed debug output + /// + /// This has the same effect as setting `RUST_LOG=qcp=debug` in the environment. + /// If present, `RUST_LOG` overrides this option. + #[arg(short, long, action, help_heading("Debug"))] + pub debug: bool, + + /// Log to a file + /// + /// By default the log receives everything printed to stderr. + /// To override this behaviour, set the environment variable `RUST_LOG_FILE_DETAIL` (same semantics as `RUST_LOG`). + #[arg(short('l'), long, action, help_heading("Debug"), value_name("FILE"))] + pub log_file: Option, + + /// Quiet mode + /// + /// Switches off progress display and statistics; reports only errors + #[arg(short, long, action, conflicts_with("debug"))] + pub quiet: bool, + + /// Outputs additional transfer statistics + #[arg(short = 's', long, alias("stats"), action, conflicts_with("quiet"))] + pub statistics: bool, + + /// Enables detailed debug output from the remote endpoint + /// (this may interfere with transfer speeds) + #[arg(long, action, help_heading("Debug"))] + pub remote_debug: bool, + + /// Prints timing profile data after completion + #[arg(long, action, help_heading("Debug"))] + pub profile: bool, + + // JOB SPECIFICAION ==================================================================== + // (POSITIONAL ARGUMENTS!) + /// The source file. This may be a local filename, or remote specified as HOST:FILE or USER@HOST:FILE. + /// + /// Exactly one of source and destination must be remote. + #[arg( + conflicts_with_all(crate::cli::MODE_OPTIONS), + required = true, + value_name = "SOURCE" + )] + pub source: Option, + + /// Destination. This may be a file or directory. It may be local or remote. + /// + /// If remote, specify as HOST:DESTINATION or USER@HOST:DESTINATION; or simply HOST: or USER@HOST: to copy to your home directory there. + /// + /// Exactly one of source and destination must be remote. + #[arg( + conflicts_with_all(crate::cli::MODE_OPTIONS), + required = true, + value_name = "DESTINATION" + )] + pub destination: Option, +} + +impl TryFrom<&Parameters> for crate::client::CopyJobSpec { + type Error = anyhow::Error; + + fn try_from(args: &Parameters) -> Result { + let source = args + .source + .as_ref() + .ok_or_else(|| anyhow::anyhow!("source and destination are required"))? + .clone(); + let destination = args + .destination + .as_ref() + .ok_or_else(|| anyhow::anyhow!("source and destination are required"))? + .clone(); + + if !(source.host.is_none() ^ destination.host.is_none()) { + anyhow::bail!("One file argument must be remote"); + } + + Ok(Self { + source, + destination, + }) + } +} diff --git a/src/config/manager.rs b/src/config/manager.rs new file mode 100644 index 0000000..69b7dc7 --- /dev/null +++ b/src/config/manager.rs @@ -0,0 +1,572 @@ +//! Configuration file wrangling +// (c) 2024 Ross Younger + +use super::Configuration; + +use anyhow::Result; +use figment::{ + providers::{Format, Serialized, Toml}, + value::Value, + Figment, Metadata, Provider, +}; +use serde::Deserialize; +use std::{ + collections::HashSet, + fmt::{Debug, Display}, + path::{Path, PathBuf}, +}; +use struct_field_names_as_array::FieldNamesAsSlice; +use tabled::{settings::style::Style, Table, Tabled}; + +use tracing::{trace, warn}; + +// PATHS ///////////////////////////////////////////////////////////////////////////////////////////////////// + +const BASE_CONFIG_FILENAME: &str = "qcp.toml"; + +#[cfg(unix)] +fn user_config_dir() -> Result { + // home directory for now + use etcetera::BaseStrategy as _; + Ok(etcetera::choose_base_strategy()?.home_dir().into()) +} + +#[cfg(windows)] +fn user_config_dir() -> Result { + use etcetera::{choose_app_strategy, AppStrategy as _, AppStrategyArgs}; + + Ok(choose_app_strategy(AppStrategyArgs { + top_level_domain: "com".to_string(), + author: "TeamQCP".to_string(), + app_name: env!("CARGO_PKG_NAME").to_string(), + })? + .config_dir()) +} + +#[cfg(unix)] +fn user_config_path() -> Result { + // ~/. for now + let mut d: PathBuf = user_config_dir()?; + d.push(format!(".{BASE_CONFIG_FILENAME}")); + Ok(d) +} + +#[cfg(unix)] +fn system_config_path() -> PathBuf { + // /etc/ for now + let mut p: PathBuf = PathBuf::new(); + p.push("/etc"); + p.push(BASE_CONFIG_FILENAME); + p +} + +// SYSTEM DEFAULTS ////////////////////////////////////////////////////////////////////////////////////////////// + +/// A `[https://docs.rs/figment/latest/figment/trait.Provider.html](figment::Provider)` that holds +/// our set of fixed system default options +#[derive(Default)] +struct SystemDefault {} + +impl SystemDefault { + const META_NAME: &str = "default"; +} + +impl Provider for SystemDefault { + fn metadata(&self) -> Metadata { + figment::Metadata::named(Self::META_NAME) + } + + fn data( + &self, + ) -> std::result::Result< + figment::value::Map, + figment::Error, + > { + Serialized::defaults(Configuration::default()).data() + } +} + +// CONFIG MANAGER ///////////////////////////////////////////////////////////////////////////////////////////// + +/// Processes and merges all possible configuration sources. +/// +/// Configuration file locations are platform-dependent. +/// To see what applies on the current platform, run `qcp --config-files`. +#[derive(Debug)] +pub struct Manager { + /// Configuration data + data: Figment, +} + +fn add_user_config(f: Figment) -> Figment { + let path = match user_config_path() { + Ok(p) => p, + Err(e) => { + warn!("could not determine user configuration file path: {e}"); + return f; + } + }; + if !path.exists() { + trace!("user configuration file {path:?} not present"); + return f; + } + f.merge(Toml::file(path.as_path())) +} + +fn add_system_config(f: Figment) -> Figment { + let path = system_config_path(); + if !path.exists() { + trace!("system configuration file {path:?} not present"); + return f; + } + f.merge(Toml::file(path.as_path())) +} + +impl Default for Manager { + /// Initialises this structure fully-empty (for new(), or testing) + fn default() -> Self { + Self { + data: Figment::default(), + } + } +} + +impl Manager { + /// Initialises this structure, reading the set of config files appropriate to the platform + /// and the current user. + #[must_use] + pub fn new() -> Self { + let mut data = Figment::new().merge(SystemDefault::default()); + data = add_system_config(data); + + // N.B. This may leave data in a fused-error state, if a data file isn't parseable. + data = add_user_config(data); + Self { + data, + //..Self::default() + } + } + + /// Returns the list of configuration files we read. + /// + /// This is a function of platform and the current user id. + pub fn config_files() -> Vec { + let inputs = vec![Ok(system_config_path()), user_config_path()]; + + inputs + .into_iter() + .filter_map(std::result::Result::ok) + .map(|p| p.into_os_string().to_string_lossy().into()) + .collect() + } + + /// Testing/internal constructor, does not read files from system + #[must_use] + #[allow(unused)] + pub(crate) fn without_files() -> Self { + let data = Figment::new().merge(SystemDefault::default()); + Self { + data, + //..Self::default() + } + } + + /// Merges in a data set, which is some sort of [figment::Provider](https://docs.rs/figment/latest/figment/trait.Provider.html). + /// + /// Within qcp, we use [crate::util::derive_deftly_template_Optionalify] to implement Provider for [Configuration]. + pub fn merge_provider(&mut self, provider: T) + where + T: Provider, + { + let f = std::mem::take(&mut self.data); + self.data = f.merge(provider); // in the error case, this leaves the provider in a fused state + } + + /// Merges in a data set from a TOML file + pub fn merge_toml_file(&mut self, toml: T) + where + T: AsRef, + { + let path = toml.as_ref(); + let provider = Toml::file_exact(path); + self.merge_provider(provider); + } + + /// Attempts to extract a particular struct from the data. + /// + /// Within qcp, `T` is usually [Configuration], but it isn't intrinsically required to be. + pub fn get<'de, T>(&self) -> anyhow::Result + where + T: Deserialize<'de>, + { + self.data.extract::() + } +} + +// PRETTY PRINT SUPPORT /////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Tabled)] +struct PrettyConfig { + field: String, + value: String, + source: String, +} + +impl PrettyConfig { + fn render_source(meta: Option<&Metadata>) -> String { + if let Some(m) = meta { + m.source + .as_ref() + .map_or_else(|| m.name.to_string(), figment::Source::to_string) + } else { + String::new() + } + } + + fn render_value(value: &Value) -> String { + match value { + Value::String(_tag, s) => s.to_string(), + Value::Char(_tag, c) => c.to_string(), + Value::Bool(_tag, b) => b.to_string(), + Value::Num(_tag, num) => { + if let Some(i) = num.to_i128() { + i.to_string() + } else if let Some(u) = num.to_u128() { + u.to_string() + } else if let Some(ff) = num.to_f64() { + ff.to_string() + } else { + todo!("unhandled Num case"); + } + } + Value::Empty(_tag, _) => "".into(), + // we don't currently support dict types + Value::Dict(_tag, _dict) => todo!(), + Value::Array(_tag, vec) => { + format!( + "[{}]", + vec.iter() + .map(PrettyConfig::render_value) + .collect::>() + .join(",") + ) + } + } + } + + fn new(field: &str, value: &Value, meta: Option<&Metadata>) -> Self { + Self { + field: field.into(), + value: PrettyConfig::render_value(value), + source: PrettyConfig::render_source(meta), + } + } +} + +impl Display for Manager { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let data = match self.data.data() { + Ok(d) => d, + Err(e) => { + // This isn't terribly helpful as it doesn't have metadata attached; BUT attempting to get() a struct does. + return write!(f, "error: {e}"); + } + }; + let data = data.get(&figment::Profile::Default).unwrap(); + + let mut fields = Vec::::new(); + + for field in data.keys() { + let value = self.data.find_value(field); + let value = match value { + Ok(v) => v, + Err(e) => { + writeln!(f, "error on field {field}: {e}")?; + continue; + } + }; + let meta = self.data.find_metadata(field); + fields.push(PrettyConfig::new(field, &value, meta)); + } + write!(f, "{}", Table::new(fields).with(Style::sharp())) + } +} + +/// Pretty-printing type wrapper to Manager +#[derive(Debug)] +pub struct DisplayAdapter<'a> { + /// Data source + source: &'a Manager, + /// Whether to warn if unused fields are present + warn_on_unused: bool, + /// The fields we want to output + fields: HashSet, +} + +impl Manager { + /// Creates a `DisplayAdapter` for this struct with the given options. + /// + /// # Returns + /// An ephemeral structure implementing `Display`. + #[must_use] + pub fn to_display_adapter<'de, T>(&self, warn_on_unused: bool) -> DisplayAdapter<'_> + where + T: Deserialize<'de> + FieldNamesAsSlice, + { + let mut fields = HashSet::::new(); + fields.extend(T::FIELD_NAMES_AS_SLICE.iter().map(|s| String::from(*s))); + DisplayAdapter { + source: self, + warn_on_unused, + fields, + } + } +} + +impl Display for DisplayAdapter<'_> { + /// Formats the contents of this structure which are relevant to a given output type. + /// + /// N.B. This function uses CLI styling. + #[allow(clippy::missing_panics_doc)] + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let warn = crate::cli::styles::warning(); + let data = match self.source.data.data() { + Ok(d) => d, + Err(e) => { + // This isn't terribly helpful as it doesn't have metadata attached; BUT attempting to get() a struct does. + return writeln!(f, "{}", warn.apply_to(format!("error: {e}"))); + } + }; + // panic is impossible on the Default profile, hence #[allow(clippy::missing_panics_doc)] + let data = data.get(&figment::Profile::Default).unwrap(); + + let mut output = Vec::::new(); + + for field in data.keys() { + let meta = self.source.data.find_metadata(field); + if self.fields.contains(field) { + let value = self.source.data.find_value(field); + let value = match value { + Ok(v) => v, + Err(e) => { + writeln!( + f, + "{}", + warn.apply_to(format!("error on field {field}: {e}")) + )?; + continue; + } + }; + output.push(PrettyConfig::new(field, &value, meta)); + } else if self.warn_on_unused { + let source = PrettyConfig::render_source(meta); + let _ = writeln!( + f, + "{}", + warn.apply_to(format!("Unrecognised field `{field}` in {source}")) + ); + } + } + write!(f, "{}", Table::new(output).with(Style::sharp())) + } +} + +#[cfg(test)] +mod test { + use std::path::PathBuf; + + use serde::Deserialize; + use tempfile::TempDir; + + use crate::util::PortRange; + + use crate::config::{Configuration, Configuration_Optional, Manager}; + + #[test] + fn defaults() { + let mgr = Manager::without_files(); + let result = mgr.get().unwrap(); + let expected = Configuration::default(); + assert_eq!(expected, result); + } + + #[test] + fn config_merge() { + // simulate a CLI + let entered = Configuration_Optional { + rx: Some(12345.into()), + ..Default::default() + }; + let expected = Configuration { + rx: 12345.into(), + ..Default::default() + }; + + let mut mgr = Manager::without_files(); + mgr.merge_provider(entered); + let result = mgr.get().unwrap(); + assert_eq!(expected, result); + } + + fn make_tempfile(data: &str, filename: &str) -> (PathBuf, TempDir) { + let tempdir = tempfile::tempdir().unwrap(); + let path = tempdir.path().join(filename); + std::fs::write(&path, data).expect("Unable to write tempfile"); + // println!("temp file is {:?}", &path); + (path, tempdir) + } + + #[test] + fn dump_config_cli_and_toml() { + // Not a unit test as such; this is a human test + let (path, _tempdir) = make_tempfile( + r#" + tx = 42 + congestion = "Bbr" + unused__ = 42 + "#, + "test.toml", + ); + let fake_cli = Configuration_Optional { + rtt: Some(999), + initial_congestion_window: Some(Some(67890)), // yeah the double-Some is a bit of a wart + ..Default::default() + }; + let mut mgr = Manager::without_files(); + mgr.merge_toml_file(path); + mgr.merge_provider(fake_cli); + println!("{mgr}"); + } + + #[test] + fn unparseable_toml() { + // This is a semi unit test; there is one assert, but the secondary goal is that it outputs something sensible + let (path, _tempdir) = make_tempfile( + r" + a = 1 + rx 123 # this line is a syntax error + b = 2 + ", + "test.toml", + ); + let mut mgr = Manager::without_files(); + mgr.merge_toml_file(path); + let get = mgr.get::(); + assert!(get.is_err()); + println!("{}", get.unwrap_err()); + // println!("{mgr}"); + } + + #[test] + fn type_error() { + // This is a semi unit test; this has a secondary goal of outputting something sensible + + #[derive(Deserialize)] + struct Test { + magic_: i32, + } + + let (path, _tempdir) = make_tempfile( + r" + rx = true # invalid + rtt = 3.14159 # also invalid + magic_ = 42 + ", + "test.toml", + ); + let mut mgr = Manager::without_files(); + mgr.merge_toml_file(path); + // This TOML successfully merges into the config, but you can't extract the struct. + let err = mgr.get::().unwrap_err(); + println!("Error: {err}"); + // TODO: Would really like a rich error message here pointing to the failing key and errant file. + // We get no metadata in the error :-( + + // But the config as a whole is not broken and other things can be extracted: + let other_struct = mgr.get::().unwrap(); + assert_eq!(other_struct.magic_, 42); + } + + #[test] + fn int_or_string() { + #[derive(Deserialize)] + struct Test { + t1: PortRange, + t2: PortRange, + t3: PortRange, + } + let (path, _tempdir) = make_tempfile( + r#" + t1 = 1234 + t2 = "2345" + t3 = "123-456" + "#, + "test.toml", + ); + let mut mgr = Manager::without_files(); + mgr.merge_toml_file(path); + let res = mgr.get::().unwrap(); + assert_eq!( + res.t1, + PortRange { + begin: 1234, + end: 1234 + } + ); + assert_eq!( + res.t2, + PortRange { + begin: 2345, + end: 2345 + } + ); + assert_eq!( + res.t3, + PortRange { + begin: 123, + end: 456 + } + ); + } + + #[test] + fn array_type() { + #[derive(Deserialize)] + struct Test { + ii: Vec, + } + + let (path, _tempdir) = make_tempfile( + r" + ii = [1,2,3,4,6] + ", + "test.toml", + ); + let mut mgr = Manager::without_files(); + mgr.merge_toml_file(path); + let result = mgr.get::().unwrap(); + assert_eq!(result.ii, vec![1, 2, 3, 4, 6]); + } + + #[test] + fn field_parse_failure() { + #[derive(Debug, Deserialize)] + struct Test { + _p: PortRange, + } + + let (path, _tempdir) = make_tempfile( + r#" + _p = "234-123" + "#, + "test.toml", + ); + let mut mgr = Manager::without_files(); + mgr.merge_toml_file(path); + let result = mgr.get::().unwrap_err(); + println!("{result}"); + assert!(result + .to_string() + .contains("invalid port range \"234-123\"")); + } +} diff --git a/src/config/mod.rs b/src/config/mod.rs new file mode 100644 index 0000000..655a583 --- /dev/null +++ b/src/config/mod.rs @@ -0,0 +1,38 @@ +// (c) 2024 Ross Younger +//! # Configuration management +//! +//! qcp obtains run-time configuration from the following sources, in order: +//! 1. Command-line options +//! 2. The user's configuration file (typically `~/.qcp.toml`) +//! 3. The system-wide configuration file (typically `/etc/qcp.toml`) +//! 4. Hard-wired defaults +//! +//! Each option may appear in multiple places, but only the first match is used. +//! +//! **Note** Configuration file locations are platform-dependent. +//! To see what applies on the current platform, run `qcp --config-files`. +//! +//! ## File format +//! +//! Configuration files use the [TOML](https://toml.io/en/) format. +//! This is a textual `key=value` format that supports comments. +//! +//! **Note** Strings are quoted; booleans and integers are not. For example: +//! +//! ```toml +//! rx="5M" # we have 40Mbit download +//! tx=1000000 # we have 8Mbit upload; we could also have written this as "1M" +//! rtt=150 # servers we care about are an ocean away +//! congestion="bbr" # this works well for us +//! ``` +//! +//! ## Configurable options +//! +//! The full list of supported fields is defined by [Configuration]. + +mod structure; +pub use structure::Configuration; +pub(crate) use structure::Configuration_Optional; + +mod manager; +pub use manager::Manager; diff --git a/src/config/structure.rs b/src/config/structure.rs new file mode 100644 index 0000000..8ecf93c --- /dev/null +++ b/src/config/structure.rs @@ -0,0 +1,252 @@ +//! Configuration structure +// (c) 2024 Ross Younger + +use std::time::Duration; + +use clap::Parser; +use human_repr::{HumanCount as _, HumanDuration as _}; +use serde::{Deserialize, Serialize}; +use struct_field_names_as_array::FieldNamesAsSlice; + +use crate::{ + transport::CongestionControllerType, + util::{derive_deftly_template_Optionalify, humanu64::HumanU64, AddressFamily, PortRange}, +}; +use derive_deftly::Deftly; + +/// The set of configurable options supported by qcp. +/// +/// **Note:** The implementation of `default()` for this struct returns qcp's hard-wired configuration defaults. +#[derive(Deftly)] +#[derive_deftly(Optionalify)] +#[deftly(visibility = "pub(crate)")] +#[derive(Debug, Clone, PartialEq, Eq, Parser, Deserialize, Serialize, FieldNamesAsSlice)] +pub struct Configuration { + // TRANSPORT PARAMETERS ============================================================================ + // System bandwidth, UDP ports, timeout. + /// The maximum network bandwidth we expect receiving data FROM the remote system. + /// [default: 12500k] + /// + /// This may be specified directly as a number of bytes, or as an SI quantity + /// like `10M` or `256k`. **Note that this is described in BYTES, not bits**; + /// if (for example) you expect to fill a 1Gbit ethernet connection, + /// 125M might be a suitable setting. + #[arg(short('b'), long, alias("rx-bw"), help_heading("Network tuning"), display_order(10), value_name="bytes", value_parser=clap::value_parser!(HumanU64))] + pub rx: HumanU64, + /// The maximum network bandwidth we expect sending data TO the remote system, + /// if it is different from the bandwidth FROM the system. + /// + /// (For example, when you are connected via an asymmetric last-mile DSL or fibre profile.) + /// + /// If not specified, uses the value of `rx`. + #[arg(short('B'), long, alias("tx-bw"), help_heading("Network tuning"), display_order(10), value_name="bytes", value_parser=clap::value_parser!(HumanU64))] + pub tx: Option, + + /// The expected network Round Trip time to the target system, in milliseconds. + /// [default: 300] + #[arg( + short('r'), + long, + help_heading("Network tuning"), + display_order(1), + value_name("ms") + )] + pub rtt: u16, + + /// Specifies the congestion control algorithm to use. + /// [default: cubic] + #[arg( + long, + action, + value_name = "alg", + help_heading("Advanced network tuning") + )] + #[clap(value_enum)] + pub congestion: CongestionControllerType, + + /// _(Network wizards only!)_ + /// The initial value for the sending congestion control window. + /// If unspecified, the active congestion control algorithm decides. + /// + /// _Setting this value too high reduces performance!_ + #[arg(long, help_heading("Advanced network tuning"), value_name = "bytes")] + pub initial_congestion_window: Option, + + /// Uses the given UDP port or range on the local endpoint. + /// This can be useful when there is a firewall between the endpoints. + /// + /// For example: `12345`, `"20000-20100"` + /// (in a configuration file, a range must be quoted) + /// + /// If unspecified, uses any available UDP port. + #[arg(short = 'p', long, value_name("M-N"), help_heading("Connection"))] + pub port: Option, + + /// Connection timeout for the QUIC endpoints [seconds; default 5] + /// + /// This needs to be long enough for your network connection, but short enough to provide + /// a timely indication that UDP may be blocked. + #[arg(short, long, value_name("sec"), help_heading("Connection"))] + pub timeout: u16, + + // CLIENT OPTIONS ================================================================================== + /// Forces use of a particular IP version when connecting to the remote. + /// + /// If unspecified, uses whatever seems suitable given the target address or the result of DNS lookup. + // (see also [CliArgs::ipv4_alias__] and [CliArgs::ipv6_alias__]) + #[arg(long, alias("ipv"), help_heading("Connection"), group("ip address"))] + pub address_family: Option, + + /// Specifies the ssh client program to use [default: `ssh`] + #[arg(long, help_heading("Connection"))] + pub ssh: String, + + /// Provides an additional option or argument to pass to the ssh client. [default: none] + /// + /// **On the command line** you must repeat `-S` for each argument. + /// For example, to pass `-i /dev/null` to ssh, specify: `-S -i -S /dev/null` + /// + /// **In a configuration file** this field is an array of strings. + /// For the same example: `ssh_opts=["-i", "/dev/null"]` + #[arg( + short = 'S', + action, + value_name("ssh-option"), + allow_hyphen_values(true), + help_heading("Connection") + )] + pub ssh_opt: Vec, + + /// Uses the given UDP port or range on the remote endpoint. + /// This can be useful when there is a firewall between the endpoints. + /// + /// For example: `12345`, `"20000-20100"` + /// (in a configuration file, a range must be quoted) + /// + /// If unspecified, uses any available UDP port. + #[arg(short = 'P', long, value_name("M-N"), help_heading("Connection"))] + pub remote_port: Option, +} + +impl Configuration { + /// Computes the theoretical bandwidth-delay product for outbound data + #[must_use] + #[allow(clippy::cast_possible_truncation)] + pub fn bandwidth_delay_product_tx(&self) -> u64 { + self.tx() * u64::from(self.rtt) / 1000 + } + /// Computes the theoretical bandwidth-delay product for inbound data + #[must_use] + #[allow(clippy::cast_possible_truncation)] + pub fn bandwidth_delay_product_rx(&self) -> u64 { + self.rx() * u64::from(self.rtt) / 1000 + } + #[must_use] + /// Receive bandwidth (accessor) + pub fn rx(&self) -> u64 { + *self.rx + } + #[must_use] + /// Transmit bandwidth (accessor) + pub fn tx(&self) -> u64 { + if let Some(tx) = self.tx { + *tx + } else { + self.rx() + } + } + /// RTT accessor as Duration + #[must_use] + pub fn rtt_duration(&self) -> Duration { + Duration::from_millis(u64::from(self.rtt)) + } + + /// UDP kernel sending buffer size to use + #[must_use] + pub fn send_buffer() -> u64 { + // UDP kernel buffers of 2MB have proven sufficient to get close to line speed on a 300Mbit downlink with 300ms RTT. + 2_097_152 + } + /// UDP kernel receive buffer size to use + #[must_use] + pub fn recv_buffer() -> u64 { + // UDP kernel buffers of 2MB have proven sufficient to get close to line speed on a 300Mbit downlink with 300ms RTT. + 2_097_152 + } + + /// QUIC receive window + #[must_use] + pub fn recv_window(&self) -> u64 { + // The theoretical in-flight limit appears to be sufficient + self.bandwidth_delay_product_rx() + } + + /// QUIC send window + #[must_use] + pub fn send_window(&self) -> u64 { + // There might be random added latency en route, so provide for a larger send window than theoretical. + 2 * self.bandwidth_delay_product_tx() + } + + /// Accessor for `timeout`, as a Duration + #[must_use] + pub fn timeout_duration(&self) -> Duration { + Duration::from_secs(self.timeout.into()) + } + + /// Formats the transport-related options for display + #[must_use] + pub fn format_transport_config(&self) -> String { + let iwind = match self.initial_congestion_window { + None => "".to_string(), + Some(s) => s.human_count_bytes().to_string(), + }; + let (tx, rx) = (self.tx(), self.rx()); + format!( + "rx {rx} ({rxbits}), tx {tx} ({txbits}), rtt {rtt}, congestion algorithm {congestion:?} with initial window {iwind}", + tx = tx.human_count_bytes(), + txbits = (tx * 8).human_count("bit"), + rx = rx.human_count_bytes(), + rxbits = (rx * 8).human_count("bit"), + rtt = self.rtt_duration().human_duration(), + congestion = self.congestion, + ) + } +} + +impl Default for Configuration { + /// **(Unusual!)** + /// Returns qcp's hard-wired configuration defaults. + fn default() -> Self { + Self { + // Transport + rx: 12_500_000.into(), + tx: None, + rtt: 300, + congestion: CongestionControllerType::Cubic, + initial_congestion_window: None, + port: None, + timeout: 5, + + // Client + address_family: None, + ssh: "ssh".into(), + ssh_opt: vec![], + remote_port: None, + } + } +} + +#[cfg(test)] +mod test { + use super::Configuration; + + #[test] + fn flattened() { + let v = Configuration::default(); + let j = serde_json::to_string(&v).unwrap(); + let d = json::parse(&j).unwrap(); + assert!(!d.has_key("bw")); + assert!(d.has_key("rtt")); + } +} diff --git a/src/lib.rs b/src/lib.rs index eaf4adc..3f192c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,6 +42,12 @@ //! //! The [protocol] documentation contains more detail and a discussion of its security properties. //! +//! ## Configuration +//! +//! On the command line, qcp has a comprehensive `--help` message. +//! +//! Most options can also be specified in a config file. See [config] for detalis. +//! //! ## πŸ“ˆ Getting the best out of qcp //! //! See [performance](doc::performance) and [troubleshooting](doc::troubleshooting). @@ -57,6 +63,7 @@ mod cli; pub use cli::cli; // needs to be re-exported for the binary crate pub mod client; +pub mod config; pub mod protocol; pub mod server; pub mod transport; @@ -67,3 +74,8 @@ pub mod doc; pub mod os; mod version; + +#[doc(hidden)] +pub use derive_deftly; +// Use the current version of derive_deftly here: +derive_deftly::template_export_semver_check!("0.14.0"); diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 509ae88..30c86d5 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -6,11 +6,11 @@ //! # The QCP protocol //! `qcp` is a **hybrid protocol**. //! The binary contains the complete protocol implementation, -//! but not the ssh binary used to establish the control channel itself. +//! except for the ssh binary used to establish the control channel itself. //! //! The protocol flow looks like this: //! -//! 1. The user runs `qcp` from the a machine we will call the _initiator_ or _client_. +//! 1. The user runs `qcp` from the machine we will call the _initiator_ or _client_. //! * qcp uses ssh to connect to the _remote_ machine and start a `qcp --server` process there. //! * We call this link between the two processes the _control channel_. //! * The _remote_ machine is also known as the _server_, in keeping with other communication protocols. diff --git a/src/server.rs b/src/server.rs index 3418a4b..9aee995 100644 --- a/src/server.rs +++ b/src/server.rs @@ -4,14 +4,12 @@ use std::path::PathBuf; use std::sync::Arc; +use crate::config::Configuration; use crate::protocol::control::{ClientMessage, ClosedownReport, ServerMessage}; use crate::protocol::session::{Command, FileHeader, FileTrailer, Response, Status}; use crate::protocol::{self, StreamPair}; -use crate::transport::BandwidthParams; -use crate::util::socket::bind_range_for_family; -use crate::util::Credentials; -use crate::util::PortRange; -use crate::{transport, util}; +use crate::transport::ThroughputMode; +use crate::util::{io, socket, Credentials}; use anyhow::Context as _; use quinn::crypto::rustls::QuicServerConfig; @@ -27,10 +25,7 @@ use tracing::{debug, error, info, trace, trace_span, warn, Instrument}; /// Server event loop #[allow(clippy::module_name_repetitions)] -pub async fn server_main( - bandwidth: crate::transport::BandwidthParams, - quic: crate::transport::QuicParams, -) -> anyhow::Result<()> { +pub async fn server_main(config: &Configuration) -> anyhow::Result<()> { let mut stdin = tokio::io::stdin(); let mut stdout = tokio::io::stdout(); // There are tricks you can use to get an unbuffered handle to stdout, but at a typing cost. @@ -53,11 +48,11 @@ pub async fn server_main( client_message.connection_type, ); - let bandwidth_info = format!("{bandwidth:?}"); - let file_buffer_size = usize::try_from(bandwidth.send_buffer())?; + let bandwidth_info = format!("{config:?}"); + let file_buffer_size = usize::try_from(Configuration::send_buffer())?; let credentials = Credentials::generate()?; - let (endpoint, warning) = create_endpoint(&credentials, client_message, bandwidth, quic.port)?; + let (endpoint, warning) = create_endpoint(&credentials, client_message, config)?; let local_addr = endpoint.local_addr()?; debug!("Local address is {local_addr}"); ServerMessage::write( @@ -79,7 +74,7 @@ pub async fn server_main( // but a timeout is useful to give the user a cue that UDP isn't getting there. trace!("waiting for QUIC"); let (stats_tx, mut stats_rx) = oneshot::channel(); - if let Some(conn) = timeout(quic.timeout, endpoint.accept()) + if let Some(conn) = timeout(config.timeout_duration(), endpoint.accept()) .await .with_context(|| "Timed out waiting for QUIC connection")? { @@ -113,8 +108,7 @@ pub async fn server_main( fn create_endpoint( credentials: &Credentials, client_message: ClientMessage, - bandwidth: BandwidthParams, - ports: Option, + transport: &Configuration, ) -> anyhow::Result<(quinn::Endpoint, Option)> { let client_cert: CertificateDer<'_> = client_message.cert.into(); @@ -127,24 +121,24 @@ fn create_endpoint( tls_config.max_early_data_size = u32::MAX; let qsc = QuicServerConfig::try_from(tls_config)?; - let mut config = quinn::ServerConfig::with_crypto(Arc::new(qsc)); - let _ = config.transport_config(crate::transport::create_config( - bandwidth, - transport::ThroughputMode::Both, + let mut server = quinn::ServerConfig::with_crypto(Arc::new(qsc)); + let _ = server.transport_config(crate::transport::create_config( + transport, + ThroughputMode::Both, )?); - let mut socket = bind_range_for_family(client_message.connection_type, ports)?; + let mut socket = socket::bind_range_for_family(client_message.connection_type, transport.port)?; // We don't know whether client will send or receive, so configure for both. - let wanted_send = Some(usize::try_from(bandwidth.send_buffer())?); - let wanted_recv = Some(usize::try_from(bandwidth.recv_buffer())?); - let warning = util::socket::set_udp_buffer_sizes(&mut socket, wanted_send, wanted_recv)? + let wanted_send = Some(usize::try_from(Configuration::send_buffer())?); + let wanted_recv = Some(usize::try_from(Configuration::recv_buffer())?); + let warning = socket::set_udp_buffer_sizes(&mut socket, wanted_send, wanted_recv)? .inspect(|s| warn!("{s}")); // SOMEDAY: allow user to specify max_udp_payload_size in endpoint config, to support jumbo frames let runtime = quinn::default_runtime().ok_or_else(|| anyhow::anyhow!("no async runtime found"))?; Ok(( - quinn::Endpoint::new(EndpointConfig::default(), Some(config), socket, runtime)?, + quinn::Endpoint::new(EndpointConfig::default(), Some(server), socket, runtime)?, warning, )) } @@ -212,7 +206,7 @@ async fn handle_get( trace!("begin"); let path = PathBuf::from(&filename); - let (file, meta) = match crate::util::io::open_file(&filename).await { + let (file, meta) = match io::open_file(&filename).await { Ok(res) => res, Err((status, message, _)) => { return send_response(&mut stream.send, status, message.as_deref()).await; @@ -271,7 +265,7 @@ async fn handle_put(mut stream: StreamPair, destination: String) -> anyhow::Resu } let append_filename = if path.is_dir() || path.is_file() { // Destination exists - if !crate::util::io::dest_is_writeable(&path).await { + if !io::dest_is_writeable(&path).await { return send_response( &mut stream.send, Status::IncorrectPermissions, @@ -290,7 +284,7 @@ async fn handle_put(mut stream: StreamPair, destination: String) -> anyhow::Resu path_test.push("."); } if path_test.is_dir() { - if !crate::util::io::dest_is_writeable(&path_test).await { + if !io::dest_is_writeable(&path_test).await { return send_response( &mut stream.send, Status::IncorrectPermissions, diff --git a/src/transport.rs b/src/transport.rs index e81c7d6..0a4b9e6 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -1,40 +1,22 @@ //! QUIC transport configuration // (c) 2024 Ross Younger -use std::{fmt::Display, sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration}; use anyhow::Result; -use clap::Parser; -use human_repr::{HumanCount, HumanDuration as _}; -use humanize_rs::bytes::Bytes; +use human_repr::HumanCount as _; use quinn::{ congestion::{BbrConfig, CubicConfig}, TransportConfig, }; +use serde::{Deserialize, Serialize}; use tracing::debug; -use crate::util::{parse_duration, PortRange}; +use crate::config::Configuration; /// Keepalive interval for the QUIC connection pub const PROTOCOL_KEEPALIVE: Duration = Duration::from_secs(5); -/// Shared parameters used to set up the QUIC UDP connection -#[derive(Copy, Clone, Debug, Parser)] -pub struct QuicParams { - /// Uses the given UDP port or range on the local endpoint. - /// - /// This can be useful when there is a firewall between the endpoints. - #[arg(short = 'p', long, value_name("M-N"), help_heading("Connection"))] - pub port: Option, - - /// Connection timeout for the QUIC endpoints. - /// - /// This needs to be long enough for your network connection, but short enough to provide - /// a timely indication that UDP may be blocked. - #[arg(short, long, default_value("5"), value_name("sec"), value_parser=parse_duration, help_heading("Connection"))] - pub timeout: Duration, -} - /// Specifies whether to configure to maximise transmission throughput, receive throughput, or both. /// Specifying `Both` for a one-way data transfer will work, but wastes kernel memory. #[derive(Copy, Clone, Debug)] @@ -48,8 +30,19 @@ pub enum ThroughputMode { } /// Selects the congestion control algorithm to use -#[derive(Copy, Clone, Debug, strum_macros::Display, clap::ValueEnum)] -#[strum(serialize_all = "kebab_case")] +#[derive( + Copy, + Clone, + Debug, + PartialEq, + Eq, + strum_macros::Display, + clap::ValueEnum, + Serialize, + Deserialize, +)] +#[serde(rename_all = "kebab-case")] +#[strum(serialize_all = "kebab_case")] // I'm not entirely sure this does anything in this particular case pub enum CongestionControllerType { /// The congestion algorithm TCP uses. This is good for most cases. Cubic, @@ -64,143 +57,8 @@ pub enum CongestionControllerType { Bbr, } -/// Parameters needed to set up transport configuration -#[derive(Copy, Clone, Debug, Parser)] -pub struct BandwidthParams { - /// The maximum network bandwidth we expect receiving data FROM the remote system. - /// - /// This may be specified directly as a number of bytes, or as an SI quantity - /// e.g. "10M" or "256k". Note that this is described in BYTES, not bits; - /// if (for example) you expect to fill a 1Gbit ethernet connection, - /// 125M might be a suitable setting. - #[arg(short('b'), long, help_heading("Network tuning"), display_order(10), default_value("12500k"), value_name="bytes", value_parser=clap::value_parser!(Bytes))] - pub rx_bw: Bytes, - - /// The maximum network bandwidth we expect sending data TO the remote system, - /// if it is different from the bandwidth FROM the system. - /// (For example, when you are connected via an asymmetric last-mile DSL or fibre profile.) - /// [default: use the value of --rx-bw] - #[arg(short('B'), long, help_heading("Network tuning"), display_order(10), value_name="bytes", value_parser=clap::value_parser!(Bytes))] - pub tx_bw: Option>, - - /// The expected network Round Trip time to the target system, in milliseconds. - #[arg( - short('r'), - long, - help_heading("Network tuning"), - display_order(1), - default_value("300"), - value_name("ms") - )] - pub rtt: u16, - - /// Specifies the congestion control algorithm to use. - #[arg( - long, - action, - value_name = "alg", - help_heading("Advanced network tuning") - )] - #[clap(value_enum, default_value_t=CongestionControllerType::Cubic)] - pub congestion: CongestionControllerType, - - /// (Network wizards only!) - /// The initial value for the sending congestion control window. - /// - /// Setting this value too high reduces performance! - /// - /// If not specified, this setting is determined by the selected - /// congestion control algorithm. - #[arg(long, help_heading("Advanced network tuning"), value_name = "bytes")] - pub initial_congestion_window: Option, -} - -impl Display for BandwidthParams { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let iwind = match self.initial_congestion_window { - None => "".to_string(), - Some(s) => s.human_count_bytes().to_string(), - }; - let (tx, rx) = (self.tx(), self.rx()); - write!( - f, - "rx {rx} ({rxbits}), tx {tx} ({txbits}), rtt {rtt}, congestion algorithm {congestion:?} with initial window {iwind}", - tx = tx.human_count_bytes(), - txbits = (tx * 8).human_count("bit"), - rx = rx.human_count_bytes(), - rxbits = (rx * 8).human_count("bit"), - rtt = self.rtt_duration().human_duration(), - congestion = self.congestion, - ) - } -} - -impl BandwidthParams { - /// Computes the theoretical bandwidth-delay product for outbound data - #[must_use] - #[allow(clippy::cast_possible_truncation)] - pub fn bandwidth_delay_product_tx(&self) -> u64 { - self.tx() * u64::from(self.rtt) / 1000 - } - /// Computes the theoretical bandwidth-delay product for inbound data - #[must_use] - #[allow(clippy::cast_possible_truncation)] - pub fn bandwidth_delay_product_rx(&self) -> u64 { - self.rx() * u64::from(self.rtt) / 1000 - } - #[must_use] - /// Receive bandwidth (accessor) - pub fn rx(&self) -> u64 { - self.rx_bw.size() - } - #[must_use] - /// Transmit bandwidth (accessor) - pub fn tx(&self) -> u64 { - if let Some(tx) = self.tx_bw { - tx.size() - } else { - self.rx() - } - } - /// RTT accessor as Duration - #[must_use] - pub fn rtt_duration(&self) -> Duration { - Duration::from_millis(u64::from(self.rtt)) - } - - /// UDP kernel sending buffer size to use - #[must_use] - pub fn send_buffer(&self) -> u64 { - // UDP kernel buffers of 2MB have proven sufficient to get close to line speed on a 300Mbit downlink with 300ms RTT. - 2_097_152 - } - /// UDP kernel receive buffer size to use - #[must_use] - pub fn recv_buffer(&self) -> u64 { - // UDP kernel buffers of 2MB have proven sufficient to get close to line speed on a 300Mbit downlink with 300ms RTT. - 2_097_152 - } - - /// QUIC receive window - #[must_use] - pub fn recv_window(&self) -> u64 { - // The theoretical in-flight limit appears to be sufficient - self.bandwidth_delay_product_rx() - } - - /// QUIC send window - #[must_use] - pub fn send_window(&self) -> u64 { - // There might be random added latency en route, so provide for a larger send window than theoretical. - 2 * self.bandwidth_delay_product_tx() - } -} - /// Creates a `quinn::TransportConfig` for the endpoint setup -pub fn create_config( - params: BandwidthParams, - mode: ThroughputMode, -) -> Result> { +pub fn create_config(params: &Configuration, mode: ThroughputMode) -> Result> { let mut config = TransportConfig::default(); let _ = config .max_concurrent_bidi_streams(1u8.into()) @@ -212,7 +70,7 @@ pub fn create_config( ThroughputMode::Tx | ThroughputMode::Both => { let _ = config .send_window(params.send_window()) - .datagram_send_buffer_size(params.send_buffer().try_into()?); + .datagram_send_buffer_size(Configuration::send_buffer().try_into()?); } ThroughputMode::Rx => (), } @@ -222,7 +80,7 @@ pub fn create_config( ThroughputMode::Rx | ThroughputMode::Both => { let _ = config .stream_receive_window(params.recv_window().try_into()?) - .datagram_receive_buffer_size(Some(params.recv_buffer() as usize)); + .datagram_receive_buffer_size(Some(Configuration::recv_buffer() as usize)); } ThroughputMode::Tx => (), } @@ -244,13 +102,16 @@ pub fn create_config( } } - debug!("Network configuration: {params}"); + debug!( + "Network configuration: {}", + params.format_transport_config() + ); debug!( "Buffer configuration: send window {sw}, buffer {sb}; recv window {rw}, buffer {rb}", sw = params.send_window().human_count_bytes(), - sb = params.send_buffer().human_count_bytes(), + sb = Configuration::send_buffer().human_count_bytes(), rw = params.recv_window().human_count_bytes(), - rb = params.recv_buffer().human_count_bytes() + rb = Configuration::recv_buffer().human_count_bytes() ); Ok(config.into()) diff --git a/src/util/address_family.rs b/src/util/address_family.rs new file mode 100644 index 0000000..240cfbf --- /dev/null +++ b/src/util/address_family.rs @@ -0,0 +1,125 @@ +//! CLI helper - Address family +// (c) 2024 Ross Younger + +use std::fmt::Display; +use std::marker::PhantomData; +use std::str::FromStr; + +use figment::error::Actual; +use serde::Serialize; + +use crate::util::cli::IntOrString; + +/// Representation an IP address family +/// +/// This is a local type with special parsing semantics to take part in the config/CLI system. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, clap::ValueEnum)] +#[serde(from = "IntOrString", into = "u64")] +pub enum AddressFamily { + /// IPv4 + #[value(name = "4")] + V4, + /// IPv6 + #[value(name = "6")] + V6, +} + +impl From for u64 { + fn from(value: AddressFamily) -> Self { + match value { + AddressFamily::V4 => 4, + AddressFamily::V6 => 6, + } + } +} + +impl Display for AddressFamily { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let u: u8 = match self { + AddressFamily::V4 => 4, + AddressFamily::V6 => 6, + }; + write!(f, "{u}") + } +} + +impl FromStr for AddressFamily { + type Err = figment::Error; + + fn from_str(s: &str) -> Result { + if s == "4" { + Ok(AddressFamily::V4) + } else if s == "6" { + Ok(AddressFamily::V6) + } else { + Err(figment::error::Kind::InvalidType(Actual::Str(s.into()), "4 or 6".into()).into()) + } + } +} + +impl TryFrom for AddressFamily { + type Error = figment::Error; + + fn try_from(value: u64) -> Result { + match value { + 4 => Ok(AddressFamily::V4), + 6 => Ok(AddressFamily::V6), + _ => Err(figment::error::Kind::InvalidValue( + Actual::Unsigned(value.into()), + "4 or 6".into(), + ) + .into()), + } + } +} + +impl<'de> serde::Deserialize<'de> for AddressFamily { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(IntOrString(PhantomData)) + } +} + +#[cfg(test)] +mod test { + use super::AddressFamily; + + #[test] + fn serialize() { + let a = AddressFamily::V4; + let b = AddressFamily::V6; + + let aa = serde_json::to_string(&a); + let bb = serde_json::to_string(&b); + assert_eq!(aa.unwrap(), "4"); + assert_eq!(bb.unwrap(), "6"); + } + + #[test] + fn deser_str() { + let a: AddressFamily = serde_json::from_str(r#" "4" "#).unwrap(); + assert_eq!(a, AddressFamily::V4); + let a: AddressFamily = serde_json::from_str(r#" "6" "#).unwrap(); + assert_eq!(a, AddressFamily::V6); + } + + #[test] + fn deser_int() { + let a: AddressFamily = serde_json::from_str("4").unwrap(); + assert_eq!(a, AddressFamily::V4); + let a: AddressFamily = serde_json::from_str("6").unwrap(); + assert_eq!(a, AddressFamily::V6); + } + + #[test] + fn deser_invalid() { + let _ = serde_json::from_str::("true").unwrap_err(); + let _ = serde_json::from_str::("5").unwrap_err(); + let _ = serde_json::from_str::(r#" "5" "#).unwrap_err(); + let _ = serde_json::from_str::("-1").unwrap_err(); + let _ = serde_json::from_str::(r#" "42" "#).unwrap_err(); + let _ = serde_json::from_str::(r#" "string" "#).unwrap_err(); + } +} diff --git a/src/util/cli.rs b/src/util/cli.rs index 0e3b5c2..d34a38b 100644 --- a/src/util/cli.rs +++ b/src/util/cli.rs @@ -1,105 +1,51 @@ -// CLI argument +//! CLI generic serialization helpers // (c) 2024 Ross Younger -use std::{fmt::Display, str::FromStr}; - -/// Represents a number or a contiguous range of positive integers -#[derive(Debug, Clone, Copy)] -pub struct PortRange { - /// First number in the range - pub begin: u16, - /// Last number in the range. - /// The caller defines whether the range is inclusive or exclusive of `end`. - pub end: u16, -} - -impl Display for PortRange { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if self.begin == self.end { - f.write_fmt(format_args!("{}", self.begin)) - } else { - f.write_fmt(format_args!("{}-{}", self.begin, self.end)) - } - } -} - -impl FromStr for PortRange { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - if let Ok(n) = s.parse::() { - // case 1: it's a number - // port 0 is allowed here (with the usual "unspecified" semantics), the user may know what they're doing. - return Ok(Self { begin: n, end: n }); - } - // case 2: it's a range - if let Some((a, b)) = s.split_once('-') { - let aa = a.parse(); - let bb = b.parse(); - if aa.is_ok() && bb.is_ok() { - let aa = aa.unwrap_or_default(); - let bb = bb.unwrap_or_default(); - anyhow::ensure!(aa != 0, "0 is not valid in a port range"); - anyhow::ensure!(aa <= bb, "invalid range"); - return Ok(Self { begin: aa, end: bb }); - } - // else failed to parse - } - // else failed to parse - anyhow::bail!("failed to parse range"); - } -} - -/// Parse helper for Duration fields specified in seconds -pub fn parse_duration(arg: &str) -> Result { - let seconds = arg.parse()?; - Ok(std::time::Duration::from_secs(seconds)) -} - -#[cfg(test)] -mod tests { - use std::str::FromStr; - - type Uut = super::PortRange; - - #[test] - fn output_single() { - let uut = Uut { - begin: 123, - end: 123, - }; - assert_eq!(format!("{uut}"), "123"); - } - #[test] - fn output_range() { - let uut = Uut { - begin: 123, - end: 456, - }; - assert_eq!(format!("{uut}"), "123-456"); - } - #[test] - fn parse_single() { - let uut = Uut::from_str("1234").unwrap(); - assert_eq!(uut.begin, 1234); - assert_eq!(uut.end, 1234); - } - #[test] - fn parse_range() { - let uut = Uut::from_str("1234-2345").unwrap(); - assert_eq!(uut.begin, 1234); - assert_eq!(uut.end, 2345); - } - #[test] - fn invalid_range() { - let _ = Uut::from_str("1000-999").expect_err("should have failed"); - } - #[test] - fn invalid_negative() { - let _ = Uut::from_str("-500").expect_err("should have failed"); - } - #[test] - fn port_range_not_zero() { - let _ = Uut::from_str("0-1000").expect_err("should have failed"); +use std::{fmt, marker::PhantomData, str::FromStr}; + +use serde::{de, de::Visitor, Deserialize}; + +/// Deserialization helper for types which might reasonably be expressed as an +/// integer or a string. +/// +/// This is a Visitor that forwards string types to T's `FromStr` impl and +/// forwards int types to T's `From` or `From` impls. The `PhantomData` is to +/// keep the compiler from complaining about T being an unused generic type +/// parameter. We need T in order to know the Value type for the Visitor +/// impl. +#[allow(missing_debug_implementations)] +pub struct IntOrString(pub PhantomData T>); + +impl<'de, T> Visitor<'de> for IntOrString +where + T: Deserialize<'de> + TryFrom + FromStr, + ::Err: std::fmt::Display, + >::Error: std::fmt::Display, +{ + type Value = T; + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("int or string") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + T::from_str(value).map_err(de::Error::custom) + } + + fn visit_u64(self, value: u64) -> Result + where + E: de::Error, + { + T::try_from(value).map_err(de::Error::custom) + } + + fn visit_i64(self, value: i64) -> Result + where + E: de::Error, + { + let u = u64::try_from(value).map_err(de::Error::custom)?; + T::try_from(u).map_err(de::Error::custom) } } diff --git a/src/util/dns.rs b/src/util/dns.rs index e78545d..41ad8f0 100644 --- a/src/util/dns.rs +++ b/src/util/dns.rs @@ -3,26 +3,24 @@ use std::net::IpAddr; -use crate::protocol::control::ConnectionType; use anyhow::Context as _; +use super::AddressFamily; + /// DNS lookup helper /// /// Results can be restricted to a given address family. /// Only the first matching result is returned. /// If there are no matching records of the required type, returns an error. -pub fn lookup_host_by_family( - host: &str, - desired: Option, -) -> anyhow::Result { +pub fn lookup_host_by_family(host: &str, desired: Option) -> anyhow::Result { let candidates = dns_lookup::lookup_host(host) .with_context(|| format!("host name lookup for {host} failed"))?; let mut it = candidates.iter(); let found = match desired { None => it.next(), - Some(ConnectionType::Ipv4) => it.find(|addr| addr.is_ipv4()), - Some(ConnectionType::Ipv6) => it.find(|addr| addr.is_ipv6()), + Some(AddressFamily::V4) => it.find(|addr| addr.is_ipv4()), + Some(AddressFamily::V6) => it.find(|addr| addr.is_ipv6()), }; found .map(std::borrow::ToOwned::to_owned) diff --git a/src/util/humanu64.rs b/src/util/humanu64.rs new file mode 100644 index 0000000..8578fa3 --- /dev/null +++ b/src/util/humanu64.rs @@ -0,0 +1,122 @@ +//! Serialization helper type - u64 parseable by humanize_rs +// (c) 2024 Ross Younger + +use std::{marker::PhantomData, ops::Deref, str::FromStr}; + +use anyhow::Context as _; +use humanize_rs::bytes::Bytes; +use serde::Serialize; + +use super::cli::IntOrString; + +/// An integer field that may also be expressed using engineering prefixes (k, M, G, etc). +/// For example, `1k` and `1000` are the same. +/// +/// (Nerdy description: This is a newtype wrapper to `u64` that adds a flexible deserializer via `humanize_rs::bytes::Bytes`.) + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(from = "IntOrString", into = "u64")] +pub struct HumanU64(pub u64); + +impl HumanU64 { + /// standard constructor + #[must_use] + pub fn new(value: u64) -> Self { + Self(value) + } +} + +impl Deref for HumanU64 { + type Target = u64; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl From for u64 { + fn from(value: HumanU64) -> Self { + value.0 + } +} + +impl FromStr for HumanU64 { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Ok(Self::new( + Bytes::from_str(s) + .with_context(|| "parsing bytes string")? + .size(), + )) + } +} + +impl From for HumanU64 { + fn from(value: u64) -> Self { + Self::new(value) + } +} + +impl<'de> serde::Deserialize<'de> for HumanU64 { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(IntOrString(PhantomData)) + } +} + +#[cfg(test)] +impl rand::prelude::Distribution for rand::distributions::Standard { + fn sample(&self, rng: &mut R) -> HumanU64 { + rng.gen::().into() + } +} + +#[cfg(test)] +mod test { + use std::str::FromStr as _; + + use serde_test::{assert_tokens, Token}; + + use super::HumanU64; + + fn test_deser_str(s: &str, n: u64) { + let foo: HumanU64 = serde_json::from_str(s).unwrap(); + assert_eq!(*foo, n); + } + + #[test] + fn deser_number_string() { + test_deser_str("\"12345\"", 12345); + } + + #[test] + fn deser_human() { + test_deser_str("\"100k\"", 100_000); + } + + #[test] + fn deser_raw_int() { + let foo: HumanU64 = serde_json::from_str("12345").unwrap(); + assert_eq!(*foo, 12345); + } + + #[test] + fn serde_test() { + let bw = HumanU64::new(42); + assert_tokens(&bw, &[Token::U64(42)]); + } + + #[test] + fn from_int() { + let result = HumanU64::from(12345); + assert_eq!(*result, 12345); + } + #[test] + fn from_str() { + let result = HumanU64::from_str("12345k").unwrap(); + assert_eq!(*result, 12_345_000); + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs index f972f28..a550563 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,12 +1,18 @@ //! General utility code that didn't fit anywhere else // (c) 2024 Ross Younger +mod address_family; +pub use address_family::AddressFamily; + mod dns; pub use dns::lookup_host_by_family; mod cert; pub use cert::Credentials; +pub mod cli; + +pub mod humanu64; pub mod io; pub mod socket; pub mod stats; @@ -15,5 +21,8 @@ pub mod time; mod tracing; pub use tracing::setup as setup_tracing; -mod cli; -pub use cli::{parse_duration, PortRange}; +mod port_range; +pub use port_range::PortRange; + +mod optionalify; +pub use optionalify::{derive_deftly_template_Optionalify, insert_if_some}; diff --git a/src/util/optionalify.rs b/src/util/optionalify.rs new file mode 100644 index 0000000..119e7c7 --- /dev/null +++ b/src/util/optionalify.rs @@ -0,0 +1,158 @@ +//! Macro to clone a structure for use with configuration data +// (c) 2024 Ross Younger + +#![allow(meta_variable_misuse)] // false positives in these macro definitions + +use derive_deftly::define_derive_deftly; +use figment::value::{Dict, Value}; + +/// Helper function for `figment::Provider` implementation +/// +/// If the given `arg` is not None, inserts it into `dict` with key `arg_name`. +pub fn insert_if_some( + dict: &mut Dict, + arg_name: &str, + arg: Option, +) -> Result<(), figment::Error> +where + T: serde::Serialize, +{ + if let Some(a) = arg { + let _ = dict.insert(arg_name.to_string(), Value::serialize(a)?); + } + Ok(()) +} + +define_derive_deftly! { + /// Clones a structure for use with CLI ([`clap`](https://docs.rs/clap/)) and options managers ([`figment`](https://docs.rs/figment/)). + /// + /// The expected use case is for configuration structs. + /// The idea is that you define your struct how you want, then automatically generate a variety of the struct + /// to make life easier. The variant: + /// * doesn't require the user to enter all parameters (everything is an `Option`) + /// * implements the [`figment::Provider`](https://docs.rs/figment/latest/figment/trait.Provider.html) helper trait + /// which makes it easy to extract only the parameters the user entered. + /// + /// Of course, you would not set `default_value` attributes, because you would normally register + /// the defaults with the configuration system at a different place (e.g. by implementing the [`Default`][std::default::Default] trait). + /// + /// The new struct: + /// * is named `{OriginalName}_Optional` + /// * has the same fields as the original, with all their attributes, but with their types wrapped + /// in [`std::option::Option`]. (Yes, even any that were already `Option<...>`.) + /// * contains exactly the same attributes as the original, plus `#[derive(Default)]` *(see note)*. + /// * has the same visibility as the original, though you can override this with `#[deftly(visibility = ...)]` + /// + /// **Note:** + /// If you already derived Default for the original struct, add the + /// attribute ```#[deftly(already_has_default)]```. + /// This tells the macro to *not* add ```#[derive(Default)]```, avoiding a compile error. + /// + ///
+ /// CAUTION: Attribute ordering is crucial. All attributes to be cloned to the new struct + /// must appear after deriving `Optionalify`. It might look something like this: + ///
+ /// + /// ```ignore + /// #[derive(Deftly)] + /// #[derive_deftly(Optionalify)] + /// #[derive(Debug, Clone /*, WhateverElseYouNeed...*/)] + /// struct MyStruct { + /// /* ... */ + /// } + /// ``` + /// + /// As with other template macros created with [`derive-deftly`](https://docs.rs/derive-deftly/), if you need + /// to see what's going on you can use `#[derive_deftly(Optionalify[dbg])]` instead of `#[derive_deftly(Optionalify)]` + /// to see the expanded output at compile time. + export Optionalify for struct, expect items: + ${define OPTIONAL_TYPE ${paste $tdeftype _Optional}} + + /// Auto-derived struct variant + #[allow(non_camel_case_types)] + ${tattrs} + ${if not(tmeta(already_has_default)){ + #[derive(Default)] + }} + ${if tmeta(visibility) { + ${tmeta(visibility) as token_stream} + } else { + ${tvis} + }} + struct $OPTIONAL_TYPE { + $( + ${fattrs} + ${fvis} $fname: Option<$ftype>, + // Yes, if $ftype is Option, the derived struct ends up with Option>. That's OK. + ) + } + + impl figment::Provider for $OPTIONAL_TYPE { + fn metadata(&self) -> figment::Metadata { + figment::Metadata::named("command-line").interpolater(|_profile, path| { + use heck::ToKebabCase; + let key = path.last().map_or("".to_string(), |s| s.to_kebab_case()); + format!("--{key}") + }) + } + + fn data(&self) -> Result, figment::Error> { + use $crate::util::insert_if_some; + use figment::{Profile, value::{Dict, Map}}; + let mut dict = Dict::new(); + + $( + insert_if_some(&mut dict, stringify!($fname), self.${fname}.clone())?; + ) + + let mut profile_map = Map::new(); + let _ = profile_map.insert(Profile::default(), dict); + + Ok(profile_map) + } + } +} + +#[allow(clippy::module_name_repetitions)] +pub use derive_deftly_template_Optionalify; + +#[cfg(test)] +mod test { + use super::derive_deftly_template_Optionalify; + use derive_deftly::Deftly; + use figment::{providers::Serialized, Figment}; + + #[derive(Deftly)] + #[derive_deftly(Optionalify)] + #[deftly(already_has_default)] + #[derive(PartialEq, Debug, Default, serde::Serialize, serde::Deserialize)] + struct Foo { + bar: i32, + baz: String, + wibble: Option, + q: Option, + } + + #[test] + fn optionality() { + let mut entered = Foo_Optional::default(); + assert!(entered.bar.is_none()); + entered.bar = Some(999); + entered.wibble = Some(Some("hi".to_string())); + entered.q = Some(Some(123)); + + //println!("simulated cli: {entered:?}"); + let f = Figment::new() + .merge(Serialized::defaults(Foo::default())) + .merge(entered); + let working: Foo = f.extract().expect("extract failed"); + + let expected = Foo { + bar: 999, + baz: String::new(), // default + wibble: Some("hi".into()), + q: Some(123), + }; + assert_eq!(expected, working); + } +} diff --git a/src/util/port_range.rs b/src/util/port_range.rs new file mode 100644 index 0000000..320ef20 --- /dev/null +++ b/src/util/port_range.rs @@ -0,0 +1,134 @@ +/// CLI argument helper - PortRange +// (c) 2024 Ross Younger +use serde::Serialize; +use std::{fmt::Display, str::FromStr}; + +use super::cli::IntOrString; + +/// A range of UDP port numbers. +/// +/// Port 0 is allowed with the usual meaning ("any available port"), but 0 may not form part of a range. +/// +/// In a configuration file, a range must be specified as a string. For example: +/// ```toml +/// remote_port=60000 # a single port can be an integer +/// remote_port="60000" # a single port can also be a string +/// remote_port="60000-60010" # a range must be specified as a string +/// ``` +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(from = "IntOrString", into = "String")] +pub struct PortRange { + /// First number in the range + pub begin: u16, + /// Last number in the range, inclusive. + pub end: u16, +} + +impl Display for PortRange { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.begin == self.end { + f.write_fmt(format_args!("{}", self.begin)) + } else { + f.write_fmt(format_args!("{}-{}", self.begin, self.end)) + } + } +} + +impl From for String { + fn from(value: PortRange) -> Self { + value.to_string() + } +} + +impl FromStr for PortRange { + type Err = figment::Error; + + fn from_str(s: &str) -> Result { + if let Ok(n) = s.parse::() { + // case 1: it's a number + // port 0 is allowed here (with the usual "unspecified" semantics), the user may know what they're doing. + return Ok(Self { begin: n, end: n }); + } + // case 2: it's a range + if let Some((a, b)) = s.split_once('-') { + let aa = a.parse(); + let bb = b.parse(); + if aa.is_ok() && bb.is_ok() { + let aa = aa.unwrap_or_default(); + let bb = bb.unwrap_or_default(); + if aa != 0 && aa <= bb { + return Ok(Self { begin: aa, end: bb }); + } + // else invalid + } + // else failed to parse + } + // else failed to parse + Err(figment::error::Kind::Message(format!("invalid port range \"{s}\"")).into()) + } +} + +impl From for PortRange { + fn from(value: u64) -> Self { + #[allow(clippy::cast_possible_truncation)] + let v = value as u16; + PortRange { begin: v, end: v } + } +} + +impl<'de> serde::Deserialize<'de> for PortRange { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(IntOrString(std::marker::PhantomData)) + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + type Uut = super::PortRange; + + #[test] + fn output_single() { + let uut = Uut { + begin: 123, + end: 123, + }; + assert_eq!(format!("{uut}"), "123"); + } + #[test] + fn output_range() { + let uut = Uut { + begin: 123, + end: 456, + }; + assert_eq!(format!("{uut}"), "123-456"); + } + #[test] + fn parse_single() { + let uut = Uut::from_str("1234").unwrap(); + assert_eq!(uut.begin, 1234); + assert_eq!(uut.end, 1234); + } + #[test] + fn parse_range() { + let uut = Uut::from_str("1234-2345").unwrap(); + assert_eq!(uut.begin, 1234); + assert_eq!(uut.end, 2345); + } + #[test] + fn invalid_range() { + let _ = Uut::from_str("1000-999").expect_err("should have failed"); + } + #[test] + fn invalid_negative() { + let _ = Uut::from_str("-500").expect_err("should have failed"); + } + #[test] + fn port_range_not_zero() { + let _ = Uut::from_str("0-1000").expect_err("should have failed"); + } +} diff --git a/src/util/stats.rs b/src/util/stats.rs index 0e09e84..cc21cfc 100644 --- a/src/util/stats.rs +++ b/src/util/stats.rs @@ -7,7 +7,7 @@ use quinn::ConnectionStats; use std::{cmp, fmt::Display, time::Duration}; use tracing::{info, warn}; -use crate::{protocol::control::ClosedownReport, transport::BandwidthParams}; +use crate::{config::Configuration, protocol::control::ClosedownReport}; /// Human friendly output helper #[derive(Debug, Clone, Copy)] @@ -56,7 +56,7 @@ pub fn process_statistics( payload_bytes: u64, transport_time: Option, remote_stats: ClosedownReport, - bandwidth: BandwidthParams, + bandwidth: &Configuration, show_statistics: bool, ) { let locale = &num_format::Locale::en; From 1ac239751528ce039f257de94357455dea833947 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sun, 8 Dec 2024 12:02:31 +1300 Subject: [PATCH 06/54] fix: Always use the same address family with ssh and quic - Correctly propagate the AddressFamily resulting from the DNS lookup down to the ssh subprocess. - remove duplicated remote_host argument from Channel::transact, extract it from Parameters iunstead - tidy up Channel::transact arguments, remove unused try_from<&CliArgs> for CopyJobSpec --- src/cli/args.rs | 10 +--------- src/client/control.rs | 18 ++++++++---------- src/client/job.rs | 15 --------------- src/client/main_loop.rs | 6 +++--- src/client/options.rs | 10 ++++++++-- 5 files changed, 20 insertions(+), 39 deletions(-) diff --git a/src/cli/args.rs b/src/cli/args.rs index 2fbbb33..32130dc 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -3,7 +3,7 @@ use clap::{ArgAction::SetTrue, Args as _, FromArgMatches as _, Parser}; -use crate::{client::CopyJobSpec, config::Manager, util::AddressFamily}; +use crate::{config::Manager, util::AddressFamily}; /// Options that switch us into another mode i.e. which don't require source/destination arguments pub(crate) const MODE_OPTIONS: &[&str] = &["server", "help_buffers", "show_config", "config_files"]; @@ -110,11 +110,3 @@ impl From<&CliArgs> for Manager { mgr } } - -impl TryFrom<&CliArgs> for CopyJobSpec { - type Error = anyhow::Error; - - fn try_from(args: &CliArgs) -> Result { - CopyJobSpec::try_from(&args.client_params) - } -} diff --git a/src/client/control.rs b/src/client/control.rs index 5df7f1d..9d414fa 100644 --- a/src/client/control.rs +++ b/src/client/control.rs @@ -14,7 +14,7 @@ use tracing::{debug, trace, warn}; use crate::{ config::Configuration, protocol::control::{ClientMessage, ClosedownReport, ConnectionType, ServerMessage, BANNER}, - util::{AddressFamily, Credentials}, + util::Credentials, }; use super::Parameters; @@ -35,17 +35,15 @@ impl Channel { } /// Opens the control channel, checks the banner, sends the Client Message, reads the Server Message. - #[allow(clippy::too_many_arguments)] pub async fn transact( credentials: &Credentials, - remote_host: &str, connection_type: ConnectionType, display: &MultiProgress, config: &Configuration, parameters: &Parameters, ) -> Result<(Channel, ServerMessage)> { trace!("opening control channel"); - let mut new1 = Self::launch(display, config, remote_host, parameters)?; + let mut new1 = Self::launch(display, config, parameters, connection_type)?; new1.wait_for_banner().await?; let mut pipe = new1 @@ -80,19 +78,19 @@ impl Channel { fn launch( display: &MultiProgress, config: &Configuration, - remote_host: &str, parameters: &Parameters, + connection_type: ConnectionType, ) -> Result { + let remote_host = parameters.remote_host()?; let mut server = tokio::process::Command::new(&config.ssh); let _ = server.kill_on_drop(true); - let _ = match config.address_family { - None => &mut server, - Some(AddressFamily::V4) => server.arg("-4"), - Some(AddressFamily::V6) => server.arg("-6"), + let _ = match connection_type { + ConnectionType::Ipv4 => server.arg("-4"), + ConnectionType::Ipv6 => server.arg("-6"), }; let _ = server.args(&config.ssh_opt); let _ = server.args([ - remote_host, + &remote_host, "qcp", "--server", // Remote receive bandwidth = our transmit bandwidth diff --git a/src/client/job.rs b/src/client/job.rs index 03e4f63..ad0e7e2 100644 --- a/src/client/job.rs +++ b/src/client/job.rs @@ -68,21 +68,6 @@ impl CopyJobSpec { } } - /* - pub(crate) fn remote_user_host(&self) -> anyhow::Result<&str> { - let src = self.source.as_ref().ok_or(anyhow::anyhow!( - "both source and destination must be specified" - ))?; - let dest = self.destination.as_ref().ok_or(anyhow::anyhow!( - "both source and destination must be specified" - ))?; - Ok(src - .host - .as_ref() - .unwrap_or_else(|| dest.host.as_ref().unwrap())) - } - */ - pub(crate) fn remote_user_host(&self) -> &str { self.source .host diff --git a/src/client/main_loop.rs b/src/client/main_loop.rs index a9b1a01..89624ca 100644 --- a/src/client/main_loop.rs +++ b/src/client/main_loop.rs @@ -48,14 +48,14 @@ pub async fn client_main( // Prep -------------------------- let job_spec = crate::client::CopyJobSpec::try_from(¶meters)?; let credentials = Credentials::generate()?; - let remote_host = job_spec.remote_host(); - let remote_address = lookup_host_by_family(remote_host, config.address_family)?; + // If the user didn't specify the address family: we do the DNS lookup, figure it out and tell ssh to use that. + // (Otherwise if we resolved a v4 and ssh a v6 - as might happen with round-robin DNS - that could be surprising.) + let remote_address = lookup_host_by_family(job_spec.remote_host(), config.address_family)?; // Control channel --------------- timers.next("control channel"); let (mut control, server_message) = Channel::transact( &credentials, - remote_host, remote_address.into(), &display, config, diff --git a/src/client/options.rs b/src/client/options.rs index f82050d..05a25f6 100644 --- a/src/client/options.rs +++ b/src/client/options.rs @@ -1,7 +1,7 @@ //! Options specific to qcp client-mode // (c) 2024 Ross Younger -use super::FileSpec; +use super::{CopyJobSpec, FileSpec}; use clap::Parser; #[derive(Debug, Parser, Clone, Default)] @@ -66,7 +66,7 @@ pub struct Parameters { pub destination: Option, } -impl TryFrom<&Parameters> for crate::client::CopyJobSpec { +impl TryFrom<&Parameters> for CopyJobSpec { type Error = anyhow::Error; fn try_from(args: &Parameters) -> Result { @@ -91,3 +91,9 @@ impl TryFrom<&Parameters> for crate::client::CopyJobSpec { }) } } + +impl Parameters { + pub(crate) fn remote_host(&self) -> anyhow::Result { + Ok(CopyJobSpec::try_from(self)?.remote_host().to_string()) + } +} From bdee71ed36a52c04658ee1b06908e257dc837d3b Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sun, 8 Dec 2024 12:58:19 +1300 Subject: [PATCH 07/54] chore: tidy up clippy warnings for rust 1.82 --- src/protocol/control.rs | 1 - src/protocol/control_capnp.rs | 1 + src/protocol/session.rs | 1 - src/protocol/session_capnp.rs | 1 + 4 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/protocol/control.rs b/src/protocol/control.rs index 920731d..a1fb3ad 100644 --- a/src/protocol/control.rs +++ b/src/protocol/control.rs @@ -56,7 +56,6 @@ impl ClientMessage { Ok(()) } /// Deserializer - pub async fn read(read: &mut R) -> Result where R: tokio::io::AsyncRead + Unpin, diff --git a/src/protocol/control_capnp.rs b/src/protocol/control_capnp.rs index 1d6b12d..5362796 100644 --- a/src/protocol/control_capnp.rs +++ b/src/protocol/control_capnp.rs @@ -10,6 +10,7 @@ clippy::missing_panics_doc, clippy::module_name_repetitions, clippy::must_use_candidate, + clippy::needless_lifetimes, clippy::semicolon_if_nothing_returned, clippy::uninlined_format_args, clippy::used_underscore_binding diff --git a/src/protocol/session.rs b/src/protocol/session.rs index 4d99c49..d0cb928 100644 --- a/src/protocol/session.rs +++ b/src/protocol/session.rs @@ -223,7 +223,6 @@ pub struct FileTrailer {} impl FileTrailer { /// One-stop serializer - #[must_use] pub fn serialize_direct() -> Vec { let mut msg = ::capnp::message::Builder::new_default(); diff --git a/src/protocol/session_capnp.rs b/src/protocol/session_capnp.rs index f897e75..90a5f0f 100644 --- a/src/protocol/session_capnp.rs +++ b/src/protocol/session_capnp.rs @@ -11,6 +11,7 @@ clippy::missing_panics_doc, clippy::module_name_repetitions, clippy::must_use_candidate, + clippy::needless_lifetimes, clippy::semicolon_if_nothing_returned, clippy::uninlined_format_args, clippy::used_underscore_binding From 44d7668bfcc011dbfcc377661bf98ba15fecc7d1 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sun, 8 Dec 2024 13:07:27 +1300 Subject: [PATCH 08/54] docs: Update project policies and notes - README.md: add notes on limitations, move most of Contributing out to its own file - CONTRIBUTING.md: created - lib.rs: add MSRV policy - troubleshooting.rs: add ssh note --- .github/workflows/ci.yml | 4 +-- CONTRIBUTING.md | 39 +++++++++++++++++++++++++++ Cargo.toml | 2 +- README.md | 55 +++++++++++++++++++++++++++++--------- src/doc/troubleshooting.rs | 9 +++++++ src/lib.rs | 7 ++++- 6 files changed, 98 insertions(+), 18 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d07f18c..7064f7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,10 +100,8 @@ jobs: - name: install packages run: scripts/install-ubuntu-packages # Checks begin here! - - run: cargo fmt --all -- --check + - run: cargo fmt --all --check - run: cargo test --locked - run: cargo clippy --locked --all-targets - # We care that the benchmarks build and run, not about their numeric output. - # To keep the CI a bit leaner, do this in the dev profile. - run: cargo build --locked --all-targets - run: cargo doc --no-deps --locked diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d31fd02 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,39 @@ +# Contributing to QCP + +## πŸ› Bug reports & Feature requests + +Bug reports and feature requests are welcome, please open an [issue]. + +- It may be useful to check the [issues list] and the [discussions] first in case somebody else has already raised it. +- Please be aware that I mostly work on this project in my own time. + +## πŸ—οΈ Pull request policy + +If you're thinking of contributing something non-trivial, it might be best to raise it in [discussions] first so you can get feedback early. This is particularly important for new features, to ensure they are aligned with the project goals and your approach is suitable. + +* Changes should normally be based on the `dev` branch. _(Exception: hotfixes may be branched against `main`.)_ +* PRs must pass the full set of CI checks (see below). No exceptions. +* Unit tests are encouraged, particularly those which fail before and pass after a fix. +* Refactoring for its own sake is OK if driven by a feature or bugfix. +* Clean commit histories are preferred, but don't be discouraged if you don't know how to do this. git can be a tricky tool. +* Commit messages should follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). + * Begin the commit message for a feature with `feat:`. If it's a bugfix, use `fix:`. + * For a full list of supported message tags, look at `commit_parsers` in [release-plz.toml](release-plz.toml). + * This policy is in force since release 0.1.0. Earlier commits are considered grandfathered in. +* Where there is an issue number, commit messages should reference it, e.g. (#12) +* Do not edit CHANGELOG.md, that will be done for you on release. + +## β˜‘οΈ CI checks applied + +| Check | How to run it yourself | Notes | +| ----- | ---------------------- | ----- | +| Code style | `cargo fmt --all --check` | For VS Code users, `editor.formatOnSave=true` is set | +| Everything must build | `cargo build --all-targets` | +| Unit tests pass | `cargo test` | +| Lints | `cargo clippy --all-targets` | This is a reasonably pedantic set of lints, which I make no apologies for | +| Docs build | `cargo doc --no-deps` | + + +[issue]: https://github.com/crazyscot/qcp/issues/new/choose +[issues list]: https://github.com/crazyscot/qcp/issues +[discussions]: https://github.com/crazyscot/qcp/discussions diff --git a/Cargo.toml b/Cargo.toml index a701da5..4d558e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "qcp" description = "Secure remote file copy utility which uses the QUIC protocol over UDP" -rust-version = "1.81.0" +rust-version = "1.81.0" # 1.81.0 was the current Rust version when this project started resolver = "2" version = "0.1.3" edition = "2021" diff --git a/README.md b/README.md index 83f1111..6cd26b7 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ high-performance remote file copy utility for long-distance internet connections - πŸ”§ Drop-in replacement for `scp` - πŸ›‘οΈ Similar security to `scp`, using existing, well-known mechanisms - πŸš€ Better throughput on congested networks +- **(New in 0.2)** Configuration file support + +For a full list of changes, see the [changelog](CHANGELOG.md). #### Platform support status @@ -23,6 +26,9 @@ high-performance remote file copy utility for long-distance internet connections ## 🧰 Getting Started * You must have ssh access to the target machine. + - You must be able to exchange UDP packets with the target on a given port. + - If the local machine is behind connection-tracking NAT, things usually work just fine. This is the case for the vast majority of home and business network connections. + - You can tell qcp to use a particular port range if you need to. * Install the `qcp` binary on both machines. It needs to be in your `PATH` on the remote machine. * Run `qcp --help-buffers` and follow its instructions. @@ -103,35 +109,56 @@ The brief version: The [protocol] documentation contains more detail and a discussion of its security properties. -## βš–οΈ License +## πŸ“˜ Project Policies + +### βš–οΈ License + +This project is released publicly under the [GNU Affero General Public License](LICENSE). + +Alternative license terms can be made available on request on a commercial basis (see below). + +### πŸ§‘β€πŸ­ Bugs, Features & Contributions + +Bug reports and feature requests are welcome, please use the [issue] tracker. + +- It may be useful to check the [issues list] and the [discussions] first in case somebody else has already raised it. +- Please be aware that I mostly work on this project in my own time. -The initial release is made under the [GNU Affero General Public License](LICENSE). +🚧 If you're thinking of contributing code, please read [CONTRIBUTING.md](CONTRIBUTING.md). -## πŸ§‘β€πŸ­ Contributing +#### Help wanted: MacOS/BSD -Feel free to report bugs via the [bug tracker]. +I'd particularly welcome performance reports from MacOS/BSD users as those are not platforms I use regularly. -I'd particularly welcome performance reports from BSD/OSX users as that's not a platform I use regularly. +### πŸ“‘ Version number and compatibility -While suggestions and feature requests are welcome, please be aware that I mostly work on this project in my own time. +This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) 2.0.0. + +In its initial experimental phase, the major number will be kept at 0. +Breaking changes will be noted in the [changelog](CHANGELOG.md) and will trigger a minor version bump. + +The project will move to version 1.x when the protocol has stabilised. After 1.0, breaking changes will trigger a major version bump. ## πŸ’Έ Supporting the project If you find this software useful and would like to say thank you, please consider [buying me a coffee] or [ko-fi]. [Github sponsorship] is also available. -If you're a business and need a formal invoice for your accountant, my freelancing company can issue the paperwork. -For this, and any other commercial enquiries (alternative licensing, support, etc) please get in touch, to `qcp@crazyscot.com`. - Please also consider supporting the galaxy of projects this work builds upon. Most notably, [Quinn] is a pure-Rust implementation of the [QUIC] protocol, without which qcp simply wouldn't exist in its current form. -### πŸ’‘ Roadmap +If you're a business and need a formal invoice for your accountant, my freelancing company can issue the paperwork. +For this, and any other commercial enquiries please get in touch, to `qcp@crazyscot.com`. We would be pleased to discuss commercial terms for: + +* Alternative licensing +* Support +* Sponsoring feature development + +## πŸ’‘ Future Directions Some ideas for the future, in no particular order: -* A local config mechanism, so you don't have to type out the network parameters every time * Support for copying multiple files (e.g. shell globs or `scp -r`) -* Windows native support, at least for client mode +* Windows native support * Firewall/NAT traversal * Interactive file transfer (akin to `ftp`) * Smart file copy using the `rsync` protocol or similar (send only the sections you need to) @@ -140,7 +167,9 @@ Some ideas for the future, in no particular order: * Bind a daemon to a fixed port, for better firewall/NAT traversal properties but at the cost of having to implement user authentication. * _The same thing we do every night, Pinky. We try to take over the world!_ -[bug tracker]: https://github.com/crazyscot/qcp/issues +[issue]: https://github.com/crazyscot/qcp/issues/new/choose +[issues list]: https://github.com/crazyscot/qcp/issues +[discussions]: https://github.com/crazyscot/qcp/discussions [quic]: https://quicwg.github.io/ [Quinn]: https://opencollective.com/quinn-rs [rfc9000]: https://www.rfc-editor.org/rfc/rfc9000.html diff --git a/src/doc/troubleshooting.rs b/src/doc/troubleshooting.rs index 49f50af..62b7110 100644 --- a/src/doc/troubleshooting.rs +++ b/src/doc/troubleshooting.rs @@ -53,3 +53,12 @@ //! If this bothers you, you might want to look into setting up QoS on your router. //! //! There might be some mileage in having qcp try to limit its bandwidth use or tune it to be less aggressive in the face of congestion, but that's not implemented at the moment. +//! +//! ### It takes a long time to set up the control channel +//! +//! The control channel is an ordinary ssh connection, so you need to figure out how to make ssh faster. +//! This is not within qcp's control. +//! +//! * Often this is due to a DNS misconfiguration at the server side, causing it to stall until a DNS lookup times out. +//! * There are a number of guides online purporting to advise you how to speed up ssh connections; I can't vouch for them. +//! * You might also look into ssh connection multiplexing. diff --git a/src/lib.rs b/src/lib.rs index 3f192c1..76138ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,7 +21,7 @@ //! I was inspired to write this when I needed to copy a load of multi-GB files from a server on the other side of the planet. //! //! #### Limitations -//! - You must be able to ssh directly to the remote machine, and exchange UDP packets with it on a given port. (If the local machine is behind connection-tracking NAT, things work just fine. This is the case for the vast majority of home and business network connections.) +//! - You must be able to ssh directly to the remote machine, and exchange UDP packets with it on a given port. (If the local machine is behind connection-tracking NAT, things work just fine. This is the case for the vast majority of home and business network connections. If need be, you can configure qcp to use a particular port range.) //! - Network security systems can't readily identify QUIC traffic as such. It's opaque, and high bandwidth. Some security systems might flag it as a potential threat. //! //! #### What qcp is not @@ -52,6 +52,11 @@ //! //! See [performance](doc::performance) and [troubleshooting](doc::troubleshooting). //! +//! ## MSRV policy +//! +//! As this is an application crate, the MSRV is not guaranteed to remain stable. +//! The MSRV may be upgraded from time to time to take advantage of new language features. +//! //! [QUIC]: https://quicwg.github.io/ //! [ssh]: https://en.wikipedia.org/wiki/Secure_Shell //! [CDN]: https://en.wikipedia.org/wiki/Content_delivery_network From 4cac73ac38d8d96aa9db864ad2efbe057b25b13d Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sun, 8 Dec 2024 13:58:36 +1300 Subject: [PATCH 09/54] chore(release): add download notes to GH releases page --- release-plz.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/release-plz.toml b/release-plz.toml index c1cd03b..8855951 100644 --- a/release-plz.toml +++ b/release-plz.toml @@ -1,6 +1,14 @@ [workspace] dependencies_update = false #allow_dirty = true +#git_release_draft = true +git_release_body = """ +{{ changelog }} + +## Download notes +* Debian and Ubuntu users may find the .deb packages convenient. +* The Linux binary builds `qcp--unknown-linux-musl.tar.gz` are static musl binaries which should work on all distributions. +""" [changelog] header = """# Changelog From 3ac4944713f76701712a05ba1dfb10a5d0b2582f Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sun, 8 Dec 2024 15:07:08 +1300 Subject: [PATCH 10/54] tidyup: ProgressWriter is really a newtype --- src/util/tracing.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/util/tracing.rs b/src/util/tracing.rs index 403d5c5..fd7e644 100644 --- a/src/util/tracing.rs +++ b/src/util/tracing.rs @@ -110,15 +110,11 @@ pub fn setup( } /// A wrapper type so tracing can output in a way that doesn't mess up `MultiProgress` -struct ProgressWriter { - display: MultiProgress, -} +struct ProgressWriter(MultiProgress); impl ProgressWriter { fn wrap(display: &MultiProgress) -> Mutex { - Mutex::new(Self { - display: display.clone(), - }) + Mutex::new(Self(display.clone())) } } @@ -126,10 +122,10 @@ impl Write for ProgressWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { let msg = std::str::from_utf8(buf) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - if self.display.is_hidden() { + if self.0.is_hidden() { eprintln!("{msg}"); } else { - self.display.println(msg)?; + self.0.println(msg)?; } Ok(buf.len()) } From 6ff1b2809b18eae5a6f1327ea47b760987bacff1 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sun, 8 Dec 2024 23:20:14 +1300 Subject: [PATCH 11/54] tidyup: consolidate use of ANSI styling as far as possible - use consistent styling - use consts and lazy statics instead of factories - we can't help where crates pull in other dependencies, but we can at least align our own use --- Cargo.lock | 42 ++++++++++++++++++++-- Cargo.toml | 4 +++ src/cli/args.rs | 2 +- src/cli/cli_main.rs | 17 ++++----- src/cli/styles.rs | 81 +++++++++++++++++++------------------------ src/config/manager.rs | 21 ++++++----- src/util/tracing.rs | 1 + 7 files changed, 100 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 15202ae..4f9dc0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,9 +28,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -47,6 +47,16 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +[[package]] +name = "anstyle-owo-colors" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b774bbe47d3bd767710315f5e57c23a769d6c35f28456f2be8d6e20339f55f34" +dependencies = [ + "anstyle", + "owo-colors", +] + [[package]] name = "anstyle-parse" version = "0.2.6" @@ -599,6 +609,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -790,6 +806,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "owo-colors" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb37767f6569cd834a413442455e0f066d0d522de8630436e2a1761d9726ba56" +dependencies = [ + "supports-color", +] + [[package]] name = "papergrid" version = "0.13.0" @@ -888,7 +913,9 @@ dependencies = [ name = "qcp" version = "0.1.3" dependencies = [ + "anstream", "anstyle", + "anstyle-owo-colors", "anyhow", "capnp", "capnp-futures", @@ -908,8 +935,10 @@ dependencies = [ "indicatif", "jemallocator", "json", + "lazy_static", "nix", "num-format", + "owo-colors", "quinn", "rand", "rcgen", @@ -1331,6 +1360,15 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "supports-color" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8775305acf21c96926c900ad056abeef436701108518cf890020387236ac5a77" +dependencies = [ + "is_ci", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index 4d558e1..f4581aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,9 @@ lto = "thin" strip = "symbols" [dependencies] +anstream = "0.6.18" anstyle = "1.0.8" +anstyle-owo-colors = "2.0.3" anyhow = "1.0.89" capnp = "0.20.1" capnp-futures = "0.20.0" @@ -33,7 +35,9 @@ heck = "0.5.0" human-repr = "1.1.0" humanize-rs = "0.1.5" indicatif = { version = "0.17.8", features = ["tokio"] } +lazy_static = "1.5.0" num-format = { version = "0.4.4" } +owo-colors = { version = "4.0.0", features = ["supports-color"] } quinn = { version = "0.11.5", default-features = false, features = ["runtime-tokio", "rustls", "ring"] } rcgen = { version = "0.13.1" } rustls-pki-types = "1.9.0" diff --git a/src/cli/args.rs b/src/cli/args.rs index 32130dc..16832a7 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -25,7 +25,7 @@ pub(crate) const MODE_OPTIONS: &[&str] = &["server", "help_buffers", "show_confi {all-args}{after-help} " ))] -#[command(styles=super::styles::get())] +#[command(styles=super::styles::CLAP_STYLES)] #[allow(clippy::struct_excessive_bools)] pub(crate) struct CliArgs { // MODE SELECTION ====================================================================== diff --git a/src/cli/cli_main.rs b/src/cli/cli_main.rs index dfb6ee2..d596c97 100644 --- a/src/cli/cli_main.rs +++ b/src/cli/cli_main.rs @@ -3,8 +3,10 @@ use std::process::ExitCode; -use super::{args::CliArgs, styles}; - +use super::{ + args::CliArgs, + styles::{ERROR_S, INFO_S}, +}; use crate::{ client::{client_main, Parameters as ClientParameters, MAX_UPDATE_FPS}, config::{Configuration, Manager}, @@ -12,7 +14,10 @@ use crate::{ server::server_main, util::setup_tracing, }; + +use anstream::{eprintln, println}; use indicatif::{MultiProgress, ProgressDrawTarget}; +use owo_colors::OwoColorize as _; use tracing::error_span; fn trace_level(args: &ClientParameters) -> &str { @@ -64,16 +69,12 @@ pub async fn cli() -> anyhow::Result { let config = match config_manager.get::() { Ok(c) => c, Err(err) => { - println!( - "{}: Failed to parse configuration", - styles::error().apply_to("ERROR") - ); + println!("{}: Failed to parse configuration", "ERROR".style(*ERROR_S)); if err.count() == 1 { println!("{err}"); } else { - let inf = styles::info(); for (i, e) in err.into_iter().enumerate() { - println!("{}: {e}", inf.apply_to(i + 1)); + println!("{}: {e}", (i + 1).style(*INFO_S)); } } return Ok(ExitCode::FAILURE); diff --git a/src/cli/styles.rs b/src/cli/styles.rs index 724a100..b301bd5 100644 --- a/src/cli/styles.rs +++ b/src/cli/styles.rs @@ -1,49 +1,38 @@ -// Default styling for CLI output +// (c) 2024 Ross Younger +//! CLI output styling +//! +//! Users of this module probably ought to use anstream's `println!` / `eprintln!` macros, -#[must_use] -/// Styling factory -pub(crate) fn get() -> clap::builder::Styles { - clap::builder::Styles::styled() - .usage( - anstyle::Style::new() - .bold() - .underline() - .fg_color(Some(anstyle::Color::Ansi(anstyle::AnsiColor::Yellow))), - ) - .header( - anstyle::Style::new() - .bold() - .underline() - .fg_color(Some(anstyle::Color::Ansi(anstyle::AnsiColor::Yellow))), - ) - .literal(anstyle::Style::new().bold()) - .invalid( - anstyle::Style::new() - .bold() - .fg_color(Some(anstyle::Color::Ansi(anstyle::AnsiColor::Yellow))), - ) - .error( - anstyle::Style::new() - .bold() - .fg_color(Some(anstyle::Color::Ansi(anstyle::AnsiColor::Red))), - ) - .valid( - anstyle::Style::new() - .bold() - .underline() - .fg_color(Some(anstyle::Color::Ansi(anstyle::AnsiColor::Cyan))), - ) - .placeholder( - anstyle::Style::new().fg_color(Some(anstyle::Color::Ansi(anstyle::AnsiColor::Cyan))), - ) -} +#[allow(clippy::enum_glob_use)] +use anstyle::AnsiColor::*; +use anstyle::Color::Ansi; +use anstyle_owo_colors::to_owo_style; +use clap::builder::styling::Styles; +use lazy_static::lazy_static; +use owo_colors::Style as OwoStyle; -pub(crate) fn error() -> console::Style { - console::Style::new().red() -} -pub(crate) fn warning() -> console::Style { - console::Style::new().yellow() -} -pub(crate) fn info() -> console::Style { - console::Style::new().cyan() +pub(crate) const ERROR: anstyle::Style = anstyle::Style::new().bold().fg_color(Some(Ansi(Red))); +pub(crate) const WARNING: anstyle::Style = + anstyle::Style::new().bold().fg_color(Some(Ansi(Yellow))); +pub(crate) const INFO: anstyle::Style = anstyle::Style::new().fg_color(Some(Ansi(Cyan))); +pub(crate) const DEBUG: anstyle::Style = anstyle::Style::new().fg_color(Some(Ansi(Blue))); + +lazy_static! { + pub(crate) static ref ERROR_S: OwoStyle = to_owo_style(ERROR); + pub(crate) static ref WARNING_S: OwoStyle = to_owo_style(WARNING); + pub(crate) static ref INFO_S: OwoStyle = to_owo_style(INFO); + pub(crate) static ref DEBUG_S: OwoStyle = to_owo_style(DEBUG); } + +pub(crate) const CALL_OUT: anstyle::Style = anstyle::Style::new() + .underline() + .fg_color(Some(Ansi(Yellow))); + +pub(crate) const CLAP_STYLES: Styles = Styles::styled() + .usage(CALL_OUT) + .header(CALL_OUT) + .literal(anstyle::Style::new().bold()) + .invalid(WARNING) + .error(ERROR) + .valid(INFO.bold().underline()) + .placeholder(INFO); diff --git a/src/config/manager.rs b/src/config/manager.rs index 69b7dc7..7b59614 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -329,12 +329,16 @@ impl Display for DisplayAdapter<'_> { /// N.B. This function uses CLI styling. #[allow(clippy::missing_panics_doc)] fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let warn = crate::cli::styles::warning(); + use crate::cli::styles::{ERROR_S, WARNING_S}; + use anstream::eprintln; + use owo_colors::OwoColorize as _; + let data = match self.source.data.data() { Ok(d) => d, Err(e) => { // This isn't terribly helpful as it doesn't have metadata attached; BUT attempting to get() a struct does. - return writeln!(f, "{}", warn.apply_to(format!("error: {e}"))); + eprintln!("{} {e}", "ERROR".style(*ERROR_S)); + return Ok(()); } }; // panic is impossible on the Default profile, hence #[allow(clippy::missing_panics_doc)] @@ -349,21 +353,16 @@ impl Display for DisplayAdapter<'_> { let value = match value { Ok(v) => v, Err(e) => { - writeln!( - f, - "{}", - warn.apply_to(format!("error on field {field}: {e}")) - )?; + eprintln!("{}: error on {field}: {e}", "WARNING".style(*WARNING_S)); continue; } }; output.push(PrettyConfig::new(field, &value, meta)); } else if self.warn_on_unused { let source = PrettyConfig::render_source(meta); - let _ = writeln!( - f, - "{}", - warn.apply_to(format!("Unrecognised field `{field}` in {source}")) + eprintln!( + "{}: unrecognised field `{field}` in {source}", + "WARNING".style(*WARNING_S) ); } } diff --git a/src/util/tracing.rs b/src/util/tracing.rs index fd7e644..d71d41a 100644 --- a/src/util/tracing.rs +++ b/src/util/tracing.rs @@ -7,6 +7,7 @@ use std::{ sync::{Arc, Mutex}, }; +use anstream::eprintln; use anyhow::Context; use indicatif::MultiProgress; use tracing_subscriber::{fmt, prelude::*, EnvFilter, Layer}; From 5987cce3a62ffcd42daaac80868cf917aa8d9ec1 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sun, 8 Dec 2024 23:35:20 +1300 Subject: [PATCH 12/54] feat: log entries use local time --- Cargo.lock | 29 +++++++++++++++++++++++++++-- Cargo.toml | 3 ++- src/util/tracing.rs | 28 +++++++++++++++++++--------- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4f9dc0b..8c18343 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -779,6 +779,15 @@ dependencies = [ "itoa", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -951,6 +960,7 @@ dependencies = [ "strum_macros", "tabled", "tempfile", + "time", "tokio", "tokio-util", "toml", @@ -1469,15 +1479,19 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", + "itoa", + "libc", "num-conv", + "num_threads", "powerfmt", "serde", "time-core", + "time-macros", ] [[package]] @@ -1486,6 +1500,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +[[package]] +name = "time-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -1633,6 +1657,7 @@ dependencies = [ "sharded-slab", "smallvec", "thread_local", + "time", "tracing", "tracing-core", "tracing-log", diff --git a/Cargo.toml b/Cargo.toml index f4581aa..ced0a86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,10 +46,11 @@ static_assertions = "1.1.0" struct-field-names-as-array = "0.3.0" strum_macros = "0.26.4" tabled = "0.17.0" +time = { version = "0.3.37", features = ["macros"] } tokio = { version = "1.40.0", default-features = true, features = ["fs", "io-std", "macros", "process", "rt", "time", "sync"] } tokio-util = { version = "0.7.12", features = ["compat"] } tracing = "0.1.40" -tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } +tracing-subscriber = { version = "0.3.18", features = ["env-filter", "time", "local-time"] } [target.'cfg(unix)'.dependencies] nix = { version = "0.29.0", features = ["socket"] } diff --git a/src/util/tracing.rs b/src/util/tracing.rs index d71d41a..3d5c10b 100644 --- a/src/util/tracing.rs +++ b/src/util/tracing.rs @@ -10,7 +10,8 @@ use std::{ use anstream::eprintln; use anyhow::Context; use indicatif::MultiProgress; -use tracing_subscriber::{fmt, prelude::*, EnvFilter, Layer}; +use time::macros::format_description; +use tracing_subscriber::{fmt::time::OffsetTime, prelude::*, EnvFilter, Layer}; const STANDARD_ENV_VAR: &str = "RUST_LOG"; const LOG_FILE_DETAIL_ENV_VAR: &str = "RUST_LOG_FILE_DETAIL"; @@ -61,18 +62,29 @@ pub fn setup( let filter = filter_for(trace_level, STANDARD_ENV_VAR)?; // If we used the environment variable, show log targets; if we did not, we're only logging qcp, so do not show targets. - let format = fmt::layer().compact().with_target(filter.used_env); + let offset = time::UtcOffset::current_local_offset()?; + let timer = OffsetTime::<_>::new( + offset, + format_description!("[year]-[month]-[day] [hour]:[minute]:[second]"), + ); + + let format_base = || { + tracing_subscriber::fmt::layer() + .with_timer(timer.clone()) + .compact() + .with_target(filter.used_env) + }; // Users must add a filter.filter after setting up their writer. match display { None => { - let format = format + let format = format_base() .with_writer(std::io::stderr) .with_filter(filter.filter) .boxed(); layers.push(format); } Some(mp) => { - let format = format + let format = format_base() .with_writer(ProgressWriter::wrap(mp)) .with_filter(filter.filter) .boxed(); @@ -92,12 +104,10 @@ pub fn setup( } else { filter_for(trace_level, STANDARD_ENV_VAR)? }; - let layer = tracing_subscriber::fmt::layer() - .with_writer(out_file) - // Same logic for if we used the environment variable. - .with_target(filter.used_env) - .compact() + // Same logic for if we used the environment variable. + let layer = format_base() .with_ansi(false) + .with_writer(out_file) .with_filter(filter.filter) .boxed(); layers.push(layer); From ff3c5ccef8c301d2418af000106568cda943bdc1 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Mon, 9 Dec 2024 21:22:30 +1300 Subject: [PATCH 13/54] refactor: tracing layer setup --- Cargo.lock | 160 ++++++++++++++++++++++++++++++++++++++------ Cargo.toml | 3 +- src/util/tracing.rs | 76 ++++++++++++--------- 3 files changed, 186 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8c18343..cff648f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.18" @@ -148,6 +163,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "bytecount" version = "0.6.8" @@ -221,6 +242,18 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-targets 0.52.6", +] + [[package]] name = "clap" version = "4.5.20" @@ -281,6 +314,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.15" @@ -576,6 +615,29 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "016b02deb8b0c415d8d56a6f0ab265e50c22df61194e37f9be75ed3a722de8a6" +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "indexmap" version = "2.6.0" @@ -656,6 +718,16 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "json" version = "0.12.4" @@ -780,12 +852,12 @@ dependencies = [ ] [[package]] -name = "num_threads" -version = "0.1.7" +name = "num-traits" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ - "libc", + "autocfg", ] [[package]] @@ -960,7 +1032,6 @@ dependencies = [ "strum_macros", "tabled", "tempfile", - "time", "tokio", "tokio-util", "toml", @@ -1484,14 +1555,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", - "itoa", - "libc", "num-conv", - "num_threads", "powerfmt", "serde", "time-core", - "time-macros", ] [[package]] @@ -1500,16 +1567,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" -[[package]] -name = "time-macros" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" -dependencies = [ - "num-conv", - "time-core", -] - [[package]] name = "tinyvec" version = "1.8.0" @@ -1650,6 +1707,7 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ + "chrono", "matchers", "nu-ansi-term", "once_cell", @@ -1657,7 +1715,6 @@ dependencies = [ "sharded-slab", "smallvec", "thread_local", - "time", "tracing", "tracing-core", "tracing-log", @@ -1732,6 +1789,60 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn 2.0.85", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" + [[package]] name = "winapi" version = "0.3.9" @@ -1754,6 +1865,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index ced0a86..ac4ce15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,11 +46,10 @@ static_assertions = "1.1.0" struct-field-names-as-array = "0.3.0" strum_macros = "0.26.4" tabled = "0.17.0" -time = { version = "0.3.37", features = ["macros"] } tokio = { version = "1.40.0", default-features = true, features = ["fs", "io-std", "macros", "process", "rt", "time", "sync"] } tokio-util = { version = "0.7.12", features = ["compat"] } tracing = "0.1.40" -tracing-subscriber = { version = "0.3.18", features = ["env-filter", "time", "local-time"] } +tracing-subscriber = { version = "0.3.18", features = ["env-filter", "chrono"] } [target.'cfg(unix)'.dependencies] nix = { version = "0.29.0", features = ["socket"] } diff --git a/src/util/tracing.rs b/src/util/tracing.rs index 3d5c10b..c92fbcf 100644 --- a/src/util/tracing.rs +++ b/src/util/tracing.rs @@ -10,8 +10,11 @@ use std::{ use anstream::eprintln; use anyhow::Context; use indicatif::MultiProgress; -use time::macros::format_description; -use tracing_subscriber::{fmt::time::OffsetTime, prelude::*, EnvFilter, Layer}; +use tracing_subscriber::{ + fmt::{time::ChronoLocal, MakeWriter}, + prelude::*, + EnvFilter, +}; const STANDARD_ENV_VAR: &str = "RUST_LOG"; const LOG_FILE_DETAIL_ENV_VAR: &str = "RUST_LOG_FILE_DETAIL"; @@ -43,6 +46,27 @@ fn filter_for(trace_level: &str, key: &str) -> anyhow::Result { }) } +fn layer_factory( + writer: W, + filter: F, + show_target: bool, + ansi: bool, +) -> Box + Send + Sync> +where + S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>, + W: for<'writer> MakeWriter<'writer> + 'static + Sync + Send, + F: tracing_subscriber::layer::Filter + 'static + Sync + Send, +{ + tracing_subscriber::fmt::layer::() + .with_timer(ChronoLocal::rfc_3339()) + .compact() + .with_target(show_target) + .with_ansi(ansi) + .with_writer(writer) + .with_filter(filter) + .boxed() +} + /// Set up rust tracing, to console (via an optional `MultiProgress`) and optionally to file. /// /// By default we log only our events (qcp), at a given trace level. @@ -62,33 +86,23 @@ pub fn setup( let filter = filter_for(trace_level, STANDARD_ENV_VAR)?; // If we used the environment variable, show log targets; if we did not, we're only logging qcp, so do not show targets. - let offset = time::UtcOffset::current_local_offset()?; - let timer = OffsetTime::<_>::new( - offset, - format_description!("[year]-[month]-[day] [hour]:[minute]:[second]"), - ); - - let format_base = || { - tracing_subscriber::fmt::layer() - .with_timer(timer.clone()) - .compact() - .with_target(filter.used_env) - }; // Users must add a filter.filter after setting up their writer. match display { None => { - let format = format_base() - .with_writer(std::io::stderr) - .with_filter(filter.filter) - .boxed(); - layers.push(format); + layers.push(layer_factory( + std::io::stderr, + filter.filter, + filter.used_env, + true, + )); } Some(mp) => { - let format = format_base() - .with_writer(ProgressWriter::wrap(mp)) - .with_filter(filter.filter) - .boxed(); - layers.push(format); + layers.push(layer_factory( + ProgressWriter::wrap(mp), + filter.filter, + filter.used_env, + true, + )); } }; @@ -104,13 +118,13 @@ pub fn setup( } else { filter_for(trace_level, STANDARD_ENV_VAR)? }; - // Same logic for if we used the environment variable. - let layer = format_base() - .with_ansi(false) - .with_writer(out_file) - .with_filter(filter.filter) - .boxed(); - layers.push(layer); + // Same logic for whether we used the environment variable. + layers.push(layer_factory( + out_file, + filter.filter, + filter.used_env, + false, + )); } //////// From 3ea187664c1290530a26767eff5cb47db6c03f20 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Mon, 9 Dec 2024 21:48:22 +1300 Subject: [PATCH 14/54] feat: allow user to specify the time stamp format for printed/logged messages * Supported time formats: local, utc, rfc3339 * On the CLI: --time-format rfc3339 * In the configuration: time_format="rfc3339" --- src/cli/cli_main.rs | 14 ++++--- src/client/options.rs | 19 +++++++--- src/config/structure.rs | 11 +++++- src/util/mod.rs | 2 +- src/util/tracing.rs | 82 +++++++++++++++++++++++++++++++++++------ 5 files changed, 103 insertions(+), 25 deletions(-) diff --git a/src/cli/cli_main.rs b/src/cli/cli_main.rs index d596c97..fac2cac 100644 --- a/src/cli/cli_main.rs +++ b/src/cli/cli_main.rs @@ -50,12 +50,6 @@ pub async fn cli() -> anyhow::Result { let progress = (!args.server).then(|| { MultiProgress::with_draw_target(ProgressDrawTarget::stderr_with_hz(MAX_UPDATE_FPS)) }); - setup_tracing( - trace_level(&args.client_params), - progress.as_ref(), - &args.client_params.log_file, - ) - .inspect_err(|e| eprintln!("{e:?}"))?; if args.config_files { // do this before attempting to read config, in case it fails @@ -81,6 +75,14 @@ pub async fn cli() -> anyhow::Result { } }; + setup_tracing( + trace_level(&args.client_params), + progress.as_ref(), + &args.client_params.log_file, + config.time_format, + ) + .inspect_err(|e| eprintln!("{e:?}"))?; + if args.show_config { println!( "{}", diff --git a/src/client/options.rs b/src/client/options.rs index 05a25f6..c5e284c 100644 --- a/src/client/options.rs +++ b/src/client/options.rs @@ -19,17 +19,24 @@ pub struct Parameters { /// /// By default the log receives everything printed to stderr. /// To override this behaviour, set the environment variable `RUST_LOG_FILE_DETAIL` (same semantics as `RUST_LOG`). - #[arg(short('l'), long, action, help_heading("Debug"), value_name("FILE"))] + #[arg(short('l'), long, action, value_name("FILE"), help_heading("Output"))] pub log_file: Option, /// Quiet mode /// /// Switches off progress display and statistics; reports only errors - #[arg(short, long, action, conflicts_with("debug"))] + #[arg(short, long, action, conflicts_with("debug"), help_heading("Output"))] pub quiet: bool, - /// Outputs additional transfer statistics - #[arg(short = 's', long, alias("stats"), action, conflicts_with("quiet"))] + /// Show additional transfer statistics + #[arg( + short = 's', + long, + alias("stats"), + action, + conflicts_with("quiet"), + help_heading("Output") + )] pub statistics: bool, /// Enables detailed debug output from the remote endpoint @@ -37,8 +44,8 @@ pub struct Parameters { #[arg(long, action, help_heading("Debug"))] pub remote_debug: bool, - /// Prints timing profile data after completion - #[arg(long, action, help_heading("Debug"))] + /// Output timing profile data after completion + #[arg(long, action, help_heading("Output"))] pub profile: bool, // JOB SPECIFICAION ==================================================================== diff --git a/src/config/structure.rs b/src/config/structure.rs index 8ecf93c..0932271 100644 --- a/src/config/structure.rs +++ b/src/config/structure.rs @@ -10,8 +10,12 @@ use struct_field_names_as_array::FieldNamesAsSlice; use crate::{ transport::CongestionControllerType, - util::{derive_deftly_template_Optionalify, humanu64::HumanU64, AddressFamily, PortRange}, + util::{ + derive_deftly_template_Optionalify, humanu64::HumanU64, AddressFamily, PortRange, + TimeFormat, + }, }; + use derive_deftly::Deftly; /// The set of configurable options supported by qcp. @@ -126,6 +130,10 @@ pub struct Configuration { /// If unspecified, uses any available UDP port. #[arg(short = 'P', long, value_name("M-N"), help_heading("Connection"))] pub remote_port: Option, + + /// Specifies the time format to use when printing messages to the console or to file + #[arg(short = 'T', long, value_name("FORMAT"), help_heading("Output"))] + pub time_format: TimeFormat, } impl Configuration { @@ -233,6 +241,7 @@ impl Default for Configuration { ssh: "ssh".into(), ssh_opt: vec![], remote_port: None, + time_format: TimeFormat::Local, } } } diff --git a/src/util/mod.rs b/src/util/mod.rs index a550563..bca7975 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -19,7 +19,7 @@ pub mod stats; pub mod time; mod tracing; -pub use tracing::setup as setup_tracing; +pub use tracing::{setup as setup_tracing, TimeFormat}; mod port_range; pub use port_range::PortRange; diff --git a/src/util/tracing.rs b/src/util/tracing.rs index c92fbcf..3afaa0d 100644 --- a/src/util/tracing.rs +++ b/src/util/tracing.rs @@ -10,15 +10,52 @@ use std::{ use anstream::eprintln; use anyhow::Context; use indicatif::MultiProgress; +use serde::{Deserialize, Serialize}; use tracing_subscriber::{ - fmt::{time::ChronoLocal, MakeWriter}, + fmt::{ + time::{ChronoLocal, ChronoUtc}, + MakeWriter, + }, prelude::*, EnvFilter, }; +const FRIENDLY_FORMAT_LOCAL: &str = "%Y-%m-%d %H:%M:%SL"; +const FRIENDLY_FORMAT_UTC: &str = "%Y-%m-%d %H:%M:%SZ"; + +/// Environment variable that controls what gets logged to stderr const STANDARD_ENV_VAR: &str = "RUST_LOG"; +/// Environment variable that controls what gets logged to file const LOG_FILE_DETAIL_ENV_VAR: &str = "RUST_LOG_FILE_DETAIL"; +/// Selects the format of time stamps in output messages +#[derive( + Copy, + Clone, + Debug, + Default, + Eq, + PartialEq, + strum_macros::Display, + clap::ValueEnum, + Serialize, + Deserialize, +)] +#[serde(rename_all = "kebab-case")] +pub enum TimeFormat { + /// Local time (as best as we can figure it out), as "year-month-day HH:MM:SS" + #[default] + Local, + /// UTC time, as "year-month-day HH:MM:SS" + Utc, + /// UTC time, in the format described in [RFC 3339](https://datatracker.ietf.org/doc/html/rfc3339). + /// + /// Examples: + /// `1997-11-12T09:55:06-06:00` + /// `2010-03-14T18:32:03Z` + Rfc3339, +} + /// Result type for `filter_for()` struct FilterResult { filter: EnvFilter, @@ -46,9 +83,10 @@ fn filter_for(trace_level: &str, key: &str) -> anyhow::Result { }) } -fn layer_factory( +fn make_tracing_layer( writer: W, filter: F, + time_format: TimeFormat, show_target: bool, ansi: bool, ) -> Box + Send + Sync> @@ -57,14 +95,32 @@ where W: for<'writer> MakeWriter<'writer> + 'static + Sync + Send, F: tracing_subscriber::layer::Filter + 'static + Sync + Send, { - tracing_subscriber::fmt::layer::() - .with_timer(ChronoLocal::rfc_3339()) + // The common bit + let layer = tracing_subscriber::fmt::layer::() .compact() .with_target(show_target) - .with_ansi(ansi) - .with_writer(writer) - .with_filter(filter) - .boxed() + .with_ansi(ansi); + + // Unfortunately, you have to add the timer before you can add the writer and filter, so + // there's a bit of duplication here: + match time_format { + TimeFormat::Local => layer + .with_timer(ChronoLocal::new(FRIENDLY_FORMAT_LOCAL.into())) + .with_writer(writer) + .with_filter(filter) + .boxed(), + TimeFormat::Utc => layer + .with_timer(ChronoUtc::new(FRIENDLY_FORMAT_UTC.into())) + .with_writer(writer) + .with_filter(filter) + .boxed(), + + TimeFormat::Rfc3339 => layer + .with_timer(ChronoLocal::rfc_3339()) + .with_writer(writer) + .with_filter(filter) + .boxed(), + } } /// Set up rust tracing, to console (via an optional `MultiProgress`) and optionally to file. @@ -79,6 +135,7 @@ pub fn setup( trace_level: &str, display: Option<&MultiProgress>, filename: &Option, + time_format: TimeFormat, ) -> anyhow::Result<()> { let mut layers = Vec::new(); @@ -89,17 +146,19 @@ pub fn setup( match display { None => { - layers.push(layer_factory( + layers.push(make_tracing_layer( std::io::stderr, filter.filter, + time_format, filter.used_env, true, )); } Some(mp) => { - layers.push(layer_factory( + layers.push(make_tracing_layer( ProgressWriter::wrap(mp), filter.filter, + time_format, filter.used_env, true, )); @@ -119,9 +178,10 @@ pub fn setup( filter_for(trace_level, STANDARD_ENV_VAR)? }; // Same logic for whether we used the environment variable. - layers.push(layer_factory( + layers.push(make_tracing_layer( out_file, filter.filter, + time_format, filter.used_env, false, )); From 3dde118ab3886db2828a40a29561b741b8923fa3 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Tue, 10 Dec 2024 20:41:42 +1300 Subject: [PATCH 15/54] fix: make all configuration members non-optional This fixes some sunspots introduced by configuration --- src/cli/args.rs | 4 ++-- src/client/control.rs | 11 +++++++---- src/config/manager.rs | 2 +- src/config/structure.rs | 34 +++++++++++++++++----------------- src/transport.rs | 9 +++++---- src/util/address_family.rs | 33 +++++++++++++++++++++++++-------- src/util/dns.rs | 8 ++++---- src/util/port_range.rs | 12 ++++++++++++ src/util/socket.rs | 10 +++------- 9 files changed, 76 insertions(+), 47 deletions(-) diff --git a/src/cli/args.rs b/src/cli/args.rs index 16832a7..c3b376d 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -93,9 +93,9 @@ impl CliArgs { CliArgs::from_arg_matches(&cli.get_matches_from(std::env::args_os())).unwrap(); // Custom logic: '-4' and '-6' convenience aliases if args.ipv4_alias__ { - args.config.address_family = Some(Some(AddressFamily::V4)); + args.config.address_family = Some(AddressFamily::V4); } else if args.ipv6_alias__ { - args.config.address_family = Some(Some(AddressFamily::V6)); + args.config.address_family = Some(AddressFamily::V6); } args } diff --git a/src/client/control.rs b/src/client/control.rs index 9d414fa..1b3b470 100644 --- a/src/client/control.rs +++ b/src/client/control.rs @@ -109,11 +109,14 @@ impl Channel { if parameters.remote_debug { let _ = server.arg("--debug"); } - if let Some(w) = config.initial_congestion_window { - let _ = server.args(["--initial-congestion-window", &w.to_string()]); + match config.initial_congestion_window { + 0 => (), + w => { + let _ = server.args(["--initial-congestion-window", &w.to_string()]); + } } - if let Some(pr) = config.remote_port { - let _ = server.args(["--port", &pr.to_string()]); + if !config.remote_port.is_default() { + let _ = server.args(["--port", &config.remote_port.to_string()]); } let _ = server .stdin(Stdio::piped()) diff --git a/src/config/manager.rs b/src/config/manager.rs index 7b59614..fef287b 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -428,7 +428,7 @@ mod test { ); let fake_cli = Configuration_Optional { rtt: Some(999), - initial_congestion_window: Some(Some(67890)), // yeah the double-Some is a bit of a wart + initial_congestion_window: Some(67890), ..Default::default() }; let mut mgr = Manager::without_files(); diff --git a/src/config/structure.rs b/src/config/structure.rs index 0932271..6e61def 100644 --- a/src/config/structure.rs +++ b/src/config/structure.rs @@ -21,6 +21,7 @@ use derive_deftly::Deftly; /// The set of configurable options supported by qcp. /// /// **Note:** The implementation of `default()` for this struct returns qcp's hard-wired configuration defaults. +// Maintainer note: None of the members of this struct should be Option. That leads to sunspots in the CLI and strange warts (Some(Some(foo))). #[derive(Deftly)] #[derive_deftly(Optionalify)] #[deftly(visibility = "pub(crate)")] @@ -42,9 +43,9 @@ pub struct Configuration { /// /// (For example, when you are connected via an asymmetric last-mile DSL or fibre profile.) /// - /// If not specified, uses the value of `rx`. + /// If not specified or 0, uses the value of `rx`. #[arg(short('B'), long, alias("tx-bw"), help_heading("Network tuning"), display_order(10), value_name="bytes", value_parser=clap::value_parser!(HumanU64))] - pub tx: Option, + pub tx: HumanU64, /// The expected network Round Trip time to the target system, in milliseconds. /// [default: 300] @@ -74,7 +75,7 @@ pub struct Configuration { /// /// _Setting this value too high reduces performance!_ #[arg(long, help_heading("Advanced network tuning"), value_name = "bytes")] - pub initial_congestion_window: Option, + pub initial_congestion_window: u64, /// Uses the given UDP port or range on the local endpoint. /// This can be useful when there is a firewall between the endpoints. @@ -84,7 +85,7 @@ pub struct Configuration { /// /// If unspecified, uses any available UDP port. #[arg(short = 'p', long, value_name("M-N"), help_heading("Connection"))] - pub port: Option, + pub port: PortRange, /// Connection timeout for the QUIC endpoints [seconds; default 5] /// @@ -99,7 +100,7 @@ pub struct Configuration { /// If unspecified, uses whatever seems suitable given the target address or the result of DNS lookup. // (see also [CliArgs::ipv4_alias__] and [CliArgs::ipv6_alias__]) #[arg(long, alias("ipv"), help_heading("Connection"), group("ip address"))] - pub address_family: Option, + pub address_family: AddressFamily, /// Specifies the ssh client program to use [default: `ssh`] #[arg(long, help_heading("Connection"))] @@ -129,7 +130,7 @@ pub struct Configuration { /// /// If unspecified, uses any available UDP port. #[arg(short = 'P', long, value_name("M-N"), help_heading("Connection"))] - pub remote_port: Option, + pub remote_port: PortRange, /// Specifies the time format to use when printing messages to the console or to file #[arg(short = 'T', long, value_name("FORMAT"), help_heading("Output"))] @@ -157,10 +158,9 @@ impl Configuration { #[must_use] /// Transmit bandwidth (accessor) pub fn tx(&self) -> u64 { - if let Some(tx) = self.tx { - *tx - } else { - self.rx() + match *self.tx { + 0 => self.rx(), + tx => tx, } } /// RTT accessor as Duration @@ -206,8 +206,8 @@ impl Configuration { #[must_use] pub fn format_transport_config(&self) -> String { let iwind = match self.initial_congestion_window { - None => "".to_string(), - Some(s) => s.human_count_bytes().to_string(), + 0 => "".to_string(), + s => s.human_count_bytes().to_string(), }; let (tx, rx) = (self.tx(), self.rx()); format!( @@ -229,18 +229,18 @@ impl Default for Configuration { Self { // Transport rx: 12_500_000.into(), - tx: None, + tx: 0.into(), rtt: 300, congestion: CongestionControllerType::Cubic, - initial_congestion_window: None, - port: None, + initial_congestion_window: 0, + port: PortRange::default(), timeout: 5, // Client - address_family: None, + address_family: AddressFamily::Any, ssh: "ssh".into(), ssh_opt: vec![], - remote_port: None, + remote_port: PortRange::default(), time_format: TimeFormat::Local, } } diff --git a/src/transport.rs b/src/transport.rs index 0a4b9e6..3d1115a 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -85,18 +85,19 @@ pub fn create_config(params: &Configuration, mode: ThroughputMode) -> Result (), } + let window = params.initial_congestion_window; match params.congestion { CongestionControllerType::Cubic => { let mut cubic = CubicConfig::default(); - if let Some(w) = params.initial_congestion_window { - let _ = cubic.initial_window(w); + if window != 0 { + let _ = cubic.initial_window(window); } let _ = config.congestion_controller_factory(Arc::new(cubic)); } CongestionControllerType::Bbr => { let mut bbr = BbrConfig::default(); - if let Some(w) = params.initial_congestion_window { - let _ = bbr.initial_window(w); + if window != 0 { + let _ = bbr.initial_window(window); } let _ = config.congestion_controller_factory(Arc::new(bbr)); } diff --git a/src/util/address_family.rs b/src/util/address_family.rs index 240cfbf..e342b4f 100644 --- a/src/util/address_family.rs +++ b/src/util/address_family.rs @@ -13,8 +13,7 @@ use crate::util::cli::IntOrString; /// Representation an IP address family /// /// This is a local type with special parsing semantics to take part in the config/CLI system. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, clap::ValueEnum)] -#[serde(from = "IntOrString", into = "u64")] +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] pub enum AddressFamily { /// IPv4 #[value(name = "4")] @@ -22,24 +21,39 @@ pub enum AddressFamily { /// IPv6 #[value(name = "6")] V6, + /// We don't mind what type of IP address + Any, } -impl From for u64 { +impl From for u8 { fn from(value: AddressFamily) -> Self { match value { AddressFamily::V4 => 4, AddressFamily::V6 => 6, + AddressFamily::Any => 0, + } + } +} + +impl Serialize for AddressFamily { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match *self { + AddressFamily::Any => serializer.serialize_str("any"), + t => serializer.serialize_u8(u8::from(t)), } } } impl Display for AddressFamily { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let u: u8 = match self { - AddressFamily::V4 => 4, - AddressFamily::V6 => 6, - }; - write!(f, "{u}") + if *self == AddressFamily::Any { + write!(f, "any") + } else { + write!(f, "{}", u8::from(*self)) + } } } @@ -51,6 +65,8 @@ impl FromStr for AddressFamily { Ok(AddressFamily::V4) } else if s == "6" { Ok(AddressFamily::V6) + } else if s == "0" || s == "any" { + Ok(AddressFamily::Any) } else { Err(figment::error::Kind::InvalidType(Actual::Str(s.into()), "4 or 6".into()).into()) } @@ -64,6 +80,7 @@ impl TryFrom for AddressFamily { match value { 4 => Ok(AddressFamily::V4), 6 => Ok(AddressFamily::V6), + 0 => Ok(AddressFamily::Any), _ => Err(figment::error::Kind::InvalidValue( Actual::Unsigned(value.into()), "4 or 6".into(), diff --git a/src/util/dns.rs b/src/util/dns.rs index 41ad8f0..875a34d 100644 --- a/src/util/dns.rs +++ b/src/util/dns.rs @@ -12,15 +12,15 @@ use super::AddressFamily; /// Results can be restricted to a given address family. /// Only the first matching result is returned. /// If there are no matching records of the required type, returns an error. -pub fn lookup_host_by_family(host: &str, desired: Option) -> anyhow::Result { +pub fn lookup_host_by_family(host: &str, desired: AddressFamily) -> anyhow::Result { let candidates = dns_lookup::lookup_host(host) .with_context(|| format!("host name lookup for {host} failed"))?; let mut it = candidates.iter(); let found = match desired { - None => it.next(), - Some(AddressFamily::V4) => it.find(|addr| addr.is_ipv4()), - Some(AddressFamily::V6) => it.find(|addr| addr.is_ipv6()), + AddressFamily::Any => it.next(), + AddressFamily::V4 => it.find(|addr| addr.is_ipv4()), + AddressFamily::V6 => it.find(|addr| addr.is_ipv6()), }; found .map(std::borrow::ToOwned::to_owned) diff --git a/src/util/port_range.rs b/src/util/port_range.rs index 320ef20..ecc1f44 100644 --- a/src/util/port_range.rs +++ b/src/util/port_range.rs @@ -76,6 +76,18 @@ impl From for PortRange { } } +impl Default for PortRange { + fn default() -> Self { + Self::from(0) + } +} + +impl PortRange { + pub(crate) fn is_default(self) -> bool { + self.begin == 0 && self.begin == self.end + } +} + impl<'de> serde::Deserialize<'de> for PortRange { fn deserialize(deserializer: D) -> Result where diff --git a/src/util/socket.rs b/src/util/socket.rs index 4c4631a..1552ceb 100644 --- a/src/util/socket.rs +++ b/src/util/socket.rs @@ -86,7 +86,7 @@ pub fn bind_unspecified_for(peer: &SocketAddr) -> anyhow::Result, + range: PortRange, ) -> anyhow::Result { let addr: IpAddr = match peer { SocketAddr::V4(_) => IpAddr::V4(Ipv4Addr::UNSPECIFIED), @@ -98,12 +98,8 @@ pub fn bind_range_for_peer( /// Creates and binds a UDP socket from a restricted range of local ports, for a given local address pub fn bind_range_for_address( addr: IpAddr, - range: Option, + range: PortRange, ) -> anyhow::Result { - let range = match range { - None => PortRange { begin: 0, end: 0 }, - Some(r) => r, - }; if range.begin == range.end { return Ok(UdpSocket::bind(SocketAddr::new(addr, range.begin))?); } @@ -119,7 +115,7 @@ pub fn bind_range_for_address( /// Creates and binds a UDP socket from a restricted range of local ports, for the unspecified address of the given address family pub fn bind_range_for_family( family: ConnectionType, - range: Option, + range: PortRange, ) -> anyhow::Result { let addr = match family { ConnectionType::Ipv4 => IpAddr::V4(Ipv4Addr::UNSPECIFIED), From 39dc1a67ee078d6c5fc63edd363343a9a24b2fd9 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Tue, 10 Dec 2024 23:18:09 +1300 Subject: [PATCH 16/54] package: Add Debian postinst script (#13) --- Cargo.toml | 4 ++- debian/config | 12 ++++++++ debian/postinst | 74 ++++++++++++++++++++++++++++++++++++++++++++++++ debian/postrm | 20 +++++++++++++ debian/templates | 16 +++++++++++ 5 files changed, 125 insertions(+), 1 deletion(-) create mode 100755 debian/config create mode 100755 debian/postinst create mode 100755 debian/postrm create mode 100644 debian/templates diff --git a/Cargo.toml b/Cargo.toml index ac4ce15..d8a5d0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,6 +126,8 @@ assets = [ [ "LICENSE", "usr/share/doc/qcp/", "644" ], # gzip -9n < CHANGELOG.md > misc/changelog.gz [ "misc/changelog.gz", "usr/share/doc/qcp/", "644" ], - [ "misc/20-qcp.conf", "etc/sysctl.d/", "644" ], + [ "misc/20-qcp.conf", "etc/sysctl.d/", "644" ], # this is automatically recognised as a conffile [ "misc/qcp.1", "usr/share/man/man1/", "644" ], ] +maintainer-scripts="debian" +depends = "$auto,debconf" diff --git a/debian/config b/debian/config new file mode 100755 index 0000000..6266c26 --- /dev/null +++ b/debian/config @@ -0,0 +1,12 @@ +#!/bin/sh + +set -e + +if [ -e /usr/share/debconf/confmodule ]; then + . /usr/share/debconf/confmodule + db_version 2.0 + db_capb + db_settitle qcp/title +fi + +# We cannot meaningfully preconfigure as postinst checks the filesystem at runtime. diff --git a/debian/postinst b/debian/postinst new file mode 100755 index 0000000..a5b68a0 --- /dev/null +++ b/debian/postinst @@ -0,0 +1,74 @@ +#!/bin/sh + +set -e + +# By Debian policy, packages must not ask unnecessary questions. +# +# Therefore, we examine the persistent sysctl directories to see +# if there are any mentions of the sysctls we want to set. +# If not, we presume that the system has no special requirements +# (this is expected to be the general case). + +SYSCTL_FILE=20-qcp.conf +SYSCTL_PATH=/etc/sysctl.d/${SYSCTL_FILE} + +. /usr/share/debconf/confmodule +db_version 2.0 +db_capb +db_settitle qcp/title + +#when testing this script, this line resets the db: +#db_fset qcp/sysctl_clash seen false + +check_for_clashing_sysctls() { + for DIR in /etc/sysctl.d /usr/lib/sysctl.d; do + if grep -qcr -e net.core.rmem_max -e net.core.wmem_max --exclude "*${SYSCTL_FILE}*" ${DIR}; then + return 0 + fi + done + return 1 +} + +activate_our_sysctls() { + sysctl -w -p ${SYSCTL_PATH} +} + +disable_our_file() { + if [ -e ${SYSCTL_PATH} ]; then + mv -f ${SYSCTL_PATH} ${SYSCTL_PATH}.disabled + fi +} + +try_to_enable_our_file() { + if [ -e ${SYSCTL_PATH}.disabled ]; then + mv -f ${SYSCTL_PATH}.disabled ${SYSCTL_PATH} + fi +} + +alert_sysadmin() { + db_input high qcp/sysctl_clash || true + db_go || true + + db_get qcp/sysctl_clash || true + case "$RET" in + "install and activate now") + try_to_enable_our_file + activate_our_sysctls + ;; + "install but do NOT activate") + try_to_enable_our_file + # do nothing + ;; + "do not install") + # they don't want it, OK + disable_our_file + ;; + esac +} + +if check_for_clashing_sysctls; then + alert_sysadmin +else + # No clashes; proceed quietly. + activate_our_sysctls +fi diff --git a/debian/postrm b/debian/postrm new file mode 100755 index 0000000..d8d3019 --- /dev/null +++ b/debian/postrm @@ -0,0 +1,20 @@ +#!/bin/sh + +set -e + +if [ -e /usr/share/debconf/confmodule ]; then +. /usr/share/debconf/confmodule +db_version 2 + +case "$1" in +purge) + # Remove my changes to the db. + if [ -e /usr/share/debconf/confmodule ]; then + . /usr/share/debconf/confmodule + db_version 2 + db_purge + fi + ;; +esac + +fi \ No newline at end of file diff --git a/debian/templates b/debian/templates new file mode 100644 index 0000000..c6774c8 --- /dev/null +++ b/debian/templates @@ -0,0 +1,16 @@ +Template: qcp/title +Type: title +Description: qcp sysctl files + +Template: qcp/sysctl_clash +Type: select +Choices: install and activate now, install but do NOT activate, do not install +Default: install and activate now +Description: Install sysctl files for qcp? + qcp has detected a possible sysctl clash! + . + qcp would like to install sysctl files to configure the kernel for good performance. + . + The sysctls used by qcp are: net.core.rmem_max net.core.wmem_max + . + sysctl config files may be found in: /etc/sysctl.d /usr/lib/sysctl.d From d888da0e0c28a20099718eafc54e12192dc6258e Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Tue, 10 Dec 2024 23:39:40 +1300 Subject: [PATCH 17/54] misc: add 'package' commit type --- release-plz.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-plz.toml b/release-plz.toml index 8855951..b769a40 100644 --- a/release-plz.toml +++ b/release-plz.toml @@ -73,7 +73,7 @@ commit_parsers = [ { message = "^chore\\(pr\\)", skip = true }, { message = "^chore\\(pull\\)", skip = true }, { message = "^chore\\(skip", skip = true }, - { message = "^build|^ci", group = "πŸ—οΈ Build & CI" }, + { message = "^build|^ci|^package", group = "πŸ—οΈ Build, packaging & CI" }, { message = "^chore|^misc|^tidyup", group = "βš™οΈ Miscellaneous Tasks" }, { message = "^revert", group = "◀️ Revert" }, ] From a12df26446ff7ea11f7adb477db68dbfd4270efe Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Tue, 10 Dec 2024 20:12:19 +1300 Subject: [PATCH 18/54] build: speed up link times - use split-debuginfo="unpacked" - add notes about setting up a local cargo config to use mold --- .cargo/config.toml | 15 ++++++++++++--- Cargo.toml | 3 +++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 29fd467..e5b2406 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,6 +1,15 @@ -[target.'cfg(target_os="linux")'] -# using mold breaks cargo cross on musl (the image is based on focal, which does not have a mold package) -#rustflags = ["-C", "link-arg=-fuse-ld=mold"] +[future-incompat-report] +frequency = 'always' + +#[target.'cfg(target_os="linux")'] +# Caution! Unconditionally specifying mold breaks cargo cross on musl. +# (the base docker image cross uses is based on focal, which does not have a mold package) +# So for now, we use defaults to not break CI. +# To speed up local builds, set up your own ~/.cargo/config.toml something like this: +# [target.'cfg(target_os="linux")'] +# linker = "clang-15" +# rustflags = ["-C", "link-arg=--ld-path=mold"] + [target.'cfg(target_os="windows")'] rustflags = ["-C", "link-arg=-fuse-ld=lld"] diff --git a/Cargo.toml b/Cargo.toml index d8a5d0f..9a881ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,9 @@ homepage = "https://github.com/crazyscot/qcp/" keywords = [ "networking", "file-transfer", "quic" ] categories = [ "command-line-utilities" ] +[profile.dev] +split-debuginfo="unpacked" + [profile.release] lto = "thin" strip = "symbols" From 68afab3d4a76638ab10e8fa17ff5f0ac4abbbbd8 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 12 Dec 2024 20:19:20 +1300 Subject: [PATCH 19/54] chore(deps): bump taiki-e/install-action from 2.44.58 to 2.46.6 --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7064f7b..2b7c764 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: toolchain: ${{ matrix.rust }} targets: ${{ matrix.target }} - name: Install tools - uses: taiki-e/install-action@v2.44.58 + uses: taiki-e/install-action@v2.46.6 with: tool: cross,cargo-deb #- name: Set minimal profile (Windows only) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 000f60f..2379eb8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: with: toolchain: ${{ matrix.rust }} targets: ${{ matrix.target }} - - uses: taiki-e/install-action@v2.44.58 + - uses: taiki-e/install-action@v2.46.6 with: tool: cross,cargo-deb - uses: Swatinem/rust-cache@v2 From 598b4423af4d5f87ae0e29c139cc63faf6349f19 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 12 Dec 2024 20:28:38 +1300 Subject: [PATCH 20/54] chore(deps): update 17 explicit dependencies --- Cargo.lock | 167 +++++++++++++++++++++++++---------------------------- Cargo.toml | 34 +++++------ 2 files changed, 97 insertions(+), 104 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cff648f..966bf83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,9 +58,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-owo-colors" @@ -102,9 +102,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.91" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "arrayvec" @@ -204,19 +204,20 @@ dependencies = [ [[package]] name = "capnp-futures" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271140b6249c5755f29c4a971f1760d5e8d3708eceac7eddc1448348bc8c1a65" +checksum = "7b70b0d44372d42654e3efac38c1643c7b0f9d3a9e9b72b635f942ff3f17e891" dependencies = [ "capnp", - "futures", + "futures-channel", + "futures-util", ] [[package]] name = "capnpc" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d13cb6e2643fd1f9fb804ba938323636ef52e07d6e495e372d933ad0cb98207" +checksum = "1aa3d5f01e69ed11656d2c7c47bf34327ea9bfb5c85c7de787fcd7b6c5e45b61" dependencies = [ "capnp", ] @@ -256,9 +257,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.20" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" dependencies = [ "clap_builder", "clap_derive", @@ -266,9 +267,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.20" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", @@ -286,14 +287,14 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "colorchoice" @@ -372,7 +373,7 @@ dependencies = [ "quote", "sha3", "strum", - "syn 2.0.85", + "syn 2.0.90", "void", ] @@ -445,9 +446,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "figment" @@ -468,20 +469,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - [[package]] name = "futures-channel" version = "0.3.31" @@ -489,7 +476,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", - "futures-sink", ] [[package]] @@ -522,10 +508,8 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ - "futures-channel", "futures-core", "futures-io", - "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -560,8 +544,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -650,25 +636,16 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.8" +version = "0.17.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" +checksum = "cbf675b85ed934d3c67b5c5469701eec7db22689d0a2139d856e0925fa28b281" dependencies = [ "console", - "instant", "number_prefix", "portable-atomic", "tokio", - "unicode-width 0.1.14", -] - -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", + "unicode-width 0.2.0", + "web-time", ] [[package]] @@ -978,14 +955,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] name = "proc-macro2" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -1041,9 +1018,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" dependencies = [ "bytes", "pin-project-lite", @@ -1059,19 +1036,22 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", + "getrandom", "rand", "ring", "rustc-hash", "rustls", + "rustls-pki-types", "slab", "thiserror", "tinyvec", "tracing", + "web-time", ] [[package]] @@ -1242,6 +1222,9 @@ name = "rustls-pki-types" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +dependencies = [ + "web-time", +] [[package]] name = "rustls-webpki" @@ -1268,22 +1251,22 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -1410,7 +1393,7 @@ checksum = "a2dbf8b57f3ce20e4bb171a11822b283bdfab6c4bb0fe64fa729f045f23a0938" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -1432,7 +1415,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -1463,9 +1446,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.85" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -1520,22 +1503,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.65" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] @@ -1584,9 +1567,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -1607,14 +1590,14 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -1660,9 +1643,9 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -1671,20 +1654,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -1703,9 +1686,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "chrono", "matchers", @@ -1810,7 +1793,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", "wasm-bindgen-shared", ] @@ -1832,7 +1815,7 @@ checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1843,6 +1826,16 @@ version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2058,7 +2051,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.90", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9a881ba..210e259 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,12 +21,12 @@ strip = "symbols" [dependencies] anstream = "0.6.18" -anstyle = "1.0.8" +anstyle = "1.0.10" anstyle-owo-colors = "2.0.3" -anyhow = "1.0.89" -capnp = "0.20.1" -capnp-futures = "0.20.0" -clap = { version = "4.5.19", features = ["wrap_help", "derive", "cargo", "help", "string"] } +anyhow = "1.0.94" +capnp = "0.20.3" +capnp-futures = "0.20.1" +clap = { version = "4.5.23", features = ["wrap_help", "derive", "cargo", "help", "string"] } console = "0.15.8" derive-deftly = "0.14.2" dns-lookup = "2.0.4" @@ -37,22 +37,22 @@ gethostname = "0.5.0" heck = "0.5.0" human-repr = "1.1.0" humanize-rs = "0.1.5" -indicatif = { version = "0.17.8", features = ["tokio"] } +indicatif = { version = "0.17.9", features = ["tokio"] } lazy_static = "1.5.0" num-format = { version = "0.4.4" } -owo-colors = { version = "4.0.0", features = ["supports-color"] } -quinn = { version = "0.11.5", default-features = false, features = ["runtime-tokio", "rustls", "ring"] } +owo-colors = { version = "4.1.0", features = ["supports-color"] } +quinn = { version = "0.11.6", default-features = false, features = ["runtime-tokio", "rustls", "ring"] } rcgen = { version = "0.13.1" } -rustls-pki-types = "1.9.0" -serde = { version = "1.0.215", features = ["derive"] } +rustls-pki-types = "1.10.0" +serde = { version = "1.0.216", features = ["derive"] } static_assertions = "1.1.0" struct-field-names-as-array = "0.3.0" strum_macros = "0.26.4" tabled = "0.17.0" -tokio = { version = "1.40.0", default-features = true, features = ["fs", "io-std", "macros", "process", "rt", "time", "sync"] } -tokio-util = { version = "0.7.12", features = ["compat"] } -tracing = "0.1.40" -tracing-subscriber = { version = "0.3.18", features = ["env-filter", "chrono"] } +tokio = { version = "1.42.0", default-features = true, features = ["fs", "io-std", "macros", "process", "rt", "time", "sync"] } +tokio-util = { version = "0.7.13", features = ["compat"] } +tracing = "0.1.41" +tracing-subscriber = { version = "0.3.19", features = ["env-filter", "chrono"] } [target.'cfg(unix)'.dependencies] nix = { version = "0.29.0", features = ["socket"] } @@ -61,13 +61,13 @@ nix = { version = "0.29.0", features = ["socket"] } jemallocator = "0.5.4" [build-dependencies] -capnpc = "0.20.0" +capnpc = "0.20.1" [dev-dependencies] -fastrand = "2.1.1" +fastrand = "2.3.0" json = "0.12.4" rand = "0.8.5" -serde_json = "1.0.132" +serde_json = "1.0.133" serde_test = "1.0.177" tempfile = "3.14.0" toml = "0.8.19" From c356930d80f5acc0de306595219639dc90983940 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 12 Dec 2024 20:47:15 +1300 Subject: [PATCH 21/54] chore(deps): update myriad implicit dependencies --- Cargo.lock | 98 +++++++++++++++++++++++++----------------------------- 1 file changed, 46 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 966bf83..6700111 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -177,9 +177,9 @@ checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" [[package]] name = "bytemuck" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" +checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" [[package]] name = "byteorder" @@ -189,9 +189,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "capnp" @@ -224,9 +224,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" dependencies = [ "shlex", ] @@ -245,9 +245,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -323,9 +323,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] @@ -425,12 +425,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -558,9 +558,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "hashbrown" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "heck" @@ -574,12 +574,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - [[package]] name = "home" version = "0.5.9" @@ -626,9 +620,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", "hashbrown", @@ -671,9 +665,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jemalloc-sys" @@ -728,9 +722,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "linux-raw-sys" @@ -779,11 +773,10 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ - "hermit-abi", "libc", "wasi", "windows-sys 0.52.0", @@ -908,9 +901,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "portable-atomic" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" [[package]] name = "powerfmt" @@ -1056,10 +1049,11 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" dependencies = [ + "cfg_aliases", "libc", "once_cell", "socket2", @@ -1127,7 +1121,7 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.9", "regex-syntax 0.8.5", ] @@ -1142,9 +1136,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -1186,28 +1180,28 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.18" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "once_cell", "ring", @@ -1350,9 +1344,9 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1426,9 +1420,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "supports-color" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8775305acf21c96926c900ad056abeef436701108518cf890020387236ac5a77" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" dependencies = [ "is_ci", ] @@ -1493,9 +1487,9 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f599bd7ca042cfdf8f4512b277c02ba102247820f9d9d4a9f521f496751a6ef" +checksum = "5352447f921fda68cf61b4101566c0bdb5104eff6804d0678e5227580ab6a4e9" dependencies = [ "rustix", "windows-sys 0.59.0", @@ -1720,9 +1714,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-width" From 21f8fb7ac0d42d2998dca6e722d0996310de0a17 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 12 Dec 2024 21:03:47 +1300 Subject: [PATCH 22/54] tidyup: comments around Optionalify --- src/config/structure.rs | 8 ++++++++ src/util/optionalify.rs | 1 + 2 files changed, 9 insertions(+) diff --git a/src/config/structure.rs b/src/config/structure.rs index 6e61def..5a3b6e6 100644 --- a/src/config/structure.rs +++ b/src/config/structure.rs @@ -21,6 +21,14 @@ use derive_deftly::Deftly; /// The set of configurable options supported by qcp. /// /// **Note:** The implementation of `default()` for this struct returns qcp's hard-wired configuration defaults. +/// +/// This structure uses the [Optionalify](derive_deftly_template_Optionalify) deftly macro to automatically +/// define the `Configuration_Optional` struct, which is the same but has all members of type `Option`. +/// This is the magic that lets us use the same underlying struct for CLI and saved configuration files: +/// the CLI uses the `_Optional` version , with everything defaulting to `None`. +/// The result is that wherever the user does not provide a value, values read from lower priority sources +/// (configuration files and system defaults) obtain. +/// // Maintainer note: None of the members of this struct should be Option. That leads to sunspots in the CLI and strange warts (Some(Some(foo))). #[derive(Deftly)] #[derive_deftly(Optionalify)] diff --git a/src/util/optionalify.rs b/src/util/optionalify.rs index 119e7c7..137d991 100644 --- a/src/util/optionalify.rs +++ b/src/util/optionalify.rs @@ -69,6 +69,7 @@ define_derive_deftly! { ${define OPTIONAL_TYPE ${paste $tdeftype _Optional}} /// Auto-derived struct variant + /// #[allow(non_camel_case_types)] ${tattrs} ${if not(tmeta(already_has_default)){ From 6513a0418e97b2e2c9186ecbc1f00fc0c206fc88 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 12 Dec 2024 21:30:58 +1300 Subject: [PATCH 23/54] fix: use correct format for the remote endpoint network config debug message --- src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.rs b/src/server.rs index 9aee995..e3678bd 100644 --- a/src/server.rs +++ b/src/server.rs @@ -48,7 +48,7 @@ pub async fn server_main(config: &Configuration) -> anyhow::Result<()> { client_message.connection_type, ); - let bandwidth_info = format!("{config:?}"); + let bandwidth_info = config.format_transport_config().to_string(); let file_buffer_size = usize::try_from(Configuration::send_buffer())?; let credentials = Credentials::generate()?; From 8f00f513c3043a2cce5ccc9cb8e77739518c07a9 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Fri, 13 Dec 2024 18:22:15 +1300 Subject: [PATCH 24/54] refactor: client works out remote_host once and passes it along --- src/client/control.rs | 7 ++++--- src/client/main_loop.rs | 5 ++++- src/client/options.rs | 6 ------ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/client/control.rs b/src/client/control.rs index 1b3b470..c51a5a3 100644 --- a/src/client/control.rs +++ b/src/client/control.rs @@ -37,13 +37,14 @@ impl Channel { /// Opens the control channel, checks the banner, sends the Client Message, reads the Server Message. pub async fn transact( credentials: &Credentials, + remote_host: &str, connection_type: ConnectionType, display: &MultiProgress, config: &Configuration, parameters: &Parameters, ) -> Result<(Channel, ServerMessage)> { trace!("opening control channel"); - let mut new1 = Self::launch(display, config, parameters, connection_type)?; + let mut new1 = Self::launch(display, config, parameters, remote_host, connection_type)?; new1.wait_for_banner().await?; let mut pipe = new1 @@ -79,9 +80,9 @@ impl Channel { display: &MultiProgress, config: &Configuration, parameters: &Parameters, + remote_host: &str, connection_type: ConnectionType, ) -> Result { - let remote_host = parameters.remote_host()?; let mut server = tokio::process::Command::new(&config.ssh); let _ = server.kill_on_drop(true); let _ = match connection_type { @@ -90,7 +91,7 @@ impl Channel { }; let _ = server.args(&config.ssh_opt); let _ = server.args([ - &remote_host, + remote_host, "qcp", "--server", // Remote receive bandwidth = our transmit bandwidth diff --git a/src/client/main_loop.rs b/src/client/main_loop.rs index 89624ca..7841b29 100644 --- a/src/client/main_loop.rs +++ b/src/client/main_loop.rs @@ -48,14 +48,17 @@ pub async fn client_main( // Prep -------------------------- let job_spec = crate::client::CopyJobSpec::try_from(¶meters)?; let credentials = Credentials::generate()?; + let remote_host = job_spec.remote_host(); + // If the user didn't specify the address family: we do the DNS lookup, figure it out and tell ssh to use that. // (Otherwise if we resolved a v4 and ssh a v6 - as might happen with round-robin DNS - that could be surprising.) - let remote_address = lookup_host_by_family(job_spec.remote_host(), config.address_family)?; + let remote_address = lookup_host_by_family(remote_host, config.address_family)?; // Control channel --------------- timers.next("control channel"); let (mut control, server_message) = Channel::transact( &credentials, + remote_host, remote_address.into(), &display, config, diff --git a/src/client/options.rs b/src/client/options.rs index c5e284c..4ae3b16 100644 --- a/src/client/options.rs +++ b/src/client/options.rs @@ -98,9 +98,3 @@ impl TryFrom<&Parameters> for CopyJobSpec { }) } } - -impl Parameters { - pub(crate) fn remote_host(&self) -> anyhow::Result { - Ok(CopyJobSpec::try_from(self)?.remote_host().to_string()) - } -} From 25d3a1e175e5cb247ea8ac9ec6292bc345b6e705 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Fri, 13 Dec 2024 20:54:27 +1300 Subject: [PATCH 25/54] refactor: move test tempfile helper function out to util --- src/config/manager.rs | 29 ++++++++--------------------- src/util/mod.rs | 11 +++++++++++ 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/config/manager.rs b/src/config/manager.rs index fef287b..90e2f46 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -372,14 +372,9 @@ impl Display for DisplayAdapter<'_> { #[cfg(test)] mod test { - use std::path::PathBuf; - - use serde::Deserialize; - use tempfile::TempDir; - - use crate::util::PortRange; - use crate::config::{Configuration, Configuration_Optional, Manager}; + use crate::util::{make_test_tempfile, PortRange}; + use serde::Deserialize; #[test] fn defaults() { @@ -407,18 +402,10 @@ mod test { assert_eq!(expected, result); } - fn make_tempfile(data: &str, filename: &str) -> (PathBuf, TempDir) { - let tempdir = tempfile::tempdir().unwrap(); - let path = tempdir.path().join(filename); - std::fs::write(&path, data).expect("Unable to write tempfile"); - // println!("temp file is {:?}", &path); - (path, tempdir) - } - #[test] fn dump_config_cli_and_toml() { // Not a unit test as such; this is a human test - let (path, _tempdir) = make_tempfile( + let (path, _tempdir) = make_test_tempfile( r#" tx = 42 congestion = "Bbr" @@ -440,7 +427,7 @@ mod test { #[test] fn unparseable_toml() { // This is a semi unit test; there is one assert, but the secondary goal is that it outputs something sensible - let (path, _tempdir) = make_tempfile( + let (path, _tempdir) = make_test_tempfile( r" a = 1 rx 123 # this line is a syntax error @@ -465,7 +452,7 @@ mod test { magic_: i32, } - let (path, _tempdir) = make_tempfile( + let (path, _tempdir) = make_test_tempfile( r" rx = true # invalid rtt = 3.14159 # also invalid @@ -494,7 +481,7 @@ mod test { t2: PortRange, t3: PortRange, } - let (path, _tempdir) = make_tempfile( + let (path, _tempdir) = make_test_tempfile( r#" t1 = 1234 t2 = "2345" @@ -535,7 +522,7 @@ mod test { ii: Vec, } - let (path, _tempdir) = make_tempfile( + let (path, _tempdir) = make_test_tempfile( r" ii = [1,2,3,4,6] ", @@ -554,7 +541,7 @@ mod test { _p: PortRange, } - let (path, _tempdir) = make_tempfile( + let (path, _tempdir) = make_test_tempfile( r#" _p = "234-123" "#, diff --git a/src/util/mod.rs b/src/util/mod.rs index bca7975..fce6cc0 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -26,3 +26,14 @@ pub use port_range::PortRange; mod optionalify; pub use optionalify::{derive_deftly_template_Optionalify, insert_if_some}; + +#[cfg(test)] +pub(crate) fn make_test_tempfile( + data: &str, + filename: &str, +) -> (std::path::PathBuf, tempfile::TempDir) { + let tempdir = tempfile::tempdir().unwrap(); + let path = tempdir.path().join(filename); + std::fs::write(&path, data).expect("Unable to write tempfile"); + (path, tempdir) +} From bf52d644a4f19319878b3fc2adb9556e7379a890 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Fri, 13 Dec 2024 21:34:43 +1300 Subject: [PATCH 26/54] misc: introduce a generic OS abstraction, populate with system_ssh_config --- src/os/mod.rs | 18 +++++++++++++++++- src/os/unix.rs | 10 ++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/os/mod.rs b/src/os/mod.rs index 97b93ee..34466d3 100644 --- a/src/os/mod.rs +++ b/src/os/mod.rs @@ -28,10 +28,26 @@ pub trait SocketOptions { fn force_recvbuf(&mut self, size: usize) -> Result<()>; } +/// General platform abstraction trait. +/// The active implementation should be pulled into this crate +/// Implementations should be called `Platform`, e.g. [unix::Platform]. +/// +/// Usage: +/// ``` +/// use qcp::os::Platform; +/// use qcp::os::AbstractPlatform as _; +/// println!("{}", Platform::system_ssh_config()); +/// ``` +pub trait AbstractPlatform { + /// Path to the system ssh config file. + /// On most platforms this will be `/etc/ssh/ssh_config` + fn system_ssh_config() -> &'static str; +} + #[cfg(any(unix, doc))] mod unix; #[cfg(any(unix, doc))] -pub(crate) use unix::*; +pub use unix::*; static_assertions::assert_cfg!(unix, "This OS is not yet supported"); diff --git a/src/os/unix.rs b/src/os/unix.rs index 078dc05..11bff2c 100644 --- a/src/os/unix.rs +++ b/src/os/unix.rs @@ -87,3 +87,13 @@ can create a file /etc/sysctl.d/20-qcp.conf containing: } // TODO add other OS-specific notes here } + +/// Concretions for Unix platforms +#[derive(Debug, Clone, Copy)] +pub struct Platform {} + +impl super::AbstractPlatform for Platform { + fn system_ssh_config() -> &'static str { + "/etc/ssh/ssh_config" + } +} From 14256922d50b52ed6b7eff0b25000f9977c4a278 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Fri, 13 Dec 2024 19:50:48 +1300 Subject: [PATCH 27/54] feat: look up host name aliases in ssh_config (#22) --- Cargo.lock | 93 +++++++++++++++++++++++++++++++-- Cargo.toml | 1 + src/client/job.rs | 8 ++- src/client/main_loop.rs | 8 +-- src/client/mod.rs | 1 + src/client/ssh.rs | 110 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 213 insertions(+), 8 deletions(-) create mode 100644 src/client/ssh.rs diff --git a/Cargo.lock b/Cargo.lock index 6700111..69be675 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -387,6 +387,27 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "dns-lookup" version = "2.0.4" @@ -726,6 +747,16 @@ version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags", + "libc", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -851,6 +882,12 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "overload" version = "0.1.1" @@ -997,6 +1034,7 @@ dependencies = [ "serde", "serde_json", "serde_test", + "ssh2-config", "static_assertions", "struct-field-names-as-array", "strum_macros", @@ -1022,7 +1060,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror", + "thiserror 2.0.6", "tokio", "tracing", ] @@ -1041,7 +1079,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.6", "tinyvec", "tracing", "web-time", @@ -1113,6 +1151,17 @@ dependencies = [ "yasna", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "regex" version = "1.11.1" @@ -1358,6 +1407,18 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "ssh2-config" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98150bad1e8fe53df07f38b53364f4d34e84a6cc2ee9f933e43629571060af65" +dependencies = [ + "bitflags", + "dirs", + "thiserror 1.0.69", + "wildmatch", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -1495,13 +1556,33 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.6", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] @@ -1830,6 +1911,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wildmatch" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ce1ab1f8c62655ebe1350f589c61e505cf94d385bc6a12899442d9081e71fd" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 210e259..c98af48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ quinn = { version = "0.11.6", default-features = false, features = ["runtime-tok rcgen = { version = "0.13.1" } rustls-pki-types = "1.10.0" serde = { version = "1.0.216", features = ["derive"] } +ssh2-config = "0.2.3" static_assertions = "1.1.0" struct-field-names-as-array = "0.3.0" strum_macros = "0.26.4" diff --git a/src/client/job.rs b/src/client/job.rs index ad0e7e2..4cc52d8 100644 --- a/src/client/job.rs +++ b/src/client/job.rs @@ -8,7 +8,9 @@ use crate::transport::ThroughputMode; /// A file source or destination specified by the user #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct FileSpec { - /// The remote host for the file. + /// The remote host for the file. This may be a hostname or an IP address. + /// It may also be a _hostname alias_ that matches a Host section in the user's ssh config file. + /// (In that case, the ssh config file must specify a HostName.) /// /// If not present, this is a local file. pub host: Option, @@ -68,13 +70,15 @@ impl CopyJobSpec { } } - pub(crate) fn remote_user_host(&self) -> &str { + /// The [user@]hostname portion of whichever of the arguments contained a hostname. + fn remote_user_host(&self) -> &str { self.source .host .as_ref() .unwrap_or_else(|| self.destination.host.as_ref().unwrap()) } + /// The hostname portion of whichever of the arguments contained one. pub(crate) fn remote_host(&self) -> &str { let user_host = self.remote_user_host(); // It might be user@host, or it might be just the hostname or IP. diff --git a/src/client/main_loop.rs b/src/client/main_loop.rs index 7841b29..c457163 100644 --- a/src/client/main_loop.rs +++ b/src/client/main_loop.rs @@ -48,17 +48,19 @@ pub async fn client_main( // Prep -------------------------- let job_spec = crate::client::CopyJobSpec::try_from(¶meters)?; let credentials = Credentials::generate()?; - let remote_host = job_spec.remote_host(); + let user_hostname = job_spec.remote_host(); + let remote_host = + super::ssh::resolve_host_alias(user_hostname).unwrap_or_else(|| user_hostname.into()); // If the user didn't specify the address family: we do the DNS lookup, figure it out and tell ssh to use that. // (Otherwise if we resolved a v4 and ssh a v6 - as might happen with round-robin DNS - that could be surprising.) - let remote_address = lookup_host_by_family(remote_host, config.address_family)?; + let remote_address = lookup_host_by_family(&remote_host, config.address_family)?; // Control channel --------------- timers.next("control channel"); let (mut control, server_message) = Channel::transact( &credentials, - remote_host, + &remote_host, remote_address.into(), &display, config, diff --git a/src/client/mod.rs b/src/client/mod.rs index 21978a4..9faf56c 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -13,6 +13,7 @@ pub use job::FileSpec; mod main_loop; mod meter; mod progress; +pub mod ssh; #[allow(clippy::module_name_repetitions)] pub use main_loop::client_main; diff --git a/src/client/ssh.rs b/src/client/ssh.rs new file mode 100644 index 0000000..1304469 --- /dev/null +++ b/src/client/ssh.rs @@ -0,0 +1,110 @@ +//! Interaction with ssh configuration +// (c) 2024 Ross Younger + +use std::{fs::File, io::BufReader}; + +use ssh2_config::{ParseRule, SshConfig}; +use tracing::{debug, warn}; + +use crate::os::{AbstractPlatform as _, Platform}; + +/// Attempts to resolve a hostname from a single OpenSSH-style config file +/// +/// If `path` is None, uses the default user ssh config file. +fn resolve_one(path: Option<&str>, host: &str) -> Option { + let source = path.unwrap_or("~/.ssh/config"); + let result = match path { + Some(p) => { + let mut reader = match File::open(p) { + Ok(f) => BufReader::new(f), + Err(e) => { + // This is not automatically an error, as the file might not exist. + debug!("Unable to read {p}; continuing without. {e}"); + return None; + } + }; + SshConfig::default().parse(&mut reader, ParseRule::ALLOW_UNKNOWN_FIELDS) + } + None => SshConfig::parse_default_file(ParseRule::ALLOW_UNKNOWN_FIELDS), + }; + let cfg = match result { + Ok(cfg) => cfg, + Err(e) => { + warn!("Unable to parse {source}; continuing without. [{e}]"); + return None; + } + }; + + cfg.query(host).host_name.inspect(|h| { + debug!("Using hostname '{h}' for '{host}' (from {source})"); + }) +} + +/// Attempts to resolve hostname aliasing from the user's and system's ssh config files to resolve aliasing. +/// +/// ## Returns +/// Some(hostname) if any config file matched. +/// None if no config files matched. +/// +/// ## ssh_config features not currently supported +/// * Include directives +/// * Match patterns +/// * CanonicalizeHostname and friends +#[must_use] +pub fn resolve_host_alias(host: &str) -> Option { + let files = vec![None, Some(Platform::system_ssh_config())]; + files.into_iter().find_map(|it| resolve_one(it, host)) +} + +#[cfg(test)] +mod test { + use super::resolve_one; + use crate::util::make_test_tempfile; + + #[test] + fn hosts_resolve() { + let (path, _dir) = make_test_tempfile( + r" + Host aaa + HostName zzz + Host bbb ccc.ddd + HostName yyy + ", + "test_ssh_config", + ); + let f = path.to_string_lossy().to_string(); + assert!(resolve_one(Some(&f), "nope").is_none()); + assert_eq!(resolve_one(Some(&f), "aaa").unwrap(), "zzz"); + assert_eq!(resolve_one(Some(&f), "bbb").unwrap(), "yyy"); + assert_eq!(resolve_one(Some(&f), "ccc.ddd").unwrap(), "yyy"); + } + + #[test] + fn wildcards_match() { + let (path, _dir) = make_test_tempfile( + r" + Host *.bar + HostName baz + Host 10.11.*.13 + # this is a silly example but it shows that wildcards match by IP + HostName wibble + Host fr?d + hostname barney + ", + "test_ssh_config", + ); + let f = path.to_string_lossy().to_string(); + assert_eq!(resolve_one(Some(&f), "foo.bar").unwrap(), "baz"); + assert_eq!(resolve_one(Some(&f), "qux.qix.bar").unwrap(), "baz"); + assert!(resolve_one(Some(&f), "qux.qix").is_none()); + assert_eq!(resolve_one(Some(&f), "10.11.12.13").unwrap(), "wibble"); + assert_eq!(resolve_one(Some(&f), "10.11.0.13").unwrap(), "wibble"); + assert_eq!(resolve_one(Some(&f), "10.11.256.13").unwrap(), "wibble"); // yes I know this isn't a real IP address + assert!(resolve_one(Some(&f), "10.11.0.130").is_none()); + + assert_eq!(resolve_one(Some(&f), "fred").unwrap(), "barney"); + assert_eq!(resolve_one(Some(&f), "frid").unwrap(), "barney"); + assert!(resolve_one(Some(&f), "freed").is_none()); + assert!(resolve_one(Some(&f), "fredd").is_none()); + } +} From c7b20d17b97a48d209265b5b1c2ccd9df7392055 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Fri, 13 Dec 2024 22:23:00 +1300 Subject: [PATCH 28/54] refactor: move config file paths into Platform abstraction use dirs instead of etcetera to remove a crate dependency --- Cargo.lock | 22 +------------- Cargo.toml | 2 +- src/config/manager.rs | 67 ++++++++++--------------------------------- src/config/mod.rs | 2 ++ src/os/mod.rs | 17 +++++++++++ src/os/unix.rs | 23 ++++++++++++++- 6 files changed, 58 insertions(+), 75 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69be675..948553c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -454,17 +454,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "etcetera" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" -dependencies = [ - "cfg-if", - "home", - "windows-sys 0.48.0", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -595,15 +584,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "home" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" -dependencies = [ - "windows-sys 0.52.0", -] - [[package]] name = "human-repr" version = "1.1.0" @@ -1011,8 +991,8 @@ dependencies = [ "clap", "console", "derive-deftly", + "dirs", "dns-lookup", - "etcetera", "fastrand", "figment", "futures-util", diff --git a/Cargo.toml b/Cargo.toml index c98af48..6ca4375 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,8 +29,8 @@ capnp-futures = "0.20.1" clap = { version = "4.5.23", features = ["wrap_help", "derive", "cargo", "help", "string"] } console = "0.15.8" derive-deftly = "0.14.2" +dirs = "5.0.1" dns-lookup = "2.0.4" -etcetera = "0.8.0" figment = { version = "0.10.19", features = ["toml"] } futures-util = { version = "0.3.31", default-features = false } gethostname = "0.5.0" diff --git a/src/config/manager.rs b/src/config/manager.rs index 90e2f46..a68bdda 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -1,9 +1,10 @@ //! Configuration file wrangling // (c) 2024 Ross Younger +use crate::os::{AbstractPlatform as _, Platform}; + use super::Configuration; -use anyhow::Result; use figment::{ providers::{Format, Serialized, Toml}, value::Value, @@ -13,53 +14,13 @@ use serde::Deserialize; use std::{ collections::HashSet, fmt::{Debug, Display}, - path::{Path, PathBuf}, + path::Path, }; use struct_field_names_as_array::FieldNamesAsSlice; use tabled::{settings::style::Style, Table, Tabled}; use tracing::{trace, warn}; -// PATHS ///////////////////////////////////////////////////////////////////////////////////////////////////// - -const BASE_CONFIG_FILENAME: &str = "qcp.toml"; - -#[cfg(unix)] -fn user_config_dir() -> Result { - // home directory for now - use etcetera::BaseStrategy as _; - Ok(etcetera::choose_base_strategy()?.home_dir().into()) -} - -#[cfg(windows)] -fn user_config_dir() -> Result { - use etcetera::{choose_app_strategy, AppStrategy as _, AppStrategyArgs}; - - Ok(choose_app_strategy(AppStrategyArgs { - top_level_domain: "com".to_string(), - author: "TeamQCP".to_string(), - app_name: env!("CARGO_PKG_NAME").to_string(), - })? - .config_dir()) -} - -#[cfg(unix)] -fn user_config_path() -> Result { - // ~/. for now - let mut d: PathBuf = user_config_dir()?; - d.push(format!(".{BASE_CONFIG_FILENAME}")); - Ok(d) -} - -#[cfg(unix)] -fn system_config_path() -> PathBuf { - // /etc/ for now - let mut p: PathBuf = PathBuf::new(); - p.push("/etc"); - p.push(BASE_CONFIG_FILENAME); - p -} - // SYSTEM DEFAULTS ////////////////////////////////////////////////////////////////////////////////////////////// /// A `[https://docs.rs/figment/latest/figment/trait.Provider.html](figment::Provider)` that holds @@ -99,13 +60,11 @@ pub struct Manager { } fn add_user_config(f: Figment) -> Figment { - let path = match user_config_path() { - Ok(p) => p, - Err(e) => { - warn!("could not determine user configuration file path: {e}"); - return f; - } + let Some(path) = Platform::user_config_path() else { + warn!("could not determine user configuration file path"); + return f; }; + if !path.exists() { trace!("user configuration file {path:?} not present"); return f; @@ -114,7 +73,11 @@ fn add_user_config(f: Figment) -> Figment { } fn add_system_config(f: Figment) -> Figment { - let path = system_config_path(); + let Some(path) = Platform::system_config_path() else { + warn!("could not determine system configuration file path"); + return f; + }; + if !path.exists() { trace!("system configuration file {path:?} not present"); return f; @@ -150,13 +113,13 @@ impl Manager { /// Returns the list of configuration files we read. /// /// This is a function of platform and the current user id. + #[must_use] pub fn config_files() -> Vec { - let inputs = vec![Ok(system_config_path()), user_config_path()]; + let inputs = vec![Platform::system_config_path(), Platform::user_config_path()]; inputs .into_iter() - .filter_map(std::result::Result::ok) - .map(|p| p.into_os_string().to_string_lossy().into()) + .filter_map(|p| Some(p?.into_os_string().to_string_lossy().to_string())) .collect() } diff --git a/src/config/mod.rs b/src/config/mod.rs index 655a583..affeefe 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -36,3 +36,5 @@ pub(crate) use structure::Configuration_Optional; mod manager; pub use manager::Manager; + +pub(crate) const BASE_CONFIG_FILENAME: &str = "qcp.toml"; diff --git a/src/os/mod.rs b/src/os/mod.rs index 34466d3..b5dc1a3 100644 --- a/src/os/mod.rs +++ b/src/os/mod.rs @@ -1,6 +1,8 @@ //! OS abstraction layer // (c) 2024 Ross Younger +use std::path::PathBuf; + use anyhow::Result; /// OS abstraction trait providing access to socket options @@ -42,6 +44,21 @@ pub trait AbstractPlatform { /// Path to the system ssh config file. /// On most platforms this will be `/etc/ssh/ssh_config` fn system_ssh_config() -> &'static str; + + /// The directory to store user configuration files in. + /// + /// On Unix platforms this is the traditional home directory. + /// + /// If somehow we could not determine the directory to use, returns None (and may emit a warning). + fn user_config_dir() -> Option; + + /// The absolute path to the user configuration file, if one is defined on this platform. + /// + /// If somehow we could not determine the path to use, returns None (and may emit a warning). + fn user_config_path() -> Option; + + /// The absolute path to the system configuration file, if one is defined on this platform. + fn system_config_path() -> Option; } #[cfg(any(unix, doc))] diff --git a/src/os/unix.rs b/src/os/unix.rs index 11bff2c..70a6705 100644 --- a/src/os/unix.rs +++ b/src/os/unix.rs @@ -1,10 +1,12 @@ // OS abstraction layer for qcp - Unix implementation // (c) 2024 Ross Younger +use crate::config::BASE_CONFIG_FILENAME; + use super::SocketOptions; use anyhow::Result; use nix::sys::socket::{self, sockopt}; -use std::net::UdpSocket; +use std::{net::UdpSocket, path::PathBuf}; fn bsdish() -> bool { cfg!(any( @@ -96,4 +98,23 @@ impl super::AbstractPlatform for Platform { fn system_ssh_config() -> &'static str { "/etc/ssh/ssh_config" } + + fn user_config_dir() -> Option { + dirs::home_dir() + } + + fn user_config_path() -> Option { + // ~/. for now + let mut d: PathBuf = Self::user_config_dir()?; + d.push(format!(".{BASE_CONFIG_FILENAME}")); + Some(d) + } + + fn system_config_path() -> Option { + // /etc/ for now + let mut p: PathBuf = PathBuf::new(); + p.push("/etc"); + p.push(BASE_CONFIG_FILENAME); + Some(p) + } } From 7b5398f4bd1271690e779b55c669f1b88ecd8e8f Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Fri, 13 Dec 2024 22:27:23 +1300 Subject: [PATCH 29/54] chore(release): rearrange changelog sections --- release-plz.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-plz.toml b/release-plz.toml index b769a40..b69e277 100644 --- a/release-plz.toml +++ b/release-plz.toml @@ -58,13 +58,13 @@ body = """ """ commit_parsers = [ - { body = ".*security", group = "πŸ›‘οΈ Security" }, - { message = "^feat", group = "⛰️ Features" }, - { message = "^fix", group = "πŸ› Bug Fixes" }, + { body = ".*security", group = "πŸ›‘οΈ Security" }, + { message = "^feat", group = "⛰️ Features" }, + { message = "^fix", group = "πŸ› Bug Fixes" }, { message = "^doc", group = "πŸ“š Documentation" }, { message = "^perf", group = "⚑ Performance" }, { message = "^refactor\\(clippy\\)", skip = true }, - { message = "^refactor", group = "🚜 Refactor" }, + { message = "^refactor", group = "🚜 Refactor" }, { message = "^style", group = "🎨 Styling" }, { message = "^test", group = "πŸ§ͺ Testing" }, { message = "^chore\\(release\\):", skip = true }, From 0fe943b4b7b6a34e12afbe05d235f0f17d00fbe6 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sat, 14 Dec 2024 14:58:44 +1300 Subject: [PATCH 30/54] test: unignore doc test on Optionalify --- src/util/optionalify.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/util/optionalify.rs b/src/util/optionalify.rs index 137d991..f5c277d 100644 --- a/src/util/optionalify.rs +++ b/src/util/optionalify.rs @@ -53,7 +53,9 @@ define_derive_deftly! { /// must appear after deriving `Optionalify`. It might look something like this: /// /// - /// ```ignore + /// ``` + /// use derive_deftly::Deftly; + /// use qcp::derive_deftly_template_Optionalify; /// #[derive(Deftly)] /// #[derive_deftly(Optionalify)] /// #[derive(Debug, Clone /*, WhateverElseYouNeed...*/)] From 384a87005494909611dd6abb65daef2cf0067f9c Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Tue, 17 Dec 2024 22:58:59 +1300 Subject: [PATCH 31/54] misc: add AbstractPlatform::user_ssh_config --- src/os/mod.rs | 9 +++++++++ src/os/unix.rs | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/src/os/mod.rs b/src/os/mod.rs index b5dc1a3..9e3f1a7 100644 --- a/src/os/mod.rs +++ b/src/os/mod.rs @@ -45,6 +45,15 @@ pub trait AbstractPlatform { /// On most platforms this will be `/etc/ssh/ssh_config` fn system_ssh_config() -> &'static str; + /// Path to the user ssh config file. + /// On most platforms this will be `${HOME}/.ssh/config` + /// # Note + /// This is a _theoretical_ path construction; it does not guarantee that the path actually exists. + /// That is up to the caller to determine and reason about. + /// # Errors + /// If the current user's home directory could not be determined + fn user_ssh_config() -> Result; + /// The directory to store user configuration files in. /// /// On Unix platforms this is the traditional home directory. diff --git a/src/os/unix.rs b/src/os/unix.rs index 70a6705..481d8a3 100644 --- a/src/os/unix.rs +++ b/src/os/unix.rs @@ -99,6 +99,15 @@ impl super::AbstractPlatform for Platform { "/etc/ssh/ssh_config" } + fn user_ssh_config() -> Result { + let Some(mut pb) = dirs::home_dir() else { + anyhow::bail!("could not determine home directory"); + }; + pb.push(".ssh"); + pb.push("config"); + Ok(pb) + } + fn user_config_dir() -> Option { dirs::home_dir() } From 8f397d92fa32b2e2e15c55f039e683d17198fb55 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sun, 15 Dec 2024 17:19:23 +1300 Subject: [PATCH 32/54] misc: implement our own ssh config file parser Use that instead of ssh2_config. This adds support for include directives. --- Cargo.lock | 168 +++++++++++-- Cargo.toml | 5 +- src/client/ssh.rs | 93 +++---- src/config/mod.rs | 2 + src/config/ssh.rs | 606 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 805 insertions(+), 69 deletions(-) create mode 100644 src/config/ssh.rs diff --git a/Cargo.lock b/Cargo.lock index 948553c..aa364f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -106,12 +106,30 @@ version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + [[package]] name = "arrayvec" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "assertables" +version = "9.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0082388b8564898f945b04215e800e800a164af15307d8dfe714b02cc69356e9" + [[package]] name = "atomic" version = "0.6.0" @@ -142,6 +160,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -154,6 +178,17 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "blake2b_simd" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" +dependencies = [ + "arrayref", + "arrayvec 0.5.2", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -315,6 +350,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -330,6 +371,12 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crypto-common" version = "0.1.6" @@ -387,6 +434,17 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +dependencies = [ + "libc", + "redox_users 0.3.5", + "winapi", +] + [[package]] name = "dirs" version = "5.0.1" @@ -404,7 +462,7 @@ checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" dependencies = [ "libc", "option-ext", - "redox_users", + "redox_users 0.4.6", "windows-sys 0.48.0", ] @@ -454,6 +512,17 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "expanduser" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e0b79235da57db6b6c2beed9af6e5de867d63a973ae3e91910ddc33ba40bc0" +dependencies = [ + "dirs 1.0.5", + "lazy_static", + "pwd", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -547,6 +616,17 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -556,7 +636,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -566,6 +646,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "hashbrown" version = "0.15.2" @@ -789,7 +875,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -828,7 +914,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" dependencies = [ - "arrayvec", + "arrayvec 0.7.6", "itoa", ] @@ -900,7 +986,7 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" dependencies = [ - "base64", + "base64 0.22.1", "serde", ] @@ -977,6 +1063,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pwd" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c71c0c79b9701efe4e1e4b563b2016dd4ee789eb99badcb09d61ac4b92e4a2" +dependencies = [ + "libc", + "thiserror 1.0.69", +] + [[package]] name = "qcp" version = "0.1.3" @@ -985,18 +1081,21 @@ dependencies = [ "anstyle", "anstyle-owo-colors", "anyhow", + "assertables", "capnp", "capnp-futures", "capnpc", "clap", "console", "derive-deftly", - "dirs", + "dirs 5.0.1", "dns-lookup", + "expanduser", "fastrand", "figment", "futures-util", "gethostname", + "glob", "heck 0.5.0", "human-repr", "humanize-rs", @@ -1014,7 +1113,6 @@ dependencies = [ "serde", "serde_json", "serde_test", - "ssh2-config", "static_assertions", "struct-field-names-as-array", "strum_macros", @@ -1025,6 +1123,7 @@ dependencies = [ "toml", "tracing", "tracing-subscriber", + "wildmatch", ] [[package]] @@ -1052,7 +1151,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", - "getrandom", + "getrandom 0.2.15", "rand", "ring", "rustc-hash", @@ -1115,7 +1214,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] @@ -1131,13 +1230,30 @@ dependencies = [ "yasna", ] +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom 0.1.16", + "redox_syscall", + "rust-argon2", +] + [[package]] name = "redox_users" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", "thiserror 1.0.69", ] @@ -1194,13 +1310,25 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "spin", "untrusted", "windows-sys 0.52.0", ] +[[package]] +name = "rust-argon2" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" +dependencies = [ + "base64 0.13.1", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -1387,18 +1515,6 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -[[package]] -name = "ssh2-config" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98150bad1e8fe53df07f38b53364f4d34e84a6cc2ee9f933e43629571060af65" -dependencies = [ - "bitflags", - "dirs", - "thiserror 1.0.69", - "wildmatch", -] - [[package]] name = "static_assertions" version = "1.1.0" @@ -1821,6 +1937,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 6ca4375..59b2fe1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,9 +31,11 @@ console = "0.15.8" derive-deftly = "0.14.2" dirs = "5.0.1" dns-lookup = "2.0.4" +expanduser = "1.2.2" figment = { version = "0.10.19", features = ["toml"] } futures-util = { version = "0.3.31", default-features = false } gethostname = "0.5.0" +glob = "0.3.1" heck = "0.5.0" human-repr = "1.1.0" humanize-rs = "0.1.5" @@ -45,7 +47,6 @@ quinn = { version = "0.11.6", default-features = false, features = ["runtime-tok rcgen = { version = "0.13.1" } rustls-pki-types = "1.10.0" serde = { version = "1.0.216", features = ["derive"] } -ssh2-config = "0.2.3" static_assertions = "1.1.0" struct-field-names-as-array = "0.3.0" strum_macros = "0.26.4" @@ -54,6 +55,7 @@ tokio = { version = "1.42.0", default-features = true, features = ["fs", "io-std tokio-util = { version = "0.7.13", features = ["compat"] } tracing = "0.1.41" tracing-subscriber = { version = "0.3.19", features = ["env-filter", "chrono"] } +wildmatch = "2.4.0" [target.'cfg(unix)'.dependencies] nix = { version = "0.29.0", features = ["socket"] } @@ -65,6 +67,7 @@ jemallocator = "0.5.4" capnpc = "0.20.1" [dev-dependencies] +assertables = "9.5.0" fastrand = "2.3.0" json = "0.12.4" rand = "0.8.5" diff --git a/src/client/ssh.rs b/src/client/ssh.rs index 1304469..48caf8a 100644 --- a/src/client/ssh.rs +++ b/src/client/ssh.rs @@ -1,43 +1,45 @@ //! Interaction with ssh configuration // (c) 2024 Ross Younger -use std::{fs::File, io::BufReader}; +use std::path::PathBuf; -use ssh2_config::{ParseRule, SshConfig}; +use crate::config::ssh::Parser; +use anyhow::Context; use tracing::{debug, warn}; use crate::os::{AbstractPlatform as _, Platform}; /// Attempts to resolve a hostname from a single OpenSSH-style config file -/// -/// If `path` is None, uses the default user ssh config file. -fn resolve_one(path: Option<&str>, host: &str) -> Option { - let source = path.unwrap_or("~/.ssh/config"); - let result = match path { - Some(p) => { - let mut reader = match File::open(p) { - Ok(f) => BufReader::new(f), - Err(e) => { - // This is not automatically an error, as the file might not exist. - debug!("Unable to read {p}; continuing without. {e}"); - return None; - } - }; - SshConfig::default().parse(&mut reader, ParseRule::ALLOW_UNKNOWN_FIELDS) +fn resolve_one(path: &PathBuf, user_config_file: bool, host: &str) -> Option { + if !std::fs::exists(path).is_ok_and(|b| b) { + // file could not be verified to exist. this is not intrinsically an error; keep quiet + return None; + } + let mut parser = match Parser::for_path(path, user_config_file) { + Ok(p) => p, + Err(e) => { + // file permissions issue? + warn!("failed to open {path:?}: {e}"); + return None; } - None => SshConfig::parse_default_file(ParseRule::ALLOW_UNKNOWN_FIELDS), }; - let cfg = match result { - Ok(cfg) => cfg, + let data = match parser + .parse_file_for(host) + .with_context(|| format!("reading configuration file {path:?}")) + { + Ok(data) => data, Err(e) => { - warn!("Unable to parse {source}; continuing without. [{e}]"); + warn!("{e}"); return None; } }; - - cfg.query(host).host_name.inspect(|h| { - debug!("Using hostname '{h}' for '{host}' (from {source})"); - }) + if let Some(s) = data.get("hostname") { + let result = s.first_arg(); + debug!("Using hostname '{result}' for '{host}' (from {})", s.source); + Some(result) + } else { + None + } } /// Attempts to resolve hostname aliasing from the user's and system's ssh config files to resolve aliasing. @@ -47,13 +49,16 @@ fn resolve_one(path: Option<&str>, host: &str) -> Option { /// None if no config files matched. /// /// ## ssh_config features not currently supported -/// * Include directives /// * Match patterns /// * CanonicalizeHostname and friends #[must_use] pub fn resolve_host_alias(host: &str) -> Option { - let files = vec![None, Some(Platform::system_ssh_config())]; - files.into_iter().find_map(|it| resolve_one(it, host)) + let f = Platform::user_ssh_config().map(|pb| resolve_one(&pb, true, host)); + if let Ok(Some(s)) = f { + return Some(s); + } + + resolve_one(&PathBuf::from(Platform::system_ssh_config()), false, host) } #[cfg(test)] @@ -72,11 +77,10 @@ mod test { ", "test_ssh_config", ); - let f = path.to_string_lossy().to_string(); - assert!(resolve_one(Some(&f), "nope").is_none()); - assert_eq!(resolve_one(Some(&f), "aaa").unwrap(), "zzz"); - assert_eq!(resolve_one(Some(&f), "bbb").unwrap(), "yyy"); - assert_eq!(resolve_one(Some(&f), "ccc.ddd").unwrap(), "yyy"); + assert!(resolve_one(&path, false, "nope").is_none()); + assert_eq!(resolve_one(&path, false, "aaa").unwrap(), "zzz"); + assert_eq!(resolve_one(&path, false, "bbb").unwrap(), "yyy"); + assert_eq!(resolve_one(&path, false, "ccc.ddd").unwrap(), "yyy"); } #[test] @@ -93,18 +97,17 @@ mod test { ", "test_ssh_config", ); - let f = path.to_string_lossy().to_string(); - assert_eq!(resolve_one(Some(&f), "foo.bar").unwrap(), "baz"); - assert_eq!(resolve_one(Some(&f), "qux.qix.bar").unwrap(), "baz"); - assert!(resolve_one(Some(&f), "qux.qix").is_none()); - assert_eq!(resolve_one(Some(&f), "10.11.12.13").unwrap(), "wibble"); - assert_eq!(resolve_one(Some(&f), "10.11.0.13").unwrap(), "wibble"); - assert_eq!(resolve_one(Some(&f), "10.11.256.13").unwrap(), "wibble"); // yes I know this isn't a real IP address - assert!(resolve_one(Some(&f), "10.11.0.130").is_none()); + assert_eq!(resolve_one(&path, false, "foo.bar").unwrap(), "baz"); + assert_eq!(resolve_one(&path, false, "qux.qix.bar").unwrap(), "baz"); + assert!(resolve_one(&path, false, "qux.qix").is_none()); + assert_eq!(resolve_one(&path, false, "10.11.12.13").unwrap(), "wibble"); + assert_eq!(resolve_one(&path, false, "10.11.0.13").unwrap(), "wibble"); + assert_eq!(resolve_one(&path, false, "10.11.256.13").unwrap(), "wibble"); // yes I know this isn't a real IP address + assert!(resolve_one(&path, false, "10.11.0.130").is_none()); - assert_eq!(resolve_one(Some(&f), "fred").unwrap(), "barney"); - assert_eq!(resolve_one(Some(&f), "frid").unwrap(), "barney"); - assert!(resolve_one(Some(&f), "freed").is_none()); - assert!(resolve_one(Some(&f), "fredd").is_none()); + assert_eq!(resolve_one(&path, false, "fred").unwrap(), "barney"); + assert_eq!(resolve_one(&path, false, "frid").unwrap(), "barney"); + assert!(resolve_one(&path, false, "freed").is_none()); + assert!(resolve_one(&path, false, "fredd").is_none()); } } diff --git a/src/config/mod.rs b/src/config/mod.rs index affeefe..e10f425 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -38,3 +38,5 @@ mod manager; pub use manager::Manager; pub(crate) const BASE_CONFIG_FILENAME: &str = "qcp.toml"; + +pub(crate) mod ssh; diff --git a/src/config/ssh.rs b/src/config/ssh.rs new file mode 100644 index 0000000..b8a0ee8 --- /dev/null +++ b/src/config/ssh.rs @@ -0,0 +1,606 @@ +//! Config file parsing, openssh-style +// (c) 2024 Ross Younger + +use std::{ + collections::BTreeMap, + fs::File, + io::{BufRead, BufReader, Read}, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result}; +use glob::{glob_with, MatchOptions}; +use tracing::warn; + +#[derive(Debug, Clone, PartialEq)] +/// A parsed line we read from an ssh config file +enum Line { + Empty, + Host { + line_number: usize, + args: Vec, + }, + Match { + line_number: usize, + args: Vec, + }, + Include { + line_number: usize, + args: Vec, + }, + Generic { + line_number: usize, + keyword: String, /*lowercase!*/ + args: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq, Default)] +/// A setting we read from a config file +pub(crate) struct Setting { + /// where the value came from + pub source: String, + /// line number within the value + pub line_number: usize, + /// the setting data itself (not parsed; we assert nothing beyond the parser has applied the ssh quoting logic) + pub args: Vec, +} + +impl Setting { + pub(crate) fn first_arg(&self) -> String { + self.args.first().cloned().unwrap_or_else(String::new) + } +} + +/// Splits a string into a list of arguments. +/// Arguments are delimited by whitespace, subject to quoting (single or double quotes), and simple escapes (\\, \", \'). +fn split_args(input: &str) -> Result> { + // We need to index over the characters of the input, but also need to be able to peek at the next token in case of escapes. + let mut i = 0; + let input: Vec = input.chars().collect(); + let mut output = Vec::::new(); + while i < input.len() { + // Strip any leading whitespace + if input[i] == ' ' || input[i] == '\t' { + i += 1; + continue; + } + if input[i] == '#' { + break; // it's a comment, we're done + } + + // We're at the start of a real token + let mut current_arg = String::new(); + let mut quote_state: char = '\0'; + + while i < input.len() { + let ch = input[i]; + match (ch, quote_state) { + ('\\', _) => { + // It might be an escape + let next = input.get(i + 1); + match next { + Some(nn @ ('\'' | '\"' | '\\')) => { + // It is an escape + current_arg.push(*nn); + i += 1; + } + Some(_) | None => current_arg.push(ch), // Ignore unrecognised escape + } + } + (' ' | '\t', '\0') => break, // end of token + (q @ ('\'' | '\"'), '\0') => quote_state = q, // start of quote + (q1, q2) if q1 == q2 => quote_state = '\0', // end of quote + (c, _) => current_arg.push(c), // nothing special + } + i += 1; + } + + // end of token + anyhow::ensure!(quote_state == '\0', "unterminated quote"); + output.push(current_arg); + i += 1; + } + Ok(output) +} + +fn evaluate_host_match(host: &str, args: &Vec) -> bool { + for arg in args { + if wildmatch::WildMatch::new(arg).matches(host) { + return true; + } + } + false +} + +/// Wildcard matching and ~ expansion for Include directives +fn find_include_files(arg: &str, is_user: bool) -> Result> { + let mut path = if arg.starts_with('~') { + anyhow::ensure!( + is_user, + "include paths may not start with ~ in a system configuration file" + ); + expanduser::expanduser(arg) + .with_context(|| format!("expanding include expression {arg}"))? + } else { + PathBuf::from(arg) + }; + if !path.is_absolute() { + if is_user { + let Some(home) = dirs::home_dir() else { + anyhow::bail!("could not determine home directory"); + }; + let mut buf = home; + buf.push(".ssh"); + buf.push(path); + path = buf; + } else { + let mut buf = PathBuf::from("/etc/ssh/"); + buf.push(path); + path = buf; + } + } + + let mut result = Vec::new(); + let options = MatchOptions { + case_sensitive: true, + require_literal_leading_dot: true, + require_literal_separator: true, + }; + for entry in (glob_with(path.to_string_lossy().as_ref(), options)?).flatten() { + if let Some(s) = entry.to_str() { + result.push(s.into()); + } + } + Ok(result) +} + +pub(crate) struct Parser +where + R: Read, +{ + line_number: usize, + reader: BufReader, + source: String, + is_user: bool, +} + +impl Parser { + pub(crate) fn for_path

(path: P, is_user: bool) -> Result + where + P: AsRef, + { + let path = path.as_ref(); + let file = File::open(path)?; + let reader = BufReader::new(file); + Ok(Self::for_reader( + reader, + path.to_string_lossy().to_string(), + is_user, + )) + } +} + +impl Parser { + fn for_reader(reader: BufReader, source: String, is_user: bool) -> Self { + Self { + line_number: 0, + reader, + source, + is_user, + } + } +} + +impl<'a> Parser<&'a [u8]> { + fn for_str(s: &'a str, is_user: bool) -> Self { + Self::for_reader(BufReader::new(s.as_bytes()), "".into(), is_user) + } +} + +impl Default for Parser<&[u8]> { + fn default() -> Self { + Parser::for_str("", false) + } +} + +impl Parser { + fn parse_line(&self, line: &str) -> Result { + let line = line.trim(); + let line_number = self.line_number; + // extract keyword, which may be delimited by whitespace (Key Value) OR equals (Key=Value) + let (keyword, rest) = { + let mut splitter = line.splitn(2, &[' ', '\t', '=']); + let keyword = match splitter.next() { + None | Some("") => return Ok(Line::Empty), + Some(kw) => kw.to_lowercase(), + }; + (keyword, splitter.next().unwrap_or_default()) + }; + if keyword.starts_with('#') { + return Ok(Line::Empty); + } + let args = split_args(rest).with_context(|| format!("at line {line_number}"))?; + anyhow::ensure!(!args.is_empty(), "missing argument at line {line_number}"); + + Ok(match keyword.as_str() { + "host" => Line::Host { line_number, args }, + "match" => Line::Match { line_number, args }, + "include" => Line::Include { line_number, args }, + _ => Line::Generic { + line_number, + keyword, + args, + }, + }) + } + + const INCLUDE_DEPTH_LIMIT: u8 = 16; + + fn parse_file_inner( + &mut self, + host: &str, + accepting: &mut bool, + depth: u8, + output: &mut BTreeMap, + ) -> Result<()> { + let mut line = String::new(); + anyhow::ensure!( + depth < Self::INCLUDE_DEPTH_LIMIT, + "too many nested includes" + ); + + loop { + line.clear(); + self.line_number += 1; + let mut line = String::new(); + if 0 == self.reader.read_line(&mut line)? { + break; // EOF + } + match self.parse_line(&line)? { + Line::Empty => (), + Line::Host { args, .. } => { + *accepting = evaluate_host_match(host, &args); + } + Line::Match { .. } => { + warn!("match expressions in ssh_config files are not yet supported"); + } + Line::Include { args, .. } => { + for arg in args { + let files = find_include_files(&arg, self.is_user)?; + for f in files { + let mut subparser = + Parser::for_path(f, self.is_user).with_context(|| { + format!( + "Include directive at {} line {}", + self.source, self.line_number + ) + })?; + subparser.parse_file_inner(host, accepting, depth + 1, output)?; + } + } + } + Line::Generic { keyword, args, .. } => { + if *accepting { + // per ssh_config(5), the first matching entry for a given key wins. + let _ = output.entry(keyword).or_insert_with(|| Setting { + source: self.source.clone(), + line_number: self.line_number, + args, + }); + } + } + } + } + Ok(()) + } + + pub(crate) fn parse_file_for(&mut self, host: &str) -> Result> { + let mut output = BTreeMap::::new(); + let mut accepting = true; + self.parse_file_inner(host, &mut accepting, 0, &mut output)?; + Ok(output) + } +} + +#[cfg(test)] +mod test { + use anyhow::{anyhow, Context, Result}; + use assertables::{assert_contains, assert_contains_as_result, assert_eq_as_result}; + + use crate::{ + os::{AbstractPlatform, Platform}, + util::make_test_tempfile, + }; + + use super::{evaluate_host_match, find_include_files, split_args, Line, Parser}; + #[test] + fn arg_splitting() -> Result<()> { + for (input, expected) in [ + ("", vec![]), + ("a", vec!["a"]), + (" a b ", vec!["a", "b"]), + (" a b # c d", vec!["a", "b"]), + (r#"a\ \' \"b"#, vec!["a\\", "'", "\"b"]), + (r#""a b" 'c d'"#, vec!["a b", "c d"]), + (r#""a \"b" '\'c d'"#, vec!["a \"b", "'c d"]), + ] { + let msg = || format!("input \"{input}\" failed"); + assert_eq_as_result!(split_args(input).with_context(msg)?, expected) + .map_err(|e| anyhow!(e)) + .with_context(msg)?; + } + for (input, expected_msg) in [ + ("aaa\"bbb", "unterminated quote"), + ("'", "unterminated quote"), + ] { + let err = split_args(input).unwrap_err(); + assert_contains_as_result!(err.to_string(), expected_msg) + .map_err(|e| anyhow!(e)) + .with_context(|| format!("input \"{input}\" failed"))?; + } + Ok(()) + } + + macro_rules! make_vec { + ($v:expr) => { + $v.into_iter().map(|s| s.into()).collect() + }; + } + + fn host_(args: Vec<&str>) -> Line { + Line::Host { + line_number: 0, + args: make_vec!(args), + } + } + fn match_(args: Vec<&str>) -> Line { + Line::Match { + line_number: 0, + args: make_vec!(args), + } + } + fn include_(args: Vec<&str>) -> Line { + Line::Include { + line_number: 0, + args: make_vec!(args), + } + } + fn generic_(kw: &str, args: Vec<&str>) -> Line { + Line::Generic { + line_number: 0, + keyword: kw.into(), + args: make_vec!(args), + } + } + + #[test] + fn line_parsing() -> Result<()> { + let p = Parser::default(); + for (input, expected) in [ + ("", Line::Empty), + (" # foo", Line::Empty), + ("Foo Bar", generic_("foo", vec!["Bar"])), + ("Foo Bar baz", generic_("foo", vec!["Bar", "baz"])), + ("Foo \"Bar baz\"", generic_("foo", vec!["Bar baz"])), + ("Foo=bar", generic_("foo", vec!["bar"])), + ("Host a b", host_(vec!["a", "b"])), + ("Match a b", match_(vec!["a", "b"])), + ("iNcluDe c d", include_(vec!["c", "d"])), + ( + "QUOTED \"abc def\" ghi", + generic_("quoted", vec!["abc def", "ghi"]), + ), + ] { + let msg = || format!("input \"{input}\" failed"); + assert_eq_as_result!(p.parse_line(input).with_context(msg)?, expected) + .map_err(|e| anyhow!(e)) + .with_context(msg)?; + } + for (input, expected_msg) in [ + ("aaa bbb \" ccc", "unterminated quote"), + ("aaa", "missing argument"), + ] { + let err = p.parse_line(input).unwrap_err(); + assert_contains_as_result!(err.root_cause().to_string(), expected_msg) + .map_err(|e| anyhow!(e)) + .with_context(|| format!("input \"{input}\" failed"))?; + } + Ok(()) + } + + #[test] + fn host_matching() -> Result<()> { + for (host, args, result) in [ + ("foo", vec!["foo"], true), + ("foo", vec![""], false), + ("foo", vec!["bar"], false), + ("foo", vec!["bar", "foo"], true), + ("foo", vec!["f?o"], true), + ("fooo", vec!["f?o"], false), + ("foo", vec!["f*"], true), + ("oof", vec!["*of"], true), + ("192.168.1.42", vec!["192.168.?.42"], true), + ("192.168.10.42", vec!["192.168.?.42"], false), + ] { + assert_eq_as_result!(evaluate_host_match(host, &make_vec!(args.clone())), result) + .map_err(|e| anyhow!(e)) + .with_context(|| format!("host {host}, args {args:?}"))?; + } + Ok(()) + } + + macro_rules! assert_1_arg { + ($left:expr, $right:expr) => { + assert_eq!(($left).unwrap().args.first().unwrap(), $right); + }; + } + + #[test] + fn defaults_without_host_block() { + let output = Parser::for_str( + r" + Foo Bar + Baz Qux + # foop is a comment + ", + true, + ) + .parse_file_for("any host") + .unwrap(); + //println!("{output:?}"); + assert_1_arg!(output.get("foo"), "Bar"); + assert_1_arg!(output.get("baz"), "Qux"); + assert_eq!(output.get("foop"), None); + } + + #[test] + fn host_block_simple() { + let output = Parser::for_str( + r" + Host Fred + Foo Bar + Host Barney + Foo Baz + ", + true, + ) + .parse_file_for("Fred") + .unwrap(); + assert_1_arg!(output.get("foo"), "Bar"); + } + + #[test] + fn earlier_match_wins() { + let output = Parser::for_str( + r" + Host Fred + Foo Bar + Host Barney + Foo Baz + Host Fred + Foo Qux + Host * + Foo Qix + ", + true, + ) + .parse_file_for("Fred") + .unwrap(); + assert_1_arg!(output.get("foo"), "Bar"); + } + + #[test] + fn later_default_works() { + let output = Parser::for_str( + r" + Host Fred + Foo Bar + Host Barney + Foo Baz + Host * + Qux Qix + ", + true, + ) + .parse_file_for("Fred") + .unwrap(); + assert_1_arg!(output.get("qux"), "Qix"); + } + + #[test] + fn read_real_file() { + let (path, _dir) = make_test_tempfile( + r" + hi there + ", + "test.conf", + ); + let output = Parser::for_path(path, true) + .unwrap() + .parse_file_for("any") + .unwrap(); + assert_1_arg!(output.get("hi"), "there"); + } + + #[test] + fn recursion_limit() { + let tempdir = tempfile::tempdir().unwrap(); + let path = tempdir.path().join("test-recursion"); + let contents = format!( + " + include {path:?} + " + ); + std::fs::write(&path, contents).unwrap(); + let err = Parser::for_path(path, true) + .unwrap() + .parse_file_for("any") + .unwrap_err(); + assert_contains!(err.to_string(), "too many nested includes"); + } + + #[test] + fn expand_globs() { + let tempdir = tempfile::tempdir().unwrap(); + let path1 = tempdir.path().join("test1"); + let path2 = tempdir.path().join("other2"); + let path3 = tempdir.path().join("other3"); + let glob = tempdir.path().join("oth*"); + std::fs::write(&path1, format!("include {glob:?}")).unwrap(); + std::fs::write(&path2, "hi there").unwrap(); + std::fs::write(&path3, "green cheese").unwrap(); + let output = Parser::for_path(path1, true) + .unwrap() + .parse_file_for("any") + .unwrap(); + assert_1_arg!(output.get("hi"), "there"); + assert_1_arg!(output.get("green"), "cheese"); + } + + #[test] + #[ignore] // this test is dependent on the current user filespace + fn tilde_expansion_current_user() { + let a = find_include_files("~/*.conf", true).expect("~ should expand to home directory"); + assert!(!a.is_empty()); + let _ = find_include_files("~/*", false) + .expect_err("~ should not be allowed in system configurations"); + } + + #[test] + #[ignore] // obviously this won't run on CI. TODO: figure out a way to make it CIable. + fn tilde_expansion_arbitrary_user() { + let a = + find_include_files("~wry/*.conf", true).expect("~ should expand to a home directory"); + println!("{a:?}"); + assert!(!a.is_empty()); + let _ = find_include_files("~/*", false) + .expect_err("~ should not be allowed in system configurations"); + } + + #[test] + #[ignore] // TODO: Make this runnable on CI + fn relative_path_expansion() { + let a = find_include_files("config", true).unwrap(); + println!("{a:?}"); + assert!(!a.is_empty()); + + let a = find_include_files("sshd_config", false).unwrap(); + println!("{a:?}"); + assert!(!a.is_empty()); + + // but the user does not have an sshd_config: + let a = find_include_files("sshd_config", true).unwrap(); + println!("{a:?}"); + assert!(a.is_empty()); + } + + #[test] + #[ignore] + fn dump_local_config() { + let path = Platform::user_ssh_config().unwrap(); + let mut parser = Parser::for_path(path, true).unwrap(); + let data = parser.parse_file_for("lapis").unwrap(); + println!("{data:#?}"); + } +} From f6860c2d2931d474d805bb7db797130d718073fc Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Wed, 18 Dec 2024 20:11:24 +1300 Subject: [PATCH 33/54] feat: option to not parse ssh config file --- src/client/main_loop.rs | 4 +- src/client/ssh.rs | 129 +++++++++++++++++++++++++++++----------- src/config/structure.rs | 13 ++++ 3 files changed, 108 insertions(+), 38 deletions(-) diff --git a/src/client/main_loop.rs b/src/client/main_loop.rs index c457163..60d89dd 100644 --- a/src/client/main_loop.rs +++ b/src/client/main_loop.rs @@ -49,8 +49,8 @@ pub async fn client_main( let job_spec = crate::client::CopyJobSpec::try_from(¶meters)?; let credentials = Credentials::generate()?; let user_hostname = job_spec.remote_host(); - let remote_host = - super::ssh::resolve_host_alias(user_hostname).unwrap_or_else(|| user_hostname.into()); + let remote_host = super::ssh::resolve_host_alias(user_hostname, &config.ssh_config) + .unwrap_or_else(|| user_hostname.into()); // If the user didn't specify the address family: we do the DNS lookup, figure it out and tell ssh to use that. // (Otherwise if we resolved a v4 and ssh a v6 - as might happen with round-robin DNS - that could be surprising.) diff --git a/src/client/ssh.rs b/src/client/ssh.rs index 48caf8a..2e1c063 100644 --- a/src/client/ssh.rs +++ b/src/client/ssh.rs @@ -1,48 +1,83 @@ //! Interaction with ssh configuration // (c) 2024 Ross Younger -use std::path::PathBuf; +use std::{path::PathBuf, str::FromStr}; use crate::config::ssh::Parser; -use anyhow::Context; +use anyhow::{Context, Result}; use tracing::{debug, warn}; use crate::os::{AbstractPlatform as _, Platform}; -/// Attempts to resolve a hostname from a single OpenSSH-style config file -fn resolve_one(path: &PathBuf, user_config_file: bool, host: &str) -> Option { - if !std::fs::exists(path).is_ok_and(|b| b) { - // file could not be verified to exist. this is not intrinsically an error; keep quiet - return None; +struct ConfigFile { + path: PathBuf, + user: bool, // this is a user file i.e. ~ expansion is allowed + warn_on_error: bool, +} + +impl ConfigFile { + fn for_path(path: PathBuf, user: bool) -> Self { + Self { + path, + user, + warn_on_error: false, + } } - let mut parser = match Parser::for_path(path, user_config_file) { - Ok(p) => p, - Err(e) => { - // file permissions issue? - warn!("failed to open {path:?}: {e}"); + fn for_str(path: &str, user: bool, warn_on_error: bool) -> Result { + Ok(Self { + path: PathBuf::from_str(path)?, + user, + warn_on_error, + }) + } + + /// Attempts to resolve a hostname from a single OpenSSH-style config file + fn resolve_one(&self, host: &str) -> Option { + let path = &self.path; + if !std::fs::exists(path).is_ok_and(|b| b) { + // file could not be verified to exist. + // This is not intrinsically an error; the user or system file might legitimately not be there. + // But if this was a file explicitly specified by the user, assume they do care and let them know. + if self.warn_on_error { + warn!("ssh-config file {path:?} not found"); + } return None; } - }; - let data = match parser - .parse_file_for(host) - .with_context(|| format!("reading configuration file {path:?}")) - { - Ok(data) => data, - Err(e) => { - warn!("{e}"); - return None; + let mut parser = match Parser::for_path(path, self.user) { + Ok(p) => p, + Err(e) => { + // file permissions issue? + warn!("failed to open {path:?}: {e}"); + return None; + } + }; + let data = match parser + .parse_file_for(host) + .with_context(|| format!("reading configuration file {path:?}")) + { + Ok(data) => data, + Err(e) => { + warn!("{e}"); + return None; + } + }; + if let Some(s) = data.get("hostname") { + let result = s.first_arg(); + debug!("Using hostname '{result}' for '{host}' (from {})", s.source); + Some(result) + } else { + None } - }; - if let Some(s) = data.get("hostname") { - let result = s.first_arg(); - debug!("Using hostname '{result}' for '{host}' (from {})", s.source); - Some(result) - } else { - None } } -/// Attempts to resolve hostname aliasing from the user's and system's ssh config files to resolve aliasing. +/// Attempts to resolve hostname aliasing from ssh config files. +/// +/// ## Arguments +/// * host: the host name alias to look up (matching a 'Host' block in ssh_config) +/// * config_files: The list of ssh config files to use, in priority order. +/// +/// If the list is empty, the user's and system's ssh config files will be used. /// /// ## Returns /// Some(hostname) if any config file matched. @@ -52,20 +87,42 @@ fn resolve_one(path: &PathBuf, user_config_file: bool, host: &str) -> Option Option { - let f = Platform::user_ssh_config().map(|pb| resolve_one(&pb, true, host)); - if let Ok(Some(s)) = f { - return Some(s); +pub fn resolve_host_alias(host: &str, config_files: &[String]) -> Option { + let files = if config_files.is_empty() { + let mut v = Vec::new(); + if let Ok(f) = Platform::user_ssh_config() { + v.push(ConfigFile::for_path(f, true)); + } + if let Ok(f) = ConfigFile::for_str(Platform::system_ssh_config(), false, false) { + v.push(f); + } + v + } else { + config_files + .iter() + .flat_map(|s| ConfigFile::for_str(s, true, true)) + .collect() + }; + for cfg in files { + let result = cfg.resolve_one(host); + if result.is_some() { + return result; + } } - - resolve_one(&PathBuf::from(Platform::system_ssh_config()), false, host) + None } #[cfg(test)] mod test { - use super::resolve_one; + use std::path::Path; + + use super::ConfigFile; use crate::util::make_test_tempfile; + fn resolve_one(path: &Path, user: bool, host: &str) -> Option { + ConfigFile::for_path(path.to_path_buf(), user).resolve_one(host) + } + #[test] fn hosts_resolve() { let (path, _dir) = make_test_tempfile( diff --git a/src/config/structure.rs b/src/config/structure.rs index 5a3b6e6..c018110 100644 --- a/src/config/structure.rs +++ b/src/config/structure.rs @@ -143,6 +143,18 @@ pub struct Configuration { /// Specifies the time format to use when printing messages to the console or to file #[arg(short = 'T', long, value_name("FORMAT"), help_heading("Output"))] pub time_format: TimeFormat, + + /// Alternative ssh config file(s) + /// + /// By default, qcp reads your user and system ssh config files to look for Hostname aliases. + /// In some cases the logic in qcp may not read them successfully; this is an escape hatch, + /// allowing you to specify one or more alternative files to read instead (which may be empty, + /// nonexistent or /dev/null). + /// + /// This option is really intended to be used in a qcp configuration file. + /// On the command line, you can repeat `--ssh-config file` as many times as needed. + #[arg(long, value_name("FILE"), help_heading("Connection"))] + pub ssh_config: Vec, } impl Configuration { @@ -250,6 +262,7 @@ impl Default for Configuration { ssh_opt: vec![], remote_port: PortRange::default(), time_format: TimeFormat::Local, + ssh_config: Vec::new(), } } } From 93bee7c5b689ad3b8efe7650d0c24ae855509d1a Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sat, 21 Dec 2024 11:16:12 +1300 Subject: [PATCH 34/54] misc: make PortRange parse errors more useful --- src/config/manager.rs | 2 +- src/util/port_range.rs | 26 +++++++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/config/manager.rs b/src/config/manager.rs index a68bdda..64144dd 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -516,6 +516,6 @@ mod test { println!("{result}"); assert!(result .to_string() - .contains("invalid port range \"234-123\"")); + .contains("must be increasing")); } } diff --git a/src/util/port_range.rs b/src/util/port_range.rs index ecc1f44..8040cd6 100644 --- a/src/util/port_range.rs +++ b/src/util/port_range.rs @@ -1,6 +1,9 @@ /// CLI argument helper - PortRange // (c) 2024 Ross Younger -use serde::Serialize; +use serde::{ + de::{Error, Unexpected}, + Serialize, +}; use std::{fmt::Display, str::FromStr}; use super::cli::IntOrString; @@ -44,11 +47,20 @@ impl FromStr for PortRange { type Err = figment::Error; fn from_str(s: &str) -> Result { + use figment::error::Error as FigmentError; + static EXPECTED: &str = "a single port number [0..65535] or a range `a-b`"; if let Ok(n) = s.parse::() { // case 1: it's a number // port 0 is allowed here (with the usual "unspecified" semantics), the user may know what they're doing. return Ok(Self { begin: n, end: n }); } + if let Ok(n) = s.parse::() { + // out of range + return Err(FigmentError::invalid_value( + Unexpected::Unsigned(n), + &EXPECTED, + )); + } // case 2: it's a range if let Some((a, b)) = s.split_once('-') { let aa = a.parse(); @@ -56,15 +68,19 @@ impl FromStr for PortRange { if aa.is_ok() && bb.is_ok() { let aa = aa.unwrap_or_default(); let bb = bb.unwrap_or_default(); - if aa != 0 && aa <= bb { - return Ok(Self { begin: aa, end: bb }); + if aa > bb { + return Err(FigmentError::custom(format!( + "invalid port range `{s}` (must be increasing)" + ))); + } else if aa == 0 && bb != 0 { + return Err(FigmentError::custom(format!("invalid port range `{s}` (port 0 means \"any\" so cannot be part of a range)"))); } - // else invalid + return Ok(Self { begin: aa, end: bb }); } // else failed to parse } // else failed to parse - Err(figment::error::Kind::Message(format!("invalid port range \"{s}\"")).into()) + Err(FigmentError::invalid_value(Unexpected::Str(s), &EXPECTED)) } } From 513c0f5d770f7ce417b0cbdf086711edc6751e27 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Sat, 21 Dec 2024 12:54:42 +1300 Subject: [PATCH 35/54] misc: make HumanU64 parse errors more useful --- src/util/humanu64.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/util/humanu64.rs b/src/util/humanu64.rs index 8578fa3..b9e97df 100644 --- a/src/util/humanu64.rs +++ b/src/util/humanu64.rs @@ -3,9 +3,11 @@ use std::{marker::PhantomData, ops::Deref, str::FromStr}; -use anyhow::Context as _; use humanize_rs::bytes::Bytes; -use serde::Serialize; +use serde::{ + de::{self, Error as _}, + Serialize, +}; use super::cli::IntOrString; @@ -41,12 +43,18 @@ impl From for u64 { } impl FromStr for HumanU64 { - type Err = anyhow::Error; + type Err = figment::Error; fn from_str(s: &str) -> Result { + use figment::error::Error as FigmentError; Ok(Self::new( Bytes::from_str(s) - .with_context(|| "parsing bytes string")? + .map_err(|_| { + FigmentError::invalid_value( + de::Unexpected::Str(s), + &"an integer with optional units (examples: `100`, `10M`, `42k`)", + ) + })? .size(), )) } From 856e2c01a3ef1611ba0cd31906fb8fdb34897622 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 19 Dec 2024 20:23:31 +1300 Subject: [PATCH 36/54] misc: enhance ssh config file parsing - Parser returns a HostConfiguration struct, which is a figment::Provider - make it clear that you can only use Parser once - extract arbitrary types from ssh config files - Use extract_lossy() to allow strings to coerce to ints and bools - Use figment Profile to specify that this is for a given host - consolidate and deduplicate pretty-printing code in manager - provide config source information - improve config error output - add a newtype wrapper for figment::Error with custom Display - custom display for figment::error::Kind - split up ssh.rs into component parts for readability --- src/client/ssh.rs | 2 +- src/config/manager.rs | 228 ++++++++++++-- src/config/ssh.rs | 619 +------------------------------------ src/config/ssh/errors.rs | 62 ++++ src/config/ssh/files.rs | 436 ++++++++++++++++++++++++++ src/config/ssh/includes.rs | 90 ++++++ src/config/ssh/lines.rs | 118 +++++++ src/config/ssh/matching.rs | 46 +++ src/config/ssh/values.rs | 78 +++++ 9 files changed, 1041 insertions(+), 638 deletions(-) create mode 100644 src/config/ssh/errors.rs create mode 100644 src/config/ssh/files.rs create mode 100644 src/config/ssh/includes.rs create mode 100644 src/config/ssh/lines.rs create mode 100644 src/config/ssh/matching.rs create mode 100644 src/config/ssh/values.rs diff --git a/src/client/ssh.rs b/src/client/ssh.rs index 2e1c063..a69a636 100644 --- a/src/client/ssh.rs +++ b/src/client/ssh.rs @@ -43,7 +43,7 @@ impl ConfigFile { } return None; } - let mut parser = match Parser::for_path(path, self.user) { + let parser = match Parser::for_path(path, self.user) { Ok(p) => p, Err(e) => { // file permissions issue? diff --git a/src/config/manager.rs b/src/config/manager.rs index 64144dd..bdd93d3 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -12,7 +12,7 @@ use figment::{ }; use serde::Deserialize; use std::{ - collections::HashSet, + collections::{BTreeMap, HashSet}, fmt::{Debug, Display}, path::Path, }; @@ -134,6 +134,16 @@ impl Manager { } } + /// Testing/internal constructor, does not read files from system + #[must_use] + #[allow(unused)] + pub(crate) fn empty() -> Self { + Self { + data: Figment::new(), + //..Self::default() + } + } + /// Merges in a data set, which is some sort of [figment::Provider](https://docs.rs/figment/latest/figment/trait.Provider.html). /// /// Within qcp, we use [crate::util::derive_deftly_template_Optionalify] to implement Provider for [Configuration]. @@ -155,14 +165,30 @@ impl Manager { self.merge_provider(provider); } + /// Merges in a data set from an ssh config file + pub fn merge_ssh_config(&mut self, file: F, host: &str) + where + F: AsRef, + { + let path = file.as_ref(); + // TODO: differentiate between user and system configs (Include rules) + let p = super::ssh::Parser::for_path(file.as_ref(), true) + .and_then(|p| p.parse_file_for(host)) + .map(|hc| self.merge_provider(hc.as_figment())); + if let Err(e) = p { + warn!("parsing {ff}: {e}", ff = path.to_string_lossy()); + } + } + /// Attempts to extract a particular struct from the data. /// /// Within qcp, `T` is usually [Configuration], but it isn't intrinsically required to be. + /// (This is useful for unit testing.) pub fn get<'de, T>(&self) -> anyhow::Result where T: Deserialize<'de>, { - self.data.extract::() + self.data.extract_lossy::() } } @@ -226,32 +252,11 @@ impl PrettyConfig { } } +static DEFAULT_EMPTY_MAP: BTreeMap = BTreeMap::new(); + impl Display for Manager { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let data = match self.data.data() { - Ok(d) => d, - Err(e) => { - // This isn't terribly helpful as it doesn't have metadata attached; BUT attempting to get() a struct does. - return write!(f, "error: {e}"); - } - }; - let data = data.get(&figment::Profile::Default).unwrap(); - - let mut fields = Vec::::new(); - - for field in data.keys() { - let value = self.data.find_value(field); - let value = match value { - Ok(v) => v, - Err(e) => { - writeln!(f, "error on field {field}: {e}")?; - continue; - } - }; - let meta = self.data.find_metadata(field); - fields.push(PrettyConfig::new(field, &value, meta)); - } - write!(f, "{}", Table::new(fields).with(Style::sharp())) + std::fmt::Display::fmt(&self.display_everything_adapter(), f) } } @@ -262,7 +267,7 @@ pub struct DisplayAdapter<'a> { source: &'a Manager, /// Whether to warn if unused fields are present warn_on_unused: bool, - /// The fields we want to output + /// The fields we want to output. (If empty, outputs everything.) fields: HashSet, } @@ -284,13 +289,25 @@ impl Manager { fields, } } + + /// Creates a generic `DisplayAdapter` that outputs everything + /// + /// # Returns + /// An ephemeral structure implementing `Display`. + #[must_use] + pub fn display_everything_adapter(&self) -> DisplayAdapter<'_> { + DisplayAdapter { + source: self, + warn_on_unused: false, + fields: HashSet::::new(), + } + } } impl Display for DisplayAdapter<'_> { /// Formats the contents of this structure which are relevant to a given output type. /// /// N.B. This function uses CLI styling. - #[allow(clippy::missing_panics_doc)] fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { use crate::cli::styles::{ERROR_S, WARNING_S}; use anstream::eprintln; @@ -304,14 +321,17 @@ impl Display for DisplayAdapter<'_> { return Ok(()); } }; - // panic is impossible on the Default profile, hence #[allow(clippy::missing_panics_doc)] - let data = data.get(&figment::Profile::Default).unwrap(); + let profile = match data.first_key_value() { + None => &figment::Profile::Default, + Some((k, _)) => k, + }; + let data = data.get(profile).unwrap_or(&DEFAULT_EMPTY_MAP); let mut output = Vec::::new(); for field in data.keys() { let meta = self.source.data.find_metadata(field); - if self.fields.contains(field) { + if self.fields.is_empty() || self.fields.contains(field) { let value = self.source.data.find_value(field); let value = match value { Ok(v) => v, @@ -335,6 +355,7 @@ impl Display for DisplayAdapter<'_> { #[cfg(test)] mod test { + use crate::config::ssh::SshConfigError; use crate::config::{Configuration, Configuration_Optional, Manager}; use crate::util::{make_test_tempfile, PortRange}; use serde::Deserialize; @@ -514,8 +535,147 @@ mod test { mgr.merge_toml_file(path); let result = mgr.get::().unwrap_err(); println!("{result}"); - assert!(result - .to_string() - .contains("must be increasing")); + assert!(result.to_string().contains("must be increasing")); + } + + #[test] + fn ssh_style() { + #[derive(Debug, Deserialize)] + struct Test { + ssh_opt: Vec, + } + + // Bear in mind: in an ssh style config file, the first match for a particular keyword wins. + let (path, _tempdir) = make_test_tempfile( + r" + host bar + ssh_opt d e f + host * + ssh_opt a b c + ", + "test.conf", + ); + let mut mgr = Manager::empty(); + mgr.merge_ssh_config(&path, "foo"); + //println!("{mgr}"); + let result = mgr.get::().unwrap(); + assert_eq!(result.ssh_opt, vec!["a", "b", "c"]); + + let mut mgr = Manager::without_files(); + mgr.merge_ssh_config(&path, "bar"); + //println!("{mgr}"); + let result = mgr.get::().unwrap(); + assert_eq!(result.ssh_opt, vec!["d", "e", "f"]); + } + + #[test] + fn types() { + use crate::transport::CongestionControllerType; + + #[derive(Debug, Deserialize, PartialEq)] + struct Test { + vecs: Vec, + s: String, + i: u32, + b: bool, + en: CongestionControllerType, + pr: PortRange, + } + + let (path, _tempdir) = make_test_tempfile( + r" + vecs a b c + s foo + i 42 + b true + en bbr + pr 123-456 + ", + "test.conf", + ); + let mut mgr = Manager::empty(); + mgr.merge_ssh_config(&path, "foo"); + println!("{mgr}"); + let result = mgr.get::().unwrap(); + assert_eq!( + result, + Test { + vecs: vec!["a".into(), "b".into(), "c".into()], + s: "foo".into(), + i: 42, + b: true, + en: CongestionControllerType::Bbr, + pr: PortRange { + begin: 123, + end: 456 + } + } + ); + } + + #[test] + fn bools() { + #[derive(Debug, Deserialize)] + struct Test { + b: bool, + } + + let tempdir = tempfile::tempdir().unwrap(); + let path = tempdir.path().join("testfile"); + + for (s, expected) in [ + ("yes", true), + ("true", true), + ("1", true), + ("no", false), + ("false", false), + ("0", false), + ] { + std::fs::write( + &path, + format!( + r" + b {s} + " + ), + ) + .expect("Unable to write tempfile"); + // ... test it + let mut mgr = Manager::empty(); + mgr.merge_ssh_config(&path, "foo"); + let result = mgr + .get::() + .inspect_err(|e| println!("ERROR: {e}")) + .unwrap(); + assert_eq!(result.b, expected); + } + } + + #[test] + fn invalid_data() { + use crate::transport::CongestionControllerType; + + #[derive(Debug, Deserialize, PartialEq)] + struct Test { + b: bool, + en: CongestionControllerType, + i: u32, + pr: PortRange, + } + + let (path, _tempdir) = make_test_tempfile( + r" + i wombat + b wombat + en wombat + pr wombat + ", + "test.conf", + ); + let mut mgr = Manager::empty(); + mgr.merge_ssh_config(&path, "foo"); + //println!("{mgr:?}"); + let err = mgr.get::().map_err(SshConfigError::from).unwrap_err(); + println!("{err}"); } } diff --git a/src/config/ssh.rs b/src/config/ssh.rs index b8a0ee8..d6013b7 100644 --- a/src/config/ssh.rs +++ b/src/config/ssh.rs @@ -1,606 +1,19 @@ //! Config file parsing, openssh-style // (c) 2024 Ross Younger -use std::{ - collections::BTreeMap, - fs::File, - io::{BufRead, BufReader, Read}, - path::{Path, PathBuf}, -}; - -use anyhow::{Context, Result}; -use glob::{glob_with, MatchOptions}; -use tracing::warn; - -#[derive(Debug, Clone, PartialEq)] -/// A parsed line we read from an ssh config file -enum Line { - Empty, - Host { - line_number: usize, - args: Vec, - }, - Match { - line_number: usize, - args: Vec, - }, - Include { - line_number: usize, - args: Vec, - }, - Generic { - line_number: usize, - keyword: String, /*lowercase!*/ - args: Vec, - }, -} - -#[derive(Debug, Clone, PartialEq, Default)] -/// A setting we read from a config file -pub(crate) struct Setting { - /// where the value came from - pub source: String, - /// line number within the value - pub line_number: usize, - /// the setting data itself (not parsed; we assert nothing beyond the parser has applied the ssh quoting logic) - pub args: Vec, -} - -impl Setting { - pub(crate) fn first_arg(&self) -> String { - self.args.first().cloned().unwrap_or_else(String::new) - } -} - -/// Splits a string into a list of arguments. -/// Arguments are delimited by whitespace, subject to quoting (single or double quotes), and simple escapes (\\, \", \'). -fn split_args(input: &str) -> Result> { - // We need to index over the characters of the input, but also need to be able to peek at the next token in case of escapes. - let mut i = 0; - let input: Vec = input.chars().collect(); - let mut output = Vec::::new(); - while i < input.len() { - // Strip any leading whitespace - if input[i] == ' ' || input[i] == '\t' { - i += 1; - continue; - } - if input[i] == '#' { - break; // it's a comment, we're done - } - - // We're at the start of a real token - let mut current_arg = String::new(); - let mut quote_state: char = '\0'; - - while i < input.len() { - let ch = input[i]; - match (ch, quote_state) { - ('\\', _) => { - // It might be an escape - let next = input.get(i + 1); - match next { - Some(nn @ ('\'' | '\"' | '\\')) => { - // It is an escape - current_arg.push(*nn); - i += 1; - } - Some(_) | None => current_arg.push(ch), // Ignore unrecognised escape - } - } - (' ' | '\t', '\0') => break, // end of token - (q @ ('\'' | '\"'), '\0') => quote_state = q, // start of quote - (q1, q2) if q1 == q2 => quote_state = '\0', // end of quote - (c, _) => current_arg.push(c), // nothing special - } - i += 1; - } - - // end of token - anyhow::ensure!(quote_state == '\0', "unterminated quote"); - output.push(current_arg); - i += 1; - } - Ok(output) -} - -fn evaluate_host_match(host: &str, args: &Vec) -> bool { - for arg in args { - if wildmatch::WildMatch::new(arg).matches(host) { - return true; - } - } - false -} - -/// Wildcard matching and ~ expansion for Include directives -fn find_include_files(arg: &str, is_user: bool) -> Result> { - let mut path = if arg.starts_with('~') { - anyhow::ensure!( - is_user, - "include paths may not start with ~ in a system configuration file" - ); - expanduser::expanduser(arg) - .with_context(|| format!("expanding include expression {arg}"))? - } else { - PathBuf::from(arg) - }; - if !path.is_absolute() { - if is_user { - let Some(home) = dirs::home_dir() else { - anyhow::bail!("could not determine home directory"); - }; - let mut buf = home; - buf.push(".ssh"); - buf.push(path); - path = buf; - } else { - let mut buf = PathBuf::from("/etc/ssh/"); - buf.push(path); - path = buf; - } - } - - let mut result = Vec::new(); - let options = MatchOptions { - case_sensitive: true, - require_literal_leading_dot: true, - require_literal_separator: true, - }; - for entry in (glob_with(path.to_string_lossy().as_ref(), options)?).flatten() { - if let Some(s) = entry.to_str() { - result.push(s.into()); - } - } - Ok(result) -} - -pub(crate) struct Parser -where - R: Read, -{ - line_number: usize, - reader: BufReader, - source: String, - is_user: bool, -} - -impl Parser { - pub(crate) fn for_path

(path: P, is_user: bool) -> Result - where - P: AsRef, - { - let path = path.as_ref(); - let file = File::open(path)?; - let reader = BufReader::new(file); - Ok(Self::for_reader( - reader, - path.to_string_lossy().to_string(), - is_user, - )) - } -} - -impl Parser { - fn for_reader(reader: BufReader, source: String, is_user: bool) -> Self { - Self { - line_number: 0, - reader, - source, - is_user, - } - } -} - -impl<'a> Parser<&'a [u8]> { - fn for_str(s: &'a str, is_user: bool) -> Self { - Self::for_reader(BufReader::new(s.as_bytes()), "".into(), is_user) - } -} - -impl Default for Parser<&[u8]> { - fn default() -> Self { - Parser::for_str("", false) - } -} - -impl Parser { - fn parse_line(&self, line: &str) -> Result { - let line = line.trim(); - let line_number = self.line_number; - // extract keyword, which may be delimited by whitespace (Key Value) OR equals (Key=Value) - let (keyword, rest) = { - let mut splitter = line.splitn(2, &[' ', '\t', '=']); - let keyword = match splitter.next() { - None | Some("") => return Ok(Line::Empty), - Some(kw) => kw.to_lowercase(), - }; - (keyword, splitter.next().unwrap_or_default()) - }; - if keyword.starts_with('#') { - return Ok(Line::Empty); - } - let args = split_args(rest).with_context(|| format!("at line {line_number}"))?; - anyhow::ensure!(!args.is_empty(), "missing argument at line {line_number}"); - - Ok(match keyword.as_str() { - "host" => Line::Host { line_number, args }, - "match" => Line::Match { line_number, args }, - "include" => Line::Include { line_number, args }, - _ => Line::Generic { - line_number, - keyword, - args, - }, - }) - } - - const INCLUDE_DEPTH_LIMIT: u8 = 16; - - fn parse_file_inner( - &mut self, - host: &str, - accepting: &mut bool, - depth: u8, - output: &mut BTreeMap, - ) -> Result<()> { - let mut line = String::new(); - anyhow::ensure!( - depth < Self::INCLUDE_DEPTH_LIMIT, - "too many nested includes" - ); - - loop { - line.clear(); - self.line_number += 1; - let mut line = String::new(); - if 0 == self.reader.read_line(&mut line)? { - break; // EOF - } - match self.parse_line(&line)? { - Line::Empty => (), - Line::Host { args, .. } => { - *accepting = evaluate_host_match(host, &args); - } - Line::Match { .. } => { - warn!("match expressions in ssh_config files are not yet supported"); - } - Line::Include { args, .. } => { - for arg in args { - let files = find_include_files(&arg, self.is_user)?; - for f in files { - let mut subparser = - Parser::for_path(f, self.is_user).with_context(|| { - format!( - "Include directive at {} line {}", - self.source, self.line_number - ) - })?; - subparser.parse_file_inner(host, accepting, depth + 1, output)?; - } - } - } - Line::Generic { keyword, args, .. } => { - if *accepting { - // per ssh_config(5), the first matching entry for a given key wins. - let _ = output.entry(keyword).or_insert_with(|| Setting { - source: self.source.clone(), - line_number: self.line_number, - args, - }); - } - } - } - } - Ok(()) - } - - pub(crate) fn parse_file_for(&mut self, host: &str) -> Result> { - let mut output = BTreeMap::::new(); - let mut accepting = true; - self.parse_file_inner(host, &mut accepting, 0, &mut output)?; - Ok(output) - } -} - -#[cfg(test)] -mod test { - use anyhow::{anyhow, Context, Result}; - use assertables::{assert_contains, assert_contains_as_result, assert_eq_as_result}; - - use crate::{ - os::{AbstractPlatform, Platform}, - util::make_test_tempfile, - }; - - use super::{evaluate_host_match, find_include_files, split_args, Line, Parser}; - #[test] - fn arg_splitting() -> Result<()> { - for (input, expected) in [ - ("", vec![]), - ("a", vec!["a"]), - (" a b ", vec!["a", "b"]), - (" a b # c d", vec!["a", "b"]), - (r#"a\ \' \"b"#, vec!["a\\", "'", "\"b"]), - (r#""a b" 'c d'"#, vec!["a b", "c d"]), - (r#""a \"b" '\'c d'"#, vec!["a \"b", "'c d"]), - ] { - let msg = || format!("input \"{input}\" failed"); - assert_eq_as_result!(split_args(input).with_context(msg)?, expected) - .map_err(|e| anyhow!(e)) - .with_context(msg)?; - } - for (input, expected_msg) in [ - ("aaa\"bbb", "unterminated quote"), - ("'", "unterminated quote"), - ] { - let err = split_args(input).unwrap_err(); - assert_contains_as_result!(err.to_string(), expected_msg) - .map_err(|e| anyhow!(e)) - .with_context(|| format!("input \"{input}\" failed"))?; - } - Ok(()) - } - - macro_rules! make_vec { - ($v:expr) => { - $v.into_iter().map(|s| s.into()).collect() - }; - } - - fn host_(args: Vec<&str>) -> Line { - Line::Host { - line_number: 0, - args: make_vec!(args), - } - } - fn match_(args: Vec<&str>) -> Line { - Line::Match { - line_number: 0, - args: make_vec!(args), - } - } - fn include_(args: Vec<&str>) -> Line { - Line::Include { - line_number: 0, - args: make_vec!(args), - } - } - fn generic_(kw: &str, args: Vec<&str>) -> Line { - Line::Generic { - line_number: 0, - keyword: kw.into(), - args: make_vec!(args), - } - } - - #[test] - fn line_parsing() -> Result<()> { - let p = Parser::default(); - for (input, expected) in [ - ("", Line::Empty), - (" # foo", Line::Empty), - ("Foo Bar", generic_("foo", vec!["Bar"])), - ("Foo Bar baz", generic_("foo", vec!["Bar", "baz"])), - ("Foo \"Bar baz\"", generic_("foo", vec!["Bar baz"])), - ("Foo=bar", generic_("foo", vec!["bar"])), - ("Host a b", host_(vec!["a", "b"])), - ("Match a b", match_(vec!["a", "b"])), - ("iNcluDe c d", include_(vec!["c", "d"])), - ( - "QUOTED \"abc def\" ghi", - generic_("quoted", vec!["abc def", "ghi"]), - ), - ] { - let msg = || format!("input \"{input}\" failed"); - assert_eq_as_result!(p.parse_line(input).with_context(msg)?, expected) - .map_err(|e| anyhow!(e)) - .with_context(msg)?; - } - for (input, expected_msg) in [ - ("aaa bbb \" ccc", "unterminated quote"), - ("aaa", "missing argument"), - ] { - let err = p.parse_line(input).unwrap_err(); - assert_contains_as_result!(err.root_cause().to_string(), expected_msg) - .map_err(|e| anyhow!(e)) - .with_context(|| format!("input \"{input}\" failed"))?; - } - Ok(()) - } - - #[test] - fn host_matching() -> Result<()> { - for (host, args, result) in [ - ("foo", vec!["foo"], true), - ("foo", vec![""], false), - ("foo", vec!["bar"], false), - ("foo", vec!["bar", "foo"], true), - ("foo", vec!["f?o"], true), - ("fooo", vec!["f?o"], false), - ("foo", vec!["f*"], true), - ("oof", vec!["*of"], true), - ("192.168.1.42", vec!["192.168.?.42"], true), - ("192.168.10.42", vec!["192.168.?.42"], false), - ] { - assert_eq_as_result!(evaluate_host_match(host, &make_vec!(args.clone())), result) - .map_err(|e| anyhow!(e)) - .with_context(|| format!("host {host}, args {args:?}"))?; - } - Ok(()) - } - - macro_rules! assert_1_arg { - ($left:expr, $right:expr) => { - assert_eq!(($left).unwrap().args.first().unwrap(), $right); - }; - } - - #[test] - fn defaults_without_host_block() { - let output = Parser::for_str( - r" - Foo Bar - Baz Qux - # foop is a comment - ", - true, - ) - .parse_file_for("any host") - .unwrap(); - //println!("{output:?}"); - assert_1_arg!(output.get("foo"), "Bar"); - assert_1_arg!(output.get("baz"), "Qux"); - assert_eq!(output.get("foop"), None); - } - - #[test] - fn host_block_simple() { - let output = Parser::for_str( - r" - Host Fred - Foo Bar - Host Barney - Foo Baz - ", - true, - ) - .parse_file_for("Fred") - .unwrap(); - assert_1_arg!(output.get("foo"), "Bar"); - } - - #[test] - fn earlier_match_wins() { - let output = Parser::for_str( - r" - Host Fred - Foo Bar - Host Barney - Foo Baz - Host Fred - Foo Qux - Host * - Foo Qix - ", - true, - ) - .parse_file_for("Fred") - .unwrap(); - assert_1_arg!(output.get("foo"), "Bar"); - } - - #[test] - fn later_default_works() { - let output = Parser::for_str( - r" - Host Fred - Foo Bar - Host Barney - Foo Baz - Host * - Qux Qix - ", - true, - ) - .parse_file_for("Fred") - .unwrap(); - assert_1_arg!(output.get("qux"), "Qix"); - } - - #[test] - fn read_real_file() { - let (path, _dir) = make_test_tempfile( - r" - hi there - ", - "test.conf", - ); - let output = Parser::for_path(path, true) - .unwrap() - .parse_file_for("any") - .unwrap(); - assert_1_arg!(output.get("hi"), "there"); - } - - #[test] - fn recursion_limit() { - let tempdir = tempfile::tempdir().unwrap(); - let path = tempdir.path().join("test-recursion"); - let contents = format!( - " - include {path:?} - " - ); - std::fs::write(&path, contents).unwrap(); - let err = Parser::for_path(path, true) - .unwrap() - .parse_file_for("any") - .unwrap_err(); - assert_contains!(err.to_string(), "too many nested includes"); - } - - #[test] - fn expand_globs() { - let tempdir = tempfile::tempdir().unwrap(); - let path1 = tempdir.path().join("test1"); - let path2 = tempdir.path().join("other2"); - let path3 = tempdir.path().join("other3"); - let glob = tempdir.path().join("oth*"); - std::fs::write(&path1, format!("include {glob:?}")).unwrap(); - std::fs::write(&path2, "hi there").unwrap(); - std::fs::write(&path3, "green cheese").unwrap(); - let output = Parser::for_path(path1, true) - .unwrap() - .parse_file_for("any") - .unwrap(); - assert_1_arg!(output.get("hi"), "there"); - assert_1_arg!(output.get("green"), "cheese"); - } - - #[test] - #[ignore] // this test is dependent on the current user filespace - fn tilde_expansion_current_user() { - let a = find_include_files("~/*.conf", true).expect("~ should expand to home directory"); - assert!(!a.is_empty()); - let _ = find_include_files("~/*", false) - .expect_err("~ should not be allowed in system configurations"); - } - - #[test] - #[ignore] // obviously this won't run on CI. TODO: figure out a way to make it CIable. - fn tilde_expansion_arbitrary_user() { - let a = - find_include_files("~wry/*.conf", true).expect("~ should expand to a home directory"); - println!("{a:?}"); - assert!(!a.is_empty()); - let _ = find_include_files("~/*", false) - .expect_err("~ should not be allowed in system configurations"); - } - - #[test] - #[ignore] // TODO: Make this runnable on CI - fn relative_path_expansion() { - let a = find_include_files("config", true).unwrap(); - println!("{a:?}"); - assert!(!a.is_empty()); - - let a = find_include_files("sshd_config", false).unwrap(); - println!("{a:?}"); - assert!(!a.is_empty()); - - // but the user does not have an sshd_config: - let a = find_include_files("sshd_config", true).unwrap(); - println!("{a:?}"); - assert!(a.is_empty()); - } - - #[test] - #[ignore] - fn dump_local_config() { - let path = Platform::user_ssh_config().unwrap(); - let mut parser = Parser::for_path(path, true).unwrap(); - let data = parser.parse_file_for("lapis").unwrap(); - println!("{data:#?}"); - } -} +mod errors; +pub(crate) use errors::SshConfigError; + +mod files; +mod includes; +mod lines; +mod matching; +mod values; + +pub(crate) use files::Parser; +pub(crate) use values::Setting; + +use includes::find_include_files; +use lines::{split_args, Line}; +use matching::evaluate_host_match; +use values::ValueProvider; diff --git a/src/config/ssh/errors.rs b/src/config/ssh/errors.rs new file mode 100644 index 0000000..c38995c --- /dev/null +++ b/src/config/ssh/errors.rs @@ -0,0 +1,62 @@ +//! Error output helpers +// (c) 2024 Ross Younger + +use figment::error::{Kind, OneOf}; + +/// A newtype wrapper implementing `Display` for errors originating from this module +#[derive(Debug)] +pub(crate) struct SshConfigError(figment::Error); +impl From for SshConfigError { + fn from(value: figment::Error) -> Self { + Self(value) + } +} + +impl SshConfigError { + fn rewrite_expected_type(s: &str) -> String { + match s { + "a boolean" => format!( + "a boolean ({})", + OneOf(&["yes", "no", "true", "false", "1", "0"]) + ), + _ => s.to_owned(), + } + } + + fn fmt_kind(kind: &Kind, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match kind { + Kind::InvalidType(v, exp) => write!( + f, + "invalid type: found {v}, expected {exp}", + exp = Self::rewrite_expected_type(exp) + ), + Kind::UnknownVariant(v, exp) => { + write!(f, "unknown variant: found {v}, expected {}", OneOf(exp)) + } + _ => std::fmt::Display::fmt(&kind, f), + } + } +} + +impl std::fmt::Display for SshConfigError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let e = &self.0; + Self::fmt_kind(&e.kind, f)?; + + if let (Some(profile), Some(md)) = (&e.profile, &e.metadata) { + if !e.path.is_empty() { + let key = md.interpolate(profile, &e.path); + write!(f, " for {key}")?; + } + } + + if let Some(md) = &e.metadata { + if let Some(source) = &md.source { + write!(f, " at {source}")?; + } else { + write!(f, " in {}", md.name)?; + } + } + Ok(()) + } +} diff --git a/src/config/ssh/files.rs b/src/config/ssh/files.rs new file mode 100644 index 0000000..474b5df --- /dev/null +++ b/src/config/ssh/files.rs @@ -0,0 +1,436 @@ +//! File parsing internals +// (c) 2024 Ross Younger + +use std::{ + collections::BTreeMap, + fs::File, + io::{BufRead, BufReader, Read}, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result}; +use figment::Figment; +use tracing::warn; + +use super::{evaluate_host_match, find_include_files, split_args, Line, Setting, ValueProvider}; + +/// The result of parsing an ssh-style configuration file, with a particular host in mind. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct HostConfiguration { + /// The host we were interested in + host: String, + /// If present, this is the file we read + source: Option, + /// Output data + data: BTreeMap, +} + +impl HostConfiguration { + fn new(host: &str, source: Option) -> Self { + Self { + host: host.into(), + source, + data: BTreeMap::default(), + } + } + pub(crate) fn get(&self, key: &str) -> Option<&Setting> { + self.data.get(key) + } + + pub(crate) fn as_figment(&self) -> Figment { + let mut figment = Figment::new(); + let profile = figment::Profile::new(&self.host); + + for (k, v) in &self.data { + figment = figment.merge(ValueProvider::new(k, v, &profile)); + } + figment + } +} + +/////////////////////////////////////////////////////////////////////////////////////// + +/// The business end of reading a config file. +/// +/// # Note +/// You can only use this struct once. If for some reason you want to re-parse a file, +/// you must create a fresh `Parser` to do so. +pub(crate) struct Parser +where + R: Read, +{ + line_number: usize, + reader: BufReader, + source: String, + path: Option, + is_user: bool, +} + +impl Parser { + pub(crate) fn for_path

(path: P, is_user: bool) -> Result + where + P: AsRef, + { + let path = path.as_ref(); + let file = File::open(path)?; + let reader = BufReader::new(file); + Ok(Self::for_reader( + reader, + path.to_string_lossy().to_string(), + Some(path.to_path_buf()), + is_user, + )) + } +} + +impl<'a> Parser<&'a [u8]> { + fn for_str(s: &'a str, is_user: bool) -> Self { + Self::for_reader( + BufReader::new(s.as_bytes()), + "".into(), + None, + is_user, + ) + } +} + +impl Default for Parser<&[u8]> { + fn default() -> Self { + Parser::for_str("", false) + } +} + +impl Parser { + fn for_reader( + reader: BufReader, + source: String, + path: Option, + is_user: bool, + ) -> Self { + Self { + line_number: 0, + reader, + source, + path, + is_user, + } + } + + fn parse_line(&self, line: &str) -> Result { + let line = line.trim(); + let line_number = self.line_number; + // extract keyword, which may be delimited by whitespace (Key Value) OR equals (Key=Value) + let (keyword, rest) = { + let mut splitter = line.splitn(2, &[' ', '\t', '=']); + let keyword = match splitter.next() { + None | Some("") => return Ok(Line::Empty), + Some(kw) => kw.to_lowercase(), + }; + (keyword, splitter.next().unwrap_or_default()) + }; + if keyword.starts_with('#') { + return Ok(Line::Empty); + } + let args = split_args(rest).with_context(|| format!("at line {line_number}"))?; + anyhow::ensure!(!args.is_empty(), "missing argument at line {line_number}"); + + Ok(match keyword.as_str() { + "host" => Line::Host { line_number, args }, + "match" => Line::Match { line_number, args }, + "include" => Line::Include { line_number, args }, + _ => Line::Generic { + line_number, + keyword, + args, + }, + }) + } + + const INCLUDE_DEPTH_LIMIT: u8 = 16; + + fn parse_file_inner( + &mut self, + accepting: &mut bool, + depth: u8, + output: &mut HostConfiguration, + ) -> Result<()> { + let mut line = String::new(); + anyhow::ensure!( + depth < Self::INCLUDE_DEPTH_LIMIT, + "too many nested includes" + ); + + loop { + line.clear(); + self.line_number += 1; + let mut line = String::new(); + if 0 == self.reader.read_line(&mut line)? { + break; // EOF + } + match self.parse_line(&line)? { + Line::Empty => (), + Line::Host { args, .. } => { + *accepting = evaluate_host_match(&output.host, &args); + } + Line::Match { .. } => { + warn!("match expressions in ssh_config files are not yet supported"); + } + Line::Include { args, .. } => { + for arg in args { + let files = find_include_files(&arg, self.is_user)?; + for f in files { + let mut subparser = + Parser::for_path(f, self.is_user).with_context(|| { + format!( + "Include directive at {} line {}", + self.source, self.line_number + ) + })?; + subparser.parse_file_inner(accepting, depth + 1, output)?; + } + } + } + Line::Generic { keyword, args, .. } => { + if *accepting { + // per ssh_config(5), the first matching entry for a given key wins. + let _ = output.data.entry(keyword).or_insert_with(|| Setting { + source: self.source.clone(), + line_number: self.line_number, + args, + }); + } + } + } + } + Ok(()) + } + + /// Interprets the source with a given hostname in mind. + /// This consumes the `Parser`. + pub(crate) fn parse_file_for(mut self, host: &str) -> Result { + let mut output = HostConfiguration::new(host, self.path.take()); + let mut accepting = true; + self.parse_file_inner(&mut accepting, 0, &mut output)?; + Ok(output) + } +} + +/////////////////////////////////////////////////////////////////////////////////////// + +#[cfg(test)] +mod test { + use anyhow::{anyhow, Context, Result}; + use assertables::{assert_contains, assert_contains_as_result, assert_eq_as_result}; + + use super::super::Line; + use super::Parser; + + use crate::{ + os::{AbstractPlatform, Platform}, + util::make_test_tempfile, + }; + + macro_rules! assert_1_arg { + ($left:expr, $right:expr) => { + assert_eq!(($left).unwrap().args.first().unwrap(), $right); + }; + } + + macro_rules! make_vec { + ($v:expr) => { + $v.into_iter().map(|s| s.into()).collect() + }; + } + + fn host_(args: Vec<&str>) -> Line { + Line::Host { + line_number: 0, + args: make_vec!(args), + } + } + fn match_(args: Vec<&str>) -> Line { + Line::Match { + line_number: 0, + args: make_vec!(args), + } + } + fn include_(args: Vec<&str>) -> Line { + Line::Include { + line_number: 0, + args: make_vec!(args), + } + } + fn generic_(kw: &str, args: Vec<&str>) -> Line { + Line::Generic { + line_number: 0, + keyword: kw.into(), + args: make_vec!(args), + } + } + + #[test] + fn line_parsing() -> Result<()> { + let p = Parser::default(); + for (input, expected) in [ + ("", Line::Empty), + (" # foo", Line::Empty), + ("Foo Bar", generic_("foo", vec!["Bar"])), + ("Foo Bar baz", generic_("foo", vec!["Bar", "baz"])), + ("Foo \"Bar baz\"", generic_("foo", vec!["Bar baz"])), + ("Foo=bar", generic_("foo", vec!["bar"])), + ("Host a b", host_(vec!["a", "b"])), + ("Match a b", match_(vec!["a", "b"])), + ("iNcluDe c d", include_(vec!["c", "d"])), + ( + "QUOTED \"abc def\" ghi", + generic_("quoted", vec!["abc def", "ghi"]), + ), + ] { + let msg = || format!("input \"{input}\" failed"); + assert_eq_as_result!(p.parse_line(input).with_context(msg)?, expected) + .map_err(|e| anyhow!(e)) + .with_context(msg)?; + } + for (input, expected_msg) in [ + ("aaa bbb \" ccc", "unterminated quote"), + ("aaa", "missing argument"), + ] { + let err = p.parse_line(input).unwrap_err(); + assert_contains_as_result!(err.root_cause().to_string(), expected_msg) + .map_err(|e| anyhow!(e)) + .with_context(|| format!("input \"{input}\" failed"))?; + } + Ok(()) + } + + #[test] + fn defaults_without_host_block() { + let output = Parser::for_str( + r" + Foo Bar + Baz Qux + # foop is a comment + ", + true, + ) + .parse_file_for("any host") + .unwrap(); + //println!("{output:?}"); + assert_1_arg!(output.get("foo"), "Bar"); + assert_1_arg!(output.get("baz"), "Qux"); + assert_eq!(output.get("foop"), None); + } + + #[test] + fn host_block_simple() { + let output = Parser::for_str( + r" + Host Fred + Foo Bar + Host Barney + Foo Baz + ", + true, + ) + .parse_file_for("Fred") + .unwrap(); + assert_1_arg!(output.get("foo"), "Bar"); + } + + #[test] + fn earlier_match_wins() { + let output = Parser::for_str( + r" + Host Fred + Foo Bar + Host Barney + Foo Baz + Host Fred + Foo Qux + Host * + Foo Qix + ", + true, + ) + .parse_file_for("Fred") + .unwrap(); + assert_1_arg!(output.get("foo"), "Bar"); + } + + #[test] + fn later_default_works() { + let output = Parser::for_str( + r" + Host Fred + Foo Bar + Host Barney + Foo Baz + Host * + Qux Qix + ", + true, + ) + .parse_file_for("Fred") + .unwrap(); + assert_1_arg!(output.get("qux"), "Qix"); + } + + #[test] + fn read_real_file() { + let (path, _dir) = make_test_tempfile( + r" + hi there + ", + "test.conf", + ); + let output = Parser::for_path(path, true) + .unwrap() + .parse_file_for("any") + .unwrap(); + assert_1_arg!(output.get("hi"), "there"); + } + + #[test] + fn recursion_limit() { + let tempdir = tempfile::tempdir().unwrap(); + let path = tempdir.path().join("test-recursion"); + let contents = format!( + " + include {path:?} + " + ); + std::fs::write(&path, contents).unwrap(); + let err = Parser::for_path(path, true) + .unwrap() + .parse_file_for("any") + .unwrap_err(); + assert_contains!(err.to_string(), "too many nested includes"); + } + + #[test] + fn expand_globs() { + let tempdir = tempfile::tempdir().unwrap(); + let path1 = tempdir.path().join("test1"); + let path2 = tempdir.path().join("other2"); + let path3 = tempdir.path().join("other3"); + let glob = tempdir.path().join("oth*"); + std::fs::write(&path1, format!("include {glob:?}")).unwrap(); + std::fs::write(&path2, "hi there").unwrap(); + std::fs::write(&path3, "green cheese").unwrap(); + let output = Parser::for_path(path1, true) + .unwrap() + .parse_file_for("any") + .unwrap(); + assert_1_arg!(output.get("hi"), "there"); + assert_1_arg!(output.get("green"), "cheese"); + } + + #[test] + #[ignore] + fn dump_local_config() { + let path = Platform::user_ssh_config().unwrap(); + let parser = Parser::for_path(path, true).unwrap(); + let data = parser.parse_file_for("lapis").unwrap(); + println!("{data:#?}"); + } +} diff --git a/src/config/ssh/includes.rs b/src/config/ssh/includes.rs new file mode 100644 index 0000000..c2627dd --- /dev/null +++ b/src/config/ssh/includes.rs @@ -0,0 +1,90 @@ +//! Include directive logic +// (c) 2024 Ross Younger + +use anyhow::{Context, Result}; +use glob::{glob_with, MatchOptions}; +use std::path::PathBuf; + +/// Wildcard matching and ~ expansion for Include directives +pub(super) fn find_include_files(arg: &str, is_user: bool) -> Result> { + let mut path = if arg.starts_with('~') { + anyhow::ensure!( + is_user, + "include paths may not start with ~ in a system configuration file" + ); + expanduser::expanduser(arg) + .with_context(|| format!("expanding include expression {arg}"))? + } else { + PathBuf::from(arg) + }; + if !path.is_absolute() { + if is_user { + let Some(home) = dirs::home_dir() else { + anyhow::bail!("could not determine home directory"); + }; + let mut buf = home; + buf.push(".ssh"); + buf.push(path); + path = buf; + } else { + let mut buf = PathBuf::from("/etc/ssh/"); + buf.push(path); + path = buf; + } + } + + let mut result = Vec::new(); + let options = MatchOptions { + case_sensitive: true, + require_literal_leading_dot: true, + require_literal_separator: true, + }; + for entry in (glob_with(path.to_string_lossy().as_ref(), options)?).flatten() { + if let Some(s) = entry.to_str() { + result.push(s.into()); + } + } + Ok(result) +} + +#[cfg(test)] +mod test { + use super::find_include_files; + + #[test] + #[ignore] // this test is dependent on the current user filespace + fn tilde_expansion_current_user() { + let a = find_include_files("~/*.conf", true).expect("~ should expand to home directory"); + assert!(!a.is_empty()); + let _ = find_include_files("~/*", false) + .expect_err("~ should not be allowed in system configurations"); + } + + #[test] + #[ignore] // obviously this won't run on CI. TODO: figure out a way to make it CIable. + fn tilde_expansion_arbitrary_user() { + let a = + find_include_files("~wry/*.conf", true).expect("~ should expand to a home directory"); + println!("{a:?}"); + assert!(!a.is_empty()); + let _ = find_include_files("~/*", false) + .expect_err("~ should not be allowed in system configurations"); + } + + #[test] + #[ignore] // TODO: Make this runnable on CI + fn relative_path_expansion() { + let a = find_include_files("config", true).unwrap(); + println!("{a:?}"); + assert!(!a.is_empty()); + + let a = find_include_files("sshd_config", false).unwrap(); + println!("{a:?}"); + assert!(!a.is_empty()); + + // but the user does not have an sshd_config: + let a = find_include_files("sshd_config", true).unwrap(); + println!("{a:?}"); + assert!(a.is_empty()); + } +} diff --git a/src/config/ssh/lines.rs b/src/config/ssh/lines.rs new file mode 100644 index 0000000..0caa95e --- /dev/null +++ b/src/config/ssh/lines.rs @@ -0,0 +1,118 @@ +//! Line parsing internals +// (c) 2024 Ross Younger + +use anyhow::Result; + +#[derive(Debug, PartialEq)] +/// A parsed line we read from an ssh config file +pub(super) enum Line { + Empty, + Host { + line_number: usize, + args: Vec, + }, + Match { + line_number: usize, + args: Vec, + }, + Include { + line_number: usize, + args: Vec, + }, + Generic { + line_number: usize, + keyword: String, /*lowercase!*/ + args: Vec, + }, +} + +/////////////////////////////////////////////////////////////////////////////////////// + +/// Splits a string into a list of arguments. +/// Arguments are delimited by whitespace, subject to quoting (single or double quotes), and simple escapes (\\, \", \'). +pub(super) fn split_args(input: &str) -> Result> { + // We need to index over the characters of the input, but also need to be able to peek at the next token in case of escapes. + let mut i = 0; + let input: Vec = input.chars().collect(); + let mut output = Vec::::new(); + while i < input.len() { + // Strip any leading whitespace + if input[i] == ' ' || input[i] == '\t' { + i += 1; + continue; + } + if input[i] == '#' { + break; // it's a comment, we're done + } + + // We're at the start of a real token + let mut current_arg = String::new(); + let mut quote_state: char = '\0'; + + while i < input.len() { + let ch = input[i]; + match (ch, quote_state) { + ('\\', _) => { + // It might be an escape + let next = input.get(i + 1); + match next { + Some(nn @ ('\'' | '\"' | '\\')) => { + // It is an escape + current_arg.push(*nn); + i += 1; + } + Some(_) | None => current_arg.push(ch), // Ignore unrecognised escape + } + } + (' ' | '\t', '\0') => break, // end of token + (q @ ('\'' | '\"'), '\0') => quote_state = q, // start of quote + (q1, q2) if q1 == q2 => quote_state = '\0', // end of quote + (c, _) => current_arg.push(c), // nothing special + } + i += 1; + } + + // end of token + anyhow::ensure!(quote_state == '\0', "unterminated quote"); + output.push(current_arg); + i += 1; + } + Ok(output) +} + +/////////////////////////////////////////////////////////////////////////////////////// + +#[cfg(test)] +mod test { + use anyhow::{anyhow, Context, Result}; + use assertables::{assert_contains_as_result, assert_eq_as_result}; + + use crate::config::ssh::split_args; + #[test] + fn arg_splitting() -> Result<()> { + for (input, expected) in [ + ("", vec![]), + ("a", vec!["a"]), + (" a b ", vec!["a", "b"]), + (" a b # c d", vec!["a", "b"]), + (r#"a\ \' \"b"#, vec!["a\\", "'", "\"b"]), + (r#""a b" 'c d'"#, vec!["a b", "c d"]), + (r#""a \"b" '\'c d'"#, vec!["a \"b", "'c d"]), + ] { + let msg = || format!("input \"{input}\" failed"); + assert_eq_as_result!(split_args(input).with_context(msg)?, expected) + .map_err(|e| anyhow!(e)) + .with_context(msg)?; + } + for (input, expected_msg) in [ + ("aaa\"bbb", "unterminated quote"), + ("'", "unterminated quote"), + ] { + let err = split_args(input).unwrap_err(); + assert_contains_as_result!(err.to_string(), expected_msg) + .map_err(|e| anyhow!(e)) + .with_context(|| format!("input \"{input}\" failed"))?; + } + Ok(()) + } +} diff --git a/src/config/ssh/matching.rs b/src/config/ssh/matching.rs new file mode 100644 index 0000000..53d3eaa --- /dev/null +++ b/src/config/ssh/matching.rs @@ -0,0 +1,46 @@ +//! Host matching +// (c) 2024 Ross Younger + +pub(super) fn evaluate_host_match(host: &str, args: &Vec) -> bool { + for arg in args { + if wildmatch::WildMatch::new(arg).matches(host) { + return true; + } + } + false +} + +/////////////////////////////////////////////////////////////////////////////////////// + +#[cfg(test)] +mod test { + use super::evaluate_host_match; + use anyhow::{anyhow, Context, Result}; + use assertables::assert_eq_as_result; + + #[test] + fn host_matching() -> Result<()> { + for (host, args, result) in [ + ("foo", vec!["foo"], true), + ("foo", vec![""], false), + ("foo", vec!["bar"], false), + ("foo", vec!["bar", "foo"], true), + ("foo", vec!["f?o"], true), + ("fooo", vec!["f?o"], false), + ("foo", vec!["f*"], true), + ("oof", vec!["*of"], true), + ("192.168.1.42", vec!["192.168.?.42"], true), + ("192.168.10.42", vec!["192.168.?.42"], false), + ] { + let vec = args + .clone() + .into_iter() + .map(std::convert::Into::into) + .collect(); + assert_eq_as_result!(evaluate_host_match(host, &vec), result) + .map_err(|e| anyhow!(e)) + .with_context(|| format!("host {host}, args {args:?}"))?; + } + Ok(()) + } +} diff --git a/src/config/ssh/values.rs b/src/config/ssh/values.rs new file mode 100644 index 0000000..dbe5039 --- /dev/null +++ b/src/config/ssh/values.rs @@ -0,0 +1,78 @@ +//! Individual configured values +// (c) 2024 Ross Younger + +use figment::{Metadata, Profile, Source}; + +#[derive(Debug, Clone, PartialEq)] +/// A setting we read from a config file +pub(crate) struct Setting { + /// where the value came from + pub source: String, + /// line number within the value + pub line_number: usize, + /// the setting data itself (not parsed; we assert nothing beyond the parser has applied the ssh quoting logic) + pub args: Vec, +} + +impl Setting { + pub(crate) fn first_arg(&self) -> String { + self.args.first().cloned().unwrap_or_else(String::new) + } +} + +/////////////////////////////////////////////////////////////////////////////////////// + +/// Wraps a Setting into something Figment can deal with +pub(super) struct ValueProvider<'a> { + key: &'a String, + value: &'a Setting, + profile: &'a Profile, +} + +impl<'a> ValueProvider<'a> { + pub(super) fn new(key: &'a String, value: &'a Setting, profile: &'a Profile) -> Self { + Self { + key, + value, + profile, + } + } +} + +impl figment::Provider for ValueProvider<'_> { + fn metadata(&self) -> figment::Metadata { + Metadata::from( + "configuration file", + Source::Custom(format!( + "line {line} of {src}", + src = self.value.source, + line = self.value.line_number + )), + ) + .interpolater(|profile, path| { + let key = path.to_vec(); + format!("key `{key}` of host `{profile}`", key = key.join(".")) + }) + } + + fn data( + &self, + ) -> std::result::Result< + figment::value::Map, + figment::Error, + > { + use figment::value::{Dict, Empty, Value}; + let mut dict = Dict::new(); + let value: Value = match self.value.args.len() { + 0 => Empty::Unit.into(), + 1 => self.value.args.first().unwrap().clone().into(), + _ => self.value.args.clone().into(), + }; + let _ = dict.insert(self.key.clone(), value); + Ok(self.profile.collect(dict)) + } + + fn profile(&self) -> Option { + Some(self.profile.clone()) + } +} From 58c2d98494d5d17cfcdceae421eb83d6907994d3 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Mon, 23 Dec 2024 12:12:46 +1300 Subject: [PATCH 37/54] tidy: Manager constructors - merge Manager::default and Manager::empty - rename Manager::new -> standard - hide Manager::without_files() behind cfg(test) --- src/cli/args.rs | 2 +- src/config/manager.rs | 22 ++++++---------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/src/cli/args.rs b/src/cli/args.rs index c3b376d..98ce82c 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -105,7 +105,7 @@ impl From<&CliArgs> for Manager { /// Merge options from the CLI into the structure. /// Any new option packs (_Optional structs) need to be added here. fn from(value: &CliArgs) -> Self { - let mut mgr = Manager::new(); + let mut mgr = Manager::standard(); mgr.merge_provider(&value.config); mgr } diff --git a/src/config/manager.rs b/src/config/manager.rs index bdd93d3..90c6e62 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -98,7 +98,7 @@ impl Manager { /// Initialises this structure, reading the set of config files appropriate to the platform /// and the current user. #[must_use] - pub fn new() -> Self { + pub fn standard() -> Self { let mut data = Figment::new().merge(SystemDefault::default()); data = add_system_config(data); @@ -125,7 +125,7 @@ impl Manager { /// Testing/internal constructor, does not read files from system #[must_use] - #[allow(unused)] + #[cfg(test)] pub(crate) fn without_files() -> Self { let data = Figment::new().merge(SystemDefault::default()); Self { @@ -134,16 +134,6 @@ impl Manager { } } - /// Testing/internal constructor, does not read files from system - #[must_use] - #[allow(unused)] - pub(crate) fn empty() -> Self { - Self { - data: Figment::new(), - //..Self::default() - } - } - /// Merges in a data set, which is some sort of [figment::Provider](https://docs.rs/figment/latest/figment/trait.Provider.html). /// /// Within qcp, we use [crate::util::derive_deftly_template_Optionalify] to implement Provider for [Configuration]. @@ -555,7 +545,7 @@ mod test { ", "test.conf", ); - let mut mgr = Manager::empty(); + let mut mgr = Manager::default(); mgr.merge_ssh_config(&path, "foo"); //println!("{mgr}"); let result = mgr.get::().unwrap(); @@ -593,7 +583,7 @@ mod test { ", "test.conf", ); - let mut mgr = Manager::empty(); + let mut mgr = Manager::default(); mgr.merge_ssh_config(&path, "foo"); println!("{mgr}"); let result = mgr.get::().unwrap(); @@ -641,7 +631,7 @@ mod test { ) .expect("Unable to write tempfile"); // ... test it - let mut mgr = Manager::empty(); + let mut mgr = Manager::default(); mgr.merge_ssh_config(&path, "foo"); let result = mgr .get::() @@ -672,7 +662,7 @@ mod test { ", "test.conf", ); - let mut mgr = Manager::empty(); + let mut mgr = Manager::default(); mgr.merge_ssh_config(&path, "foo"); //println!("{mgr:?}"); let err = mgr.get::().map_err(SshConfigError::from).unwrap_err(); From ec3a245f94330ed1ff890f5c329e871a19441de3 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Mon, 23 Dec 2024 12:54:46 +1300 Subject: [PATCH 38/54] refactor: unify add_*_config into Manager --- src/config/manager.rs | 58 ++++++++++++++++--------------------------- 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/src/config/manager.rs b/src/config/manager.rs index 90c6e62..ce1999d 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -14,12 +14,12 @@ use serde::Deserialize; use std::{ collections::{BTreeMap, HashSet}, fmt::{Debug, Display}, - path::Path, + path::{Path, PathBuf}, }; use struct_field_names_as_array::FieldNamesAsSlice; use tabled::{settings::style::Style, Table, Tabled}; -use tracing::{trace, warn}; +use tracing::{debug, warn}; // SYSTEM DEFAULTS ////////////////////////////////////////////////////////////////////////////////////////////// @@ -59,32 +59,6 @@ pub struct Manager { data: Figment, } -fn add_user_config(f: Figment) -> Figment { - let Some(path) = Platform::user_config_path() else { - warn!("could not determine user configuration file path"); - return f; - }; - - if !path.exists() { - trace!("user configuration file {path:?} not present"); - return f; - } - f.merge(Toml::file(path.as_path())) -} - -fn add_system_config(f: Figment) -> Figment { - let Some(path) = Platform::system_config_path() else { - warn!("could not determine system configuration file path"); - return f; - }; - - if !path.exists() { - trace!("system configuration file {path:?} not present"); - return f; - } - f.merge(Toml::file(path.as_path())) -} - impl Default for Manager { /// Initialises this structure fully-empty (for new(), or testing) fn default() -> Self { @@ -99,15 +73,26 @@ impl Manager { /// and the current user. #[must_use] pub fn standard() -> Self { - let mut data = Figment::new().merge(SystemDefault::default()); - data = add_system_config(data); - - // N.B. This may leave data in a fused-error state, if a data file isn't parseable. - data = add_user_config(data); - Self { - data, + let mut new1 = Self { + data: Figment::new(), //..Self::default() + }; + new1.merge_provider(SystemDefault::default()); + // N.B. This may leave data in a fused-error state, if a config file isn't parseable. + new1.add_config("system", Platform::system_config_path()); + new1.add_config("user", Platform::user_config_path()); + new1 + } + fn add_config(&mut self, what: &str, path: Option) { + let Some(path) = path else { + warn!("could not determine {what} configuration file path"); + return; + }; + if !path.exists() { + debug!("{what} configuration file {path:?} not present"); + return; } + self.merge_provider(Toml::file(path.as_path())); } /// Returns the list of configuration files we read. @@ -146,7 +131,8 @@ impl Manager { } /// Merges in a data set from a TOML file - pub fn merge_toml_file(&mut self, toml: T) + #[allow(unused)] + pub(crate) fn merge_toml_file(&mut self, toml: T) where T: AsRef, { From 2672ed7f173d10b0ad4bc5b0df7c93f130e2c5ba Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Mon, 23 Dec 2024 13:22:28 +1300 Subject: [PATCH 39/54] misc: allow config extraction for an unspecified host i.e. return all-host settings only --- src/client/ssh.rs | 2 +- src/config/manager.rs | 2 +- src/config/ssh/files.rs | 34 +++++++++++--------- src/config/ssh/matching.rs | 66 +++++++++++++++++++++++++------------- 4 files changed, 64 insertions(+), 40 deletions(-) diff --git a/src/client/ssh.rs b/src/client/ssh.rs index a69a636..1e10249 100644 --- a/src/client/ssh.rs +++ b/src/client/ssh.rs @@ -52,7 +52,7 @@ impl ConfigFile { } }; let data = match parser - .parse_file_for(host) + .parse_file_for(Some(host)) .with_context(|| format!("reading configuration file {path:?}")) { Ok(data) => data, diff --git a/src/config/manager.rs b/src/config/manager.rs index ce1999d..8663791 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -149,7 +149,7 @@ impl Manager { let path = file.as_ref(); // TODO: differentiate between user and system configs (Include rules) let p = super::ssh::Parser::for_path(file.as_ref(), true) - .and_then(|p| p.parse_file_for(host)) + .and_then(|p| p.parse_file_for(Some(host))) .map(|hc| self.merge_provider(hc.as_figment())); if let Err(e) = p { warn!("parsing {ff}: {e}", ff = path.to_string_lossy()); diff --git a/src/config/ssh/files.rs b/src/config/ssh/files.rs index 474b5df..b38f43f 100644 --- a/src/config/ssh/files.rs +++ b/src/config/ssh/files.rs @@ -17,8 +17,8 @@ use super::{evaluate_host_match, find_include_files, split_args, Line, Setting, /// The result of parsing an ssh-style configuration file, with a particular host in mind. #[derive(Debug, Clone, PartialEq)] pub(crate) struct HostConfiguration { - /// The host we were interested in - host: String, + /// The host we were interested in. If None, this is "unspecified", i.e. we return data in `Host *` sections or in an unqualified section at the top of the file. + host: Option, /// If present, this is the file we read source: Option, /// Output data @@ -26,9 +26,9 @@ pub(crate) struct HostConfiguration { } impl HostConfiguration { - fn new(host: &str, source: Option) -> Self { + fn new(host: Option<&str>, source: Option) -> Self { Self { - host: host.into(), + host: host.map(std::borrow::ToOwned::to_owned), source, data: BTreeMap::default(), } @@ -39,8 +39,10 @@ impl HostConfiguration { pub(crate) fn as_figment(&self) -> Figment { let mut figment = Figment::new(); - let profile = figment::Profile::new(&self.host); - + let profile = self + .host + .as_deref() + .map_or(figment::Profile::Default, figment::Profile::new); for (k, v) in &self.data { figment = figment.merge(ValueProvider::new(k, v, &profile)); } @@ -170,7 +172,7 @@ impl Parser { match self.parse_line(&line)? { Line::Empty => (), Line::Host { args, .. } => { - *accepting = evaluate_host_match(&output.host, &args); + *accepting = evaluate_host_match(output.host.as_deref(), &args); } Line::Match { .. } => { warn!("match expressions in ssh_config files are not yet supported"); @@ -207,7 +209,7 @@ impl Parser { /// Interprets the source with a given hostname in mind. /// This consumes the `Parser`. - pub(crate) fn parse_file_for(mut self, host: &str) -> Result { + pub(crate) fn parse_file_for(mut self, host: Option<&str>) -> Result { let mut output = HostConfiguration::new(host, self.path.take()); let mut accepting = true; self.parse_file_inner(&mut accepting, 0, &mut output)?; @@ -313,7 +315,7 @@ mod test { ", true, ) - .parse_file_for("any host") + .parse_file_for(None) .unwrap(); //println!("{output:?}"); assert_1_arg!(output.get("foo"), "Bar"); @@ -332,7 +334,7 @@ mod test { ", true, ) - .parse_file_for("Fred") + .parse_file_for(Some("Fred")) .unwrap(); assert_1_arg!(output.get("foo"), "Bar"); } @@ -352,7 +354,7 @@ mod test { ", true, ) - .parse_file_for("Fred") + .parse_file_for(Some("Fred")) .unwrap(); assert_1_arg!(output.get("foo"), "Bar"); } @@ -370,7 +372,7 @@ mod test { ", true, ) - .parse_file_for("Fred") + .parse_file_for(Some("Fred")) .unwrap(); assert_1_arg!(output.get("qux"), "Qix"); } @@ -385,7 +387,7 @@ mod test { ); let output = Parser::for_path(path, true) .unwrap() - .parse_file_for("any") + .parse_file_for(None) .unwrap(); assert_1_arg!(output.get("hi"), "there"); } @@ -402,7 +404,7 @@ mod test { std::fs::write(&path, contents).unwrap(); let err = Parser::for_path(path, true) .unwrap() - .parse_file_for("any") + .parse_file_for(None) .unwrap_err(); assert_contains!(err.to_string(), "too many nested includes"); } @@ -419,7 +421,7 @@ mod test { std::fs::write(&path3, "green cheese").unwrap(); let output = Parser::for_path(path1, true) .unwrap() - .parse_file_for("any") + .parse_file_for(None) .unwrap(); assert_1_arg!(output.get("hi"), "there"); assert_1_arg!(output.get("green"), "cheese"); @@ -430,7 +432,7 @@ mod test { fn dump_local_config() { let path = Platform::user_ssh_config().unwrap(); let parser = Parser::for_path(path, true).unwrap(); - let data = parser.parse_file_for("lapis").unwrap(); + let data = parser.parse_file_for(Some("lapis")).unwrap(); println!("{data:#?}"); } } diff --git a/src/config/ssh/matching.rs b/src/config/ssh/matching.rs index 53d3eaa..287c6c1 100644 --- a/src/config/ssh/matching.rs +++ b/src/config/ssh/matching.rs @@ -1,13 +1,14 @@ //! Host matching // (c) 2024 Ross Younger -pub(super) fn evaluate_host_match(host: &str, args: &Vec) -> bool { - for arg in args { - if wildmatch::WildMatch::new(arg).matches(host) { - return true; - } +pub(super) fn evaluate_host_match(host: Option<&str>, args: &[String]) -> bool { + if let Some(host) = host { + args.iter() + .any(|arg| wildmatch::WildMatch::new(arg).matches(host)) + } else { + // host is None i.e. unspecified; match only on '*' + args.iter().any(|arg| arg == "*") } - false } /////////////////////////////////////////////////////////////////////////////////////// @@ -18,29 +19,50 @@ mod test { use anyhow::{anyhow, Context, Result}; use assertables::assert_eq_as_result; + /// helper macro: concise notation to create a Vec + /// + /// # Example + /// ``` + /// use vec_of_strings as sv; + /// assert_eq!(sv["a","b"], vec![String::from("a"), String::from("b")]); + /// ``` + macro_rules! vec_of_strings { + ($($x:expr),*) => (vec![$($x.to_string()),*]); + } + + use vec_of_strings as sv; + #[test] fn host_matching() -> Result<()> { for (host, args, result) in [ - ("foo", vec!["foo"], true), - ("foo", vec![""], false), - ("foo", vec!["bar"], false), - ("foo", vec!["bar", "foo"], true), - ("foo", vec!["f?o"], true), - ("fooo", vec!["f?o"], false), - ("foo", vec!["f*"], true), - ("oof", vec!["*of"], true), - ("192.168.1.42", vec!["192.168.?.42"], true), - ("192.168.10.42", vec!["192.168.?.42"], false), + ("foo", sv!["foo"], true), + ("foo", sv![""], false), + ("foo", sv!["bar"], false), + ("foo", sv!["bar", "foo"], true), + ("foo", sv!["f?o"], true), + ("fooo", sv!["f?o"], false), + ("foo", sv!["f*"], true), + ("oof", sv!["*of"], true), + ("192.168.1.42", sv!["192.168.?.42"], true), + ("192.168.10.42", sv!["192.168.?.42"], false), ] { - let vec = args - .clone() - .into_iter() - .map(std::convert::Into::into) - .collect(); - assert_eq_as_result!(evaluate_host_match(host, &vec), result) + assert_eq_as_result!(evaluate_host_match(Some(host), &args), result) .map_err(|e| anyhow!(e)) .with_context(|| format!("host {host}, args {args:?}"))?; } Ok(()) } + #[test] + fn unspecified_host() -> Result<()> { + for (args, result) in [ + (sv!["foo", "bar", "baz"], false), + (sv!["*"], true), + (sv!["foo", "bar", "*", "baz"], true), // silly case but we ought to get it right + ] { + assert_eq_as_result!(evaluate_host_match(None, &args), result) + .map_err(|e| anyhow!(e)) + .with_context(|| format!("host , args {args:?}"))?; + } + Ok(()) + } } From 9f4d03ae51dedb7cb2f6d533ede314c11889fd30 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Mon, 23 Dec 2024 13:05:47 +1300 Subject: [PATCH 40/54] feat: move to ssh style config files, with options for the targetted Host - Update error message output to suit - --show-config outputs settings for the job where possible - This overhauled DisplayAdapter::fmt() - Tweak source display for values read from files - add cli_beats_config_file test - remove toml scaffolding - update config module docs for file format --- Cargo.lock | 28 ---- Cargo.toml | 3 +- src/cli/args.rs | 25 ++- src/cli/cli_main.rs | 28 ++-- src/client/options.rs | 30 +++- src/config/manager.rs | 346 +++++++++++++-------------------------- src/config/mod.rs | 56 +++++-- src/config/ssh/errors.rs | 19 +++ src/config/ssh/values.rs | 2 +- src/util/port_range.rs | 9 +- 10 files changed, 241 insertions(+), 305 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aa364f2..5cda830 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -537,7 +537,6 @@ checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" dependencies = [ "atomic", "serde", - "toml", "uncased", "version_check", ] @@ -1120,7 +1119,6 @@ dependencies = [ "tempfile", "tokio", "tokio-util", - "toml", "tracing", "tracing-subscriber", "wildmatch", @@ -1432,15 +1430,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_spanned" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" -dependencies = [ - "serde", -] - [[package]] name = "serde_test" version = "1.0.177" @@ -1778,26 +1767,11 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml" -version = "0.8.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - [[package]] name = "toml_datetime" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" -dependencies = [ - "serde", -] [[package]] name = "toml_edit" @@ -1806,8 +1780,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap", - "serde", - "serde_spanned", "toml_datetime", "winnow", ] diff --git a/Cargo.toml b/Cargo.toml index 59b2fe1..0d0c5eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ derive-deftly = "0.14.2" dirs = "5.0.1" dns-lookup = "2.0.4" expanduser = "1.2.2" -figment = { version = "0.10.19", features = ["toml"] } +figment = { version = "0.10.19" } futures-util = { version = "0.3.31", default-features = false } gethostname = "0.5.0" glob = "0.3.1" @@ -74,7 +74,6 @@ rand = "0.8.5" serde_json = "1.0.133" serde_test = "1.0.177" tempfile = "3.14.0" -toml = "0.8.19" [lints.rust] dead_code = "warn" diff --git a/src/cli/args.rs b/src/cli/args.rs index 98ce82c..d6be003 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -6,7 +6,7 @@ use clap::{ArgAction::SetTrue, Args as _, FromArgMatches as _, Parser}; use crate::{config::Manager, util::AddressFamily}; /// Options that switch us into another mode i.e. which don't require source/destination arguments -pub(crate) const MODE_OPTIONS: &[&str] = &["server", "help_buffers", "show_config", "config_files"]; +pub(crate) const MODE_OPTIONS: &[&str] = &["server", "help_buffers", "config_files", "show_config"]; #[derive(Debug, Parser, Clone)] #[command( @@ -44,7 +44,14 @@ pub(crate) struct CliArgs { )] pub server: bool, - /// Outputs the configuration, then exits + /// Outputs the configuration, then exits. + /// + /// If a remote `SOURCE` or `DESTINATION` argument is given, outputs the configuration we would use + /// for operations to that host. + /// + /// If not, outputs only global settings from configuration, which may be overridden in + /// `Host` blocks in configuration files. + /// #[arg(long, help_heading("Configuration"))] pub show_config: bool, /// Outputs the paths to configuration file(s), then exits @@ -101,12 +108,14 @@ impl CliArgs { } } -impl From<&CliArgs> for Manager { - /// Merge options from the CLI into the structure. - /// Any new option packs (_Optional structs) need to be added here. - fn from(value: &CliArgs) -> Self { - let mut mgr = Manager::standard(); +impl TryFrom<&CliArgs> for Manager { + type Error = anyhow::Error; + + fn try_from(value: &CliArgs) -> Result { + let host = value.client_params.remote_host_lossy()?; + + let mut mgr = Manager::standard(host.as_deref()); mgr.merge_provider(&value.config); - mgr + Ok(mgr) } } diff --git a/src/cli/cli_main.rs b/src/cli/cli_main.rs index fac2cac..4b5bfa7 100644 --- a/src/cli/cli_main.rs +++ b/src/cli/cli_main.rs @@ -3,10 +3,7 @@ use std::process::ExitCode; -use super::{ - args::CliArgs, - styles::{ERROR_S, INFO_S}, -}; +use super::{args::CliArgs, styles::ERROR_S}; use crate::{ client::{client_main, Parameters as ClientParameters, MAX_UPDATE_FPS}, config::{Configuration, Manager}, @@ -58,19 +55,19 @@ pub async fn cli() -> anyhow::Result { } // Now fold the arguments in with the CLI config (which may fail) - let config_manager = Manager::from(&args); + let config_manager = match Manager::try_from(&args) { + Ok(m) => m, + Err(err) => { + eprintln!("{}: {err}", "ERROR".style(*ERROR_S)); + return Ok(ExitCode::FAILURE); + } + }; let config = match config_manager.get::() { Ok(c) => c, Err(err) => { - println!("{}: Failed to parse configuration", "ERROR".style(*ERROR_S)); - if err.count() == 1 { - println!("{err}"); - } else { - for (i, e) in err.into_iter().enumerate() { - println!("{}: {e}", (i + 1).style(*INFO_S)); - } - } + eprintln!("{}: Failed to parse configuration", "ERROR".style(*ERROR_S)); + err.into_iter().for_each(|e| eprintln!("{e}")); return Ok(ExitCode::FAILURE); } }; @@ -84,10 +81,7 @@ pub async fn cli() -> anyhow::Result { .inspect_err(|e| eprintln!("{e:?}"))?; if args.show_config { - println!( - "{}", - config_manager.to_display_adapter::(true) - ); + println!("{}", config_manager.to_display_adapter::()); Ok(ExitCode::SUCCESS) } else if args.server { let _span = error_span!("REMOTE").entered(); diff --git a/src/client/options.rs b/src/client/options.rs index 4ae3b16..c38b807 100644 --- a/src/client/options.rs +++ b/src/client/options.rs @@ -54,8 +54,7 @@ pub struct Parameters { /// /// Exactly one of source and destination must be remote. #[arg( - conflicts_with_all(crate::cli::MODE_OPTIONS), - required = true, + required_unless_present_any(crate::cli::MODE_OPTIONS), value_name = "SOURCE" )] pub source: Option, @@ -66,8 +65,7 @@ pub struct Parameters { /// /// Exactly one of source and destination must be remote. #[arg( - conflicts_with_all(crate::cli::MODE_OPTIONS), - required = true, + required_unless_present_any(crate::cli::MODE_OPTIONS), value_name = "DESTINATION" )] pub destination: Option, @@ -98,3 +96,27 @@ impl TryFrom<&Parameters> for CopyJobSpec { }) } } + +impl Parameters { + /// A best-effort attempt to extract a single remote host string from the parameters. + /// + /// # Output + /// If neither source nor dest are present, Ok("") + /// If at most one of source and dest contains a remote host, Ok() + /// + /// # Errors + /// If both source and dest contain a remote host, Err("Only one remote file argument is supported") + pub(crate) fn remote_host_lossy(&self) -> anyhow::Result> { + let src_host = self.source.as_ref().and_then(|fs| fs.host.as_ref()); + let dst_host = self.destination.as_ref().and_then(|fs| fs.host.as_ref()); + Ok(if let Some(src_host) = src_host { + if dst_host.is_some() { + anyhow::bail!("Only one remote file argument is supported"); + } + Some(src_host.to_string()) + } else { + // Destination without source would be an exotic situation, but do our best anyway: + dst_host.map(std::string::ToString::to_string) + }) + } +} diff --git a/src/config/manager.rs b/src/config/manager.rs index 8663791..2fee980 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -3,21 +3,20 @@ use crate::os::{AbstractPlatform as _, Platform}; -use super::Configuration; +use super::{ssh::SshConfigError, Configuration}; -use figment::{ - providers::{Format, Serialized, Toml}, - value::Value, - Figment, Metadata, Provider, -}; +use figment::{providers::Serialized, value::Value, Figment, Metadata, Provider}; use serde::Deserialize; use std::{ - collections::{BTreeMap, HashSet}, + collections::HashSet, fmt::{Debug, Display}, path::{Path, PathBuf}, }; use struct_field_names_as_array::FieldNamesAsSlice; -use tabled::{settings::style::Style, Table, Tabled}; +use tabled::{ + settings::{object::Rows, style::Style, Color}, + Table, Tabled, +}; use tracing::{debug, warn}; @@ -57,6 +56,8 @@ impl Provider for SystemDefault { pub struct Manager { /// Configuration data data: Figment, + /// The host argument this data was read for, if applicable + host: Option, } impl Default for Manager { @@ -64,6 +65,7 @@ impl Default for Manager { fn default() -> Self { Self { data: Figment::default(), + host: None, } } } @@ -72,18 +74,24 @@ impl Manager { /// Initialises this structure, reading the set of config files appropriate to the platform /// and the current user. #[must_use] - pub fn standard() -> Self { + pub fn standard(for_host: Option<&str>) -> Self { let mut new1 = Self { data: Figment::new(), - //..Self::default() + host: for_host.map(std::borrow::ToOwned::to_owned), }; new1.merge_provider(SystemDefault::default()); // N.B. This may leave data in a fused-error state, if a config file isn't parseable. - new1.add_config("system", Platform::system_config_path()); - new1.add_config("user", Platform::user_config_path()); + new1.add_config(false, "system", Platform::system_config_path(), for_host); + new1.add_config(true, "user", Platform::user_config_path(), for_host); new1 } - fn add_config(&mut self, what: &str, path: Option) { + fn add_config( + &mut self, + is_user: bool, + what: &str, + path: Option, + for_host: Option<&str>, + ) { let Some(path) = path else { warn!("could not determine {what} configuration file path"); return; @@ -92,7 +100,7 @@ impl Manager { debug!("{what} configuration file {path:?} not present"); return; } - self.merge_provider(Toml::file(path.as_path())); + self.merge_ssh_config(path, for_host, is_user); } /// Returns the list of configuration files we read. @@ -111,12 +119,10 @@ impl Manager { /// Testing/internal constructor, does not read files from system #[must_use] #[cfg(test)] - pub(crate) fn without_files() -> Self { + pub(crate) fn without_files(host: Option<&str>) -> Self { let data = Figment::new().merge(SystemDefault::default()); - Self { - data, - //..Self::default() - } + let host = host.map(std::string::ToString::to_string); + Self { data, host } } /// Merges in a data set, which is some sort of [figment::Provider](https://docs.rs/figment/latest/figment/trait.Provider.html). @@ -130,26 +136,14 @@ impl Manager { self.data = f.merge(provider); // in the error case, this leaves the provider in a fused state } - /// Merges in a data set from a TOML file - #[allow(unused)] - pub(crate) fn merge_toml_file(&mut self, toml: T) - where - T: AsRef, - { - let path = toml.as_ref(); - let provider = Toml::file_exact(path); - self.merge_provider(provider); - } - /// Merges in a data set from an ssh config file - pub fn merge_ssh_config(&mut self, file: F, host: &str) + pub fn merge_ssh_config(&mut self, file: F, host: Option<&str>, is_user: bool) where F: AsRef, { let path = file.as_ref(); - // TODO: differentiate between user and system configs (Include rules) - let p = super::ssh::Parser::for_path(file.as_ref(), true) - .and_then(|p| p.parse_file_for(Some(host))) + let p = super::ssh::Parser::for_path(file.as_ref(), is_user) + .and_then(|p| p.parse_file_for(host)) .map(|hc| self.merge_provider(hc.as_figment())); if let Err(e) = p { warn!("parsing {ff}: {e}", ff = path.to_string_lossy()); @@ -160,11 +154,21 @@ impl Manager { /// /// Within qcp, `T` is usually [Configuration], but it isn't intrinsically required to be. /// (This is useful for unit testing.) - pub fn get<'de, T>(&self) -> anyhow::Result + pub(crate) fn get<'de, T>(&self) -> anyhow::Result where T: Deserialize<'de>, { - self.data.extract_lossy::() + let profile = if let Some(host) = &self.host { + figment::Profile::new(host) + } else { + figment::Profile::Default + }; + + self.data + .clone() + .select(profile) + .extract_lossy::() + .map_err(SshConfigError::from) } } @@ -228,21 +232,11 @@ impl PrettyConfig { } } -static DEFAULT_EMPTY_MAP: BTreeMap = BTreeMap::new(); - -impl Display for Manager { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(&self.display_everything_adapter(), f) - } -} - /// Pretty-printing type wrapper to Manager #[derive(Debug)] pub struct DisplayAdapter<'a> { /// Data source source: &'a Manager, - /// Whether to warn if unused fields are present - warn_on_unused: bool, /// The fields we want to output. (If empty, outputs everything.) fields: HashSet, } @@ -253,7 +247,7 @@ impl Manager { /// # Returns /// An ephemeral structure implementing `Display`. #[must_use] - pub fn to_display_adapter<'de, T>(&self, warn_on_unused: bool) -> DisplayAdapter<'_> + pub fn to_display_adapter<'de, T>(&self) -> DisplayAdapter<'_> where T: Deserialize<'de> + FieldNamesAsSlice, { @@ -261,23 +255,9 @@ impl Manager { fields.extend(T::FIELD_NAMES_AS_SLICE.iter().map(|s| String::from(*s))); DisplayAdapter { source: self, - warn_on_unused, fields, } } - - /// Creates a generic `DisplayAdapter` that outputs everything - /// - /// # Returns - /// An ephemeral structure implementing `Display`. - #[must_use] - pub fn display_everything_adapter(&self) -> DisplayAdapter<'_> { - DisplayAdapter { - source: self, - warn_on_unused: false, - fields: HashSet::::new(), - } - } } impl Display for DisplayAdapter<'_> { @@ -285,47 +265,39 @@ impl Display for DisplayAdapter<'_> { /// /// N.B. This function uses CLI styling. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use crate::cli::styles::{ERROR_S, WARNING_S}; - use anstream::eprintln; - use owo_colors::OwoColorize as _; - - let data = match self.source.data.data() { - Ok(d) => d, - Err(e) => { - // This isn't terribly helpful as it doesn't have metadata attached; BUT attempting to get() a struct does. - eprintln!("{} {e}", "ERROR".style(*ERROR_S)); - return Ok(()); - } - }; - let profile = match data.first_key_value() { - None => &figment::Profile::Default, - Some((k, _)) => k, - }; - let data = data.get(profile).unwrap_or(&DEFAULT_EMPTY_MAP); + let mut data = self.source.data.clone(); let mut output = Vec::::new(); - - for field in data.keys() { - let meta = self.source.data.find_metadata(field); - if self.fields.is_empty() || self.fields.contains(field) { - let value = self.source.data.find_value(field); - let value = match value { - Ok(v) => v, - Err(e) => { - eprintln!("{}: error on {field}: {e}", "WARNING".style(*WARNING_S)); - continue; - } - }; + // First line of the table is special + let (host_string, host_colour) = if let Some(host) = &self.source.host { + let profile = figment::Profile::new(host); + data = data.select(profile); + (host.clone(), Color::FG_GREEN) + } else { + ("* (globals)".into(), Color::FG_CYAN) + }; + output.push(PrettyConfig { + field: "(Remote host)".into(), + value: host_string, + source: String::new(), + }); + + let mut keys = self.fields.iter().collect::>(); + keys.sort(); + + for field in keys { + if let Ok(value) = data.find_value(field) { + let meta = data.get_metadata(value.tag()); output.push(PrettyConfig::new(field, &value, meta)); - } else if self.warn_on_unused { - let source = PrettyConfig::render_source(meta); - eprintln!( - "{}: unrecognised field `{field}` in {source}", - "WARNING".style(*WARNING_S) - ); } } - write!(f, "{}", Table::new(output).with(Style::sharp())) + write!( + f, + "{}", + Table::new(output) + .modify(Rows::single(1), host_colour) + .with(Style::sharp()) + ) } } @@ -338,7 +310,7 @@ mod test { #[test] fn defaults() { - let mgr = Manager::without_files(); + let mgr = Manager::without_files(None); let result = mgr.get().unwrap(); let expected = Configuration::default(); assert_eq!(expected, result); @@ -356,53 +328,12 @@ mod test { ..Default::default() }; - let mut mgr = Manager::without_files(); + let mut mgr = Manager::without_files(None); mgr.merge_provider(entered); let result = mgr.get().unwrap(); assert_eq!(expected, result); } - #[test] - fn dump_config_cli_and_toml() { - // Not a unit test as such; this is a human test - let (path, _tempdir) = make_test_tempfile( - r#" - tx = 42 - congestion = "Bbr" - unused__ = 42 - "#, - "test.toml", - ); - let fake_cli = Configuration_Optional { - rtt: Some(999), - initial_congestion_window: Some(67890), - ..Default::default() - }; - let mut mgr = Manager::without_files(); - mgr.merge_toml_file(path); - mgr.merge_provider(fake_cli); - println!("{mgr}"); - } - - #[test] - fn unparseable_toml() { - // This is a semi unit test; there is one assert, but the secondary goal is that it outputs something sensible - let (path, _tempdir) = make_test_tempfile( - r" - a = 1 - rx 123 # this line is a syntax error - b = 2 - ", - "test.toml", - ); - let mut mgr = Manager::without_files(); - mgr.merge_toml_file(path); - let get = mgr.get::(); - assert!(get.is_err()); - println!("{}", get.unwrap_err()); - // println!("{mgr}"); - } - #[test] fn type_error() { // This is a semi unit test; this has a secondary goal of outputting something sensible @@ -414,86 +345,23 @@ mod test { let (path, _tempdir) = make_test_tempfile( r" - rx = true # invalid - rtt = 3.14159 # also invalid - magic_ = 42 + rx true # invalid + rtt 3.14159 # also invalid + magic_ 42 ", - "test.toml", + "test.conf", ); - let mut mgr = Manager::without_files(); - mgr.merge_toml_file(path); - // This TOML successfully merges into the config, but you can't extract the struct. + let mut mgr = Manager::without_files(None); + mgr.merge_ssh_config(path, None, false); + // This file successfully merges into the config, but you can't extract the struct. let err = mgr.get::().unwrap_err(); println!("Error: {err}"); - // TODO: Would really like a rich error message here pointing to the failing key and errant file. - // We get no metadata in the error :-( // But the config as a whole is not broken and other things can be extracted: let other_struct = mgr.get::().unwrap(); assert_eq!(other_struct.magic_, 42); } - #[test] - fn int_or_string() { - #[derive(Deserialize)] - struct Test { - t1: PortRange, - t2: PortRange, - t3: PortRange, - } - let (path, _tempdir) = make_test_tempfile( - r#" - t1 = 1234 - t2 = "2345" - t3 = "123-456" - "#, - "test.toml", - ); - let mut mgr = Manager::without_files(); - mgr.merge_toml_file(path); - let res = mgr.get::().unwrap(); - assert_eq!( - res.t1, - PortRange { - begin: 1234, - end: 1234 - } - ); - assert_eq!( - res.t2, - PortRange { - begin: 2345, - end: 2345 - } - ); - assert_eq!( - res.t3, - PortRange { - begin: 123, - end: 456 - } - ); - } - - #[test] - fn array_type() { - #[derive(Deserialize)] - struct Test { - ii: Vec, - } - - let (path, _tempdir) = make_test_tempfile( - r" - ii = [1,2,3,4,6] - ", - "test.toml", - ); - let mut mgr = Manager::without_files(); - mgr.merge_toml_file(path); - let result = mgr.get::().unwrap(); - assert_eq!(result.ii, vec![1, 2, 3, 4, 6]); - } - #[test] fn field_parse_failure() { #[derive(Debug, Deserialize)] @@ -502,13 +370,13 @@ mod test { } let (path, _tempdir) = make_test_tempfile( - r#" - _p = "234-123" - "#, - "test.toml", + r" + _p 234-123 + ", + "test.conf", ); - let mut mgr = Manager::without_files(); - mgr.merge_toml_file(path); + let mut mgr = Manager::without_files(None); + mgr.merge_ssh_config(path, None, true); let result = mgr.get::().unwrap_err(); println!("{result}"); assert!(result.to_string().contains("must be increasing")); @@ -520,7 +388,6 @@ mod test { struct Test { ssh_opt: Vec, } - // Bear in mind: in an ssh style config file, the first match for a particular keyword wins. let (path, _tempdir) = make_test_tempfile( r" @@ -531,15 +398,14 @@ mod test { ", "test.conf", ); - let mut mgr = Manager::default(); - mgr.merge_ssh_config(&path, "foo"); - //println!("{mgr}"); + let mut mgr = Manager::without_files(Some("foo")); + mgr.merge_ssh_config(&path, Some("foo"), false); + //println!("{}", mgr.to_display_adapter::(false)); let result = mgr.get::().unwrap(); assert_eq!(result.ssh_opt, vec!["a", "b", "c"]); - let mut mgr = Manager::without_files(); - mgr.merge_ssh_config(&path, "bar"); - //println!("{mgr}"); + let mut mgr = Manager::without_files(Some("bar")); + mgr.merge_ssh_config(&path, Some("bar"), false); let result = mgr.get::().unwrap(); assert_eq!(result.ssh_opt, vec!["d", "e", "f"]); } @@ -569,9 +435,9 @@ mod test { ", "test.conf", ); - let mut mgr = Manager::default(); - mgr.merge_ssh_config(&path, "foo"); - println!("{mgr}"); + let mut mgr = Manager::without_files(Some("foo")); + mgr.merge_ssh_config(&path, Some("foo"), false); + // println!("{mgr}"); let result = mgr.get::().unwrap(); assert_eq!( result, @@ -617,8 +483,8 @@ mod test { ) .expect("Unable to write tempfile"); // ... test it - let mut mgr = Manager::default(); - mgr.merge_ssh_config(&path, "foo"); + let mut mgr = Manager::without_files(Some("foo")); + mgr.merge_ssh_config(&path, Some("foo"), false); let result = mgr .get::() .inspect_err(|e| println!("ERROR: {e}")) @@ -649,9 +515,31 @@ mod test { "test.conf", ); let mut mgr = Manager::default(); - mgr.merge_ssh_config(&path, "foo"); + mgr.merge_ssh_config(&path, Some("foo"), false); //println!("{mgr:?}"); let err = mgr.get::().map_err(SshConfigError::from).unwrap_err(); println!("{err}"); } + + #[test] + fn cli_beats_config_file() { + // simulate a CLI + let entered = Configuration_Optional { + rx: Some(12345.into()), + ..Default::default() + }; + let (path, _tempdir) = make_test_tempfile( + r" + rx 66666 + ", + "test.conf", + ); + + let mut mgr = Manager::without_files(None); + mgr.merge_ssh_config(&path, Some("foo"), false); + // The order of merging mirrors what happens in Manager::try_from(&CliArgs) + mgr.merge_provider(entered); + let result = mgr.get::().unwrap(); + assert_eq!(12345, *result.rx); + } } diff --git a/src/config/mod.rs b/src/config/mod.rs index e10f425..584c765 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -3,8 +3,8 @@ //! //! qcp obtains run-time configuration from the following sources, in order: //! 1. Command-line options -//! 2. The user's configuration file (typically `~/.qcp.toml`) -//! 3. The system-wide configuration file (typically `/etc/qcp.toml`) +//! 2. The user's configuration file (typically `~/.qcp.conf`) +//! 3. The system-wide configuration file (typically `/etc/qcp.conf`) //! 4. Hard-wired defaults //! //! Each option may appear in multiple places, but only the first match is used. @@ -14,21 +14,55 @@ //! //! ## File format //! -//! Configuration files use the [TOML](https://toml.io/en/) format. -//! This is a textual `key=value` format that supports comments. +//! Configuration files use the same format as OpenSSH configuration files. +//! This is a textual `Key Value` format that supports comments. //! -//! **Note** Strings are quoted; booleans and integers are not. For example: +//! qcp supports `Host` directives with wildcard matching, and `Include` directives. +//! This allows you to tune your configuration for a range of network hosts. //! -//! ```toml -//! rx="5M" # we have 40Mbit download -//! tx=1000000 # we have 8Mbit upload; we could also have written this as "1M" -//! rtt=150 # servers we care about are an ocean away -//! congestion="bbr" # this works well for us +//! ### Example +//! +//! ```text +//! Host old-faithful +//! # This is an old server with a very limited CPU which we do not want to overstress +//! rx 125k +//! tx 0 +//! +//! Host *.internal.corp +//! # This is a nearby data centre which we have a dedicated 1Gbit connection to. +//! # We don't need to use qcp, but it's convenient to use one tool in our scripts. +//! rx 125M +//! tx 0 +//! rtt 10 +//! +//! # For all other hosts, try to maximise our VDSL +//! Host * +//! rx 5M # we have 40Mbit download +//! tx 1000000 # we have 8Mbit upload; we could also have written this as "1M" +//! rtt 150 # most servers we care about are an ocean away +//! congestion bbr # this works well for us //! ``` //! //! ## Configurable options //! //! The full list of supported fields is defined by [Configuration]. +//! +//! On the command line: +//! * `qcp --show-config` outputs a list of supported fields, their current values, and where each value came from. +//! * For an explanation of each field, refer to `qcp --help` . +//! * `qcp --config-files` outputs the list of configuration files for the current user and platform. +//! +//! ### Traps and tips +//! 1. Like OpenSSH, for each setting we use the value from the _first_ Host block we find that matches the remote hostname. +//! 1. Each setting is evaluated independently. +//! - In the example above, the `Host old-faithful` block sets an `rx` but does not set `rtt`. Any operations to `old-faithful` inherit `rtt 150` from the `Host *` block. +//! 1. The `tx` setting has a default value of 0, which means "use the active rx value". If you set `tx` in a `Host *` block, you probably want to set it explicitly everywhere you set `rx`. +//! +//! If you have a complicated config file we recommend you structure it as follows: +//! * Any global settings that are intended to apply to all hosts +//! * `Host` blocks; if you use wildcards, from most-specific to least-specific +//! * A `Host *` block to provide default settings to apply where no more specific value has been given +//! mod structure; pub use structure::Configuration; @@ -37,6 +71,6 @@ pub(crate) use structure::Configuration_Optional; mod manager; pub use manager::Manager; -pub(crate) const BASE_CONFIG_FILENAME: &str = "qcp.toml"; +pub(crate) const BASE_CONFIG_FILENAME: &str = "qcp.conf"; pub(crate) mod ssh; diff --git a/src/config/ssh/errors.rs b/src/config/ssh/errors.rs index c38995c..4e365fc 100644 --- a/src/config/ssh/errors.rs +++ b/src/config/ssh/errors.rs @@ -60,3 +60,22 @@ impl std::fmt::Display for SshConfigError { Ok(()) } } + +/// An iterator over all errors in an [`SshConfigError`] +pub(crate) struct IntoIter(::IntoIter); +impl Iterator for IntoIter { + type Item = SshConfigError; + + fn next(&mut self) -> Option { + self.0.next().map(std::convert::Into::into) + } +} + +impl IntoIterator for SshConfigError { + type Item = SshConfigError; + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + IntoIter(self.0.into_iter()) + } +} diff --git a/src/config/ssh/values.rs b/src/config/ssh/values.rs index dbe5039..3f581c1 100644 --- a/src/config/ssh/values.rs +++ b/src/config/ssh/values.rs @@ -44,7 +44,7 @@ impl figment::Provider for ValueProvider<'_> { Metadata::from( "configuration file", Source::Custom(format!( - "line {line} of {src}", + "{src} (line {line})", src = self.value.source, line = self.value.line_number )), diff --git a/src/util/port_range.rs b/src/util/port_range.rs index 8040cd6..7e9b356 100644 --- a/src/util/port_range.rs +++ b/src/util/port_range.rs @@ -12,11 +12,10 @@ use super::cli::IntOrString; /// /// Port 0 is allowed with the usual meaning ("any available port"), but 0 may not form part of a range. /// -/// In a configuration file, a range must be specified as a string. For example: -/// ```toml -/// remote_port=60000 # a single port can be an integer -/// remote_port="60000" # a single port can also be a string -/// remote_port="60000-60010" # a range must be specified as a string +/// In a configuration file, a range may specified as an integer or as a pair of ports. For example: +/// ```text +/// remote_port 60000 # a single port +/// remote_port 60000-60010 # a range /// ``` #[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize)] #[serde(from = "IntOrString", into = "String")] From eb935b69c2926565e2b308c2cf8d3f6f0898f6e2 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Tue, 24 Dec 2024 20:43:58 +1300 Subject: [PATCH 41/54] refactor: overhaul & simplify internal types - AddressFamily - - rename enum variants to Inet / Inet6 / Any - - serialize as "inet4" / "inet6" to align with ssh config - - accept aliases "4" / "6" - - no need to use IntOrString now we've changed config file format - PortRange - - no need to deserialise via IntOrString - - drop From - - derive Default - HumanU64 - - no need to deserialize from IntOrString - - ser/de as string for now - drop IntOrString as no longer used --- src/cli/args.rs | 4 +- src/util/address_family.rs | 139 +++++++++++++------------------------ src/util/cli.rs | 51 -------------- src/util/dns.rs | 4 +- src/util/humanu64.rs | 34 ++++----- src/util/mod.rs | 2 - src/util/port_range.rs | 25 ++----- 7 files changed, 69 insertions(+), 190 deletions(-) delete mode 100644 src/util/cli.rs diff --git a/src/cli/args.rs b/src/cli/args.rs index d6be003..5589a7d 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -100,9 +100,9 @@ impl CliArgs { CliArgs::from_arg_matches(&cli.get_matches_from(std::env::args_os())).unwrap(); // Custom logic: '-4' and '-6' convenience aliases if args.ipv4_alias__ { - args.config.address_family = Some(AddressFamily::V4); + args.config.address_family = Some(AddressFamily::Inet); } else if args.ipv6_alias__ { - args.config.address_family = Some(AddressFamily::V6); + args.config.address_family = Some(AddressFamily::Inet6); } args } diff --git a/src/util/address_family.rs b/src/util/address_family.rs index e342b4f..2b340c0 100644 --- a/src/util/address_family.rs +++ b/src/util/address_family.rs @@ -1,142 +1,97 @@ //! CLI helper - Address family // (c) 2024 Ross Younger -use std::fmt::Display; -use std::marker::PhantomData; use std::str::FromStr; -use figment::error::Actual; -use serde::Serialize; +use figment::error::{Actual, OneOf}; +use serde::{de, Deserialize, Serialize}; -use crate::util::cli::IntOrString; - -/// Representation an IP address family +/// Representation of an IP address family /// -/// This is a local type with special parsing semantics to take part in the config/CLI system. -#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +/// This is a local type with special parsing semantics and aliasing to take part in the config/CLI system. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum, Serialize)] +#[serde(rename_all = "kebab-case")] // to match clap::ValueEnum pub enum AddressFamily { /// IPv4 - #[value(name = "4")] - V4, + #[value(alias("4"), alias("inet4"))] + Inet, /// IPv6 - #[value(name = "6")] - V6, + #[value(alias("6"))] + Inet6, /// We don't mind what type of IP address Any, } -impl From for u8 { - fn from(value: AddressFamily) -> Self { - match value { - AddressFamily::V4 => 4, - AddressFamily::V6 => 6, - AddressFamily::Any => 0, - } - } -} - -impl Serialize for AddressFamily { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match *self { - AddressFamily::Any => serializer.serialize_str("any"), - t => serializer.serialize_u8(u8::from(t)), - } - } -} - -impl Display for AddressFamily { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if *self == AddressFamily::Any { - write!(f, "any") - } else { - write!(f, "{}", u8::from(*self)) - } - } -} - impl FromStr for AddressFamily { type Err = figment::Error; fn from_str(s: &str) -> Result { - if s == "4" { - Ok(AddressFamily::V4) - } else if s == "6" { - Ok(AddressFamily::V6) - } else if s == "0" || s == "any" { - Ok(AddressFamily::Any) - } else { - Err(figment::error::Kind::InvalidType(Actual::Str(s.into()), "4 or 6".into()).into()) - } - } -} - -impl TryFrom for AddressFamily { - type Error = figment::Error; - - fn try_from(value: u64) -> Result { - match value { - 4 => Ok(AddressFamily::V4), - 6 => Ok(AddressFamily::V6), - 0 => Ok(AddressFamily::Any), - _ => Err(figment::error::Kind::InvalidValue( - Actual::Unsigned(value.into()), - "4 or 6".into(), + match s { + "4" | "inet" | "inet4" => Ok(AddressFamily::Inet), + "6" | "inet6" => Ok(AddressFamily::Inet6), + "any" => Ok(AddressFamily::Any), + _ => Err(figment::error::Kind::InvalidType( + Actual::Str(s.into()), + OneOf(&["inet", "4", "inet6", "6"]).to_string(), ) .into()), } } } -impl<'de> serde::Deserialize<'de> for AddressFamily { +impl<'de> Deserialize<'de> for AddressFamily { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { - deserializer.deserialize_any(IntOrString(PhantomData)) + let s = String::deserialize(deserializer)?; + FromStr::from_str(&s).map_err(de::Error::custom) } } #[cfg(test)] mod test { + use std::str::FromStr; + use super::AddressFamily; #[test] fn serialize() { - let a = AddressFamily::V4; - let b = AddressFamily::V6; + let a = AddressFamily::Inet; + let b = AddressFamily::Inet6; + let c = AddressFamily::Any; let aa = serde_json::to_string(&a); let bb = serde_json::to_string(&b); - assert_eq!(aa.unwrap(), "4"); - assert_eq!(bb.unwrap(), "6"); + let cc = serde_json::to_string(&c); + assert_eq!(aa.unwrap(), "\"inet\""); + assert_eq!(bb.unwrap(), "\"inet6\""); + assert_eq!(cc.unwrap(), "\"any\""); } #[test] fn deser_str() { - let a: AddressFamily = serde_json::from_str(r#" "4" "#).unwrap(); - assert_eq!(a, AddressFamily::V4); - let a: AddressFamily = serde_json::from_str(r#" "6" "#).unwrap(); - assert_eq!(a, AddressFamily::V6); - } - - #[test] - fn deser_int() { - let a: AddressFamily = serde_json::from_str("4").unwrap(); - assert_eq!(a, AddressFamily::V4); - let a: AddressFamily = serde_json::from_str("6").unwrap(); - assert_eq!(a, AddressFamily::V6); + use AddressFamily::*; + for (str, expected) in &[ + ("4", Inet), + ("inet", Inet), + ("inet4", Inet), + ("6", Inet6), + ("inet6", Inet6), + ("any", Any), + ] { + let raw = AddressFamily::from_str(str).expect(str); + let json = format!(r#""{str}""#); + let output = serde_json::from_str::(&json).expect(str); + assert_eq!(raw, *expected); + assert_eq!(output, *expected); + } } #[test] fn deser_invalid() { - let _ = serde_json::from_str::("true").unwrap_err(); - let _ = serde_json::from_str::("5").unwrap_err(); - let _ = serde_json::from_str::(r#" "5" "#).unwrap_err(); - let _ = serde_json::from_str::("-1").unwrap_err(); - let _ = serde_json::from_str::(r#" "42" "#).unwrap_err(); - let _ = serde_json::from_str::(r#" "string" "#).unwrap_err(); + for s in &["true", "5", r#""5""#, "-1", r#""42"#, r#""string"#] { + let _ = serde_json::from_str::(s).expect_err(s); + } } } diff --git a/src/util/cli.rs b/src/util/cli.rs deleted file mode 100644 index d34a38b..0000000 --- a/src/util/cli.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! CLI generic serialization helpers -// (c) 2024 Ross Younger - -use std::{fmt, marker::PhantomData, str::FromStr}; - -use serde::{de, de::Visitor, Deserialize}; - -/// Deserialization helper for types which might reasonably be expressed as an -/// integer or a string. -/// -/// This is a Visitor that forwards string types to T's `FromStr` impl and -/// forwards int types to T's `From` or `From` impls. The `PhantomData` is to -/// keep the compiler from complaining about T being an unused generic type -/// parameter. We need T in order to know the Value type for the Visitor -/// impl. -#[allow(missing_debug_implementations)] -pub struct IntOrString(pub PhantomData T>); - -impl<'de, T> Visitor<'de> for IntOrString -where - T: Deserialize<'de> + TryFrom + FromStr, - ::Err: std::fmt::Display, - >::Error: std::fmt::Display, -{ - type Value = T; - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("int or string") - } - - fn visit_str(self, value: &str) -> Result - where - E: de::Error, - { - T::from_str(value).map_err(de::Error::custom) - } - - fn visit_u64(self, value: u64) -> Result - where - E: de::Error, - { - T::try_from(value).map_err(de::Error::custom) - } - - fn visit_i64(self, value: i64) -> Result - where - E: de::Error, - { - let u = u64::try_from(value).map_err(de::Error::custom)?; - T::try_from(u).map_err(de::Error::custom) - } -} diff --git a/src/util/dns.rs b/src/util/dns.rs index 875a34d..70a7e12 100644 --- a/src/util/dns.rs +++ b/src/util/dns.rs @@ -19,8 +19,8 @@ pub fn lookup_host_by_family(host: &str, desired: AddressFamily) -> anyhow::Resu let found = match desired { AddressFamily::Any => it.next(), - AddressFamily::V4 => it.find(|addr| addr.is_ipv4()), - AddressFamily::V6 => it.find(|addr| addr.is_ipv6()), + AddressFamily::Inet => it.find(|addr| addr.is_ipv4()), + AddressFamily::Inet6 => it.find(|addr| addr.is_ipv6()), }; found .map(std::borrow::ToOwned::to_owned) diff --git a/src/util/humanu64.rs b/src/util/humanu64.rs index b9e97df..ad0445c 100644 --- a/src/util/humanu64.rs +++ b/src/util/humanu64.rs @@ -1,7 +1,7 @@ //! Serialization helper type - u64 parseable by humanize_rs // (c) 2024 Ross Younger -use std::{marker::PhantomData, ops::Deref, str::FromStr}; +use std::{ops::Deref, str::FromStr}; use humanize_rs::bytes::Bytes; use serde::{ @@ -9,15 +9,13 @@ use serde::{ Serialize, }; -use super::cli::IntOrString; - -/// An integer field that may also be expressed using engineering prefixes (k, M, G, etc). +/// An integer field that may also be expressed using SI notation (k, M, G, etc). /// For example, `1k` and `1000` are the same. /// /// (Nerdy description: This is a newtype wrapper to `u64` that adds a flexible deserializer via `humanize_rs::bytes::Bytes`.) #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] -#[serde(from = "IntOrString", into = "u64")] +#[serde(from = "String", into = "String")] pub struct HumanU64(pub u64); impl HumanU64 { @@ -42,6 +40,12 @@ impl From for u64 { } } +impl From for String { + fn from(value: HumanU64) -> Self { + format!("{}", *value) + } +} + impl FromStr for HumanU64 { type Err = figment::Error; @@ -71,14 +75,8 @@ impl<'de> serde::Deserialize<'de> for HumanU64 { where D: serde::Deserializer<'de>, { - deserializer.deserialize_any(IntOrString(PhantomData)) - } -} - -#[cfg(test)] -impl rand::prelude::Distribution for rand::distributions::Standard { - fn sample(&self, rng: &mut R) -> HumanU64 { - rng.gen::().into() + let s = String::deserialize(deserializer)?; + FromStr::from_str(&s).map_err(de::Error::custom) } } @@ -105,21 +103,15 @@ mod test { test_deser_str("\"100k\"", 100_000); } - #[test] - fn deser_raw_int() { - let foo: HumanU64 = serde_json::from_str("12345").unwrap(); - assert_eq!(*foo, 12345); - } - #[test] fn serde_test() { let bw = HumanU64::new(42); - assert_tokens(&bw, &[Token::U64(42)]); + assert_tokens(&bw, &[Token::Str("42")]); } #[test] fn from_int() { - let result = HumanU64::from(12345); + let result = HumanU64::new(12345); assert_eq!(*result, 12345); } #[test] diff --git a/src/util/mod.rs b/src/util/mod.rs index fce6cc0..a4d9b53 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -10,8 +10,6 @@ pub use dns::lookup_host_by_family; mod cert; pub use cert::Credentials; -pub mod cli; - pub mod humanu64; pub mod io; pub mod socket; diff --git a/src/util/port_range.rs b/src/util/port_range.rs index 7e9b356..ddb1a48 100644 --- a/src/util/port_range.rs +++ b/src/util/port_range.rs @@ -1,13 +1,11 @@ /// CLI argument helper - PortRange // (c) 2024 Ross Younger use serde::{ - de::{Error, Unexpected}, + de::{self, Error, Unexpected}, Serialize, }; use std::{fmt::Display, str::FromStr}; -use super::cli::IntOrString; - /// A range of UDP port numbers. /// /// Port 0 is allowed with the usual meaning ("any available port"), but 0 may not form part of a range. @@ -17,8 +15,8 @@ use super::cli::IntOrString; /// remote_port 60000 # a single port /// remote_port 60000-60010 # a range /// ``` -#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize)] -#[serde(from = "IntOrString", into = "String")] +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, Serialize)] +#[serde(from = "String", into = "String")] pub struct PortRange { /// First number in the range pub begin: u16, @@ -83,20 +81,6 @@ impl FromStr for PortRange { } } -impl From for PortRange { - fn from(value: u64) -> Self { - #[allow(clippy::cast_possible_truncation)] - let v = value as u16; - PortRange { begin: v, end: v } - } -} - -impl Default for PortRange { - fn default() -> Self { - Self::from(0) - } -} - impl PortRange { pub(crate) fn is_default(self) -> bool { self.begin == 0 && self.begin == self.end @@ -108,7 +92,8 @@ impl<'de> serde::Deserialize<'de> for PortRange { where D: serde::Deserializer<'de>, { - deserializer.deserialize_any(IntOrString(std::marker::PhantomData)) + let s = String::deserialize(deserializer)?; + FromStr::from_str(&s).map_err(de::Error::custom) } } From 92089bedacf4f9fa0d9a694d3929d5f37557a45d Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 26 Dec 2024 10:59:56 +1300 Subject: [PATCH 42/54] feat: negative host matching --- src/config/ssh/matching.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/config/ssh/matching.rs b/src/config/ssh/matching.rs index 287c6c1..8b84770 100644 --- a/src/config/ssh/matching.rs +++ b/src/config/ssh/matching.rs @@ -1,10 +1,17 @@ //! Host matching // (c) 2024 Ross Younger +fn match_one_pattern(host: &str, pattern: &str) -> bool { + if let Some(negative_pattern) = pattern.strip_prefix('!') { + !wildmatch::WildMatch::new(negative_pattern).matches(host) + } else { + wildmatch::WildMatch::new(pattern).matches(host) + } +} + pub(super) fn evaluate_host_match(host: Option<&str>, args: &[String]) -> bool { if let Some(host) = host { - args.iter() - .any(|arg| wildmatch::WildMatch::new(arg).matches(host)) + args.iter().any(|arg| match_one_pattern(host, arg)) } else { // host is None i.e. unspecified; match only on '*' args.iter().any(|arg| arg == "*") @@ -45,6 +52,8 @@ mod test { ("oof", sv!["*of"], true), ("192.168.1.42", sv!["192.168.?.42"], true), ("192.168.10.42", sv!["192.168.?.42"], false), + ("xyzy", sv!["!xyzzy"], true), + ("xyzy", sv!["!xyzy"], false), ] { assert_eq_as_result!(evaluate_host_match(Some(host), &args), result) .map_err(|e| anyhow!(e)) From 02de3abe448d592df0ae61fa66915f041651362e Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 26 Dec 2024 11:00:14 +1300 Subject: [PATCH 43/54] feat: apply Postel's Law to config files - Allow fields in config files to be specified in randomised case, including snake or kebab. - Render fields in CamelCase to make the human experience closer to ssh. - Parse enums case insensitively --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/config/manager.rs | 16 ++++---- src/config/ssh/files.rs | 84 +++++++++++++++++++++++++++++++++++--- src/config/ssh/lines.rs | 2 +- src/config/structure.rs | 2 +- src/protocol/session.rs | 2 +- src/transport.rs | 26 +++++++++--- src/util/address_family.rs | 3 +- src/util/tracing.rs | 22 ++++++++-- 10 files changed, 133 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5cda830..cbfc9c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1114,7 +1114,7 @@ dependencies = [ "serde_test", "static_assertions", "struct-field-names-as-array", - "strum_macros", + "strum", "tabled", "tempfile", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 0d0c5eb..be71213 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,7 @@ rustls-pki-types = "1.10.0" serde = { version = "1.0.216", features = ["derive"] } static_assertions = "1.1.0" struct-field-names-as-array = "0.3.0" -strum_macros = "0.26.4" +strum = { version = "0.26.3", features = ["derive"]} tabled = "0.17.0" tokio = { version = "1.42.0", default-features = true, features = ["fs", "io-std", "macros", "process", "rt", "time", "sync"] } tokio-util = { version = "0.7.13", features = ["compat"] } diff --git a/src/config/manager.rs b/src/config/manager.rs index 2fee980..13b159e 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -6,6 +6,7 @@ use crate::os::{AbstractPlatform as _, Platform}; use super::{ssh::SshConfigError, Configuration}; use figment::{providers::Serialized, value::Value, Figment, Metadata, Provider}; +use heck::ToUpperCamelCase; use serde::Deserialize; use std::{ collections::HashSet, @@ -223,7 +224,7 @@ impl PrettyConfig { } } - fn new(field: &str, value: &Value, meta: Option<&Metadata>) -> Self { + fn new>(field: F, value: &Value, meta: Option<&Metadata>) -> Self { Self { field: field.into(), value: PrettyConfig::render_value(value), @@ -288,7 +289,7 @@ impl Display for DisplayAdapter<'_> { for field in keys { if let Ok(value) = data.find_value(field) { let meta = data.get_metadata(value.tag()); - output.push(PrettyConfig::new(field, &value, meta)); + output.push(PrettyConfig::new(field.to_upper_camel_case(), &value, meta)); } } write!( @@ -340,14 +341,14 @@ mod test { #[derive(Deserialize)] struct Test { - magic_: i32, + magic: i32, } let (path, _tempdir) = make_test_tempfile( r" rx true # invalid rtt 3.14159 # also invalid - magic_ 42 + magic 42 ", "test.conf", ); @@ -359,19 +360,20 @@ mod test { // But the config as a whole is not broken and other things can be extracted: let other_struct = mgr.get::().unwrap(); - assert_eq!(other_struct.magic_, 42); + assert_eq!(other_struct.magic, 42); } #[test] fn field_parse_failure() { #[derive(Debug, Deserialize)] + #[allow(dead_code)] struct Test { - _p: PortRange, + p: PortRange, } let (path, _tempdir) = make_test_tempfile( r" - _p 234-123 + p 234-123 ", "test.conf", ); diff --git a/src/config/ssh/files.rs b/src/config/ssh/files.rs index b38f43f..a1ea0bc 100644 --- a/src/config/ssh/files.rs +++ b/src/config/ssh/files.rs @@ -10,6 +10,8 @@ use std::{ use anyhow::{Context, Result}; use figment::Figment; +use lazy_static::lazy_static; +use struct_field_names_as_array::FieldNamesAsSlice as _; use tracing::warn; use super::{evaluate_host_match, find_include_files, split_args, Line, Setting, ValueProvider}; @@ -21,10 +23,26 @@ pub(crate) struct HostConfiguration { host: Option, /// If present, this is the file we read source: Option, - /// Output data + /// Output data. Field names have been canonicalised (see [`CanonicalIntermediate`]), + /// then mapped back to fields in [`super::super::Configuration`] if they match. data: BTreeMap, } +/// Creates a reverse mapping of intermediate-canonical keywords to field names for a struct. +fn create_field_name_map(fields: &'_ [&'_ str]) -> BTreeMap { + BTreeMap::::from_iter( + fields + .iter() + .map(|s| (CanonicalIntermediate::from(*s), (*s).to_string())) + .collect::>(), + ) +} + +lazy_static! { + static ref CONFIGURATION_FIELDS_MAP: BTreeMap = + create_field_name_map(crate::config::Configuration::FIELD_NAMES_AS_SLICE); +} + impl HostConfiguration { fn new(host: Option<&str>, source: Option) -> Self { Self { @@ -52,6 +70,40 @@ impl HostConfiguration { /////////////////////////////////////////////////////////////////////////////////////// +/// A keyword in an _intermediate canonical format_. +/// This format is lowercase and contains no underscores or hyphens. +/// +/// To convert from this format to snake case requires a lookup. +/// See [`CanonicalIntermediate::to_configuration_field`]. +#[derive(PartialOrd, Ord, PartialEq, Eq, Debug, Clone)] +struct CanonicalIntermediate(String); + +impl CanonicalIntermediate { + /// Attempt to reverse-map the canonicalised field to one from Configuration. + /// If the field is not known, return it unchanged. + fn to_configuration_field(&self) -> String { + CONFIGURATION_FIELDS_MAP + .get(self) + .unwrap_or(&self.0) + .clone() + } +} + +impl From<&str> for CanonicalIntermediate { + /// Converts a keyword into the inner canonical form defined by this module. + fn from(input: &str) -> Self { + Self( + input + .chars() + .map(|ch| ch.to_ascii_lowercase()) + .filter(|ch| *ch != '_' && *ch != '-') + .collect(), + ) + } +} + +/////////////////////////////////////////////////////////////////////////////////////// + /// The business end of reading a config file. /// /// # Note @@ -126,23 +178,23 @@ impl Parser { let mut splitter = line.splitn(2, &[' ', '\t', '=']); let keyword = match splitter.next() { None | Some("") => return Ok(Line::Empty), - Some(kw) => kw.to_lowercase(), + Some(kw) => CanonicalIntermediate::from(kw), }; (keyword, splitter.next().unwrap_or_default()) }; - if keyword.starts_with('#') { + if keyword.0.starts_with('#') { return Ok(Line::Empty); } let args = split_args(rest).with_context(|| format!("at line {line_number}"))?; anyhow::ensure!(!args.is_empty(), "missing argument at line {line_number}"); - Ok(match keyword.as_str() { + Ok(match keyword.0.as_str() { "host" => Line::Host { line_number, args }, "match" => Line::Match { line_number, args }, "include" => Line::Include { line_number, args }, _ => Line::Generic { line_number, - keyword, + keyword: keyword.to_configuration_field(), args, }, }) @@ -223,11 +275,13 @@ impl Parser { mod test { use anyhow::{anyhow, Context, Result}; use assertables::{assert_contains, assert_contains_as_result, assert_eq_as_result}; + use struct_field_names_as_array::FieldNamesAsSlice; - use super::super::Line; use super::Parser; + use super::{super::Line, CanonicalIntermediate}; use crate::{ + config::Configuration, os::{AbstractPlatform, Platform}, util::make_test_tempfile, }; @@ -287,6 +341,15 @@ mod test { "QUOTED \"abc def\" ghi", generic_("quoted", vec!["abc def", "ghi"]), ), + // Fields unknown to Configuration, are converted to CanonicalIntermediate: + ("kebab-case foo", generic_("kebabcase", vec!["foo"])), + ("snake_case foo", generic_("snakecase", vec!["foo"])), + ( + "RanDomcaPitaLiZATion foo", + generic_("randomcapitalization", vec!["foo"]), + ), + // Fields known to Configuration are resolved back to their names from the structure + ("AddressFamily foo", generic_("address_family", vec!["foo"])), ] { let msg = || format!("input \"{input}\" failed"); assert_eq_as_result!(p.parse_line(input).with_context(msg)?, expected) @@ -435,4 +498,13 @@ mod test { let data = parser.parse_file_for(Some("lapis")).unwrap(); println!("{data:#?}"); } + + #[test] + fn config_fields_pairwise() { + for f in Configuration::FIELD_NAMES_AS_SLICE { + let intermed = CanonicalIntermediate::from(*f); + let result = intermed.to_configuration_field(); + assert_eq!(result, *f); + } + } } diff --git a/src/config/ssh/lines.rs b/src/config/ssh/lines.rs index 0caa95e..5bb4e3c 100644 --- a/src/config/ssh/lines.rs +++ b/src/config/ssh/lines.rs @@ -21,7 +21,7 @@ pub(super) enum Line { }, Generic { line_number: usize, - keyword: String, /*lowercase!*/ + keyword: String, /*canonicalised!*/ args: Vec, }, } diff --git a/src/config/structure.rs b/src/config/structure.rs index c018110..5a0a637 100644 --- a/src/config/structure.rs +++ b/src/config/structure.rs @@ -107,7 +107,7 @@ pub struct Configuration { /// /// If unspecified, uses whatever seems suitable given the target address or the result of DNS lookup. // (see also [CliArgs::ipv4_alias__] and [CliArgs::ipv6_alias__]) - #[arg(long, alias("ipv"), help_heading("Connection"), group("ip address"))] + #[arg(long, help_heading("Connection"), group("ip address"))] pub address_family: AddressFamily, /// Specifies the ssh client program to use [default: `ssh`] diff --git a/src/protocol/session.rs b/src/protocol/session.rs index d0cb928..06a612a 100644 --- a/src/protocol/session.rs +++ b/src/protocol/session.rs @@ -47,7 +47,7 @@ use std::fmt::Display; use tokio_util::compat::TokioAsyncReadCompatExt as _; /// Command packet -#[derive(Debug, strum_macros::Display)] +#[derive(Debug, strum::Display)] #[allow(missing_docs)] pub enum Command { Get(GetArgs), diff --git a/src/transport.rs b/src/transport.rs index 3d1115a..9593d39 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -1,7 +1,7 @@ //! QUIC transport configuration // (c) 2024 Ross Younger -use std::{sync::Arc, time::Duration}; +use std::{str::FromStr, sync::Arc, time::Duration}; use anyhow::Result; use human_repr::HumanCount as _; @@ -9,7 +9,8 @@ use quinn::{ congestion::{BbrConfig, CubicConfig}, TransportConfig, }; -use serde::{Deserialize, Serialize}; +use serde::{de, Deserialize, Serialize}; +use strum::VariantNames; use tracing::debug; use crate::config::Configuration; @@ -36,13 +37,13 @@ pub enum ThroughputMode { Debug, PartialEq, Eq, - strum_macros::Display, + strum::Display, + strum::EnumString, + strum::VariantNames, clap::ValueEnum, Serialize, - Deserialize, )] -#[serde(rename_all = "kebab-case")] -#[strum(serialize_all = "kebab_case")] // I'm not entirely sure this does anything in this particular case +#[strum(serialize_all = "lowercase")] // N.B. this applies to EnumString, not Display pub enum CongestionControllerType { /// The congestion algorithm TCP uses. This is good for most cases. Cubic, @@ -57,6 +58,19 @@ pub enum CongestionControllerType { Bbr, } +impl<'de> Deserialize<'de> for CongestionControllerType { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let lower = s.to_ascii_lowercase(); + // requires strum::EnumString && strum::VariantNames && #[strum(serialize_all = "lowercase")] + FromStr::from_str(&lower) + .map_err(|_| de::Error::unknown_variant(&s, CongestionControllerType::VARIANTS)) + } +} + /// Creates a `quinn::TransportConfig` for the endpoint setup pub fn create_config(params: &Configuration, mode: ThroughputMode) -> Result> { let mut config = TransportConfig::default(); diff --git a/src/util/address_family.rs b/src/util/address_family.rs index 2b340c0..02f9f59 100644 --- a/src/util/address_family.rs +++ b/src/util/address_family.rs @@ -26,7 +26,8 @@ impl FromStr for AddressFamily { type Err = figment::Error; fn from_str(s: &str) -> Result { - match s { + let lc = s.to_ascii_lowercase(); + match lc.as_str() { "4" | "inet" | "inet4" => Ok(AddressFamily::Inet), "6" | "inet6" => Ok(AddressFamily::Inet6), "any" => Ok(AddressFamily::Any), diff --git a/src/util/tracing.rs b/src/util/tracing.rs index 3afaa0d..1c4b624 100644 --- a/src/util/tracing.rs +++ b/src/util/tracing.rs @@ -10,7 +10,8 @@ use std::{ use anstream::eprintln; use anyhow::Context; use indicatif::MultiProgress; -use serde::{Deserialize, Serialize}; +use serde::{de, Deserialize, Serialize}; +use strum::VariantNames as _; use tracing_subscriber::{ fmt::{ time::{ChronoLocal, ChronoUtc}, @@ -36,11 +37,13 @@ const LOG_FILE_DETAIL_ENV_VAR: &str = "RUST_LOG_FILE_DETAIL"; Default, Eq, PartialEq, - strum_macros::Display, + strum::Display, + strum::EnumString, + strum::VariantNames, clap::ValueEnum, Serialize, - Deserialize, )] +#[strum(serialize_all = "lowercase")] #[serde(rename_all = "kebab-case")] pub enum TimeFormat { /// Local time (as best as we can figure it out), as "year-month-day HH:MM:SS" @@ -56,6 +59,19 @@ pub enum TimeFormat { Rfc3339, } +impl<'de> Deserialize<'de> for TimeFormat { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let lower = s.to_ascii_lowercase(); + // requires strum::EnumString && strum::VariantNames && #[strum(serialize_all = "lowercase")] + std::str::FromStr::from_str(&lower) + .map_err(|_| de::Error::unknown_variant(&s, TimeFormat::VARIANTS)) + } +} + /// Result type for `filter_for()` struct FilterResult { filter: EnvFilter, From 2623bbbd0117d1c3c525287acfae2465dc1b7497 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 26 Dec 2024 16:20:55 +1300 Subject: [PATCH 44/54] fix: command-line takes precedence over config when a host is selected --- src/config/manager.rs | 11 ++++++++++- src/util/optionalify.rs | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/config/manager.rs b/src/config/manager.rs index 13b159e..6148d95 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -86,6 +86,13 @@ impl Manager { new1.add_config(true, "user", Platform::user_config_path(), for_host); new1 } + + /// Accessor (only used in tests at the moment) + #[cfg(test)] + fn host(&self) -> Option { + self.host.clone() + } + fn add_config( &mut self, is_user: bool, @@ -532,15 +539,17 @@ mod test { }; let (path, _tempdir) = make_test_tempfile( r" + Host foo rx 66666 ", "test.conf", ); - let mut mgr = Manager::without_files(None); + let mut mgr = Manager::without_files(Some("foo")); mgr.merge_ssh_config(&path, Some("foo"), false); // The order of merging mirrors what happens in Manager::try_from(&CliArgs) mgr.merge_provider(entered); + assert_eq!(mgr.host(), Some("foo".to_string())); let result = mgr.get::().unwrap(); assert_eq!(12345, *result.rx); } diff --git a/src/util/optionalify.rs b/src/util/optionalify.rs index f5c277d..bf8d4a8 100644 --- a/src/util/optionalify.rs +++ b/src/util/optionalify.rs @@ -109,7 +109,7 @@ define_derive_deftly! { ) let mut profile_map = Map::new(); - let _ = profile_map.insert(Profile::default(), dict); + let _ = profile_map.insert(Profile::Global, dict); Ok(profile_map) } From 8f0b958acd8f55b4e4757fc870d22bed09c2a1b9 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 26 Dec 2024 13:21:06 +1300 Subject: [PATCH 45/54] misc: rename ssh_opt to ssh_options, for better UX --- src/cli/args.rs | 2 +- src/client/control.rs | 2 +- src/config/manager.rs | 10 +++++----- src/config/structure.rs | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/cli/args.rs b/src/cli/args.rs index 5589a7d..536fb3a 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -38,7 +38,7 @@ pub(crate) struct CliArgs { conflicts_with_all([ "help_buffers", "show_config", "config_files", "quiet", "statistics", "remote_debug", "profile", - "ssh", "ssh_opt", "remote_port", + "ssh", "ssh_options", "remote_port", "source", "destination", ]) )] diff --git a/src/client/control.rs b/src/client/control.rs index c51a5a3..d900c1e 100644 --- a/src/client/control.rs +++ b/src/client/control.rs @@ -89,7 +89,7 @@ impl Channel { ConnectionType::Ipv4 => server.arg("-4"), ConnectionType::Ipv6 => server.arg("-6"), }; - let _ = server.args(&config.ssh_opt); + let _ = server.args(&config.ssh_options); let _ = server.args([ remote_host, "qcp", diff --git a/src/config/manager.rs b/src/config/manager.rs index 6148d95..df61777 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -395,15 +395,15 @@ mod test { fn ssh_style() { #[derive(Debug, Deserialize)] struct Test { - ssh_opt: Vec, + ssh_options: Vec, } // Bear in mind: in an ssh style config file, the first match for a particular keyword wins. let (path, _tempdir) = make_test_tempfile( r" host bar - ssh_opt d e f + ssh_options d e f host * - ssh_opt a b c + ssh_options a b c ", "test.conf", ); @@ -411,12 +411,12 @@ mod test { mgr.merge_ssh_config(&path, Some("foo"), false); //println!("{}", mgr.to_display_adapter::(false)); let result = mgr.get::().unwrap(); - assert_eq!(result.ssh_opt, vec!["a", "b", "c"]); + assert_eq!(result.ssh_options, vec!["a", "b", "c"]); let mut mgr = Manager::without_files(Some("bar")); mgr.merge_ssh_config(&path, Some("bar"), false); let result = mgr.get::().unwrap(); - assert_eq!(result.ssh_opt, vec!["d", "e", "f"]); + assert_eq!(result.ssh_options, vec!["d", "e", "f"]); } #[test] diff --git a/src/config/structure.rs b/src/config/structure.rs index 5a0a637..8b2a3be 100644 --- a/src/config/structure.rs +++ b/src/config/structure.rs @@ -128,7 +128,7 @@ pub struct Configuration { allow_hyphen_values(true), help_heading("Connection") )] - pub ssh_opt: Vec, + pub ssh_options: Vec, /// Uses the given UDP port or range on the remote endpoint. /// This can be useful when there is a firewall between the endpoints. @@ -259,7 +259,7 @@ impl Default for Configuration { // Client address_family: AddressFamily::Any, ssh: "ssh".into(), - ssh_opt: vec![], + ssh_options: vec![], remote_port: PortRange::default(), time_format: TimeFormat::Local, ssh_config: Vec::new(), From 4530565f219c0c008cea871bf490b048ff8dc89e Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Tue, 24 Dec 2024 21:50:04 +1300 Subject: [PATCH 46/54] build: add cargo doc task to include private items; fix that build --- .vscode/tasks.json | 13 +++++++++++++ src/client/job.rs | 2 +- src/client/options.rs | 4 ++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index af22b87..8364bf0 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -13,6 +13,19 @@ "isDefault": false }, "label": "rust: cargo doc" + }, + { + "type": "cargo", + "command": "doc", + "args": ["--no-deps", "--locked", "--document-private-items"], + "problemMatcher": [ + "$rustc" + ], + "group": { + "kind": "build", + "isDefault": false + }, + "label": "rust: cargo doc --document-private-items" } ] } diff --git a/src/client/job.rs b/src/client/job.rs index 4cc52d8..f2501f3 100644 --- a/src/client/job.rs +++ b/src/client/job.rs @@ -70,7 +70,7 @@ impl CopyJobSpec { } } - /// The [user@]hostname portion of whichever of the arguments contained a hostname. + /// The `[user@]hostname` portion of whichever of the arguments contained a hostname. fn remote_user_host(&self) -> &str { self.source .host diff --git a/src/client/options.rs b/src/client/options.rs index c38b807..e22dd5b 100644 --- a/src/client/options.rs +++ b/src/client/options.rs @@ -101,8 +101,8 @@ impl Parameters { /// A best-effort attempt to extract a single remote host string from the parameters. /// /// # Output - /// If neither source nor dest are present, Ok("") - /// If at most one of source and dest contains a remote host, Ok() + /// If neither source nor dest are present, `Ok("")` + /// If at most one of source and dest contains a remote host, `Ok()` /// /// # Errors /// If both source and dest contain a remote host, Err("Only one remote file argument is supported") From 14655bdb5dd494e5c7459b7b110e56767dad3698 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Tue, 24 Dec 2024 21:52:31 +1300 Subject: [PATCH 47/54] ci: set git_release_draft=true, update MAINTENANCE.md --- MAINTENANCE.md | 3 +++ release-plz.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/MAINTENANCE.md b/MAINTENANCE.md index e9a84ae..0335d5a 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -1,12 +1,15 @@ ## Creating a release * Create PR: + * _(Optional)_ `release-plz update` to preview updates to the changelog and version * ```release-plz release-pr --git-token $GITHUB_QCP_TOKEN``` * _if this token has expired, you'll need to generate a fresh one; walk back through the release-plz setup steps_ +* Review changelog, edit if necessary. * Merge the PR (rebase strategy preferred) * Delete the PR branch * `git fetch && git merge --ff-only` * Finalise the release: * ```release-plz release --git-token $GITHUB_QCP_TOKEN``` + * Check the new (draft) Github release page; update notes as necessary, publish when ready (when artifacts have all uploaded). * Merge `dev` into `main`, or whatever suits the current branching strategy * Check the docs built, follow up on the release workflow, etc. diff --git a/release-plz.toml b/release-plz.toml index b69e277..f01000c 100644 --- a/release-plz.toml +++ b/release-plz.toml @@ -1,7 +1,7 @@ [workspace] dependencies_update = false #allow_dirty = true -#git_release_draft = true +git_release_draft = true git_release_body = """ {{ changelog }} From c0b3e49cae1868ce21c1903ab2c2aa5cdbc8b2e1 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Tue, 24 Dec 2024 21:55:24 +1300 Subject: [PATCH 48/54] ci: build rust binaries with --locked --- .github/workflows/ci.yml | 3 ++- .github/workflows/release.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b7c764..34208e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: #- name: Build # run: cross build --release --locked --target ${{ matrix.target }} - - uses: taiki-e/upload-rust-binary-action@v1.22.1 + - uses: taiki-e/upload-rust-binary-action@v1.24.0 id: build with: bin: qcp @@ -66,6 +66,7 @@ jobs: target: ${{ matrix.target }} include: README.md,LICENSE,CHANGELOG.md leading-dir: true + locked: true tar: unix zip: windows dry_run: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2379eb8..e2310be 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,7 +49,7 @@ jobs: #- name: Build # run: cross build --release --locked --target ${{ matrix.target }} - - uses: taiki-e/upload-rust-binary-action@v1.22.1 + - uses: taiki-e/upload-rust-binary-action@v1.24.0 id: build with: bin: qcp @@ -57,6 +57,7 @@ jobs: target: ${{ matrix.target }} include: README.md,LICENSE,CHANGELOG.md leading-dir: true + locked: true tar: unix zip: windows dry_run: ${{ github.event_name != 'release' }} From 38bcaad554cf01748a20395665830db217375aa7 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 26 Dec 2024 11:01:28 +1300 Subject: [PATCH 49/54] docs: tidy up --help ordering, update man pages, tidy up doc comments Adds qcp_config(5). --- Cargo.toml | 1 + README.md | 10 +- misc/qcp.1 | 238 ++++++++++++++++++++++++------------- misc/qcp_config.5 | 182 ++++++++++++++++++++++++++++ src/cli/args.rs | 44 ++++--- src/cli/cli_main.rs | 1 + src/cli/mod.rs | 4 +- src/client/main_loop.rs | 6 +- src/client/meter.rs | 2 +- src/client/options.rs | 23 ++-- src/client/progress.rs | 40 ++++--- src/client/ssh.rs | 6 +- src/config/manager.rs | 10 +- src/config/mod.rs | 73 +++++++++--- src/config/structure.rs | 76 ++++++++---- src/doc/troubleshooting.rs | 18 +++ src/lib.rs | 10 +- src/os/unix.rs | 3 +- src/protocol/mod.rs | 2 +- src/util/address_family.rs | 2 +- src/util/dns.rs | 2 +- src/util/port_range.rs | 2 +- 22 files changed, 584 insertions(+), 171 deletions(-) create mode 100644 misc/qcp_config.5 diff --git a/Cargo.toml b/Cargo.toml index be71213..50ddae6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,6 +134,7 @@ assets = [ [ "misc/changelog.gz", "usr/share/doc/qcp/", "644" ], [ "misc/20-qcp.conf", "etc/sysctl.d/", "644" ], # this is automatically recognised as a conffile [ "misc/qcp.1", "usr/share/man/man1/", "644" ], + [ "misc/qcp_config.5", "usr/share/man/man5/", "644" ], ] maintainer-scripts="debian" depends = "$auto,debconf" diff --git a/README.md b/README.md index 6cd26b7..9e7df52 100644 --- a/README.md +++ b/README.md @@ -76,9 +76,17 @@ $ qcp my-server:/tmp/testfile /tmp/ testfile β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘ 1s @ 6.71 MB/s [10.49 MB] ``` -**The program uses the ssh binary on your system to connect to the target machine**. +Things you should know: + +* **qcp uses the ssh binary on your system to connect to the target machine**. ssh will check the remote host key and prompt you for a password or passphrase in the usual way. +* **qcp will read your ssh config file** to resolve any Hostname aliases you may have defined there. +The idea is, if you can `ssh` to a host, you should also be able to `qcp` to it. +However, some particularly complicated ssh config files may be too much for qcp to understand. +(In particular, `Match` directives are not currently supported.) +In that case, you can use `--ssh-config` to provide an alternative configuration (or set it in your qcp configuration file). + #### Tuning By default qcp is tuned for a 100Mbit connection, with 300ms round-trip time to the target server. diff --git a/misc/qcp.1 b/misc/qcp.1 index 90cd1d2..8b087e9 100644 --- a/misc/qcp.1 +++ b/misc/qcp.1 @@ -1,59 +1,67 @@ .ie \n(.g .ds Aq \(aq .el .ds Aq ' -.TH qcp 1 "qcp v0.1" +.TH qcp 1 "qcp v0.2" .SH NAME -qcp \- Secure remote file copy utility which uses the QUIC protocol over UDP +qcp β€” Secure remote file copy utility which uses the QUIC protocol over UDP .SH SYNOPSIS \fBqcp\fR [\fB-46qsd\fR] [\fB--ssh\fR ssh-command] [\fB-S ssh-options\fR] [\fB-p local-port-range\fR] [\fB-P remote-port-range\fR] -[\fB--rx-bw bandwidth\fR] -[\fB--tx-bw bandwidth\fR] +[\fB--rx bandwidth\fR] +[\fB--tx bandwidth\fR] [\fB-r rtt\fR] [\fB-t timeout\fR] +[\fI\fR] <\fISOURCE\fR> <\fIDESTINATION\fR> .TP \fBqcp\fR [-h|--help|--help-buffers|-V|--version] .SH DESCRIPTION .TP -The QUIC Copier (\fIqcp\fR) is an experimental high-performance remote file copy utility for long-distance internet connections. -.TP -It is intended as a drop-in replacement for scp. +The QUIC Copier (\fIqcp\fR) is an experimental high-performance remote file copy utility for long-distance internet connections. It is intended as a drop-in replacement for scp. .TP qcp offers similar security to scp using existing, well-known mechanisms, and better throughput on congested networks. .SH LIMITATIONS .TP -.RS 14 .IP \(bu 2 -You must be able to ssh directly to the remote machine, and exchange UDP packets with it on a given port. (If the local machine is behind connection-tracking NAT, things work just fine. This is the case for the vast majority of home and business network connections.) +You must be able to ssh directly to the remote machine, and exchange UDP packets with it on a given port. +(If the local machine is behind connection-tracking NAT, things work just fine. +This is the case for the vast majority of home and business network connections.) .IP \(bu 2 -Network security systems can’t readily identify QUIC traffic as such. It’s opaque, and high bandwidth. Some security systems might flag it as a potential threat. -.RE +Network security systems can’t readily identify QUIC traffic as such. +It’s opaque, and high bandwidth. +Some security systems might flag it as a potential threat. .SH ARGUMENTS .TP <\fISOURCE\fR> -The source file. This may be a local filename, or remote specified as HOST:FILE or USER@HOST:FILE. +The source file. +This may be a local filename, or remote specified as HOST:FILE or USER@HOST:FILE. -Exactly one of source and destination must be remote. .TP <\fIDESTINATION\fR> -Destination. This may be a file or directory. It may be local or remote. +Destination. +This may be a file or directory. It may be local or remote. If remote, specify as HOST:DESTINATION or USER@HOST:DESTINATION; or simply HOST: or USER@HOST: to copy to your home directory there. -Exactly one of source and destination must be remote. - -.SH OPTIONS .TP -\fB\-q\fR, \fB\-\-quiet\fR -Quiet mode +Exactly one of \fIsource\fR and \fIdestination\fR must be remote. .TP -\fB\-s\fR, \fB\-\-statistics\fR -Outputs additional transfer statistics +qcp will read your ssh config file to resolve any host name aliases you may have defined. The idea is, if you can ssh directly to a given host, you should be able to qcp to it by the same name. However, some particularly complicated ssh config files may be too much for qcp to understand. (In particular, \fIMatch\fR directives are not currently supported.) In that case, you can use \fI--ssh-config\fR to provide an alternative configuration (or set it in your qcp configuration file). + +.SH CONFIGURATION +Many of qcp's configuration options may be set persistently via configuration files. +See \fBqcp_config\fR(5) for details. + +.SH OPTIONS + +\fINote: this man page is currently maintained by hand.\fR +In the event of discrepancies or confusion, the help text within the program (\fIqcp --help\fR) +is more likely to be correct. + .TP \fB\-h\fR, \fB\-\-help\fR Print help (see a summary with \*(Aq\-h\*(Aq) @@ -61,51 +69,105 @@ Print help (see a summary with \*(Aq\-h\*(Aq) \fB\-V\fR, \fB\-\-version\fR Print version -.SH Network tuning options +.SS Network tuning options .TP -\fB\-r\fR, \fB\-\-rtt\fR=\fIms\fR [default: 300] -The expected network Round Trip time to the target system, in milliseconds -.TP -\fB\-b\fR, \fB\-\-rx\-bw\fR=\fIbytes\fR [default: 12500k] +\fB\-b\fR, \fB\-\-rx\fR=\fIbytes\fR [default: 12500k] The maximum network bandwidth we expect receiving data FROM the remote system. -This may be specified directly as a number of bytes, or as an SI quantity e.g. "10M" or "256k". Note that this is described in BYTES, not bits; if (for example) you expect to fill a 1Gbit ethernet connection, 125M might be a suitable setting. +This may be specified directly as a number of bytes, or as an SI quantity e.g. "10M" or "256k". +Note that this is described in \fIBYTES\fR, not bits; if (for example) you expect to fill a 1Gbit ethernet connection, 125M might be a suitable setting. .TP -\fB\-B\fR, \fB\-\-tx\-bw\fR=\fIbytes\fR -The maximum network bandwidth we expect sending data TO the remote system, -if it is different from the bandwidth FROM the system. (For example, when you are connected via an asymmetric last\-mile DSL or fibre profile.) [default: same as \-\-rx\-bw] +\fB\-B\fR, \fB\-\-tx\fR=\fIbytes\fR +The maximum network bandwidth (in bytes) we expect sending data TO the remote system, +if it is different from the bandwidth FROM the system. +(For example, when you are connected via an asymmetric last\-mile DSL or fibre profile.) +[default: same as \-\-rx\-bw] +.TP +\fB\-r\fR, \fB\-\-rtt\fR=\fIms\fR [default: 300] +The expected network Round Trip time to the target system, in milliseconds .TP \fB\-\-help\-buffers\fR Outputs additional information about kernel UDP buffer sizes and platform\-specific tips -.SH Connection options +.SS Advanced network tuning .TP -\fB\-4\fR, \fB\-\-ipv4\fR +\fB\-\-congestion\fR=\fIalg\fR [default: cubic] +Specifies the congestion control algorithm to use +.br + +.br +\fIPossible values:\fR +.RS 8 +.IP \(bu 2 +cubic: The congestion algorithm TCP uses. This is good for most cases +.IP \(bu 2 +bbr: (Use with caution!) An experimental algorithm created by Google, which increases goodput in some situations (particularly long and fat connections where the intervening buffers are shallow). +However this comes at the cost of having more data in\-flight, and much greater packet retransmission. + +See +.UR https://blog.apnic.net/2020/01/10/when\-to\-use\-and\-not\-use\-bbr/ +.UE +for more discussion. +.RE +.TP +\fB\-\-initial\-congestion\-window\fR=\fIbytes\fR +(Network wizards only!) The initial value for the sending congestion control window. + +Setting this value too high reduces performance! + +If not specified, this setting is determined by the selected congestion control algorithm. + +.SS Connection options +.TP +\fB\-4\fR Forces IPv4 connection [default: autodetect] + .TP -\fB\-6\fR, \fB\-\-ipv6\fR +\fB\-6\fR Forces IPv6 connection [default: autodetect] +.TP +\fB\-\-address\-family\fR=\fIoption\fR [default: any] + +\fIPossible values:\fR +.RS 8 +.IP \(bu 2 +inet: IPv4 +.IP \(bu 2 +inet6: IPv6 +.IP \(bu 2 +any: Unspecified. qcp will use whatever seems suitable given the target address or the result of DNS lookup +.RE .TP -\fB\-\-ssh\fR=\fISSH\fR [default: ssh] -Specifies the ssh client program to use +\fB\-p\fR, \fB\-\-port\fR=\fIM\-N\fR +Uses the given UDP port or range on the local endpoint. + +This can be useful when there is a firewall between the endpoints. +.TP +\fB\-P\fR, \fB\-\-remote\-port\fR=\fIM\-N\fR +Uses the given UDP port or range on the remote endpoint. +This can be useful when there is a firewall between the endpoints. + .TP \fB\-S\fR=\fIssh\-option\fR Provides an additional option or argument to pass to the ssh client. -Note that you must repeat `\-S` for each. For example, to pass `\-i /dev/null` to ssh, specify: `\-S \-i \-S /dev/null` +Note that you must repeat \fI\-S\fR for each. For example, to pass \fI\-i /dev/null\fR to ssh, specify: \fI\-S \-i \-S /dev/null\fR .TP -\fB\-P\fR, \fB\-\-remote\-port\fR=\fIM\-N\fR -Uses the given UDP port or range on the remote endpoint. +\fB\-\-ssh\fR=\fISSH\fR [default: ssh] +Specifies the ssh client program to use -This can be useful when there is a firewall between the endpoints. .TP -\fB\-p\fR, \fB\-\-port\fR=\fIM\-N\fR -Uses the given UDP port or range on the local endpoint. +\fB\-\-ssh\-config\fR=\fIFILE\fR +Alternative ssh config file(s) + +By default, qcp reads your user and system ssh config files to look for Hostname aliases. +In some cases the logic in qcp may not read them successfully; this is an escape hatch, allowing you to specify one or more alternative files to read instead (which may be empty, nonexistent or \fI/dev/null\fR). + +This option is really intended to be used in a qcp configuration file. On the command line, you can repeat \fI\-\-ssh\-config file\fR as many times as needed. -This can be useful when there is a firewall between the endpoints. .TP \fB\-t\fR, \fB\-\-timeout\fR=\fIsec\fR [default: 5] Connection timeout for the QUIC endpoints. @@ -113,85 +175,99 @@ Connection timeout for the QUIC endpoints. This needs to be long enough for your network connection, but short enough to provide a timely indication that UDP may be blocked. -.SH Debug options +.SS Output options + .TP -\fB\-\-remote\-debug\fR -Enables detailed debug output from the remote endpoint +\fB\-l\fR, \fB\-\-log\-file\fR +Logs to a file. By default the log receives everything printed to stderr. To override this behaviour, set the environment variable \fIRUST_LOG_FILE_DETAIL\fR (same semantics as \fIRUST_LOG\fR). + .TP \fB\-\-profile\fR Prints timing profile data after completion -.TP -\fB\-d\fR, \fB\-\-debug\fR -Enable detailed debug output -This has the same effect as setting `RUST_LOG=qcp=debug` in the environment. If present, `RUST_LOG` overrides this option. .TP -\fB\-l\fR, \fB\-\-log\-file\fR=\fIFILE\fR -Log to a file - -By default the log receives everything printed to stderr. To override this behaviour, set the environment variable `RUST_LOG_FILE_DETAIL` (same semantics as `RUST_LOG`). - +\fB\-s\fR, \fB\-\-statistics\fR +Outputs additional transfer statistics -.SH Advanced network options .TP -\fB\-\-congestion\fR=\fIalg\fR [default: cubic] -Specifies the congestion control algorithm to use -.br +\fB\-T\fR, \fB\-\-time\-format\fR +Specifies the time format to use when printing messages to the console or to file [default: local] -.br \fIPossible values:\fR -.RS 14 +.RS 8 .IP \(bu 2 -cubic: The congestion algorithm TCP uses. This is good for most cases +local: Local time (as best as we can figure it out), as "year-month-day HH:MM:SS" .IP \(bu 2 -bbr: (Use with caution!) An experimental algorithm created by Google, which increases goodput in some situations (particularly long and fat connections where the intervening buffers are shallow). However this comes at the cost of having more data in\-flight, and much greater packet retransmission. See `https://blog.apnic.net/2020/01/10/when\-to\-use\-and\-not\-use\-bbr/` for more discussion +utc: UTC time, as "year-month-day HH:MM:SS" +.IP \(bu 2 +rfc3339: UTC time, in the format described in RFC3339 .RE + .TP -\fB\-\-initial\-congestion\-window\fR=\fIbytes\fR -(Network wizards only!) The initial value for the sending congestion control window. +\fB\-q\fR, \fB\-\-quiet\fR +Quiet mode -Setting this value too high reduces performance! +.SS Configuration options -If not specified, this setting is determined by the selected congestion control algorithm. +.TP +\fB\-\-config\-files\fR +Outputs the paths to configuration file(s), then exits + +.TP +\fB\-\-show\-config\fR +Outputs the configuration, then exits. + +If a remote \fISOURCE\fR or \fIDESTINATION\fR argument is given, outputs the configuration +we would use for operations to that host. + +If not, outputs only global settings from configuration, which may be overridden in +\fIHost\fR blocks in configuration files. + +.SS Debug options +.TP +\fB\-d\fR, \fB\-\-debug\fR +Enable detailed debug output + +This has the same effect as setting \fIRUST_LOG=qcp=debug\fR in the environment. If present, \fIRUST_LOG\fR overrides this option. + +.TP +\fB\-\-remote\-debug\fR +Enables detailed debug output from the remote endpoint .SH EXIT STATUS The qcp utility exits 0 on success, and >0 if an error occurs. .SH PROTOCOL -qcp is a \fIhybrid\fR protocol. We use \fIssh\fR to establish a control channel and exchange ephemeral TLS certificates, -then a \fIQUIC\fR connection to transport data. +qcp is a \fIhybrid\fR protocol. +We use \fIssh\fR to establish a control channel and exchange ephemeral TLS certificates, then a \fIQUIC\fR connection to transport data. Detailed protocol documentation can be found at -.nh -https://docs\.rs/qcp/latest/qcp/protocol/ -.hy -. +.UR https://docs\.rs/qcp/latest/qcp/protocol/ +.UE . .SH PERFORMANCE TUNING See -.nh -https://docs.rs/qcp/latest/qcp/doc/performance/index.html -.hy +.UR https://docs.rs/qcp/latest/qcp/doc/performance/index.html +.UE .SH TROUBLESHOOTING See -.nh -https://docs.rs/qcp/latest/qcp/doc/troubleshooting/index.html -.hy +.UR https://docs.rs/qcp/latest/qcp/doc/troubleshooting/index.html +.UE .SH SEE ALSO .sp -\fBssh(1)\fP, \fI\%RFC 4254\fP, \fI\%RFC 9000\fP, \fI\%RFC 9001\fP. +\fBssh\fR(1), \fBqcp_config\fR(5), \fI\%RFC 4254\fP, \fI\%RFC 9000\fP, \fI\%RFC 9001\fP. .SH AUTHOR Ross Younger .SH BUGS Please report via the issue tracker: -.nh +.UR https://github\.com/crazyscot/qcp/issues -.hy +.UE .SH CAVEATS diff --git a/misc/qcp_config.5 b/misc/qcp_config.5 new file mode 100644 index 0000000..9354336 --- /dev/null +++ b/misc/qcp_config.5 @@ -0,0 +1,182 @@ +.ie \n(.g .ds Aq \(aq +.el .ds Aq ' +.TH qcp_config 5 "December 2024" +.SH NAME +qcp_config β€” Configuration options for qcp +.SH DESCRIPTION +\fBqcp\fR(1) obtains run-time configuration from the following sources, in order: + +.RS 0 +.IP +1. Command-line options +.IP +2. The user's configuration file (typically \fI~/.qcp.conf\fR) +.IP +3. The system-wide configuration file (typically \fI/etc/qcp.conf\fR) +.IP +4. Hard-wired defaults +.RE + +Each option may appear in multiple places, but only the first match is used. + +\fBNote:\fR Configuration file locations are platform-dependent. To see what applies on the current platform, run \fIqcp --config-files\fR. + +.SH FILE FORMAT + +qcp uses the same basic format as OpenSSH configuration files. + +Each line contains a keyword and one or more arguments. +Keywords are single words and are case-insensitive. + +Arguments are separated from keywords, and each other, by whitespace. +(It is also possible to write \fIKey=Value\fR or \fIKey = Value\fR.) + +Arguments may be surrounded by double quotes ("); this allows you to set an argument containing spaces. +If a backslash, double or single quote forms part of an argument it must be backslash-escaped i.e. \\" or \\\\ + +Comments are supported; # introduces a comment (up to the end of the line). + +Empty lines are ignored. + +qcp supports \fIHost\fR directives with wildcard and negative matching, as well as \fIInclude\fR directives. This allows you to tune your configuration for a range of network hosts. + +.SH EXAMPLE + Host old-faithful + # This is an old server with a very limited CPU which we do not want to overstress + rx 125k + tx 0 + + Host *.internal.corp + # This is a nearby data centre which we have a dedicated 1Gbit connection to. + # We don't need to use qcp, but it's convenient to use one tool in our scripts. + rx 125M + tx 0 + rtt 10 + + # For all other hosts, try to maximise our VDSL + Host * + rx 5M # we have 40Mbit download + tx 1000000 # we have 8Mbit upload; we could also have written this as "1M" + rtt 150 # most servers we care about are an ocean away + congestion bbr # this works well for us + +.SH CONFIGURATION DIRECTIVES + +.TP +\fBHost\fR \fIpattern [pattern ...]\fR +Introduces a \fIhost block\fR. +All following options - up to the next Host - only apply to hosts matching any of the patterns given. + +Pattern matching uses '*' and '?' as wildcards in the usual way. + +A single asterisk '*' matches all hosts; this is used to provide defaults. + +A pattern beginning with '!' is a \fInegative\fR match; it matches all remote hosts \fIexcept\fR those matching the rest of the pattern. + +Pattern matching is applied directly to the remote host given on the QCP command line, before DNS or alias resolution. +If you connect to hosts by IP address, a pattern of \fI10.11.12.*\fR works in the obvious way. + +.TP +\fBInclude\fR \fIfile [file ...]\fR + +Include the specified file(s) in the configuration at the current point. Glob wildcards ('*' and '?') are supported in filenames. + +User configuration files may refer to pathnames relative to '~' (the user's home directory). + +Filenames with relative paths are assumed to be in \fI~/.ssh/\fR if read from a user configuration file, or \fI/etc/ssh/\fR if read from a system configuration file. + +An Include directive inside a Host block retains the Host context. +This may be useful to apply common directives to multiple hosts with minimal repetition. +Note that if an included file begins a new Host block, that will continue to apply on return to the including file. + +It is possible for included files to themselves include additional files; there is a brake that prevents infinite recursion. + +.SH CONFIGURATION OPTIONS + +The following options from the CLI are supported in configuration files: + +\fIrx, tx, rtt, congestion, initial_congestion_window, port, timeout, address_family, ssh, ssh_options, remote_port, time_format, ssh_config\fR + +Refer to \fBqcp\fR(1) for details. + +In configuration files, option keywords are case insensitive and ignore hyphens and underscores. +(On the command line, options must be specified in kebab-case.) +For example, these declarations are all equivalent: + RemotePort 12345 + remoteport 12345 + remote_port 12345 + Remote_Port 12345 + ReMoTePoRt 12345 + rEmOtE-pOrT 12345 + +.SH CONFIGURATION EXPLAINER + +As configurations can get quite complex, it may be useful to understand where a particular value came from. + +qcp will do this for you, with the \fI--show-config\fR option. +Provide a source and destination as if you were copying a file to/from a host to see the configuration that would apply. For example: + + $ qcp --show-config myserver:some-file /tmp/ + + .---------------------------.-------------.------------------------------. + | field | value | source | + .---------------------------.-------------.------------------------------. + | (Remote host) | myserver | | + | AddressFamily | inet | /home/xyz/.qcp.conf (line 9) | + | Congestion | cubic | default | + | InitialCongestionWindow | 0 | default | + | Port | 0 | default | + | RemotePort | 60500-61000 | /home/xyz/.qcp.conf (line 8) | + | Rtt | 300 | /etc/qcp.conf (line 4) | + | Rx | 38M | /etc/qcp.conf (line 2) | + | Ssh | ssh | default | + | SshConfig | [] | default | + | SshOption | [] | default | + | TimeFormat | local | default | + | Timeout | 5 | default | + | Tx | 12M | /etc/qcp.conf (line 3) | + .---------------------------.-------------.------------------------------. + +.SH TIPS AND TRAPS +1. Like OpenSSH, for each setting we use the value from the \fIfirst\fR Host block we find that matches the remote hostname. + +2. Each setting is evaluated independently. +In the example above, the \fIHost old-faithful\fR block sets rx but does not set rtt. +Any operations to old-faithful inherit \fIrtt 150\fR from the Host * block. + +3. The tx setting has a default value of 0, which means β€œuse the active rx value”. +\fIIf you set tx in a Host * block, you probably want to set it explicitly everywhere you set rx.\fR + +If you have a complicated config file we suggest you structure it as follows: +.RS 0 +.IP +1. Any global settings that are intended to apply to all hosts +.IP +2. Host blocks; if you use wildcards, from most-specific to least-specific +.IP +3. A Host * block to provide default settings to apply where no more specific value has been given +.RE + +.SH FILES + +.TP +~/.qcp.conf +The user configuration file (on most platforms) + +.TP +/etc/qcp.conf +The system configuration file (on most platforms) + +.TP +~/.ssh/ssh_config +The user ssh configuration file + +.TP +/etc/ssh/ssh_config +The system ssh configuration file + +.SH SEE ALSO +\fBqcp\fR(1), \fBssh_config\fR(5) + +.SH AUTHOR +Ross Younger diff --git a/src/cli/args.rs b/src/cli/args.rs index 536fb3a..974608c 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -1,4 +1,4 @@ -// QCP top-level command-line arguments +//! Command-line argument definition and processing // (c) 2024 Ross Younger use clap::{ArgAction::SetTrue, Args as _, FromArgMatches as _, Parser}; @@ -8,12 +8,18 @@ use crate::{config::Manager, util::AddressFamily}; /// Options that switch us into another mode i.e. which don't require source/destination arguments pub(crate) const MODE_OPTIONS: &[&str] = &["server", "help_buffers", "config_files", "show_config"]; +/// CLI argument definition #[derive(Debug, Parser, Clone)] #[command( author, // we set short/long version strings explicitly, see custom_parse() about, - before_help = "e.g. qcp some/file my-server:some-directory/", + before_help = r"e.g. qcp some/file my-server:some-directory/ + + Exactly one of source and destination must be remote. + + qcp will read your ssh config file to resolve any host name aliases you may have defined. The idea is, if you can ssh directly to a given host, you should be able to qcp to it by the same name. However, some particularly complicated ssh config files may be too much for qcp to understand. (In particular, Match directives are not currently supported.) In that case, you can use --ssh-config to provide an alternative configuration (or set it in your qcp configuration file). + ", infer_long_args(true) )] #[command(help_template( @@ -29,7 +35,7 @@ pub(crate) const MODE_OPTIONS: &[&str] = &["server", "help_buffers", "config_fil #[allow(clippy::struct_excessive_bools)] pub(crate) struct CliArgs { // MODE SELECTION ====================================================================== - /// Operates in server mode. + /// Operate in server mode. /// /// This is what we run on the remote machine; it is not /// intended for interactive use. @@ -44,6 +50,13 @@ pub(crate) struct CliArgs { )] pub server: bool, + // CONFIGURABLE OPTIONS ================================================================ + #[command(flatten)] + /// The set of options which may be set in a config file or via command-line. + pub config: crate::config::Configuration_Optional, + + // MODE SELECTION, part 2 ============================================================== + // (These are down here to control clap's output ordering.) /// Outputs the configuration, then exits. /// /// If a remote `SOURCE` or `DESTINATION` argument is given, outputs the configuration we would use @@ -52,41 +65,44 @@ pub(crate) struct CliArgs { /// If not, outputs only global settings from configuration, which may be overridden in /// `Host` blocks in configuration files. /// - #[arg(long, help_heading("Configuration"))] + #[arg(long, help_heading("Configuration"), display_order(0))] pub show_config: bool, /// Outputs the paths to configuration file(s), then exits - #[arg(long, help_heading("Configuration"))] + #[arg(long, help_heading("Configuration"), display_order(0))] pub config_files: bool, /// Outputs additional information about kernel UDP buffer sizes and platform-specific tips - #[arg(long, action, help_heading("Network tuning"), display_order(50))] + #[arg(long, action, help_heading("Network tuning"), display_order(100))] pub help_buffers: bool, - // CONFIGURABLE OPTIONS ================================================================ - #[command(flatten)] - pub config: crate::config::Configuration_Optional, - // CLIENT-SIDE NON-CONFIGURABLE OPTIONS ================================================ // (including positional arguments!) #[command(flatten)] + /// The set of options which may only be provided via command-line. pub client_params: crate::client::Parameters, - /// Convenience alias for `--address-family 4` + /// Forces use of IPv4 + /// + /// This is a convenience alias for `--address-family inet` // this is actioned by our custom parser #[arg( short = '4', help_heading("Connection"), group("ip address"), - action(SetTrue) + action(SetTrue), + display_order(0) )] pub ipv4_alias__: bool, - /// Convenience alias for `--address-family 6` + /// Forces use of IPv6 + /// + /// This is a convenience alias for `--address-family inet6` // this is actioned by our custom parser #[arg( short = '6', help_heading("Connection"), group("ip address"), - action(SetTrue) + action(SetTrue), + display_order(0) )] pub ipv6_alias__: bool, } diff --git a/src/cli/cli_main.rs b/src/cli/cli_main.rs index 4b5bfa7..2d8ffcb 100644 --- a/src/cli/cli_main.rs +++ b/src/cli/cli_main.rs @@ -17,6 +17,7 @@ use indicatif::{MultiProgress, ProgressDrawTarget}; use owo_colors::OwoColorize as _; use tracing::error_span; +/// Computes the trace level for a given set of [ClientParameters] fn trace_level(args: &ClientParameters) -> &str { if args.debug { "debug" diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 447d74e..647c8c1 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1,5 +1,5 @@ -/// Command Line Interface for qcp -/// (c) 2024 Ross Younger +//! Command Line Interface for qcp +// (c) 2024 Ross Younger mod args; mod cli_main; pub(crate) mod styles; diff --git a/src/client/main_loop.rs b/src/client/main_loop.rs index 60d89dd..eb3e98d 100644 --- a/src/client/main_loop.rs +++ b/src/client/main_loop.rs @@ -1,4 +1,4 @@ -// qcp client event loop +//! Main client mode event loop // (c) 2024 Ross Younger use crate::{ @@ -30,6 +30,7 @@ use tracing::{debug, error, info, span, trace, trace_span, warn, Instrument as _ use super::job::CopyJobSpec; use super::Parameters as ClientParameters; +/// a shared definition string used in a couple of places const SHOW_TIME: &str = "file transfer"; /// Main client mode event loop @@ -224,6 +225,7 @@ async fn manage_request( } } +/// Adds a progress bar to the stack (in `MultiProgress`) for the current job fn progress_bar_for( display: &MultiProgress, job: &CopyJobSpec, @@ -301,6 +303,7 @@ pub(crate) fn create_endpoint( Ok(endpoint) } +/// Actions a GET command async fn do_get( sp: RawStreamPair, job: &CopyJobSpec, @@ -366,6 +369,7 @@ async fn do_get( Ok(header.size) } +/// Actions a PUT command async fn do_put( sp: RawStreamPair, job: &CopyJobSpec, diff --git a/src/client/meter.rs b/src/client/meter.rs index 2532c35..60092c2 100644 --- a/src/client/meter.rs +++ b/src/client/meter.rs @@ -1,4 +1,4 @@ -// Instant progress read-out +//! Instant progress read-out // (c) 2024 Ross Younger //! # Rationale diff --git a/src/client/options.rs b/src/client/options.rs index e22dd5b..2699c8a 100644 --- a/src/client/options.rs +++ b/src/client/options.rs @@ -12,14 +12,22 @@ pub struct Parameters { /// /// This has the same effect as setting `RUST_LOG=qcp=debug` in the environment. /// If present, `RUST_LOG` overrides this option. - #[arg(short, long, action, help_heading("Debug"))] + #[arg(short, long, action, help_heading("Debug"), display_order(0))] pub debug: bool, /// Log to a file /// /// By default the log receives everything printed to stderr. /// To override this behaviour, set the environment variable `RUST_LOG_FILE_DETAIL` (same semantics as `RUST_LOG`). - #[arg(short('l'), long, action, value_name("FILE"), help_heading("Output"))] + #[arg( + short('l'), + long, + action, + value_name("FILE"), + help_heading("Output"), + next_line_help(true), + display_order(0) + )] pub log_file: Option, /// Quiet mode @@ -35,24 +43,23 @@ pub struct Parameters { alias("stats"), action, conflicts_with("quiet"), - help_heading("Output") + help_heading("Output"), + display_order(0) )] pub statistics: bool, /// Enables detailed debug output from the remote endpoint /// (this may interfere with transfer speeds) - #[arg(long, action, help_heading("Debug"))] + #[arg(long, action, help_heading("Debug"), display_order(0))] pub remote_debug: bool, /// Output timing profile data after completion - #[arg(long, action, help_heading("Output"))] + #[arg(long, action, help_heading("Output"), display_order(0))] pub profile: bool, // JOB SPECIFICAION ==================================================================== // (POSITIONAL ARGUMENTS!) /// The source file. This may be a local filename, or remote specified as HOST:FILE or USER@HOST:FILE. - /// - /// Exactly one of source and destination must be remote. #[arg( required_unless_present_any(crate::cli::MODE_OPTIONS), value_name = "SOURCE" @@ -62,8 +69,6 @@ pub struct Parameters { /// Destination. This may be a file or directory. It may be local or remote. /// /// If remote, specify as HOST:DESTINATION or USER@HOST:DESTINATION; or simply HOST: or USER@HOST: to copy to your home directory there. - /// - /// Exactly one of source and destination must be remote. #[arg( required_unless_present_any(crate::cli::MODE_OPTIONS), value_name = "DESTINATION" diff --git a/src/client/progress.rs b/src/client/progress.rs index aa64aa3..56104e8 100644 --- a/src/client/progress.rs +++ b/src/client/progress.rs @@ -7,33 +7,43 @@ pub const MAX_UPDATE_FPS: u8 = 20; use console::Term; use indicatif::ProgressStyle; +/// A single-line style format for Indicatif which should cover most situations. +/// +/// ```text +/// 11111111111111111111111111111111111111111111111111111111111111111111111111111111 +/// filename [========================== ] 2m30s @ 123.4MB/s [70%/1.24GB] +/// fairly-long-filename [==================== ] 2m30s @ 123.4MB/s [70%/1.24GB] +/// extremely-long-filename-no-really-very-long [== ] 2m30s @ 123.4MB/s [70%/1.24GB] +/// 11111111111111111111111111111111111111111111111111111111111111111111111111111111 +/// const PROGRESS_STYLE_COMPACT: &str = "{msg:.dim} {wide_bar:.cyan} {eta} @ {decimal_bytes_per_sec} [{decimal_total_bytes:.dim}]"; -// 11111111111111111111111111111111111111111111111111111111111111111111111111111111 -// filename [========================== ] 2m30s @ 123.4MB/s [70%/1.24GB] -// fairly-long-filename [==================== ] 2m30s @ 123.4MB/s [70%/1.24GB] -// extremely-long-filename-no-really-very-long [== ] 2m30s @ 123.4MB/s [70%/1.24GB] -// 11111111111111111111111111111111111111111111111111111111111111111111111111111111 - -// We need about 35 characters for the data readout. -// A useful progress bar needs maybe 20 characters. -// This informs how much space we can allow for the filename. +/// Space to allow for the filename +/// +/// We need about 35 characters for the data readout. +/// A useful progress bar needs maybe 20 characters. +/// This informs how much space we can allow for the filename. const DATA_AND_PROGRESS: usize = 55; -// 11111111111111111111111111111111111111111111111111111111111111111111111111111111 -// extremely-long-filename-no-really-very-long [70%/1.24GB] -// [========================== ] 2m30s @ 123.4MB/s -// 11111111111111111111111111111111111111111111111111111111111111111111111111111111 - +/// A double-line style format for Indicatif for use when the filename is too long. +/// +/// ```text +/// 11111111111111111111111111111111111111111111111111111111111111111111111111111111 +/// extremely-long-filename-no-really-very-long [70%/1.24GB] +/// [========================== ] 2m30s @ 123.4MB/s +/// 11111111111111111111111111111111111111111111111111111111111111111111111111111111 +/// ``` const PROGRESS_STYLE_OVERLONG: &str = "{wide_msg:.dim} [{decimal_total_bytes:.dim}]\n{wide_bar:.cyan} {eta} @ {decimal_bytes_per_sec}"; +/// Determine the appropriate progress style to use fn use_long_style(terminal: &Term, msg_size: usize) -> bool { let term_width = terminal.size().1 as usize; // this returns a reasonable default if it can't detect msg_size + DATA_AND_PROGRESS > term_width } +/// Determine and retrieve the appropriate progress style to use pub(crate) fn progress_style_for(terminal: &Term, msg_size: usize) -> &str { if use_long_style(terminal, msg_size) { PROGRESS_STYLE_OVERLONG @@ -42,8 +52,10 @@ pub(crate) fn progress_style_for(terminal: &Term, msg_size: usize) -> &str { } } +/// Indicatif template for spinner lines pub(crate) const SPINNER_TEMPLATE: &str = "{spinner} {wide_msg} {prefix}"; +/// Indicatif template for spinner lines pub(crate) fn spinner_style() -> anyhow::Result { Ok(ProgressStyle::with_template(SPINNER_TEMPLATE)?) } diff --git a/src/client/ssh.rs b/src/client/ssh.rs index 1e10249..04eab63 100644 --- a/src/client/ssh.rs +++ b/src/client/ssh.rs @@ -9,9 +9,13 @@ use tracing::{debug, warn}; use crate::os::{AbstractPlatform as _, Platform}; +/// Metadata representing a QCP config file struct ConfigFile { + /// The file to read path: PathBuf, - user: bool, // this is a user file i.e. ~ expansion is allowed + /// if set, this is a user file i.e. ~ expansion is allowed + user: bool, + /// if set, warns on various failures and attempts to keep going warn_on_error: bool, } diff --git a/src/config/manager.rs b/src/config/manager.rs index df61777..2423466 100644 --- a/src/config/manager.rs +++ b/src/config/manager.rs @@ -23,8 +23,8 @@ use tracing::{debug, warn}; // SYSTEM DEFAULTS ////////////////////////////////////////////////////////////////////////////////////////////// -/// A `[https://docs.rs/figment/latest/figment/trait.Provider.html](figment::Provider)` that holds -/// our set of fixed system default options +/// A [`figment::Provider`](https://docs.rs/figment/latest/figment/trait.Provider.html) that holds +/// the set of system default options #[derive(Default)] struct SystemDefault {} @@ -144,7 +144,10 @@ impl Manager { self.data = f.merge(provider); // in the error case, this leaves the provider in a fused state } - /// Merges in a data set from an ssh config file + /// Merges in a data set from an ssh config file. + /// + /// The caller is expected to specify the destination host. + /// This simplifies parsing dramatically, as it means we can apply host wildcard matching immediately. pub fn merge_ssh_config(&mut self, file: F, host: Option<&str>, is_user: bool) where F: AsRef, @@ -182,6 +185,7 @@ impl Manager { // PRETTY PRINT SUPPORT /////////////////////////////////////////////////////////////////////////////////////// +/// Data type used when rendering the config table #[derive(Tabled)] struct PrettyConfig { field: String, diff --git a/src/config/mod.rs b/src/config/mod.rs index 584c765..22633c3 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1,5 +1,5 @@ // (c) 2024 Ross Younger -//! # Configuration management +//! # πŸ“– Configuration management //! //! qcp obtains run-time configuration from the following sources, in order: //! 1. Command-line options @@ -14,19 +14,68 @@ //! //! ## File format //! -//! Configuration files use the same format as OpenSSH configuration files. -//! This is a textual `Key Value` format that supports comments. +//! qcp uses the same basic format as OpenSSH configuration files. //! -//! qcp supports `Host` directives with wildcard matching, and `Include` directives. +//! Each line contains a keyword and one or more arguments. +//! Keywords are single words and are case-insensitive. +//! +//! Arguments are separated from keywords, and each other, by whitespace. +//! (It is also possible to write `Key=Value` or `Key = Value`.) +//! +//! Arguments may be surrounded by double quotes (`"`); this allows you to set an argument containing spaces. +//! If a backslash, double or single quote forms part of an argument it must be backslash-escaped i.e. `\"` or `\\`. +//! +//! Empty lines are ignored. +//! +//! **qcp supports Host and Include directives in way that is intended to be compatible with OpenSSH.** //! This allows you to tune your configuration for a range of network hosts. //! -//! ### Example +//! #### Host +//! +//! `Host host [host2 host3...]` +//! +//! This directive introduces a _host block_. +//! All following options - up to the next `Host` - only apply to hosts matching any of the patterns given. +//! +//! * Pattern matching uses `*` and `?` as wildcards in the usual way. +//! * A single asterisk `*` matches all hosts; this is used to provide defaults. +//! * A pattern beginning with `!` is a _negative_ match; it matches all remote hosts _except_ those matching the rest of the pattern. +//! * Pattern matching is applied directly to the remote host given on the QCP command line, before DNS or alias resolution. +//! If you connect to hosts by IP address, a pattern of `10.11.12.*` works in the obvious way. +//! +//! #### Include +//! +//! `Include file [file2 file3...]` +//! +//! Include the specified file(s) in the configuration at the current point. +//! +//! * Glob wildcards ('*' and '?') are supported in filenames. +//! * User configuration files may refer to pathnames relative to '~' (the user's home directory). +//! * Filenames with relative paths are assumed to be in `~/.ssh/` if read from a user configuration file, or `/etc/ssh/` if read from a system configuration file. +//! * An Include directive inside a Host block retains the Host context. +//! This may be useful to apply common directives to multiple hosts with minimal repetition. +//! Note that if an included file begins a new Host block, that will continue to apply on return to the including file. +//! * It is possible for included files to themselves include additional files; there is a brake that prevents infinite recursion. +//! +//! ## Configurable options +//! +//! The set of supported fields is defined by [Configuration]. +//! +//! In configuration files, option keywords are case insensitive and ignore hyphens and underscores. +//! (On the command line, they must be specified in kebab-case.) +//! +//! * `qcp --show-config` outputs a list of supported fields, their current values, and where each value came from. +//! * For an explanation of each field, refer to `qcp --help` . +//! * `qcp --config-files` outputs the list of configuration files for the current user and platform. +//! +//! ## Example //! //! ```text //! Host old-faithful //! # This is an old server with a very limited CPU which we do not want to overstress //! rx 125k //! tx 0 +//! RemotePort 65400-65500 # allowed in firewall config //! //! Host *.internal.corp //! # This is a nearby data centre which we have a dedicated 1Gbit connection to. @@ -43,19 +92,11 @@ //! congestion bbr # this works well for us //! ``` //! -//! ## Configurable options -//! -//! The full list of supported fields is defined by [Configuration]. -//! -//! On the command line: -//! * `qcp --show-config` outputs a list of supported fields, their current values, and where each value came from. -//! * For an explanation of each field, refer to `qcp --help` . -//! * `qcp --config-files` outputs the list of configuration files for the current user and platform. -//! -//! ### Traps and tips +//! ## Tips and traps //! 1. Like OpenSSH, for each setting we use the value from the _first_ Host block we find that matches the remote hostname. //! 1. Each setting is evaluated independently. -//! - In the example above, the `Host old-faithful` block sets an `rx` but does not set `rtt`. Any operations to `old-faithful` inherit `rtt 150` from the `Host *` block. +//! In the example above, the `Host old-faithful` block sets an `rx` but does not set `rtt`. +//! Any operations to `old-faithful` therefore inherit `rtt 150` from the `Host *` block. //! 1. The `tx` setting has a default value of 0, which means "use the active rx value". If you set `tx` in a `Host *` block, you probably want to set it explicitly everywhere you set `rx`. //! //! If you have a complicated config file we recommend you structure it as follows: diff --git a/src/config/structure.rs b/src/config/structure.rs index 8b2a3be..45c0fa6 100644 --- a/src/config/structure.rs +++ b/src/config/structure.rs @@ -44,7 +44,7 @@ pub struct Configuration { /// like `10M` or `256k`. **Note that this is described in BYTES, not bits**; /// if (for example) you expect to fill a 1Gbit ethernet connection, /// 125M might be a suitable setting. - #[arg(short('b'), long, alias("rx-bw"), help_heading("Network tuning"), display_order(10), value_name="bytes", value_parser=clap::value_parser!(HumanU64))] + #[arg(short('b'), long, alias("rx-bw"), help_heading("Network tuning"), display_order(1), value_name="bytes", value_parser=clap::value_parser!(HumanU64))] pub rx: HumanU64, /// The maximum network bandwidth we expect sending data TO the remote system, /// if it is different from the bandwidth FROM the system. @@ -52,7 +52,7 @@ pub struct Configuration { /// (For example, when you are connected via an asymmetric last-mile DSL or fibre profile.) /// /// If not specified or 0, uses the value of `rx`. - #[arg(short('B'), long, alias("tx-bw"), help_heading("Network tuning"), display_order(10), value_name="bytes", value_parser=clap::value_parser!(HumanU64))] + #[arg(short('B'), long, alias("tx-bw"), help_heading("Network tuning"), display_order(1), value_name="bytes", value_parser=clap::value_parser!(HumanU64))] pub tx: HumanU64, /// The expected network Round Trip time to the target system, in milliseconds. @@ -61,7 +61,7 @@ pub struct Configuration { short('r'), long, help_heading("Network tuning"), - display_order(1), + display_order(10), value_name("ms") )] pub rtt: u16, @@ -72,7 +72,8 @@ pub struct Configuration { long, action, value_name = "alg", - help_heading("Advanced network tuning") + help_heading("Advanced network tuning"), + display_order(0) )] #[clap(value_enum)] pub congestion: CongestionControllerType, @@ -82,66 +83,97 @@ pub struct Configuration { /// If unspecified, the active congestion control algorithm decides. /// /// _Setting this value too high reduces performance!_ - #[arg(long, help_heading("Advanced network tuning"), value_name = "bytes")] + #[arg( + long, + help_heading("Advanced network tuning"), + value_name = "bytes", + display_order(0) + )] pub initial_congestion_window: u64, /// Uses the given UDP port or range on the local endpoint. /// This can be useful when there is a firewall between the endpoints. /// - /// For example: `12345`, `"20000-20100"` - /// (in a configuration file, a range must be quoted) + /// For example: `12345`, `20000-20100` /// /// If unspecified, uses any available UDP port. - #[arg(short = 'p', long, value_name("M-N"), help_heading("Connection"))] + #[arg( + short = 'p', + long, + value_name("M-N"), + help_heading("Connection"), + display_order(0) + )] pub port: PortRange, /// Connection timeout for the QUIC endpoints [seconds; default 5] /// /// This needs to be long enough for your network connection, but short enough to provide /// a timely indication that UDP may be blocked. - #[arg(short, long, value_name("sec"), help_heading("Connection"))] + #[arg( + short, + long, + value_name("sec"), + help_heading("Connection"), + display_order(0) + )] pub timeout: u16, // CLIENT OPTIONS ================================================================================== - /// Forces use of a particular IP version when connecting to the remote. + /// Forces use of a particular IP version when connecting to the remote. [default: any] /// - /// If unspecified, uses whatever seems suitable given the target address or the result of DNS lookup. // (see also [CliArgs::ipv4_alias__] and [CliArgs::ipv6_alias__]) - #[arg(long, help_heading("Connection"), group("ip address"))] + #[arg( + long, + help_heading("Connection"), + group("ip address"), + display_order(0) + )] pub address_family: AddressFamily, /// Specifies the ssh client program to use [default: `ssh`] - #[arg(long, help_heading("Connection"))] + #[arg(long, help_heading("Connection"), display_order(0))] pub ssh: String, /// Provides an additional option or argument to pass to the ssh client. [default: none] /// /// **On the command line** you must repeat `-S` for each argument. /// For example, to pass `-i /dev/null` to ssh, specify: `-S -i -S /dev/null` - /// - /// **In a configuration file** this field is an array of strings. - /// For the same example: `ssh_opts=["-i", "/dev/null"]` #[arg( short = 'S', action, value_name("ssh-option"), allow_hyphen_values(true), - help_heading("Connection") + help_heading("Connection"), + display_order(0) )] pub ssh_options: Vec, /// Uses the given UDP port or range on the remote endpoint. /// This can be useful when there is a firewall between the endpoints. /// - /// For example: `12345`, `"20000-20100"` - /// (in a configuration file, a range must be quoted) + /// For example: `12345`, `20000-20100` /// /// If unspecified, uses any available UDP port. - #[arg(short = 'P', long, value_name("M-N"), help_heading("Connection"))] + #[arg( + short = 'P', + long, + value_name("M-N"), + help_heading("Connection"), + display_order(0) + )] pub remote_port: PortRange, /// Specifies the time format to use when printing messages to the console or to file - #[arg(short = 'T', long, value_name("FORMAT"), help_heading("Output"))] + /// [default: local] + #[arg( + short = 'T', + long, + value_name("FORMAT"), + help_heading("Output"), + next_line_help(true), + display_order(0) + )] pub time_format: TimeFormat, /// Alternative ssh config file(s) @@ -153,7 +185,7 @@ pub struct Configuration { /// /// This option is really intended to be used in a qcp configuration file. /// On the command line, you can repeat `--ssh-config file` as many times as needed. - #[arg(long, value_name("FILE"), help_heading("Connection"))] + #[arg(long, value_name("FILE"), help_heading("Connection"), display_order(0))] pub ssh_config: Vec, } diff --git a/src/doc/troubleshooting.rs b/src/doc/troubleshooting.rs index 62b7110..21bb4ef 100644 --- a/src/doc/troubleshooting.rs +++ b/src/doc/troubleshooting.rs @@ -62,3 +62,21 @@ //! * Often this is due to a DNS misconfiguration at the server side, causing it to stall until a DNS lookup times out. //! * There are a number of guides online purporting to advise you how to speed up ssh connections; I can't vouch for them. //! * You might also look into ssh connection multiplexing. +//! +//! ### qcp isn't using the network parameters you expected it to +//! * Parameters specified on the command line always override those in config files. +//! * Settings in the user config file take precedence over those in the system config file. +//! * For each setting, the first value found in a matching Host block wins. +//! * Add `--show-config` to your command line to see the settings qcp is using and where it got them from: +//! ```text +//! $ qcp myserver:some-file /tmp/ --show-config +//! β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +//! β”‚ field β”‚ value β”‚ source β”‚ +//! β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +//! β”‚ (Remote host) β”‚ myserver β”‚ β”‚ +//! β”‚ address_family β”‚ inet β”‚ /home/xyz/.qcp.conf (line 9) β”‚ +//! β”‚ congestion β”‚ cubic β”‚ /etc/qcp.conf (line 4) β”‚ +//! β”‚ initial_congestion_window β”‚ 0 β”‚ default β”‚ +//! β”‚ port β”‚ 0 β”‚ default β”‚ +//! ... +//! ``` diff --git a/src/lib.rs b/src/lib.rs index 76138ea..620bcf7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,11 +42,19 @@ //! //! The [protocol] documentation contains more detail and a discussion of its security properties. //! +//! * **qcp uses the ssh binary on your system to connect to the target machine**. +//! ssh will check the remote host key and prompt you for a password or passphrase in the usual way. +//! * **qcp will read your ssh config file** to resolve any Hostname aliases you may have defined there. +//! The idea is, if you can `ssh` to a host, you should also be able to `qcp` to it. +//! However, some particularly complicated ssh config files may be too much for qcp to understand. +//! (In particular, `Match` directives are not currently supported.) +//! In that case, you can use `--ssh-config` to provide an alternative configuration (or set it in your qcp configuration file). +//! //! ## Configuration //! //! On the command line, qcp has a comprehensive `--help` message. //! -//! Most options can also be specified in a config file. See [config] for detalis. +//! Many options can also be specified in a config file. See [config] for detalis. //! //! ## πŸ“ˆ Getting the best out of qcp //! diff --git a/src/os/unix.rs b/src/os/unix.rs index 481d8a3..3ccaa9c 100644 --- a/src/os/unix.rs +++ b/src/os/unix.rs @@ -1,4 +1,4 @@ -// OS abstraction layer for qcp - Unix implementation +//! OS concretions for Unix platforms // (c) 2024 Ross Younger use crate::config::BASE_CONFIG_FILENAME; @@ -8,6 +8,7 @@ use anyhow::Result; use nix::sys::socket::{self, sockopt}; use std::{net::UdpSocket, path::PathBuf}; +/// Is this platform BSDish? fn bsdish() -> bool { cfg!(any( target_os = "netbsd", diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 30c86d5..64db8f8 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -1,6 +1,6 @@ // (c) 2024 Ross Younger -//! Protocol defininitions +//! πŸ“– Protocol defininitions //! #![allow(clippy::doc_markdown)] //! # The QCP protocol diff --git a/src/util/address_family.rs b/src/util/address_family.rs index 02f9f59..9dbfe35 100644 --- a/src/util/address_family.rs +++ b/src/util/address_family.rs @@ -18,7 +18,7 @@ pub enum AddressFamily { /// IPv6 #[value(alias("6"))] Inet6, - /// We don't mind what type of IP address + /// Unspecified. qcp will use whatever seems suitable given the target address or the result of DNS lookup. Any, } diff --git a/src/util/dns.rs b/src/util/dns.rs index 70a7e12..c7faee4 100644 --- a/src/util/dns.rs +++ b/src/util/dns.rs @@ -1,4 +1,4 @@ -// DNS helpers +//! DNS helpers // (c) 2024 Ross Younger use std::net::IpAddr; diff --git a/src/util/port_range.rs b/src/util/port_range.rs index ddb1a48..3cac9d9 100644 --- a/src/util/port_range.rs +++ b/src/util/port_range.rs @@ -1,4 +1,4 @@ -/// CLI argument helper - PortRange +//! CLI argument helper type - a range of UDP port numnbers. // (c) 2024 Ross Younger use serde::{ de::{self, Error, Unexpected}, From fa4fbc929f0d1c7e7d6556486bb5505d69ff1f8a Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 26 Dec 2024 16:50:37 +1300 Subject: [PATCH 50/54] misc: add template system configuration file --- Cargo.toml | 1 + misc/qcp.conf | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 misc/qcp.conf diff --git a/Cargo.toml b/Cargo.toml index 50ddae6..6d24df2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,6 +135,7 @@ assets = [ [ "misc/20-qcp.conf", "etc/sysctl.d/", "644" ], # this is automatically recognised as a conffile [ "misc/qcp.1", "usr/share/man/man1/", "644" ], [ "misc/qcp_config.5", "usr/share/man/man5/", "644" ], + [ "misc/qcp.conf", "etc/", "644" ], # this is automatically recognised as a conffile ] maintainer-scripts="debian" depends = "$auto,debconf" diff --git a/misc/qcp.conf b/misc/qcp.conf new file mode 100644 index 0000000..366d8ef --- /dev/null +++ b/misc/qcp.conf @@ -0,0 +1,29 @@ +# This is the system-wide configuration file for QCP. +# The values set here can be overridden in per-user configuration files, or on the command line. +# +# This file is a very similar format to ssh_config. +# You can use Host and Include directives in the same way. +# See qcp_config(5) for more information. +# +# For explanations of the options you can set, see qcp(1) or `qcp --help`. +# + +Host * +# Rx 12500000 +# Tx 0 +# Rtt 300 + +# AddressFamily any + +# Port 0 +# RemotePort 0 + +# Congestion cubic +# InitialCongestionWindow 0 + +# Ssh ssh +# SshConfig +# SshOptions + +# TimeFormat local +# Timeout 5 From 7a8deac56bf97dacb3485e1a335a30c02b82739d Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 26 Dec 2024 18:47:41 +1300 Subject: [PATCH 51/54] style: show Opening control channel message --- src/client/main_loop.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/client/main_loop.rs b/src/client/main_loop.rs index eb3e98d..e8cb3d0 100644 --- a/src/client/main_loop.rs +++ b/src/client/main_loop.rs @@ -46,7 +46,15 @@ pub async fn client_main( let _guard = trace_span!("CLIENT").entered(); let mut timers = StopwatchChain::new_running("setup"); + let spinner = if parameters.quiet { + ProgressBar::hidden() + } else { + display.add(ProgressBar::new_spinner().with_style(spinner_style()?)) + }; + spinner.enable_steady_tick(Duration::from_millis(150)); + // Prep -------------------------- + spinner.set_message("Preparing"); let job_spec = crate::client::CopyJobSpec::try_from(¶meters)?; let credentials = Credentials::generate()?; let user_hostname = job_spec.remote_host(); @@ -58,6 +66,8 @@ pub async fn client_main( let remote_address = lookup_host_by_family(&remote_host, config.address_family)?; // Control channel --------------- + spinner.set_message("Opening control channel"); + spinner.disable_steady_tick(); // otherwise the spinner messes with ssh passphrase prompting; as we're using tokio spinner.suspend() isn't helpful timers.next("control channel"); let (mut control, server_message) = Channel::transact( &credentials, @@ -75,11 +85,6 @@ pub async fn client_main( std::net::IpAddr::V6(ip) => SocketAddrV6::new(ip, server_message.port, 0, 0).into(), }; - let spinner = if parameters.quiet { - ProgressBar::hidden() - } else { - display.add(ProgressBar::new_spinner().with_style(spinner_style()?)) - }; spinner.enable_steady_tick(Duration::from_millis(150)); spinner.set_message("Establishing data channel"); timers.next("data channel setup"); From b54ee504d3b951ce6e0f36e25b55a8ab45098fa2 Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 26 Dec 2024 18:55:15 +1300 Subject: [PATCH 52/54] misc: add feature flag to enable rustls logging (on by default) --- Cargo.lock | 17 +++++++++++++++++ Cargo.toml | 6 ++++++ src/lib.rs | 3 +++ 3 files changed, 26 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index cbfc9c7..766e504 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -478,6 +478,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "document-features" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6969eaabd2421f8a2775cfd2471a2b634372b4a25d41e3bd647b79912850a0" +dependencies = [ + "litrs", +] + [[package]] name = "either" version = "1.13.0" @@ -828,6 +837,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litrs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" + [[package]] name = "log" version = "0.4.22" @@ -1089,6 +1104,7 @@ dependencies = [ "derive-deftly", "dirs 5.0.1", "dns-lookup", + "document-features", "expanduser", "fastrand", "figment", @@ -1358,6 +1374,7 @@ version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", diff --git a/Cargo.toml b/Cargo.toml index 6d24df2..b8e826a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,11 @@ split-debuginfo="unpacked" lto = "thin" strip = "symbols" +[features] +default = ["rustls-log"] +## Enables rustls debug messages. You still have to request them using the environment variable, e.g. `RUST_LOG="rustls=debug"`. +rustls-log = ["quinn/rustls-log"] + [dependencies] anstream = "0.6.18" anstyle = "1.0.10" @@ -31,6 +36,7 @@ console = "0.15.8" derive-deftly = "0.14.2" dirs = "5.0.1" dns-lookup = "2.0.4" +document-features = "0.2.10" expanduser = "1.2.2" figment = { version = "0.10.19" } futures-util = { version = "0.3.31", default-features = false } diff --git a/src/lib.rs b/src/lib.rs index 620bcf7..c9d5b2c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -71,6 +71,9 @@ //! [BitTorrent]: https://en.wikipedia.org/wiki/BitTorrent //! [rsync]: https://en.wikipedia.org/wiki/Rsync //! [mosh]: https://mosh.org/ +//! +//! ## Feature flags +#![doc = document_features::document_features!()] mod cli; pub use cli::cli; // needs to be re-exported for the binary crate From 1194c247d37bdcda35d4a2423e6aa51af86a394c Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 26 Dec 2024 19:20:17 +1300 Subject: [PATCH 53/54] tidyup: Reduce unnecessary complexity in terminal styling --- Cargo.lock | 36 ------------------------------------ Cargo.toml | 2 -- src/cli/cli_main.rs | 7 +++---- src/cli/styles.rs | 12 +----------- 4 files changed, 4 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 766e504..a8b51eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,16 +62,6 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" -[[package]] -name = "anstyle-owo-colors" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b774bbe47d3bd767710315f5e57c23a769d6c35f28456f2be8d6e20339f55f34" -dependencies = [ - "anstyle", - "owo-colors", -] - [[package]] name = "anstyle-parse" version = "0.2.6" @@ -737,12 +727,6 @@ dependencies = [ "web-time", ] -[[package]] -name = "is_ci" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" - [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -974,15 +958,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "owo-colors" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb37767f6569cd834a413442455e0f066d0d522de8630436e2a1761d9726ba56" -dependencies = [ - "supports-color", -] - [[package]] name = "papergrid" version = "0.13.0" @@ -1093,7 +1068,6 @@ version = "0.1.3" dependencies = [ "anstream", "anstyle", - "anstyle-owo-colors", "anyhow", "assertables", "capnp", @@ -1120,7 +1094,6 @@ dependencies = [ "lazy_static", "nix", "num-format", - "owo-colors", "quinn", "rand", "rcgen", @@ -1581,15 +1554,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "supports-color" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" -dependencies = [ - "is_ci", -] - [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index b8e826a..ee97d35 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,6 @@ rustls-log = ["quinn/rustls-log"] [dependencies] anstream = "0.6.18" anstyle = "1.0.10" -anstyle-owo-colors = "2.0.3" anyhow = "1.0.94" capnp = "0.20.3" capnp-futures = "0.20.1" @@ -48,7 +47,6 @@ humanize-rs = "0.1.5" indicatif = { version = "0.17.9", features = ["tokio"] } lazy_static = "1.5.0" num-format = { version = "0.4.4" } -owo-colors = { version = "4.1.0", features = ["supports-color"] } quinn = { version = "0.11.6", default-features = false, features = ["runtime-tokio", "rustls", "ring"] } rcgen = { version = "0.13.1" } rustls-pki-types = "1.10.0" diff --git a/src/cli/cli_main.rs b/src/cli/cli_main.rs index 2d8ffcb..f3dd4cb 100644 --- a/src/cli/cli_main.rs +++ b/src/cli/cli_main.rs @@ -3,7 +3,7 @@ use std::process::ExitCode; -use super::{args::CliArgs, styles::ERROR_S}; +use super::args::CliArgs; use crate::{ client::{client_main, Parameters as ClientParameters, MAX_UPDATE_FPS}, config::{Configuration, Manager}, @@ -14,7 +14,6 @@ use crate::{ use anstream::{eprintln, println}; use indicatif::{MultiProgress, ProgressDrawTarget}; -use owo_colors::OwoColorize as _; use tracing::error_span; /// Computes the trace level for a given set of [ClientParameters] @@ -59,7 +58,7 @@ pub async fn cli() -> anyhow::Result { let config_manager = match Manager::try_from(&args) { Ok(m) => m, Err(err) => { - eprintln!("{}: {err}", "ERROR".style(*ERROR_S)); + eprintln!("ERROR: {err}"); return Ok(ExitCode::FAILURE); } }; @@ -67,7 +66,7 @@ pub async fn cli() -> anyhow::Result { let config = match config_manager.get::() { Ok(c) => c, Err(err) => { - eprintln!("{}: Failed to parse configuration", "ERROR".style(*ERROR_S)); + eprintln!("ERROR: Failed to parse configuration"); err.into_iter().for_each(|e| eprintln!("{e}")); return Ok(ExitCode::FAILURE); } diff --git a/src/cli/styles.rs b/src/cli/styles.rs index b301bd5..162b505 100644 --- a/src/cli/styles.rs +++ b/src/cli/styles.rs @@ -6,23 +6,13 @@ #[allow(clippy::enum_glob_use)] use anstyle::AnsiColor::*; use anstyle::Color::Ansi; -use anstyle_owo_colors::to_owo_style; use clap::builder::styling::Styles; -use lazy_static::lazy_static; -use owo_colors::Style as OwoStyle; pub(crate) const ERROR: anstyle::Style = anstyle::Style::new().bold().fg_color(Some(Ansi(Red))); pub(crate) const WARNING: anstyle::Style = anstyle::Style::new().bold().fg_color(Some(Ansi(Yellow))); pub(crate) const INFO: anstyle::Style = anstyle::Style::new().fg_color(Some(Ansi(Cyan))); -pub(crate) const DEBUG: anstyle::Style = anstyle::Style::new().fg_color(Some(Ansi(Blue))); - -lazy_static! { - pub(crate) static ref ERROR_S: OwoStyle = to_owo_style(ERROR); - pub(crate) static ref WARNING_S: OwoStyle = to_owo_style(WARNING); - pub(crate) static ref INFO_S: OwoStyle = to_owo_style(INFO); - pub(crate) static ref DEBUG_S: OwoStyle = to_owo_style(DEBUG); -} +// pub(crate) const DEBUG: anstyle::Style = anstyle::Style::new().fg_color(Some(Ansi(Blue))); pub(crate) const CALL_OUT: anstyle::Style = anstyle::Style::new() .underline() From 73bf6d9bf953d157889c5a85d76867ea64a970fd Mon Sep 17 00:00:00 2001 From: Ross Younger Date: Thu, 26 Dec 2024 19:25:37 +1300 Subject: [PATCH 54/54] chore(release): update dependencies & github actions --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- Cargo.lock | 116 ++++++++++++++++------------------ 3 files changed, 57 insertions(+), 63 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34208e1..cf6da85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: toolchain: ${{ matrix.rust }} targets: ${{ matrix.target }} - name: Install tools - uses: taiki-e/install-action@v2.46.6 + uses: taiki-e/install-action@v2.46.20 with: tool: cross,cargo-deb #- name: Set minimal profile (Windows only) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e2310be..3c2b290 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: with: toolchain: ${{ matrix.rust }} targets: ${{ matrix.target }} - - uses: taiki-e/install-action@v2.46.6 + - uses: taiki-e/install-action@v2.46.20 with: tool: cross,cargo-deb - uses: Swatinem/rust-cache@v2 diff --git a/Cargo.lock b/Cargo.lock index a8b51eb..0765e9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,9 +92,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "arrayref" @@ -202,9 +202,9 @@ checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" [[package]] name = "bytemuck" -version = "1.20.0" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" [[package]] name = "byteorder" @@ -249,9 +249,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.3" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" +checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" dependencies = [ "shlex", ] @@ -312,7 +312,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -329,15 +329,15 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "console" -version = "0.15.8" +version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" dependencies = [ "encode_unicode", - "lazy_static", "libc", - "unicode-width 0.1.14", - "windows-sys 0.52.0", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", ] [[package]] @@ -410,7 +410,7 @@ dependencies = [ "quote", "sha3", "strum", - "syn 2.0.90", + "syn 2.0.91", "void", ] @@ -491,9 +491,9 @@ checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" [[package]] name = "encode_unicode" -version = "0.3.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "equivalent" @@ -723,7 +723,7 @@ dependencies = [ "number_prefix", "portable-atomic", "tokio", - "unicode-width 0.2.0", + "unicode-width", "web-time", ] @@ -801,9 +801,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.168" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libredox" @@ -859,9 +859,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -933,9 +933,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -966,7 +966,7 @@ checksum = "d2b0f8def1f117e13c895f3eda65a7b5650688da29d6ad04635f61bc7b92eebd" dependencies = [ "bytecount", "fnv", - "unicode-width 0.2.0", + "unicode-width", ] [[package]] @@ -1040,7 +1040,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -1126,7 +1126,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.6", + "thiserror 2.0.9", "tokio", "tracing", ] @@ -1145,7 +1145,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.6", + "thiserror 2.0.9", "tinyvec", "tracing", "web-time", @@ -1153,9 +1153,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", @@ -1167,9 +1167,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -1206,9 +1206,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54077e1872c46788540de1ea3d7f4ccb1983d12f9aa909b234468676c1a36779" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" dependencies = [ "pem", "ring", @@ -1358,9 +1358,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" dependencies = [ "web-time", ] @@ -1405,14 +1405,14 @@ checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "itoa", "memchr", @@ -1523,7 +1523,7 @@ checksum = "a2dbf8b57f3ce20e4bb171a11822b283bdfab6c4bb0fe64fa729f045f23a0938" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -1545,7 +1545,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -1567,9 +1567,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.90" +version = "2.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" dependencies = [ "proc-macro2", "quote", @@ -1633,11 +1633,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.6" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" dependencies = [ - "thiserror-impl 2.0.6", + "thiserror-impl 2.0.9", ] [[package]] @@ -1648,18 +1648,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] name = "thiserror-impl" -version = "2.0.6" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -1693,9 +1693,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -1731,7 +1731,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -1784,7 +1784,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]] @@ -1848,12 +1848,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "unicode-width" version = "0.2.0" @@ -1923,7 +1917,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", "wasm-bindgen-shared", ] @@ -1945,7 +1939,7 @@ checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2187,7 +2181,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.91", ] [[package]]