From 248736143e4903fdd274f8e934bcf79f71748ccc Mon Sep 17 00:00:00 2001 From: neonphog Date: Thu, 21 Nov 2024 13:16:28 -0700 Subject: [PATCH 01/13] deps and other changes in prep for bootstrap srv PRs --- .github/workflows/test.yaml | 14 +- Cargo.lock | 946 +++++++++++++++++++++++++++++++- Cargo.toml | 13 + Makefile | 22 + crates/bootstrap_srv/Cargo.toml | 13 + 5 files changed, 990 insertions(+), 18 deletions(-) create mode 100644 Makefile diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cfcdd41..7fc12d5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -20,20 +20,20 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Format - run: cargo fmt --all -- --check - - - name: Build - run: cargo build + run: make fmt - name: Lint - run: cargo clippy --all-targets -- --deny warnings + run: make clippy - name: Doc if: matrix.os == 'ubuntu-latest' - run: RUSTDOCFLAGS="-D warnings" cargo doc --no-deps + run: make doc + + - name: Build + run: make build - name: Test - run: cargo test + run: make test ci_pass: if: ${{ always() }} diff --git a/Cargo.lock b/Cargo.lock index cd5f77c..01e0259 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,61 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "autocfg" version = "1.4.0" @@ -44,12 +99,27 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bytes" version = "1.8.0" @@ -59,29 +129,442 @@ dependencies = [ "serde", ] +[[package]] +name = "cc" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" +dependencies = [ + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + +[[package]] +name = "clap" +version = "4.5.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "cpufeatures" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctrlc" +version = "3.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3" +dependencies = [ + "nix", + "windows-sys 0.59.0", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "flate2" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "gimli" version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "7a73e9fe3c49d7afb2ace819fa181a287ce54a0983eda4e0eb05c22f82ffe534" [[package]] name = "kitsune2_api" @@ -97,12 +580,37 @@ dependencies = [ [[package]] name = "kitsune2_bootstrap_srv" version = "0.0.1-alpha" +dependencies = [ + "base64", + "bytes", + "clap", + "ctrlc", + "ed25519-dalek", + "num_cpus", + "serde", + "serde_json", + "tempfile", + "tiny_http", + "ureq", +] [[package]] name = "libc" -version = "0.2.162" +version = "0.2.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" [[package]] name = "lock_api" @@ -114,6 +622,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + [[package]] name = "memchr" version = "2.7.4" @@ -138,7 +652,29 @@ dependencies = [ "hermit-abi", "libc", "wasi", - "windows-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", ] [[package]] @@ -150,6 +686,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + [[package]] name = "parking_lot" version = "0.12.3" @@ -173,12 +715,28 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + [[package]] name = "pin-project-lite" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "proc-macro2" version = "1.0.89" @@ -197,6 +755,15 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "redox_syscall" version = "0.5.7" @@ -206,12 +773,81 @@ dependencies = [ "bitflags", ] +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f1a745511c54ba6d4465e8d5dfbd81b45791756de28d4981af70d6dca128f1e" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.18" @@ -224,6 +860,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + [[package]] name = "serde" version = "1.0.215" @@ -246,9 +888,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", @@ -256,6 +898,23 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -265,6 +924,15 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -278,9 +946,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.87" @@ -292,6 +994,62 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "terminal_size" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f599bd7ca042cfdf8f4512b277c02ba102247820f9d9d4a9f521f496751a6ef" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tokio" version = "1.41.1" @@ -307,7 +1065,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -321,11 +1079,74 @@ dependencies = [ "syn", ] +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" +dependencies = [ + "base64", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots", +] + +[[package]] +name = "url" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wasi" @@ -333,6 +1154,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "webpki-roots" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -342,6 +1172,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -405,3 +1244,88 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index fe492e0..1974351 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,14 +12,27 @@ base64 = "0.22.1" # shallow-clone byte arrays is a solved problem. # bytes is the crate that solves it. bytes = { version = "1.8.0", features = ["serde"] } +# boot_srv for cli param parsing +clap = "4.5.21" +# boot_srv uses this to make a best-effort to clean up tempfiles on shutdown. +ctrlc = { version = "3.4.5", features = ["termination"] } +# boot_srv for signature verification. +ed25519-dalek = "2.1.1" +# boot_srv uses this to determine worker thread count. +num_cpus = "1.16.0" # kitsune types need to be serializable for network transmission. serde = { version = "1.0.215", features = ["derive"] } # kitsune2 agent info is serialized as json to improve debugability of # bootstrapping. So, we need a json library. serde_json = "1.0.132" +# boot_srv uses tempfiles as virtual memory for storage instead of RAM. +tempfile = "3.14.0" +# this is used by boot_srv as the http server implementation. +tiny_http = "0.12.0" # --- dev-dependencies --- # The following workspace dependencies are used in crate dev-dependencies. # Please be careful to only include them in dev dependencies or move them # above this section. # --- dev-dependencies --- tokio = "1.41.1" +ureq = "2.10.1" diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0e75337 --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +# A temporary Makefile to hold a single source of truth for running CI +# tasks both in automation and locally until we figure out better +# release automation tools. + +.PHONY: all fmt clippy doc build test + +all: fmt clippy doc test + +fmt: + cargo fmt --all -- --check + +clippy: + cargo clippy --all-targets -- --deny warnings + +doc: + RUSTDOCFLAGS="-D warnings" cargo doc --no-deps + +build: + cargo build + +test: + cargo test diff --git a/crates/bootstrap_srv/Cargo.toml b/crates/bootstrap_srv/Cargo.toml index 14a7bfe..29e0659 100644 --- a/crates/bootstrap_srv/Cargo.toml +++ b/crates/bootstrap_srv/Cargo.toml @@ -11,3 +11,16 @@ categories = ["network-programming"] edition = "2021" [dependencies] +base64 = { workspace = true } +bytes = { workspace = true } +clap = { workspace = true, features = ["derive", "wrap_help"] } +ctrlc = { workspace = true } +ed25519-dalek = { workspace = true } +num_cpus = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tempfile = { workspace = true } +tiny_http = { workspace = true } + +[dev-dependencies] +ureq = { workspace = true } From a5da1ed4a10f0d55b356d80842ec5e06b48dd38a Mon Sep 17 00:00:00 2001 From: neonphog Date: Thu, 21 Nov 2024 13:22:10 -0700 Subject: [PATCH 02/13] core kitsune2 bootstrap server code --- .../src/bin/kitsune2-bootstrap-srv.rs | 49 +++ crates/bootstrap_srv/src/config.rs | 84 +++++ crates/bootstrap_srv/src/lib.rs | 22 ++ crates/bootstrap_srv/src/parse.rs | 83 +++++ crates/bootstrap_srv/src/server.rs | 333 ++++++++++++++++++ crates/bootstrap_srv/src/space.rs | 228 ++++++++++++ crates/bootstrap_srv/src/store.rs | 191 ++++++++++ 7 files changed, 990 insertions(+) create mode 100644 crates/bootstrap_srv/src/bin/kitsune2-bootstrap-srv.rs create mode 100644 crates/bootstrap_srv/src/config.rs create mode 100644 crates/bootstrap_srv/src/parse.rs create mode 100644 crates/bootstrap_srv/src/server.rs create mode 100644 crates/bootstrap_srv/src/space.rs create mode 100644 crates/bootstrap_srv/src/store.rs diff --git a/crates/bootstrap_srv/src/bin/kitsune2-bootstrap-srv.rs b/crates/bootstrap_srv/src/bin/kitsune2-bootstrap-srv.rs new file mode 100644 index 0000000..3640b2c --- /dev/null +++ b/crates/bootstrap_srv/src/bin/kitsune2-bootstrap-srv.rs @@ -0,0 +1,49 @@ +//! The binary kitsune2-bootstrap-srv. + +use kitsune2_bootstrap_srv::*; + +#[derive(clap::Parser, Debug)] +#[command(version)] +pub struct Args { + /// By default kitsune2-boot-srv runs in "testing" configuration + /// with much lighter resource usage settings. This testing mode + /// should be more than enough for most developer application testing + /// and continuous integration or automated tests. + /// + /// To setup the server to be ready to use most of the resources available + /// on a single given machine, you can set this "production" mode. + #[arg(long)] + pub production: bool, + // TODO - Implement the ability to specify TLS certificates + // TODO - Implement the ability to specify the listening address + // TODO - Implement the ability to override any other relevant + // config params that we wish to expose +} + +fn main() { + let args = ::parse(); + + let config = if args.production { + Config::production() + } else { + Config::testing() + }; + + println!("{args:?}--{config:?}"); + + let (send, recv) = std::sync::mpsc::channel(); + + ctrlc::set_handler(move || { + send.send(()).unwrap(); + }) + .unwrap(); + + let srv = BootstrapSrv::new(config); + + let _ = recv.recv(); + + println!("Terminating..."); + drop(srv); + println!("Done."); + std::process::exit(0); +} diff --git a/crates/bootstrap_srv/src/config.rs b/crates/bootstrap_srv/src/config.rs new file mode 100644 index 0000000..1eb1c86 --- /dev/null +++ b/crates/bootstrap_srv/src/config.rs @@ -0,0 +1,84 @@ +//! config types. + +/// Configuration for running a BootstrapSrv. +#[derive(Debug)] +pub struct Config { + /// Worker thread count. + /// + /// This server is currently built using blocking io and filesystem + /// storage. It is therefore beneficial to have more worker threads + /// than system cpus, since the workers will be bound on io, not + /// on cpu. On the other hand, increasing this will also increase + /// memory overhead and tempfile handle count, so we don't want to + /// set it too high. + /// + /// Defaults: + /// - `testing = 2` + /// - `production = 4 * cpu_count` + pub worker_thread_count: usize, + + /// The maximum agent info entry count per space. + /// + /// All entries will be returned in a get space request, so + /// this count should be low enough to reasonably send this response + /// over http without needing pagination. + /// + /// Defaults: + /// - `testing = 32` + /// - `production = 32` + pub max_entries_per_space: usize, + + /// The duration worker threads will block waiting for incoming connections + /// before checking to see if the server is shutting down. + /// + /// Setting this very high will cause ctrl-c / server shutdown to be slow. + /// Setting this very low will increase cpu overhead (and in extreme + /// conditions, could cause a lack of responsiveness in the server). + /// + /// Defaults: + /// - `testing = 10ms` + /// - `production = 2s` + pub request_listen_duration: std::time::Duration, + + /// The address(es) at which to listen. + /// + /// Defaults: + /// - `testing = "127.0.0.1:0"` + /// - `production = "0.0.0.0:443"` + pub listen_address: std::net::SocketAddr, + + /// The interval at which expired agents are purged from the cache. + /// This is a fairly expensive operation that requires iterating + /// through every registered space and loading all the infos off the disk, + /// so it should not be undertaken too frequently. + /// + /// Defaults: + /// + /// - `testing = 10s` + /// - `production = 60s` + pub prune_interval: std::time::Duration, +} + +impl Config { + /// Get a boot_srv config suitable for testing. + pub fn testing() -> Self { + Self { + worker_thread_count: 2, + max_entries_per_space: 32, + request_listen_duration: std::time::Duration::from_millis(10), + listen_address: ([127, 0, 0, 1], 0).into(), + prune_interval: std::time::Duration::from_secs(10), + } + } + + /// Get a boot_srv config suitable for production. + pub fn production() -> Self { + Self { + worker_thread_count: num_cpus::get() * 4, + max_entries_per_space: 32, + request_listen_duration: std::time::Duration::from_secs(2), + listen_address: ([0, 0, 0, 0], 443).into(), + prune_interval: std::time::Duration::from_secs(60), + } + } +} diff --git a/crates/bootstrap_srv/src/lib.rs b/crates/bootstrap_srv/src/lib.rs index a3eede7..3d725c1 100644 --- a/crates/bootstrap_srv/src/lib.rs +++ b/crates/bootstrap_srv/src/lib.rs @@ -175,3 +175,25 @@ /// interval. #[cfg(doc)] pub mod spec {} + +fn now() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .expect("InvalidSystemTime") + .as_micros() as i64 +} + +mod config; +pub use config::*; + +mod parse; +use parse::*; + +mod store; +use store::*; + +mod space; +use space::*; + +mod server; +pub use server::*; diff --git a/crates/bootstrap_srv/src/parse.rs b/crates/bootstrap_srv/src/parse.rs new file mode 100644 index 0000000..2970ac1 --- /dev/null +++ b/crates/bootstrap_srv/src/parse.rs @@ -0,0 +1,83 @@ +/// An entry with known content. +#[derive(Debug)] +pub struct ParsedEntry { + /// agent + pub agent: ed25519_dalek::VerifyingKey, + + /// space + pub space: bytes::Bytes, + + /// created_at + pub created_at: i64, + + /// expires_at + pub expires_at: i64, + + /// is_tombstone + pub is_tombstone: bool, + + /// encoded + pub encoded: String, + + /// signature + pub signature: ed25519_dalek::Signature, +} + +impl ParsedEntry { + /// Parse entry from a slice. + pub fn from_slice(slice: &[u8]) -> std::io::Result { + use base64::prelude::*; + + #[derive(serde::Deserialize)] + #[serde(rename_all = "camelCase")] + struct Outer { + agent_info: String, + signature: String, + } + + #[derive(serde::Deserialize)] + #[serde(rename_all = "camelCase")] + struct Inner { + agent: String, + space: String, + created_at: String, + expires_at: String, + is_tombstone: bool, + } + + let out: Outer = serde_json::from_slice(slice)?; + let inn: Inner = serde_json::from_str(&out.agent_info)?; + + let agent: [u8; 32] = BASE64_URL_SAFE_NO_PAD + .decode(inn.agent) + .map_err(std::io::Error::other)? + .try_into() + .map_err(|_| std::io::Error::other("InvalidAgentPubKey"))?; + + let signature: [u8; 64] = BASE64_URL_SAFE_NO_PAD + .decode(out.signature) + .map_err(std::io::Error::other)? + .try_into() + .map_err(|_| std::io::Error::other("InvalidSignature"))?; + + Ok(ParsedEntry { + agent: ed25519_dalek::VerifyingKey::from_bytes(&agent) + .map_err(std::io::Error::other)?, + space: BASE64_URL_SAFE_NO_PAD + .decode(inn.space) + .map_err(std::io::Error::other)? + .into(), + created_at: inn + .created_at + .parse() + .map_err(std::io::Error::other)?, + expires_at: inn + .expires_at + .parse() + .map_err(std::io::Error::other)?, + is_tombstone: inn.is_tombstone, + encoded: out.agent_info, + signature: ed25519_dalek::Signature::from_bytes(&signature), + }) + } +} diff --git a/crates/bootstrap_srv/src/server.rs b/crates/bootstrap_srv/src/server.rs new file mode 100644 index 0000000..208bfed --- /dev/null +++ b/crates/bootstrap_srv/src/server.rs @@ -0,0 +1,333 @@ +//! bootstrap http server types. + +use std::sync::Arc; + +use crate::*; +use tiny_http::*; + +/// An actual kitsune2_bootstrap_srv server instance. +/// +/// This server is built to be direct, light-weight, and responsive. +/// On the server-side, as one aspect toward accomplishing this, +/// we are eschewing async code in favor of os thread workers. +pub struct BootstrapSrv { + cont: Arc, + workers: Vec>>, + addr: std::net::SocketAddr, +} + +impl Drop for BootstrapSrv { + fn drop(&mut self) { + self.cont.store(false, std::sync::atomic::Ordering::SeqCst); + for worker in self.workers.drain(..) { + let _ = worker.join().expect("Failure shutting down worker thread"); + } + } +} + +impl BootstrapSrv { + /// Construct a new BootstrapSrv instance. + pub fn new(config: Config) -> std::io::Result { + let config = Arc::new(config); + + // atomic flag for telling worker threads to shutdown + let cont = Arc::new(std::sync::atomic::AtomicBool::new(true)); + + // synchronization type for managing infos in spaces + let space_map = crate::SpaceMap::default(); + + // tiny_http configuration + let sconf = ServerConfig { + addr: ConfigListenAddr::IP(vec![config.listen_address]), + // TODO make the server able to accept TLS certificates + ssl: None, + }; + + // virtual-memory-like file system storage for infos + let store = Arc::new(crate::Store::default()); + + // start the actual http server + let server = + Arc::new(Server::new(sconf).map_err(std::io::Error::other)?); + + // get the address that was assigned + let addr = server.server_addr().to_ip().expect("BadAddress"); + println!("Listening at {:?}", addr); + + // spawn our worker threads + let mut workers = Vec::with_capacity(config.worker_thread_count + 1); + for _ in 0..config.worker_thread_count { + let config = config.clone(); + let cont = cont.clone(); + let store = store.clone(); + let server = server.clone(); + let space_map = space_map.clone(); + workers.push(std::thread::spawn(move || { + worker(config, cont, store, server, space_map) + })); + } + + // also set up a worker for pruning expired infos + let maint_cont = cont.clone(); + let maint_space_map = space_map.clone(); + workers.push(std::thread::spawn(move || { + maint_worker(config, maint_cont, maint_space_map) + })); + + Ok(Self { + cont, + workers, + addr, + }) + } + + /// Get the bound listinging address of this server. + pub fn listen_addr(&self) -> std::net::SocketAddr { + self.addr + } +} + +fn maint_worker( + config: Arc, + cont: Arc, + space_map: crate::SpaceMap, +) -> std::io::Result<()> { + let mut last_check = std::time::Instant::now(); + + while cont.load(std::sync::atomic::Ordering::SeqCst) { + std::thread::sleep(config.request_listen_duration); + + if last_check.elapsed() >= config.prune_interval { + last_check = std::time::Instant::now(); + + space_map.update_all(config.max_entries_per_space); + } + } + + Ok(()) +} + +fn worker( + config: Arc, + cont: Arc, + store: Arc, + server: Arc, + space_map: crate::SpaceMap, +) -> std::io::Result<()> { + while cont.load(std::sync::atomic::Ordering::SeqCst) { + let req = match server.recv_timeout(config.request_listen_duration)? { + Some(req) => req, + None => continue, + }; + + let path = req + .url() + .split('/') + .rev() + .filter_map(|p| { + if p.is_empty() { + None + } else { + Some(p.to_string()) + } + }) + .collect::>(); + + let handler = Handler { + config: &config, + store: &store, + space_map: &space_map, + method: req.method().as_str().to_string(), + path, + req, + }; + + handler.handle()?; + } + Ok(()) +} + +struct Handler<'lt> { + config: &'lt Config, + store: &'lt crate::Store, + space_map: &'lt crate::SpaceMap, + method: String, + path: Vec, + req: tiny_http::Request, +} + +impl<'lt> Handler<'lt> { + /// Wrap the handle call so we can respond to the client with errors. + pub fn handle(mut self) -> std::io::Result<()> { + match self.handle_inner() { + Ok((status, body)) => self.respond(status, body), + Err(err) => self.respond( + 500, + serde_json::to_string(&serde_json::json!({ + "error": format!("{err:?}"), + }))? + .into_bytes(), + ), + } + } + + /// Dispatch to the correct handlers. + fn handle_inner(&mut self) -> std::io::Result<(u16, Vec)> { + if let Some(cmd) = self.path.pop() { + match (self.method.as_str(), cmd.as_str()) { + ("GET", "health") => { + return Ok((200, b"{}".to_vec())); + } + ("GET", "bootstrap") => { + return self.handle_boot_get(); + } + ("PUT", "bootstrap") => { + return self.handle_boot_put(); + } + _ => (), + } + } + Ok((400, b"{\"error\":\"bad request\"}".to_vec())) + } + + /// Respond to a request for the agent infos within a space. + fn handle_boot_get(&mut self) -> std::io::Result<(u16, Vec)> { + let space = self.path_to_bytes()?; + + let res = self.space_map.read(&space)?; + + Ok((200, res)) + } + + /// Validate an incoming agent info and put it in the store if appropriate. + fn handle_boot_put(&mut self) -> std::io::Result<(u16, Vec)> { + use ed25519_dalek::*; + + let now = crate::now(); + + let space = self.path_to_bytes()?; + let agent = self.path_to_bytes()?; + + let info_raw = self.read_body()?; + let info = crate::ParsedEntry::from_slice(&info_raw)?; + + // validate agent matches url path + if *agent != *info.agent.as_bytes() { + return Err(std::io::Error::other("InvalidAgent")); + } + + // validate space matches url path + if space != info.space { + return Err(std::io::Error::other("InvalidSpace")); + } + + // validate created at is not older than 3 min ago + if info.created_at + < now - (std::time::Duration::from_secs(60 * 3).as_micros() as i64) + { + return Err(std::io::Error::other("InvalidCreatedAt")); + } + + // validate created at is less than 3 min in the future + if info.created_at + > now + (std::time::Duration::from_secs(60 * 3).as_micros() as i64) + { + return Err(std::io::Error::other("InvalidCreatedAt")); + } + + // validate not expired + if info.expires_at < now { + return Err(std::io::Error::other("InvalidExpiresAt")); + } + + // validate expires_at is not before (or equal to) created_at + if info.expires_at <= info.created_at { + return Err(std::io::Error::other("InvalidExpiresAt")); + } + + // validate expires_at is not more than 30 min after created_at + if info.expires_at - info.created_at + > (std::time::Duration::from_secs(60 * 30).as_micros() as i64) + { + return Err(std::io::Error::other("InvalidExpiresAt")); + } + + // validate signature (do this at the end because it's more expensive + info.agent + .verify(info.encoded.as_bytes(), &info.signature) + .map_err(|err| { + std::io::Error::other(format!("InvalidSignature: {err:?}")) + })?; + + let r = if info.is_tombstone { + None + } else { + Some(self.store.write(&info_raw)?) + }; + + self.space_map.update( + self.config.max_entries_per_space, + space, + Some((info, r)), + ); + + Ok((200, b"{}".to_vec())) + } + + /// Helper to get the next path segment as Bytes. + fn path_to_bytes(&mut self) -> std::io::Result { + use base64::prelude::*; + + let p = match self.path.pop() { + Some(p) => p, + None => return Err(std::io::Error::other("InvalidPathSegment")), + }; + + Ok(bytes::Bytes::copy_from_slice( + &BASE64_URL_SAFE_NO_PAD + .decode(p) + .map_err(std::io::Error::other)?, + )) + } + + /// Read the body while respecting our max message size. + fn read_body(&mut self) -> std::io::Result> { + // these are the same right now, but *could* be different + const MAX_INFO_SIZE: usize = 1024; + const READ_BUF_SIZE: usize = 1024; + + let mut buf = [0; READ_BUF_SIZE]; + let mut out = Vec::new(); + loop { + let read = match self.req.as_reader().read(&mut buf[..]) { + Ok(read) => read, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => { + continue; + } + Err(e) => return Err(e), + }; + if read == 0 { + return Ok(out); + } + out.extend_from_slice(&buf[..read]); + if out.len() > MAX_INFO_SIZE { + return Err(std::io::Error::other("InfoTooLarge")); + } + } + } + + /// Process the response. + fn respond(self, status: u16, bytes: Vec) -> std::io::Result<()> { + let len = bytes.len(); + self.req.respond(Response::new( + StatusCode(status), + vec![Header { + field: HeaderField::from_bytes(b"Content-Type").unwrap(), + value: std::str::FromStr::from_str("application/json").unwrap(), + }], + std::io::Cursor::new(bytes), + Some(len), + None, + )) + } +} diff --git a/crates/bootstrap_srv/src/space.rs b/crates/bootstrap_srv/src/space.rs new file mode 100644 index 0000000..52b0025 --- /dev/null +++ b/crates/bootstrap_srv/src/space.rs @@ -0,0 +1,228 @@ +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use crate::store::*; + +/// A map of spaces. +#[derive(Clone)] +pub struct SpaceMap(Arc>>); + +impl Default for SpaceMap { + fn default() -> Self { + Self(Arc::new(Mutex::new(HashMap::new()))) + } +} + +impl SpaceMap { + /// Read the content of a space. + pub fn read(&self, space: &bytes::Bytes) -> std::io::Result> { + // minimize outer mutex lock time + let space = self.0.lock().unwrap().get(space).cloned(); + + match space { + Some(space) => space.read(), + None => Ok(b"[]".to_vec()), + } + } + + /// Update all the spaces stored in this map. + pub fn update_all(&self, max_entries: usize) { + // minimize outer mutex lock time + let all = self.0.lock().unwrap().keys().cloned().collect::>(); + + for space in all { + self.update(max_entries, space, None); + } + } + + /// Update a space, clearing out any expired infos and optionally + /// adding a new incoming info. If adding a new info, perform all + /// validation before calling this function. + pub fn update( + &self, + max_entries: usize, + space: bytes::Bytes, + new_info: Option<(crate::ParsedEntry, Option)>, + ) { + // minimize outer mutex lock time + let space: Space = { + use std::collections::hash_map::Entry; + + let mut map = self.0.lock().unwrap(); + + if new_info.is_none() && !map.contains_key(&space) { + // we don't have this space, and we're not + // adding an info to it... nothing to do + return; + } + + match map.entry(space) { + Entry::Occupied(e) => { + // most other naive methods for clearing out dead spaces + // result in either races or deadlock, or they require + // us to hold the outer mutex lock for an unacceptably + // long time. So, just doing this for now. + if new_info.is_none() + && e.get().readable.lock().unwrap().is_empty() + { + e.remove(); + return; + } + + e.get().clone() + } + Entry::Vacant(e) => e.insert(Space::default()).clone(), + } + }; + + // do the actual update without the outer mutex locked + space.update(max_entries, new_info); + } +} + +/// A concurrent single space. +#[derive(Clone)] +struct Space { + // Using a separate write lock instead of a RwLock, because + // we still want to allow reads while the write_lock is locked. + // The write function will lock the readable when it is done processing + // and quickly update the reader, then drop both locks (read first). + write_lock: Arc>, + + // This list of store refs can be cheaply and quickly cloned, + // minimizing the mutex lock time. Then the actual data content can be + // read without blocking other threads. + readable: Arc>>, +} + +impl Default for Space { + fn default() -> Self { + Self { + write_lock: Arc::new(Mutex::new(())), + readable: Arc::new(Mutex::new(Vec::new())), + } + } +} + +impl Space { + /// Read the content of this space. + pub fn read(&self) -> std::io::Result> { + // keep the mutext lock time to a minimum + let list = self.readable.lock().unwrap().clone(); + + let mut len: usize = list.iter().map(StoreEntryRef::len).sum(); + + // plus square brackets + len += 2; + + // plus commas + if !list.is_empty() { + len += list.len() - 1; + } + + let mut out = Vec::with_capacity(len); + + out.push(b'['); + + let mut first = true; + + for r in list { + if first { + first = false; + } else { + out.push(b','); + } + r.read(&mut out)?; + } + + out.push(b']'); + + debug_assert_eq!(len, out.len()); + + Ok(out) + } + + /// Update a space, clearing out any expired infos and optionally + /// adding a new incoming info. If adding a new info, perform all + /// validation before calling this function. + pub fn update( + &self, + max_entries: usize, + mut new_info: Option<(crate::ParsedEntry, Option)>, + ) { + // get the current system time + let now = std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .expect("InvalidSystemTime") + .as_micros() as i64; + + // only allow one write at a time + let write_lock = self.write_lock.lock().unwrap(); + + // keep the readable lock time to a minimum + let list = self.readable.lock().unwrap().clone(); + + // if we have a new info to add, pull out the agent key + // so we can check to replace an existing one while + // iterating the existing infos + let replace_agent: Option = + new_info.as_ref().map(|(p, _)| p.agent); + + // parse the current entries + let mut list = list + .into_iter() + .filter_map(|mut store| { + let mut parsed = match store.parse() { + Ok(p) => p, + Err(_) => return None, + }; + + // if this matches an existing info, and the new one is newer + // use the new one. note, we'll still check for expiration + // after this step. + if Some(parsed.agent) == replace_agent { + // can unwrap because this can only be true if is_some + let (new_p, new_s) = new_info.take().unwrap(); + if new_p.created_at > parsed.created_at { + if new_p.is_tombstone { + return None; + } + + parsed = new_p; + store = new_s.expect("RequireStoreRef"); + } + } + + // check for expiration + if parsed.expires_at <= now { + return None; + } + + Some((parsed, store)) + }) + .collect::>(); + + // if we still have a new info to add, it must not have matched + // any existing ones + if let Some((parsed, store)) = new_info { + if !parsed.is_tombstone { + // if we are full, follow the "default" strategy rule of + // deleting the half-way info + if list.len() >= max_entries { + list.remove(max_entries / 2); + } + + // push the new info onto the stack + list.push((parsed, store.expect("RequireStoreRef"))); + } + } + + // drop the parsed versions + let list = list.into_iter().map(|e| e.1).collect::>(); + + // update the reader at the end + *self.readable.lock().unwrap() = list; + + drop(write_lock); + } +} diff --git a/crates/bootstrap_srv/src/store.rs b/crates/bootstrap_srv/src/store.rs new file mode 100644 index 0000000..ca1ad0e --- /dev/null +++ b/crates/bootstrap_srv/src/store.rs @@ -0,0 +1,191 @@ +//! This is a virtual-memory inspired tempfile store. +//! +//! ### Rationale +//! +//! - We don't need to persist anything in the store beyond a single +//! process invocation, because the infos are going to expire after +//! a matter of minutes anyways, and peers will continue to re-publish them. +//! - We would like a server to be able to store more spaces and infos +//! than would reasonably fit in RAM. +//! +//! ### Implementation +//! +//! - Have a pool of tempfiles that can grow to match the worker thread +//! count if needed. +//! - Have the ability to Clone (reopen) these tempfile handles. +//! - Designate one file handle only per pool entry as writable. +//! - All other handles will be readonly. +//! - After writing an entry, return a reference with a readonly file handle +//! to the tempfile that was written to, and an offset/length to +//! allow readback of exactly the bytes written for that entry. +//! - Once a tempfile reaches a certain size, drop the write handle and +//! open a new tempfile for writing. +//! - The older read handles will persist the existence of the older tempfiles +//! until the last read reference is dropped, at which point the tempfile +//! will be cleaned up by the os. +//! - We can trust these files will cycle through at a rate similar to the +//! max expiration time on the infos they contain (30 minutes). + +use std::sync::{Arc, Mutex}; + +/// How large we should allow individual tempfiles to grow. +/// This is a balance between using up too much disk space +/// and having too many file handles open. +/// +/// Start with 10MiB? +const MAX_PER_TEMPFILE: u64 = 1024 * 1024 * 10; + +struct Writer { + writer: std::fs::File, + read_clone: Arc, +} + +type WriterPool = Arc>>; + +/// A reference to previously written data. +#[derive(Clone)] +pub struct StoreEntryRef { + read_clone: Arc, + offset: u64, + length: usize, +} + +impl StoreEntryRef { + /// Get the length of data to be read. + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.length + } + + /// Read the content of this entry. This will extend the provided + /// buffer with the bytes read from the store. + pub fn read(&self, buf: &mut Vec) -> std::io::Result<()> { + use std::io::{Read, Seek}; + let mut reader = self.read_clone.reopen()?; + reader.seek(std::io::SeekFrom::Start(self.offset))?; + + buf.reserve(self.length); + + unsafe { + let offset = buf.len(); + buf.set_len(offset + self.length); + + if let Err(err) = + reader.read_exact(&mut buf[offset..offset + self.length]) + { + // On read error, undo the set_len. + buf.set_len(offset); + + return Err(err); + } + } + + Ok(()) + } + + /// Parse this entry. + pub fn parse(&self) -> std::io::Result { + let mut tmp = Vec::with_capacity(self.length); + self.read(&mut tmp)?; + crate::ParsedEntry::from_slice(&tmp) + } +} + +/// Tempfile-based virtual memory solution. +pub struct Store { + writer_pool: WriterPool, +} + +impl Default for Store { + fn default() -> Self { + Self { + writer_pool: Arc::new( + Mutex::new(std::collections::VecDeque::new()), + ), + } + } +} + +impl Store { + /// Write an entry to the virtual memory store, getting back + /// a reference that will allow future reading. + pub fn write(&self, content: &[u8]) -> std::io::Result { + use std::io::{Seek, Write}; + let mut writer = { + match self.writer_pool.lock().unwrap().pop_front() { + Some(writer) => writer, + None => { + let read_clone = Arc::new(tempfile::NamedTempFile::new()?); + let writer = read_clone.reopen()?; + Writer { writer, read_clone } + } + } + }; + let read_clone = writer.read_clone.clone(); + let offset = writer.writer.stream_position()?; + let length = content.len(); + + writer.writer.write_all(content)?; + writer.writer.sync_data()?; + + if offset + (length as u64) < MAX_PER_TEMPFILE { + self.writer_pool.lock().unwrap().push_back(writer); + } + + Ok(StoreEntryRef { + read_clone, + offset, + length, + }) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn happy_sanity() { + let s = Store::default(); + + let hello = s.write(b"Hello ").unwrap(); + let world = s.write(b"world!").unwrap(); + + let mut buf = Vec::new(); + hello.read(&mut buf).unwrap(); + world.read(&mut buf).unwrap(); + + assert_eq!(b"Hello world!", buf.as_slice()); + } + + #[test] + fn happy_multi_thread_sanity() { + const COUNT: usize = 10; + let mut all = Vec::with_capacity(COUNT); + + let s = Arc::new(Store::default()); + let b = Arc::new(std::sync::Barrier::new(COUNT)); + + let (send, recv) = std::sync::mpsc::channel(); + + for i in 0..COUNT { + let send = send.clone(); + let s = s.clone(); + let b = b.clone(); + + all.push(std::thread::spawn(move || { + b.wait(); + let wrote = format!("index:{i}"); + let r = s.write(wrote.as_bytes()).unwrap(); + send.send((wrote, r)).unwrap(); + })); + } + + for _ in 0..COUNT { + let (wrote, r) = recv.recv().unwrap(); + let mut read = Vec::new(); + r.read(&mut read).unwrap(); + assert_eq!(wrote.as_bytes(), read); + } + } +} From 59abeeab1c6392848ed5f1d01fae1228d04e0656 Mon Sep 17 00:00:00 2001 From: David Braden Date: Fri, 22 Nov 2024 09:48:26 -0700 Subject: [PATCH 03/13] Update crates/bootstrap_srv/src/space.rs Co-authored-by: mattyg --- crates/bootstrap_srv/src/space.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bootstrap_srv/src/space.rs b/crates/bootstrap_srv/src/space.rs index 52b0025..608da8e 100644 --- a/crates/bootstrap_srv/src/space.rs +++ b/crates/bootstrap_srv/src/space.rs @@ -107,7 +107,7 @@ impl Default for Space { impl Space { /// Read the content of this space. pub fn read(&self) -> std::io::Result> { - // keep the mutext lock time to a minimum + // keep the mutex lock time to a minimum let list = self.readable.lock().unwrap().clone(); let mut len: usize = list.iter().map(StoreEntryRef::len).sum(); From 47296a7bf3081360dc6330343bb93ae17d3f639f Mon Sep 17 00:00:00 2001 From: David Braden Date: Fri, 22 Nov 2024 10:10:03 -0700 Subject: [PATCH 04/13] Update crates/bootstrap_srv/src/store.rs --- crates/bootstrap_srv/src/store.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/bootstrap_srv/src/store.rs b/crates/bootstrap_srv/src/store.rs index ca1ad0e..2d51d92 100644 --- a/crates/bootstrap_srv/src/store.rs +++ b/crates/bootstrap_srv/src/store.rs @@ -7,6 +7,8 @@ //! a matter of minutes anyways, and peers will continue to re-publish them. //! - We would like a server to be able to store more spaces and infos //! than would reasonably fit in RAM. +//! - We would like to avoid use of an external database, to reduce +//! dependencies and ease deployment and testing. //! //! ### Implementation //! From 263d5ab3f9e742b19c46ce7a11d370205ad904ae Mon Sep 17 00:00:00 2001 From: neonphog Date: Fri, 22 Nov 2024 13:54:52 -0700 Subject: [PATCH 05/13] address review comments --- crates/bootstrap_srv/src/server.rs | 28 +++++++++++++++++----------- crates/bootstrap_srv/src/space.rs | 9 ++++++--- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/crates/bootstrap_srv/src/server.rs b/crates/bootstrap_srv/src/server.rs index 208bfed..1cb2cf1 100644 --- a/crates/bootstrap_srv/src/server.rs +++ b/crates/bootstrap_srv/src/server.rs @@ -5,6 +5,16 @@ use std::sync::Arc; use crate::*; use tiny_http::*; +/// Don't allow created_at to be greater or less than this far away from now. +/// 3 minutes. +const CREATED_AT_CLOCK_SKEW_ALLOWED_MICROS: i64 = + std::time::Duration::from_secs(60 * 3).as_micros() as i64; + +/// Don't allow expires_at - created_at to be greater than this duration. +/// 30 minutes. +const EXPIRES_AT_DURATION_MAX_ALLOWED_MICROS: i64 = + std::time::Duration::from_secs(60 * 30).as_micros() as i64; + /// An actual kitsune2_bootstrap_srv server instance. /// /// This server is built to be direct, light-weight, and responsive. @@ -68,10 +78,10 @@ impl BootstrapSrv { } // also set up a worker for pruning expired infos - let maint_cont = cont.clone(); - let maint_space_map = space_map.clone(); + let prune_cont = cont.clone(); + let prune_space_map = space_map.clone(); workers.push(std::thread::spawn(move || { - maint_worker(config, maint_cont, maint_space_map) + prune_worker(config, prune_cont, prune_space_map) })); Ok(Self { @@ -87,7 +97,7 @@ impl BootstrapSrv { } } -fn maint_worker( +fn prune_worker( config: Arc, cont: Arc, space_map: crate::SpaceMap, @@ -222,16 +232,12 @@ impl<'lt> Handler<'lt> { } // validate created at is not older than 3 min ago - if info.created_at - < now - (std::time::Duration::from_secs(60 * 3).as_micros() as i64) - { + if info.created_at < now - CREATED_AT_CLOCK_SKEW_ALLOWED_MICROS { return Err(std::io::Error::other("InvalidCreatedAt")); } // validate created at is less than 3 min in the future - if info.created_at - > now + (std::time::Duration::from_secs(60 * 3).as_micros() as i64) - { + if info.created_at > now + CREATED_AT_CLOCK_SKEW_ALLOWED_MICROS { return Err(std::io::Error::other("InvalidCreatedAt")); } @@ -247,7 +253,7 @@ impl<'lt> Handler<'lt> { // validate expires_at is not more than 30 min after created_at if info.expires_at - info.created_at - > (std::time::Duration::from_secs(60 * 30).as_micros() as i64) + > EXPIRES_AT_DURATION_MAX_ALLOWED_MICROS { return Err(std::io::Error::other("InvalidExpiresAt")); } diff --git a/crates/bootstrap_srv/src/space.rs b/crates/bootstrap_srv/src/space.rs index 608da8e..05943c8 100644 --- a/crates/bootstrap_srv/src/space.rs +++ b/crates/bootstrap_srv/src/space.rs @@ -3,9 +3,12 @@ use std::sync::{Arc, Mutex}; use crate::store::*; +/// The space identifier. +pub type SpaceId = bytes::Bytes; + /// A map of spaces. #[derive(Clone)] -pub struct SpaceMap(Arc>>); +pub struct SpaceMap(Arc>>); impl Default for SpaceMap { fn default() -> Self { @@ -15,7 +18,7 @@ impl Default for SpaceMap { impl SpaceMap { /// Read the content of a space. - pub fn read(&self, space: &bytes::Bytes) -> std::io::Result> { + pub fn read(&self, space: &SpaceId) -> std::io::Result> { // minimize outer mutex lock time let space = self.0.lock().unwrap().get(space).cloned(); @@ -41,7 +44,7 @@ impl SpaceMap { pub fn update( &self, max_entries: usize, - space: bytes::Bytes, + space: SpaceId, new_info: Option<(crate::ParsedEntry, Option)>, ) { // minimize outer mutex lock time From d265c14d98740c2c65a39d1fc0a9ff4d1530ef4c Mon Sep 17 00:00:00 2001 From: David Braden Date: Mon, 25 Nov 2024 13:00:24 -0700 Subject: [PATCH 06/13] Update crates/bootstrap_srv/src/server.rs Co-authored-by: Jost Schulte <28270981+jost-s@users.noreply.github.com> --- crates/bootstrap_srv/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bootstrap_srv/src/server.rs b/crates/bootstrap_srv/src/server.rs index 1cb2cf1..10953d8 100644 --- a/crates/bootstrap_srv/src/server.rs +++ b/crates/bootstrap_srv/src/server.rs @@ -91,7 +91,7 @@ impl BootstrapSrv { }) } - /// Get the bound listinging address of this server. + /// Get the bound listening address of this server. pub fn listen_addr(&self) -> std::net::SocketAddr { self.addr } From a4ac1e1156bc1026f9df6e19743edb0b22ad1e15 Mon Sep 17 00:00:00 2001 From: neonphog Date: Mon, 25 Nov 2024 13:06:35 -0700 Subject: [PATCH 07/13] review comment --- crates/bootstrap_srv/src/parse.rs | 2 +- crates/bootstrap_srv/src/server.rs | 2 +- crates/bootstrap_srv/src/store.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/bootstrap_srv/src/parse.rs b/crates/bootstrap_srv/src/parse.rs index 2970ac1..96a64bf 100644 --- a/crates/bootstrap_srv/src/parse.rs +++ b/crates/bootstrap_srv/src/parse.rs @@ -25,7 +25,7 @@ pub struct ParsedEntry { impl ParsedEntry { /// Parse entry from a slice. - pub fn from_slice(slice: &[u8]) -> std::io::Result { + pub fn try_from_slice(slice: &[u8]) -> std::io::Result { use base64::prelude::*; #[derive(serde::Deserialize)] diff --git a/crates/bootstrap_srv/src/server.rs b/crates/bootstrap_srv/src/server.rs index 10953d8..0bd2def 100644 --- a/crates/bootstrap_srv/src/server.rs +++ b/crates/bootstrap_srv/src/server.rs @@ -219,7 +219,7 @@ impl<'lt> Handler<'lt> { let agent = self.path_to_bytes()?; let info_raw = self.read_body()?; - let info = crate::ParsedEntry::from_slice(&info_raw)?; + let info = crate::ParsedEntry::try_from_slice(&info_raw)?; // validate agent matches url path if *agent != *info.agent.as_bytes() { diff --git a/crates/bootstrap_srv/src/store.rs b/crates/bootstrap_srv/src/store.rs index 2d51d92..860bb5d 100644 --- a/crates/bootstrap_srv/src/store.rs +++ b/crates/bootstrap_srv/src/store.rs @@ -89,7 +89,7 @@ impl StoreEntryRef { pub fn parse(&self) -> std::io::Result { let mut tmp = Vec::with_capacity(self.length); self.read(&mut tmp)?; - crate::ParsedEntry::from_slice(&tmp) + crate::ParsedEntry::try_from_slice(&tmp) } } From ebac6ff5dcaff4f2b78c66fab4e696e75c504847 Mon Sep 17 00:00:00 2001 From: neonphog Date: Tue, 26 Nov 2024 13:25:30 -0700 Subject: [PATCH 08/13] review comment --- crates/bootstrap_srv/src/server.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/bootstrap_srv/src/server.rs b/crates/bootstrap_srv/src/server.rs index 0bd2def..f562864 100644 --- a/crates/bootstrap_srv/src/server.rs +++ b/crates/bootstrap_srv/src/server.rs @@ -28,10 +28,7 @@ pub struct BootstrapSrv { impl Drop for BootstrapSrv { fn drop(&mut self) { - self.cont.store(false, std::sync::atomic::Ordering::SeqCst); - for worker in self.workers.drain(..) { - let _ = worker.join().expect("Failure shutting down worker thread"); - } + let _ = self.shutdown(); } } @@ -91,6 +88,23 @@ impl BootstrapSrv { }) } + /// Shutdown the server, returning an error result if any + /// of the worker threads had panicked. + pub fn shutdown(&mut self) -> std::io::Result<()> { + let mut is_err = false; + self.cont.store(false, std::sync::atomic::Ordering::SeqCst); + for worker in self.workers.drain(..) { + if worker.join().is_err() { + is_err = true; + } + } + if is_err { + Err(std::io::Error::other("Failure shutting down worker thread")) + } else { + Ok(()) + } + } + /// Get the bound listening address of this server. pub fn listen_addr(&self) -> std::net::SocketAddr { self.addr From 85dffc79aec3d3d5493b4cab9c9ddabce0bc6bfb Mon Sep 17 00:00:00 2001 From: neonphog Date: Tue, 26 Nov 2024 13:29:22 -0700 Subject: [PATCH 09/13] review comment --- crates/bootstrap_srv/src/parse.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bootstrap_srv/src/parse.rs b/crates/bootstrap_srv/src/parse.rs index 96a64bf..73e1c11 100644 --- a/crates/bootstrap_srv/src/parse.rs +++ b/crates/bootstrap_srv/src/parse.rs @@ -1,4 +1,4 @@ -/// An entry with known content. +/// An entry with known content: [crate::spec#1-types]. #[derive(Debug)] pub struct ParsedEntry { /// agent From b407784ebfeca5aa999c249cce3f2871ed1350ff Mon Sep 17 00:00:00 2001 From: neonphog Date: Tue, 26 Nov 2024 13:30:39 -0700 Subject: [PATCH 10/13] review comment --- crates/bootstrap_srv/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bootstrap_srv/src/server.rs b/crates/bootstrap_srv/src/server.rs index f562864..a041d1c 100644 --- a/crates/bootstrap_srv/src/server.rs +++ b/crates/bootstrap_srv/src/server.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use crate::*; use tiny_http::*; -/// Don't allow created_at to be greater or less than this far away from now. +/// Don't allow created_at to be greater than this far away from now. /// 3 minutes. const CREATED_AT_CLOCK_SKEW_ALLOWED_MICROS: i64 = std::time::Duration::from_secs(60 * 3).as_micros() as i64; From 79232349486ffdc6839cdc54abc9e96a87b4eae4 Mon Sep 17 00:00:00 2001 From: neonphog Date: Tue, 26 Nov 2024 13:34:00 -0700 Subject: [PATCH 11/13] review comment --- crates/bootstrap_srv/src/server.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/bootstrap_srv/src/server.rs b/crates/bootstrap_srv/src/server.rs index a041d1c..47f63c9 100644 --- a/crates/bootstrap_srv/src/server.rs +++ b/crates/bootstrap_srv/src/server.rs @@ -15,6 +15,15 @@ const CREATED_AT_CLOCK_SKEW_ALLOWED_MICROS: i64 = const EXPIRES_AT_DURATION_MAX_ALLOWED_MICROS: i64 = std::time::Duration::from_secs(60 * 30).as_micros() as i64; +/// Print out a message if this thread dies. +struct ThreadGuard(&'static str); + +impl Drop for ThreadGuard { + fn drop(&mut self) { + eprintln!("{}", self.0); + } +} + /// An actual kitsune2_bootstrap_srv server instance. /// /// This server is built to be direct, light-weight, and responsive. @@ -116,6 +125,8 @@ fn prune_worker( cont: Arc, space_map: crate::SpaceMap, ) -> std::io::Result<()> { + let _g = ThreadGuard("WARN: prune_worker thread has ended"); + let mut last_check = std::time::Instant::now(); while cont.load(std::sync::atomic::Ordering::SeqCst) { @@ -138,6 +149,8 @@ fn worker( server: Arc, space_map: crate::SpaceMap, ) -> std::io::Result<()> { + let _g = ThreadGuard("WARN: worker thread has ended"); + while cont.load(std::sync::atomic::Ordering::SeqCst) { let req = match server.recv_timeout(config.request_listen_duration)? { Some(req) => req, From 68c1004a59438347f8498756f89870514f7e7fe5 Mon Sep 17 00:00:00 2001 From: neonphog Date: Tue, 26 Nov 2024 13:36:29 -0700 Subject: [PATCH 12/13] review comment --- crates/bootstrap_srv/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bootstrap_srv/src/server.rs b/crates/bootstrap_srv/src/server.rs index 47f63c9..a1bcea0 100644 --- a/crates/bootstrap_srv/src/server.rs +++ b/crates/bootstrap_srv/src/server.rs @@ -285,7 +285,7 @@ impl<'lt> Handler<'lt> { return Err(std::io::Error::other("InvalidExpiresAt")); } - // validate signature (do this at the end because it's more expensive + // validate signature (do this at the end because it's more expensive) info.agent .verify(info.encoded.as_bytes(), &info.signature) .map_err(|err| { From ea028b02bacba5c3010e18b1998d9aa7f669c282 Mon Sep 17 00:00:00 2001 From: neonphog Date: Tue, 26 Nov 2024 13:41:50 -0700 Subject: [PATCH 13/13] review comment --- crates/bootstrap_srv/src/store.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/bootstrap_srv/src/store.rs b/crates/bootstrap_srv/src/store.rs index 860bb5d..e23a8d2 100644 --- a/crates/bootstrap_srv/src/store.rs +++ b/crates/bootstrap_srv/src/store.rs @@ -24,7 +24,10 @@ //! open a new tempfile for writing. //! - The older read handles will persist the existence of the older tempfiles //! until the last read reference is dropped, at which point the tempfile -//! will be cleaned up by the os. +//! will be cleaned up by the os. The drop impls on the +//! [tempfile::NamedTempFile] instances themselves will attempt the cleanup. +//! If we find some systems (looking at you Windows...) fail to do the +//! cleanup, we can add something more explicit in our code here. //! - We can trust these files will cycle through at a rate similar to the //! max expiration time on the infos they contain (30 minutes).