From 9b4aea86846a5131bc6f672023cae5064bf9645c Mon Sep 17 00:00:00 2001 From: Joe Runde Date: Mon, 20 May 2024 15:35:56 -0700 Subject: [PATCH] :recycle: move metrics into one file (#97) #### Motivation This PR accomplishes two things: - Upgrades the `metrics` crate to the latest version, which causes API breaking changes - Duplicates all counter metrics to include `{metric_name}_total` to align with the prometheus metrics exported by vLLM #### Modifications This refactors all usages of the `metrics` crate into a single file, so that changes to how it's used can be made in one place. This lets us easily duplicate all the counter metrics. #### Result No existing behavior should change, only new `*_total` counters should be added to the /metrics endpoint. --------- Signed-off-by: Joe Runde --- Cargo.lock | 748 +++++++++++++++++++++++--------------- Dockerfile | 4 +- router/Cargo.toml | 4 +- router/src/batcher.rs | 55 +-- router/src/grpc_server.rs | 55 ++- router/src/lib.rs | 1 + router/src/metrics.rs | 34 ++ router/src/queue.rs | 23 +- router/src/validation.rs | 11 +- 9 files changed, 559 insertions(+), 376 deletions(-) create mode 100644 router/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 7a086a05..76c96c74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,47 +40,48 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.13" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" dependencies = [ "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" dependencies = [ "anstyle", "windows-sys 0.52.0", @@ -88,9 +89,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.81" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" +checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" [[package]] name = "async-lock" @@ -127,9 +128,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.78" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "461abc97219de0eaaf81fe3ef974a540158f3d079c2ab200f891f1a2ef201e85" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", @@ -138,15 +139,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "aws-lc-rs" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5509d663b2c00ee421bda8d6a24d6c42e15970957de1701b8df9f6fbe5707df1" +checksum = "8487b59d62764df8231cb371c459314df895b41756df457a1fb1243d65c89195" dependencies = [ "aws-lc-sys", "mirai-annotations", @@ -156,9 +157,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5d317212c2a78d86ba6622e969413c38847b62f48111f8b763af3dac2f9840" +checksum = "c15eb61145320320eb919d9bab524617a7aa4216c78d342fae3a758bc33073e4" dependencies = [ "bindgen", "cc", @@ -180,9 +181,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit", "memchr", @@ -210,8 +211,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -226,7 +227,7 @@ checksum = "164b95427e83b79583c7699a72b4a6b485a12bbdef5b5c054ee5ff2296d82f52" dependencies = [ "axum", "futures", - "http", + "http 0.2.12", "opentelemetry 0.18.0", "tower", "tower-http", @@ -236,9 +237,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.70" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95d8e92cac0961e91dbd517496b00f7e9b92363dbe6d42c3198268323798860c" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -261,6 +262,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bindgen" version = "0.69.4" @@ -298,24 +305,25 @@ checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "bumpalo" -version = "3.15.4" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.90" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" dependencies = [ "jobserver", "libc", + "once_cell", ] [[package]] @@ -369,7 +377,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.0", + "strsim", ] [[package]] @@ -378,7 +386,7 @@ version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "syn", @@ -401,15 +409,15 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" [[package]] name = "concurrent-queue" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" dependencies = [ "crossbeam-utils", ] @@ -427,6 +435,22 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + [[package]] name = "crossbeam-channel" version = "0.5.12" @@ -473,9 +497,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" +checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" dependencies = [ "darling_core", "darling_macro", @@ -483,23 +507,23 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" +checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", + "strsim", "syn", ] [[package]] name = "darling_macro" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" +checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ "darling_core", "quote", @@ -513,7 +537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -558,9 +582,9 @@ checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" [[package]] name = "either" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" [[package]] name = "encode_unicode" @@ -576,9 +600,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", @@ -627,9 +651,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "fixedbitset" @@ -776,9 +800,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "js-sys", @@ -810,8 +834,27 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", - "indexmap 2.2.5", + "http 0.2.12", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.1.0", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -826,25 +869,13 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.13.1" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", ] -[[package]] -name = "hashbrown" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -877,6 +908,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -884,7 +926,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -916,9 +981,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -930,18 +995,75 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-timeout" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.28", "pin-project-lite", "tokio", "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.3.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -960,12 +1082,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.5" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -996,6 +1118,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + [[package]] name = "itertools" version = "0.11.0" @@ -1016,15 +1144,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" dependencies = [ "libc", ] @@ -1052,9 +1180,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "libloading" @@ -1063,7 +1191,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -1074,9 +1202,9 @@ checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -1088,15 +1216,6 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" -[[package]] -name = "mach2" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" -dependencies = [ - "libc", -] - [[package]] name = "macro_rules_attribute" version = "0.2.0" @@ -1130,62 +1249,53 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "metrics" -version = "0.21.1" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" +checksum = "2be3cbd384d4e955b231c895ce10685e3d8260c5ccffae898c96c723b0772835" dependencies = [ "ahash", - "metrics-macros", "portable-atomic", ] [[package]] name = "metrics-exporter-prometheus" -version = "0.12.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950" +checksum = "5d58e362dc7206e9456ddbcdbd53c71ba441020e62104703075a69151e38d85f" dependencies = [ - "base64 0.21.7", - "hyper", - "indexmap 1.9.3", + "base64 0.22.1", + "http-body-util", + "hyper 1.3.1", + "hyper-tls", + "hyper-util", + "indexmap 2.2.6", "ipnet", "metrics", "metrics-util", - "quanta 0.11.1", + "quanta", "thiserror", "tokio", "tracing", ] -[[package]] -name = "metrics-macros" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "metrics-util" -version = "0.15.1" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" +checksum = "8b07a5eb561b8cbc16be2d216faf7757f9baf3bfb94dbb0fae3df8387a5bb47f" dependencies = [ "crossbeam-epoch", "crossbeam-utils", - "hashbrown 0.13.1", + "hashbrown 0.14.5", "metrics", "num_cpus", - "quanta 0.11.1", + "quanta", "sketches-ddsketch", ] @@ -1229,9 +1339,9 @@ checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" [[package]] name = "moka" -version = "0.12.6" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87bfd249f570638bfb0b4f9d258e6b8cddd2a5a7d0ed47e8bb8b176bfc0e7a17" +checksum = "9e0d88686dc561d743b40de8269b26eaf0dc58781bde087b0984646602021d08" dependencies = [ "async-lock", "async-trait", @@ -1242,7 +1352,7 @@ dependencies = [ "futures-util", "once_cell", "parking_lot", - "quanta 0.12.2", + "quanta", "rustc_version", "smallvec", "tagptr", @@ -1253,9 +1363,9 @@ dependencies = [ [[package]] name = "monostate" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20fffcd8ca4c69d31e036a71abc400147b41f90895df4edcb36497a1f8af8bf" +checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e" dependencies = [ "monostate-impl", "serde", @@ -1263,9 +1373,9 @@ dependencies = [ [[package]] name = "monostate-impl" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf307cbbbd777a9c10cec88ddafee572b3484caad5cce0c9236523c3803105a6" +checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" dependencies = [ "proc-macro2", "quote", @@ -1274,9 +1384,9 @@ dependencies = [ [[package]] name = "multimap" -version = "0.8.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" [[package]] name = "nanorand" @@ -1287,6 +1397,24 @@ dependencies = [ "getrandom", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.28.0" @@ -1327,9 +1455,9 @@ dependencies = [ [[package]] name = "num" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135b08af27d103b0a51f2ae0f8632117b7b185ccf931445affa8df530576a41" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ "num-bigint", "num-complex", @@ -1341,20 +1469,19 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" dependencies = [ - "autocfg", "num-integer", "num-traits", ] [[package]] name = "num-complex" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", ] @@ -1370,9 +1497,9 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", "num-integer", @@ -1381,11 +1508,10 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "autocfg", "num-bigint", "num-integer", "num-traits", @@ -1393,9 +1519,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] @@ -1479,6 +1605,12 @@ dependencies = [ "syn", ] +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "openssl-sys" version = "0.9.102" @@ -1524,7 +1656,7 @@ checksum = "1a016b8d9495c639af2145ac22387dcb88e44118e45320d9238fbf4e7889abcb" dependencies = [ "async-trait", "futures-core", - "http", + "http 0.2.12", "opentelemetry 0.22.0", "opentelemetry-proto", "opentelemetry-semantic-conventions", @@ -1636,9 +1768,9 @@ checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" dependencies = [ "lock_api", "parking_lot_core", @@ -1646,22 +1778,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.5", ] [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "percent-encoding" @@ -1671,12 +1803,12 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.2.5", + "indexmap 2.2.6", ] [[package]] @@ -1701,9 +1833,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -1731,9 +1863,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.16" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", "syn", @@ -1741,9 +1873,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.81" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" +checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" dependencies = [ "unicode-ident", ] @@ -1760,13 +1892,13 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" +checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1" dependencies = [ "bytes", - "heck 0.4.1", - "itertools 0.11.0", + "heck", + "itertools 0.12.1", "log", "multimap", "once_cell", @@ -1777,14 +1909,13 @@ dependencies = [ "regex", "syn", "tempfile", - "which", ] [[package]] name = "prost-derive" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" +checksum = "9554e3ab233f0a932403704f1a1d08c30d5ccd931adfdfa1e8b5a19b52c1d55a" dependencies = [ "anyhow", "itertools 0.12.1", @@ -1795,39 +1926,23 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" +checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe" dependencies = [ "prost", ] [[package]] name = "quanta" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" -dependencies = [ - "crossbeam-utils", - "libc", - "mach2", - "once_cell", - "raw-cpuid 10.7.0", - "wasi", - "web-sys", - "winapi", -] - -[[package]] -name = "quanta" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca0b7bac0b97248c40bb77288fc52029cf1459c0461ea1b05ee32ccf011de2c" +checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5" dependencies = [ "crossbeam-utils", "libc", "once_cell", - "raw-cpuid 11.0.1", + "raw-cpuid", "wasi", "web-sys", "winapi", @@ -1835,9 +1950,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -1874,18 +1989,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "10.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "raw-cpuid" -version = "11.0.1" +version = "11.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" +checksum = "e29830cbb1290e404f24c73af91c5d8d631ce7e128691e9477556b540cd01ecd" dependencies = [ "bitflags 2.5.0", ] @@ -1923,23 +2029,23 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", ] [[package]] name = "regex" -version = "1.10.3" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", "regex-automata 0.4.6", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", ] [[package]] @@ -1959,7 +2065,7 @@ checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", ] [[package]] @@ -1970,9 +2076,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "ring" @@ -1991,9 +2097,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" @@ -2012,9 +2118,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.32" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.5.0", "errno", @@ -2039,9 +2145,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.4" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c4d6d8ad9f2492485e13453acbb291dd08f64441b6609c491f1c2cd2c6b4fe1" +checksum = "afabcee0551bd1aa3e18e5adbf2c0544722014b899adb31bd186ec638d3da97e" dependencies = [ "aws-lc-rs", "log", @@ -2054,25 +2160,25 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.4.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868e20fada228fefaf6b652e00cc73623d54f8171e7352c18bb281571f2d92da" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" -version = "0.102.2" +version = "0.102.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" dependencies = [ "aws-lc-rs", "ring", @@ -2082,15 +2188,24 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "ryu" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] [[package]] name = "scopeguard" @@ -2098,26 +2213,49 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +dependencies = [ + "bitflags 2.5.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.198" +version = "1.0.202" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" +checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.198" +version = "1.0.202" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" +checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838" dependencies = [ "proc-macro2", "quote", @@ -2126,9 +2264,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "itoa", "ryu", @@ -2174,9 +2312,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] @@ -2204,9 +2342,9 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2235,15 +2373,9 @@ dependencies = [ [[package]] name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "strsim" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "subtle" @@ -2253,9 +2385,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.60" +version = "2.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +checksum = "bf5be731623ca1a1fb7d8be6f261a3be6d3e2337b8a1f97be944d020c8fcb704" dependencies = [ "proc-macro2", "quote", @@ -2323,8 +2455,8 @@ dependencies = [ "clap", "flume", "futures", - "h2", - "hyper", + "h2 0.3.26", + "hyper 0.14.28", "metrics", "metrics-exporter-prometheus", "moka", @@ -2359,18 +2491,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" dependencies = [ "proc-macro2", "quote", @@ -2409,7 +2541,7 @@ dependencies = [ "rayon", "rayon-cond", "regex", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", "serde", "serde_json", "spm_precompiled", @@ -2459,6 +2591,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.25.0" @@ -2476,7 +2618,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.4", + "rustls 0.23.5", "rustls-pki-types", "tokio", ] @@ -2494,16 +2636,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -2517,10 +2658,10 @@ dependencies = [ "axum", "base64 0.21.7", "bytes", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", @@ -2579,8 +2720,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "http-range-header", "pin-project-lite", "tower-layer", @@ -2772,9 +2913,9 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" [[package]] name = "unicode_categories" @@ -2966,7 +3107,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -2986,17 +3127,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.4", - "windows_aarch64_msvc 0.52.4", - "windows_i686_gnu 0.52.4", - "windows_i686_msvc 0.52.4", - "windows_x86_64_gnu 0.52.4", - "windows_x86_64_gnullvm 0.52.4", - "windows_x86_64_msvc 0.52.4", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -3007,9 +3149,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -3019,9 +3161,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -3031,9 +3173,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.4" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -3043,9 +3191,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -3055,9 +3203,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -3067,9 +3215,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -3079,24 +3227,24 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", diff --git a/Dockerfile b/Dockerfile index b70d9634..624f1bd5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,8 +85,8 @@ ENV LIBRARY_PATH="$CUDA_HOME/lib64/stubs" ## Rust builder ################################################################ -# Specific debian version so that compatible glibc version is used -FROM rust:1.77.2-bullseye as rust-builder +# Using bookworm for compilation so the rust binaries get linked against libssl.so.3 +FROM rust:1.78-bookworm as rust-builder ARG PROTOC_VERSION ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse diff --git a/router/Cargo.toml b/router/Cargo.toml index 7c86ae5f..99f62b5f 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -20,8 +20,8 @@ text-generation-client = { path = "client" } clap = { version = "^4.5.4", features = ["derive", "env"] } futures = "^0.3.30" flume = "^0.11.0" -metrics = "0.21.1" -metrics-exporter-prometheus = { version = "0.12.2", features = [] } +metrics = "0.22.3" +metrics-exporter-prometheus = { version = "0.14.0", features = ["http-listener"] } moka = { version = "0.12.6", features = ["future"] } nohash-hasher = "^0.2.0" num = "^0.4.2" diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 0b395ffd..32a5359f 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -52,6 +52,7 @@ use crate::{ validation::RequestSize, ErrorResponse, GenerateRequest, }; +use crate::metrics::{increment_counter, increment_labeled_counter, observe_histogram, observe_labeled_histogram, set_gauge}; /// Batcher #[derive(Clone)] @@ -447,9 +448,9 @@ async fn batching_task( batch_size, ); - metrics::gauge!("tgi_batch_current_size", batch_size as f64); - metrics::gauge!("tgi_batch_input_tokens", batch_tokens as f64); - metrics::gauge!( + set_gauge("tgi_batch_current_size", batch_size as f64); + set_gauge("tgi_batch_input_tokens", batch_tokens as f64); + set_gauge( "tgi_batch_max_remaining_tokens", batch_max_remaining_tokens.unwrap() as f64 ); @@ -529,7 +530,7 @@ async fn batching_task( "Extending batch #{} of {} with additional batch #{} of {}", batch_id, batch_size, new_batch_id, added_batch_size ); - metrics::increment_counter!("tgi_batch_concatenation_count"); + increment_counter("tgi_batch_concatenation_count", 1); } } else { combined_batch_id = new_batch_id; @@ -560,9 +561,9 @@ async fn batching_task( } } - metrics::gauge!("tgi_batch_current_size", 0.0); - metrics::gauge!("tgi_batch_input_tokens", 0.0); - metrics::gauge!("tgi_batch_max_remaining_tokens", 0.0); + set_gauge("tgi_batch_current_size", 0.0); + set_gauge("tgi_batch_input_tokens", 0.0); + set_gauge("tgi_batch_max_remaining_tokens", 0.0); } info!("Batching loop exiting"); @@ -625,9 +626,9 @@ impl<'a> TokenProcessor<'a> { let batch_size = batch.requests.len(); let batch_tokens = batch.total_tokens; let start_time = Instant::now(); - metrics::histogram!("tgi_batch_next_tokens", batch_tokens as f64); - metrics::histogram!( - "tgi_batch_inference_batch_size", batch_size as f64, "method" => "prefill" + observe_histogram("tgi_batch_next_tokens", batch_tokens as f64); + observe_labeled_histogram( + "tgi_batch_inference_batch_size", &[("method", "prefill")], batch_size as f64 ); let (result, prefill_time) = self ._wrap_future( @@ -648,8 +649,8 @@ impl<'a> TokenProcessor<'a> { batches: Vec, queue: &mut Queue, ) -> (Option, Duration) { - metrics::histogram!( - "tgi_batch_inference_batch_size", self.entries.len() as f64, "method" => "next_token" + observe_labeled_histogram( + "tgi_batch_inference_batch_size", &[("method", "next_token")], self.entries.len() as f64 ); let start_time = Instant::now(); self._wrap_future( @@ -672,7 +673,7 @@ impl<'a> TokenProcessor<'a> { start_id: Option, queue: &mut Queue, ) -> (Option, Duration) { - metrics::increment_counter!("tgi_batch_inference_count", "method" => method); + increment_labeled_counter("tgi_batch_inference_count", &[("method", method)], 1); // We process the shared queue while waiting for the response from the python shard(s) let queue_servicer = queue.service_queue().fuse(); @@ -692,27 +693,27 @@ impl<'a> TokenProcessor<'a> { let completed_request_ids = self.process_next_tokens(generated_tokens, errors); // Update health self.generation_health.store(true, Ordering::SeqCst); - metrics::histogram!( + observe_labeled_histogram( "tgi_batch_inference_duration", + &[("method", method), + ("makeup", "single_only")], elapsed.as_secs_f64(), - "method" => method, - "makeup" => "single_only", // later will possibly be beam_only or mixed ); - metrics::histogram!( + observe_labeled_histogram( "tgi_batch_inference_forward_duration", - forward_duration, - "method" => method, - "makeup" => "single_only", // later will possibly be beam_only or mixed + &[("method", method), + ("makeup", "single_only")], + forward_duration.as_secs_f64(), ); - metrics::histogram!( + observe_labeled_histogram( "tgi_batch_inference_tokproc_duration", + &[("method", method), + ("makeup", "single_only")], pre_token_process_time.elapsed().as_secs_f64(), - "method" => method, - "makeup" => "single_only", // later will possibly be beam_only or mixed ); // Probably don't need this additional counter because the duration histogram // records a total count - metrics::increment_counter!("tgi_batch_inference_success", "method" => method); + increment_labeled_counter("tgi_batch_inference_success", &[("method", method)], 1); Some(CachedBatch { batch_id: next_batch_id, status: completed_request_ids.map(|c| RequestsStatus { completed_ids: c }), @@ -729,7 +730,7 @@ impl<'a> TokenProcessor<'a> { ClientError::Connection(_) => "connection", _ => "error", }; - metrics::increment_counter!("tgi_batch_inference_failure", "method" => method, "reason" => reason); + increment_labeled_counter("tgi_batch_inference_failure", &[("method", method), ("reason", reason)], 1); self.send_errors(err, start_id); None } @@ -980,7 +981,7 @@ impl<'a> TokenProcessor<'a> { // If receiver closed (request cancelled), cancel this entry let e = self.entries.remove(&request_id).unwrap(); stop_reason = Cancelled; - metrics::increment_counter!("tgi_request_failure", "err" => "cancelled"); + increment_labeled_counter("tgi_request_failure", &[("err", "cancelled")], 1); //TODO include request context in log message warn!( "Aborted streaming request {request_id} cancelled by client \ @@ -994,7 +995,7 @@ impl<'a> TokenProcessor<'a> { // If receiver closed (request cancelled), cancel this entry let e = self.entries.remove(&request_id).unwrap(); stop_reason = Cancelled; - metrics::increment_counter!("tgi_request_failure", "err" => "cancelled"); + increment_labeled_counter("tgi_request_failure", &[("err", "cancelled")], 1); //TODO include request context in log message warn!( "Aborted request {request_id} cancelled by client \ diff --git a/router/src/grpc_server.rs b/router/src/grpc_server.rs index f6cfbb7f..f5f8d0de 100644 --- a/router/src/grpc_server.rs +++ b/router/src/grpc_server.rs @@ -32,6 +32,7 @@ use crate::{ validation::{RequestSize, ValidationError}, GenerateParameters, GenerateRequest, }; +use crate::metrics::{increment_counter, increment_labeled_counter, observe_histogram}; use crate::pb::fmaas::tokenize_response::Offset; /// Whether to fail if sampling parameters are provided in greedy-mode requests @@ -67,8 +68,6 @@ pub(crate) async fn start_grpc_server + Send + 'static>( let grpc_service = GenerationServicer { state: shared_state, tokenizer, - input_counter: metrics::register_counter!("tgi_request_input_count"), - tokenize_input_counter: metrics::register_counter!("tgi_tokenize_request_input_count"), }; let grpc_server = builder .add_service(GenerationServiceServer::new(grpc_service)) @@ -92,8 +91,6 @@ async fn load_pem(path: String, name: &str) -> Vec { pub struct GenerationServicer { state: ServerState, tokenizer: AsyncTokenizer, - input_counter: metrics::Counter, - tokenize_input_counter: metrics::Counter, } #[tonic::async_trait] @@ -124,20 +121,20 @@ impl GenerationService for GenerationServicer { let br = request.into_inner(); let batch_size = br.requests.len(); let kind = if batch_size == 1 { "single" } else { "batch" }; - metrics::increment_counter!("tgi_request_count", "kind" => kind); + increment_labeled_counter("tgi_request_count", &[("kind", kind)], 1); if batch_size == 0 { return Ok(Response::new(BatchedGenerationResponse { responses: vec![], })); } - self.input_counter.increment(batch_size as u64); + increment_counter("tgi_request_input_count", batch_size as u64); // Limit concurrent requests by acquiring a permit from the semaphore let _permit = self .state .limit_concurrent_requests .try_acquire_many(batch_size as u32) .map_err(|_| { - metrics::increment_counter!("tgi_request_failure", "err" => "conc_limit"); + increment_labeled_counter("tgi_request_failure", &[("err", "conc_limit")], 1); tracing::error!("Model is overloaded"); Status::resource_exhausted("Model is overloaded") })?; @@ -217,11 +214,11 @@ impl GenerationService for GenerationServicer { } .map_err(|err| match err { InferError::RequestQueueFull() => { - metrics::increment_counter!("tgi_request_failure", "err" => "queue_full"); + increment_labeled_counter("tgi_request_failure", &[("err", "queue_full")], 1); Status::resource_exhausted(err.to_string()) } _ => { - metrics::increment_counter!("tgi_request_failure", "err" => "generate"); + increment_labeled_counter("tgi_request_failure", &[("err", "generate")], 1); tracing::error!("{err}"); Status::from_error(Box::new(err)) } @@ -254,15 +251,15 @@ impl GenerationService for GenerationServicer { ) -> Result, Status> { let start_time = Instant::now(); let request = request.extract_context(); - metrics::increment_counter!("tgi_request_count", "kind" => "stream"); - self.input_counter.increment(1); + increment_labeled_counter("tgi_request_count", &[("kind", "stream")], 1); + increment_counter("tgi_request_input_count", 1); let permit = self .state .limit_concurrent_requests .clone() .try_acquire_owned() .map_err(|_| { - metrics::increment_counter!("tgi_request_failure", "err" => "conc_limit"); + increment_labeled_counter("tgi_request_failure", &[("err", "conc_limit")], 1); tracing::error!("Model is overloaded"); Status::resource_exhausted("Model is overloaded") })?; @@ -292,7 +289,7 @@ impl GenerationService for GenerationServicer { |ctx, count, reason, request_id, times, out, err| { let _enter = ctx.span.enter(); if let Some(e) = err { - metrics::increment_counter!("tgi_request_failure", "err" => "generate"); + increment_labeled_counter("tgi_request_failure", &[("err", "generate")], 1); tracing::error!( "Streaming response failed after {count} tokens, \ output so far: '{:?}': {e}", @@ -322,11 +319,11 @@ impl GenerationService for GenerationServicer { .await .map_err(|err| match err { InferError::RequestQueueFull() => { - metrics::increment_counter!("tgi_request_failure", "err" => "queue_full"); + increment_labeled_counter("tgi_request_failure", &[("err", "queue_full")], 1); Status::resource_exhausted(err.to_string()) } _ => { - metrics::increment_counter!("tgi_request_failure", "err" => "unknown"); + increment_labeled_counter("tgi_request_failure", &[("err", "unknown")], 1); tracing::error!("{err}"); Status::from_error(Box::new(err)) } @@ -341,9 +338,9 @@ impl GenerationService for GenerationServicer { request: Request, ) -> Result, Status> { let br = request.into_inner(); - metrics::increment_counter!("tgi_tokenize_request_count"); + increment_counter("tgi_tokenize_request_count", 1); let start_time = Instant::now(); - self.tokenize_input_counter.increment(br.requests.len() as u64); + increment_counter("tgi_tokenize_request_input_count", br.requests.len() as u64); let truncate_to = match br.truncate_input_tokens { 0 => u32::MAX, @@ -378,8 +375,8 @@ impl GenerationService for GenerationServicer { .await?; let token_total: u32 = responses.iter().map(|tr| tr.token_count).sum(); - metrics::histogram!("tgi_tokenize_request_tokens", token_total as f64); - metrics::histogram!( + observe_histogram("tgi_tokenize_request_tokens", token_total as f64); + observe_histogram( "tgi_tokenize_request_duration", start_time.elapsed().as_secs_f64() ); @@ -428,12 +425,12 @@ impl GenerationServicer { Err(err) => Err(err), } .map_err(|err| { - metrics::increment_counter!("tgi_request_failure", "err" => "validation"); + increment_labeled_counter("tgi_request_failure", &[("err", "validation")], 1); tracing::error!("{err}"); Status::invalid_argument(err.to_string()) }) .map(|requests| { - metrics::histogram!( + observe_histogram( "tgi_request_validation_duration", start_time.elapsed().as_secs_f64() ); @@ -474,11 +471,11 @@ fn log_response( span.record("total_time", format!("{total_time:?}")); span.record("input_toks", input_tokens); - metrics::histogram!( + observe_histogram( "tgi_request_inference_duration", inference_time.as_secs_f64() ); - metrics::histogram!( + observe_histogram( "tgi_request_mean_time_per_token_duration", time_per_token.as_secs_f64() ); @@ -486,15 +483,15 @@ fn log_response( // Metrics match reason { - Error => metrics::increment_counter!("tgi_request_failure", "err" => "generate"), + Error => increment_labeled_counter("tgi_request_failure", &[("err", "generate")], 1), Cancelled => (), // recorded where cancellation is detected _ => { - metrics::increment_counter!( - "tgi_request_success", "stop_reason" => reason.as_str_name(), "kind" => kind + increment_labeled_counter( + "tgi_request_success", &[("stop_reason", reason.as_str_name()), ("kind", kind)], 1 ); - metrics::histogram!("tgi_request_duration", total_time.as_secs_f64()); - metrics::histogram!("tgi_request_generated_tokens", generated_tokens as f64); - metrics::histogram!( + observe_histogram("tgi_request_duration", total_time.as_secs_f64()); + observe_histogram("tgi_request_generated_tokens", generated_tokens as f64); + observe_histogram( "tgi_request_total_tokens", (generated_tokens as usize + input_tokens) as f64 ); diff --git a/router/src/lib.rs b/router/src/lib.rs index ad09b8c5..572a9ffd 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -11,6 +11,7 @@ pub mod server; mod tokenizer; mod validation; mod tracing; +mod metrics; use batcher::Batcher; use serde::{Deserialize, Serialize}; diff --git a/router/src/metrics.rs b/router/src/metrics.rs new file mode 100644 index 00000000..820d5abe --- /dev/null +++ b/router/src/metrics.rs @@ -0,0 +1,34 @@ +// Small helpers for using the metrics crate. +// This aims to collect all usages of the metrics crate so that future api-breaking changes can be handled in one place. + + +// These counter helper methods will actually increment a second counter with `_total` appended to the name. +// This is for compatibility with other runtimes that use prometheus directly, which is very +// opinionated that all counters should end with the suffix _total. +// Cite: https://prometheus.github.io/client_python/instrumenting/counter/ + +pub fn increment_counter(name: &'static str, value: u64) { + metrics::counter!(name).increment(value); + metrics::counter!(format!("{name}_total")).increment(value); +} + + +pub fn increment_labeled_counter(name: &'static str, labels: &[(&'static str, &'static str)], value: u64) { + metrics::counter!(name, labels).increment(value); + metrics::counter!(format!("{name}_total"), labels).increment(value); +} + + +pub fn set_gauge(name: &'static str, value: f64) { + metrics::gauge!(name).set(value); +} + + +pub fn observe_histogram(name: &'static str, value: f64) { + metrics::histogram!(name).record(value); +} + + +pub fn observe_labeled_histogram(name: &'static str, labels: &[(&'static str, &'static str)], value: f64) { + metrics::histogram!(name, labels).record(value); +} diff --git a/router/src/queue.rs b/router/src/queue.rs index 89834733..8e32e57f 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -25,6 +25,7 @@ use crate::{ batch_types::BatchType, batcher::InferResponse, decoder::IncrementalDecoderWrapper, GenerateParameters, GenerateRequest, }; +use crate::metrics::{increment_counter, increment_labeled_counter, observe_histogram, set_gauge}; // Requests that fit into the next batch can overtake others // that don't as long as they arrive within this amount of time after @@ -199,13 +200,13 @@ impl Queue { let mut pruned = false; self.buffer.retain_mut(|entry| match entry { entry if entry.is_cancelled() => { - metrics::increment_counter!("tgi_request_failure", "err" => "cancelled"); + increment_labeled_counter("tgi_request_failure", &[("err", "cancelled")], 1); pruned = true; false } entry if entry.deadline_exceeded() => { // Send timeout response - metrics::increment_counter!("tgi_request_failure", "err" => "timeout"); + increment_labeled_counter("tgi_request_failure", &[("err", "timeout")], 1); entry.batch_time = Some(Instant::now()); entry .send_final(Ok(InferResponse::early_timeout(entry))) @@ -217,7 +218,7 @@ impl Queue { }); if pruned { - metrics::gauge!("tgi_queue_size", self.buffer.len() as f64); + set_gauge("tgi_queue_size", self.buffer.len() as f64); } while let Some(ents) = self.receiver.recv().await { @@ -227,7 +228,7 @@ impl Queue { fn add_to_buffer(&mut self, new_entries: Vec) { self.buffer.extend(new_entries); - metrics::gauge!("tgi_queue_size", self.buffer.len() as f64); + set_gauge("tgi_queue_size", self.buffer.len() as f64); } /// Get the next batch without blocking. @@ -340,14 +341,14 @@ impl Queue { time_cutoff.get_or_insert_with(|| entry.queue_time.add(CUTOFF_DURATION)); continue; } - metrics::increment_counter!("tgi_granular_batch_addition"); + increment_counter("tgi_granular_batch_addition", 1); } else if let Some(tree) = btree.as_mut() { // If we initialized the btree for a prior request, keep it updated tree.insert((output_len, input_len, tree.len())); } // Here, we can add this request to the batch without breaching memory limit if time_cutoff.is_some() { - metrics::increment_counter!("tgi_queue_jump"); + increment_counter("tgi_queue_jump", 1); } // Also check whether adding this request will breach the prefill weight limit @@ -361,7 +362,7 @@ impl Queue { .prefill_weight(&next_prefill_stats, batch_size); if prefill_weight > effective_prefill_weight_limit { skip = true; - metrics::increment_counter!("tgi_prefill_weight_limit_exceeded"); + increment_counter("tgi_prefill_weight_limit_exceeded", 1); } } if !skip && max_prefill_padding < 1.0 { @@ -370,7 +371,7 @@ impl Queue { skip = true; //TODO if we skip due to padding and added other requests from queue, // we could consider doing another pass since the padding proportion may have decreased - metrics::increment_counter!("tgi_prefill_padding_limit_exceeded"); + increment_counter("tgi_prefill_padding_limit_exceeded", 1); } } if skip { @@ -379,7 +380,7 @@ impl Queue { tree.remove(&(output_len, input_len, tree.len() - 1)); } time_cutoff.get_or_insert_with(|| entry.queue_time.add(CUTOFF_DURATION)); - metrics::increment_counter!("tgi_prefill_weight_limit_exceeded"); + increment_counter("tgi_prefill_weight_limit_exceeded", 1); continue; } prefill_stats = next_prefill_stats; @@ -434,7 +435,7 @@ impl Queue { }; // Set batch_time entry.batch_time = some_now; - metrics::histogram!( + observe_histogram( "tgi_request_queue_duration", (now - entry.queue_time).as_secs_f64() ); @@ -448,7 +449,7 @@ impl Queue { requests.iter().map(|r| r.input_length as usize), chosen_count, ); - metrics::gauge!("tgi_queue_size", self.buffer.len() as f64); + set_gauge("tgi_queue_size", self.buffer.len() as f64); let batch = Batch { id: self.next_batch_id, requests, diff --git a/router/src/validation.rs b/router/src/validation.rs index 75ac4548..28a43c05 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -11,6 +11,7 @@ use thiserror::Error; use tokio::time::Instant; use crate::{tokenizer::AsyncTokenizer, ErrorResponse, GenerateParameters, GenerateRequest}; +use crate::metrics::{increment_counter, observe_histogram}; const MAX_STOP_SEQS: usize = 6; const MAX_STOP_SEQ_LENGTH: usize = 240; @@ -125,7 +126,7 @@ impl Validation { self.tokenizer .tokenize(input, false) .map_ok(|(input, input_length, _)| { - metrics::histogram!("tgi_request_raw_input_length", input_length as f64); + observe_histogram("tgi_request_raw_input_length", input_length as f64); (input, input_length) }) }); @@ -200,11 +201,11 @@ impl Validation { .map(|results| { // Only record these for successful validation for (request_size, _) in &results { - metrics::histogram!( + observe_histogram( "tgi_request_input_length", request_size.input_length as f64 ); - metrics::histogram!( + observe_histogram( "tgi_request_max_new_tokens", max_new_tokens as f64 ); @@ -224,12 +225,12 @@ async fn prompt_prefix_lookup( let start_time = Instant::now(); let result = client.clone().prefix_lookup(prefix_id).await; if result.is_ok() { - metrics::histogram!( + observe_histogram( "tgi_prompt_load_duration", start_time.elapsed().as_secs_f64() ); } else { - metrics::increment_counter!("tgi_prompt_load_failure"); + increment_counter("tgi_prompt_load_failure", 1); } result }