From bd4ffd938ac6fd693cbfab595043ea6fd41eb4af Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 24 Jan 2023 16:13:42 +0100 Subject: [PATCH 01/47] Added runtime settings calculation struct --- rust/crd/src/lib.rs | 1 + rust/crd/src/memory.rs | 98 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 rust/crd/src/memory.rs diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 0755b8ed..3ab674b8 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -1,5 +1,6 @@ pub mod authentication; pub mod authorization; +pub mod memory; pub mod resource; pub mod security; pub mod storage; diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs new file mode 100644 index 00000000..09ff39e8 --- /dev/null +++ b/rust/crd/src/memory.rs @@ -0,0 +1,98 @@ +//! Calculate how much memory to allocate where + +pub struct RuntimeSettings { + total_memory: usize, + cpu_millis: usize, + min_heap_ratio: f64, + max_buffer_size: usize, + os_reserved_memory: usize, +} + +impl RuntimeSettings { + pub fn new(total_memory: usize, cpu_millis: usize) -> Self { + Self { + total_memory, + cpu_millis, + min_heap_ratio: 0.75, + os_reserved_memory: 300_000_000, // 300MB + max_buffer_size: 2_000_000_000, // 2GB, Druid recommended + } + } + + /// The total memory we use for druid. This is what's left after we take out the OS reserved memory. + pub fn allocatable_memory(&self) -> usize { + self.total_memory - self.os_reserved_memory + } + + pub fn heap_memory(&self) -> usize { + self.allocatable_memory() - self.direct_access_memory() + } + + /// The memory that is available to allocate for direct access. + pub fn allocatable_direct_access_memory(&self) -> usize { + ((self.allocatable_memory() as f64) * (1. - self.min_heap_ratio)).round() as usize + } + + /// The max memory to allocate to direct access. This is based on the max buffer size of a single buffer. + pub fn max_direct_access_memory(&self) -> usize { + self.max_buffer_size * (self.num_merge_buffers() + self.num_threads() + 1) + } + + /// How much to allocate (or keep free) for direct access. + pub fn direct_access_memory(&self) -> usize { + self.allocatable_direct_access_memory().min(self.max_direct_access_memory()) + } + + /// The number of threads to use, based on the CPU millis. + /// leaves at least 500m available to core functionalities. + pub fn num_threads(&self) -> usize { + (((self.cpu_millis as f64) / 1000.).round() as usize - 1).max(1) + } + + pub fn num_merge_buffers(&self) -> usize { + ((self.num_threads() as f64 / 4.).floor() as usize).max(2) + } + + pub fn buffer_size(&self) -> usize { + ((self.direct_access_memory() as f64) + / (self.num_threads() + self.num_merge_buffers() + 1) as f64) + .round() as usize + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::*; + + #[rstest] + #[case(1000, 1)] + #[case(1400, 1)] + #[case(1600, 1)] + #[case(2000, 1)] + #[case(2400, 1)] + #[case(2600, 2)] + #[case(3000, 2)] + #[case(3400, 2)] + #[case(3600, 3)] + #[case(32_000, 31)] + fn test_num_threads(#[case] cpu_millis: usize, #[case] expected_num_threads: usize) { + let s = RuntimeSettings::new(2_000_000_000, cpu_millis); + assert_eq!(s.num_threads(), expected_num_threads); + } + + #[rstest] + #[case(1000, 2)] + #[case(2000, 2)] + #[case(4000, 2)] + #[case(8000, 2)] + #[case(15_000, 3)] + #[case(16_000, 3)] + #[case(17_000, 4)] + #[case(32_000, 7)] + fn test_num_merge_buffers(#[case] cpu_millis: usize, #[case] expected_num_merge_buffers: usize) { + let s = RuntimeSettings::new(2_000_000_000, cpu_millis); + assert_eq!(s.num_merge_buffers(), expected_num_merge_buffers); + } + +} \ No newline at end of file From 743e54f4d30d1601ad300909426b3b75427221fb Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 24 Jan 2023 16:19:57 +0100 Subject: [PATCH 02/47] fmt --- rust/crd/src/memory.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 09ff39e8..2469b8af 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -15,7 +15,7 @@ impl RuntimeSettings { cpu_millis, min_heap_ratio: 0.75, os_reserved_memory: 300_000_000, // 300MB - max_buffer_size: 2_000_000_000, // 2GB, Druid recommended + max_buffer_size: 2_000_000_000, // 2GB, Druid recommended } } @@ -40,7 +40,8 @@ impl RuntimeSettings { /// How much to allocate (or keep free) for direct access. pub fn direct_access_memory(&self) -> usize { - self.allocatable_direct_access_memory().min(self.max_direct_access_memory()) + self.allocatable_direct_access_memory() + .min(self.max_direct_access_memory()) } /// The number of threads to use, based on the CPU millis. @@ -81,7 +82,7 @@ mod tests { assert_eq!(s.num_threads(), expected_num_threads); } - #[rstest] + #[rstest] #[case(1000, 2)] #[case(2000, 2)] #[case(4000, 2)] @@ -90,9 +91,11 @@ mod tests { #[case(16_000, 3)] #[case(17_000, 4)] #[case(32_000, 7)] - fn test_num_merge_buffers(#[case] cpu_millis: usize, #[case] expected_num_merge_buffers: usize) { + fn test_num_merge_buffers( + #[case] cpu_millis: usize, + #[case] expected_num_merge_buffers: usize, + ) { let s = RuntimeSettings::new(2_000_000_000, cpu_millis); assert_eq!(s.num_merge_buffers(), expected_num_merge_buffers); } - -} \ No newline at end of file +} From ebcc8c9ae601dd1803f29f12521d9fda5d009115 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 24 Jan 2023 16:37:24 +0100 Subject: [PATCH 03/47] Added setting the config settings --- rust/crd/src/lib.rs | 4 ++++ rust/crd/src/memory.rs | 23 ++++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 3ab674b8..ffcb6e58 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -88,6 +88,10 @@ pub const MD_ST_USER: &str = "druid.metadata.storage.connector.user"; pub const MD_ST_PASSWORD: &str = "druid.metadata.storage.connector.password"; // indexer properties pub const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; +// historical settings +pub const PROCESSING_BUFFER_SIZEBYTES: &str = "druid.processing.buffer.sizeBytes"; +pub const PROCESSING_NUMMERGEBUFFERS: &str = "druid.processing.numMergeBuffers"; +pub const PROCESSING_NUMTHREADS: &str = "druid.processing.numThreads"; // extra pub const CREDENTIALS_SECRET_PROPERTY: &str = "credentialsSecret"; // metrics diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 2469b8af..ccfec298 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -1,4 +1,8 @@ -//! Calculate how much memory to allocate where +use std::collections::HashMap; + +use stackable_operator::memory::MemoryQuantity; + +use crate::{PROCESSING_BUFFER_SIZEBYTES, PROCESSING_NUMMERGEBUFFERS, PROCESSING_NUMTHREADS}; pub struct RuntimeSettings { total_memory: usize, @@ -59,6 +63,23 @@ impl RuntimeSettings { / (self.num_threads() + self.num_merge_buffers() + 1) as f64) .round() as usize } + + pub fn get_settings(&self) -> HashMap { + HashMap::from([ + ( + PROCESSING_NUMTHREADS.to_owned(), + self.num_threads().to_string(), + ), + ( + PROCESSING_NUMMERGEBUFFERS.to_owned(), + self.num_merge_buffers().to_string(), + ), + ( + PROCESSING_BUFFER_SIZEBYTES.to_owned(), + self.buffer_size().to_string(), + ), + ]) + } } #[cfg(test)] From 69aa5e10ba403613986258c81bf1eec43ff402b9 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 25 Jan 2023 13:00:35 +0100 Subject: [PATCH 04/47] moved to operator-rs branch --- Cargo.lock | 107 +++++++++++++++++--------------- rust/crd/Cargo.toml | 2 +- rust/operator-binary/Cargo.toml | 4 +- 3 files changed, 60 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc24dedd..99173a8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,6 +72,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" + [[package]] name = "bit-set" version = "0.5.3" @@ -167,9 +173,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.29" +version = "4.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d" +checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" dependencies = [ "bitflags", "clap_derive", @@ -182,9 +188,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.0.21" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" dependencies = [ "heck", "proc-macro-error", @@ -905,12 +911,13 @@ dependencies = [ [[package]] name = "json-patch" -version = "0.2.6" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f995a3c8f2bc3dd52a18a583e90f9ec109c047fa1603a853e46bcda14d2e279d" +checksum = "e712e62827c382a77b87f590532febb1f8b2fdbc3eefa1ee37fe7281687075ef" dependencies = [ "serde", "serde_json", + "thiserror", "treediff", ] @@ -927,11 +934,11 @@ dependencies = [ [[package]] name = "k8s-openapi" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9455388f4977de4d0934efa9f7d36296295537d774574113a20f6082de03da" +checksum = "3d1985030683a2bac402cbda61222195de80d3f66b4c87ab56e5fea379bd98c3" dependencies = [ - "base64", + "base64 0.20.0", "bytes", "chrono", "schemars", @@ -942,9 +949,9 @@ dependencies = [ [[package]] name = "kube" -version = "0.76.0" +version = "0.78.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf241a3a42bca4a2d1c21f2f34a659655032a7858270c7791ad4433aa8d79cb" +checksum = "53ee2ba94546e32a5aef943e5831c6ac25592ff8dcfa8b2a06e0aaea90c69188" dependencies = [ "k8s-openapi", "kube-client", @@ -955,11 +962,11 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.76.0" +version = "0.78.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e442b4e6d55c4b3d0c0c70d79a8865bf17e2c33725f9404bfcb8a29ee002ffe" +checksum = "7c9ca1f597bd48ed26f45f601bf2fa3aaa0933b8d1652d883b8444519b72af4a" dependencies = [ - "base64", + "base64 0.20.0", "bytes", "chrono", "dirs-next", @@ -990,9 +997,9 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.76.0" +version = "0.78.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca2e1b1528287ba61602bbd17d0aa717fbb4d0fb257f4fa3a5fa884116ef778" +checksum = "61f2c6d1a2d1584859499eb05a41c5a44713818041621fa7515cfdbdf4769ea7" dependencies = [ "chrono", "form_urlencoded", @@ -1008,9 +1015,9 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.76.0" +version = "0.78.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1af50996adb7e1251960d278859772fa30df99879dc154d792e36832209637cb" +checksum = "3e1dfe288fd87029f87c5713ddf585a4221e1b5be8f8c7c02ba28f5211f2a6d7" dependencies = [ "darling", "proc-macro2", @@ -1021,9 +1028,9 @@ dependencies = [ [[package]] name = "kube-runtime" -version = "0.76.0" +version = "0.78.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9b312c38884a3f41d67e2f7580824b6f45d360b98497325b5630664b3a359d" +checksum = "7995aaf15656bf3694cd455578d59119acc3ca55f71e9a1b579a34a47d17ecce" dependencies = [ "ahash", "backoff", @@ -1372,7 +1379,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03c64931a1a212348ec4f3b4362585eca7159d0d09cbdf4a7f74f02173596fd4" dependencies = [ - "base64", + "base64 0.13.1", ] [[package]] @@ -1451,9 +1458,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" dependencies = [ "unicode-ident", ] @@ -1476,9 +1483,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -1535,9 +1542,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -1677,9 +1684,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] @@ -1696,9 +1703,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -1742,9 +1749,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.14" +version = "0.9.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d232d893b10de3eb7258ff01974d6ee20663d8e833263c99409d4b13a0209da" +checksum = "8fb06d4b6cdaef0e0c51fa881acb721bed3c924cfaa71d9c94a3b771dfdf6567" dependencies = [ "indexmap", "itoa", @@ -1788,9 +1795,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "snafu" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152ba99b054b22972ee794cf04e5ef572da1229e33b65f3c57abbff0525a454" +checksum = "cb0656e7e3ffb70f6c39b3c2a86332bb74aa3c679da781642590f3c1118c5045" dependencies = [ "doc-comment", "snafu-derive", @@ -1798,9 +1805,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5e79cdebbabaebb06a9bdbaedc7f159b410461f63611d4d0e3fb0fab8fed850" +checksum = "475b3bbe5245c26f2d8a6f62d67c1f30eb9fffeccee721c45d162c3ebbdf81b2" dependencies = [ "heck", "proc-macro2", @@ -1850,7 +1857,7 @@ dependencies = [ "semver", "serde", "serde_json", - "serde_yaml 0.9.14", + "serde_yaml 0.9.17", "snafu", "stackable-druid-crd", "stackable-operator", @@ -1861,8 +1868,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.30.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.30.1#34cc76200b6432f7b143ca1350e665ddb169df88" +version = "0.32.1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#f40cc141f6a0ab901c38722bd33eca2589f355ab" dependencies = [ "chrono", "clap", @@ -1882,7 +1889,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "serde_yaml 0.9.14", + "serde_yaml 0.9.17", "snafu", "stackable-operator-derive", "strum", @@ -1895,8 +1902,8 @@ dependencies = [ [[package]] name = "stackable-operator-derive" -version = "0.30.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.30.1#34cc76200b6432f7b143ca1350e665ddb169df88" +version = "0.32.1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#f40cc141f6a0ab901c38722bd33eca2589f355ab" dependencies = [ "darling", "proc-macro2", @@ -1954,18 +1961,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", @@ -2140,7 +2147,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" dependencies = [ - "base64", + "base64 0.13.1", "bitflags", "bytes", "futures-core", @@ -2245,9 +2252,9 @@ dependencies = [ [[package]] name = "treediff" -version = "3.0.2" +version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "761e8d5ad7ce14bb82b7e61ccc0ca961005a275a060b9644a2431aa11553c2ff" +checksum = "52984d277bdf2a751072b5df30ec0377febdb02f7696d64c2d7d54630bac4303" dependencies = [ "serde_json", ] diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index ecce9756..a02a1679 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -9,7 +9,7 @@ version = "0.9.0-nightly" publish = false [dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.1" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/better-memory-quantity" } semver = "1.0" serde = { version = "1.0", features = ["derive"] } diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index cc6f4867..6b150f9a 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,7 @@ version = "0.9.0-nightly" publish = false [dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.1" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/better-memory-quantity" } stackable-druid-crd = { path = "../crd" } anyhow = "1.0" clap = "4.0" @@ -28,7 +28,7 @@ tracing = "0.1" [build-dependencies] built = { version = "0.5", features = ["chrono", "git2"] } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.1" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/better-memory-quantity" } stackable-druid-crd = { path = "../crd" } [dev-dependencies] From b2fc4e62ef7a347225b6651ef5489d54b0012bb5 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 25 Jan 2023 14:23:55 +0100 Subject: [PATCH 05/47] moved to quantities --- Cargo.lock | 4 +- rust/crd/src/memory.rs | 135 +++++++++++++------ rust/operator-binary/src/druid_controller.rs | 2 + 3 files changed, 95 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99173a8c..5ac6428d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1869,7 +1869,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.32.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#f40cc141f6a0ab901c38722bd33eca2589f355ab" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#3ad2a6e342b554d96d1f8023cbf3d3ec7b2e4a5d" dependencies = [ "chrono", "clap", @@ -1903,7 +1903,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.32.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#f40cc141f6a0ab901c38722bd33eca2589f355ab" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#3ad2a6e342b554d96d1f8023cbf3d3ec7b2e4a5d" dependencies = [ "darling", "proc-macro2", diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index ccfec298..b3cd1f66 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -1,49 +1,71 @@ -use std::collections::HashMap; - -use stackable_operator::memory::MemoryQuantity; - -use crate::{PROCESSING_BUFFER_SIZEBYTES, PROCESSING_NUMMERGEBUFFERS, PROCESSING_NUMTHREADS}; +use std::{collections::BTreeMap, str::FromStr}; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + commons::resources::{NoRuntimeLimits, Resources}, + config::fragment, + cpu::CpuQuantity, + memory::{BinaryMultiple, MemoryQuantity}, + role_utils::RoleGroupRef, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use crate::{ + storage::HistoricalStorage, PROCESSING_BUFFER_SIZEBYTES, PROCESSING_NUMMERGEBUFFERS, + PROCESSING_NUMTHREADS, +}; + +/// This Error cannot derive PartialEq because fragment::ValidationError doesn't derive it +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +#[allow(clippy::enum_variant_names)] +pub enum Error { + #[snafu(display("failed to parse memory limits"))] + ParsingMemoryLimitFailure { + source: stackable_operator::error::Error, + }, +} -pub struct RuntimeSettings { - total_memory: usize, - cpu_millis: usize, - min_heap_ratio: f64, - max_buffer_size: usize, - os_reserved_memory: usize, +pub struct HistoricalDerivedSettings { + total_memory: MemoryQuantity, + cpu_millis: CpuQuantity, + min_heap_ratio: f32, + max_buffer_size: MemoryQuantity, + os_reserved_memory: MemoryQuantity, } -impl RuntimeSettings { - pub fn new(total_memory: usize, cpu_millis: usize) -> Self { +impl HistoricalDerivedSettings { + pub fn new(total_memory: MemoryQuantity, cpu_millis: CpuQuantity) -> Self { Self { total_memory, cpu_millis, min_heap_ratio: 0.75, - os_reserved_memory: 300_000_000, // 300MB - max_buffer_size: 2_000_000_000, // 2GB, Druid recommended + os_reserved_memory: MemoryQuantity::from_str("300Mi").unwrap(), // 300MB + max_buffer_size: MemoryQuantity::from_str("2Gi").unwrap(), // 2GB, Druid recommended } } /// The total memory we use for druid. This is what's left after we take out the OS reserved memory. - pub fn allocatable_memory(&self) -> usize { + pub fn allocatable_memory(&self) -> MemoryQuantity { self.total_memory - self.os_reserved_memory } - pub fn heap_memory(&self) -> usize { + pub fn heap_memory(&self) -> MemoryQuantity { self.allocatable_memory() - self.direct_access_memory() } /// The memory that is available to allocate for direct access. - pub fn allocatable_direct_access_memory(&self) -> usize { - ((self.allocatable_memory() as f64) * (1. - self.min_heap_ratio)).round() as usize + pub fn allocatable_direct_access_memory(&self) -> MemoryQuantity { + self.allocatable_memory() * (1. - self.min_heap_ratio) } /// The max memory to allocate to direct access. This is based on the max buffer size of a single buffer. - pub fn max_direct_access_memory(&self) -> usize { - self.max_buffer_size * (self.num_merge_buffers() + self.num_threads() + 1) + pub fn max_direct_access_memory(&self) -> MemoryQuantity { + self.max_buffer_size * (self.num_merge_buffers() + self.num_threads() + 1) as f32 } /// How much to allocate (or keep free) for direct access. - pub fn direct_access_memory(&self) -> usize { + pub fn direct_access_memory(&self) -> MemoryQuantity { self.allocatable_direct_access_memory() .min(self.max_direct_access_memory()) } @@ -51,34 +73,55 @@ impl RuntimeSettings { /// The number of threads to use, based on the CPU millis. /// leaves at least 500m available to core functionalities. pub fn num_threads(&self) -> usize { - (((self.cpu_millis as f64) / 1000.).round() as usize - 1).max(1) + (self.cpu_millis.as_cpu_count().round() - 1.).max(1.) as usize } pub fn num_merge_buffers(&self) -> usize { ((self.num_threads() as f64 / 4.).floor() as usize).max(2) } - pub fn buffer_size(&self) -> usize { - ((self.direct_access_memory() as f64) - / (self.num_threads() + self.num_merge_buffers() + 1) as f64) - .round() as usize + pub fn buffer_size(&self) -> MemoryQuantity { + self.direct_access_memory() / (self.num_threads() + self.num_merge_buffers() + 1) as f32 + } + + pub fn add_settings(&self, config: &mut BTreeMap>) { + config.insert( + PROCESSING_NUMTHREADS.to_owned(), + Some(self.num_threads().to_string()), + ); + config.insert( + PROCESSING_NUMMERGEBUFFERS.to_owned(), + Some(self.num_merge_buffers().to_string()), + ); + config.insert( + PROCESSING_BUFFER_SIZEBYTES.to_owned(), + Some(self.buffer_size().druid_byte_format()), + ); + } +} + +impl TryFrom<&Resources> for HistoricalDerivedSettings { + type Error = Error; + + fn try_from(r: &Resources) -> Result { + let total_memory = MemoryQuantity::try_from(r.memory.limit.clone().unwrap()) + .context(ParsingMemoryLimitFailureSnafu)?; + let cpu_millis = CpuQuantity::try_from(r.cpu.max.clone().unwrap()).unwrap(); // TODO no unwrap + Ok(HistoricalDerivedSettings::new(total_memory, cpu_millis)) } +} + +/// A trait to format something as the Druid Byte format: ``. +/// It supports human readable units, but only integer values, i.e. "1.5Gi" does not work, use "1536Mi" instead. +trait AsDruidByteFormat { + fn druid_byte_format(&self) -> String; +} - pub fn get_settings(&self) -> HashMap { - HashMap::from([ - ( - PROCESSING_NUMTHREADS.to_owned(), - self.num_threads().to_string(), - ), - ( - PROCESSING_NUMMERGEBUFFERS.to_owned(), - self.num_merge_buffers().to_string(), - ), - ( - PROCESSING_BUFFER_SIZEBYTES.to_owned(), - self.buffer_size().to_string(), - ), - ]) +impl AsDruidByteFormat for MemoryQuantity { + fn druid_byte_format(&self) -> String { + let k = self.scale_to(BinaryMultiple::Kibi); + let v = k.value.round() as usize; + format!("{v}Ki") } } @@ -99,7 +142,9 @@ mod tests { #[case(3600, 3)] #[case(32_000, 31)] fn test_num_threads(#[case] cpu_millis: usize, #[case] expected_num_threads: usize) { - let s = RuntimeSettings::new(2_000_000_000, cpu_millis); + let mem = MemoryQuantity::from_str("2Gi").unwrap(); + let cpu = CpuQuantity::from_millis(cpu_millis); + let s = HistoricalDerivedSettings::new(mem, cpu); assert_eq!(s.num_threads(), expected_num_threads); } @@ -116,7 +161,9 @@ mod tests { #[case] cpu_millis: usize, #[case] expected_num_merge_buffers: usize, ) { - let s = RuntimeSettings::new(2_000_000_000, cpu_millis); + let mem = MemoryQuantity::from_str("2Gi").unwrap(); + let cpu = CpuQuantity::from_millis(cpu_millis); + let s = HistoricalDerivedSettings::new(mem, cpu); assert_eq!(s.num_merge_buffers(), expected_num_merge_buffers); } } diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index b8b9c90e..a55117be 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -488,6 +488,8 @@ fn build_rolegroup_config_map( deep_storage_bucket_name.map(str::to_string), ); + // ... + // add tls encryption / auth properties druid_tls_security.add_tls_config_properties(&mut transformed_config, &role); From d5e3b8af9730e20abaf3d600a719650aa7344133 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 25 Jan 2023 14:42:43 +0100 Subject: [PATCH 06/47] feat: added memory setting to the reconcile loop --- rust/crd/src/memory.rs | 2 -- rust/operator-binary/src/druid_controller.rs | 19 +++++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index b3cd1f66..6dd4a7c8 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -3,10 +3,8 @@ use std::{collections::BTreeMap, str::FromStr}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ commons::resources::{NoRuntimeLimits, Resources}, - config::fragment, cpu::CpuQuantity, memory::{BinaryMultiple, MemoryQuantity}, - role_utils::RoleGroupRef, }; use strum::{EnumDiscriminants, IntoStaticStr}; diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index a55117be..42cb9eb9 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -8,11 +8,12 @@ use crate::{ use crate::OPERATOR_NAME; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_druid_crd::{ - authorization::DruidAuthorization, build_string_list, security::DruidTlsSecurity, - DeepStorageSpec, DruidCluster, DruidRole, APP_NAME, AUTH_AUTHORIZER_OPA_URI, CERTS_DIR, - CREDENTIALS_SECRET_PROPERTY, DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, - HDFS_CONFIG_DIRECTORY, JVM_CONFIG, LOG4J2_CONFIG, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, - S3_ENDPOINT_URL, S3_PATH_STYLE_ACCESS, S3_SECRET_DIR_NAME, ZOOKEEPER_CONNECTION_STRING, + authorization::DruidAuthorization, build_string_list, memory::HistoricalDerivedSettings, + security::DruidTlsSecurity, DeepStorageSpec, DruidCluster, DruidRole, APP_NAME, + AUTH_AUTHORIZER_OPA_URI, CERTS_DIR, CREDENTIALS_SECRET_PROPERTY, DRUID_CONFIG_DIRECTORY, + DS_BUCKET, EXTENSIONS_LOADLIST, HDFS_CONFIG_DIRECTORY, JVM_CONFIG, LOG4J2_CONFIG, + RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ENDPOINT_URL, S3_PATH_STYLE_ACCESS, S3_SECRET_DIR_NAME, + ZOOKEEPER_CONNECTION_STRING, }; use stackable_druid_crd::{ build_recommended_labels, @@ -488,7 +489,13 @@ fn build_rolegroup_config_map( deep_storage_bucket_name.map(str::to_string), ); - // ... + match resources { + RoleResource::Historical(r) => { + let settings = HistoricalDerivedSettings::try_from(r).unwrap(); // TODO fix unwrap + settings.add_settings(&mut transformed_config); + } + RoleResource::Druid(_) => todo!(), + } // add tls encryption / auth properties druid_tls_security.add_tls_config_properties(&mut transformed_config, &role); From 6ac8d3d9e8f0ac579301a443e9d9e93ec5e905ac Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 25 Jan 2023 15:43:15 +0100 Subject: [PATCH 07/47] WIP --- rust/operator-binary/src/config.rs | 3 ++- rust/operator-binary/src/druid_controller.rs | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 4d4e7143..15457702 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -2,6 +2,7 @@ use indoc::formatdoc; use stackable_druid_crd::{DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD}; pub fn get_jvm_config(role: &DruidRole, heap_in_mebi: u32) -> String { + // TODO heap and direct memory should be configured differently let common_config = formatdoc! {" -server -Duser.timezone=UTC @@ -36,7 +37,7 @@ pub fn get_jvm_config(role: &DruidRole, heap_in_mebi: u32) -> String { {common_config} -Xms{heap_in_mebi}m -Xmx{heap_in_mebi}m - -XX:MaxDirectMemorySize=400m + -XX:MaxDirectMemorySize=4000m "} } DruidRole::MiddleManager => { diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 42cb9eb9..969879cc 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -60,7 +60,7 @@ use strum::{EnumDiscriminants, IntoStaticStr}; pub const CONTROLLER_NAME: &str = "druidcluster"; -const JVM_HEAP_FACTOR: f32 = 0.8; +const JVM_HEAP_FACTOR: f32 = 0.7; const DOCKER_IMAGE_BASE_NAME: &str = "druid"; pub struct Ctx { @@ -492,9 +492,10 @@ fn build_rolegroup_config_map( match resources { RoleResource::Historical(r) => { let settings = HistoricalDerivedSettings::try_from(r).unwrap(); // TODO fix unwrap + println!("Updating Settings ################"); settings.add_settings(&mut transformed_config); } - RoleResource::Druid(_) => todo!(), + RoleResource::Druid(_) => (), } // add tls encryption / auth properties @@ -508,6 +509,15 @@ fn build_rolegroup_config_map( cm_conf_data.insert(RUNTIME_PROPS.to_string(), runtime_properties); } PropertyNameKind::File(file_name) if file_name == JVM_CONFIG => { + match resources { + RoleResource::Historical(r) => { + let settings = HistoricalDerivedSettings::try_from(r).unwrap(); // TODO fix unwrap + println!("Updating Settings ################"); + settings.add_settings(&mut transformed_config); + + } + RoleResource::Druid(_) => (), + } let heap_in_mebi = to_java_heap_value( resources .as_memory_limits() From 4cc9cf007ca9474ca408f9b2f61b20bc0d8195ee Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 25 Jan 2023 16:11:33 +0100 Subject: [PATCH 08/47] WIP --- rust/crd/src/memory.rs | 14 ++++++++++++++ rust/crd/src/resource.rs | 2 ++ 2 files changed, 16 insertions(+) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 6dd4a7c8..63b7ee84 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -24,6 +24,12 @@ pub enum Error { }, } +/// This struct takes the resource limits of the Pod and derives Druid settings from it. +/// For mentioned Druid properties, consult the +/// [Druid Configuration Reference](https://druid.apache.org/docs/latest/configuration/index.html) +/// for additional information. +/// Also have a look at the "Basic Cluster Tuning" documentation: +/// `` pub struct HistoricalDerivedSettings { total_memory: MemoryQuantity, cpu_millis: CpuQuantity, @@ -48,7 +54,9 @@ impl HistoricalDerivedSettings { self.total_memory - self.os_reserved_memory } + /// How much memory to set for the JVM to use. pub fn heap_memory(&self) -> MemoryQuantity { + // TODO also implement max limit of 24Gi, as recommended by Druid self.allocatable_memory() - self.direct_access_memory() } @@ -63,6 +71,7 @@ impl HistoricalDerivedSettings { } /// How much to allocate (or keep free) for direct access. + /// this is the amount to configure in the JVM as the `MaxDirectMemorySize`. pub fn direct_access_memory(&self) -> MemoryQuantity { self.allocatable_direct_access_memory() .min(self.max_direct_access_memory()) @@ -70,14 +79,19 @@ impl HistoricalDerivedSettings { /// The number of threads to use, based on the CPU millis. /// leaves at least 500m available to core functionalities. + /// Druid Property: `druid.processing.numThreads` pub fn num_threads(&self) -> usize { (self.cpu_millis.as_cpu_count().round() - 1.).max(1.) as usize } + /// Druid property: `druid.processing.numMergeBuffers` pub fn num_merge_buffers(&self) -> usize { ((self.num_threads() as f64 / 4.).floor() as usize).max(2) } + /// The buffer size for intermediate result storage. By setting it ourselves, we can set it up to 2Gi. + /// If we leave it on the `auto` default, we only get up to 1Gi. + /// Druid property: `druid.processing.buffer.sizeBytes` pub fn buffer_size(&self) -> MemoryQuantity { self.direct_access_memory() / (self.num_threads() + self.num_merge_buffers() + 1) as f32 } diff --git a/rust/crd/src/resource.rs b/rust/crd/src/resource.rs index b4916a19..1244f910 100644 --- a/rust/crd/src/resource.rs +++ b/rust/crd/src/resource.rs @@ -104,7 +104,9 @@ impl RoleResource { )) }); } + // TODO put my code here } + // TODO add a new if case for the JVM config and generate it here as well } pub fn update_volumes_and_volume_mounts(&self, cb: &mut ContainerBuilder, pb: &mut PodBuilder) { From aed1d50c6eeaf582a78a705cb0d54c79a085c4d0 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 26 Jan 2023 09:41:46 +0100 Subject: [PATCH 09/47] WIP: Added DerivedHistoricalSettings to RoleResource --- rust/crd/src/resource.rs | 23 ++++++++++++-------- rust/operator-binary/src/config.rs | 9 +++++++- rust/operator-binary/src/druid_controller.rs | 22 +++---------------- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/rust/crd/src/resource.rs b/rust/crd/src/resource.rs index 1244f910..8eb7eec0 100644 --- a/rust/crd/src/resource.rs +++ b/rust/crd/src/resource.rs @@ -1,12 +1,15 @@ use std::collections::BTreeMap; +use crate::memory::HistoricalDerivedSettings; use crate::storage::{self, FreePercentageEmptyDirFragment}; use crate::{ - DruidCluster, DruidRole, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS, RUNTIME_PROPS, + DruidCluster, DruidRole, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS, RUNTIME_PROPS, JVM_CONFIG, }; use lazy_static::lazy_static; use snafu::{ResultExt, Snafu}; use stackable_operator::config::fragment; +use stackable_operator::memory::MemoryQuantity; +use stackable_operator::product_config::types::PropertyNameKind; use stackable_operator::role_utils::RoleGroupRef; use stackable_operator::{ builder::{ContainerBuilder, PodBuilder, VolumeBuilder}, @@ -86,13 +89,13 @@ impl RoleResource { /// Update the given configuration file with resource properties. /// Currently it only adds the segment cache location property for historicals to runtime.properties. + /// This is only for runtime properties file pub fn update_druid_config_file( &self, - file: &str, config: &mut BTreeMap>, ) { - if let Self::Historical(r) = self { - if RUNTIME_PROPS == file { + match self { + RoleResource::Historical(r) => { let free_percentage = r.storage.segment_cache.free_percentage.unwrap_or(5u16); let capacity = &r.storage.segment_cache.empty_dir.capacity; config @@ -103,10 +106,12 @@ impl RoleResource { PATH_SEGMENT_CACHE, capacity.0, free_percentage )) }); - } - // TODO put my code here + + let settings = HistoricalDerivedSettings::try_from(r).unwrap(); // TODO fix unwrap + settings.add_settings(config); + }, + RoleResource::Druid(_) => (), } - // TODO add a new if case for the JVM config and generate it here as well } pub fn update_volumes_and_volume_mounts(&self, cb: &mut ContainerBuilder, pb: &mut PodBuilder) { @@ -593,7 +598,7 @@ mod test { let res = resources(&cluster, &DruidRole::Historical, &rolegroup_ref)?; let mut got = BTreeMap::new(); - res.update_druid_config_file(RUNTIME_PROPS, &mut got); + res.update_druid_config_file(&mut got); let expected: BTreeMap> = vec![ (PROP_SEGMENT_CACHE_LOCATIONS.to_string(), Some(r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"5g","freeSpacePercent":"3"}]"#.to_string())) @@ -612,7 +617,7 @@ mod test { }; let res = resources(&cluster, &DruidRole::Historical, &rolegroup_ref)?; let mut got = BTreeMap::new(); - res.update_druid_config_file(RUNTIME_PROPS, &mut got); + res.update_druid_config_file(&mut got); let expected = vec![ (PROP_SEGMENT_CACHE_LOCATIONS.to_string(), Some(r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"2g","freeSpacePercent":"7"}]"#.to_string())) diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 15457702..a33b5a58 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -1,7 +1,14 @@ use indoc::formatdoc; use stackable_druid_crd::{DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD}; +use stackable_operator::memory::MemoryQuantity; -pub fn get_jvm_config(role: &DruidRole, heap_in_mebi: u32) -> String { +pub fn get_jvm_config2(role: &DruidRole, heap: MemoryQuantity, direct_memory: MemoryQuantity) -> String { + // TODO + + todo!() +} + +pub fn get_jvm_config(role: &DruidRole, heap_in_mebi: u32, direct_memory_in_mebi: Option) -> String { // TODO heap and direct memory should be configured differently let common_config = formatdoc! {" -server diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 969879cc..10f7c660 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -439,13 +439,14 @@ fn build_rolegroup_config_map( .map(|(k, v)| (k.clone(), Some(v.clone()))) .collect(); + match property_name_kind { PropertyNameKind::File(file_name) if file_name == RUNTIME_PROPS => { // Add any properties derived from storage manifests, such as segment cache locations. // This has to be done here since there is no other suitable place for it. // Previously such properties were added in the compute_files() function, // but that code path is now incompatible with the design of fragment merging. - resources.update_druid_config_file(file_name.as_str(), &mut transformed_config); + resources.update_druid_config_file(&mut transformed_config); // NOTE: druid.host can be set manually - if it isn't, the canonical host name of // the local host is used. This should work with the agent and k8s host networking // but might need to be revisited in the future @@ -489,15 +490,6 @@ fn build_rolegroup_config_map( deep_storage_bucket_name.map(str::to_string), ); - match resources { - RoleResource::Historical(r) => { - let settings = HistoricalDerivedSettings::try_from(r).unwrap(); // TODO fix unwrap - println!("Updating Settings ################"); - settings.add_settings(&mut transformed_config); - } - RoleResource::Druid(_) => (), - } - // add tls encryption / auth properties druid_tls_security.add_tls_config_properties(&mut transformed_config, &role); @@ -509,15 +501,7 @@ fn build_rolegroup_config_map( cm_conf_data.insert(RUNTIME_PROPS.to_string(), runtime_properties); } PropertyNameKind::File(file_name) if file_name == JVM_CONFIG => { - match resources { - RoleResource::Historical(r) => { - let settings = HistoricalDerivedSettings::try_from(r).unwrap(); // TODO fix unwrap - println!("Updating Settings ################"); - settings.add_settings(&mut transformed_config); - } - RoleResource::Druid(_) => (), - } let heap_in_mebi = to_java_heap_value( resources .as_memory_limits() @@ -531,7 +515,7 @@ fn build_rolegroup_config_map( unit: BinaryMultiple::Mebi.to_java_memory_unit(), })?; - let jvm_config = get_jvm_config(&role, heap_in_mebi); + let jvm_config = get_jvm_config(&role, heap_in_mebi, todo!()); cm_conf_data.insert(JVM_CONFIG.to_string(), jvm_config); } PropertyNameKind::File(file_name) if file_name == LOG4J2_CONFIG => { From 191fabe437b12b76c132d27d438552c61323f515 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 26 Jan 2023 09:50:22 +0100 Subject: [PATCH 10/47] modified get_jvm_config to take an optional direct_memory setting --- rust/operator-binary/src/config.rs | 77 ++++++++++-------------------- 1 file changed, 25 insertions(+), 52 deletions(-) diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index a33b5a58..a659f0f0 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -9,60 +9,33 @@ pub fn get_jvm_config2(role: &DruidRole, heap: MemoryQuantity, direct_memory: Me } pub fn get_jvm_config(role: &DruidRole, heap_in_mebi: u32, direct_memory_in_mebi: Option) -> String { - // TODO heap and direct memory should be configured differently - let common_config = formatdoc! {" - -server - -Duser.timezone=UTC - -Dfile.encoding=UTF-8 - -Djava.io.tmpdir=/tmp - -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager - -XX:+UseG1GC - -XX:+ExitOnOutOfMemoryError - -Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE} - -Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD} - -Djavax.net.ssl.trustStoreType=pkcs12"}; + // TODO heap and direct memory should be configured differently + let mut config = formatdoc! {" + -server + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Djava.io.tmpdir=/tmp + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + -XX:+UseG1GC + -XX:+ExitOnOutOfMemoryError + -Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE} + -Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD} + -Djavax.net.ssl.trustStoreType=pkcs12 + -Xms{heap_in_mebi}m + -Xmx{heap_in_mebi}m"}; - match role { - DruidRole::Broker => { - formatdoc! {" - {common_config} - -Xms{heap_in_mebi}m - -Xmx{heap_in_mebi}m - -XX:MaxDirectMemorySize=400m - "} - } - DruidRole::Coordinator => { - formatdoc! {" - {common_config} - -Xms{heap_in_mebi}m - -Xmx{heap_in_mebi}m - -Dderby.stream.error.file=/stackable/var/druid/derby.log - "} - } - DruidRole::Historical => { - formatdoc! {" - {common_config} - -Xms{heap_in_mebi}m - -Xmx{heap_in_mebi}m - -XX:MaxDirectMemorySize=4000m - "} - } - DruidRole::MiddleManager => { - formatdoc! {" - {common_config} - -Xms{heap_in_mebi}m - -Xmx{heap_in_mebi}m - "} - } - DruidRole::Router => { - formatdoc! {" - {common_config} - -Xms{heap_in_mebi}m - -Xmx{heap_in_mebi}m - -XX:MaxDirectMemorySize=128m - "} - } + if let Some(direct_memory) = direct_memory_in_mebi { + config += &formatdoc! {" + -XX:MaxDirectMemorySize={direct_memory}m + "}; } + + if role == &DruidRole::Coordinator { + config += &formatdoc! {" + -Dderby.stream.error.file=/stackable/var/druid/derby.log + "}; + } + config } pub fn get_log4j_config(_role: &DruidRole) -> String { From 492bfcfe818b78d5eaac3afcecf03d0844ee862e Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 26 Jan 2023 16:19:27 +0100 Subject: [PATCH 11/47] JVM config refactored; a version that SHOULD run --- Cargo.lock | 4 +- rust/crd/src/memory.rs | 2 +- rust/crd/src/resource.rs | 12 ++-- rust/operator-binary/src/config.rs | 23 ++++---- rust/operator-binary/src/druid_controller.rs | 60 ++++++++++++++------ 5 files changed, 62 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ac6428d..f16870a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1869,7 +1869,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.32.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#3ad2a6e342b554d96d1f8023cbf3d3ec7b2e4a5d" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#ee5cedec5d9658f8402e57e31303091710b7c6d0" dependencies = [ "chrono", "clap", @@ -1903,7 +1903,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.32.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#3ad2a6e342b554d96d1f8023cbf3d3ec7b2e4a5d" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#ee5cedec5d9658f8402e57e31303091710b7c6d0" dependencies = [ "darling", "proc-macro2", diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 63b7ee84..8490fb8c 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -54,7 +54,7 @@ impl HistoricalDerivedSettings { self.total_memory - self.os_reserved_memory } - /// How much memory to set for the JVM to use. + /// How much memory to set for the JVM to use. pub fn heap_memory(&self) -> MemoryQuantity { // TODO also implement max limit of 24Gi, as recommended by Druid self.allocatable_memory() - self.direct_access_memory() diff --git a/rust/crd/src/resource.rs b/rust/crd/src/resource.rs index 8eb7eec0..ea108c53 100644 --- a/rust/crd/src/resource.rs +++ b/rust/crd/src/resource.rs @@ -3,7 +3,8 @@ use std::collections::BTreeMap; use crate::memory::HistoricalDerivedSettings; use crate::storage::{self, FreePercentageEmptyDirFragment}; use crate::{ - DruidCluster, DruidRole, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS, RUNTIME_PROPS, JVM_CONFIG, + DruidCluster, DruidRole, JVM_CONFIG, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS, + RUNTIME_PROPS, }; use lazy_static::lazy_static; use snafu::{ResultExt, Snafu}; @@ -90,10 +91,7 @@ impl RoleResource { /// Update the given configuration file with resource properties. /// Currently it only adds the segment cache location property for historicals to runtime.properties. /// This is only for runtime properties file - pub fn update_druid_config_file( - &self, - config: &mut BTreeMap>, - ) { + pub fn update_druid_config_file(&self, config: &mut BTreeMap>) { match self { RoleResource::Historical(r) => { let free_percentage = r.storage.segment_cache.free_percentage.unwrap_or(5u16); @@ -106,10 +104,10 @@ impl RoleResource { PATH_SEGMENT_CACHE, capacity.0, free_percentage )) }); - + let settings = HistoricalDerivedSettings::try_from(r).unwrap(); // TODO fix unwrap settings.add_settings(config); - }, + } RoleResource::Druid(_) => (), } } diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index a659f0f0..00784ba7 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -2,14 +2,13 @@ use indoc::formatdoc; use stackable_druid_crd::{DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD}; use stackable_operator::memory::MemoryQuantity; -pub fn get_jvm_config2(role: &DruidRole, heap: MemoryQuantity, direct_memory: MemoryQuantity) -> String { - // TODO - - todo!() -} - -pub fn get_jvm_config(role: &DruidRole, heap_in_mebi: u32, direct_memory_in_mebi: Option) -> String { - // TODO heap and direct memory should be configured differently +pub fn get_jvm_config( + role: &DruidRole, + heap: MemoryQuantity, + direct_memory: Option, +) -> String { + let heap_str = heap.format_for_java().unwrap(); // TODO fix unwrap + let direct_memory_str = direct_memory.map(|m| m.format_for_java().unwrap()); // TODO fix unwrap let mut config = formatdoc! {" -server -Duser.timezone=UTC @@ -21,12 +20,12 @@ pub fn get_jvm_config(role: &DruidRole, heap_in_mebi: u32, direct_memory_in_mebi -Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE} -Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD} -Djavax.net.ssl.trustStoreType=pkcs12 - -Xms{heap_in_mebi}m - -Xmx{heap_in_mebi}m"}; + -Xms{heap_str} + -Xmx{heap_str}"}; - if let Some(direct_memory) = direct_memory_in_mebi { + if let Some(direct_memory) = direct_memory_str { config += &formatdoc! {" - -XX:MaxDirectMemorySize={direct_memory}m + -XX:MaxDirectMemorySize={direct_memory} "}; } diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 10f7c660..62e8c596 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -19,7 +19,6 @@ use stackable_druid_crd::{ build_recommended_labels, resource::{self, RoleResource}, }; -use stackable_operator::commons::product_image_selection::ResolvedProductImage; use stackable_operator::{ builder::{ ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder, @@ -49,6 +48,9 @@ use stackable_operator::{ product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::RoleGroupRef, }; +use stackable_operator::{ + commons::product_image_selection::ResolvedProductImage, memory::MemoryQuantity, +}; use std::{ collections::{BTreeMap, HashMap}, ops::Deref, @@ -439,7 +441,6 @@ fn build_rolegroup_config_map( .map(|(k, v)| (k.clone(), Some(v.clone()))) .collect(); - match property_name_kind { PropertyNameKind::File(file_name) if file_name == RUNTIME_PROPS => { // Add any properties derived from storage manifests, such as segment cache locations. @@ -501,21 +502,46 @@ fn build_rolegroup_config_map( cm_conf_data.insert(RUNTIME_PROPS.to_string(), runtime_properties); } PropertyNameKind::File(file_name) if file_name == JVM_CONFIG => { - - let heap_in_mebi = to_java_heap_value( - resources - .as_memory_limits() - .limit - .as_ref() - .context(InvalidJavaHeapConfigSnafu)?, - JVM_HEAP_FACTOR, - BinaryMultiple::Mebi, - ) - .context(FailedToConvertJavaHeapSnafu { - unit: BinaryMultiple::Mebi.to_java_memory_unit(), - })?; - - let jvm_config = get_jvm_config(&role, heap_in_mebi, todo!()); + // This large match structure computes the heap and direct access memory requirements for each role. + // Both parameters are then used to construct the JVM config. + let (heap, direct) = match resources { + RoleResource::Historical(r) => { + let settings = HistoricalDerivedSettings::try_from(r).unwrap(); // TODO fix unwrap + ( + settings.heap_memory(), + Some(settings.direct_access_memory()), + ) + } + RoleResource::Druid(r) => { + let total_memory = + MemoryQuantity::try_from(r.memory.limit.as_ref().unwrap()).unwrap(); // TODO fix unwrap + let os_reserved = MemoryQuantity::from_mebi(300.); + match role { + DruidRole::Coordinator => { + // The coordinator needs no direct memory + let heap_memory = total_memory - os_reserved; + (heap_memory, None) + } + DruidRole::Broker => { + let direct_memory = MemoryQuantity::from_mebi(400.); + let heap_memory = total_memory - os_reserved - direct_memory; + (heap_memory, Some(direct_memory)) + } + DruidRole::Historical => panic!(), // TODO fix panic; we cannot reach this arm here + DruidRole::MiddleManager => { + // The middle manager needs no direct memory + let heap_memory = total_memory - os_reserved; + (heap_memory, None) + } + DruidRole::Router => { + let direct_memory = MemoryQuantity::from_mebi(128.); + let heap_memory = total_memory - os_reserved - direct_memory; + (heap_memory, Some(direct_memory)) + } + } + } + }; + let jvm_config = get_jvm_config(&role, heap, direct); cm_conf_data.insert(JVM_CONFIG.to_string(), jvm_config); } PropertyNameKind::File(file_name) if file_name == LOG4J2_CONFIG => { From 3f9ea062e63dd0243a2e30019265020c086891de Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 26 Jan 2023 16:36:07 +0100 Subject: [PATCH 12/47] fix: JVM config formatting --- rust/operator-binary/src/config.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 00784ba7..41041b3e 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -24,15 +24,11 @@ pub fn get_jvm_config( -Xmx{heap_str}"}; if let Some(direct_memory) = direct_memory_str { - config += &formatdoc! {" - -XX:MaxDirectMemorySize={direct_memory} - "}; + config += &format!("\n-XX:MaxDirectMemorySize={direct_memory}"); } if role == &DruidRole::Coordinator { - config += &formatdoc! {" - -Dderby.stream.error.file=/stackable/var/druid/derby.log - "}; + config += "\n-Dderby.stream.error.file=/stackable/var/druid/derby.log"; } config } From 11e0f8d4ff54052c235fead99ed1ae3e6228d3f3 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 26 Jan 2023 16:41:32 +0100 Subject: [PATCH 13/47] chore: fixed checks --- Cargo.lock | 1 + rust/crd/src/resource.rs | 7 +------ rust/operator-binary/Cargo.toml | 1 + rust/operator-binary/src/druid_controller.rs | 17 ++++++++++------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f16870a9..87273e7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1852,6 +1852,7 @@ dependencies = [ "fnv", "futures 0.3.25", "indoc", + "lazy_static", "pin-project", "rstest", "semver", diff --git a/rust/crd/src/resource.rs b/rust/crd/src/resource.rs index ea108c53..77022849 100644 --- a/rust/crd/src/resource.rs +++ b/rust/crd/src/resource.rs @@ -2,15 +2,10 @@ use std::collections::BTreeMap; use crate::memory::HistoricalDerivedSettings; use crate::storage::{self, FreePercentageEmptyDirFragment}; -use crate::{ - DruidCluster, DruidRole, JVM_CONFIG, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS, - RUNTIME_PROPS, -}; +use crate::{DruidCluster, DruidRole, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS}; use lazy_static::lazy_static; use snafu::{ResultExt, Snafu}; use stackable_operator::config::fragment; -use stackable_operator::memory::MemoryQuantity; -use stackable_operator::product_config::types::PropertyNameKind; use stackable_operator::role_utils::RoleGroupRef; use stackable_operator::{ builder::{ContainerBuilder, PodBuilder, VolumeBuilder}, diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 6b150f9a..035a334f 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -25,6 +25,7 @@ snafu = "0.7" strum = { version = "0.24", features = ["derive"] } tokio = { version = "1.24", features = ["full"] } tracing = "0.1" +lazy_static = "1.4" [build-dependencies] built = { version = "0.5", features = ["chrono", "git2"] } diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 62e8c596..561c7684 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -6,6 +6,7 @@ use crate::{ }; use crate::OPERATOR_NAME; +use lazy_static::lazy_static; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_druid_crd::{ authorization::DruidAuthorization, build_string_list, memory::HistoricalDerivedSettings, @@ -43,7 +44,6 @@ use stackable_operator::{ }, labels::{role_group_selector_labels, role_selector_labels}, logging::controller::ReconcilerError, - memory::{to_java_heap_value, BinaryMultiple}, product_config::{types::PropertyNameKind, ProductConfigManager}, product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::RoleGroupRef, @@ -62,8 +62,10 @@ use strum::{EnumDiscriminants, IntoStaticStr}; pub const CONTROLLER_NAME: &str = "druidcluster"; -const JVM_HEAP_FACTOR: f32 = 0.7; const DOCKER_IMAGE_BASE_NAME: &str = "druid"; +lazy_static! { + static ref RESERVED_OS_MEMORY: MemoryQuantity = MemoryQuantity::from_mebi(300.); +} pub struct Ctx { pub client: stackable_operator::client::Client, @@ -515,27 +517,28 @@ fn build_rolegroup_config_map( RoleResource::Druid(r) => { let total_memory = MemoryQuantity::try_from(r.memory.limit.as_ref().unwrap()).unwrap(); // TODO fix unwrap - let os_reserved = MemoryQuantity::from_mebi(300.); match role { DruidRole::Coordinator => { // The coordinator needs no direct memory - let heap_memory = total_memory - os_reserved; + let heap_memory = total_memory - *RESERVED_OS_MEMORY; (heap_memory, None) } DruidRole::Broker => { let direct_memory = MemoryQuantity::from_mebi(400.); - let heap_memory = total_memory - os_reserved - direct_memory; + let heap_memory = + total_memory - *RESERVED_OS_MEMORY - direct_memory; (heap_memory, Some(direct_memory)) } DruidRole::Historical => panic!(), // TODO fix panic; we cannot reach this arm here DruidRole::MiddleManager => { // The middle manager needs no direct memory - let heap_memory = total_memory - os_reserved; + let heap_memory = total_memory - *RESERVED_OS_MEMORY; (heap_memory, None) } DruidRole::Router => { let direct_memory = MemoryQuantity::from_mebi(128.); - let heap_memory = total_memory - os_reserved - direct_memory; + let heap_memory = + total_memory - *RESERVED_OS_MEMORY - direct_memory; (heap_memory, Some(direct_memory)) } } From 9ee6c7fcb3f3cafe2da58b8242ea65a251f2d871 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 26 Jan 2023 16:42:30 +0100 Subject: [PATCH 14/47] minor change --- rust/operator-binary/src/druid_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 561c7684..12d17a58 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -518,6 +518,7 @@ fn build_rolegroup_config_map( let total_memory = MemoryQuantity::try_from(r.memory.limit.as_ref().unwrap()).unwrap(); // TODO fix unwrap match role { + DruidRole::Historical => panic!(), // TODO fix panic; we cannot reach this arm here DruidRole::Coordinator => { // The coordinator needs no direct memory let heap_memory = total_memory - *RESERVED_OS_MEMORY; @@ -529,7 +530,6 @@ fn build_rolegroup_config_map( total_memory - *RESERVED_OS_MEMORY - direct_memory; (heap_memory, Some(direct_memory)) } - DruidRole::Historical => panic!(), // TODO fix panic; we cannot reach this arm here DruidRole::MiddleManager => { // The middle manager needs no direct memory let heap_memory = total_memory - *RESERVED_OS_MEMORY; From 069f6f54a9badfc8d9b475d6a9e74bb69286c855 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 30 Jan 2023 10:38:47 +0100 Subject: [PATCH 15/47] fixed some unwraps --- Cargo.lock | 4 +- deploy/helm/druid-operator/crds/crds.yaml | 1056 +---------------- .../getting_started/examples/code/druid.yaml | 4 + rust/crd/src/memory.rs | 40 +- rust/operator-binary/src/druid_controller.rs | 10 +- 5 files changed, 70 insertions(+), 1044 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87273e7e..5b94d9dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1870,7 +1870,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.32.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#ee5cedec5d9658f8402e57e31303091710b7c6d0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#6f4bb27dcbddcbfacc8f7db50d7d7661478b3992" dependencies = [ "chrono", "clap", @@ -1904,7 +1904,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.32.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#ee5cedec5d9658f8402e57e31303091710b7c6d0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#6f4bb27dcbddcbfacc8f7db50d7d7661478b3992" dependencies = [ "darling", "proc-macro2", diff --git a/deploy/helm/druid-operator/crds/crds.yaml b/deploy/helm/druid-operator/crds/crds.yaml index 236ec3d8..5743ee99 100644 --- a/deploy/helm/druid-operator/crds/crds.yaml +++ b/deploy/helm/druid-operator/crds/crds.yaml @@ -44,111 +44,18 @@ spec: max: null properties: max: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string min: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string type: object memory: properties: limit: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string runtimeLimits: @@ -190,111 +97,18 @@ spec: max: null properties: max: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string min: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string type: object memory: properties: limit: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string runtimeLimits: @@ -713,111 +527,18 @@ spec: max: null properties: max: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string min: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string type: object memory: properties: limit: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string runtimeLimits: @@ -859,111 +580,18 @@ spec: max: null properties: max: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string min: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string type: object memory: properties: limit: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string runtimeLimits: @@ -1046,111 +674,18 @@ spec: max: null properties: max: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string min: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string type: object memory: properties: limit: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string runtimeLimits: @@ -1170,38 +705,7 @@ spec: capacity: null properties: capacity: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string medium: @@ -1249,111 +753,18 @@ spec: max: null properties: max: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string min: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string type: object memory: properties: limit: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string runtimeLimits: @@ -1373,38 +784,7 @@ spec: capacity: null properties: capacity: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string medium: @@ -1536,111 +916,18 @@ spec: max: null properties: max: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string min: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string type: object memory: properties: limit: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string runtimeLimits: @@ -1682,111 +969,18 @@ spec: max: null properties: max: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string min: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string type: object memory: properties: limit: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string runtimeLimits: @@ -1869,111 +1063,18 @@ spec: max: null properties: max: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string min: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string type: object memory: properties: limit: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string runtimeLimits: @@ -2015,111 +1116,18 @@ spec: max: null properties: max: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string min: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string type: object memory: properties: limit: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string runtimeLimits: diff --git a/docs/modules/getting_started/examples/code/druid.yaml b/docs/modules/getting_started/examples/code/druid.yaml index 41713e50..d51f30a9 100644 --- a/docs/modules/getting_started/examples/code/druid.yaml +++ b/docs/modules/getting_started/examples/code/druid.yaml @@ -31,6 +31,10 @@ spec: roleGroups: default: replicas: 1 + config: + resources: + memory: + limit: 4Gi middleManagers: roleGroups: default: diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 8490fb8c..bbb64eca 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -1,11 +1,11 @@ -use std::{collections::BTreeMap, str::FromStr}; - -use snafu::{ResultExt, Snafu}; +use lazy_static::lazy_static; +use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ commons::resources::{NoRuntimeLimits, Resources}, cpu::CpuQuantity, memory::{BinaryMultiple, MemoryQuantity}, }; +use std::{collections::BTreeMap}; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ @@ -13,6 +13,14 @@ use crate::{ PROCESSING_NUMTHREADS, }; +static MIN_HEAP_RATIO: f32 = 0.75; +lazy_static! { + pub static ref RESERVED_OS_MEMORY: MemoryQuantity = MemoryQuantity::from_mebi(300.); + /// Max size for direct access buffers. This is defined in Druid to be 2GB: + /// https://druid.apache.org/docs/latest/configuration/index.html#processing-1 + static ref MAX_DIRECT_BUFFER_SIZE: MemoryQuantity = MemoryQuantity::from_gibi(2.); +} + /// This Error cannot derive PartialEq because fragment::ValidationError doesn't derive it #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] @@ -22,6 +30,14 @@ pub enum Error { ParsingMemoryLimitFailure { source: stackable_operator::error::Error, }, + #[snafu(display("failed to parse CPU limits"))] + ParsingCpuLimitFailure { + source: stackable_operator::error::Error, + }, + #[snafu(display("could not derive memory distribution, no memory limits defined"))] + NoMemoryLimitsDefined, + #[snafu(display("could not derive memory distribution, no CPU limits defined"))] + NoCpuLimitsDefined, } /// This struct takes the resource limits of the Pod and derives Druid settings from it. @@ -43,9 +59,9 @@ impl HistoricalDerivedSettings { Self { total_memory, cpu_millis, - min_heap_ratio: 0.75, - os_reserved_memory: MemoryQuantity::from_str("300Mi").unwrap(), // 300MB - max_buffer_size: MemoryQuantity::from_str("2Gi").unwrap(), // 2GB, Druid recommended + min_heap_ratio: MIN_HEAP_RATIO, + os_reserved_memory: *RESERVED_OS_MEMORY, + max_buffer_size: *MAX_DIRECT_BUFFER_SIZE, } } @@ -116,9 +132,11 @@ impl TryFrom<&Resources> for HistoricalDeriv type Error = Error; fn try_from(r: &Resources) -> Result { - let total_memory = MemoryQuantity::try_from(r.memory.limit.clone().unwrap()) - .context(ParsingMemoryLimitFailureSnafu)?; - let cpu_millis = CpuQuantity::try_from(r.cpu.max.clone().unwrap()).unwrap(); // TODO no unwrap + let total_memory = + MemoryQuantity::try_from(r.memory.limit.clone().context(NoMemoryLimitsDefinedSnafu)?) + .context(ParsingMemoryLimitFailureSnafu)?; + let cpu_millis = CpuQuantity::try_from(r.cpu.max.clone().context(NoCpuLimitsDefinedSnafu)?) + .context(ParsingCpuLimitFailureSnafu)?; Ok(HistoricalDerivedSettings::new(total_memory, cpu_millis)) } } @@ -154,7 +172,7 @@ mod tests { #[case(3600, 3)] #[case(32_000, 31)] fn test_num_threads(#[case] cpu_millis: usize, #[case] expected_num_threads: usize) { - let mem = MemoryQuantity::from_str("2Gi").unwrap(); + let mem = MemoryQuantity::from_gibi(2.); let cpu = CpuQuantity::from_millis(cpu_millis); let s = HistoricalDerivedSettings::new(mem, cpu); assert_eq!(s.num_threads(), expected_num_threads); @@ -173,7 +191,7 @@ mod tests { #[case] cpu_millis: usize, #[case] expected_num_merge_buffers: usize, ) { - let mem = MemoryQuantity::from_str("2Gi").unwrap(); + let mem = MemoryQuantity::from_gibi(2.); let cpu = CpuQuantity::from_millis(cpu_millis); let s = HistoricalDerivedSettings::new(mem, cpu); assert_eq!(s.num_merge_buffers(), expected_num_merge_buffers); diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 12d17a58..8a2f3d4c 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -1,15 +1,14 @@ //! Ensures that `Pod`s are configured and running for each [`DruidCluster`] use crate::{ + OPERATOR_NAME, config::{get_jvm_config, get_log4j_config}, discovery::{self, build_discovery_configmaps}, extensions::get_extension_list, }; -use crate::OPERATOR_NAME; -use lazy_static::lazy_static; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_druid_crd::{ - authorization::DruidAuthorization, build_string_list, memory::HistoricalDerivedSettings, + authorization::DruidAuthorization, build_string_list, memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}, security::DruidTlsSecurity, DeepStorageSpec, DruidCluster, DruidRole, APP_NAME, AUTH_AUTHORIZER_OPA_URI, CERTS_DIR, CREDENTIALS_SECRET_PROPERTY, DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, HDFS_CONFIG_DIRECTORY, JVM_CONFIG, LOG4J2_CONFIG, @@ -63,9 +62,6 @@ use strum::{EnumDiscriminants, IntoStaticStr}; pub const CONTROLLER_NAME: &str = "druidcluster"; const DOCKER_IMAGE_BASE_NAME: &str = "druid"; -lazy_static! { - static ref RESERVED_OS_MEMORY: MemoryQuantity = MemoryQuantity::from_mebi(300.); -} pub struct Ctx { pub client: stackable_operator::client::Client, @@ -504,7 +500,7 @@ fn build_rolegroup_config_map( cm_conf_data.insert(RUNTIME_PROPS.to_string(), runtime_properties); } PropertyNameKind::File(file_name) if file_name == JVM_CONFIG => { - // This large match structure computes the heap and direct access memory requirements for each role. + // This match structure computes the heap and direct access memory requirements for each role. // Both parameters are then used to construct the JVM config. let (heap, direct) = match resources { RoleResource::Historical(r) => { From 0c9cb81b00617bd1915c59b1efe2d8bc123f6836 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 30 Jan 2023 10:56:18 +0100 Subject: [PATCH 16/47] fix: fixed test --- rust/crd/src/memory.rs | 2 +- rust/crd/src/resource.rs | 29 ++++++++------------ rust/operator-binary/src/druid_controller.rs | 16 ++++++----- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index bbb64eca..4ab129ae 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -5,7 +5,7 @@ use stackable_operator::{ cpu::CpuQuantity, memory::{BinaryMultiple, MemoryQuantity}, }; -use std::{collections::BTreeMap}; +use std::collections::BTreeMap; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ diff --git a/rust/crd/src/resource.rs b/rust/crd/src/resource.rs index 77022849..e60d568f 100644 --- a/rust/crd/src/resource.rs +++ b/rust/crd/src/resource.rs @@ -589,18 +589,14 @@ mod test { role_group: "default".into(), }; let res = resources(&cluster, &DruidRole::Historical, &rolegroup_ref)?; - let mut got = BTreeMap::new(); + res.update_druid_config_file(&mut got); - let expected: BTreeMap> = vec![ - (PROP_SEGMENT_CACHE_LOCATIONS.to_string(), - Some(r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"5g","freeSpacePercent":"3"}]"#.to_string())) - ].into_iter().collect(); + assert!(got.contains_key(PROP_SEGMENT_CACHE_LOCATIONS)); - assert_eq!( - got, expected, - "role: historical, group: default, segment cache config" - ); + let value = got.get(PROP_SEGMENT_CACHE_LOCATIONS).unwrap(); + let expected = Some(r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"5g","freeSpacePercent":"3"}]"#.to_string()); + assert_eq!(value, &expected, "primary"); // ---------- secondary role group let rolegroup_ref = RoleGroupRef { @@ -610,16 +606,13 @@ mod test { }; let res = resources(&cluster, &DruidRole::Historical, &rolegroup_ref)?; let mut got = BTreeMap::new(); + res.update_druid_config_file(&mut got); - let expected = vec![ - (PROP_SEGMENT_CACHE_LOCATIONS.to_string(), - Some(r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"2g","freeSpacePercent":"7"}]"#.to_string())) - ].into_iter().collect(); - - assert_eq!( - got, expected, - "role: historical, group: secondary, segment cache config" - ); + assert!(got.contains_key(PROP_SEGMENT_CACHE_LOCATIONS)); + + let value = got.get(PROP_SEGMENT_CACHE_LOCATIONS).unwrap(); + let expected = Some(r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"2g","freeSpacePercent":"7"}]"#.to_string()); + assert_eq!(value, &expected, "secondary"); Ok(()) } diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 8a2f3d4c..4b1376f8 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -1,19 +1,21 @@ //! Ensures that `Pod`s are configured and running for each [`DruidCluster`] use crate::{ - OPERATOR_NAME, config::{get_jvm_config, get_log4j_config}, discovery::{self, build_discovery_configmaps}, extensions::get_extension_list, + OPERATOR_NAME, }; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_druid_crd::{ - authorization::DruidAuthorization, build_string_list, memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}, - security::DruidTlsSecurity, DeepStorageSpec, DruidCluster, DruidRole, APP_NAME, - AUTH_AUTHORIZER_OPA_URI, CERTS_DIR, CREDENTIALS_SECRET_PROPERTY, DRUID_CONFIG_DIRECTORY, - DS_BUCKET, EXTENSIONS_LOADLIST, HDFS_CONFIG_DIRECTORY, JVM_CONFIG, LOG4J2_CONFIG, - RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ENDPOINT_URL, S3_PATH_STYLE_ACCESS, S3_SECRET_DIR_NAME, - ZOOKEEPER_CONNECTION_STRING, + authorization::DruidAuthorization, + build_string_list, + memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}, + security::DruidTlsSecurity, + DeepStorageSpec, DruidCluster, DruidRole, APP_NAME, AUTH_AUTHORIZER_OPA_URI, CERTS_DIR, + CREDENTIALS_SECRET_PROPERTY, DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, + HDFS_CONFIG_DIRECTORY, JVM_CONFIG, LOG4J2_CONFIG, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, + S3_ENDPOINT_URL, S3_PATH_STYLE_ACCESS, S3_SECRET_DIR_NAME, ZOOKEEPER_CONNECTION_STRING, }; use stackable_druid_crd::{ build_recommended_labels, From bc24d6c62c15dd874db4981304e1fccf9d9060b9 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 30 Jan 2023 11:17:42 +0100 Subject: [PATCH 17/47] fix: removed some more unwraps --- rust/operator-binary/src/config.rs | 19 +++++++++++++++---- rust/operator-binary/src/druid_controller.rs | 7 ++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 41041b3e..76d777e3 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -1,4 +1,6 @@ +use crate::druid_controller::{Error, FormatMemoryStringForJavaSnafu}; use indoc::formatdoc; +use snafu::ResultExt; use stackable_druid_crd::{DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD}; use stackable_operator::memory::MemoryQuantity; @@ -6,9 +8,18 @@ pub fn get_jvm_config( role: &DruidRole, heap: MemoryQuantity, direct_memory: Option, -) -> String { - let heap_str = heap.format_for_java().unwrap(); // TODO fix unwrap - let direct_memory_str = direct_memory.map(|m| m.format_for_java().unwrap()); // TODO fix unwrap +) -> Result { + let heap_str = heap + .format_for_java() + .context(FormatMemoryStringForJavaSnafu)?; + let direct_memory_str = if let Some(m) = direct_memory { + Some( + m.format_for_java() + .context(FormatMemoryStringForJavaSnafu)?, + ) + } else { + None + }; let mut config = formatdoc! {" -server -Duser.timezone=UTC @@ -30,7 +41,7 @@ pub fn get_jvm_config( if role == &DruidRole::Coordinator { config += "\n-Dderby.stream.error.file=/stackable/var/druid/derby.log"; } - config + Ok(config) } pub fn get_log4j_config(_role: &DruidRole) -> String { diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 4b1376f8..2f3203f6 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -71,6 +71,7 @@ pub struct Ctx { } #[derive(Snafu, Debug, EnumDiscriminants)] +#[snafu(visibility(pub(crate)))] #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] pub enum Error { @@ -186,6 +187,10 @@ pub enum Error { FailedToInitializeSecurityContext { source: stackable_druid_crd::security::Error, }, + #[snafu(display("failed to format memory quantity for Java"))] + FormatMemoryStringForJava { + source: stackable_operator::error::Error, + }, } type Result = std::result::Result; @@ -543,7 +548,7 @@ fn build_rolegroup_config_map( } }; let jvm_config = get_jvm_config(&role, heap, direct); - cm_conf_data.insert(JVM_CONFIG.to_string(), jvm_config); + cm_conf_data.insert(JVM_CONFIG.to_string(), jvm_config?); } PropertyNameKind::File(file_name) if file_name == LOG4J2_CONFIG => { let log_config = get_log4j_config(&role); From d255efa67254e9acbaad39dc5232b10ca24056ec Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 30 Jan 2023 11:38:30 +0100 Subject: [PATCH 18/47] fix: fixed more todos --- rust/operator-binary/src/druid_controller.rs | 23 ++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 2f3203f6..07065aef 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -191,6 +191,18 @@ pub enum Error { FormatMemoryStringForJava { source: stackable_operator::error::Error, }, + #[snafu(display("failed to derive Druid memory settings from resources"))] + DeriveMemorySettings { + source: stackable_druid_crd::memory::Error, + }, + #[snafu(display("failed to get memory limits"))] + GetMemoryLimit, + #[snafu(display("failed to parse memory quantity"))] + ParseMemoryQuantity { + source: stackable_operator::error::Error, + }, + #[snafu(display("the operator produced an internally inconsistent state"))] + InconsistentConfiguration, } type Result = std::result::Result; @@ -511,17 +523,20 @@ fn build_rolegroup_config_map( // Both parameters are then used to construct the JVM config. let (heap, direct) = match resources { RoleResource::Historical(r) => { - let settings = HistoricalDerivedSettings::try_from(r).unwrap(); // TODO fix unwrap + let settings = HistoricalDerivedSettings::try_from(r) + .context(DeriveMemorySettingsSnafu)?; ( settings.heap_memory(), Some(settings.direct_access_memory()), ) } RoleResource::Druid(r) => { - let total_memory = - MemoryQuantity::try_from(r.memory.limit.as_ref().unwrap()).unwrap(); // TODO fix unwrap + let total_memory = MemoryQuantity::try_from( + r.memory.limit.as_ref().context(GetMemoryLimitSnafu)?, + ) + .context(ParseMemoryQuantitySnafu)?; match role { - DruidRole::Historical => panic!(), // TODO fix panic; we cannot reach this arm here + DruidRole::Historical => return Err(Error::InconsistentConfiguration), DruidRole::Coordinator => { // The coordinator needs no direct memory let heap_memory = total_memory - *RESERVED_OS_MEMORY; From 5b3a50e21510e68414fa8b707de3b01e13de0d00 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 30 Jan 2023 11:46:03 +0100 Subject: [PATCH 19/47] reverted an accidental change in the docs --- docs/modules/getting_started/examples/code/druid.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/modules/getting_started/examples/code/druid.yaml b/docs/modules/getting_started/examples/code/druid.yaml index d51f30a9..41713e50 100644 --- a/docs/modules/getting_started/examples/code/druid.yaml +++ b/docs/modules/getting_started/examples/code/druid.yaml @@ -31,10 +31,6 @@ spec: roleGroups: default: replicas: 1 - config: - resources: - memory: - limit: 4Gi middleManagers: roleGroups: default: From 8232b3472c4a5c727c6b938ddee559bf7d57d930 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 30 Jan 2023 11:54:53 +0100 Subject: [PATCH 20/47] Some more docs --- rust/crd/src/memory.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 4ab129ae..73ed6151 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -21,7 +21,6 @@ lazy_static! { static ref MAX_DIRECT_BUFFER_SIZE: MemoryQuantity = MemoryQuantity::from_gibi(2.); } -/// This Error cannot derive PartialEq because fragment::ValidationError doesn't derive it #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] @@ -70,7 +69,9 @@ impl HistoricalDerivedSettings { self.total_memory - self.os_reserved_memory } - /// How much memory to set for the JVM to use. + /// How much memory to set for the JVM to use. The minimum ratio can be defined in the struct. + /// Once the direct memory is maxed out, all the remaining allocatable memory will be assigned + /// as heap memory. pub fn heap_memory(&self) -> MemoryQuantity { // TODO also implement max limit of 24Gi, as recommended by Druid self.allocatable_memory() - self.direct_access_memory() @@ -83,7 +84,7 @@ impl HistoricalDerivedSettings { /// The max memory to allocate to direct access. This is based on the max buffer size of a single buffer. pub fn max_direct_access_memory(&self) -> MemoryQuantity { - self.max_buffer_size * (self.num_merge_buffers() + self.num_threads() + 1) as f32 + self.max_buffer_size * self.total_num_buffers() as f32 } /// How much to allocate (or keep free) for direct access. @@ -105,13 +106,18 @@ impl HistoricalDerivedSettings { ((self.num_threads() as f64 / 4.).floor() as usize).max(2) } + fn total_num_buffers(&self) -> usize { + self.num_merge_buffers() + self.num_threads() + 1 + } + /// The buffer size for intermediate result storage. By setting it ourselves, we can set it up to 2Gi. /// If we leave it on the `auto` default, we only get up to 1Gi. /// Druid property: `druid.processing.buffer.sizeBytes` pub fn buffer_size(&self) -> MemoryQuantity { - self.direct_access_memory() / (self.num_threads() + self.num_merge_buffers() + 1) as f32 + self.direct_access_memory() / self.total_num_buffers() as f32 } + /// Adds derived runtime settings to the given config pub fn add_settings(&self, config: &mut BTreeMap>) { config.insert( PROCESSING_NUMTHREADS.to_owned(), From 9a204a03746812af74a616c6aaa0881af2d86690 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 30 Jan 2023 12:07:32 +0100 Subject: [PATCH 21/47] fixed an unwrap --- rust/crd/src/resource.rs | 11 +++++++++-- rust/operator-binary/src/druid_controller.rs | 8 +++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/rust/crd/src/resource.rs b/rust/crd/src/resource.rs index e60d568f..9b9ffe5f 100644 --- a/rust/crd/src/resource.rs +++ b/rust/crd/src/resource.rs @@ -38,6 +38,8 @@ pub enum Error { source: Box, rolegroup_ref: RoleGroupRef, }, + #[snafu(display("failed to derive Druid settings from resources"))] + DeriveMemorySettings { source: crate::memory::Error }, } /// The sole purpose of this enum is to handle merging. It's needed because currently @@ -86,7 +88,10 @@ impl RoleResource { /// Update the given configuration file with resource properties. /// Currently it only adds the segment cache location property for historicals to runtime.properties. /// This is only for runtime properties file - pub fn update_druid_config_file(&self, config: &mut BTreeMap>) { + pub fn update_druid_config_file( + &self, + config: &mut BTreeMap>, + ) -> Result<(), Error> { match self { RoleResource::Historical(r) => { let free_percentage = r.storage.segment_cache.free_percentage.unwrap_or(5u16); @@ -100,11 +105,13 @@ impl RoleResource { )) }); - let settings = HistoricalDerivedSettings::try_from(r).unwrap(); // TODO fix unwrap + let settings = + HistoricalDerivedSettings::try_from(r).context(DeriveMemorySettingsSnafu)?; settings.add_settings(config); } RoleResource::Druid(_) => (), } + Ok(()) } pub fn update_volumes_and_volume_mounts(&self, cb: &mut ContainerBuilder, pb: &mut PodBuilder) { diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 07065aef..01505ec4 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -203,6 +203,10 @@ pub enum Error { }, #[snafu(display("the operator produced an internally inconsistent state"))] InconsistentConfiguration, + #[snafu(display("failed to update Druid config from resources"))] + UpdateDruidConfigFromResources { + source: stackable_druid_crd::resource::Error, + }, } type Result = std::result::Result; @@ -464,7 +468,9 @@ fn build_rolegroup_config_map( // This has to be done here since there is no other suitable place for it. // Previously such properties were added in the compute_files() function, // but that code path is now incompatible with the design of fragment merging. - resources.update_druid_config_file(&mut transformed_config); + resources + .update_druid_config_file(&mut transformed_config) + .context(UpdateDruidConfigFromResourcesSnafu)?; // NOTE: druid.host can be set manually - if it isn't, the canonical host name of // the local host is used. This should work with the agent and k8s host networking // but might need to be revisited in the future From ecb50c4ae23c5f7810f89f10109229a6e3cbf8fc Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 30 Jan 2023 12:13:47 +0100 Subject: [PATCH 22/47] clippy fix in tests --- rust/crd/src/resource.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/crd/src/resource.rs b/rust/crd/src/resource.rs index 9b9ffe5f..aeaa84ef 100644 --- a/rust/crd/src/resource.rs +++ b/rust/crd/src/resource.rs @@ -598,7 +598,7 @@ mod test { let res = resources(&cluster, &DruidRole::Historical, &rolegroup_ref)?; let mut got = BTreeMap::new(); - res.update_druid_config_file(&mut got); + assert!(res.update_druid_config_file(&mut got).is_ok()); assert!(got.contains_key(PROP_SEGMENT_CACHE_LOCATIONS)); let value = got.get(PROP_SEGMENT_CACHE_LOCATIONS).unwrap(); @@ -614,7 +614,7 @@ mod test { let res = resources(&cluster, &DruidRole::Historical, &rolegroup_ref)?; let mut got = BTreeMap::new(); - res.update_druid_config_file(&mut got); + assert!(res.update_druid_config_file(&mut got).is_ok()); assert!(got.contains_key(PROP_SEGMENT_CACHE_LOCATIONS)); let value = got.get(PROP_SEGMENT_CACHE_LOCATIONS).unwrap(); From 3778f0b715da9d3bdd518beb8e6dd6514f35681f Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 30 Jan 2023 13:16:12 +0100 Subject: [PATCH 23/47] tests: update resource test to test for the bug we're fixing --- .../kuttl/resources/20-install-druid.yaml.j2 | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/templates/kuttl/resources/20-install-druid.yaml.j2 b/tests/templates/kuttl/resources/20-install-druid.yaml.j2 index dce0a12f..dede7824 100644 --- a/tests/templates/kuttl/resources/20-install-druid.yaml.j2 +++ b/tests/templates/kuttl/resources/20-install-druid.yaml.j2 @@ -41,12 +41,15 @@ spec: default: replicas: 1 config: - resources: - storage: - segmentCache: - freePercentage: 3 - emptyDir: - capacity: 2G + resources: + memory: + limit: 4Gi + storage: + segmentCache: + freePercentage: 3 + emptyDir: + capacity: 2G + middleManagers: config: resources: From 0f5c44b1733c4d609ba7019fe55eefa9a7499660 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 30 Jan 2023 14:28:17 +0100 Subject: [PATCH 24/47] fix tests --- tests/templates/kuttl/resources/20-assert.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/templates/kuttl/resources/20-assert.yaml b/tests/templates/kuttl/resources/20-assert.yaml index dff64083..a0ecb6c8 100644 --- a/tests/templates/kuttl/resources/20-assert.yaml +++ b/tests/templates/kuttl/resources/20-assert.yaml @@ -55,10 +55,10 @@ spec: resources: requests: cpu: 200m - memory: 2Gi + memory: 4Gi limits: cpu: "4" - memory: 2Gi + memory: 4Gi volumes: - name: tls-mount ephemeral: From 143822a67576a57e86ce6cf20e1cb9a52af32222 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 30 Jan 2023 15:44:08 +0100 Subject: [PATCH 25/47] fix tests ... this time really? --- tests/templates/kuttl/resources/30-assert.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/resources/30-assert.yaml b/tests/templates/kuttl/resources/30-assert.yaml index cfdba52e..afee00b3 100644 --- a/tests/templates/kuttl/resources/30-assert.yaml +++ b/tests/templates/kuttl/resources/30-assert.yaml @@ -5,7 +5,7 @@ timeout: 600 commands: - script: kubectl get cm -n $NAMESPACE druid-resources-broker-default -o yaml | grep -- '-Xmx1638m' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-coordinator-default -o yaml | grep -- '-Xmx1638m' | xargs test ! -z - - script: kubectl get cm -n $NAMESPACE druid-resources-historical-default -o yaml | grep -- '-Xmx1638m' | xargs test ! -z + - script: kubectl get cm -n $NAMESPACE druid-resources-historical-default -o yaml | grep -- '-Xmx2847m' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-middlemanager-resources-from-role -o yaml | grep -- '-Xmx819m' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-middlemanager-resources-from-role-group -o yaml | grep -- '-Xmx2457m' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-router-default -o yaml | grep -- '-Xmx1638m' | xargs test ! -z From c0ae45e77a0cc60190ef8efcf00f8ae695d76e06 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 30 Jan 2023 17:08:23 +0100 Subject: [PATCH 26/47] fix tests ... this time really, really? --- tests/templates/kuttl/resources/30-assert.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/templates/kuttl/resources/30-assert.yaml b/tests/templates/kuttl/resources/30-assert.yaml index afee00b3..9f4a9c6d 100644 --- a/tests/templates/kuttl/resources/30-assert.yaml +++ b/tests/templates/kuttl/resources/30-assert.yaml @@ -3,9 +3,9 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 commands: - - script: kubectl get cm -n $NAMESPACE druid-resources-broker-default -o yaml | grep -- '-Xmx1638m' | xargs test ! -z - - script: kubectl get cm -n $NAMESPACE druid-resources-coordinator-default -o yaml | grep -- '-Xmx1638m' | xargs test ! -z + - script: kubectl get cm -n $NAMESPACE druid-resources-broker-default -o yaml | grep -- '-Xmx1348m' | xargs test ! -z + - script: kubectl get cm -n $NAMESPACE druid-resources-coordinator-default -o yaml | grep -- '-Xmx1738m' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-historical-default -o yaml | grep -- '-Xmx2847m' | xargs test ! -z - - script: kubectl get cm -n $NAMESPACE druid-resources-middlemanager-resources-from-role -o yaml | grep -- '-Xmx819m' | xargs test ! -z - - script: kubectl get cm -n $NAMESPACE druid-resources-middlemanager-resources-from-role-group -o yaml | grep -- '-Xmx2457m' | xargs test ! -z - - script: kubectl get cm -n $NAMESPACE druid-resources-router-default -o yaml | grep -- '-Xmx1638m' | xargs test ! -z + - script: kubectl get cm -n $NAMESPACE druid-resources-middlemanager-resources-from-role -o yaml | grep -- '-Xmx724m' | xargs test ! -z + - script: kubectl get cm -n $NAMESPACE druid-resources-middlemanager-resources-from-role-group -o yaml | grep -- '-Xmx2772m' | xargs test ! -z + - script: kubectl get cm -n $NAMESPACE druid-resources-router-default -o yaml | grep -- '-Xmx1620m' | xargs test ! -z From aa2ffb5957ccbb9930a931ca4aca1e16bbbb4db9 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 31 Jan 2023 09:14:57 +0100 Subject: [PATCH 27/47] fix test typo --- tests/templates/kuttl/resources/30-assert.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/resources/30-assert.yaml b/tests/templates/kuttl/resources/30-assert.yaml index 9f4a9c6d..b7e983d7 100644 --- a/tests/templates/kuttl/resources/30-assert.yaml +++ b/tests/templates/kuttl/resources/30-assert.yaml @@ -4,7 +4,7 @@ kind: TestAssert timeout: 600 commands: - script: kubectl get cm -n $NAMESPACE druid-resources-broker-default -o yaml | grep -- '-Xmx1348m' | xargs test ! -z - - script: kubectl get cm -n $NAMESPACE druid-resources-coordinator-default -o yaml | grep -- '-Xmx1738m' | xargs test ! -z + - script: kubectl get cm -n $NAMESPACE druid-resources-coordinator-default -o yaml | grep -- '-Xmx1748m' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-historical-default -o yaml | grep -- '-Xmx2847m' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-middlemanager-resources-from-role -o yaml | grep -- '-Xmx724m' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-middlemanager-resources-from-role-group -o yaml | grep -- '-Xmx2772m' | xargs test ! -z From f3e3715c159e5d502d84b6e91139e1ca6aea4101 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 31 Jan 2023 09:47:16 +0100 Subject: [PATCH 28/47] fix: adapt to changes in operator-rs after review --- Cargo.lock | 4 ++-- rust/crd/src/memory.rs | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b94d9dc..a71c7423 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1870,7 +1870,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.32.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#6f4bb27dcbddcbfacc8f7db50d7d7661478b3992" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#1b27941292c8fb9d3379d999f8fe8738cd460250" dependencies = [ "chrono", "clap", @@ -1904,7 +1904,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.32.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#6f4bb27dcbddcbfacc8f7db50d7d7661478b3992" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#1b27941292c8fb9d3379d999f8fe8738cd460250" dependencies = [ "darling", "proc-macro2", diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 73ed6151..8b68af4e 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -90,8 +90,11 @@ impl HistoricalDerivedSettings { /// How much to allocate (or keep free) for direct access. /// this is the amount to configure in the JVM as the `MaxDirectMemorySize`. pub fn direct_access_memory(&self) -> MemoryQuantity { - self.allocatable_direct_access_memory() - .min(self.max_direct_access_memory()) + if self.max_direct_access_memory() < self.allocatable_direct_access_memory() { + self.max_direct_access_memory() + } else { + self.allocatable_direct_access_memory() + } } /// The number of threads to use, based on the CPU millis. From a68068219ec8bc03a2fbce41b758615f46481ad3 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 31 Jan 2023 09:58:18 +0100 Subject: [PATCH 29/47] chore: clippy fixes --- rust/operator-binary/src/druid_controller.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 5a96a8e5..f9ca5234 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -19,7 +19,6 @@ use stackable_druid_crd::{ PLACEHOLDER_LDAP_BIND_USER, }, memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}, - security::DruidTlsSecurity, security::{resolve_authentication_classes, DruidTlsSecurity}, DeepStorageSpec, DruidCluster, DruidRole, APP_NAME, AUTH_AUTHORIZER_OPA_URI, CERTS_DIR, CREDENTIALS_SECRET_PROPERTY, DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, @@ -55,13 +54,11 @@ use stackable_operator::{ }, labels::{role_group_selector_labels, role_selector_labels}, logging::controller::ReconcilerError, + memory::MemoryQuantity, product_config::{types::PropertyNameKind, ProductConfigManager}, product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::RoleGroupRef, }; -use stackable_operator::{ - commons::product_image_selection::ResolvedProductImage, memory::MemoryQuantity, -}; use std::{ collections::{BTreeMap, HashMap}, ops::Deref, From 823f952b1b5bece9728c435b70a41ed53f1b0ba9 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:16:34 +0100 Subject: [PATCH 30/47] Update rust/crd/src/memory.rs Co-authored-by: Malte Sander --- rust/crd/src/memory.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 8b68af4e..293a1875 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -6,7 +6,6 @@ use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, }; use std::collections::BTreeMap; -use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ storage::HistoricalStorage, PROCESSING_BUFFER_SIZEBYTES, PROCESSING_NUMMERGEBUFFERS, @@ -21,9 +20,7 @@ lazy_static! { static ref MAX_DIRECT_BUFFER_SIZE: MemoryQuantity = MemoryQuantity::from_gibi(2.); } -#[derive(Snafu, Debug, EnumDiscriminants)] -#[strum_discriminants(derive(IntoStaticStr))] -#[allow(clippy::enum_variant_names)] +#[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to parse memory limits"))] ParsingMemoryLimitFailure { From ccd80c3590a5b965daa8fc8f5c7be2e475cba4ca Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:16:54 +0100 Subject: [PATCH 31/47] Update rust/crd/src/memory.rs Co-authored-by: Malte Sander --- rust/crd/src/memory.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 293a1875..5c251f15 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -40,8 +40,8 @@ pub enum Error { /// For mentioned Druid properties, consult the /// [Druid Configuration Reference](https://druid.apache.org/docs/latest/configuration/index.html) /// for additional information. -/// Also have a look at the "Basic Cluster Tuning" documentation: -/// `` +/// Also have a look at the documentation for +/// [Basic Cluster Tuning](). pub struct HistoricalDerivedSettings { total_memory: MemoryQuantity, cpu_millis: CpuQuantity, From d16776fc0eb7840969eb8d41c41fbd9b7551d9cd Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:17:19 +0100 Subject: [PATCH 32/47] Update rust/crd/src/memory.rs Co-authored-by: Malte Sander --- rust/crd/src/memory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 5c251f15..019508aa 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -62,7 +62,7 @@ impl HistoricalDerivedSettings { } /// The total memory we use for druid. This is what's left after we take out the OS reserved memory. - pub fn allocatable_memory(&self) -> MemoryQuantity { + fn allocatable_memory(&self) -> MemoryQuantity { self.total_memory - self.os_reserved_memory } From 0c01471666f0b3054afda3580864e62fe08b689f Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:17:34 +0100 Subject: [PATCH 33/47] Update rust/crd/src/memory.rs Co-authored-by: Malte Sander --- rust/crd/src/memory.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 019508aa..a7d1b7b7 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -75,12 +75,12 @@ impl HistoricalDerivedSettings { } /// The memory that is available to allocate for direct access. - pub fn allocatable_direct_access_memory(&self) -> MemoryQuantity { + fn allocatable_direct_access_memory(&self) -> MemoryQuantity { self.allocatable_memory() * (1. - self.min_heap_ratio) } /// The max memory to allocate to direct access. This is based on the max buffer size of a single buffer. - pub fn max_direct_access_memory(&self) -> MemoryQuantity { + fn max_direct_access_memory(&self) -> MemoryQuantity { self.max_buffer_size * self.total_num_buffers() as f32 } From 52e2c5ec73762cfafc7e80bb1279ca1224c009d2 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:27:32 +0100 Subject: [PATCH 34/47] renamed consts --- rust/crd/src/lib.rs | 6 +++--- rust/crd/src/memory.rs | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 18b25859..5288a97e 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -90,9 +90,9 @@ pub const MD_ST_PASSWORD: &str = "druid.metadata.storage.connector.password"; // indexer properties pub const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; // historical settings -pub const PROCESSING_BUFFER_SIZEBYTES: &str = "druid.processing.buffer.sizeBytes"; -pub const PROCESSING_NUMMERGEBUFFERS: &str = "druid.processing.numMergeBuffers"; -pub const PROCESSING_NUMTHREADS: &str = "druid.processing.numThreads"; +pub const PROCESSING_BUFFER_SIZE_BYTES: &str = "druid.processing.buffer.sizeBytes"; +pub const PROCESSING_NUM_MERGE_BUFFERS: &str = "druid.processing.numMergeBuffers"; +pub const PROCESSING_NUM_THREADS: &str = "druid.processing.numThreads"; // extra pub const CREDENTIALS_SECRET_PROPERTY: &str = "credentialsSecret"; // metrics diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index a7d1b7b7..5e95333b 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -8,8 +8,8 @@ use stackable_operator::{ use std::collections::BTreeMap; use crate::{ - storage::HistoricalStorage, PROCESSING_BUFFER_SIZEBYTES, PROCESSING_NUMMERGEBUFFERS, - PROCESSING_NUMTHREADS, + storage::HistoricalStorage, PROCESSING_BUFFER_SIZE_BYTES, PROCESSING_NUM_MERGE_BUFFERS, + PROCESSING_NUM_THREADS, }; static MIN_HEAP_RATIO: f32 = 0.75; @@ -120,15 +120,15 @@ impl HistoricalDerivedSettings { /// Adds derived runtime settings to the given config pub fn add_settings(&self, config: &mut BTreeMap>) { config.insert( - PROCESSING_NUMTHREADS.to_owned(), + PROCESSING_NUM_THREADS.to_owned(), Some(self.num_threads().to_string()), ); config.insert( - PROCESSING_NUMMERGEBUFFERS.to_owned(), + PROCESSING_NUM_MERGE_BUFFERS.to_owned(), Some(self.num_merge_buffers().to_string()), ); config.insert( - PROCESSING_BUFFER_SIZEBYTES.to_owned(), + PROCESSING_BUFFER_SIZE_BYTES.to_owned(), Some(self.buffer_size().druid_byte_format()), ); } From 9cfee695a0351fb817997a814c0bff48b11a5cc1 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:28:30 +0100 Subject: [PATCH 35/47] Update rust/crd/src/memory.rs Co-authored-by: Malte Sander --- rust/crd/src/memory.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 5e95333b..386b26e7 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -97,12 +97,12 @@ impl HistoricalDerivedSettings { /// The number of threads to use, based on the CPU millis. /// leaves at least 500m available to core functionalities. /// Druid Property: `druid.processing.numThreads` - pub fn num_threads(&self) -> usize { + fn num_threads(&self) -> usize { (self.cpu_millis.as_cpu_count().round() - 1.).max(1.) as usize } /// Druid property: `druid.processing.numMergeBuffers` - pub fn num_merge_buffers(&self) -> usize { + fn num_merge_buffers(&self) -> usize { ((self.num_threads() as f64 / 4.).floor() as usize).max(2) } From 0047dad4c9301ddb525ab9be98669df43fe31b6c Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:28:44 +0100 Subject: [PATCH 36/47] Update rust/crd/src/memory.rs Co-authored-by: Malte Sander --- rust/crd/src/memory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 386b26e7..15f8c303 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -113,7 +113,7 @@ impl HistoricalDerivedSettings { /// The buffer size for intermediate result storage. By setting it ourselves, we can set it up to 2Gi. /// If we leave it on the `auto` default, we only get up to 1Gi. /// Druid property: `druid.processing.buffer.sizeBytes` - pub fn buffer_size(&self) -> MemoryQuantity { + fn buffer_size(&self) -> MemoryQuantity { self.direct_access_memory() / self.total_num_buffers() as f32 } From d884dcce8ad4615fd6cb8d48ae89426f5ca6d374 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:29:48 +0100 Subject: [PATCH 37/47] Update rust/crd/src/memory.rs Co-authored-by: Malte Sander --- rust/crd/src/memory.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 15f8c303..37ce49c6 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -138,11 +138,16 @@ impl TryFrom<&Resources> for HistoricalDeriv type Error = Error; fn try_from(r: &Resources) -> Result { - let total_memory = - MemoryQuantity::try_from(r.memory.limit.clone().context(NoMemoryLimitsDefinedSnafu)?) - .context(ParsingMemoryLimitFailureSnafu)?; - let cpu_millis = CpuQuantity::try_from(r.cpu.max.clone().context(NoCpuLimitsDefinedSnafu)?) - .context(ParsingCpuLimitFailureSnafu)?; + let total_memory = MemoryQuantity::try_from( + r.memory + .limit + .as_ref() + .context(NoMemoryLimitsDefinedSnafu)?, + ) + .context(ParsingMemoryLimitFailureSnafu)?; + let cpu_millis = + CpuQuantity::try_from(r.cpu.max.as_ref().context(NoCpuLimitsDefinedSnafu)?) + .context(ParsingCpuLimitFailureSnafu)?; Ok(HistoricalDerivedSettings::new(total_memory, cpu_millis)) } } From 9885c69a78d149905041b88c2e1491f180f12951 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:35:14 +0100 Subject: [PATCH 38/47] refactor: changed Druid formatting from trait to function --- rust/crd/src/memory.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/rust/crd/src/memory.rs b/rust/crd/src/memory.rs index 37ce49c6..0dd194eb 100644 --- a/rust/crd/src/memory.rs +++ b/rust/crd/src/memory.rs @@ -129,7 +129,7 @@ impl HistoricalDerivedSettings { ); config.insert( PROCESSING_BUFFER_SIZE_BYTES.to_owned(), - Some(self.buffer_size().druid_byte_format()), + Some(format_for_druid(&self.buffer_size())), ); } } @@ -152,18 +152,15 @@ impl TryFrom<&Resources> for HistoricalDeriv } } -/// A trait to format something as the Druid Byte format: ``. -/// It supports human readable units, but only integer values, i.e. "1.5Gi" does not work, use "1536Mi" instead. -trait AsDruidByteFormat { - fn druid_byte_format(&self) -> String; -} - -impl AsDruidByteFormat for MemoryQuantity { - fn druid_byte_format(&self) -> String { - let k = self.scale_to(BinaryMultiple::Kibi); - let v = k.value.round() as usize; - format!("{v}Ki") - } +/// A function to format something as the Druid Byte format: +/// ``. +/// Only KiB precision is supported. Upd to 1KiB will be rounded away. +fn format_for_druid(memory_quantity: &MemoryQuantity) -> String { + let k = memory_quantity.scale_to(BinaryMultiple::Kibi); + // floor instead of round so we don't accidently make the memory quantity + // bigger than it should be + let v = k.value.floor() as usize; + format!("{v}Ki") } #[cfg(test)] From 3628d319050c9e5bbfecf276af91aa758bac3f23 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:38:01 +0100 Subject: [PATCH 39/47] docs: updated outdated docs comment --- rust/crd/src/resource.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/crd/src/resource.rs b/rust/crd/src/resource.rs index aeaa84ef..d3ae3617 100644 --- a/rust/crd/src/resource.rs +++ b/rust/crd/src/resource.rs @@ -86,8 +86,7 @@ impl RoleResource { } /// Update the given configuration file with resource properties. - /// Currently it only adds the segment cache location property for historicals to runtime.properties. - /// This is only for runtime properties file + /// Currently it only adds historical-specific configs for direct memory buffers, thread counts and segment cache. pub fn update_druid_config_file( &self, config: &mut BTreeMap>, From 7d82248972776f9a4f67ff5a144d8f53f5dc4dea Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:39:09 +0100 Subject: [PATCH 40/47] refactor: changed an enum spec back to Self --- rust/crd/src/resource.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/crd/src/resource.rs b/rust/crd/src/resource.rs index d3ae3617..a4f652e4 100644 --- a/rust/crd/src/resource.rs +++ b/rust/crd/src/resource.rs @@ -92,7 +92,7 @@ impl RoleResource { config: &mut BTreeMap>, ) -> Result<(), Error> { match self { - RoleResource::Historical(r) => { + Self::Historical(r) => { let free_percentage = r.storage.segment_cache.free_percentage.unwrap_or(5u16); let capacity = &r.storage.segment_cache.empty_dir.capacity; config @@ -108,7 +108,7 @@ impl RoleResource { HistoricalDerivedSettings::try_from(r).context(DeriveMemorySettingsSnafu)?; settings.add_settings(config); } - RoleResource::Druid(_) => (), + Self::Druid(_) => (), } Ok(()) } From 257f41274947a5229a465b54915a4a04cc812afb Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:52:11 +0100 Subject: [PATCH 41/47] refactor: moved match statement from the controller into RoleResource --- rust/crd/src/resource.rs | 60 +++++++++++++++++++- rust/operator-binary/src/druid_controller.rs | 50 ++-------------- 2 files changed, 62 insertions(+), 48 deletions(-) diff --git a/rust/crd/src/resource.rs b/rust/crd/src/resource.rs index a4f652e4..146183de 100644 --- a/rust/crd/src/resource.rs +++ b/rust/crd/src/resource.rs @@ -1,11 +1,12 @@ use std::collections::BTreeMap; -use crate::memory::HistoricalDerivedSettings; +use crate::memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}; use crate::storage::{self, FreePercentageEmptyDirFragment}; use crate::{DruidCluster, DruidRole, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS}; use lazy_static::lazy_static; -use snafu::{ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::config::fragment; +use stackable_operator::memory::MemoryQuantity; use stackable_operator::role_utils::RoleGroupRef; use stackable_operator::{ builder::{ContainerBuilder, PodBuilder, VolumeBuilder}, @@ -40,6 +41,14 @@ pub enum Error { }, #[snafu(display("failed to derive Druid settings from resources"))] DeriveMemorySettings { source: crate::memory::Error }, + #[snafu(display("failed to get memory limits"))] + GetMemoryLimit, + #[snafu(display("failed to parse memory quantity"))] + ParseMemoryQuantity { + source: stackable_operator::error::Error, + }, + #[snafu(display("the operator produced an internally inconsistent state"))] + InconsistentConfiguration, } /// The sole purpose of this enum is to handle merging. It's needed because currently @@ -126,6 +135,53 @@ impl RoleResource { ); } } + + /// Computes the heap and direct access memory sizes per role. The settings can be used to configure + /// the JVM accordingly. The direct memory size is an [`Option`] because not all roles require + /// direct access memory. + pub fn get_memory_sizes( + &self, + role: &DruidRole, + ) -> Result<(MemoryQuantity, Option), Error> { + match self { + Self::Historical(r) => { + let settings = + HistoricalDerivedSettings::try_from(r).context(DeriveMemorySettingsSnafu)?; + Ok(( + settings.heap_memory(), + Some(settings.direct_access_memory()), + )) + } + Self::Druid(r) => { + let total_memory = + MemoryQuantity::try_from(r.memory.limit.as_ref().context(GetMemoryLimitSnafu)?) + .context(ParseMemoryQuantitySnafu)?; + match role { + DruidRole::Historical => Err(Error::InconsistentConfiguration), + DruidRole::Coordinator => { + // The coordinator needs no direct memory + let heap_memory = total_memory - *RESERVED_OS_MEMORY; + Ok((heap_memory, None)) + } + DruidRole::Broker => { + let direct_memory = MemoryQuantity::from_mebi(400.); + let heap_memory = total_memory - *RESERVED_OS_MEMORY - direct_memory; + Ok((heap_memory, Some(direct_memory))) + } + DruidRole::MiddleManager => { + // The middle manager needs no direct memory + let heap_memory = total_memory - *RESERVED_OS_MEMORY; + Ok((heap_memory, None)) + } + DruidRole::Router => { + let direct_memory = MemoryQuantity::from_mebi(128.); + let heap_memory = total_memory - *RESERVED_OS_MEMORY - direct_memory; + Ok((heap_memory, Some(direct_memory))) + } + } + } + } + } } lazy_static! { diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index f9ca5234..dbeb3d71 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -18,7 +18,6 @@ use stackable_druid_crd::{ DruidLdapSettings, PLACEHOLDER_INTERNAL_CLIENT_PASSWORD, PLACEHOLDER_LDAP_BIND_PASSWORD, PLACEHOLDER_LDAP_BIND_USER, }, - memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}, security::{resolve_authentication_classes, DruidTlsSecurity}, DeepStorageSpec, DruidCluster, DruidRole, APP_NAME, AUTH_AUTHORIZER_OPA_URI, CERTS_DIR, CREDENTIALS_SECRET_PROPERTY, DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, @@ -54,7 +53,6 @@ use stackable_operator::{ }, labels::{role_group_selector_labels, role_selector_labels}, logging::controller::ReconcilerError, - memory::MemoryQuantity, product_config::{types::PropertyNameKind, ProductConfigManager}, product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::RoleGroupRef, @@ -200,7 +198,7 @@ pub enum Error { }, #[snafu(display("failed to derive Druid memory settings from resources"))] DeriveMemorySettings { - source: stackable_druid_crd::memory::Error, + source: stackable_druid_crd::resource::Error, }, #[snafu(display("failed to get memory limits"))] GetMemoryLimit, @@ -557,49 +555,9 @@ fn build_rolegroup_config_map( cm_conf_data.insert(RUNTIME_PROPS.to_string(), runtime_properties); } PropertyNameKind::File(file_name) if file_name == JVM_CONFIG => { - // This match structure computes the heap and direct access memory requirements for each role. - // Both parameters are then used to construct the JVM config. - let (heap, direct) = match resources { - RoleResource::Historical(r) => { - let settings = HistoricalDerivedSettings::try_from(r) - .context(DeriveMemorySettingsSnafu)?; - ( - settings.heap_memory(), - Some(settings.direct_access_memory()), - ) - } - RoleResource::Druid(r) => { - let total_memory = MemoryQuantity::try_from( - r.memory.limit.as_ref().context(GetMemoryLimitSnafu)?, - ) - .context(ParseMemoryQuantitySnafu)?; - match role { - DruidRole::Historical => return Err(Error::InconsistentConfiguration), - DruidRole::Coordinator => { - // The coordinator needs no direct memory - let heap_memory = total_memory - *RESERVED_OS_MEMORY; - (heap_memory, None) - } - DruidRole::Broker => { - let direct_memory = MemoryQuantity::from_mebi(400.); - let heap_memory = - total_memory - *RESERVED_OS_MEMORY - direct_memory; - (heap_memory, Some(direct_memory)) - } - DruidRole::MiddleManager => { - // The middle manager needs no direct memory - let heap_memory = total_memory - *RESERVED_OS_MEMORY; - (heap_memory, None) - } - DruidRole::Router => { - let direct_memory = MemoryQuantity::from_mebi(128.); - let heap_memory = - total_memory - *RESERVED_OS_MEMORY - direct_memory; - (heap_memory, Some(direct_memory)) - } - } - } - }; + let (heap, direct) = resources + .get_memory_sizes(&role) + .context(DeriveMemorySettingsSnafu)?; let jvm_config = get_jvm_config(&role, heap, direct); cm_conf_data.insert(JVM_CONFIG.to_string(), jvm_config?); } From 2891e9cf5281330e4f7e0bb065d0e3b6d9a3efca Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 12:53:53 +0100 Subject: [PATCH 42/47] removed obsolete error definitions --- rust/operator-binary/src/druid_controller.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index dbeb3d71..67add1ca 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -200,14 +200,6 @@ pub enum Error { DeriveMemorySettings { source: stackable_druid_crd::resource::Error, }, - #[snafu(display("failed to get memory limits"))] - GetMemoryLimit, - #[snafu(display("failed to parse memory quantity"))] - ParseMemoryQuantity { - source: stackable_operator::error::Error, - }, - #[snafu(display("the operator produced an internally inconsistent state"))] - InconsistentConfiguration, #[snafu(display("failed to update Druid config from resources"))] UpdateDruidConfigFromResources { source: stackable_druid_crd::resource::Error, From f6a28a74dd965f238f6ac11edb32ef62a2e0c849 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 13:01:55 +0100 Subject: [PATCH 43/47] Added dedicated Error in the config.rs file --- rust/operator-binary/src/config.rs | 12 ++++++++++-- rust/operator-binary/src/druid_controller.rs | 11 ++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 76d777e3..1f3c7424 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -1,9 +1,17 @@ -use crate::druid_controller::{Error, FormatMemoryStringForJavaSnafu}; use indoc::formatdoc; -use snafu::ResultExt; +use snafu::{ResultExt, Snafu}; use stackable_druid_crd::{DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD}; use stackable_operator::memory::MemoryQuantity; +#[derive(Snafu, Debug)] +#[allow(clippy::enum_variant_names)] +pub enum Error { + #[snafu(display("failed to format memory quantity for Java"))] + FormatMemoryStringForJava { + source: stackable_operator::error::Error, + }, +} + pub fn get_jvm_config( role: &DruidRole, heap: MemoryQuantity, diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 67add1ca..54f301fe 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -76,7 +76,6 @@ pub struct Ctx { } #[derive(Snafu, Debug, EnumDiscriminants)] -#[snafu(visibility(pub(crate)))] #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] pub enum Error { @@ -192,10 +191,8 @@ pub enum Error { FailedToInitializeSecurityContext { source: stackable_druid_crd::security::Error, }, - #[snafu(display("failed to format memory quantity for Java"))] - FormatMemoryStringForJava { - source: stackable_operator::error::Error, - }, + #[snafu(display("failed to get JVM config"))] + GetJvmConfig { source: crate::config::Error }, #[snafu(display("failed to derive Druid memory settings from resources"))] DeriveMemorySettings { source: stackable_druid_crd::resource::Error, @@ -550,8 +547,8 @@ fn build_rolegroup_config_map( let (heap, direct) = resources .get_memory_sizes(&role) .context(DeriveMemorySettingsSnafu)?; - let jvm_config = get_jvm_config(&role, heap, direct); - cm_conf_data.insert(JVM_CONFIG.to_string(), jvm_config?); + let jvm_config = get_jvm_config(&role, heap, direct).context(GetJvmConfigSnafu)?; + cm_conf_data.insert(JVM_CONFIG.to_string(), jvm_config); } PropertyNameKind::File(file_name) if file_name == LOG4J2_CONFIG => { let log_config = get_log4j_config(&role); From a52e25ecd0a192e4754b3c0eabc3335fb59595fb Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 13:58:10 +0100 Subject: [PATCH 44/47] update to tagged operator-rs --- Cargo.lock | 8 ++++---- rust/crd/Cargo.toml | 2 +- rust/operator-binary/Cargo.toml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce6a6a48..bed986eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1856,8 +1856,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.32.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#1b27941292c8fb9d3379d999f8fe8738cd460250" +version = "0.33.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.33.0#600bc948f2763d070a0f3c354a2b66434cf9f953" dependencies = [ "chrono", "clap", @@ -1890,8 +1890,8 @@ dependencies = [ [[package]] name = "stackable-operator-derive" -version = "0.32.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat/better-memory-quantity#1b27941292c8fb9d3379d999f8fe8738cd460250" +version = "0.33.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.33.0#600bc948f2763d070a0f3c354a2b66434cf9f953" dependencies = [ "darling", "proc-macro2", diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index a02a1679..5e5308c9 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -9,7 +9,7 @@ version = "0.9.0-nightly" publish = false [dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/better-memory-quantity" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.33.0" } semver = "1.0" serde = { version = "1.0", features = ["derive"] } diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index b40c464d..525dc0b7 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,7 @@ version = "0.9.0-nightly" publish = false [dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/better-memory-quantity" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.33.0" } stackable-druid-crd = { path = "../crd" } anyhow = "1.0" clap = "4.0" @@ -30,7 +30,7 @@ lazy_static = "1.4" [build-dependencies] built = { version = "0.5", features = ["chrono", "git2"] } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/better-memory-quantity" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.33.0" } stackable-druid-crd = { path = "../crd" } [dev-dependencies] From 52d7fb5b8ee459a79f8a2759533145b165c9a0e7 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 1 Feb 2023 15:48:16 +0100 Subject: [PATCH 45/47] better error message --- rust/operator-binary/src/config.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 1f3c7424..532a9d7b 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -6,8 +6,11 @@ use stackable_operator::memory::MemoryQuantity; #[derive(Snafu, Debug)] #[allow(clippy::enum_variant_names)] pub enum Error { - #[snafu(display("failed to format memory quantity for Java"))] + #[snafu(display( + "failed to format memory quantity '{value:?}' for Java. try increasing the memory limit" + ))] FormatMemoryStringForJava { + value: MemoryQuantity, source: stackable_operator::error::Error, }, } @@ -19,11 +22,11 @@ pub fn get_jvm_config( ) -> Result { let heap_str = heap .format_for_java() - .context(FormatMemoryStringForJavaSnafu)?; + .with_context(|_| FormatMemoryStringForJavaSnafu { value: heap })?; let direct_memory_str = if let Some(m) = direct_memory { Some( m.format_for_java() - .context(FormatMemoryStringForJavaSnafu)?, + .with_context(|_| FormatMemoryStringForJavaSnafu { value: m })?, ) } else { None From 02b1dc5a0937c03a839fda14be168788d1c1224d Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 2 Feb 2023 09:32:03 +0100 Subject: [PATCH 46/47] removed unnecessary clippy-allow --- rust/operator-binary/src/config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 532a9d7b..e9b75498 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -4,7 +4,6 @@ use stackable_druid_crd::{DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STOR use stackable_operator::memory::MemoryQuantity; #[derive(Snafu, Debug)] -#[allow(clippy::enum_variant_names)] pub enum Error { #[snafu(display( "failed to format memory quantity '{value:?}' for Java. try increasing the memory limit" From 314d4c018a70729e923380fcb1f916ebd81294f8 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 2 Feb 2023 09:54:31 +0100 Subject: [PATCH 47/47] updated resources test --- tests/templates/kuttl/resources/30-assert.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/templates/kuttl/resources/30-assert.yaml b/tests/templates/kuttl/resources/30-assert.yaml index b7e983d7..86825db2 100644 --- a/tests/templates/kuttl/resources/30-assert.yaml +++ b/tests/templates/kuttl/resources/30-assert.yaml @@ -6,6 +6,9 @@ commands: - script: kubectl get cm -n $NAMESPACE druid-resources-broker-default -o yaml | grep -- '-Xmx1348m' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-coordinator-default -o yaml | grep -- '-Xmx1748m' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-historical-default -o yaml | grep -- '-Xmx2847m' | xargs test ! -z + - script: kubectl get cm -n $NAMESPACE druid-resources-historical-default -o yaml | grep -- 'druid.processing.numThreads=3' | xargs test ! -z + - script: kubectl get cm -n $NAMESPACE druid-resources-historical-default -o yaml | grep -- 'druid.processing.numMergeBuffers=2' | xargs test ! -z + - script: kubectl get cm -n $NAMESPACE druid-resources-historical-default -o yaml | grep -- 'druid.processing.buffer.sizeBytes=161962Ki' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-middlemanager-resources-from-role -o yaml | grep -- '-Xmx724m' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-middlemanager-resources-from-role-group -o yaml | grep -- '-Xmx2772m' | xargs test ! -z - script: kubectl get cm -n $NAMESPACE druid-resources-router-default -o yaml | grep -- '-Xmx1620m' | xargs test ! -z