diff --git a/Cargo.lock b/Cargo.lock index d627c43d9fc99..f9c441016cd52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -222,8 +222,8 @@ name = "aws-util" version = "0.0.0" dependencies = [ "anyhow", - "http-util", "log", + "mz-http-proxy", "rusoto_core", "rusoto_credential", "rusoto_kinesis", @@ -385,9 +385,9 @@ version = "0.0.0" dependencies = [ "anyhow", "futures", - "http-util", "hyper", "lazy_static", + "mz-http-proxy", "native-tls", "openssl", "reqwest", @@ -1644,20 +1644,6 @@ dependencies = [ "http", ] -[[package]] -name = "http-util" -version = "0.0.0" -dependencies = [ - "http", - "hyper", - "hyper-proxy", - "hyper-tls", - "ipnet", - "lazy_static", - "log", - "reqwest", -] - [[package]] name = "httparse" version = "1.4.0" @@ -2132,7 +2118,6 @@ dependencies = [ "futures", "hex", "hex-literal", - "http-util", "hyper", "hyper-openssl", "include_dir", @@ -2142,6 +2127,7 @@ dependencies = [ "lazy_static", "libc", "log", + "mz-http-proxy", "mz-process-collector", "nix", "num_cpus", @@ -2327,6 +2313,20 @@ dependencies = [ "uuid", ] +[[package]] +name = "mz-http-proxy" +version = "0.1.0" +dependencies = [ + "http", + "hyper", + "hyper-proxy", + "hyper-tls", + "ipnet", + "lazy_static", + "log", + "reqwest", +] + [[package]] name = "mz-process-collector" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 2084478745e4a..88a13dd0712e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ members = [ "src/dataflow", "src/expr", "src/expr-test-util", - "src/http-util", + "src/http-proxy", "src/interchange", "src/kafka-util", "src/lowertest", diff --git a/bin/doc b/bin/doc index 36a196f862953..b11692d589f29 100755 --- a/bin/doc +++ b/bin/doc @@ -11,13 +11,28 @@ # # doc — renders API documentation. +# When using a nightly toolchain, this script will automatically enables the use +# of some unstable niceties, like the `doc_cfg` annotations to indicate in the +# docs what features must be enabled to use an item. +# +# CI always uses a nightly toolchain to run this script. To use one locally: +# +# $ RUSTUP_TOOLCHAIN=nightly bin/doc +# + set -euo pipefail cd "$(dirname "$0")/.." . misc/shlib/shlib.bash -RUSTDOCFLAGS="-D warnings" cargo doc "$@" +RUSTDOCFLAGS="-D warnings" +if [[ $(cargo -V) = *nightly* ]]; then + RUSTDOCFLAGS+=" --cfg nightly_doc_features" +fi +export RUSTDOCFLAGS + +cargo doc --all-features "$@" crates=$(cargo metadata --format-version=1 \ | jq -r -f misc/doc/crates.jq --arg pwd "$(pwd)") diff --git a/ci/deploy/pipeline.yml b/ci/deploy/pipeline.yml index 02ff676cff3c8..844f155cea6e8 100644 --- a/ci/deploy/pipeline.yml +++ b/ci/deploy/pipeline.yml @@ -8,7 +8,7 @@ # by the Apache License, Version 2.0. steps: - - command: bin/ci-builder run stable ci/deploy/devsite.sh + - command: bin/ci-builder run nightly ci/deploy/devsite.sh branches: main timeout_in_minutes: 30 agents: diff --git a/doc/user/content/cli/_index.md b/doc/user/content/cli/_index.md index 5aaf7a8392f9b..74066b31d4c82 100644 --- a/doc/user/content/cli/_index.md +++ b/doc/user/content/cli/_index.md @@ -362,8 +362,8 @@ variables specify a proxy to use for outgoing HTTP and HTTPS traffic. There is no precise specification of how these variables behave, but Materialize's behavior generally matches the behavior of other HTTP clients. -For precise details of Materialize's behavior, consult our -[developer docs](https://dev.materialize.com/api/rust/http_util/index.html#System_proxy_configuration). +For precise details of Materialize's behavior, consult the documentation of +the [`mz_http_proxy`](https://docs.rs/mz_http_proxy) crate. [gh-feature]: https://github.com/MaterializeInc/materialize/issues/new?labels=C-feature&template=feature.md [scv]: /sql/show-create-view diff --git a/src/aws-util/Cargo.toml b/src/aws-util/Cargo.toml index fd4ef78b0d140..755e716f66181 100644 --- a/src/aws-util/Cargo.toml +++ b/src/aws-util/Cargo.toml @@ -7,8 +7,8 @@ publish = false [dependencies] anyhow = "1.0.42" -http-util = { path = "../http-util" } log = "0.4.13" +mz-http-proxy = { path = "../http-proxy", features = ["hyper"] } rusoto_core = "0.47.0" rusoto_credential = "0.47.0" rusoto_kinesis = "0.47.0" diff --git a/src/aws-util/src/client.rs b/src/aws-util/src/client.rs index 9ebb2bb0c19b5..297548948b8df 100644 --- a/src/aws-util/src/client.rs +++ b/src/aws-util/src/client.rs @@ -25,9 +25,9 @@ use rusoto_sqs::SqsClient; use crate::aws::ConnectInfo; /// Gets an [`HttpClient`] that respects the system proxy configuration. -pub(crate) fn http() -> Result, anyhow::Error> { +pub(crate) fn http() -> Result, anyhow::Error> { Ok(HttpClient::from_connector( - http_util::hyper::connector().map_err(|e| anyhow!(e))?, + mz_http_proxy::hyper::connector().map_err(|e| anyhow!(e))?, )) } diff --git a/src/ccsr/Cargo.toml b/src/ccsr/Cargo.toml index 6bb5ed15f5f69..260b7e60a229d 100644 --- a/src/ccsr/Cargo.toml +++ b/src/ccsr/Cargo.toml @@ -8,7 +8,7 @@ publish = false [dependencies] anyhow = "1.0.42" futures = "0.3.16" -http-util = { path = "../http-util" } +mz-http-proxy = { path = "../http-proxy", features = ["reqwest"] } native-tls = "0.2.7" openssl = { version = "0.10.35", features = ["vendored"] } reqwest = { version = "0.11.4", features = ["blocking", "json", "native-tls-vendored"] } diff --git a/src/ccsr/src/config.rs b/src/ccsr/src/config.rs index bf9ac2fe4abe0..54a033446cfba 100644 --- a/src/ccsr/src/config.rs +++ b/src/ccsr/src/config.rs @@ -63,7 +63,7 @@ impl ClientConfig { /// Builds the [`Client`]. pub fn build(self) -> Result { - let mut builder = http_util::reqwest::client_builder(); + let mut builder = mz_http_proxy::reqwest::client_builder(); for root_cert in self.root_certs { builder = builder.add_root_certificate(root_cert.into()); diff --git a/src/http-proxy/Cargo.toml b/src/http-proxy/Cargo.toml new file mode 100644 index 0000000000000..e8e170c9247ad --- /dev/null +++ b/src/http-proxy/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "mz-http-proxy" +description = "HTTP proxy adapters that respect the system proxy configuration." +authors = ["Materialize, Inc."] +repository = "https://github.com/MaterializeInc/materialize/tree/main/src/http-proxy" +keywords = ["http", "proxy"] +categories = ["network-programming", "web-programming::http-client"] +version = "0.1.0" +license = "Apache-2.0" +edition = "2018" + +[dependencies] +http = "0.2.4" +hyper-dep = { package = "hyper", version = "0.14.11", optional = true } +hyper-proxy = { version = "0.9.1", optional = true } +hyper-tls = { version = "0.5.0", optional = true } +ipnet = "2.3.1" +lazy_static = "1.1.1" +log = "0.4.13" +reqwest = { version = "0.11.4", optional = true } + +[features] +hyper = ["hyper-dep", "hyper-proxy", "hyper-tls"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "nightly_doc_features"] diff --git a/src/http-proxy/README.md b/src/http-proxy/README.md new file mode 100644 index 0000000000000..61631b0a08abf --- /dev/null +++ b/src/http-proxy/README.md @@ -0,0 +1,18 @@ +# mz-http-proxy + +[![crates.io](https://img.shields.io/crates/v/mz-http-proxy.svg)](https://crates.io/crates/mz-http-proxy) +[![Rust Documentation](https://img.shields.io/badge/api-rustdoc-blue.svg)][docs] + +HTTP proxy adapters that respect the system's proxy configuration. + +**[View documentation.][docs]** + +## Installation + +```toml +# Cargo.toml +[dependencies] +mz-http-proxy = { version = "0.1", features = ["hyper", "reqwest"] } +``` + +[docs]: https://docs.rs/mz-http-proxy/0.1.0/ diff --git a/src/http-util/src/hyper.rs b/src/http-proxy/src/hyper.rs similarity index 68% rename from src/http-util/src/hyper.rs rename to src/http-proxy/src/hyper.rs index 70956dc0356f1..a812b6b51ed41 100644 --- a/src/http-util/src/hyper.rs +++ b/src/http-proxy/src/hyper.rs @@ -1,17 +1,23 @@ // Copyright Materialize, Inc. and contributors. All rights reserved. // -// Use of this software is governed by the Business Source License -// included in the LICENSE file. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file at the +// root of this repository, or online at // -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -//! Adapters for [`hyper`]. +//! Proxy adapters for [`hyper`](hyper_dep). use std::error::Error; -use hyper::client::HttpConnector; +use hyper_dep::client::HttpConnector; use hyper_proxy::{Proxy, ProxyConnector}; use hyper_tls::HttpsConnector; diff --git a/src/http-util/src/lib.rs b/src/http-proxy/src/lib.rs similarity index 61% rename from src/http-util/src/lib.rs rename to src/http-proxy/src/lib.rs index a81f1bbcbd2de..7aeeb232799db 100644 --- a/src/http-util/src/lib.rs +++ b/src/http-proxy/src/lib.rs @@ -1,19 +1,53 @@ // Copyright Materialize, Inc. and contributors. All rights reserved. // -// Use of this software is governed by the Business Source License -// included in the LICENSE file. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file at the +// root of this repository, or online at // -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #![deny(missing_docs)] +#![cfg_attr(nightly_doc_features, feature(doc_cfg))] -//! HTTP utilities. +//! [](https://materialize.com) +//! +//! HTTP proxy adapters. //! //! This crate constructs HTTP clients that respect the system's proxy //! configuration. //! +//! # Maintainership +//! +//! This crate is developed as part of [Materialize], the streaming data +//! warehouse. Contributions are encouraged: +//! +//! * [View source code](https://github.com/MaterializeInc/materialize/tree/main/src/http-proxy) +//! * [Report an issue](https://github.com/MaterializeInc/materialize/issues/new/choose) +//! * [Submit a pull request](https://github.com/MaterializeInc/materialize/compare) +//! +//! # Features +//! +//! All features are disabled by default. You will likely want to enable at +//! least one of the following features depending on the HTTP client library you +//! use: +//! +//! * The **`hyper`** feature enables the proxy adapters for use with the +//! [`hyper` crate](https://docs.rs/hyper). +//! +//! * The **`reqwest`** feature enables the proxy adapters for use with the +//! [`reqwest` crate](https://docs.rs/reqwest). +//! +//! Note that `reqwest` by default will perform its own determination of the +//! system proxy configuration, but its support for `no_proxy` is not as +//! complete as the implementation in this crate. +//! //! # System proxy configuration //! //! The system's proxy configuration is governed by four environment variables, @@ -80,8 +114,16 @@ //! //! [golang/go#16405]: https://github.com/golang/go/issues/16405 //! [gitlab-blog]: https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/ +//! [Materialize]: https://materialize.com +//! [repo]: https://github.com/MaterializeInc/materialize +#[cfg(any(feature = "hyper", feature = "reqwest"))] mod proxy; +#[cfg_attr(nightly_doc_features, doc(cfg(feature = "hyper")))] +#[cfg(feature = "hyper")] pub mod hyper; + +#[cfg_attr(nightly_doc_features, doc(cfg(feature = "reqwest")))] +#[cfg(feature = "reqwest")] pub mod reqwest; diff --git a/src/http-util/src/proxy.rs b/src/http-proxy/src/proxy.rs similarity index 95% rename from src/http-util/src/proxy.rs rename to src/http-proxy/src/proxy.rs index 962a189b2309f..cde831dd61bb7 100644 --- a/src/http-util/src/proxy.rs +++ b/src/http-proxy/src/proxy.rs @@ -1,11 +1,17 @@ // Copyright Materialize, Inc. and contributors. All rights reserved. // -// Use of this software is governed by the Business Source License -// included in the LICENSE file. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file at the +// root of this repository, or online at // -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. //! System proxy configuration guts. //! diff --git a/src/http-util/src/reqwest.rs b/src/http-proxy/src/reqwest.rs similarity index 68% rename from src/http-util/src/reqwest.rs rename to src/http-proxy/src/reqwest.rs index c4e81c050b2ce..ee0bbaec6504c 100644 --- a/src/http-util/src/reqwest.rs +++ b/src/http-proxy/src/reqwest.rs @@ -1,13 +1,19 @@ // Copyright Materialize, Inc. and contributors. All rights reserved. // -// Use of this software is governed by the Business Source License -// included in the LICENSE file. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file at the +// root of this repository, or online at // -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -//! Adapters for `reqwest`. +//! Proxy adapters for [`reqwest`]. use reqwest::ClientBuilder; diff --git a/src/http-util/Cargo.toml b/src/http-util/Cargo.toml deleted file mode 100644 index 3650051d3980e..0000000000000 --- a/src/http-util/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "http-util" -description = "HTTP utility library for Materialize." -version = "0.0.0" -license = "Apache-2.0" -edition = "2018" -publish = false - -[dependencies] -http = "0.2.4" -hyper = "0.14.11" -hyper-proxy = "0.9.1" -hyper-tls = "0.5.0" -ipnet = "2.3.1" -lazy_static = "1.1.1" -log = "0.4.13" -reqwest = "0.11.4" diff --git a/src/materialized/Cargo.toml b/src/materialized/Cargo.toml index d2344bd4325ff..5cdbd3333610b 100644 --- a/src/materialized/Cargo.toml +++ b/src/materialized/Cargo.toml @@ -44,7 +44,6 @@ dataflow-types = { path = "../dataflow-types" } differential-dataflow = { git = "https://github.com/TimelyDataflow/differential-dataflow.git" } futures = "0.3.16" hex = "0.4.3" -http-util = { path = "../http-util" } hyper = { version = "0.14.11", features = ["http1", "server"] } hyper-openssl = "0.9.1" include_dir = "0.6.1" @@ -53,6 +52,7 @@ krb5-src = { version = "0.2.3", features = ["binaries"] } lazy_static = "1.4.0" libc = "0.2.98" log = "0.4.13" +mz-http-proxy = { path = "../http-proxy", features = ["reqwest"] } mz-process-collector = { path = "../mz-process-collector" } nix = "0.20.0" num_cpus = "1.0.0" diff --git a/src/materialized/src/telemetry.rs b/src/materialized/src/telemetry.rs index a1d2d3e11ea0f..452c5db2b8996 100644 --- a/src/materialized/src/telemetry.rs +++ b/src/materialized/src/telemetry.rs @@ -110,7 +110,7 @@ async fn report_one(config: &Config) -> Result { .coord_client .system_execute_one(&TELEMETRY_QUERY) .await?; - let response = http_util::reqwest::client() + let response = mz_http_proxy::reqwest::client() .post(format!( "https://{}/api/telemetry/{}", config.domain, config.cluster_id