http-util: prepare for open-source release as mz-http-proxy

The http_proxy, https_proxy, and no_proxy support we built in the
http-util crate will be useful to at least one other person [0], and
probably many more. This commit prepares the crate for release to
crates.io.

Specifically:

  * The crate is renamed to "mz-http-proxy", for consistency with
    "mz-avro".

  * The license headers are updated to indicate the crate is Apache 2.0
    licensed.

  * The adapters are placed behind feature flags, so that you don't need
    to bring in reqwest if you just want hyper, or vice versa.

  * The documentation is improved, with a smidge of Materialize branding
    to boot.

I also switched `bin/doc` back to using the nightly toolchain in CI,
with support for optionally doing so locally too, to enable the
`doc_cfg` attributes that indicate which features must be enabled for
which items.

[0]: awslabs/aws-sdk-rust#169 (comment)

Cargo.toml

@@ -16,7 +16,7 @@ members = [
     "src/dataflow",
     "src/expr",
     "src/expr-test-util",
-    "src/http-util",
+    "src/http-proxy",
     "src/interchange",
     "src/kafka-util",
     "src/lowertest",

bin/doc

@@ -11,13 +11,28 @@
 #
 # doc — renders API documentation.
 
+# When using a nightly toolchain, this script will automatically enables the use
+# of some unstable niceties, like the `doc_cfg` annotations to indicate in the
+# docs what features must be enabled to use an item.
+#
+# CI always uses a nightly toolchain to run this script. To use one locally:
+#
+#     $ RUSTUP_TOOLCHAIN=nightly bin/doc
+#
+
 set -euo pipefail
 
 cd "$(dirname "$0")/.."
 
 . misc/shlib/shlib.bash
 
-RUSTDOCFLAGS="-D warnings" cargo doc "$@"
+RUSTDOCFLAGS="-D warnings"
+if [[ $(cargo -V) = *nightly* ]]; then
+  RUSTDOCFLAGS+=" --cfg nightly_doc_features"
+fi
+export RUSTDOCFLAGS
+
+cargo doc --all-features "$@"
 
 crates=$(cargo metadata --format-version=1 \
   | jq -r -f misc/doc/crates.jq --arg pwd "$(pwd)")

ci/deploy/pipeline.yml

@@ -8,7 +8,7 @@
 # by the Apache License, Version 2.0.
 
 steps:
-  - command: bin/ci-builder run stable ci/deploy/devsite.sh
+  - command: bin/ci-builder run nightly ci/deploy/devsite.sh
     branches: main
     timeout_in_minutes: 30
     agents:

doc/user/content/cli/_index.md

@@ -362,8 +362,8 @@ variables specify a proxy to use for outgoing HTTP and HTTPS traffic. There is
 no precise specification of how these variables behave, but Materialize's
 behavior generally matches the behavior of other HTTP clients.
 
-For precise details of Materialize's behavior, consult our
-[developer docs](https://dev.materialize.com/api/rust/http_util/index.html#System_proxy_configuration).
+For precise details of Materialize's behavior, consult the documentation of
+the [`mz_http_proxy`](https://docs.rs/mz_http_proxy) crate.
 
 [gh-feature]: https://github.com/MaterializeInc/materialize/issues/new?labels=C-feature&template=feature.md
 [scv]: /sql/show-create-view

src/aws-util/Cargo.toml

@@ -7,8 +7,8 @@ publish = false
 
 [dependencies]
 anyhow = "1.0.42"
-http-util = { path = "../http-util" }
 log = "0.4.13"
+mz-http-proxy = { path = "../http-proxy", features = ["hyper"] }
 rusoto_core = "0.47.0"
 rusoto_credential = "0.47.0"
 rusoto_kinesis = "0.47.0"

src/aws-util/src/client.rs

@@ -25,9 +25,9 @@ use rusoto_sqs::SqsClient;
 use crate::aws::ConnectInfo;
 
 /// Gets an [`HttpClient`] that respects the system proxy configuration.
-pub(crate) fn http() -> Result<HttpClient<http_util::hyper::Connector>, anyhow::Error> {
+pub(crate) fn http() -> Result<HttpClient<mz_http_proxy::hyper::Connector>, anyhow::Error> {
     Ok(HttpClient::from_connector(
-        http_util::hyper::connector().map_err(|e| anyhow!(e))?,
+        mz_http_proxy::hyper::connector().map_err(|e| anyhow!(e))?,
     ))
 }
 

src/ccsr/Cargo.toml

@@ -8,7 +8,7 @@ publish = false
 [dependencies]
 anyhow = "1.0.42"
 futures = "0.3.16"
-http-util = { path = "../http-util" }
+mz-http-proxy = { path = "../http-proxy", features = ["reqwest"] }
 native-tls = "0.2.7"
 openssl = { version = "0.10.35", features = ["vendored"] }
 reqwest = { version = "0.11.4", features = ["blocking", "json", "native-tls-vendored"] }

src/ccsr/src/config.rs

@@ -63,7 +63,7 @@ impl ClientConfig {
 
     /// Builds the [`Client`].
     pub fn build(self) -> Result<Client, anyhow::Error> {
-        let mut builder = http_util::reqwest::client_builder();
+        let mut builder = mz_http_proxy::reqwest::client_builder();
 
         for root_cert in self.root_certs {
             builder = builder.add_root_certificate(root_cert.into());

src/http-proxy/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "mz-http-proxy"
+description = "HTTP proxy adapters that respect the system proxy configuration."
+authors = ["Materialize, Inc."]
+repository = "https://github.com/MaterializeInc/materialize/tree/main/src/http-proxy"
+keywords = ["http", "proxy"]
+categories = ["network-programming", "web-programming::http-client"]
+version = "0.1.0"
+license = "Apache-2.0"
+edition = "2018"
+
+[dependencies]
+http = "0.2.4"
+hyper-dep = { package = "hyper", version = "0.14.11", optional = true }
+hyper-proxy = { version = "0.9.1", optional = true }
+hyper-tls = { version = "0.5.0", optional = true }
+ipnet = "2.3.1"
+lazy_static = "1.1.1"
+log = "0.4.13"
+reqwest = { version = "0.11.4", optional = true }
+
+[features]
+hyper = ["hyper-dep", "hyper-proxy", "hyper-tls"]
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "nightly_doc_features"]

src/http-proxy/README.md

@@ -0,0 +1,18 @@
+# mz-http-proxy
+
+[![crates.io](https://img.shields.io/crates/v/mz-http-proxy.svg)](https://crates.io/crates/mz-http-proxy)
+[![Rust Documentation](https://img.shields.io/badge/api-rustdoc-blue.svg)][docs]
+
+HTTP proxy adapters that respect the system's proxy configuration.
+
+**[View documentation.][docs]**
+
+## Installation
+
+```toml
+# Cargo.toml
+[dependencies]
+mz-http-proxy = { version = "0.1", features = ["hyper", "reqwest"] }
+```
+
+[docs]: https://docs.rs/mz-http-proxy/0.1.0/

src/http-util/src/hyper.rs → src/http-proxy/src/hyper.rs

@@ -1,17 +1,23 @@
 // Copyright Materialize, Inc. and contributors. All rights reserved.
 //
-// Use of this software is governed by the Business Source License
-// included in the LICENSE file.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License in the LICENSE file at the
+// root of this repository, or online at
 //
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0.
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
-//! Adapters for [`hyper`].
+//! Proxy adapters for [`hyper`](hyper_dep).
 
 use std::error::Error;
 
-use hyper::client::HttpConnector;
+use hyper_dep::client::HttpConnector;
 use hyper_proxy::{Proxy, ProxyConnector};
 use hyper_tls::HttpsConnector;
 

src/http-util/src/lib.rs → src/http-proxy/src/lib.rs

@@ -1,19 +1,53 @@
 // Copyright Materialize, Inc. and contributors. All rights reserved.
 //
-// Use of this software is governed by the Business Source License
-// included in the LICENSE file.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License in the LICENSE file at the
+// root of this repository, or online at
 //
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0.
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 #![deny(missing_docs)]
+#![cfg_attr(nightly_doc_features, feature(doc_cfg))]
 
-//! HTTP utilities.
+//! [<img src="https://materialize.com/wp-content/uploads/2020/01/materialize_logo_primary.png" width=180 align=right>](https://materialize.com)
+//!
+//! HTTP proxy adapters.
 //!
 //! This crate constructs HTTP clients that respect the system's proxy
 //! configuration.
 //!
+//! # Maintainership
+//!
+//! This crate is developed as part of [Materialize], the streaming data
+//! warehouse. Contributions are encouraged:
+//!
+//! * [View source code](https://github.com/MaterializeInc/materialize/tree/main/src/http-proxy)
+//! * [Report an issue](https://github.com/MaterializeInc/materialize/issues/new/choose)
+//! * [Submit a pull request](https://github.com/MaterializeInc/materialize/compare)
+//!
+//! # Features
+//!
+//! All features are disabled by default. You will likely want to enable at
+//! least one of the following features depending on the HTTP client library you
+//! use:
+//!
+//! * The **`hyper`** feature enables the proxy adapters for use with the
+//!   [`hyper` crate](https://docs.rs/hyper).
+//!
+//! * The **`reqwest`** feature enables the proxy adapters for use with the
+//!   [`reqwest` crate](https://docs.rs/reqwest).
+//!
+//! Note that `reqwest` by default will perform its own determination of the
+//! system proxy configuration, but its support for `no_proxy` is not as
+//! complete as the implementation in this crate.
+//!
 //! # System proxy configuration
 //!
 //! The system's proxy configuration is governed by four environment variables,
@@ -80,8 +114,16 @@
 //!
 //! [golang/go#16405]: https://github.com/golang/go/issues/16405
 //! [gitlab-blog]: https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/
+//! [Materialize]: https://materialize.com
+//! [repo]: https://github.com/MaterializeInc/materialize
 
+#[cfg(any(feature = "hyper", feature = "reqwest"))]
 mod proxy;
 
+#[cfg_attr(nightly_doc_features, doc(cfg(feature = "hyper")))]
+#[cfg(feature = "hyper")]
 pub mod hyper;
+
+#[cfg_attr(nightly_doc_features, doc(cfg(feature = "reqwest")))]
+#[cfg(feature = "reqwest")]
 pub mod reqwest;

src/http-util/src/proxy.rs → src/http-proxy/src/proxy.rs

@@ -1,11 +1,17 @@
 // Copyright Materialize, Inc. and contributors. All rights reserved.
 //
-// Use of this software is governed by the Business Source License
-// included in the LICENSE file.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License in the LICENSE file at the
+// root of this repository, or online at
 //
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0.
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 //! System proxy configuration guts.
 //!

src/http-util/src/reqwest.rs → src/http-proxy/src/reqwest.rs

@@ -1,13 +1,19 @@
 // Copyright Materialize, Inc. and contributors. All rights reserved.
 //
-// Use of this software is governed by the Business Source License
-// included in the LICENSE file.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License in the LICENSE file at the
+// root of this repository, or online at
 //
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0.
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
-//! Adapters for `reqwest`.
+//! Proxy adapters for [`reqwest`].
 
 use reqwest::ClientBuilder;