Add support for direct URL fetching. (#21)

The scie-pants project will leverage this in an install binding that
needs to fetch Pants version information from PyPI and GitHub in certain
circumstances.

Along the way, improve CLI help and switch to anyhow for better error
handling.

CHANGES.md

@@ -1,5 +1,11 @@
 # Release Notes
 
+## 0.3.0
+
+This release adds support for direct use by passing a single URL to
+fetch and improves command line help to fully explain both the existing
+scie binding file source mode and the new direct URL mode.
+
 ## 0.2.0
 
 This release brings fully static binaries for Linux with zero runtime

Cargo.toml

@@ -5,7 +5,7 @@ members = [
 
 [package]
 name = "ptex"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 authors = [
     "John Sirois <[email protected]>",
@@ -19,6 +19,7 @@ lto = "fat"
 codegen-units = 1
 
 [dependencies]
+anyhow = "1.0"
 indicatif = "0.17"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"

src/main.rs

@@ -2,9 +2,11 @@
 // Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 use std::collections::BTreeMap;
+use std::env;
 use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
 
+use anyhow::{Context, Result};
 use curl::easy::{Easy2, Handler, NetRc, WriteError};
 use indicatif::{ProgressBar, ProgressState, ProgressStyle};
 use serde::Deserialize;
@@ -15,9 +17,9 @@ struct Config {
 }
 
 impl Config {
-    fn parse<R: Read>(reader: R) -> Result<Self, String> {
-        let config: Self = serde_json::from_reader(reader)
-            .map_err(|e| format!("Failed to parse ptex config: {e}"))?;
+    fn parse<R: Read>(reader: R) -> Result<Self> {
+        let config: Self =
+            serde_json::from_reader(reader).context("Failed to parse ptex config")?;
         Ok(config)
     }
 }
@@ -70,67 +72,158 @@ impl<W: Write> Handler for FetchHandler<W> {
     }
 }
 
-fn fetch<R: Read, W: Write>(lift_manifest: R, file_path: &Path, output: W) -> Result<(), String> {
+fn fetch_manifest<R: Read, W: Write>(lift_manifest: R, file_path: &Path, output: W) -> Result<()> {
     let config = Config::parse(lift_manifest)?;
-    let url = config.ptex.get(file_path).ok_or_else(|| {
+    let url = config.ptex.get(file_path).with_context(|| {
         format!(
             "Did not find an URL mapping for file {path}.",
             path = file_path.display()
         )
     })?;
+    fetch(url, output)
+        .with_context(|| format!("Failed to source file {file}", file = file_path.display()))
+}
 
+fn fetch<W: Write>(url: &str, output: W) -> Result<()> {
     let mut easy = Easy2::new(FetchHandler::new(url, output));
     easy.follow_location(true)
-        .map_err(|e| format!("Failed to configure re-direct following: {e}"))?;
+        .context("Failed to configure re-direct following")?;
     easy.fail_on_error(true)
-        .map_err(|e| format!("Failed to configure fail on error behavior: {e}"))?;
+        .context("Failed to configure fail on error behavior")?;
     easy.netrc(NetRc::Optional)
-        .map_err(|e| format!("Failed to enable ~/.netrc parsing: {e}"))?;
+        .context("Failed to enable ~/.netrc parsing")?;
     easy.url(url)
-        .map_err(|e| format!("Failed to configure URL to fetch from as {url}: {e}"))?;
+        .context("Failed to configure URL to fetch from as {url}")?;
     easy.progress(true)
-        .map_err(|e| format!("Failed to enable progress meter: {e}"))?;
-    easy.perform().map_err(|e| {
-        format!(
-            "Failed to fetch {file} from {url}: {e}",
-            file = file_path.display()
-        )
-    })
+        .context("Failed to enable progress meter")?;
+    easy.perform()
+        .with_context(|| format!("Failed to fetch {url}"))
 }
 
-fn usage() -> Result<(), String> {
-    let current_exe = std::env::current_exe()
-        .map_err(|e| format!("Failed to determine current executable: {e}"))?;
+fn usage() -> Result<()> {
+    let current_exe = env::current_exe().context("Failed to determine current executable")?;
     eprintln!(
-        "Usage: {current_exe} [lift manifest path] [file path]",
+        r#"Usage:
+    {current_exe} [lift manifest path] [file name]
+    {current_exe} [URL]
+
+    The `ptex` binary is a statically compiled URL fetcher based on
+    libcurl. It supports the HTTP protocol up through HTTP/2, the FTP
+    protocol and TLS via OpenSSL. It follows redirects, uses credentials
+    from ~/.netrc if available and can perform NTLM authentication. It
+    exits with a non-zero status if there was a network or protocol
+    error.
+
+{current_exe} [lift manifest path] [file name]
+
+    For use in a scie file source binding. The first argument is the
+    path to the scie lift manifest and the second argument is the file
+    name to source. You configure this use in a scie by fully specifying
+    file metadata, including size, hash and type and setting the source
+    to the name of a binding command that uses `ptex` as its executable.
+
+    The relevant parts of the lift manifest look like so:
+
+    {{
+      "scie": {{
+        "lift": {{
+          "files": [
+            {{
+              "name": "ptex-linux-x86_64"
+              "executable": true
+            }},
+            {{
+              "name": "some-file-to-be-fetched.tar.gz",
+              "size": 123,
+              "hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+              "type": "tar.gz",
+              "source": "ptex-fetch"
+            }},
+          ]
+          "boot": {{
+            "bindings": {{
+              "ptex-fetch": {{
+                "exe": "{{ptex-linux-x86_64}}",
+                "args": [
+                  "{{scie.lift}}"
+                ]
+              }}
+            }}
+          }}
+        }}
+      }},
+      "ptex": {{
+        "some-file-to-be-fetched.tar.gz":
+          "https://example.org/downloads/some-file-to-be-fetched.tar.gz"
+      }}
+    }}
+
+    The file name is passed in as a second argument to the source
+    binding by the `scie-jump` and `ptex` uses that file name to look up
+    the URL to fetch the file from in the top-level "ptex" URL database
+    object.
+
+    See more documentation on scie packaging configuration here:
+     https://github.com/a-scie/jump/blob/main/docs/packaging.md
+
+{current_exe} [URL]
+
+    For use as a fully self-contained curl-like binary. The given URL is
+    fetched and the response is streamed to stdout.
+"#,
         current_exe = current_exe.display()
     );
     Ok(())
 }
 
-fn main() -> Result<(), String> {
-    if std::env::args().len() != 3 {
-        usage()?;
-        std::process::exit(1);
+trait OrExit<T> {
+    fn or_exit(self) -> T;
+}
+
+impl<T> OrExit<T> for Result<T> {
+    fn or_exit(self) -> T {
+        match self {
+            Ok(item) => item,
+            Err(err) => {
+                eprintln!("{:#}", err);
+                std::process::exit(1)
+            }
+        }
     }
-    let lift_manifest_path = PathBuf::from(
-        std::env::args()
+}
+
+fn main() {
+    if env::args().len() == 3 {
+        let lift_manifest_path = PathBuf::from(
+            env::args()
+                .nth(1)
+                .expect("We checked there were 3 args just above"),
+        );
+        let file_path = PathBuf::from(
+            env::args()
+                .nth(2)
+                .expect("We checked there were 3 args just above"),
+        );
+
+        let lift_manifest = std::fs::File::open(&lift_manifest_path)
+            .with_context(|| {
+                format!(
+                    "ailed to open lift manifest at {lift_manifest}",
+                    lift_manifest = lift_manifest_path.display()
+                )
+            })
+            .or_exit();
+        fetch_manifest(&lift_manifest, &file_path, std::io::stdout()).or_exit()
+    } else if env::args().len() == 2 {
+        let url = env::args()
             .nth(1)
-            .expect("We checked there were 3 args just above"),
-    );
-    let file_path = PathBuf::from(
-        std::env::args()
-            .nth(2)
-            .expect("We checked there were 3 args just above"),
-    );
+            .expect("We checked there were 2 args just above");
 
-    let lift_manifest = std::fs::File::open(&lift_manifest_path).map_err(|e| {
-        format!(
-            "ailed to open lift manifest at {lift_manifest}: {e}",
-            lift_manifest = lift_manifest_path.display()
-        )
-    })?;
-    fetch(&lift_manifest, &file_path, std::io::stdout())
+        fetch(url.as_str(), std::io::stdout()).or_exit()
+    } else {
+        usage().or_exit();
+        std::process::exit(1);
+    }
 }
 
 #[cfg(test)]
@@ -140,22 +233,37 @@ mod tests {
 
     use sha2::{Digest, Sha256};
 
-    #[test]
-    fn fetch() {
-        let manifest = r#"
-{
-    "ptex": {
-        "scie-jump":
-            "https://github.com/a-scie/jump/releases/download/v0.2.1/scie-jump-linux-aarch64"
-    }
-}
-"#;
-        let mut buffer: Vec<u8> = Vec::new();
-        super::fetch(Cursor::new(manifest), Path::new("scie-jump"), &mut buffer).unwrap();
+    const URL: &str =
+        "https://github.com/a-scie/jump/releases/download/v0.2.1/scie-jump-linux-aarch64";
+
+    fn assert_fetched_buffer(buffer: &[u8]) {
         assert_eq!(1205568, buffer.len());
         assert_eq!(
             "937683255e98caf10745a674d7063bd38e9cbeb523b9f8ef4dbe8807abc35382".to_string(),
             format!("{digest:x}", digest = Sha256::digest(buffer))
         );
     }
+
+    #[test]
+    fn fetch_manifest() {
+        let manifest = format!(
+            r#"
+{{
+    "ptex": {{
+        "scie-jump": "{URL}"
+    }}
+}}
+"#
+        );
+        let mut buffer: Vec<u8> = Vec::new();
+        super::fetch_manifest(Cursor::new(manifest), Path::new("scie-jump"), &mut buffer).unwrap();
+        assert_fetched_buffer(buffer.as_slice());
+    }
+
+    #[test]
+    fn fetch() {
+        let mut buffer: Vec<u8> = Vec::new();
+        super::fetch(URL, &mut buffer).unwrap();
+        assert_fetched_buffer(buffer.as_slice());
+    }
 }