Fetch: support custom http headers

docs/Fetch.md

@@ -86,6 +86,28 @@ HTTP 404 - Not Found
 
 ```
 
+### Fetch with debug trace enabled to see problem with extra whitespace in URL column
+
+Note: if URL is quoted, then there cannot be extra whitespace before quotes
+
+```
+$ cat test4.csv 
+City,URL
+Beverley Hills, http://geodb-free-service.wirefreethought.com/v1/geo/locations/+34.0901-118.4065/nearbyCities
+San Francisco, "http://geodb-free-service.wirefreethought.com/v1/geo/locations/+37.7864-122.3892/nearbyCities"
+Anaheim," http://geodb-free-service.wirefreethought.com/v1/geo/locations/+33.8085-117.9228/nearbyCities"
+
+$ QSV_LOG_LEVEL=debug qsv fetch URL test4.csv  --store-error --jql '"data".[0]."name","data".[1]."name","data".[2]."name"' 
+[00:00:01] [==================== 100% of 3 records. Cache hit ratio: 0.00% - 3 entries] (7/sec)
+"Universal City, Hollywood, Sherman Oaks"
+builder error: relative URL without a base
+"Anaheim, Garden Grove, Fullerton"
+
+$ grep ERROR qsv_rCURRENT.log | tail -1
+[2022-01-06 20:55:49.814944 +08:00] ERROR [qsv::cmd::fetch] src/cmd/fetch.rs:238: Cannot fetch url: "\"http://geodb-free-service.wirefreethought.com/v1/geo/locations/+37.7864-122.3892/nearbyCities\"", error: reqwest::Error { kind: Builder, source: RelativeUrlWithoutBase }
+
+```
+
 ### Fetch with explicit rate limit, and pipe output to a new csv
 
 ```
@@ -99,3 +121,31 @@ US,94105,http://api.zippopotam.us/us/94105,"-122.3892, 37.7864"
 US,92802,http://api.zippopotam.us/us/92802,"-117.9228, 33.8085"
 ```
 
+### Fetch using custom headers for api key
+
+```
+$ cat test5.csv
+URL
+http://httpbin.org/get
+
+$ qsv fetch URL test5.csv --jql '"headers"."X-Api-Key","headers"."X-Api-Secret"' --store-error --http-header "X-Api-Key:mykey" --http-header "X-Api-Secret  : nottelling"
+[00:00:00] [==================== 100% of 1 records. Cache hit ratio: 0.00% - 1 entries] (1,151/sec)
+"mykey, nottelling"
+
+$ qsv fetch URL test5.csv --store-error --http-header "X-Api-Key:mykey" --http-header "X-Api-Secret  : nottelling"
+[00:00:00] [==================== 100% of 1 records. Cache hit ratio: 0.00% - 1 entries] (1,105/sec)
+"{
+  ""args"": {}, 
+  ""headers"": {
+    ""Accept"": ""*/*"", 
+    ""Host"": ""httpbin.org"", 
+    ""User-Agent"": ""qsv/0.28.0 (https://github.com/jqnatividad/qsv)"", 
+    ""X-Amzn-Trace-Id"": ""Root=1-61d8d957-054da2374e304c7c7395cacc"", 
+    ""X-Api-Key"": ""mykey"", 
+    ""X-Api-Secret"": ""nottelling""
+  }, 
+  ""origin"": ""1.163.34.120"", 
+  ""url"": ""http://httpbin.org/get""
+}
+"
+```

src/cmd/fetch.rs

@@ -20,13 +20,13 @@ URL column must contain full and valid URL path, which can be constructed via th
 To set proxy, please set env var HTTP_PROXY and HTTPS_PROXY (eg export HTTPS_PROXY=socks5://127.0.0.1:1086)
 
 Usage:
-    qsv fetch [options] [<column>] [<input>]
+    qsv fetch [options] [--http-header <k:v>...] [<column>] [<input>]
 
 fetch options:
     -c, --new-column <name>    Put the fetched values in a new column instead.
     --jql <selector>           Apply jql selector to API returned JSON value.
     --rate-limit <qps>         Rate Limit in Queries Per Second. [default: 5]
-    --header <file>            File containing additional HTTP Request Headers. Useful for setting Authorization or overriding User Agent.
+    --http-header <k:v>        Pass custom header(s) to the server.
     --store-error              On error, store error code/message instead of blank value.
     --cookies                  Allow cookies.
 
@@ -47,7 +47,7 @@ struct Args {
     flag_new_column: Option<String>,
     flag_jql: Option<String>,
     flag_rate_limit: Option<u32>,
-    flag_header: Option<String>,
+    flag_http_header: Vec<String>,
     flag_store_error: bool,
     flag_cookies: bool,
     flag_output: Option<String>,
@@ -74,7 +74,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
             new column: {:?}, 
             jql: {:?},
             rate limit: {:?},
-            http header file: {:?},
+            http headers: {:?},
             store error: {:?},
             cookies: {:?},
             output: {:?}, 
@@ -86,7 +86,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
         &args.flag_new_column,
         &args.flag_jql,
         &args.flag_rate_limit,
-        &args.flag_header,
+        &args.flag_http_header,
         &args.flag_store_error,
         &args.flag_cookies,
         &args.flag_output,
@@ -95,10 +95,6 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
         &args.flag_quiet
     );
 
-    if let Some(_header) = &args.flag_header {
-        panic!("Param not yet supported: header")
-    }
-
     let rconfig = Config::new(&args.arg_input)
         .delimiter(args.flag_delimiter)
         .no_headers(args.flag_no_headers)
@@ -128,9 +124,31 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
         rate_limit = NonZeroU32::new(qps).unwrap();
     }
 
+    use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
+
+    let http_headers: HeaderMap = {
+        let mut map = HeaderMap::new();
+        for header in args.flag_http_header {
+            let vals: Vec<&str> = header.split(':').collect();
+
+            // allocate new String for header key to put into map
+            let k: String = String::from(vals[0].trim());
+            let header_name: HeaderName = HeaderName::from_lowercase(k.to_lowercase().as_bytes()).unwrap();
+
+            // allocate new String for header value to put into map
+            let v: String = String::from(vals[1].trim());
+            let header_val: HeaderValue = HeaderValue::from_str(v.as_str()).unwrap();
+
+            map.append(header_name,header_val);
+        }
+
+        map
+    };
+
     use reqwest::blocking::Client;
     let client = Client::builder()
         .user_agent(DEFAULT_USER_AGENT)
+        .default_headers(http_headers)
         .cookie_store(args.flag_cookies)
         .build()
         .unwrap();
@@ -168,7 +186,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
         }
 
         let selected_col_value = record[column_index].to_owned();
-        let url = String::from_utf8_lossy(&selected_col_value).to_string();
+        let url = String::from_utf8_lossy(&selected_col_value).trim().to_string();
         debug!("Fetching URL: {:?}", &url);
 
         let final_value = get_cached_response(&url, &client, &limiter, &args.flag_jql, args.flag_store_error);

tests/test_fetch.rs

@@ -8,7 +8,7 @@ fn fetch_simple() {
         vec![
             svec!["URL"],
             svec!["https://api.zippopotam.us/us/99999"],
-            svec!["http://api.zippopotam.us/us/90210"],
+            svec!["  http://api.zippopotam.us/us/90210"],
             svec!["https://api.zippopotam.us/us/94105"],
             svec!["http://api.zippopotam.us/us/92802"],
             svec!["https://query.wikidata.org/sparql?query=SELECT%20?dob%20WHERE%20{wd:Q42%20wdt:P569%20?dob.}&format=json"],
@@ -113,6 +113,33 @@ fn fetch_jql_multiple() {
     assert_eq!(got, expected);
 }
 
+#[test]
+fn fetch_custom_header() {
+    let wrk = Workdir::new("fetch");
+    wrk.create(
+        "data.csv",
+        vec![
+            svec!["URL"],
+            svec!["http://httpbin.org/get"],
+        ],
+    );
+    let mut cmd = wrk.command("fetch");
+    cmd.arg("URL")
+        .arg("--http-header")
+        .arg(" X-Api-Key :  DEMO_KEY")
+        .arg("--http-header")
+        .arg("X-Api-Secret :ABC123XYZ")
+        .arg("--jql")
+        .arg(r#""headers"."X-Api-Key","headers"."X-Api-Secret""#)
+        .arg("data.csv");
+
+    let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
+    let expected = vec![
+        svec!["DEMO_KEY, ABC123XYZ"],
+    ];
+    assert_eq!(got, expected);
+}
+
 use std::{sync::mpsc, thread};
 
 use actix_web::{