From a8ef611f2c1a11f08977c733739e447f1cf4c497 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Tue, 3 Oct 2023 14:12:32 +0200 Subject: [PATCH 01/63] HTTPie v4 --- .github/workflows/tests.yml | 7 +- CHANGELOG.md | 22 +- docs/README.md | 159 ++++++++-- docs/contributors/fetch.py | 8 +- httpie/__init__.py | 4 +- httpie/adapters.py | 2 +- httpie/cli/argparser.py | 21 +- httpie/cli/definition.py | 69 +++++ httpie/cli/dicts.py | 171 +++++++++-- httpie/client.py | 133 +++++--- httpie/context.py | 3 +- httpie/core.py | 55 +++- httpie/downloads.py | 11 +- httpie/internal/encoder.py | 472 +++++++++++++++++++++++++++++ httpie/internal/update_warnings.py | 4 +- httpie/models.py | 133 ++++++-- httpie/output/lexers/metadata.py | 4 +- httpie/output/streams.py | 11 +- httpie/output/writer.py | 4 +- httpie/plugins/base.py | 6 +- httpie/plugins/builtin.py | 18 +- httpie/sessions.py | 4 +- httpie/ssl_.py | 93 +++++- httpie/uploads.py | 14 +- httpie/utils.py | 6 +- pytest.ini | 7 - setup.cfg | 11 +- setup.py | 12 +- tests/conftest.py | 28 +- tests/test_auth.py | 6 +- tests/test_binary.py | 4 +- tests/test_cli.py | 5 +- tests/test_cookie.py | 15 +- tests/test_downloads.py | 6 +- tests/test_encoding.py | 2 +- tests/test_errors.py | 2 +- tests/test_exit_status.py | 8 +- tests/test_httpie.py | 9 + tests/test_json.py | 3 +- tests/test_output.py | 30 +- tests/test_redirects.py | 2 +- tests/test_regressions.py | 1 - tests/test_ssl.py | 37 +-- tests/test_stream.py | 2 +- tests/test_tokens.py | 6 +- tests/test_transport_plugin.py | 6 +- tests/test_update_warnings.py | 2 +- tests/test_uploads.py | 9 +- tests/utils/__init__.py | 23 +- tests/utils/http_server.py | 3 +- tests/utils/matching/parsing.py | 5 +- tests/utils/matching/tokens.py | 2 + 52 files changed, 1364 insertions(+), 316 deletions(-) create mode 100644 httpie/internal/encoder.py delete mode 100644 pytest.ini diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e35f4eead..14bfa86a89 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,8 +25,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.7, 3.8, 3.9, "3.10"] - pyopenssl: [0, 1] + python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -39,12 +38,8 @@ jobs: python -m pip install --upgrade pip wheel python -m pip install --upgrade '.[dev]' python -m pytest --verbose ./httpie ./tests - env: - HTTPIE_TEST_WITH_PYOPENSSL: ${{ matrix.pyopenssl }} - name: Linux & Mac setup if: matrix.os != 'windows-latest' run: | make install make test - env: - HTTPIE_TEST_WITH_PYOPENSSL: ${{ matrix.pyopenssl }} diff --git a/CHANGELOG.md b/CHANGELOG.md index fd80c096f2..f18adec8ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,25 @@ This document records all notable changes to [HTTPie](https://httpie.io). This project adheres to [Semantic Versioning](https://semver.org/). -## [3.3.0-dev](https://github.com/httpie/cli/compare/3.2.2...master) (unreleased) - -- Make it possible to [unset](https://httpie.io/docs/cli/default-request-headers) the `User-Agent`, `Accept-Encoding`, and `Host` request headers. ([#1502](https://github.com/httpie/cli/issues/1502)) +## [4.0.0.b1](https://github.com/httpie/cli/compare/3.2.2...master) (unreleased) + +- Make it possible to [unset](https://httpie.io/docs/cli/default-request-headers) the `User-Agent`, and `Accept-Encoding` headers. ([#1502](https://github.com/httpie/cli/issues/1502)) +- Dependency on requests was changed in favor of compatible niquests. ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for HTTP/2, and HTTP/3 protocols. ([#523](https://github.com/httpie/cli/issues/523)) ([#692](https://github.com/httpie/cli/issues/692)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added request metadata for the TLS certificate, negotiated version with cipher, the revocation status and the remote peer IP address. ([#1495](https://github.com/httpie/cli/issues/1495)) ([#1023](https://github.com/httpie/cli/issues/1023)) ([#826](https://github.com/httpie/cli/issues/826)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added support to load the operating system trust store for the peer certificate validation. ([#480](https://github.com/httpie/cli/issues/480)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added detailed timings in response metadata with DNS resolution, established, TLS handshake, and request sending delays. ([#1023](https://github.com/httpie/cli/issues/1023)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for using alternative DNS resolver using `--resolver`. DNS over HTTPS, DNS over TLS, DNS over QUIC, and DNS over UDP are accepted. ([#99](https://github.com/httpie/cli/issues/99)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for binding to a specific network adapter with `--interface`. ([#1422](https://github.com/httpie/cli/issues/1422)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for specifying the local port with `--local-port`. ([#1456](https://github.com/httpie/cli/issues/1456)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for forcing either IPv4 or IPv6 to reach the remote HTTP server with `-6` or `-4`. ([#94](https://github.com/httpie/cli/issues/94)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Removed support for pyopenssl. ([#1531](https://github.com/httpie/cli/pull/1531)) +- Dropped dependency on `requests_toolbelt` in favor of directly including `MultipartEncoder` into HTTPie due to its direct dependency to requests. ([#1531](https://github.com/httpie/cli/pull/1531)) +- Dropped dependency on `multidict` in favor of implementing an internal one due to often missing pre-built wheels. ([#1522](https://github.com/httpie/cli/issues/1522)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Fixed the case when multiple headers where concatenated in the response output. ([#1413](https://github.com/httpie/cli/issues/1413)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Fixed an edge case where HTTPie could be lead to believe data was passed in stdin, thus sending a POST by default. ([#1551](https://github.com/httpie/cli/issues/1551)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Slightly improved performance while downloading by setting chunk size to `-1` to retrieve packets as they arrive. ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for using the system trust store to retrieve root CAs for verifying TLS certificates. ([#1531](https://github.com/httpie/cli/pull/1531)) ## [3.2.2](https://github.com/httpie/cli/compare/3.2.1...3.2.2) (2022-05-19) diff --git a/docs/README.md b/docs/README.md index 3b12e98f4c..0ffa110134 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1562,9 +1562,9 @@ be printed via several options: |---------------------------:|----------------------------------------------------------------------------------------------------| | `--headers, -h` | Only the response headers are printed | | `--body, -b` | Only the response body is printed | -| `--meta, -m` | Only the [response metadata](#response-meta) is printed | +| `--meta, -m` | Only the [request, response metadata](#response-meta) are printed | | `--verbose, -v` | Print the whole HTTP exchange (request and response). This option also enables `--all` (see below) | -| `--verbose --verbose, -vv` | Just like `-v`, but also include the response metadata. | +| `--verbose --verbose, -vv` | Just like `-v`, but also include the request, and response metadata. | | `--print, -p` | Selects parts of the HTTP exchange | | `--quiet, -q` | Don’t print anything to `stdout` and `stderr` | @@ -1573,13 +1573,13 @@ be printed via several options: All the other [output options](#output-options) are under the hood just shortcuts for the more powerful `--print, -p`. It accepts a string of characters each of which represents a specific part of the HTTP exchange: -| Character | Stands for | -|----------:|---------------------------------| -| `H` | request headers | -| `B` | request body | -| `h` | response headers | -| `b` | response body | -| `m` | [response meta](#response-meta) | +| Character | Stands for | +|----------:|------------------------------------------| +| `H` | request headers | +| `B` | request body | +| `h` | response headers | +| `b` | response body | +| `m` | [request, response meta](#response-meta) | Print request and response headers: @@ -1592,27 +1592,49 @@ $ http --print=Hh PUT pie.dev/put hello=world The response metadata section currently includes the total time elapsed. It’s the number of seconds between opening the network connection and downloading the last byte of response the body. -To _only_ show the response metadata, use `--meta, -m` (analogically to `--headers, -h` and `--body, -b`): +To _only_ show the request, and response metadata, use `--meta, -m` (analogically to `--headers, -h` and `--body, -b`): ```bash $ http --meta pie.dev/delay/1 ``` ```console -Elapsed time: 1.099171542s +Connected to: 2a06:98c1:3120::2 port 443 +Connection secured using: TLSv1.3 with AES-256-GCM-SHA384 +Server certificate: commonName="pie.dev"; DNS="*.pie.dev"; DNS="pie.dev" +Certificate validity: "Nov 11 01:14:24 2023 UTC" to "Feb 09 01:14:23 2024 UTC" +Issuer: countryName="US"; organizationName="Let's Encrypt"; commonName="E1" +Revocation status: Good + +Elapsed DNS: 0.11338s +Elapsed established connection: 3.8e-05s +Elapsed TLS handshake: 0.057503s +Elapsed emitting request: 0.000275s +Elapsed time: 0.292854214s ``` The [extra verbose `-vv` output](#extra-verbose-output) includes the meta section by default. You can also show it in combination with other parts of the exchange via [`--print=m`](#what-parts-of-the-http-exchange-should-be-printed). For example, here we print it together with the response headers: ```bash -$ http --print=hm pie.dev/get +$ https --print=hm pie.dev/get ``` ```http -HTTP/1.1 200 OK +Connected to: 2a06:98c1:3120::2 port 443 +Connection secured using: TLSv1.3 with AES-256-GCM-SHA384 +Server certificate: commonName="pie.dev"; DNS="*.pie.dev"; DNS="pie.dev" +Certificate validity: "Nov 11 01:14:24 2023 UTC" to "Feb 09 01:14:23 2024 UTC" +Issuer: countryName="US"; organizationName="Let's Encrypt"; commonName="E1" +Revocation status: Good + +HTTP/2 200 OK Content-Type: application/json -Elapsed time: 0.077538375s +Elapsed DNS: 0.11338s +Elapsed established connection: 3.8e-05s +Elapsed TLS handshake: 0.057503s +Elapsed emitting request: 0.000275s +Elapsed time: 0.292854214s ``` @@ -1626,19 +1648,19 @@ If you [use `--style` with one of the Pie themes](#colors-and-formatting), you `--verbose` can often be useful for debugging the request and generating documentation examples: ```bash -$ http --verbose PUT pie.dev/put hello=world -PUT /put HTTP/1.1 +$ https --verbose PUT pie.dev/put hello=world +PUT /put HTTP/2 Accept: application/json, */*;q=0.5 Accept-Encoding: gzip, deflate Content-Type: application/json Host: pie.dev -User-Agent: HTTPie/0.2.7dev +User-Agent: HTTPie/4.0.0 { "hello": "world" } -HTTP/1.1 200 OK +HTTP/2 200 OK Connection: keep-alive Content-Length: 477 Content-Type: application/json @@ -1652,10 +1674,10 @@ Server: gunicorn/0.13.4 #### Extra verbose output -If you run HTTPie with `-vv` or `--verbose --verbose`, then it would also display the [response metadata](#response-meta). +If you run HTTPie with `-vv` or `--verbose --verbose`, then it would also display the [response and request metadata](#response-meta). ```bash -# Just like the above, but with additional columns like the total elapsed time +# Just like the above, but with additional columns like the total elapsed time, remote peer connection informations $ http -vv pie.dev/get ``` @@ -1833,6 +1855,101 @@ $ http --chunked pie.dev/post @files/data.xml $ cat files/data.xml | http --chunked pie.dev/post ``` +## Disable HTTP/2, or HTTP/3 + +You can at your own discretion toggle on and off HTTP/2, or/and HTTP/3. + +```bash +$ https --disable-http2 PUT pie.dev/put hello=world +``` + +```bash +$ https --disable-http3 PUT pie.dev/put hello=world +``` + +## Force HTTP/3 + +By opposition to the previous section, you can force the HTTP/3 negotiation. + +```bash +$ https --http3 pie.dev/get +``` + +By default, HTTPie cannot negotiate HTTP/3 without a first HTTP/1.1, or HTTP/2 successful response unless the +remote host specified a DNS HTTPS record that indicate its support. + +The remote server yield its support for HTTP/3 in the Alt-Svc header, if present HTTPie will issue +the successive requests via HTTP/3. You may use that argument in case the remote peer does not support +either HTTP/1.1 or HTTP/2. + +## Custom DNS resolver + +### Using DNS url + +You can specify one or many custom DNS resolvers using the `--resolver` flag. They will be tested in +presented order to resolver given hostname. + +```bash +$ https --resolver "doh+cloudflare://" pie.dev/get +``` + +To know more about DNS url and supported protocols, visit [Niquests documentation](https://niquests.readthedocs.io/en/stable/user/quickstart.html#dns-resolution). + +### Forcing hostname to resolve with a manual entry + +It is possible to fake DNS resolution using a virtual resolver. We'll make use of the `--resolver` flag +using the `in-memory` provider. + +```bash +$ https --resolver "in-memory://default/?hosts=pie.dev:10.10.4.1" pie.dev/get +``` + +In that example, `pie.dev` will resolve to `10.10.4.1`. The TLS HELLO / SNI will be set with host = `pie.dev`. + +HTTPie allows to pass directly the hostname and associated IPs directly as a shortcut to previous the example like so: + +```bash +$ https --resolver "pie.dev:10.10.4.1" pie.dev/get +``` + +You can specify multiple entries, concatenated with a comma: + +```bash +$ https --resolver "pie.dev:10.10.4.1,re.pie.dev:10.10.8.1" pie.dev/get +``` + +## Attach to a specific network adapter + +In order to bind emitted request from a specific network adapter you can use the `--interface` flag. + +```bash +$ https --interface 172.17.0.1 pie.dev/get +``` + +## Local port + +You can choose to select the outgoing port manually by passing the `--local-port` flag. + +```bash +$ https --local-port 5411 pie.dev/get +``` + +or using a range. + +```bash +$ https --local-port 5000-10000 pie.dev/get +``` + +Beware that some ports requires elevated privileges. + +## Enforcing IPv4 or IPv6 + +Since HTTPie 4, you may pass the flags `--ipv4, -4` or `--ipv6, -6` to enforce connecting to an IPv4 or IPv6 address. + +```bash +$ https -4 pie.dev/get +``` + ## Compressed request body You can use the `--compress, -x` flag to instruct HTTPie to use `Content-Encoding: deflate` and compress the request data: @@ -2556,7 +2673,7 @@ HTTPie has the following community channels: Under the hood, HTTPie uses these two amazing libraries: -- [Requests](https://requests.readthedocs.io/en/latest/) — Python HTTP library for humans +- [Niquests](https://niquests.readthedocs.io/en/latest/) — Python HTTP library for humans - [Pygments](https://pygments.org/) — Python syntax highlighter #### HTTPie friends diff --git a/docs/contributors/fetch.py b/docs/contributors/fetch.py index ba94c28183..1ea1e8d05a 100644 --- a/docs/contributors/fetch.py +++ b/docs/contributors/fetch.py @@ -1,7 +1,7 @@ """ Generate the contributors database. -FIXME: replace `requests` calls with the HTTPie API, when available. +FIXME: replace `niquests` calls with the HTTPie API, when available. """ import json import os @@ -14,7 +14,7 @@ from time import sleep from typing import Any, Dict, Optional, Set -import requests +import niquests FullNames = Set[str] GitHubLogins = Set[str] @@ -197,10 +197,10 @@ def fetch(url: str, params: Optional[Dict[str, str]] = None) -> UserInfo: } for retry in range(1, 6): debug(f'[{retry}/5]', f'{url = }', f'{params = }') - with requests.get(url, params=params, headers=headers) as req: + with niquests.get(url, params=params, headers=headers) as req: try: req.raise_for_status() - except requests.exceptions.HTTPError as exc: + except niquests.exceptions.HTTPError as exc: if exc.response.status_code == 403: # 403 Client Error: rate limit exceeded for url: ... now = int(datetime.utcnow().timestamp()) diff --git a/httpie/__init__.py b/httpie/__init__.py index ffe0d35419..b1c1a48bcc 100644 --- a/httpie/__init__.py +++ b/httpie/__init__.py @@ -3,7 +3,7 @@ """ -__version__ = '3.2.2' -__date__ = '2022-05-06' +__version__ = '4.0.0.b1' +__date__ = '2024-01-01' __author__ = 'Jakub Roztocil' __licence__ = 'BSD' diff --git a/httpie/adapters.py b/httpie/adapters.py index 8e2dd7397f..fa6cfcec89 100644 --- a/httpie/adapters.py +++ b/httpie/adapters.py @@ -1,5 +1,5 @@ from httpie.cli.dicts import HTTPHeadersDict -from requests.adapters import HTTPAdapter +from niquests.adapters import HTTPAdapter class HTTPieHTTPAdapter(HTTPAdapter): diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index 9bf09b3b73..a0099601c6 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -3,11 +3,12 @@ import os import re import sys +import threading from argparse import RawDescriptionHelpFormatter from textwrap import dedent from urllib.parse import urlsplit -from requests.utils import get_netrc_auth +from niquests.utils import get_netrc_auth from .argtypes import ( AuthCredentials, SSLCredentials, KeyValueArgType, @@ -27,6 +28,7 @@ from ..context import Environment from ..plugins.registry import plugin_manager from ..utils import ExplicitNullAuth, get_content_type +from ..uploads import observe_stdin_for_data_thread class HTTPieHelpFormatter(RawDescriptionHelpFormatter): @@ -164,7 +166,6 @@ def parse_args( and not self.args.ignore_stdin and not self.env.stdin_isatty ) - self.has_input_data = self.has_stdin_data or self.args.raw is not None # Arguments processing and environment setup. self._apply_no_options(no_options) self._process_request_type() @@ -173,6 +174,22 @@ def parse_args( self._process_output_options() self._process_pretty_options() self._process_format_options() + + # bellow is a fix for detecting "false-or empty" stdin + if self.has_stdin_data: + read_event = threading.Event() + observe_stdin_for_data_thread(env, self.env.stdin, read_event) + if ( + hasattr(self.env.stdin, 'buffer') + and hasattr(self.env.stdin.buffer, "peek") + and not self.env.stdin.buffer.peek(1) + ): + self.has_stdin_data = False + + read_event.set() + + self.has_input_data = self.has_stdin_data or self.args.raw is not None + self._guess_method() self._parse_items() self._process_url() diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 843b29c9cf..52addbd3c5 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -726,6 +726,20 @@ def format_auth_help(auth_plugins_mapping, *, isolation_mode: bool = False): """, ) +network.add_argument( + '--ipv6', + '-6', + default=False, + action='store_true', + short_help='Force using a IPv6 address to reach the remote peer.' +) +network.add_argument( + '--ipv4', + '-4', + default=False, + action='store_true', + short_help='Force using a IPv4 address to reach the remote peer.' +) network.add_argument( '--follow', '-F', @@ -802,6 +816,61 @@ def format_auth_help(auth_plugins_mapping, *, isolation_mode: bool = False): 'The Transfer-Encoding header is set to chunked.' ) ) +network.add_argument( + "--disable-http2", + default=False, + action="store_true", + short_help="Disable the HTTP/2 protocol." +) +network.add_argument( + "--disable-http3", + default=False, + action="store_true", + short_help="Disable the HTTP/3 over QUIC protocol." +) +network.add_argument( + "--http3", + default=False, + dest="force_http3", + action="store_true", + short_help="Use the HTTP/3 protocol for the request.", + help=""" + By default, HTTPie cannot negotiate HTTP/3 without a first HTTP/1.1, or HTTP/2 successful response unless the + remote host specified a DNS HTTPS record that indicate its support. + + The remote server yield its support for HTTP/3 in the Alt-Svc header, if present HTTPie will issue + the successive requests via HTTP/3. You may use that argument in case the remote peer does not support + either HTTP/1.1 or HTTP/2. + + """ +) +network.add_argument( + "--resolver", + default=[], + action='append', + short_help="Specify a DNS resolver url to resolve hostname.", + help=""" + By default, HTTPie use the system DNS through Python standard library. + You can specify an alternative DNS server to be used. (e.g. doh://cloudflare-dns.com or doh://google.dns). + You can specify multiple resolvers with different protocols. The environment + variable $NIQUESTS_DNS_URL is supported as well. + + """ +) +network.add_argument( + "--interface", + default=None, + short_help="Bind to a specific network interface.", +) +network.add_argument( + "--local-port", + default=None, + short_help="Set the local port to be used for the outgoing request.", + help=""" + It can be either a port range (e.g. "11221-14555") or a single port. + Some port may require root privileges (e.g. < 1024). + """ +) ####################################################################### # SSL diff --git a/httpie/cli/dicts.py b/httpie/cli/dicts.py index 6b6d4736d2..53faa234a0 100644 --- a/httpie/cli/dicts.py +++ b/httpie/cli/dicts.py @@ -1,49 +1,168 @@ +from __future__ import annotations + +import typing from collections import OrderedDict +from typing import Union, TypeVar -from multidict import MultiDict, CIMultiDict +T = TypeVar("T") -class BaseMultiDict(MultiDict): +class BaseMultiDictKeyView: """ - Base class for all MultiDicts. + Basic key view for BaseMultiDict. """ + def __init__(self, o: BaseMultiDict) -> None: + self._container = o + + def __iter__(self): + for key in self._container: + yield key + + def __contains__(self, item: str) -> bool: + return item in self._container -class HTTPHeadersDict(CIMultiDict, BaseMultiDict): + +class BaseMultiDict(typing.MutableMapping[str, Union[str, bytes]]): """ - Headers are case-insensitive and multiple values are supported - through the `add()` API. + This follow the multidict (case-insensitive) implementation but does not implement it fully. + We scoped this class according to our needs. In the future we should be able to refactor + HTTPie in order to use either kiss_headers.Headers or urllib3.HTTPHeaderDict. + The main constraints are: We use bytes sometime in values, and relly on multidict specific behaviors. """ - def add(self, key, value): - """ - Add or update a new header. + def __init__(self, d: BaseMultiDict | typing.MutableMapping[str, str | bytes] | None = None, **kwargs: str | bytes) -> None: + super().__init__() + self._container: typing.MutableMapping[str, list[tuple[str, str | bytes]] | str] = {} - If the given `value` is `None`, then all the previous - values will be overwritten and the value will be set - to `None`. - """ - if value is None: - self[key] = value + if d is not None: + self.update(d) + + for key, value in kwargs.items(): + self.add(key, value) + + def items(self) -> typing.Iterator[str, str | bytes | None]: + for key_i in self._container: + + if isinstance(self._container[key_i], str): + yield key_i, None + continue + + for original_key, value in self._container[key_i]: + yield original_key, value + + def keys(self) -> BaseMultiDictKeyView: + return BaseMultiDictKeyView(self) + + def copy(self: T) -> T: + return BaseMultiDict(self) + + def __delitem__(self, __key: str) -> None: + del self._container[__key.lower()] + + def __len__(self) -> int: + return len(self._container) + + def __iter__(self) -> typing.Iterator[str]: + for key_i in self._container: + if isinstance(self._container[key_i], list): + yield self._container[key_i][0][0] + else: + yield self._container[key_i] + + def __contains__(self, item: str) -> bool: + return item.lower() in self._container + + def update(self, __m, **kwargs) -> None: + if hasattr(__m, "items"): + for k in __m: + self[k] = None + for k, v in __m.items(): + self.add(k, v) + else: + for k, v in __m: + self.add(k, v) + + def getlist(self, key: str) -> list[str | bytes]: + key_lower = key.lower() + values = self._container[key_lower] + + if isinstance(values, str): + return [] + + return [_[-1] for _ in self._container[key_lower]] + + def __setitem__(self, key: str | bytes, val: str | bytes | None) -> None: + if isinstance(key, bytes): + key = key.decode("latin-1") + if val is not None: + self._container[key.lower()] = [(key, val,)] + else: + self._container[key.lower()] = key + + def __getitem__(self, key: str) -> str | None: + values = self._container[key.lower()] + if isinstance(values, str): return None + return ",".join([_[-1].decode() if isinstance(_[-1], bytes) else _[-1] for _ in values]) + + def popone(self, key: str) -> str | bytes: + key_lower = key.lower() + + val = self._container[key_lower].pop() + + if not self._container[key_lower]: + self._container[key_lower] = key + + return val[-1] + + def popall(self, key: str) -> list[str]: + key_lower = key.lower() + values = self._container[key_lower] + + self._container[key_lower] = values[0][0] + + return [_[-1] for _ in values] + + def add(self, key: str | bytes, val: str | bytes | None) -> None: + if isinstance(key, bytes): + key = key.decode("latin-1") - # If the previous value for the given header is `None` - # then discard it since we are explicitly giving a new - # value for it. - if key in self and self.getone(key) is None: - self.popone(key) + key_lower = key.lower() - super().add(key, value) + if val is None: + self._container[key_lower] = key + return - def remove_item(self, key, value): + if key_lower not in self._container or isinstance(self._container[key_lower], str): + self._container[key_lower] = [] + + self._container[key_lower].append((key, val,)) + + def remove_item(self, key: str, value: str | bytes) -> None: """ Remove a (key, value) pair from the dict. """ - existing_values = self.popall(key) - existing_values.remove(value) + key_lower = key.lower() - for value in existing_values: - self.add(key, value) + to_remove = None + + for k, v in self._container[key_lower]: + if (key == k or key == key_lower) and v == value: + to_remove = (k, v) + break + + if to_remove: + self._container[key_lower].remove(to_remove) + if not self._container[key_lower]: + del self._container[key_lower] + + +class HTTPHeadersDict(BaseMultiDict): + """ + Headers are case-insensitive and multiple values are supported + through the `add()` API. + """ class RequestJSONDataDict(OrderedDict): diff --git a/httpie/client.py b/httpie/client.py index a1da284a7c..72c9964c3a 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -1,16 +1,23 @@ import argparse -import http.client import json import sys -from contextlib import contextmanager +import typing +from random import randint from time import monotonic from typing import Any, Dict, Callable, Iterable from urllib.parse import urlparse, urlunparse -import requests -# noinspection PyPackageRequirements -import urllib3 -from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS +import niquests +from niquests._compat import HAS_LEGACY_URLLIB3 + +if not HAS_LEGACY_URLLIB3: + # noinspection PyPackageRequirements + import urllib3 + from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS, parse_url +else: + # noinspection PyPackageRequirements + import urllib3_future as urllib3 + from urllib3_future.util import SKIP_HEADER, SKIPPABLE_HEADERS, parse_url from . import __version__ from .adapters import HTTPieHTTPAdapter @@ -22,7 +29,7 @@ from .models import RequestsMessage from .plugins.registry import plugin_manager from .sessions import get_httpie_session -from .ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieCertificate, HTTPieHTTPSAdapter +from .ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieCertificate, HTTPieHTTPSAdapter, QuicCapabilityCache from .uploads import ( compress_request, prepare_request_body, get_multipart_data_and_content_type, @@ -44,6 +51,7 @@ def collect_messages( env: Environment, args: argparse.Namespace, request_body_read_callback: Callable[[bytes], None] = None, + prepared_request_readiness: Callable[[niquests.PreparedRequest], None] = None, ) -> Iterable[RequestsMessage]: httpie_session = None httpie_session_headers = None @@ -65,12 +73,34 @@ def collect_messages( ) send_kwargs = make_send_kwargs(args) send_kwargs_mergeable_from_env = make_send_kwargs_mergeable_from_env(args) + + source_address = None + + if args.interface: + source_address = (args.interface, 0) + if args.local_port: + if '-' not in args.local_port: + source_address = (args.interface or "0.0.0.0", int(args.local_port)) + else: + min_port, max_port = args.local_port.split('-', 1) + source_address = (args.interface or "0.0.0.0", randint(int(min_port), int(max_port))) + requests_session = build_requests_session( ssl_version=args.ssl_version, ciphers=args.ciphers, - verify=bool(send_kwargs_mergeable_from_env['verify']) + verify=bool(send_kwargs_mergeable_from_env['verify']), + disable_http2=args.disable_http2, + disable_http3=args.disable_http3, + resolver=args.resolver or None, + disable_ipv6=args.ipv4, + disable_ipv4=args.ipv6, + source_address=source_address, ) + if args.disable_http3 is False and args.force_http3 is True: + url = parse_url(args.url) + requests_session.quic_cache_layer[(url.host, url.port or 443)] = (url.host, url.port or 443) + if httpie_session: httpie_session.update_headers(request_kwargs['headers']) requests_session.cookies = httpie_session.cookies @@ -88,7 +118,12 @@ def collect_messages( # TODO: reflect the split between request and send kwargs. dump_request(request_kwargs) - request = requests.Request(**request_kwargs) + hooks = None + + if prepared_request_readiness: + hooks = {"pre_send": [prepared_request_readiness]} + + request = niquests.Request(**request_kwargs, hooks=hooks) prepared_request = requests_session.prepare_request(request) transform_headers(request, prepared_request) if args.path_as_is: @@ -110,12 +145,13 @@ def collect_messages( url=prepared_request.url, **send_kwargs_mergeable_from_env, ) - with max_headers(args.max_headers): - response = requests_session.send( - request=prepared_request, - **send_kwargs_merged, - **send_kwargs, - ) + response = requests_session.send( + request=prepared_request, + **send_kwargs_merged, + **send_kwargs, + ) + if args.max_headers and len(response.headers) > args.max_headers: + raise niquests.ConnectionError(f"got more than {args.max_headers} headers") response._httpie_headers_parsed_at = monotonic() expired_cookies += get_expired_cookies( response.headers.get('Set-Cookie', '') @@ -124,7 +160,7 @@ def collect_messages( response_count += 1 if response.next: if args.max_redirects and response_count == args.max_redirects: - raise requests.TooManyRedirects + raise niquests.TooManyRedirects if args.follow: prepared_request = response.next if args.all: @@ -140,28 +176,36 @@ def collect_messages( httpie_session.save() -# noinspection PyProtectedMember -@contextmanager -def max_headers(limit): - # - # noinspection PyUnresolvedReferences - orig = http.client._MAXHEADERS - http.client._MAXHEADERS = limit or float('Inf') - try: - yield - finally: - http.client._MAXHEADERS = orig - - def build_requests_session( verify: bool, ssl_version: str = None, ciphers: str = None, -) -> requests.Session: - requests_session = requests.Session() + disable_http2: bool = False, + disable_http3: bool = False, + resolver: typing.List[str] = None, + disable_ipv4: bool = False, + disable_ipv6: bool = False, + source_address: typing.Tuple[str, int] = None, +) -> niquests.Session: + requests_session = niquests.Session() + requests_session.quic_cache_layer = QuicCapabilityCache() + + if resolver: + resolver_rebuilt = [] + for r in resolver: + # assume it is the in-memory resolver + if "://" not in r: + r = f"in-memory://default/?hosts={r}" + resolver_rebuilt.append(r) + resolver = resolver_rebuilt # Install our adapter. - http_adapter = HTTPieHTTPAdapter() + http_adapter = HTTPieHTTPAdapter( + resolver=resolver, + disable_ipv4=disable_ipv4, + disable_ipv6=disable_ipv6, + source_address=source_address, + ) https_adapter = HTTPieHTTPSAdapter( ciphers=ciphers, verify=verify, @@ -169,6 +213,13 @@ def build_requests_session( AVAILABLE_SSL_VERSION_ARG_MAPPING[ssl_version] if ssl_version else None ), + disable_http2=disable_http2, + disable_http3=disable_http3, + resolver=resolver, + disable_ipv4=disable_ipv4, + disable_ipv6=disable_ipv6, + source_address=source_address, + quic_cache_layer=requests_session.quic_cache_layer, ) requests_session.mount('http://', http_adapter) requests_session.mount('https://', https_adapter) @@ -186,7 +237,7 @@ def build_requests_session( def dump_request(kwargs: dict): sys.stderr.write( - f'\n>>> requests.request(**{repr_dict(kwargs)})\n\n') + f'\n>>> niquests.request(**{repr_dict(kwargs)})\n\n') def finalize_headers(headers: HTTPHeadersDict) -> HTTPHeadersDict: @@ -210,13 +261,13 @@ def finalize_headers(headers: HTTPHeadersDict) -> HTTPHeadersDict: def transform_headers( - request: requests.Request, - prepared_request: requests.PreparedRequest + request: niquests.Request, + prepared_request: niquests.PreparedRequest ) -> None: """Apply various transformations on top of the `prepared_requests`'s headers to change the request prepreation behavior.""" - # Remove 'Content-Length' when it is misplaced by requests. + # Remove 'Content-Length' when it is misplaced by niquests. if ( prepared_request.method in IGNORE_CONTENT_LENGTH_METHODS and prepared_request.headers.get('Content-Length') == '0' @@ -232,7 +283,7 @@ def transform_headers( def apply_missing_repeated_headers( original_headers: HTTPHeadersDict, - prepared_request: requests.PreparedRequest + prepared_request: niquests.PreparedRequest ) -> None: """Update the given `prepared_request`'s headers with the original ones. This allows the requests to be prepared as usual, and then later @@ -290,12 +341,6 @@ def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict: if args.cert: cert = args.cert if args.cert_key: - # Having a client certificate key passphrase is not supported - # by requests. So we are using our own transportation structure - # which is compatible with their format (a tuple of minimum two - # items). - # - # See: https://github.com/psf/requests/issues/2519 cert = HTTPieCertificate(cert, args.cert_key, args.cert_key_pass.value) return { @@ -329,7 +374,7 @@ def make_request_kwargs( request_body_read_callback=lambda chunk: chunk ) -> dict: """ - Translate our `args` into `requests.Request` keyword arguments. + Translate our `args` into `niquests.Request` keyword arguments. """ files = args.files diff --git a/httpie/context.py b/httpie/context.py index 2a54f46916..b853339963 100644 --- a/httpie/context.py +++ b/httpie/context.py @@ -99,8 +99,9 @@ def __init__(self, devnull=None, **kwargs): assert all(hasattr(type(self), attr) for attr in kwargs.keys()) self.__dict__.update(**kwargs) - # The original STDERR unaffected by --quiet’ing. + # The original STDERR/STDOUT unaffected by --quiet’ing. self._orig_stderr = self.stderr + self._orig_stdout = self.stdout self._devnull = devnull # Keyword arguments > stream.encoding > default UTF-8 diff --git a/httpie/core.py b/httpie/core.py index d0c26dcbcc..9505185fef 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -5,9 +5,9 @@ import socket from typing import List, Optional, Union, Callable -import requests +import niquests from pygments import __version__ as pygments_version -from requests import __version__ as requests_version +from niquests import __version__ as requests_version from . import __version__ as httpie_version from .cli.constants import OUT_REQ_BODY @@ -112,16 +112,16 @@ def handle_generic_error(e, annotation=None): if include_traceback: raise exit_status = ExitStatus.ERROR - except requests.Timeout: + except niquests.Timeout: exit_status = ExitStatus.ERROR_TIMEOUT env.log_error(f'Request timed out ({parsed_args.timeout}s).') - except requests.TooManyRedirects: + except niquests.TooManyRedirects: exit_status = ExitStatus.ERROR_TOO_MANY_REDIRECTS env.log_error( f'Too many redirects' f' (--max-redirects={parsed_args.max_redirects}).' ) - except requests.exceptions.ConnectionError as exc: + except niquests.exceptions.ConnectionError as exc: annotation = None original_exc = unwrap_context(exc) if isinstance(original_exc, socket.gaierror): @@ -175,8 +175,8 @@ def program(args: argparse.Namespace, env: Environment) -> ExitStatus: # TODO: Refactor and drastically simplify, especially so that the separator logic is elsewhere. exit_status = ExitStatus.SUCCESS downloader = None - initial_request: Optional[requests.PreparedRequest] = None - final_response: Optional[requests.Response] = None + initial_request: Optional[niquests.PreparedRequest] = None + final_response: Optional[niquests.Response] = None processing_options = ProcessingOptions.from_raw_args(args) def separate(): @@ -204,8 +204,35 @@ def request_body_read_callback(chunk: bytes): args.follow = True # --download implies --follow. downloader = Downloader(env, output_file=args.output_file, resume=args.download_resume) downloader.pre_request(args.headers) - messages = collect_messages(env, args=args, - request_body_read_callback=request_body_read_callback) + + def prepared_request_readiness(pr): + + oo = OutputOptions.from_message( + pr, + args.output_options + ) + + oo = oo._replace( + body=isinstance(pr.body, (str, bytes)) and (args.verbose or oo.body) + ) + + write_message( + requests_message=pr, + env=env, + output_options=oo, + processing_options=processing_options + ) + + if oo.body > 1: + separate() + + messages = collect_messages( + env, + args=args, + request_body_read_callback=request_body_read_callback, + prepared_request_readiness=prepared_request_readiness + ) + force_separator = False prev_with_body = False @@ -225,6 +252,9 @@ def request_body_read_callback(chunk: bytes): is_streamed_upload = not isinstance(message.body, (str, bytes)) do_write_body = not is_streamed_upload force_separator = is_streamed_upload and env.stdout_isatty + if message.conn_info is None and not args.offline: + prev_with_body = output_options.body + continue else: final_response = message if args.check_status or downloader: @@ -261,6 +291,11 @@ def request_body_read_callback(chunk: bytes): return exit_status finally: + if args.data and hasattr(args.data, "close"): + args.data.close() + if args.files and hasattr(args.files, "items"): + for fd in args.files.items(): + fd[1][1].close() if downloader and not downloader.finished: downloader.failed() if args.output_file and args.output_file_specified: @@ -270,7 +305,7 @@ def request_body_read_callback(chunk: bytes): def print_debug_info(env: Environment): env.stderr.writelines([ f'HTTPie {httpie_version}\n', - f'Requests {requests_version}\n', + f'Niquests {requests_version}\n', f'Pygments {pygments_version}\n', f'Python {sys.version}\n{sys.executable}\n', f'{platform.system()} {platform.release()}', diff --git a/httpie/downloads.py b/httpie/downloads.py index 9c4b895e6f..d987b0c989 100644 --- a/httpie/downloads.py +++ b/httpie/downloads.py @@ -10,7 +10,7 @@ from typing import IO, Optional, Tuple from urllib.parse import urlsplit -import requests +import niquests from .models import HTTPResponse, OutputOptions from .output.streams import RawStream @@ -179,6 +179,7 @@ def __init__( """ self.finished = False self.status = DownloadStatus(env=env) + self._output_file_created = False self._output_file = output_file self._resume = resume self._resumed_from = 0 @@ -202,7 +203,7 @@ def pre_request(self, request_headers: dict): def start( self, initial_url: str, - final_response: requests.Response + final_response: niquests.Response ) -> Tuple[RawStream, IO]: """ Initiate and return a stream for `response` body with progress @@ -228,6 +229,7 @@ def start( initial_url=initial_url, final_response=final_response, ) + self._output_file_created = True else: # `--output, -o` provided if self._resume and final_response.status_code == PARTIAL_CONTENT: @@ -263,6 +265,9 @@ def finish(self): assert not self.finished self.finished = True self.status.finished() + # we created the output file in the process, closing it now. + if self._output_file_created: + self._output_file.close() def failed(self): self.status.terminate() @@ -288,7 +293,7 @@ def chunk_downloaded(self, chunk: bytes): @staticmethod def _get_output_file_from_response( initial_url: str, - final_response: requests.Response, + final_response: niquests.Response, ) -> IO: # Output file not specified. Pick a name that doesn't exist yet. filename = None diff --git a/httpie/internal/encoder.py b/httpie/internal/encoder.py new file mode 100644 index 0000000000..5da4dd8119 --- /dev/null +++ b/httpie/internal/encoder.py @@ -0,0 +1,472 @@ +""" +This program is part of the requests_toolbelt package. + +Copyright 2014 Ian Cordasco, Cory Benfield + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import contextlib +import io +import os +from uuid import uuid4 + +from niquests._compat import HAS_LEGACY_URLLIB3 + +if HAS_LEGACY_URLLIB3: + from urllib3_future.fields import RequestField +else: + from urllib3.fields import RequestField + + +class MultipartEncoder(object): + """ + The ``MultipartEncoder`` object is a generic interface to the engine that + will create a ``multipart/form-data`` body for you. + + The basic usage is: + + .. code-block:: python + + import requests + from requests_toolbelt import MultipartEncoder + + encoder = MultipartEncoder({'field': 'value', + 'other_field': 'other_value'}) + r = requests.post('https://httpbin.org/post', data=encoder, + headers={'Content-Type': encoder.content_type}) + + If you do not need to take advantage of streaming the post body, you can + also do: + + .. code-block:: python + + r = requests.post('https://httpbin.org/post', + data=encoder.to_string(), + headers={'Content-Type': encoder.content_type}) + + If you want the encoder to use a specific order, you can use an + OrderedDict or more simply, a list of tuples: + + .. code-block:: python + + encoder = MultipartEncoder([('field', 'value'), + ('other_field', 'other_value')]) + + .. versionchanged:: 0.4.0 + + You can also provide tuples as part values as you would provide them to + requests' ``files`` parameter. + + .. code-block:: python + + encoder = MultipartEncoder({ + 'field': ('file_name', b'{"a": "b"}', 'application/json', + {'X-My-Header': 'my-value'}) + ]) + + .. warning:: + + This object will end up directly in :mod:`httplib`. Currently, + :mod:`httplib` has a hard-coded read size of **8192 bytes**. This + means that it will loop until the file has been read and your upload + could take a while. This is **not** a bug in requests. A feature is + being considered for this object to allow you, the user, to specify + what size should be returned on a read. If you have opinions on this, + please weigh in on `this issue`_. + + .. _this issue: + https://github.com/requests/toolbelt/issues/75 + + """ + + def __init__(self, fields, boundary=None, encoding='utf-8'): + #: Boundary value either passed in by the user or created + self.boundary_value = boundary or uuid4().hex + + # Computed boundary + self.boundary = '--{}'.format(self.boundary_value) + + #: Encoding of the data being passed in + self.encoding = encoding + + # Pre-encoded boundary + self._encoded_boundary = b''.join([ + self.boundary.encode(self.encoding), + '\r\n'.encode(self.encoding) + ]) + + #: Fields provided by the user + self.fields = fields + + #: Whether or not the encoder is finished + self.finished = False + + #: Pre-computed parts of the upload + self.parts = [] + + # Pre-computed parts iterator + self._iter_parts = iter([]) + + # The part we're currently working with + self._current_part = None + + # Cached computation of the body's length + self._len = None + + # Our buffer + self._buffer = CustomBytesIO(encoding=encoding) + + # Pre-compute each part's headers + self._prepare_parts() + + # Load boundary into buffer + self._write_boundary() + + @property + def len(self): + """Length of the multipart/form-data body. + + requests will first attempt to get the length of the body by calling + ``len(body)`` and then by checking for the ``len`` attribute. + + On 32-bit systems, the ``__len__`` method cannot return anything + larger than an integer (in C) can hold. If the total size of the body + is even slightly larger than 4GB users will see an OverflowError. This + manifested itself in `bug #80`_. + + As such, we now calculate the length lazily as a property. + + .. _bug #80: + https://github.com/requests/toolbelt/issues/80 + """ + # If _len isn't already calculated, calculate, return, and set it + return self._len or self._calculate_length() + + def __repr__(self): + return ''.format(self.fields) + + def _calculate_length(self): + """ + This uses the parts to calculate the length of the body. + + This returns the calculated length so __len__ can be lazy. + """ + boundary_len = len(self.boundary) # Length of --{boundary} + # boundary length + header length + body length + len('\r\n') * 2 + + self._len = sum( + (boundary_len + total_len(p) + 4) for p in self.parts + ) + boundary_len + 4 + + return self._len + + def _calculate_load_amount(self, read_size): + """This calculates how many bytes need to be added to the buffer. + + When a consumer read's ``x`` from the buffer, there are two cases to + satisfy: + + 1. Enough data in the buffer to return the requested amount + 2. Not enough data + + This function uses the amount of unread bytes in the buffer and + determines how much the Encoder has to load before it can return the + requested amount of bytes. + + :param int read_size: the number of bytes the consumer requests + :returns: int -- the number of bytes that must be loaded into the + buffer before the read can be satisfied. This will be strictly + non-negative + """ + amount = read_size - total_len(self._buffer) + return amount if amount > 0 else 0 + + def _load(self, amount): + """Load ``amount`` number of bytes into the buffer.""" + self._buffer.smart_truncate() + part = self._current_part or self._next_part() + while amount == -1 or amount > 0: + written = 0 + if part and not part.bytes_left_to_write(): + written += self._write(b'\r\n') + written += self._write_boundary() + part = self._next_part() + + if not part: + written += self._write_closing_boundary() + self.finished = True + break + + written += part.write_to(self._buffer, amount) + + if amount != -1: + amount -= written + + def _next_part(self): + try: + p = self._current_part = next(self._iter_parts) + except StopIteration: + p = None + return p + + def _iter_fields(self): + _fields = self.fields + if hasattr(self.fields, 'items'): + _fields = list(self.fields.items()) + for k, v in _fields: + file_name = None + file_type = None + file_headers = None + if isinstance(v, (list, tuple)): + if len(v) == 2: + file_name, file_pointer = v + elif len(v) == 3: + file_name, file_pointer, file_type = v + else: + file_name, file_pointer, file_type, file_headers = v + else: + file_pointer = v + + field = RequestField( + name=k, + data=file_pointer, + filename=file_name, + headers=file_headers + ) + + field.make_multipart(content_type=file_type) + yield field + + def _prepare_parts(self): + """This uses the fields provided by the user and creates Part objects. + + It populates the `parts` attribute and uses that to create a + generator for iteration. + """ + enc = self.encoding + self.parts = [Part.from_field(f, enc) for f in self._iter_fields()] + self._iter_parts = iter(self.parts) + + def _write(self, bytes_to_write): + """Write the bytes to the end of the buffer. + + :param bytes bytes_to_write: byte-string (or bytearray) to append to + the buffer + :returns: int -- the number of bytes written + """ + return self._buffer.append(bytes_to_write) + + def _write_boundary(self): + """Write the boundary to the end of the buffer.""" + return self._write(self._encoded_boundary) + + def _write_closing_boundary(self): + """Write the bytes necessary to finish a multipart/form-data body.""" + with reset(self._buffer): + self._buffer.seek(-2, 2) + self._buffer.write(b'--\r\n') + return 2 + + def _write_headers(self, headers): + """Write the current part's headers to the buffer.""" + return self._write(headers.encode(self.encoding) if isinstance(headers, str) else headers) + + @property + def content_type(self): + return str( + 'multipart/form-data; boundary={}'.format(self.boundary_value) + ) + + def to_string(self): + """Return the entirety of the data in the encoder. + + .. note:: + + This simply reads all of the data it can. If you have started + streaming or reading data from the encoder, this method will only + return whatever data is left in the encoder. + + .. note:: + + This method affects the internal state of the encoder. Calling + this method will exhaust the encoder. + + :returns: the multipart message + :rtype: bytes + """ + + return self.read() + + def read(self, size=-1): + """Read data from the streaming encoder. + + :param int size: (optional), If provided, ``read`` will return exactly + that many bytes. If it is not provided, it will return the + remaining bytes. + :returns: bytes + """ + if self.finished: + return self._buffer.read(size) + + bytes_to_load = size + if bytes_to_load != -1 and bytes_to_load is not None: + bytes_to_load = self._calculate_load_amount(int(size)) + + self._load(bytes_to_load) + return self._buffer.read(size) + + +class Part(object): + def __init__(self, headers, body): + self.headers = headers + self.body = body + self.headers_unread = True + self.len = len(self.headers) + total_len(self.body) + + @classmethod + def from_field(cls, field, encoding): + """Create a part from a Request Field generated by urllib3.""" + headers = field.render_headers().encode(encoding) + body = coerce_data(field.data, encoding) + return cls(headers, body) + + def bytes_left_to_write(self): + """Determine if there are bytes left to write. + + :returns: bool -- ``True`` if there are bytes left to write, otherwise + ``False`` + """ + to_read = 0 + if self.headers_unread: + to_read += len(self.headers) + + return (to_read + total_len(self.body)) > 0 + + def write_to(self, buffer, size): + """Write the requested amount of bytes to the buffer provided. + + The number of bytes written may exceed size on the first read since we + load the headers ambitiously. + + :param CustomBytesIO buffer: buffer we want to write bytes to + :param int size: number of bytes requested to be written to the buffer + :returns: int -- number of bytes actually written + """ + written = 0 + if self.headers_unread: + written += buffer.append(self.headers) + self.headers_unread = False + + while total_len(self.body) > 0 and (size == -1 or written < size): + amount_to_read = size + if size != -1: + amount_to_read = size - written + written += buffer.append(self.body.read(amount_to_read)) + + return written + + +class CustomBytesIO(io.BytesIO): + def __init__(self, buffer=None, encoding='utf-8'): + buffer = buffer.encode(encoding) if buffer else b"" + super(CustomBytesIO, self).__init__(buffer) + + def _get_end(self): + current_pos = self.tell() + self.seek(0, 2) + length = self.tell() + self.seek(current_pos, 0) + return length + + @property + def len(self): + length = self._get_end() + return length - self.tell() + + def append(self, bytes): + with reset(self): + written = self.write(bytes) + return written + + def smart_truncate(self): + to_be_read = total_len(self) + already_read = self._get_end() - to_be_read + + if already_read >= to_be_read: + old_bytes = self.read() + self.seek(0, 0) + self.truncate() + self.write(old_bytes) + self.seek(0, 0) # We want to be at the beginning + + +class FileWrapper(object): + def __init__(self, file_object): + self.fd = file_object + + @property + def len(self): + return total_len(self.fd) - self.fd.tell() + + def read(self, length=-1): + return self.fd.read(length) + + +@contextlib.contextmanager +def reset(buffer): + """Keep track of the buffer's current position and write to the end. + + This is a context manager meant to be used when adding data to the buffer. + It eliminates the need for every function to be concerned with the + position of the cursor in the buffer. + """ + original_position = buffer.tell() + buffer.seek(0, 2) + yield + buffer.seek(original_position, 0) + + +def coerce_data(data, encoding): + """Ensure that every object's __len__ behaves uniformly.""" + if not isinstance(data, CustomBytesIO): + if hasattr(data, 'getvalue'): + return CustomBytesIO(data.getvalue(), encoding) + + if hasattr(data, 'fileno'): + return FileWrapper(data) + + if not hasattr(data, 'read'): + return CustomBytesIO(data, encoding) + + return data + + +def total_len(o): + if hasattr(o, '__len__'): + return len(o) + + if hasattr(o, 'len'): + return o.len + + if hasattr(o, 'fileno'): + try: + fileno = o.fileno() + except io.UnsupportedOperation: + pass + else: + return os.fstat(fileno).st_size + + if hasattr(o, 'getvalue'): + # e.g. BytesIO, cStringIO.StringIO + return len(o.getvalue()) diff --git a/httpie/internal/update_warnings.py b/httpie/internal/update_warnings.py index a4b80d46b5..c684bb80ad 100644 --- a/httpie/internal/update_warnings.py +++ b/httpie/internal/update_warnings.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Any, Optional, Callable -import requests +import niquests import httpie from httpie.context import Environment, LogLevel @@ -41,7 +41,7 @@ def _fetch_updates(env: Environment) -> str: file = env.config.version_info_file data = _read_data_error_free(file) - response = requests.get(PACKAGE_INDEX_LINK, verify=False) + response = niquests.get(PACKAGE_INDEX_LINK, verify=False) response.raise_for_status() data.setdefault('last_warned_date', None) diff --git a/httpie/models.py b/httpie/models.py index a0a68c8ddc..142fd69710 100644 --- a/httpie/models.py +++ b/httpie/models.py @@ -1,7 +1,17 @@ from time import monotonic -import requests -from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS +import niquests + +from niquests._compat import HAS_LEGACY_URLLIB3 + +if not HAS_LEGACY_URLLIB3: + from urllib3 import ConnectionInfo + from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS +else: + from urllib3_future import ConnectionInfo + from urllib3_future.util import SKIP_HEADER, SKIPPABLE_HEADERS + +from kiss_headers.utils import prettify_header_name from enum import Enum, auto from typing import Iterable, Union, NamedTuple @@ -18,6 +28,10 @@ from .utils import split_cookies, parse_content_type_header ELAPSED_TIME_LABEL = 'Elapsed time' +ELAPSED_DNS_RESOLUTION_LABEL = 'Elapsed DNS' +ELAPSED_TLS_HANDSHAKE = 'Elapsed TLS handshake' +ELAPSED_REQUEST_SEND = 'Elapsed emitting request' +ELAPSED_ESTABLISH_CONN = 'Elapsed established connection' class HTTPMessage: @@ -59,7 +73,7 @@ def content_type(self) -> str: class HTTPResponse(HTTPMessage): - """A :class:`requests.models.Response` wrapper.""" + """A :class:`niquests.models.Response` wrapper.""" def iter_body(self, chunk_size=1): return self._orig.iter_content(chunk_size=chunk_size) @@ -70,18 +84,19 @@ def iter_lines(self, chunk_size): @property def headers(self): original = self._orig + http_headers = original.raw.headers if original.raw and hasattr(original.raw, "headers") else original.headers status_line = f'HTTP/{self.version} {original.status_code} {original.reason}' headers = [status_line] headers.extend( - ': '.join(header) - for header in original.headers.items() - if header[0] != 'Set-Cookie' + ': '.join([prettify_header_name(header), value]) + for header, value in http_headers.items() + if header.lower() != 'set-cookie' ) headers.extend( f'Set-Cookie: {cookie}' - for header, value in original.headers.items() + for header, value in http_headers.items() for cookie in split_cookies(value) - if header == 'Set-Cookie' + if header.lower() == 'set-cookie' ) return '\r\n'.join(headers) @@ -89,12 +104,23 @@ def headers(self): def metadata(self) -> str: data = {} time_to_parse_headers = self._orig.elapsed.total_seconds() + # noinspection PyProtectedMember time_since_headers_parsed = monotonic() - self._orig._httpie_headers_parsed_at time_elapsed = time_to_parse_headers + time_since_headers_parsed - # data['Headers time'] = str(round(time_to_parse_headers, 5)) + 's' - # data['Body time'] = str(round(time_since_headers_parsed, 5)) + 's' + + if hasattr(self._orig, "conn_info") and self._orig.conn_info: + if self._orig.conn_info.resolution_latency: + data[ELAPSED_DNS_RESOLUTION_LABEL] = str(round(self._orig.conn_info.resolution_latency.total_seconds(), 10)) + 's' + if self._orig.conn_info.established_latency: + data[ELAPSED_ESTABLISH_CONN] = str(round(self._orig.conn_info.established_latency.total_seconds(), 10)) + 's' + if self._orig.conn_info.tls_handshake_latency: + data[ELAPSED_TLS_HANDSHAKE] = str(round(self._orig.conn_info.tls_handshake_latency.total_seconds(), 10)) + 's' + if self._orig.conn_info.request_sent_latency: + data[ELAPSED_REQUEST_SEND] = str(round(self._orig.conn_info.request_sent_latency.total_seconds(), 10)) + 's' + data[ELAPSED_TIME_LABEL] = str(round(time_elapsed, 10)) + 's' + return '\n'.join( f'{key}: {value}' for key, value in data.items() @@ -108,27 +134,11 @@ def version(self) -> str: Assume HTTP/1.1 if version is not available. """ - mapping = { - 9: '0.9', - 10: '1.0', - 11: '1.1', - 20: '2.0', - } - fallback = 11 - version = None - try: - raw = self._orig.raw - if getattr(raw, '_original_response', None): - version = raw._original_response.version - else: - version = raw.version - except AttributeError: - pass - return mapping[version or fallback] + return self._orig.conn_info.http_version.value.replace("HTTP/", "").replace(".0", "") if self._orig.conn_info and self._orig.conn_info.http_version else "1.1" class HTTPRequest(HTTPMessage): - """A :class:`requests.models.Request` wrapper.""" + """A :class:`niquests.models.Request` wrapper.""" def iter_body(self, chunk_size): yield self.body @@ -136,14 +146,69 @@ def iter_body(self, chunk_size): def iter_lines(self, chunk_size): yield self.body, b'' + @property + def metadata(self) -> str: + conn_info: ConnectionInfo = self._orig.conn_info + + metadatum = f"Connected to: {conn_info.destination_address[0]} port {conn_info.destination_address[1]}\n" + + if conn_info.certificate_dict: + metadatum += ( + f"Connection secured using: {conn_info.tls_version.name.replace('_', '.')} with {conn_info.cipher.replace('TLS_', '').replace('_', '-')}\n" + f"Server certificate: " + ) + + for entry in conn_info.certificate_dict['subject']: + if len(entry) == 2: + rdns, value = entry + elif len(entry) == 1: + rdns, value = entry[0] + else: + continue + + metadatum += f'{rdns}="{value}"; ' + + if "subjectAltName" in conn_info.certificate_dict: + for entry in conn_info.certificate_dict['subjectAltName']: + if len(entry) == 2: + rdns, value = entry + metadatum += f'{rdns}="{value}"; ' + + metadatum = metadatum[:-2] + "\n" + + metadatum += f'Certificate validity: "{conn_info.certificate_dict["notBefore"]}" to "{conn_info.certificate_dict["notAfter"]}"\n' + + if "issuer" in conn_info.certificate_dict: + metadatum += "Issuer: " + + for entry in conn_info.certificate_dict['issuer']: + if len(entry) == 2: + rdns, value = entry + elif len(entry) == 1: + rdns, value = entry[0] + else: + continue + + metadatum += f'{rdns}="{value}"; ' + + metadatum = metadatum[:-2] + "\n" + + if self._orig.ocsp_verified is None: + metadatum += "Revocation status: Unverified\n" + elif self._orig.ocsp_verified is True: + metadatum += "Revocation status: Good\n" + + return metadatum[:-1] + @property def headers(self): url = urlsplit(self._orig.url) - request_line = '{method} {path}{query} HTTP/1.1'.format( + request_line = '{method} {path}{query} {http_version}'.format( method=self._orig.method, path=url.path or '/', - query=f'?{url.query}' if url.query else '' + query=f'?{url.query}' if url.query else '', + http_version=self._orig.conn_info.http_version.value.replace(".0", "") if self._orig.conn_info and self._orig.conn_info.http_version else "HTTP/1.1" ) headers = self._orig.headers.copy() @@ -158,6 +223,7 @@ def headers(self): headers.insert(0, request_line) headers = '\r\n'.join(headers).strip() + return headers @property @@ -169,7 +235,7 @@ def body(self): return body or b'' -RequestsMessage = Union[requests.PreparedRequest, requests.Response] +RequestsMessage = Union[niquests.PreparedRequest, niquests.Response] class RequestsMessageKind(Enum): @@ -178,9 +244,9 @@ class RequestsMessageKind(Enum): def infer_requests_message_kind(message: RequestsMessage) -> RequestsMessageKind: - if isinstance(message, requests.PreparedRequest): + if isinstance(message, niquests.PreparedRequest): return RequestsMessageKind.REQUEST - elif isinstance(message, requests.Response): + elif isinstance(message, niquests.Response): return RequestsMessageKind.RESPONSE else: raise TypeError(f"Unexpected message type: {type(message).__name__}") @@ -190,6 +256,7 @@ def infer_requests_message_kind(message: RequestsMessage) -> RequestsMessageKind RequestsMessageKind.REQUEST: { 'headers': OUT_REQ_HEAD, 'body': OUT_REQ_BODY, + 'meta': OUT_RESP_META }, RequestsMessageKind.RESPONSE: { 'headers': OUT_RESP_HEAD, diff --git a/httpie/output/lexers/metadata.py b/httpie/output/lexers/metadata.py index fa68e45762..7f5c77f54d 100644 --- a/httpie/output/lexers/metadata.py +++ b/httpie/output/lexers/metadata.py @@ -1,6 +1,6 @@ import pygments -from httpie.models import ELAPSED_TIME_LABEL +from httpie.models import ELAPSED_TIME_LABEL, ELAPSED_DNS_RESOLUTION_LABEL, ELAPSED_TLS_HANDSHAKE, ELAPSED_REQUEST_SEND, ELAPSED_ESTABLISH_CONN from httpie.output.lexers.common import precise SPEED_TOKENS = { @@ -36,7 +36,7 @@ class MetadataLexer(pygments.lexer.RegexLexer): tokens = { 'root': [ ( - fr'({ELAPSED_TIME_LABEL})( *)(:)( *)(\d+\.\d+)(s)', pygments.lexer.bygroups( + fr'({ELAPSED_TIME_LABEL}|{ELAPSED_DNS_RESOLUTION_LABEL}|{ELAPSED_REQUEST_SEND}|{ELAPSED_TLS_HANDSHAKE}|{ELAPSED_ESTABLISH_CONN})( *)(:)( *)(\d+\.[\de\-]+)(s)', pygments.lexer.bygroups( pygments.token.Name.Decorator, # Name pygments.token.Text, pygments.token.Operator, # Colon diff --git a/httpie/output/streams.py b/httpie/output/streams.py index 811093808a..ea7867a3ef 100644 --- a/httpie/output/streams.py +++ b/httpie/output/streams.py @@ -5,7 +5,7 @@ from .processing import Conversion, Formatting from ..context import Environment from ..encoding import smart_decode, smart_encode, UTF8 -from ..models import HTTPMessage, OutputOptions +from ..models import HTTPMessage, OutputOptions, RequestsMessageKind from ..utils import parse_content_type_header @@ -62,6 +62,10 @@ def iter_body(self) -> Iterable[bytes]: def __iter__(self) -> Iterable[bytes]: """Return an iterator over `self.msg`.""" + if self.output_options.meta and self.output_options.kind is RequestsMessageKind.REQUEST: + yield self.get_metadata() + yield b'\n\n' + if self.output_options.headers: yield self.get_headers() yield b'\r\n\r\n' @@ -77,18 +81,17 @@ def __iter__(self) -> Iterable[bytes]: yield b'\n' yield e.message - if self.output_options.meta: + if self.output_options.meta and self.output_options.kind is RequestsMessageKind.RESPONSE: if self.output_options.body: yield b'\n\n' yield self.get_metadata() - yield b'\n\n' class RawStream(BaseStream): """The message is streamed in chunks with no processing.""" - CHUNK_SIZE = 1024 * 100 + CHUNK_SIZE = -1 CHUNK_SIZE_BY_LINE = 1 def __init__(self, chunk_size=CHUNK_SIZE, **kwargs): diff --git a/httpie/output/writer.py b/httpie/output/writer.py index 4a2949bce2..4e4071cd83 100644 --- a/httpie/output/writer.py +++ b/httpie/output/writer.py @@ -1,5 +1,5 @@ import errno -import requests +import niquests from typing import Any, Dict, IO, Optional, TextIO, Tuple, Type, Union from ..cli.dicts import HTTPHeadersDict @@ -105,7 +105,7 @@ def write_raw_data( headers: Optional[HTTPHeadersDict] = None, stream_kwargs: Optional[Dict[str, Any]] = None ): - msg = requests.PreparedRequest() + msg = niquests.PreparedRequest() msg.is_body_upload_chunk = True msg.body = data msg.headers = headers or HTTPHeadersDict() diff --git a/httpie/plugins/base.py b/httpie/plugins/base.py index 1b44e5aec5..4e26242bc7 100644 --- a/httpie/plugins/base.py +++ b/httpie/plugins/base.py @@ -63,7 +63,7 @@ def get_auth(self, username: str = None, password: str = None): Use `self.raw_auth` to access the raw value passed through `--auth, -a`. - Return a ``requests.auth.AuthBase`` subclass instance. + Return a ``niquests.auth.AuthBase`` subclass instance. """ raise NotImplementedError() @@ -73,7 +73,7 @@ class TransportPlugin(BasePlugin): """ Requests transport adapter docs: - + See httpie-unixsocket for an example transport plugin: @@ -86,7 +86,7 @@ class TransportPlugin(BasePlugin): def get_adapter(self): """ - Return a ``requests.adapters.BaseAdapter`` subclass instance to be + Return a ``niquests.adapters.BaseAdapter`` subclass instance to be mounted to ``self.prefix``. """ diff --git a/httpie/plugins/builtin.py b/httpie/plugins/builtin.py index 860aebf7f9..ad79d0a53f 100644 --- a/httpie/plugins/builtin.py +++ b/httpie/plugins/builtin.py @@ -1,6 +1,6 @@ from base64 import b64encode -import requests.auth +import niquests.auth from .base import AuthPlugin @@ -10,12 +10,12 @@ class BuiltinAuthPlugin(AuthPlugin): package_name = '(builtin)' -class HTTPBasicAuth(requests.auth.HTTPBasicAuth): +class HTTPBasicAuth(niquests.auth.HTTPBasicAuth): def __call__( self, - request: requests.PreparedRequest - ) -> requests.PreparedRequest: + request: niquests.PreparedRequest + ) -> niquests.PreparedRequest: """ Override username/password serialization to allow unicode. @@ -34,12 +34,12 @@ def make_header(username: str, password: str) -> str: return f'Basic {token}' -class HTTPBearerAuth(requests.auth.AuthBase): +class HTTPBearerAuth(niquests.auth.AuthBase): def __init__(self, token: str) -> None: self.token = token - def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest: + def __call__(self, request: niquests.PreparedRequest) -> niquests.PreparedRequest: request.headers['Authorization'] = f'Bearer {self.token}' return request @@ -64,8 +64,8 @@ def get_auth( self, username: str, password: str - ) -> requests.auth.HTTPDigestAuth: - return requests.auth.HTTPDigestAuth(username, password) + ) -> niquests.auth.HTTPDigestAuth: + return niquests.auth.HTTPDigestAuth(username, password) class BearerAuthPlugin(BuiltinAuthPlugin): @@ -75,5 +75,5 @@ class BearerAuthPlugin(BuiltinAuthPlugin): auth_parse = False # noinspection PyMethodOverriding - def get_auth(self, **kwargs) -> requests.auth.HTTPDigestAuth: + def get_auth(self, **kwargs) -> niquests.auth.HTTPDigestAuth: return HTTPBearerAuth(self.raw_auth) diff --git a/httpie/sessions.py b/httpie/sessions.py index 99dcdba92e..5351959a9b 100644 --- a/httpie/sessions.py +++ b/httpie/sessions.py @@ -10,8 +10,8 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Union -from requests.auth import AuthBase -from requests.cookies import RequestsCookieJar, remove_cookie_by_name +from niquests.auth import AuthBase +from niquests.cookies import RequestsCookieJar, remove_cookie_by_name from .context import Environment, LogLevel from .cookies import HTTPieCookiePolicy diff --git a/httpie/ssl_.py b/httpie/ssl_.py index af5ca548db..937b3eab70 100644 --- a/httpie/ssl_.py +++ b/httpie/ssl_.py @@ -1,6 +1,12 @@ import ssl -from typing import NamedTuple, Optional +from typing import NamedTuple, Optional, Tuple, MutableMapping +import json +import os.path +from os import makedirs +from niquests import Response + +from httpie.config import DEFAULT_CONFIG_DIR from httpie.adapters import HTTPAdapter # noinspection PyPackageRequirements from urllib3.util.ssl_ import ( @@ -10,10 +16,6 @@ SSL_VERSION_ARG_MAPPING = { - 'ssl2.3': 'PROTOCOL_SSLv23', - 'ssl3': 'PROTOCOL_SSLv3', - 'tls1': 'PROTOCOL_TLSv1', - 'tls1.1': 'PROTOCOL_TLSv1_1', 'tls1.2': 'PROTOCOL_TLSv1_2', 'tls1.3': 'PROTOCOL_TLSv1_3', } @@ -24,6 +26,50 @@ } +class QuicCapabilityCache( + MutableMapping[Tuple[str, int], Optional[Tuple[str, int]]] +): + + def __init__(self): + self._cache = {} + if not os.path.exists(DEFAULT_CONFIG_DIR): + makedirs(DEFAULT_CONFIG_DIR, exist_ok=True) + if os.path.exists(os.path.join(DEFAULT_CONFIG_DIR, "quic.json")): + with open(os.path.join(DEFAULT_CONFIG_DIR, "quic.json"), "r") as fp: + self._cache = json.load(fp) + + def save(self): + with open(os.path.join(DEFAULT_CONFIG_DIR, "quic.json"), "w+") as fp: + json.dump(self._cache, fp) + + def __contains__(self, item: Tuple[str, int]): + return f"QUIC_{item[0]}_{item[1]}" in self._cache + + def __setitem__(self, key: Tuple[str, int], value: Optional[Tuple[str, int]]): + self._cache[f"QUIC_{key[0]}_{key[1]}"] = f"{value[0]}:{value[1]}" + self.save() + + def __getitem__(self, item: Tuple[str, int]): + key: str = f"QUIC_{item[0]}_{item[1]}" + if key in self._cache: + host, port = self._cache[key].split(":") + return host, int(port) + + return None + + def __delitem__(self, key: Tuple[str, int]): + key: str = f"QUIC_{key[0]}_{key[1]}" + if key in self._cache: + del self._cache[key] + self.save() + + def __len__(self): + return len(self._cache) + + def __iter__(self): + yield from self._cache.items() + + class HTTPieCertificate(NamedTuple): cert_file: Optional[str] = None key_file: Optional[str] = None @@ -32,7 +78,9 @@ class HTTPieCertificate(NamedTuple): def to_raw_cert(self): """Synthesize a requests-compatible (2-item tuple of cert and key file) object from HTTPie's internal representation of a certificate.""" - return (self.cert_file, self.key_file) + if self.key_password: + return self.cert_file, self.key_file, self.key_password + return self.cert_file, self.key_file class HTTPieHTTPSAdapter(HTTPAdapter): @@ -43,24 +91,38 @@ def __init__( ciphers: str = None, **kwargs ): - self._ssl_context = self._create_ssl_context( - verify=verify, - ssl_version=ssl_version, - ciphers=ciphers, - ) + self._ssl_context = None + self._verify = None + + if ssl_version or ciphers: + # Only set the custom context if user supplied one. + # Because urllib3-future set his own secure ctx with a set of + # ciphers (moz recommended list). thus avoiding excluding QUIC + # in case some ciphers are accidentally excluded. + self._ssl_context = self._create_ssl_context( + verify=verify, + ssl_version=ssl_version, + ciphers=ciphers, + ) + else: + self._verify = verify + super().__init__(**kwargs) def init_poolmanager(self, *args, **kwargs): kwargs['ssl_context'] = self._ssl_context + if self._verify is not None: + kwargs['cert_reqs'] = ssl.CERT_REQUIRED if self._verify else ssl.CERT_NONE return super().init_poolmanager(*args, **kwargs) def proxy_manager_for(self, *args, **kwargs): kwargs['ssl_context'] = self._ssl_context + if self._verify is not None: + kwargs['cert_reqs'] = ssl.CERT_REQUIRED if self._verify else ssl.CERT_NONE return super().proxy_manager_for(*args, **kwargs) def cert_verify(self, conn, url, verify, cert): if isinstance(cert, HTTPieCertificate): - conn.key_password = cert.key_password cert = cert.to_raw_cert() return super().cert_verify(conn, url, verify, cert) @@ -84,6 +146,13 @@ def _create_ssl_context( def get_default_ciphers_names(cls): return [cipher['name'] for cipher in cls._create_ssl_context(verify=False).get_ciphers()] + def send( + self, + *args, + **kwargs + ) -> Response: + return super().send(*args, **kwargs) + def _is_key_file_encrypted(key_file): """Detects if a key file is encrypted or not. diff --git a/httpie/uploads.py b/httpie/uploads.py index 4a993b3a25..3de4fd3716 100644 --- a/httpie/uploads.py +++ b/httpie/uploads.py @@ -3,18 +3,16 @@ import zlib import functools import threading -from typing import Any, Callable, IO, Iterable, Optional, Tuple, Union, TYPE_CHECKING +from typing import Any, Callable, IO, Iterable, Optional, Tuple, Union from urllib.parse import urlencode -import requests -from requests.utils import super_len - -if TYPE_CHECKING: - from requests_toolbelt import MultipartEncoder +import niquests +from niquests.utils import super_len from .context import Environment from .cli.dicts import MultipartRequestDataDict, RequestDataDict from .compat import is_windows +from .internal.encoder import MultipartEncoder class ChunkedStream: @@ -172,7 +170,6 @@ def _prepare_file_for_upload( ) if chunked: - from requests_toolbelt import MultipartEncoder if isinstance(file, MultipartEncoder): return ChunkedMultipartUploadStream( encoder=file, @@ -232,7 +229,6 @@ def get_multipart_data_and_content_type( boundary: str = None, content_type: str = None, ) -> Tuple['MultipartEncoder', str]: - from requests_toolbelt import MultipartEncoder encoder = MultipartEncoder( fields=data.items(), @@ -250,7 +246,7 @@ def get_multipart_data_and_content_type( def compress_request( - request: requests.PreparedRequest, + request: niquests.PreparedRequest, always: bool, ): deflater = zlib.compressobj() diff --git a/httpie/utils.py b/httpie/utils.py index 4735b2be5d..33d8158568 100644 --- a/httpie/utils.py +++ b/httpie/utils.py @@ -16,7 +16,7 @@ from urllib.parse import urlsplit from typing import Any, List, Optional, Tuple, Generator, Callable, Iterable, IO, TypeVar -import requests.auth +import niquests.auth RE_COOKIE_SPLIT = re.compile(r', (?=[^ ;]+=)') Item = Tuple[str, Any] @@ -121,7 +121,7 @@ def humanize_bytes(n, precision=2): return f'{n / factor:.{precision}f} {suffix}' -class ExplicitNullAuth(requests.auth.AuthBase): +class ExplicitNullAuth(niquests.auth.AuthBase): """Forces requests to ignore the ``.netrc``. """ @@ -201,7 +201,7 @@ def _max_age_to_expires(cookies, now): def parse_content_type_header(header): - """Borrowed from requests.""" + """Borrowed from niquests.""" tokens = header.split(';') content_type, params = tokens[0].strip(), tokens[1:] params_dict = {} diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index ced65979b1..0000000000 --- a/pytest.ini +++ /dev/null @@ -1,7 +0,0 @@ -[pytest] -markers = - # If you want to run tests without a full HTTPie installation - # we advise you to disable the markers below, e.g: - # pytest -m 'not requires_installation and not requires_external_processes' - requires_installation - requires_external_processes diff --git a/setup.cfg b/setup.cfg index 86c41ff308..67b12ad0f2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,16 @@ testpaths = httpie tests norecursedirs = tests/fixtures addopts = --tb=native --doctest-modules --verbose xfail_strict = True - +markers = + # If you want to run tests without a full HTTPie installation + # we advise you to disable the markers below, e.g: + # pytest -m 'not requires_installation and not requires_external_processes' + requires_installation + requires_external_processes +filterwarnings = + default + ignore:Passing msg=\.\. is deprecated:DeprecationWarning + ignore:Unverified HTTPS request is being made to host:urllib3.exceptions.InsecureRequestWarning [flake8] # diff --git a/setup.py b/setup.py index 93bdb8f957..cd18032f2f 100644 --- a/setup.py +++ b/setup.py @@ -9,12 +9,13 @@ # Note: keep requirements here to ease distributions packaging tests_require = [ - 'pytest', + 'pytest<8', 'pytest-httpbin>=0.0.6', 'pytest-lazy-fixture>=0.0.6', 'responses', 'pytest-mock', - 'werkzeug<2.1.0' + 'werkzeug<2.1.0', + 'flaky', ] dev_require = [ *tests_require, @@ -23,7 +24,6 @@ 'flake8-deprecated', 'flake8-mutable', 'flake8-tuple', - 'pyopenssl', 'pytest-cov', 'pyyaml', 'twine', @@ -34,13 +34,11 @@ 'pip', 'charset_normalizer>=2.0.0', 'defusedxml>=0.6.0', - 'requests[socks]>=2.22.0', + 'niquests[socks]>=3.4.0,<4', 'Pygments>=2.5.2', - 'requests-toolbelt>=0.9.1', - 'multidict>=4.7.0', 'setuptools', 'importlib-metadata>=1.4.0; python_version < "3.8"', - 'rich>=9.10.0' + 'rich>=9.10.0', ] install_requires_win_only = [ 'colorama>=0.2.4', diff --git a/tests/conftest.py b/tests/conftest.py index 7ca172a867..fa8642edd9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,6 @@ HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, HTTPBIN_WITH_CHUNKED_SUPPORT, REMOTE_HTTPBIN_DOMAIN, - IS_PYOPENSSL, mock_env ) from .utils.plugins_cli import ( # noqa @@ -20,6 +19,17 @@ ) from .utils.http_server import http_server, localhost_http_server # noqa +from sys import modules + +import niquests +import urllib3 + +# the mock utility 'response' only works with 'requests' +modules["requests"] = niquests +modules["requests.adapters"] = niquests.adapters +modules["requests.exceptions"] = niquests.exceptions +modules["requests.packages.urllib3"] = urllib3 + @pytest.fixture(scope='function', autouse=True) def httpbin_add_ca_bundle(monkeypatch): @@ -73,19 +83,3 @@ def remote_httpbin(_remote_httpbin_available): if _remote_httpbin_available: return 'http://' + REMOTE_HTTPBIN_DOMAIN pytest.skip(f'{REMOTE_HTTPBIN_DOMAIN} not resolvable') - - -@pytest.fixture(autouse=True, scope='session') -def pyopenssl_inject(): - """ - Injects `pyOpenSSL` module to make sure `requests` will use it. - - """ - if IS_PYOPENSSL: - try: - import urllib3.contrib.pyopenssl - urllib3.contrib.pyopenssl.inject_into_urllib3() - except ModuleNotFoundError: - pytest.fail('Missing "pyopenssl" module.') - - yield diff --git a/tests/test_auth.py b/tests/test_auth.py index 696fb22826..3f9b742cd7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -93,8 +93,8 @@ def test_missing_auth(httpbin): def test_netrc(httpbin_both): # This one gets handled by requests (no --auth, --auth-type present), - # that’s why we patch inside `requests.sessions`. - with mock.patch('requests.sessions.get_netrc_auth') as get_netrc_auth: + # that’s why we patch inside `niquests.sessions`. + with mock.patch('niquests.sessions.get_netrc_auth') as get_netrc_auth: get_netrc_auth.return_value = ('httpie', 'password') r = http(httpbin_both + '/basic-auth/httpie/password') assert get_netrc_auth.call_count == 1 @@ -106,7 +106,7 @@ def test_ignore_netrc(httpbin_both): get_netrc_auth.return_value = ('httpie', 'password') r = http('--ignore-netrc', httpbin_both + '/basic-auth/httpie/password') assert get_netrc_auth.call_count == 0 - assert 'HTTP/1.1 401 UNAUTHORIZED' in r + assert 'HTTP/1.1 401 Unauthorized' in r def test_ignore_netrc_together_with_auth(): diff --git a/tests/test_binary.py b/tests/test_binary.py index ca51aa1686..9e5747ad22 100644 --- a/tests/test_binary.py +++ b/tests/test_binary.py @@ -1,5 +1,5 @@ """Tests for dealing with binary request and response data.""" -import requests +import niquests from .fixtures import BIN_FILE_PATH, BIN_FILE_CONTENT, BIN_FILE_PATH_ARG from httpie.output.streams import BINARY_SUPPRESSED_NOTICE @@ -46,5 +46,5 @@ def test_binary_included_and_correct_when_suitable(self, httpbin): env = MockEnvironment(stdin_isatty=True, stdout_isatty=False) url = httpbin + '/bytes/1024?seed=1' r = http('GET', url, env=env) - expected = requests.get(url).content + expected = niquests.get(url).content assert r == expected diff --git a/tests/test_cli.py b/tests/test_cli.py index 6504c8a980..1cbc46d75a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,7 +2,7 @@ import argparse import pytest -from requests.exceptions import InvalidSchema +from niquests.exceptions import InvalidSchema import httpie.cli.argparser from httpie.cli import constants @@ -134,6 +134,9 @@ def test_multiple_file_fields_with_same_field_name(self): ]) assert len(items.files['file_field']) == 2 + for md in items.multipart_data['file_field']: + md[1].close() + def test_multiple_text_fields_with_same_field_name(self): items = RequestItems.from_args( request_item_args=[ diff --git a/tests/test_cookie.py b/tests/test_cookie.py index c2a9746509..9499119fa9 100644 --- a/tests/test_cookie.py +++ b/tests/test_cookie.py @@ -16,9 +16,19 @@ def setup_mock_server(self, handler): # Start running mock server in a separate thread. # Daemon threads automatically shut down when the main process exits. self.mock_server_thread = Thread(target=self.mock_server.serve_forever) - self.mock_server_thread.setDaemon(True) + self.mock_server_thread.daemon = True self.mock_server_thread.start() + def shutdown_mock_server(self): + if self.mock_server is None: + return + self.mock_server.socket.close() + self.mock_server.shutdown() + self.mock_server_thread.join() + + self.mock_server = None + self.mock_server_port = None + def test_cookie_parser(self): """Not directly testing HTTPie but `requests` to ensure their cookies handling is still as expected by `get_expired_cookies()`. @@ -28,7 +38,7 @@ class MockServerRequestHandler(BaseHTTPRequestHandler): """"HTTP request handler.""" def do_GET(self): - """Handle GET requests.""" + """Handle GET niquests.""" # Craft multiple cookies cookie = SimpleCookie() cookie['hello'] = 'world' @@ -45,3 +55,4 @@ def do_GET(self): response = http(f'http://localhost:{self.mock_server_port}/') assert 'Set-Cookie: hello=world; Path=/' in response assert 'Set-Cookie: oatmeal_raisin="is the best"; Path=/' in response + self.shutdown_mock_server() diff --git a/tests/test_downloads.py b/tests/test_downloads.py index d6e98867bc..180e702d43 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -1,12 +1,12 @@ import os import tempfile import time -import requests +import niquests from unittest import mock from urllib.request import urlopen import pytest -from requests.structures import CaseInsensitiveDict +from niquests.structures import CaseInsensitiveDict from httpie.downloads import ( parse_content_range, filename_from_content_disposition, filename_from_url, @@ -15,7 +15,7 @@ from .utils import http, MockEnvironment -class Response(requests.Response): +class Response(niquests.Response): # noinspection PyDefaultArgument def __init__(self, url, headers={}, status_code=200): self.url = url diff --git a/tests/test_encoding.py b/tests/test_encoding.py index e9f50dc9bb..b16de3c846 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -168,7 +168,7 @@ def test_terminal_output_response_content_type_charset_with_stream(charset, text method=responses.GET, url=DUMMY_URL, body=f'\n{text}'.encode(charset), - stream=True, + # stream=True, content_type=f'text/xml; charset={charset.upper()}', ) r = http('--pretty', pretty, '--stream', DUMMY_URL) diff --git a/tests/test_errors.py b/tests/test_errors.py index fca48fff15..fb9f030dcf 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -3,7 +3,7 @@ from unittest import mock from pytest import raises from requests import Request -from requests.exceptions import ConnectionError +from niquests.exceptions import ConnectionError from httpie.status import ExitStatus from .utils import HTTP_OK, http diff --git a/tests/test_exit_status.py b/tests/test_exit_status.py index 4438d3485c..97f071da54 100644 --- a/tests/test_exit_status.py +++ b/tests/test_exit_status.py @@ -26,7 +26,7 @@ def test_ok_response_exits_0(httpbin): def test_error_response_exits_0_without_check_status(httpbin): r = http('GET', httpbin.url + '/status/500') - assert '500 INTERNAL SERVER ERROR' in r + assert '500 Internal Server Error' in r assert r.exit_status == ExitStatus.SUCCESS assert not r.stderr @@ -44,7 +44,7 @@ def test_3xx_check_status_exits_3_and_stderr_when_stdout_redirected( r = http('--check-status', '--headers', 'GET', httpbin.url + '/status/301', env=env, tolerate_error_exit_status=True) - assert '301 MOVED PERMANENTLY' in r + assert '301 Moved Permanently' in r assert r.exit_status == ExitStatus.ERROR_HTTP_3XX assert '301 moved permanently' in r.stderr.lower() @@ -61,7 +61,7 @@ def test_3xx_check_status_redirects_allowed_exits_0(httpbin): def test_4xx_check_status_exits_4(httpbin): r = http('--check-status', 'GET', httpbin.url + '/status/401', tolerate_error_exit_status=True) - assert '401 UNAUTHORIZED' in r + assert '401 Unauthorized' in r assert r.exit_status == ExitStatus.ERROR_HTTP_4XX # Also stderr should be empty since stdout isn't redirected. assert not r.stderr @@ -70,5 +70,5 @@ def test_4xx_check_status_exits_4(httpbin): def test_5xx_check_status_exits_5(httpbin): r = http('--check-status', 'GET', httpbin.url + '/status/500', tolerate_error_exit_status=True) - assert '500 INTERNAL SERVER ERROR' in r + assert '500 Internal Server Error' in r assert r.exit_status == ExitStatus.ERROR_HTTP_5XX diff --git a/tests/test_httpie.py b/tests/test_httpie.py index 5824340cda..1f0806c721 100644 --- a/tests/test_httpie.py +++ b/tests/test_httpie.py @@ -116,6 +116,15 @@ def test_POST_stdin(httpbin_both): assert FILE_CONTENT in r +def test_empty_stdin(httpbin_both): + env = MockEnvironment( + stdin=io.TextIOWrapper(StdinBytesIO(b"")), + stdin_isatty=False, + ) + r = http(httpbin_both + '/get', env=env) + assert HTTP_OK in r + + def test_POST_file(httpbin_both): r = http('--form', 'POST', httpbin_both + '/post', f'file@{FILE_PATH}') assert HTTP_OK in r diff --git a/tests/test_json.py b/tests/test_json.py index e758ebe7f4..bf1b3857e9 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -338,13 +338,14 @@ def test_complex_json_arguments_with_non_json(httpbin, request_type, value): [ r'foo\[key\]:=1', r'bar\[1\]:=2', - r'baz\[\]:3', + r'baz\[\]:=3', r'quux[key\[escape\]]:=4', r'quux[key 2][\\][\\\\][\\\[\]\\\]\\\[\n\\]:=5', ], { 'foo[key]': 1, 'bar[1]': 2, + 'baz[]': 3, 'quux': { 'key[escape]': 4, 'key 2': {'\\': {'\\\\': {'\\[]\\]\\[\\n\\': 5}}}, diff --git a/tests/test_output.py b/tests/test_output.py index f85f38fa72..ac07bba7fc 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -9,7 +9,7 @@ from urllib.request import urlopen import pytest -import requests +import niquests import responses from httpie.cli.argtypes import ( @@ -97,18 +97,22 @@ def test_quiet_quiet_with_check_status_non_zero_pipe(self, httpbin): (['-q'], 1), (['-qq'], 0), ]) - # Might fail on Windows due to interference from other warnings. - @pytest.mark.xfail def test_quiet_on_python_warnings(self, test_patch, httpbin, flags, expected_warnings): def warn_and_run(*args, **kwargs): warnings.warn('warning!!') return ExitStatus.SUCCESS test_patch.side_effect = warn_and_run - with pytest.warns(None) as record: - http(*flags, httpbin + '/get') - assert len(record) == expected_warnings + if expected_warnings == 0: + with warnings.catch_warnings(): + warnings.simplefilter("error") + http(*flags, httpbin + '/get') + else: + with pytest.warns(Warning) as record: + http(*flags, httpbin + '/get') + + assert len(record) >= expected_warnings def test_double_quiet_on_error(self, httpbin): r = http( @@ -116,7 +120,7 @@ def test_double_quiet_on_error(self, httpbin): tolerate_error_exit_status=True, ) assert not r - assert 'Couldn’t resolve the given hostname' in r.stderr + assert 'Couldn’t resolve the given hostname' in r.stderr or 'Name or service not known' in r.stderr @pytest.mark.parametrize('quiet_flags', QUIET_SCENARIOS) @mock.patch('httpie.cli.argtypes.AuthCredentials._getpass', @@ -160,7 +164,7 @@ def test_quiet_with_output_redirection(self, tmp_path, httpbin, quiet_flags, wit output_path = Path('output.txt') env = MockEnvironment() orig_cwd = os.getcwd() - output = requests.get(url).text + output = niquests.get(url).text extra_args = ['--download'] if with_download else [] os.chdir(tmp_path) try: @@ -214,7 +218,7 @@ def test_verbose_json(self, httpbin): def test_verbose_implies_all(self, httpbin): r = http('--verbose', '--follow', httpbin + '/redirect/1') assert 'GET /redirect/1 HTTP/1.1' in r - assert 'HTTP/1.1 302 FOUND' in r + assert 'HTTP/1.1 302 Found' in r assert 'GET /get HTTP/1.1' in r assert HTTP_OK in r @@ -281,8 +285,14 @@ def test_ensure_status_code_is_shown_on_all_themes(http_server, style, msg): http_server + '/status/msg', '--raw', msg, env=env) + # Custom reason phrase are most likely to disappear, + # due to HTTP/2+ protocols. urllib3.future replace them anyway in HTTP/1.1 + # for uniformity across protocols. + if 'CUSTOM' in msg: + msg = ' OK' + # Trailing space is stripped away. - assert 'HTTP/1.0 200' + msg.rstrip() in strip_colors(r) + assert 'HTTP/1.1 200' + msg.rstrip() in strip_colors(r) class TestPrettyOptions: diff --git a/tests/test_redirects.py b/tests/test_redirects.py index a761fa2571..692bb8ef68 100644 --- a/tests/test_redirects.py +++ b/tests/test_redirects.py @@ -15,7 +15,7 @@ def test_follow_all_redirects_shown(httpbin): r = http('--follow', '--all', httpbin.url + '/redirect/2') assert r.count('HTTP/1.1') == 3 - assert r.count('HTTP/1.1 302 FOUND', 2) + assert r.count('HTTP/1.1 302 Found', 2) assert HTTP_OK in r diff --git a/tests/test_regressions.py b/tests/test_regressions.py index 07d60a583b..7d7f3e66d5 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -30,7 +30,6 @@ def test_output_devnull(httpbin): def test_verbose_redirected_stdout_separator(httpbin): """ - """ r = http( diff --git a/tests/test_ssl.py b/tests/test_ssl.py index 6fb983785a..6a6ba5c86f 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -2,7 +2,7 @@ import pytest import pytest_httpbin.certs -import requests.exceptions +import niquests.exceptions import urllib3 from unittest import mock @@ -10,23 +10,11 @@ from httpie.ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, DEFAULT_SSL_CIPHERS_STRING from httpie.status import ExitStatus -from .utils import HTTP_OK, TESTS_ROOT, IS_PYOPENSSL, http +from .utils import HTTP_OK, TESTS_ROOT, http - -try: - # Handle OpenSSL errors, if installed. - # See - # noinspection PyUnresolvedReferences - import OpenSSL.SSL - ssl_errors = ( - requests.exceptions.SSLError, - OpenSSL.SSL.Error, - ValueError, # TODO: Remove with OSS-65 - ) -except ImportError: - ssl_errors = ( - requests.exceptions.SSLError, - ) +ssl_errors = ( + niquests.exceptions.SSLError, +) CERTS_ROOT = TESTS_ROOT / 'client_certs' CLIENT_CERT = str(CERTS_ROOT / 'client.crt') @@ -59,10 +47,7 @@ def test_ssl_version(httpbin_secure, ssl_version): ) assert HTTP_OK in r except ssl_errors as e: - if ssl_version == 'ssl3': - # pytest-httpbin doesn't support ssl3 - pass - elif e.__context__ is not None: # Check if root cause was an unsupported TLS version + if e.__context__ is not None: # Check if root cause was an unsupported TLS version root = e.__context__ while root.__context__ is not None: root = root.__context__ @@ -151,7 +136,6 @@ def test_ciphers(httpbin_secure): assert HTTP_OK in r -@pytest.mark.skipif(IS_PYOPENSSL, reason='pyOpenSSL uses a different message format.') def test_ciphers_none_can_be_selected(httpbin_secure): r = http( httpbin_secure.url + '/get', @@ -168,15 +152,6 @@ def test_ciphers_none_can_be_selected(httpbin_secure): assert 'cipher' in r.stderr -def test_pyopenssl_presence(): - if not IS_PYOPENSSL: - assert not urllib3.util.ssl_.IS_PYOPENSSL - assert not urllib3.util.IS_PYOPENSSL - else: - assert urllib3.util.ssl_.IS_PYOPENSSL - assert urllib3.util.IS_PYOPENSSL - - @mock.patch('httpie.cli.argtypes.SSLCredentials._prompt_password', new=lambda self, prompt: PWD_CLIENT_PASS) def test_password_protected_cert_prompt(httpbin_secure): diff --git a/tests/test_stream.py b/tests/test_stream.py index 45b8e4dd32..c3b50a758d 100644 --- a/tests/test_stream.py +++ b/tests/test_stream.py @@ -72,7 +72,7 @@ def test_pretty_options_with_and_without_stream_with_converter(pretty, stream): body = b'\x00{"foo":42,\n"bar":"baz"}' responses.add(responses.GET, DUMMY_URL, body=body, - stream=True, content_type='json/bytes') + content_type='json/bytes') args = ['--pretty=' + pretty, 'GET', DUMMY_URL] if stream: diff --git a/tests/test_tokens.py b/tests/test_tokens.py index 655445ce49..7001510074 100644 --- a/tests/test_tokens.py +++ b/tests/test_tokens.py @@ -92,10 +92,10 @@ def test_redirected_headers_multipart_no_separator(): def test_verbose_chunked(httpbin_with_chunked_support): - r = http('--verbose', '--chunked', httpbin_with_chunked_support + '/post', 'hello=world') + r = http('-vv', '--chunked', httpbin_with_chunked_support + '/post', 'hello=world') assert HTTP_OK in r assert 'Transfer-Encoding: chunked' in r - assert_output_matches(r, ExpectSequence.TERMINAL_EXCHANGE) + assert_output_matches(r, ExpectSequence.TERMINAL_EXCHANGE_META) def test_request_headers_response_body(httpbin): @@ -115,4 +115,4 @@ def test_request_double_verbose(httpbin): def test_request_meta(httpbin): r = http('--meta', httpbin + '/get') - assert_output_matches(r, [Expect.RESPONSE_META]) + assert_output_matches(r, [Expect.REQUEST_META, Expect.RESPONSE_META]) diff --git a/tests/test_transport_plugin.py b/tests/test_transport_plugin.py index b71592df8d..5f04ec6203 100644 --- a/tests/test_transport_plugin.py +++ b/tests/test_transport_plugin.py @@ -1,8 +1,8 @@ from io import BytesIO -from requests.adapters import BaseAdapter -from requests.models import Response -from requests.utils import get_encoding_from_headers +from niquests.adapters import BaseAdapter +from niquests.models import Response +from niquests.utils import get_encoding_from_headers from httpie.plugins import TransportPlugin from httpie.plugins.registry import plugin_manager diff --git a/tests/test_update_warnings.py b/tests/test_update_warnings.py index b2c24c36de..36e2596355 100644 --- a/tests/test_update_warnings.py +++ b/tests/test_update_warnings.py @@ -213,7 +213,7 @@ def fetch_update_mock(mocker): @pytest.fixture def static_fetch_data(mocker): - mock_get = mocker.patch('requests.get') + mock_get = mocker.patch('niquests.get') mock_get.return_value.status_code = 200 mock_get.return_value.json.return_value = { BUILD_CHANNEL: HIGHEST_VERSION, diff --git a/tests/test_uploads.py b/tests/test_uploads.py index d0156063d4..055b6d2a21 100644 --- a/tests/test_uploads.py +++ b/tests/test_uploads.py @@ -4,6 +4,9 @@ import subprocess import time import contextlib + +from flaky import flaky + import httpie.__main__ as main import pytest @@ -125,6 +128,7 @@ def stdin_processes(httpbin, *args, warn_threshold=0.1): @pytest.mark.parametrize("wait", (True, False)) @pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") +@flaky(max_runs=6) def test_reading_from_stdin(httpbin, wait): with stdin_processes(httpbin) as (process_1, process_2): process_1.communicate(timeout=0.1, input=b"bleh") @@ -143,6 +147,7 @@ def test_reading_from_stdin(httpbin, wait): @pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") +@flaky(max_runs=6) def test_stdin_read_warning(httpbin): with stdin_processes(httpbin) as (process_1, process_2): # Wait before sending any data @@ -159,6 +164,7 @@ def test_stdin_read_warning(httpbin): @pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") +@flaky(max_runs=6) def test_stdin_read_warning_with_quiet(httpbin): with stdin_processes(httpbin, "-qq") as (process_1, process_2): # Wait before sending any data @@ -175,6 +181,7 @@ def test_stdin_read_warning_with_quiet(httpbin): @pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") +@flaky(max_runs=6) def test_stdin_read_warning_blocking_exit(httpbin): # Use a very large number. with stdin_processes(httpbin, warn_threshold=999) as (process_1, process_2): @@ -284,7 +291,7 @@ def test_multipart_custom_content_type_boundary_added(self, httpbin): assert r.count(boundary) == 4 def test_multipart_custom_content_type_boundary_preserved(self, httpbin): - # Allow explicit nonsense requests. + # Allow explicit nonsense niquests. boundary_in_header = 'HEADER_BOUNDARY' boundary_in_body = 'BODY_BOUNDARY' r = http( diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index ada0905ff2..63d7668ad2 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,17 +1,16 @@ """Utilities for HTTPie test suite.""" import re import shlex -import os import sys import time import json import tempfile -import warnings import pytest from contextlib import suppress from io import BytesIO from pathlib import Path from typing import Any, Optional, Union, List, Iterable +from shutil import rmtree import httpie.core as core import httpie.manager.__main__ as manager @@ -31,8 +30,6 @@ HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN = 'pie.dev' HTTPBIN_WITH_CHUNKED_SUPPORT = 'http://' + HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN -IS_PYOPENSSL = os.getenv('HTTPIE_TEST_WITH_PYOPENSSL', '0') == '1' - TESTS_ROOT = Path(__file__).parent.parent CRLF = '\r\n' COLOR = '\x1b[' @@ -125,6 +122,11 @@ class StdinBytesIO(BytesIO): """To be used for `MockEnvironment.stdin`""" len = 0 # See `prepare_request_body()` + def peek(self, size): + buf = self.read(size) + self.seek(0) + return buf + class MockEnvironment(Environment): """Environment subclass with reasonable defaults for testing.""" @@ -139,7 +141,7 @@ def __init__(self, create_temp_config_dir=True, **kwargs): if 'stdout' not in kwargs: kwargs['stdout'] = tempfile.NamedTemporaryFile( mode='w+t', - prefix='httpie_stderr', + prefix='httpie_stdout', newline='', encoding=UTF8, ) @@ -170,10 +172,15 @@ def cleanup(self): self.devnull.close() self.stdout.close() self.stderr.close() - warnings.resetwarnings() + if self._orig_stdout and self._orig_stdout != self.stdout: + self._orig_stdout.close() + if self._orig_stderr and self.stderr != self._orig_stderr: + self._orig_stderr.close() + self.devnull.close() + # it breaks without reasons pytest filterwarnings + # warnings.resetwarnings() if self._delete_config_dir: assert self._temp_dir in self.config_dir.parents - from shutil import rmtree rmtree(self.config_dir, ignore_errors=True) def __del__(self): @@ -210,7 +217,7 @@ class BaseCLIResponse: complete_args: List[str] = [] @property - def command(self): + def command(self): # noqa: F811 cmd = ' '.join(shlex.quote(arg) for arg in ['http', *self.args]) # pytest-httpbin to real httpbin. return re.sub(r'127\.0\.0\.1:\d+', 'httpbin.org', cmd) diff --git a/tests/utils/http_server.py b/tests/utils/http_server.py index 86cc069c57..728946f555 100644 --- a/tests/utils/http_server.py +++ b/tests/utils/http_server.py @@ -135,7 +135,8 @@ def _http_server(): thread = threading.Thread(target=server.serve_forever) thread.start() yield server - server.shutdown() + server.socket.close() + server.shutdown() # shutdown seems only to stop the thread, not closing the socket. thread.join() diff --git a/tests/utils/matching/parsing.py b/tests/utils/matching/parsing.py index e502d76bc8..b574aa2395 100644 --- a/tests/utils/matching/parsing.py +++ b/tests/utils/matching/parsing.py @@ -8,6 +8,7 @@ SEPARATOR_RE = re.compile(f'^{MESSAGE_SEPARATOR}') KEY_VALUE_RE = re.compile(r'[\n]*((.*?):(.+)[\n]?)+[\n]*') +KEY_VALUE_RE_NO_LF = re.compile(r'((.*?):(.+)(\n))+(\n)') def make_headers_re(message_type: Expect): @@ -18,7 +19,7 @@ def make_headers_re(message_type: Expect): non_crlf = rf'[^{CRLF}]' # language=RegExp - http_version = r'HTTP/\d+\.\d+' + http_version = r'HTTP/((\d+\.\d+)|\d+)' if message_type is Expect.REQUEST_HEADERS: # POST /post HTTP/1.1 start_line_re = fr'{non_crlf}*{http_version}{crlf}' @@ -42,6 +43,7 @@ def make_headers_re(message_type: Expect): CRLF, # Not really but useful for testing (just remember not to include it in a body). ] TOKEN_REGEX_MAP = { + Expect.REQUEST_META: KEY_VALUE_RE_NO_LF, Expect.REQUEST_HEADERS: make_headers_re(Expect.REQUEST_HEADERS), Expect.RESPONSE_HEADERS: make_headers_re(Expect.RESPONSE_HEADERS), Expect.RESPONSE_META: KEY_VALUE_RE, @@ -56,6 +58,7 @@ class OutputMatchingError(ValueError): def expect_tokens(tokens: Iterable[Expect], s: str): for token in tokens: s = expect_token(token, s) + # print(token, "OK") if s: raise OutputMatchingError(f'Unmatched remaining output for {tokens} in {s!r}') diff --git a/tests/utils/matching/tokens.py b/tests/utils/matching/tokens.py index c82dafedc2..1dfe7d0c57 100644 --- a/tests/utils/matching/tokens.py +++ b/tests/utils/matching/tokens.py @@ -6,6 +6,7 @@ class Expect(Enum): Predefined token types we can expect in the output. """ + REQUEST_META = auto() REQUEST_HEADERS = auto() RESPONSE_HEADERS = auto() RESPONSE_META = auto() @@ -47,6 +48,7 @@ class ExpectSequence: *TERMINAL_RESPONSE, ] TERMINAL_EXCHANGE_META = [ + Expect.REQUEST_META, *TERMINAL_EXCHANGE, Expect.RESPONSE_META ] From bb1acd9eb22da7c366bdd0014a7e4155dcb2b35a Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Mon, 18 Mar 2024 18:35:46 +0100 Subject: [PATCH 02/63] resolve merge conflicts --- setup.cfg | 1 - tests/test_cli.py | 8 ++++---- tests/test_uploads.py | 6 ------ 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/setup.cfg b/setup.cfg index d195027e65..fa95a47458 100644 --- a/setup.cfg +++ b/setup.cfg @@ -98,7 +98,6 @@ dev = flake8-deprecated flake8-mutable flake8-tuple - pyopenssl pytest-cov pyyaml twine diff --git a/tests/test_cli.py b/tests/test_cli.py index b431c6f1ce..59a03940e8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,7 +2,7 @@ import argparse import pytest -from niquests.exceptions import InvalidSchema +from niquests.exceptions import InvalidSchema, MissingSchema import httpie.cli.argparser from httpie.cli import constants @@ -363,13 +363,13 @@ def test_invalid_custom_scheme(self): # InvalidSchema is expected because HTTPie # shouldn't touch a formally valid scheme. with pytest.raises(InvalidSchema): - http('foo+bar-BAZ.123://bah') + http('foo+bar://bah') def test_invalid_scheme_via_via_default_scheme(self): # InvalidSchema is expected because HTTPie # shouldn't touch a formally valid scheme. - with pytest.raises(InvalidSchema): - http('bah', '--default=scheme=foo+bar-BAZ.123') + with pytest.raises((InvalidSchema, MissingSchema,)): + http('bah', '--default=scheme=foo+bar') def test_default_scheme_option(self, httpbin_secure): url = f'{httpbin_secure.host}:{httpbin_secure.port}' diff --git a/tests/test_uploads.py b/tests/test_uploads.py index 7cf98636ed..18b8558965 100644 --- a/tests/test_uploads.py +++ b/tests/test_uploads.py @@ -5,8 +5,6 @@ import time import contextlib -from flaky import flaky - import httpie.__main__ as main import pytest @@ -128,7 +126,6 @@ def stdin_processes(httpbin, *args, warn_threshold=0.1): @pytest.mark.parametrize("wait", (True, False)) @pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") -@flaky(max_runs=6) def test_reading_from_stdin(httpbin, wait): with stdin_processes(httpbin) as (process_1, process_2): process_1.communicate(timeout=0.1, input=b"bleh") @@ -147,7 +144,6 @@ def test_reading_from_stdin(httpbin, wait): @pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") -@flaky(max_runs=6) def test_stdin_read_warning(httpbin): with stdin_processes(httpbin) as (process_1, process_2): # Wait before sending any data @@ -164,7 +160,6 @@ def test_stdin_read_warning(httpbin): @pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") -@flaky(max_runs=6) def test_stdin_read_warning_with_quiet(httpbin): with stdin_processes(httpbin, "-qq") as (process_1, process_2): # Wait before sending any data @@ -181,7 +176,6 @@ def test_stdin_read_warning_with_quiet(httpbin): @pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") -@flaky(max_runs=6) def test_stdin_read_warning_blocking_exit(httpbin): # Use a very large number. with stdin_processes(httpbin, warn_threshold=999) as (process_1, process_2): From 7be49cfa34d3d3e73ab276c9d0453c8ca04c6d98 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Mon, 18 Mar 2024 18:38:57 +0100 Subject: [PATCH 03/63] flake8 lint fix --- tests/conftest.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ec332faff0..6ddc696e13 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,10 @@ import pytest from pytest_httpbin import certs from pytest_httpbin.serve import Server as PyTestHttpBinServer +from sys import modules + +import niquests +import urllib3 from .utils import ( # noqa HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, @@ -24,12 +28,10 @@ # Patch to support `url = str(server)` in addition to `url = server + '/foo'`. PyTestHttpBinServer.__str__ = lambda self: self.url -from sys import modules - -import niquests -import urllib3 - # the mock utility 'response' only works with 'requests' +# we're trying to fool it, thinking requests is there. +# to remove when a similar (or same, but compatible) +# utility emerge for Niquests. modules["requests"] = niquests modules["requests.adapters"] = niquests.adapters modules["requests.exceptions"] = niquests.exceptions From 4f3b46897020a983cf5289f74949fb0b85f2a07d Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 20 Mar 2024 06:45:56 +0100 Subject: [PATCH 04/63] code cleanup --- docs/README.md | 68 ++++++++++++++++++------------ httpie/internal/update_warnings.py | 2 - httpie/ssl_.py | 9 ---- tests/test_auth.py | 2 +- tests/test_cookie.py | 2 +- tests/test_encoding.py | 1 - tests/test_uploads.py | 2 +- tests/utils/matching/parsing.py | 1 - 8 files changed, 43 insertions(+), 44 deletions(-) diff --git a/docs/README.md b/docs/README.md index 7c458bc275..c3133ddff8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1599,18 +1599,12 @@ $ http --meta pie.dev/delay/1 ``` ```console -Connected to: 2a06:98c1:3120::2 port 443 -Connection secured using: TLSv1.3 with AES-256-GCM-SHA384 -Server certificate: commonName="pie.dev"; DNS="*.pie.dev"; DNS="pie.dev" -Certificate validity: "Nov 11 01:14:24 2023 UTC" to "Feb 09 01:14:23 2024 UTC" -Issuer: countryName="US"; organizationName="Let's Encrypt"; commonName="E1" -Revocation status: Good +Connected to: 2a06:98c1:3120::2 port 80 -Elapsed DNS: 0.11338s -Elapsed established connection: 3.8e-05s -Elapsed TLS handshake: 0.057503s -Elapsed emitting request: 0.000275s -Elapsed time: 0.292854214s +Elapsed DNS: 0.047945s +Elapsed established connection: 0.013063s +Elapsed emitting request: 0.000115s +Elapsed time: 1.1325035701s ``` The [extra verbose `-vv` output](#extra-verbose-output) includes the meta section by default. You can also show it in combination with other parts of the exchange via [`--print=m`](#what-parts-of-the-http-exchange-should-be-printed). For example, here we print it together with the response headers: @@ -1623,18 +1617,26 @@ $ https --print=hm pie.dev/get Connected to: 2a06:98c1:3120::2 port 443 Connection secured using: TLSv1.3 with AES-256-GCM-SHA384 Server certificate: commonName="pie.dev"; DNS="*.pie.dev"; DNS="pie.dev" -Certificate validity: "Nov 11 01:14:24 2023 UTC" to "Feb 09 01:14:23 2024 UTC" +Certificate validity: "Mar 08 00:16:33 2024 UTC" to "Jun 06 00:16:32 2024 UTC" Issuer: countryName="US"; organizationName="Let's Encrypt"; commonName="E1" Revocation status: Good -HTTP/2 200 OK +HTTP/3 200 OK +Access-Control-Allow-Credentials: true +Access-Control-Allow-Origin: * +Alt-Svc: h3=":443"; ma=86400 +Cf-Cache-Status: DYNAMIC +Cf-Ray: 867351f9cf37d4fe-CDG +Content-Encoding: br Content-Type: application/json +Date: Wed, 20 Mar 2024 05:32:11 GMT +Server: cloudflare -Elapsed DNS: 0.11338s -Elapsed established connection: 3.8e-05s -Elapsed TLS handshake: 0.057503s -Elapsed emitting request: 0.000275s -Elapsed time: 0.292854214s +Elapsed DNS: 0.000682s +Elapsed established connection: 1.7e-05s +Elapsed TLS handshake: 0.043641s +Elapsed emitting request: 0.000397s +Elapsed time: 0.1677905799s ``` @@ -1876,7 +1878,7 @@ $ https --http3 pie.dev/get ``` By default, HTTPie cannot negotiate HTTP/3 without a first HTTP/1.1, or HTTP/2 successful response unless the -remote host specified a DNS HTTPS record that indicate its support. +remote host specified a DNS HTTPS record that indicate its support (and by using a custom DNS resolver, see bellow section). The remote server yield its support for HTTP/3 in the Alt-Svc header, if present HTTPie will issue the successive requests via HTTP/3. You may use that argument in case the remote peer does not support @@ -1893,7 +1895,7 @@ presented order to resolver given hostname. $ https --resolver "doh+cloudflare://" pie.dev/get ``` -To know more about DNS url and supported protocols, visit [Niquests documentation](https://niquests.readthedocs.io/en/stable/user/quickstart.html#dns-resolution). +To know more about DNS url and supported protocols, visit the [Niquests documentation](https://niquests.readthedocs.io/en/stable/user/quickstart.html#dns-resolution). ### Forcing hostname to resolve with a manual entry @@ -1928,13 +1930,13 @@ $ https --interface 172.17.0.1 pie.dev/get ## Local port -You can choose to select the outgoing port manually by passing the `--local-port` flag. +You can choose to select the outgoing (local) network port manually by passing the `--local-port` flag. ```bash $ https --local-port 5411 pie.dev/get ``` -or using a range. +or using a range. (bellow example will pick a free port between 5000 and 10000) ```bash $ https --local-port 5000-10000 pie.dev/get @@ -2123,13 +2125,23 @@ $ http --download https://github.com/httpie/cli/archive/master.tar.gz ``` ```http -HTTP/1.1 200 OK -Content-Disposition: attachment; filename=httpie-master.tar.gz -Content-Length: 257336 +HTTP/2 200 OK +Access-Control-Allow-Origin: https://render.githubusercontent.com +Content-Disposition: attachment; filename=cli-master.tar.gz +Content-Security-Policy: default-src 'none'; style-src 'unsafe-inline'; sandbox Content-Type: application/x-gzip - -Downloading 251.30 kB to "httpie-master.tar.gz" -Done. 251.30 kB in 2.73862s (91.76 kB/s) +Cross-Origin-Resource-Policy: cross-origin +Date: Wed, 20 Mar 2024 05:37:32 GMT +Etag: "68a7a50930daac494551b4576eef285f86da741c9b4a0f3a1deeac9fce4e80d4" +Strict-Transport-Security: max-age=31536000 +Vary: Authorization,Accept-Encoding,Origin +X-Content-Type-Options: nosniff +X-Frame-Options: deny +X-Github-Request-Id: 9E06:20C449:55A32E:636ADA:65FA761C +X-Xss-Protection: 1; mode=block + +Downloading to cli-master.tar.gz +Done. 1.3 MB in 00:0.54267 (2.4 MB/s) ``` ### Downloaded filename diff --git a/httpie/internal/update_warnings.py b/httpie/internal/update_warnings.py index cc698660c3..c684bb80ad 100644 --- a/httpie/internal/update_warnings.py +++ b/httpie/internal/update_warnings.py @@ -41,8 +41,6 @@ def _fetch_updates(env: Environment) -> str: file = env.config.version_info_file data = _read_data_error_free(file) - print("fetch update...?") - response = niquests.get(PACKAGE_INDEX_LINK, verify=False) response.raise_for_status() diff --git a/httpie/ssl_.py b/httpie/ssl_.py index 937b3eab70..06f9180319 100644 --- a/httpie/ssl_.py +++ b/httpie/ssl_.py @@ -4,8 +4,6 @@ import os.path from os import makedirs -from niquests import Response - from httpie.config import DEFAULT_CONFIG_DIR from httpie.adapters import HTTPAdapter # noinspection PyPackageRequirements @@ -146,13 +144,6 @@ def _create_ssl_context( def get_default_ciphers_names(cls): return [cipher['name'] for cipher in cls._create_ssl_context(verify=False).get_ciphers()] - def send( - self, - *args, - **kwargs - ) -> Response: - return super().send(*args, **kwargs) - def _is_key_file_encrypted(key_file): """Detects if a key file is encrypted or not. diff --git a/tests/test_auth.py b/tests/test_auth.py index 8a11554b0a..1b72f90043 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -92,7 +92,7 @@ def test_missing_auth(httpbin): def test_netrc(httpbin_both): - # This one gets handled by requests (no --auth, --auth-type present), + # This one gets handled by niquests (no --auth, --auth-type present), # that’s why we patch inside `niquests.sessions`. with mock.patch('niquests.sessions.get_netrc_auth') as get_netrc_auth: get_netrc_auth.return_value = ('httpie', 'password') diff --git a/tests/test_cookie.py b/tests/test_cookie.py index 9499119fa9..3d0f96dfd5 100644 --- a/tests/test_cookie.py +++ b/tests/test_cookie.py @@ -38,7 +38,7 @@ class MockServerRequestHandler(BaseHTTPRequestHandler): """"HTTP request handler.""" def do_GET(self): - """Handle GET niquests.""" + """Handle GET requests.""" # Craft multiple cookies cookie = SimpleCookie() cookie['hello'] = 'world' diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 5afcc23f07..cfd2183f21 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -168,7 +168,6 @@ def test_terminal_output_response_content_type_charset_with_stream(charset, text method=responses.GET, url=DUMMY_URL, body=f'\n{text}'.encode(charset), - # stream=True, content_type=f'text/xml; charset={charset.upper()}', ) r = http('--pretty', pretty, '--stream', DUMMY_URL) diff --git a/tests/test_uploads.py b/tests/test_uploads.py index 18b8558965..e4723d6f6b 100644 --- a/tests/test_uploads.py +++ b/tests/test_uploads.py @@ -285,7 +285,7 @@ def test_multipart_custom_content_type_boundary_added(self, httpbin): assert r.count(boundary) == 4 def test_multipart_custom_content_type_boundary_preserved(self, httpbin): - # Allow explicit nonsense niquests. + # Allow explicit nonsense requests. boundary_in_header = 'HEADER_BOUNDARY' boundary_in_body = 'BODY_BOUNDARY' r = http( diff --git a/tests/utils/matching/parsing.py b/tests/utils/matching/parsing.py index b574aa2395..642d5d6b56 100644 --- a/tests/utils/matching/parsing.py +++ b/tests/utils/matching/parsing.py @@ -58,7 +58,6 @@ class OutputMatchingError(ValueError): def expect_tokens(tokens: Iterable[Expect], s: str): for token in tokens: s = expect_token(token, s) - # print(token, "OK") if s: raise OutputMatchingError(f'Unmatched remaining output for {tokens} in {s!r}') From 793e3732afd7bc3de5c9705b9466638dadab623a Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 20 Mar 2024 07:36:22 +0100 Subject: [PATCH 05/63] fix warnings in PersistentMockEnvironment due to missing cleanup --- tests/test_update_warnings.py | 6 ++++-- tests/utils/__init__.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/test_update_warnings.py b/tests/test_update_warnings.py index 60a1120c08..af528727ff 100644 --- a/tests/test_update_warnings.py +++ b/tests/test_update_warnings.py @@ -197,7 +197,8 @@ def with_warnings(tmp_path): env = PersistentMockEnvironment() env.config['version_info_file'] = tmp_path / 'version.json' env.config['disable_update_warnings'] = False - return env + yield env + env.cleanup(force=True) @pytest.fixture @@ -205,7 +206,8 @@ def without_warnings(tmp_path): env = PersistentMockEnvironment() env.config['version_info_file'] = tmp_path / 'version.json' env.config['disable_update_warnings'] = True - return env + yield env + env.cleanup(force=True) @pytest.fixture diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 43839dbbea..962dfc5aec 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -192,8 +192,16 @@ def __del__(self): class PersistentMockEnvironment(MockEnvironment): - def cleanup(self): - pass + def cleanup(self, *, force: bool = False): + if force: + self.devnull.close() + self.stdout.close() + self.stderr.close() + if self._orig_stdout and self._orig_stdout != self.stdout: + self._orig_stdout.close() + if self._orig_stderr and self.stderr != self._orig_stderr: + self._orig_stderr.close() + self.devnull.close() class BaseCLIResponse: From 417adcd818b223c56fa5a19cebe05ee83d49f40f Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 20 Mar 2024 07:37:12 +0100 Subject: [PATCH 06/63] fix warnings in TestItemParsing::test_escape_separator due to an unclosed file --- tests/test_cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 59a03940e8..ad0e52b8e7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -51,6 +51,9 @@ def test_escape_separator(self): } assert 'bar@baz' in items.files + # ensure we close the fixture file + items.multipart_data['bar@baz'][1].close() + @pytest.mark.parametrize('string, key, sep, value', [ ('path=c:\\windows', 'path', '=', 'c:\\windows'), ('path=c:\\windows\\', 'path', '=', 'c:\\windows\\'), From 3ccbdbf48487830a21e108861c3a519b0c2ff6d9 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 20 Mar 2024 07:37:44 +0100 Subject: [PATCH 07/63] fix warnings due leaving the http session open --- httpie/client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/httpie/client.py b/httpie/client.py index 72c9964c3a..d5c2f6956d 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -175,6 +175,11 @@ def collect_messages( httpie_session.remove_cookies(expired_cookies) httpie_session.save() + try: + requests_session.close() + except NotImplementedError: + pass + def build_requests_session( verify: bool, From d39752a02c6c5528c870f4f299c162cb8d95bf22 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 20 Mar 2024 07:38:02 +0100 Subject: [PATCH 08/63] add tests to cover recently added cli flags --- tests/conftest.py | 7 ++++++ tests/test_h2n3.py | 34 ++++++++++++++++++++++++++++++ tests/test_meta.py | 16 +++++++++++++- tests/test_network.py | 48 ++++++++++++++++++++++++++++++++++++++++++ tests/test_resolver.py | 25 ++++++++++++++++++++++ 5 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 tests/test_h2n3.py create mode 100644 tests/test_network.py create mode 100644 tests/test_resolver.py diff --git a/tests/conftest.py b/tests/conftest.py index 6ddc696e13..aac94e3269 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -90,3 +90,10 @@ def remote_httpbin(_remote_httpbin_available): if _remote_httpbin_available: return 'http://' + REMOTE_HTTPBIN_DOMAIN pytest.skip(f'{REMOTE_HTTPBIN_DOMAIN} not resolvable') + + +@pytest.fixture +def remote_httpbin_secure(_remote_httpbin_available): + if _remote_httpbin_available: + return 'https://' + REMOTE_HTTPBIN_DOMAIN + pytest.skip(f'{REMOTE_HTTPBIN_DOMAIN} not resolvable') diff --git a/tests/test_h2n3.py b/tests/test_h2n3.py new file mode 100644 index 0000000000..a9563aa916 --- /dev/null +++ b/tests/test_h2n3.py @@ -0,0 +1,34 @@ +from .utils import HTTP_OK, http + + +def test_should_not_do_http1_by_default(remote_httpbin_secure): + r = http( + "--verify=no", + remote_httpbin_secure + '/get' + ) + + assert "HTTP/1" not in r + assert HTTP_OK in r + + +def test_disable_http2n3(remote_httpbin_secure): + r = http( + "--verify=no", + '--disable-http2', + '--disable-http3', + remote_httpbin_secure + '/get' + ) + + assert "HTTP/1.1" in r + assert HTTP_OK in r + + +def test_force_http3(remote_httpbin_secure): + r = http( + "--verify=no", + '--http3', + remote_httpbin_secure + '/get' + ) + + assert "HTTP/3" in r + assert HTTP_OK in r diff --git a/tests/test_meta.py b/tests/test_meta.py index a57b510f0d..66b70dd1ba 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -1,6 +1,6 @@ import pytest -from httpie.models import ELAPSED_TIME_LABEL +from httpie.models import ELAPSED_TIME_LABEL, ELAPSED_DNS_RESOLUTION_LABEL, ELAPSED_ESTABLISH_CONN, ELAPSED_REQUEST_SEND, ELAPSED_TLS_HANDSHAKE from httpie.output.formatters.colors import PIE_STYLE_NAMES from .utils import http, MockEnvironment, COLOR @@ -8,6 +8,20 @@ def test_meta_elapsed_time(httpbin): r = http('--meta', httpbin + '/delay/1') assert f'{ELAPSED_TIME_LABEL}: 1.' in r + assert ELAPSED_DNS_RESOLUTION_LABEL in r + assert ELAPSED_ESTABLISH_CONN in r + assert ELAPSED_REQUEST_SEND in r + + +def test_meta_extended_tls(remote_httpbin_secure): + r = http('--verify=no', '--meta', remote_httpbin_secure + '/get') + assert 'Connected to' in r + assert 'Connection secured using' in r + assert 'Server certificate' in r + assert 'Certificate validity' in r + assert 'Issuer' in r + assert 'Revocation status' in r + assert ELAPSED_TLS_HANDSHAKE in r @pytest.mark.parametrize('style', ['auto', 'fruity', *PIE_STYLE_NAMES]) diff --git a/tests/test_network.py b/tests/test_network.py new file mode 100644 index 0000000000..1769a40a93 --- /dev/null +++ b/tests/test_network.py @@ -0,0 +1,48 @@ +from .utils import HTTP_OK, http + + +def test_ensure_interface_parameter(httpbin): + """We ensure that HTTPie properly wire interface by passing an interface that + does not exist. thus, we expect an error.""" + r = http( + "--interface=1.1.1.1", + httpbin + "/get", + tolerate_error_exit_status=True + ) + + assert "Cannot assign requested address" in r.stderr + + +def test_ensure_local_port_parameter(httpbin): + """We ensure that HTTPie properly wire local-port by passing a port that + require elevated privilege. thus, we expect an error.""" + r = http( + "--local-port=89", + httpbin + "/get", + tolerate_error_exit_status=True + ) + + assert "Permission denied" in r.stderr + + +def test_ensure_interface_and_port_parameters(httpbin): + r = http( + "--interface=0.0.0.0", # it's valid, setting 0.0.0.0 means "take the default" here. + "--local-port=0", # this will automatically pick a free port in range 1024-65535 + httpbin + "/get", + ) + + assert HTTP_OK in r + + +def test_ensure_ipv6_toggle_parameter(httpbin): + """This test is made to ensure we are effectively passing the IPv6-only flag to + Niquests. Our test server listen exclusively on IPv4.""" + r = http( + "-6", + httpbin + "/get", + tolerate_error_exit_status=True, + ) + + assert r.exit_status != 0 + assert "Name or service not known" in r.stderr diff --git a/tests/test_resolver.py b/tests/test_resolver.py new file mode 100644 index 0000000000..9f634747e1 --- /dev/null +++ b/tests/test_resolver.py @@ -0,0 +1,25 @@ +from .utils import http + + +def test_ensure_resolver_used(remote_httpbin_secure): + """This test ensure we're using specified resolver to get into pie.dev. + Using a custom resolver with Niquests enable direct HTTP/3 negotiation and pie.dev + (DNS) is handled by Cloudflare (NS) services.""" + r = http( + "--verify=no", + "--resolver=doh+cloudflare://", + remote_httpbin_secure + "/get" + ) + + assert "HTTP/3" in r + + +def test_ensure_override_resolver_used(remote_httpbin): + """Just an additional check to ensure we are wired properly to Niquests resolver parameter.""" + r = http( + "--resolver=pie.dev:240.0.0.0", # override DNS response to TARPIT net addr. + remote_httpbin + "/get", + tolerate_error_exit_status=True + ) + + assert "Request timed out" in r.stderr From f025f66835f79206d27ea0f5d68008d58ddd06d6 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 20 Mar 2024 07:54:36 +0100 Subject: [PATCH 09/63] fix multipart stream encoder not closing (file) part after completing them --- httpie/internal/encoder.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/httpie/internal/encoder.py b/httpie/internal/encoder.py index 5da4dd8119..714f0ab3c1 100644 --- a/httpie/internal/encoder.py +++ b/httpie/internal/encoder.py @@ -215,6 +215,8 @@ def _load(self, amount): def _next_part(self): try: + if self._current_part is not None: + self._current_part.close() p = self._current_part = next(self._iter_parts) except StopIteration: p = None @@ -334,6 +336,12 @@ def __init__(self, headers, body): self.headers_unread = True self.len = len(self.headers) + total_len(self.body) + def close(self): + if hasattr(self.body, "fd") and hasattr(self.body.fd, "close"): + self.body.fd.close() + elif hasattr(self.body, "close"): + self.body.close() + @classmethod def from_field(cls, field, encoding): """Create a part from a Request Field generated by urllib3.""" From 1ff2be7ac506cafee42dbad21ec5169cac2e243c Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 20 Mar 2024 07:55:02 +0100 Subject: [PATCH 10/63] fix warning about unclosed file in TestItemParsing::test_valid_items --- tests/test_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index ad0e52b8e7..63c485d525 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -130,6 +130,8 @@ def test_valid_items(self): assert (items.files['file'][1].read().strip(). decode() == FILE_CONTENT) + items.files['file'][1].close() + def test_multiple_file_fields_with_same_field_name(self): items = RequestItems.from_args([ self.key_value_arg('file_field@' + FILE_PATH_ARG), From 803748538bb913af994c44aa6325fa7ca21ae51e Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 20 Mar 2024 08:05:26 +0100 Subject: [PATCH 11/63] adapt tests in OS specifics behaviors (err msg) --- tests/test_meta.py | 6 +----- tests/test_network.py | 24 +++++++----------------- tests/test_resolver.py | 2 +- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/tests/test_meta.py b/tests/test_meta.py index 66b70dd1ba..e0912958c6 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -1,6 +1,6 @@ import pytest -from httpie.models import ELAPSED_TIME_LABEL, ELAPSED_DNS_RESOLUTION_LABEL, ELAPSED_ESTABLISH_CONN, ELAPSED_REQUEST_SEND, ELAPSED_TLS_HANDSHAKE +from httpie.models import ELAPSED_TIME_LABEL from httpie.output.formatters.colors import PIE_STYLE_NAMES from .utils import http, MockEnvironment, COLOR @@ -8,9 +8,6 @@ def test_meta_elapsed_time(httpbin): r = http('--meta', httpbin + '/delay/1') assert f'{ELAPSED_TIME_LABEL}: 1.' in r - assert ELAPSED_DNS_RESOLUTION_LABEL in r - assert ELAPSED_ESTABLISH_CONN in r - assert ELAPSED_REQUEST_SEND in r def test_meta_extended_tls(remote_httpbin_secure): @@ -21,7 +18,6 @@ def test_meta_extended_tls(remote_httpbin_secure): assert 'Certificate validity' in r assert 'Issuer' in r assert 'Revocation status' in r - assert ELAPSED_TLS_HANDSHAKE in r @pytest.mark.parametrize('style', ['auto', 'fruity', *PIE_STYLE_NAMES]) diff --git a/tests/test_network.py b/tests/test_network.py index 1769a40a93..0be34d04d5 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -10,19 +10,21 @@ def test_ensure_interface_parameter(httpbin): tolerate_error_exit_status=True ) - assert "Cannot assign requested address" in r.stderr + assert r.exit_status != 0 + assert "assign requested address" in r.stderr or "The requested address is not valid in its context" in r.stderr def test_ensure_local_port_parameter(httpbin): """We ensure that HTTPie properly wire local-port by passing a port that - require elevated privilege. thus, we expect an error.""" + does not exist. thus, we expect an error.""" r = http( - "--local-port=89", + "--local-port=70000", httpbin + "/get", tolerate_error_exit_status=True ) - assert "Permission denied" in r.stderr + assert r.exit_status != 0 + assert "port must be 0-65535" in r.stderr def test_ensure_interface_and_port_parameters(httpbin): @@ -32,17 +34,5 @@ def test_ensure_interface_and_port_parameters(httpbin): httpbin + "/get", ) + assert r.exit_status == 0 assert HTTP_OK in r - - -def test_ensure_ipv6_toggle_parameter(httpbin): - """This test is made to ensure we are effectively passing the IPv6-only flag to - Niquests. Our test server listen exclusively on IPv4.""" - r = http( - "-6", - httpbin + "/get", - tolerate_error_exit_status=True, - ) - - assert r.exit_status != 0 - assert "Name or service not known" in r.stderr diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 9f634747e1..06bc1a104c 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -22,4 +22,4 @@ def test_ensure_override_resolver_used(remote_httpbin): tolerate_error_exit_status=True ) - assert "Request timed out" in r.stderr + assert "Request timed out" in r.stderr or "A socket operation was attempted to an unreachable network" in r.stderr From 7eb10b5146e4b11c5dd7a68a07226af5a397e551 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 20 Mar 2024 08:48:13 +0100 Subject: [PATCH 12/63] add comments to guide future developers --- docs/README.md | 8 ++++---- httpie/cli/argparser.py | 3 ++- httpie/client.py | 6 ++++++ httpie/core.py | 6 ++++++ httpie/internal/encoder.py | 2 ++ httpie/models.py | 4 ++++ httpie/output/streams.py | 2 +- httpie/ssl_.py | 12 +++++++++++- 8 files changed, 36 insertions(+), 7 deletions(-) diff --git a/docs/README.md b/docs/README.md index c3133ddff8..6fcb3012cd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1530,13 +1530,13 @@ $ http --cert=client.pem --cert-key=client.key --cert-key-pass=my_password https ### SSL version Use the `--ssl=` option to specify the desired protocol version to use. -This will default to SSL v2.3 which will negotiate the highest protocol that both the server and your installation of OpenSSL support. -The available protocols are `ssl2.3`, `ssl3`, `tls1`, `tls1.1`, `tls1.2`, `tls1.3`. +This will default to TLS v1.0 which will negotiate the highest protocol that both the server and your installation of OpenSSL support. +The available protocols are `tls1`, `tls1.1`, `tls1.2`, `tls1.3`. (The actually available set of protocols may vary depending on your OpenSSL installation.) ```bash -# Specify the vulnerable SSL v3 protocol to talk to an outdated server: -$ http --ssl=ssl3 https://vulnerable.example.org +# Specify the vulnerable TLS 1 protocol to talk to an outdated server: +$ http --ssl=tls1 https://vulnerable.example.org ``` ### SSL ciphers diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index a0099601c6..d79d0c4cfd 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -175,7 +175,8 @@ def parse_args( self._process_pretty_options() self._process_format_options() - # bellow is a fix for detecting "false-or empty" stdin + # bellow is a fix for detecting "false-or empty" stdin. + # see https://github.com/httpie/cli/issues/1551 for more information. if self.has_stdin_data: read_event = threading.Event() observe_stdin_for_data_thread(env, self.env.stdin, read_event) diff --git a/httpie/client.py b/httpie/client.py index d5c2f6956d..c05ee746f3 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -8,6 +8,8 @@ from urllib.parse import urlparse, urlunparse import niquests +# to understand why this is required +# see https://niquests.readthedocs.io/en/latest/community/faq.html#what-is-urllib3-future from niquests._compat import HAS_LEGACY_URLLIB3 if not HAS_LEGACY_URLLIB3: @@ -120,6 +122,10 @@ def collect_messages( hooks = None + # The hook set up bellow is crucial for HTTPie. + # It will help us yield the request before it is + # actually sent. This will permit us to know about + # the connection information for example. if prepared_request_readiness: hooks = {"pre_send": [prepared_request_readiness]} diff --git a/httpie/core.py b/httpie/core.py index 9505185fef..070046a67e 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -206,6 +206,8 @@ def request_body_read_callback(chunk: bytes): downloader.pre_request(args.headers) def prepared_request_readiness(pr): + """This callback is meant to output the request part. It is triggered by + the underlying Niquests library just after establishing the connection.""" oo = OutputOptions.from_message( pr, @@ -252,7 +254,11 @@ def prepared_request_readiness(pr): is_streamed_upload = not isinstance(message.body, (str, bytes)) do_write_body = not is_streamed_upload force_separator = is_streamed_upload and env.stdout_isatty + # We're in a REQUEST message, we rather output the message + # in prepared_request_readiness because we want "message.conn_info" + # to be set appropriately. (e.g. know about HTTP protocol version, etc...) if message.conn_info is None and not args.offline: + # bellow variable will be accessed by prepared_request_readiness just after. prev_with_body = output_options.body continue else: diff --git a/httpie/internal/encoder.py b/httpie/internal/encoder.py index 714f0ab3c1..4af687fe79 100644 --- a/httpie/internal/encoder.py +++ b/httpie/internal/encoder.py @@ -21,6 +21,8 @@ import os from uuid import uuid4 +# to understand why this is required +# see https://niquests.readthedocs.io/en/latest/community/faq.html#what-is-urllib3-future from niquests._compat import HAS_LEGACY_URLLIB3 if HAS_LEGACY_URLLIB3: diff --git a/httpie/models.py b/httpie/models.py index 142fd69710..ea3f00b40d 100644 --- a/httpie/models.py +++ b/httpie/models.py @@ -2,6 +2,8 @@ import niquests +# to understand why this is required +# see https://niquests.readthedocs.io/en/latest/community/faq.html#what-is-urllib3-future from niquests._compat import HAS_LEGACY_URLLIB3 if not HAS_LEGACY_URLLIB3: @@ -109,6 +111,8 @@ def metadata(self) -> str: time_since_headers_parsed = monotonic() - self._orig._httpie_headers_parsed_at time_elapsed = time_to_parse_headers + time_since_headers_parsed + # metrics aren't guaranteed to be there. act with caution. + # see https://niquests.readthedocs.io/en/latest/user/advanced.html#event-hooks for more. if hasattr(self._orig, "conn_info") and self._orig.conn_info: if self._orig.conn_info.resolution_latency: data[ELAPSED_DNS_RESOLUTION_LABEL] = str(round(self._orig.conn_info.resolution_latency.total_seconds(), 10)) + 's' diff --git a/httpie/output/streams.py b/httpie/output/streams.py index ea7867a3ef..b88886ab40 100644 --- a/httpie/output/streams.py +++ b/httpie/output/streams.py @@ -91,7 +91,7 @@ def __iter__(self) -> Iterable[bytes]: class RawStream(BaseStream): """The message is streamed in chunks with no processing.""" - CHUNK_SIZE = -1 + CHUNK_SIZE = -1 # '-1' means that we want to receive chunks exactly as they arrive. CHUNK_SIZE_BY_LINE = 1 def __init__(self, chunk_size=CHUNK_SIZE, **kwargs): diff --git a/httpie/ssl_.py b/httpie/ssl_.py index 06f9180319..9fe02abd1b 100644 --- a/httpie/ssl_.py +++ b/httpie/ssl_.py @@ -12,11 +12,17 @@ resolve_ssl_version, ) - +# the minimum one may hope to negotiate with Python 3.7+ is tls1+ +# anything else would be unsupported. SSL_VERSION_ARG_MAPPING = { + 'tls1': 'PROTOCOL_TLSv1', + 'tls1.1': 'PROTOCOL_TLSv1_1', 'tls1.2': 'PROTOCOL_TLSv1_2', 'tls1.3': 'PROTOCOL_TLSv1_3', } +# todo: we'll need to update this in preparation for Python 3.13+ +# could be a removal (after a long deprecation about constants +# PROTOCOL_TLSv1, PROTOCOL_TLSv1_1, ...). AVAILABLE_SSL_VERSION_ARG_MAPPING = { arg: getattr(ssl, constant_name) for arg, constant_name in SSL_VERSION_ARG_MAPPING.items() @@ -27,6 +33,9 @@ class QuicCapabilityCache( MutableMapping[Tuple[str, int], Optional[Tuple[str, int]]] ): + """This class will help us keep (persistent across runs) what hosts are QUIC capable. + See https://urllib3future.readthedocs.io/en/latest/advanced-usage.html#remembering-http-3-over-quic-support for + the implementation guide.""" def __init__(self): self._cache = {} @@ -77,6 +86,7 @@ def to_raw_cert(self): """Synthesize a requests-compatible (2-item tuple of cert and key file) object from HTTPie's internal representation of a certificate.""" if self.key_password: + # Niquests support 3-tuple repr in addition to the 2-tuple repr return self.cert_file, self.key_file, self.key_password return self.cert_file, self.key_file From 8c3c77ac18c2a15888417d8280c4fa030626ebe0 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 20 Mar 2024 18:40:45 +0100 Subject: [PATCH 13/63] apply initial suggestions --- CHANGELOG.md | 7 +++++++ httpie/client.py | 28 ++++++++++++++-------------- httpie/compat.py | 17 +++++++++++++++++ httpie/core.py | 10 +++++++++- httpie/internal/encoder.py | 9 +-------- httpie/models.py | 15 ++------------- httpie/ssl_.py | 21 +++++++++++++++------ setup.cfg | 7 +++++++ 8 files changed, 72 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f18adec8ad..c4b6f8aae2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,12 +16,19 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Added support for specifying the local port with `--local-port`. ([#1456](https://github.com/httpie/cli/issues/1456)) ([#1531](https://github.com/httpie/cli/pull/1531)) - Added support for forcing either IPv4 or IPv6 to reach the remote HTTP server with `-6` or `-4`. ([#94](https://github.com/httpie/cli/issues/94)) ([#1531](https://github.com/httpie/cli/pull/1531)) - Removed support for pyopenssl. ([#1531](https://github.com/httpie/cli/pull/1531)) +- Removed support for dead SSL protocols < TLS 1.0 (e.g. sslv3) as per pyopenssl removal. ([#1531](https://github.com/httpie/cli/pull/1531)) - Dropped dependency on `requests_toolbelt` in favor of directly including `MultipartEncoder` into HTTPie due to its direct dependency to requests. ([#1531](https://github.com/httpie/cli/pull/1531)) - Dropped dependency on `multidict` in favor of implementing an internal one due to often missing pre-built wheels. ([#1522](https://github.com/httpie/cli/issues/1522)) ([#1531](https://github.com/httpie/cli/pull/1531)) - Fixed the case when multiple headers where concatenated in the response output. ([#1413](https://github.com/httpie/cli/issues/1413)) ([#1531](https://github.com/httpie/cli/pull/1531)) - Fixed an edge case where HTTPie could be lead to believe data was passed in stdin, thus sending a POST by default. ([#1551](https://github.com/httpie/cli/issues/1551)) ([#1531](https://github.com/httpie/cli/pull/1531)) + This fix has the particularity to consider 0 byte long stdin buffer as absent stdin. Empty stdin buffer will be ignored. - Slightly improved performance while downloading by setting chunk size to `-1` to retrieve packets as they arrive. ([#1531](https://github.com/httpie/cli/pull/1531)) - Added support for using the system trust store to retrieve root CAs for verifying TLS certificates. ([#1531](https://github.com/httpie/cli/pull/1531)) +- Removed support for keeping the original casing of HTTP headers. This come from an outer constraint by newer protocols, namely HTTP/2+ that normalize header keys by default. + From the HTTPie user perspective, they are "prettified" on the output by default. e.g. "x-hello-world" is displayed as "X-Hello-World". + +The plugins are expected to work without any changes. The only caveat would be that certain plugin explicitly require `requests`. +Future contributions may be made in order to relax the constraints where applicable. ## [3.2.2](https://github.com/httpie/cli/compare/3.2.1...3.2.2) (2022-05-19) diff --git a/httpie/client.py b/httpie/client.py index c05ee746f3..2191cf1291 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -8,21 +8,10 @@ from urllib.parse import urlparse, urlunparse import niquests -# to understand why this is required -# see https://niquests.readthedocs.io/en/latest/community/faq.html#what-is-urllib3-future -from niquests._compat import HAS_LEGACY_URLLIB3 - -if not HAS_LEGACY_URLLIB3: - # noinspection PyPackageRequirements - import urllib3 - from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS, parse_url -else: - # noinspection PyPackageRequirements - import urllib3_future as urllib3 - from urllib3_future.util import SKIP_HEADER, SKIPPABLE_HEADERS, parse_url from . import __version__ from .adapters import HTTPieHTTPAdapter +from .compat import urllib3, SKIP_HEADER, SKIPPABLE_HEADERS, parse_url, Timeout from .cli.constants import HTTP_OPTIONS from .cli.dicts import HTTPHeadersDict from .cli.nested_json import unwrap_top_level_list_if_needed @@ -99,9 +88,20 @@ def collect_messages( source_address=source_address, ) + parsed_url = parse_url(args.url) + if args.disable_http3 is False and args.force_http3 is True: - url = parse_url(args.url) - requests_session.quic_cache_layer[(url.host, url.port or 443)] = (url.host, url.port or 443) + requests_session.quic_cache_layer[(parsed_url.host, parsed_url.port or 443)] = (parsed_url.host, parsed_url.port or 443) + # well, this one is tricky. If we allow HTTP/3, and remote host was marked as QUIC capable + # but is not anymore, we may face an indefinite hang if timeout isn't set. This could surprise some user. + elif ( + args.disable_http3 is False + and requests_session.quic_cache_layer.get((parsed_url.host, parsed_url.port or 443)) is not None + and send_kwargs["timeout"] is None + ): + # we only set the connect timeout, the rest is still indefinite. + send_kwargs["timeout"] = Timeout(connect=3) + setattr(args, "_failsafe_http3", True) if httpie_session: httpie_session.update_headers(request_kwargs['headers']) diff --git a/httpie/compat.py b/httpie/compat.py index fcf167ca7d..7afc490279 100644 --- a/httpie/compat.py +++ b/httpie/compat.py @@ -4,6 +4,23 @@ from httpie.cookies import HTTPieCookiePolicy from http import cookiejar # noqa +from niquests._compat import HAS_LEGACY_URLLIB3 + +# to understand why this is required +# see https://niquests.readthedocs.io/en/latest/community/faq.html#what-is-urllib3-future +# short story, urllib3 (import/top-level import) may be the legacy one https://github.com/urllib3/urllib3 +# instead of urllib3-future https://github.com/jawah/urllib3.future used by Niquests +# or only the secondary entry point could be available (e.g. urllib3_future on some distro without urllib3) +if not HAS_LEGACY_URLLIB3: + # noinspection PyPackageRequirements + import urllib3 # noqa: F401 + from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS, parse_url, Timeout # noqa: F401 + from urllib3.fields import RequestField # noqa: F401 +else: + # noinspection PyPackageRequirements + import urllib3_future as urllib3 # noqa: F401 + from urllib3_future.util import SKIP_HEADER, SKIPPABLE_HEADERS, parse_url, Timeout # noqa: F401 + from urllib3_future.fields import RequestField # noqa: F401 # Request does not carry the original policy attached to the # cookie jar, so until it is resolved we change the global cookie diff --git a/httpie/core.py b/httpie/core.py index 070046a67e..2cece5b3c7 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -26,6 +26,7 @@ from .utils import unwrap_context from .internal.update_warnings import check_updates from .internal.daemon_runner import is_daemon_mode, run_daemon_task +from .ssl_ import QuicCapabilityCache # noinspection PyDefaultArgument @@ -114,7 +115,14 @@ def handle_generic_error(e, annotation=None): exit_status = ExitStatus.ERROR except niquests.Timeout: exit_status = ExitStatus.ERROR_TIMEOUT - env.log_error(f'Request timed out ({parsed_args.timeout}s).') + # this detects if we tried to connect with HTTP/3 when the remote isn't compatible anymore. + if hasattr(parsed_args, "_failsafe_http3"): + env.log_error( + f'Unable to connect. Was the remote specified HTTP/3 compatible but is not anymore? ' + f'Remove "{QuicCapabilityCache.__file__}" to clear it out. Or set --disable-http3 flag.' + ) + else: + env.log_error(f'Request timed out ({parsed_args.timeout}s).') except niquests.TooManyRedirects: exit_status = ExitStatus.ERROR_TOO_MANY_REDIRECTS env.log_error( diff --git a/httpie/internal/encoder.py b/httpie/internal/encoder.py index 4af687fe79..a6867223df 100644 --- a/httpie/internal/encoder.py +++ b/httpie/internal/encoder.py @@ -21,14 +21,7 @@ import os from uuid import uuid4 -# to understand why this is required -# see https://niquests.readthedocs.io/en/latest/community/faq.html#what-is-urllib3-future -from niquests._compat import HAS_LEGACY_URLLIB3 - -if HAS_LEGACY_URLLIB3: - from urllib3_future.fields import RequestField -else: - from urllib3.fields import RequestField +from ..compat import RequestField class MultipartEncoder(object): diff --git a/httpie/models.py b/httpie/models.py index ea3f00b40d..76ccc8c748 100644 --- a/httpie/models.py +++ b/httpie/models.py @@ -2,17 +2,6 @@ import niquests -# to understand why this is required -# see https://niquests.readthedocs.io/en/latest/community/faq.html#what-is-urllib3-future -from niquests._compat import HAS_LEGACY_URLLIB3 - -if not HAS_LEGACY_URLLIB3: - from urllib3 import ConnectionInfo - from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS -else: - from urllib3_future import ConnectionInfo - from urllib3_future.util import SKIP_HEADER, SKIPPABLE_HEADERS - from kiss_headers.utils import prettify_header_name from enum import Enum, auto @@ -26,7 +15,7 @@ OUT_RESP_HEAD, OUT_RESP_META ) -from .compat import cached_property +from .compat import urllib3, SKIP_HEADER, SKIPPABLE_HEADERS, cached_property from .utils import split_cookies, parse_content_type_header ELAPSED_TIME_LABEL = 'Elapsed time' @@ -152,7 +141,7 @@ def iter_lines(self, chunk_size): @property def metadata(self) -> str: - conn_info: ConnectionInfo = self._orig.conn_info + conn_info: urllib3.ConnectionInfo = self._orig.conn_info metadatum = f"Connected to: {conn_info.destination_address[0]} port {conn_info.destination_address[1]}\n" diff --git a/httpie/ssl_.py b/httpie/ssl_.py index 9fe02abd1b..292f03602c 100644 --- a/httpie/ssl_.py +++ b/httpie/ssl_.py @@ -1,4 +1,5 @@ import ssl +import typing from typing import NamedTuple, Optional, Tuple, MutableMapping import json import os.path @@ -37,16 +38,21 @@ class QuicCapabilityCache( See https://urllib3future.readthedocs.io/en/latest/advanced-usage.html#remembering-http-3-over-quic-support for the implementation guide.""" + __file__ = os.path.join(DEFAULT_CONFIG_DIR, "quic.json") + def __init__(self): self._cache = {} if not os.path.exists(DEFAULT_CONFIG_DIR): makedirs(DEFAULT_CONFIG_DIR, exist_ok=True) - if os.path.exists(os.path.join(DEFAULT_CONFIG_DIR, "quic.json")): - with open(os.path.join(DEFAULT_CONFIG_DIR, "quic.json"), "r") as fp: - self._cache = json.load(fp) + if os.path.exists(QuicCapabilityCache.__file__): + with open(QuicCapabilityCache.__file__, "r") as fp: + try: + self._cache = json.load(fp) + except json.JSONDecodeError: # if the file is corrupted (invalid json) then, ignore it. + pass def save(self): - with open(os.path.join(DEFAULT_CONFIG_DIR, "quic.json"), "w+") as fp: + with open(QuicCapabilityCache.__file__, "w+") as fp: json.dump(self._cache, fp) def __contains__(self, item: Tuple[str, int]): @@ -82,8 +88,11 @@ class HTTPieCertificate(NamedTuple): key_file: Optional[str] = None key_password: Optional[str] = None - def to_raw_cert(self): - """Synthesize a requests-compatible (2-item tuple of cert and key file) + def to_raw_cert(self) -> typing.Union[ + typing.Tuple[typing.Optional[str], typing.Optional[str], typing.Optional[str]], # with password + typing.Tuple[typing.Optional[str], typing.Optional[str]] # without password + ]: + """Synthesize a niquests-compatible (2(or 3)-item tuple of cert, key file and optionally password) object from HTTPie's internal representation of a certificate.""" if self.key_password: # Niquests support 3-tuple repr in addition to the 2-tuple repr diff --git a/setup.cfg b/setup.cfg index fa95a47458..64aa59d373 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,8 +19,15 @@ markers = requires_external_processes filterwarnings = default + # due to urllib3.future no longer needing http.client! nothing to be concerned about. ignore:Passing msg=\.\. is deprecated:DeprecationWarning + # this only concern the test suite / local test server with a self signed certificate. ignore:Unverified HTTPS request is being made to host:urllib3.exceptions.InsecureRequestWarning + # the constant themselves are deprecated in the ssl module, we want to silent them in the test suite until we + # change the concerned code. Python 3.13 may remove them, so we'll need to think about it soon. + ignore:ssl\.PROTOCOL_(TLSv1|TLSv1_1|TLSv1_2) is deprecated:DeprecationWarning + ignore:ssl\.TLSVersion\.(TLSv1|TLSv1_1|TLSv1_2) is deprecated:DeprecationWarning + [metadata] name = httpie From dfe3dbe5bdaad03cc362893cf087a25fe1791649 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Thu, 21 Mar 2024 17:40:30 +0100 Subject: [PATCH 14/63] apply cosmetic fixes --- httpie/cli/argparser.py | 11 +++++++++++ httpie/client.py | 9 ++++++++- httpie/models.py | 32 ++++++++++++++++++++++---------- httpie/output/lexers/http.py | 4 ++-- httpie/output/lexers/metadata.py | 2 +- httpie/output/streams.py | 1 + setup.cfg | 3 +++ tests/test_meta.py | 6 ++++++ tests/test_uploads.py | 2 ++ 9 files changed, 56 insertions(+), 14 deletions(-) diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index d79d0c4cfd..bf981900fd 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -616,6 +616,17 @@ def print_usage(self, file): def error(self, message): """Prints a usage message incorporating the message to stderr and exits.""" + + # We shall release the files in that case + # the process is going to quit early anyway. + if hasattr(self.args, "multipart_data"): + for f in self.args.multipart_data: + if isinstance(self.args.multipart_data[f], tuple): + self.args.multipart_data[f][1].close() + elif isinstance(self.args.multipart_data[f], list): + for item in self.args.multipart_data[f]: + item[1].close() + self.print_usage(sys.stderr) self.env.rich_error_console.print( dedent( diff --git a/httpie/client.py b/httpie/client.py index 2191cf1291..7a2dc0a28a 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -157,6 +157,13 @@ def collect_messages( **send_kwargs, ) if args.max_headers and len(response.headers) > args.max_headers: + try: + requests_session.close() + # we consume the content to allow the connection to be put back into the pool, and closed! + response.content + except NotImplementedError: # We allow custom transports that may not implement close. + pass + raise niquests.ConnectionError(f"got more than {args.max_headers} headers") response._httpie_headers_parsed_at = monotonic() expired_cookies += get_expired_cookies( @@ -183,7 +190,7 @@ def collect_messages( try: requests_session.close() - except NotImplementedError: + except NotImplementedError: # We allow custom transports that may not implement close. pass diff --git a/httpie/models.py b/httpie/models.py index 76ccc8c748..f20363f588 100644 --- a/httpie/models.py +++ b/httpie/models.py @@ -103,16 +103,28 @@ def metadata(self) -> str: # metrics aren't guaranteed to be there. act with caution. # see https://niquests.readthedocs.io/en/latest/user/advanced.html#event-hooks for more. if hasattr(self._orig, "conn_info") and self._orig.conn_info: - if self._orig.conn_info.resolution_latency: - data[ELAPSED_DNS_RESOLUTION_LABEL] = str(round(self._orig.conn_info.resolution_latency.total_seconds(), 10)) + 's' - if self._orig.conn_info.established_latency: - data[ELAPSED_ESTABLISH_CONN] = str(round(self._orig.conn_info.established_latency.total_seconds(), 10)) + 's' - if self._orig.conn_info.tls_handshake_latency: - data[ELAPSED_TLS_HANDSHAKE] = str(round(self._orig.conn_info.tls_handshake_latency.total_seconds(), 10)) + 's' - if self._orig.conn_info.request_sent_latency: - data[ELAPSED_REQUEST_SEND] = str(round(self._orig.conn_info.request_sent_latency.total_seconds(), 10)) + 's' - - data[ELAPSED_TIME_LABEL] = str(round(time_elapsed, 10)) + 's' + if self._orig.conn_info.resolution_latency is not None: + if self._orig.conn_info.resolution_latency: + data[ELAPSED_DNS_RESOLUTION_LABEL] = f"{round(self._orig.conn_info.resolution_latency.total_seconds(), 10):6f}s" + else: + data[ELAPSED_DNS_RESOLUTION_LABEL] = "0s" + if self._orig.conn_info.established_latency is not None: + if self._orig.conn_info.established_latency: + data[ELAPSED_ESTABLISH_CONN] = f"{round(self._orig.conn_info.established_latency.total_seconds(), 10):6f}s" + else: + data[ELAPSED_ESTABLISH_CONN] = "0s" + if self._orig.conn_info.tls_handshake_latency is not None: + if self._orig.conn_info.tls_handshake_latency: + data[ELAPSED_TLS_HANDSHAKE] = f"{round(self._orig.conn_info.tls_handshake_latency.total_seconds(), 10):6f}s" + else: + data[ELAPSED_TLS_HANDSHAKE] = "0s" + if self._orig.conn_info.request_sent_latency is not None: + if self._orig.conn_info.request_sent_latency: + data[ELAPSED_REQUEST_SEND] = f"{round(self._orig.conn_info.request_sent_latency.total_seconds(), 10):6f}s" + else: + data[ELAPSED_REQUEST_SEND] = "0s" + + data[ELAPSED_TIME_LABEL] = f"{round(time_elapsed, 10):6f}s" return '\n'.join( f'{key}: {value}' diff --git a/httpie/output/lexers/http.py b/httpie/output/lexers/http.py index aea827401e..728490115f 100644 --- a/httpie/output/lexers/http.py +++ b/httpie/output/lexers/http.py @@ -66,7 +66,7 @@ class SimplifiedHTTPLexer(pygments.lexer.RegexLexer): tokens = { 'root': [ # Request-Line - (r'([A-Z]+)( +)([^ ]+)( +)(HTTP)(/)(\d+\.\d+)', + (r'([A-Z]+)( +)([^ ]+)( +)(HTTP)(/)([0-9].?[0-9]?)', pygments.lexer.bygroups( request_method, pygments.token.Text, @@ -77,7 +77,7 @@ class SimplifiedHTTPLexer(pygments.lexer.RegexLexer): pygments.token.Number )), # Response Status-Line - (r'(HTTP)(/)(\d+\.\d+)( +)(.+)', + (r'(HTTP)(/)([0-9].?[0-9]?)( +)(.+)', pygments.lexer.bygroups( pygments.token.Keyword.Reserved, # 'HTTP' pygments.token.Operator, # '/' diff --git a/httpie/output/lexers/metadata.py b/httpie/output/lexers/metadata.py index 7f5c77f54d..1d41a67446 100644 --- a/httpie/output/lexers/metadata.py +++ b/httpie/output/lexers/metadata.py @@ -36,7 +36,7 @@ class MetadataLexer(pygments.lexer.RegexLexer): tokens = { 'root': [ ( - fr'({ELAPSED_TIME_LABEL}|{ELAPSED_DNS_RESOLUTION_LABEL}|{ELAPSED_REQUEST_SEND}|{ELAPSED_TLS_HANDSHAKE}|{ELAPSED_ESTABLISH_CONN})( *)(:)( *)(\d+\.[\de\-]+)(s)', pygments.lexer.bygroups( + fr'({ELAPSED_TIME_LABEL}|{ELAPSED_DNS_RESOLUTION_LABEL}|{ELAPSED_REQUEST_SEND}|{ELAPSED_TLS_HANDSHAKE}|{ELAPSED_ESTABLISH_CONN})( *)(:)( *)([\d]+[.\d]{{0,}})(s)', pygments.lexer.bygroups( pygments.token.Name.Decorator, # Name pygments.token.Text, pygments.token.Operator, # Colon diff --git a/httpie/output/streams.py b/httpie/output/streams.py index b88886ab40..1686a97913 100644 --- a/httpie/output/streams.py +++ b/httpie/output/streams.py @@ -86,6 +86,7 @@ def __iter__(self) -> Iterable[bytes]: yield b'\n\n' yield self.get_metadata() + yield b'\n\n' class RawStream(BaseStream): diff --git a/setup.cfg b/setup.cfg index 64aa59d373..5f2933e0cd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,9 @@ filterwarnings = # change the concerned code. Python 3.13 may remove them, so we'll need to think about it soon. ignore:ssl\.PROTOCOL_(TLSv1|TLSv1_1|TLSv1_2) is deprecated:DeprecationWarning ignore:ssl\.TLSVersion\.(TLSv1|TLSv1_1|TLSv1_2) is deprecated:DeprecationWarning + # Happen in Windows. Oppose no threats to our test suite. + # "An operation was attempted on something that is not a socket" during shutdown + ignore:Exception in thread:pytest.PytestUnhandledThreadExceptionWarning [metadata] diff --git a/tests/test_meta.py b/tests/test_meta.py index e0912958c6..7a5f57eb4d 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -19,6 +19,12 @@ def test_meta_extended_tls(remote_httpbin_secure): assert 'Issuer' in r assert 'Revocation status' in r + # If this fail, you missed two extraneous RC after the metadata render. + # see output/streams.py L89 + # why do we need two? short story, in case of redirect, expect metadata to appear multiple times, + # and we don't want them glued to the request line for example. + assert str(r).endswith("\n\n") + @pytest.mark.parametrize('style', ['auto', 'fruity', *PIE_STYLE_NAMES]) def test_meta_elapsed_time_colors(httpbin, style): diff --git a/tests/test_uploads.py b/tests/test_uploads.py index e4723d6f6b..01128bdf9c 100644 --- a/tests/test_uploads.py +++ b/tests/test_uploads.py @@ -145,6 +145,8 @@ def test_reading_from_stdin(httpbin, wait): @pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") def test_stdin_read_warning(httpbin): + """This test is flaky. Expect random failure in the CI under MacOS. + It's mainly due to the poor VM performance.""" with stdin_processes(httpbin) as (process_1, process_2): # Wait before sending any data time.sleep(1) From cce24fea7f933b5978461b31da5bdc5a93c590ae Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 22 Mar 2024 07:04:43 +0100 Subject: [PATCH 15/63] add xfail when CI+MacOS for test_stdin_read_warning --- tests/test_uploads.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_uploads.py b/tests/test_uploads.py index 01128bdf9c..9258f6e7ca 100644 --- a/tests/test_uploads.py +++ b/tests/test_uploads.py @@ -1,5 +1,6 @@ import os import json +import platform import sys import subprocess import time @@ -144,6 +145,11 @@ def test_reading_from_stdin(httpbin, wait): @pytest.mark.requires_external_processes @pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") +@pytest.mark.xfail( + platform.system() == "Darwin" and os.environ.get("CI") is not None, + reason="GitHub CI and MacOS raises random failures", + strict=False, +) def test_stdin_read_warning(httpbin): """This test is flaky. Expect random failure in the CI under MacOS. It's mainly due to the poor VM performance.""" From 8827c42bd37fa88564c5fec5683132019ccb2b2e Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 22 Mar 2024 07:49:32 +0100 Subject: [PATCH 16/63] update man files --- extras/man/http.1 | 64 ++++++++++++++++++++++++++++++++++++++-- extras/man/httpie.1 | 4 +-- extras/man/https.1 | 64 ++++++++++++++++++++++++++++++++++++++-- httpie/cli/definition.py | 3 +- 4 files changed, 128 insertions(+), 7 deletions(-) diff --git a/extras/man/http.1 b/extras/man/http.1 index 65fa133ab0..634cb84372 100644 --- a/extras/man/http.1 +++ b/extras/man/http.1 @@ -1,5 +1,5 @@ .\" This file is auto-generated from the parser declaration in httpie/cli/definition.py by extras/scripts/generate_man_pages.py. -.TH http 1 "2022-05-06" "HTTPie 3.2.2" "HTTPie Manual" +.TH http 1 "2024-01-01" "HTTPie 4.0.0.b1" "HTTPie Manual" .SH NAME http .SH SYNOPSIS @@ -427,6 +427,18 @@ and $HTTPS_proxy are supported as well. +.IP "\fB\,--ipv6\/\fR, \fB\,-6\/\fR" + + +Force using a IPv6 address to reach the remote peer. + + +.IP "\fB\,--ipv4\/\fR, \fB\,-4\/\fR" + + +Force using a IPv4 address to reach the remote peer. + + .IP "\fB\,--follow\/\fR, \fB\,-F\/\fR" @@ -484,6 +496,54 @@ Bypass dot segment (/../ or /./) URL squashing. Enable streaming via chunked transfer encoding. The Transfer-Encoding header is set to chunked. +.IP "\fB\,--disable-http2\/\fR" + + +Disable the HTTP/2 protocol. + + +.IP "\fB\,--disable-http3\/\fR" + + +Disable the HTTP/3 over QUIC protocol. + + +.IP "\fB\,--http3\/\fR" + + +By default, HTTPie cannot negotiate HTTP/3 without a first HTTP/1.1, or HTTP/2 successful response unless the +remote host specified a DNS HTTPS record that indicate its support. + +The remote server yield its support for HTTP/3 in the Alt-Svc header, if present HTTPie will issue +the successive requests via HTTP/3. You may use that argument in case the remote peer does not support +either HTTP/1.1 or HTTP/2. + + + +.IP "\fB\,--resolver\/\fR" + + +By default, HTTPie use the system DNS through Python standard library. +You can specify an alternative DNS server to be used. (e.g. doh://cloudflare-dns.com or doh://google.dns). +You can specify multiple resolvers with different protocols. The environment +variable $NIQUESTS_DNS_URL is supported as well. This flag also support overriding DNS resolution +e.g. passing \[dq]pie.dev:1.1.1.1\[dq] will resolve pie.dev to 1.1.1.1 IPv4. + + + +.IP "\fB\,--interface\/\fR" + + +Bind to a specific network interface. + + +.IP "\fB\,--local-port\/\fR" + + +It can be either a port range (e.g. \[dq]11221-14555\[dq]) or a single port. +Some port may require root privileges (e.g. < 1024). + + .PP .SH SSL .IP "\fB\,--verify\/\fR" @@ -597,4 +657,4 @@ For every \fB\,--OPTION\/\fR there is also a \fB\,--no-OPTION\/\fR that reverts to its default value. Suggestions and bug reports are greatly appreciated: -https://github.com/httpie/cli/issues \ No newline at end of file +https://github.com/httpie/cli/issues diff --git a/extras/man/httpie.1 b/extras/man/httpie.1 index 0536d61bfa..69bd76265f 100644 --- a/extras/man/httpie.1 +++ b/extras/man/httpie.1 @@ -1,5 +1,5 @@ .\" This file is auto-generated from the parser declaration in httpie/manager/cli.py by extras/scripts/generate_man_pages.py. -.TH httpie 1 "2022-05-06" "HTTPie 3.2.2" "HTTPie Manual" +.TH httpie 1 "2024-01-01" "HTTPie 4.0.0.b1" "HTTPie Manual" .SH NAME httpie .SH SYNOPSIS @@ -97,4 +97,4 @@ targets to install .PP .SH httpie plugins list List all installed HTTPie plugins. -.PP \ No newline at end of file +.PP diff --git a/extras/man/https.1 b/extras/man/https.1 index c91290a245..6a7370e071 100644 --- a/extras/man/https.1 +++ b/extras/man/https.1 @@ -1,5 +1,5 @@ .\" This file is auto-generated from the parser declaration in httpie/cli/definition.py by extras/scripts/generate_man_pages.py. -.TH https 1 "2022-05-06" "HTTPie 3.2.2" "HTTPie Manual" +.TH https 1 "2024-01-01" "HTTPie 4.0.0.b1" "HTTPie Manual" .SH NAME https .SH SYNOPSIS @@ -427,6 +427,18 @@ and $HTTPS_proxy are supported as well. +.IP "\fB\,--ipv6\/\fR, \fB\,-6\/\fR" + + +Force using a IPv6 address to reach the remote peer. + + +.IP "\fB\,--ipv4\/\fR, \fB\,-4\/\fR" + + +Force using a IPv4 address to reach the remote peer. + + .IP "\fB\,--follow\/\fR, \fB\,-F\/\fR" @@ -484,6 +496,54 @@ Bypass dot segment (/../ or /./) URL squashing. Enable streaming via chunked transfer encoding. The Transfer-Encoding header is set to chunked. +.IP "\fB\,--disable-http2\/\fR" + + +Disable the HTTP/2 protocol. + + +.IP "\fB\,--disable-http3\/\fR" + + +Disable the HTTP/3 over QUIC protocol. + + +.IP "\fB\,--http3\/\fR" + + +By default, HTTPie cannot negotiate HTTP/3 without a first HTTP/1.1, or HTTP/2 successful response unless the +remote host specified a DNS HTTPS record that indicate its support. + +The remote server yield its support for HTTP/3 in the Alt-Svc header, if present HTTPie will issue +the successive requests via HTTP/3. You may use that argument in case the remote peer does not support +either HTTP/1.1 or HTTP/2. + + + +.IP "\fB\,--resolver\/\fR" + + +By default, HTTPie use the system DNS through Python standard library. +You can specify an alternative DNS server to be used. (e.g. doh://cloudflare-dns.com or doh://google.dns). +You can specify multiple resolvers with different protocols. The environment +variable $NIQUESTS_DNS_URL is supported as well. This flag also support overriding DNS resolution +e.g. passing \[dq]pie.dev:1.1.1.1\[dq] will resolve pie.dev to 1.1.1.1 IPv4. + + + +.IP "\fB\,--interface\/\fR" + + +Bind to a specific network interface. + + +.IP "\fB\,--local-port\/\fR" + + +It can be either a port range (e.g. \[dq]11221-14555\[dq]) or a single port. +Some port may require root privileges (e.g. < 1024). + + .PP .SH SSL .IP "\fB\,--verify\/\fR" @@ -597,4 +657,4 @@ For every \fB\,--OPTION\/\fR there is also a \fB\,--no-OPTION\/\fR that reverts to its default value. Suggestions and bug reports are greatly appreciated: -https://github.com/httpie/cli/issues \ No newline at end of file +https://github.com/httpie/cli/issues diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 52addbd3c5..1a412dc50b 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -853,7 +853,8 @@ def format_auth_help(auth_plugins_mapping, *, isolation_mode: bool = False): By default, HTTPie use the system DNS through Python standard library. You can specify an alternative DNS server to be used. (e.g. doh://cloudflare-dns.com or doh://google.dns). You can specify multiple resolvers with different protocols. The environment - variable $NIQUESTS_DNS_URL is supported as well. + variable $NIQUESTS_DNS_URL is supported as well. This flag also support overriding DNS resolution + e.g. passing "pie.dev:1.1.1.1" will resolve pie.dev to 1.1.1.1 IPv4. """ ) From 74d62b82bc0e18878730badf53af3b95cadd7f88 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 22 Mar 2024 07:50:05 +0100 Subject: [PATCH 17/63] display http3 error message even if user set timeout flag --- httpie/client.py | 5 +++-- tests/test_resolver.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/httpie/client.py b/httpie/client.py index 7a2dc0a28a..d2149b23f3 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -97,10 +97,11 @@ def collect_messages( elif ( args.disable_http3 is False and requests_session.quic_cache_layer.get((parsed_url.host, parsed_url.port or 443)) is not None - and send_kwargs["timeout"] is None + and args.force_http3 is False ): # we only set the connect timeout, the rest is still indefinite. - send_kwargs["timeout"] = Timeout(connect=3) + if send_kwargs["timeout"] is None: + send_kwargs["timeout"] = Timeout(connect=3) setattr(args, "_failsafe_http3", True) if httpie_session: diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 06bc1a104c..b9c6c8b690 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -18,6 +18,7 @@ def test_ensure_override_resolver_used(remote_httpbin): """Just an additional check to ensure we are wired properly to Niquests resolver parameter.""" r = http( "--resolver=pie.dev:240.0.0.0", # override DNS response to TARPIT net addr. + "--disable-http3", remote_httpbin + "/get", tolerate_error_exit_status=True ) From 1ac0f320baf3683912896222549764b4cd966a36 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 22 Mar 2024 08:00:13 +0100 Subject: [PATCH 18/63] update docs with fixed metadata render --- docs/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/README.md b/docs/README.md index 6fcb3012cd..0432b66f8b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1601,10 +1601,10 @@ $ http --meta pie.dev/delay/1 ```console Connected to: 2a06:98c1:3120::2 port 80 -Elapsed DNS: 0.047945s -Elapsed established connection: 0.013063s -Elapsed emitting request: 0.000115s -Elapsed time: 1.1325035701s +Elapsed DNS: 0.000833s +Elapsed established connection: 0.020144s +Elapsed emitting request: 0.000121s +Elapsed time: 1.080282s ``` The [extra verbose `-vv` output](#extra-verbose-output) includes the meta section by default. You can also show it in combination with other parts of the exchange via [`--print=m`](#what-parts-of-the-http-exchange-should-be-printed). For example, here we print it together with the response headers: @@ -1632,11 +1632,11 @@ Content-Type: application/json Date: Wed, 20 Mar 2024 05:32:11 GMT Server: cloudflare -Elapsed DNS: 0.000682s -Elapsed established connection: 1.7e-05s -Elapsed TLS handshake: 0.043641s -Elapsed emitting request: 0.000397s -Elapsed time: 0.1677905799s +Elapsed DNS: 0.000629s +Elapsed established connection: 0.000013s +Elapsed TLS handshake: 0.043979s +Elapsed emitting request: 0.000257s +Elapsed time: 0.159567s ``` From bfd7f3737238b262e4269a2ef689a09a7d29895d Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Mon, 25 Mar 2024 07:40:53 +0100 Subject: [PATCH 19/63] add comment --- tests/test_h2n3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_h2n3.py b/tests/test_h2n3.py index a9563aa916..9219f3dfac 100644 --- a/tests/test_h2n3.py +++ b/tests/test_h2n3.py @@ -13,7 +13,8 @@ def test_should_not_do_http1_by_default(remote_httpbin_secure): def test_disable_http2n3(remote_httpbin_secure): r = http( - "--verify=no", + # Only for DEV environment! + "--verify=no", # we have REQUESTS_CA_BUNDLE environment set, so we must disable ext verify. '--disable-http2', '--disable-http3', remote_httpbin_secure + '/get' From b010effb46cda4626ce10bfa8f39e9cc3408e0cb Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 3 Apr 2024 07:45:49 +0200 Subject: [PATCH 20/63] respect httpie config to generate path for "quic.json" cache layer --- httpie/client.py | 5 ++++- httpie/config.py | 6 +++++- httpie/core.py | 3 +-- httpie/ssl_.py | 16 ++++++---------- tests/test_h2n3.py | 44 +++++++++++++++++++++++++++++++++++++++++++- tests/test_meta.py | 6 +++++- 6 files changed, 64 insertions(+), 16 deletions(-) diff --git a/httpie/client.py b/httpie/client.py index d2149b23f3..397de49b5c 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -2,6 +2,7 @@ import json import sys import typing +from pathlib import Path from random import randint from time import monotonic from typing import Any, Dict, Callable, Iterable @@ -86,6 +87,7 @@ def collect_messages( disable_ipv6=args.ipv4, disable_ipv4=args.ipv6, source_address=source_address, + quic_cache=env.config.quic_file, ) parsed_url = parse_url(args.url) @@ -205,9 +207,10 @@ def build_requests_session( disable_ipv4: bool = False, disable_ipv6: bool = False, source_address: typing.Tuple[str, int] = None, + quic_cache: typing.Optional[Path] = None, ) -> niquests.Session: requests_session = niquests.Session() - requests_session.quic_cache_layer = QuicCapabilityCache() + requests_session.quic_cache_layer = QuicCapabilityCache(quic_cache) if resolver: resolver_rebuilt = [] diff --git a/httpie/config.py b/httpie/config.py index 27bc0a784d..5e6fc1245e 100644 --- a/httpie/config.py +++ b/httpie/config.py @@ -149,7 +149,7 @@ def __init__(self, directory: Union[str, Path] = DEFAULT_CONFIG_DIR): def default_options(self) -> list: return self['default_options'] - def _configured_path(self, config_option: str, default: str) -> None: + def _configured_path(self, config_option: str, default: str) -> Path: return Path( self.get(config_option, self.directory / default) ).expanduser().resolve() @@ -162,6 +162,10 @@ def plugins_dir(self) -> Path: def version_info_file(self) -> Path: return self._configured_path('version_info_file', 'version_info.json') + @property + def quic_file(self) -> Path: + return self._configured_path('quic_file', 'quic.json') + @property def developer_mode(self) -> bool: """This is a special setting for the development environment. It is diff --git a/httpie/core.py b/httpie/core.py index 2cece5b3c7..3193d0190b 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -26,7 +26,6 @@ from .utils import unwrap_context from .internal.update_warnings import check_updates from .internal.daemon_runner import is_daemon_mode, run_daemon_task -from .ssl_ import QuicCapabilityCache # noinspection PyDefaultArgument @@ -119,7 +118,7 @@ def handle_generic_error(e, annotation=None): if hasattr(parsed_args, "_failsafe_http3"): env.log_error( f'Unable to connect. Was the remote specified HTTP/3 compatible but is not anymore? ' - f'Remove "{QuicCapabilityCache.__file__}" to clear it out. Or set --disable-http3 flag.' + f'Remove "{env.config.quic_file}" to clear it out. Or set --disable-http3 flag.' ) else: env.log_error(f'Request timed out ({parsed_args.timeout}s).') diff --git a/httpie/ssl_.py b/httpie/ssl_.py index 292f03602c..715a203ff0 100644 --- a/httpie/ssl_.py +++ b/httpie/ssl_.py @@ -1,11 +1,10 @@ import ssl import typing +from pathlib import Path from typing import NamedTuple, Optional, Tuple, MutableMapping import json import os.path -from os import makedirs -from httpie.config import DEFAULT_CONFIG_DIR from httpie.adapters import HTTPAdapter # noinspection PyPackageRequirements from urllib3.util.ssl_ import ( @@ -38,21 +37,18 @@ class QuicCapabilityCache( See https://urllib3future.readthedocs.io/en/latest/advanced-usage.html#remembering-http-3-over-quic-support for the implementation guide.""" - __file__ = os.path.join(DEFAULT_CONFIG_DIR, "quic.json") - - def __init__(self): + def __init__(self, path: Path): + self._path = path self._cache = {} - if not os.path.exists(DEFAULT_CONFIG_DIR): - makedirs(DEFAULT_CONFIG_DIR, exist_ok=True) - if os.path.exists(QuicCapabilityCache.__file__): - with open(QuicCapabilityCache.__file__, "r") as fp: + if os.path.exists(path): + with open(path, "r") as fp: try: self._cache = json.load(fp) except json.JSONDecodeError: # if the file is corrupted (invalid json) then, ignore it. pass def save(self): - with open(QuicCapabilityCache.__file__, "w+") as fp: + with open(self._path, "w") as fp: json.dump(self._cache, fp) def __contains__(self, item: Tuple[str, int]): diff --git a/tests/test_h2n3.py b/tests/test_h2n3.py index 9219f3dfac..b7f24b36c0 100644 --- a/tests/test_h2n3.py +++ b/tests/test_h2n3.py @@ -1,4 +1,7 @@ -from .utils import HTTP_OK, http +import pytest +import json + +from .utils import HTTP_OK, http, PersistentMockEnvironment def test_should_not_do_http1_by_default(remote_httpbin_secure): @@ -33,3 +36,42 @@ def test_force_http3(remote_httpbin_secure): assert "HTTP/3" in r assert HTTP_OK in r + + +@pytest.fixture +def with_quic_cache_persistent(tmp_path): + env = PersistentMockEnvironment() + env.config['quic_file'] = tmp_path / 'quic.json' + yield env + env.cleanup(force=True) + + +def test_ensure_quic_cache(remote_httpbin_secure, with_quic_cache_persistent): + """ + This test aim to verify that the QuicCapabilityCache work as intended. + """ + r = http( + "--verify=no", + remote_httpbin_secure + '/get', + env=with_quic_cache_persistent + ) + + assert "HTTP/2" in r + assert HTTP_OK in r + + r = http( + "--verify=no", + remote_httpbin_secure + '/get', + env=with_quic_cache_persistent + ) + + assert "HTTP/3" in r + assert HTTP_OK in r + + tmp_path = with_quic_cache_persistent.config['quic_file'] + + with open(tmp_path, "r") as fp: + cache = json.load(fp) + + assert len(cache) == 1 + assert "pie.dev" in list(cache.keys())[0] diff --git a/tests/test_meta.py b/tests/test_meta.py index 7a5f57eb4d..fa8a7e4d20 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -11,7 +11,11 @@ def test_meta_elapsed_time(httpbin): def test_meta_extended_tls(remote_httpbin_secure): - r = http('--verify=no', '--meta', remote_httpbin_secure + '/get') + # using --verify=no may cause the certificate information not to display with Python < 3.10 + # it is guaranteed to be there when using HTTP/3 over QUIC. That's why we set the '--http3' flag. + # it's a known CPython limitation with getpeercert(binary_form=False). + r = http('--verify=no', '--http3', '--meta', remote_httpbin_secure + '/get') + assert 'Connected to' in r assert 'Connection secured using' in r assert 'Server certificate' in r From e3c5acd0fcc284c9125cfd3e5605d9c5b94f9d5f Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Thu, 4 Apr 2024 07:03:39 +0200 Subject: [PATCH 21/63] add test for remote h3 not compatible anymore --- tests/test_h2n3.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_h2n3.py b/tests/test_h2n3.py index b7f24b36c0..3f3ddd82b4 100644 --- a/tests/test_h2n3.py +++ b/tests/test_h2n3.py @@ -1,6 +1,8 @@ import pytest import json +from httpie.ssl_ import QuicCapabilityCache + from .utils import HTTP_OK, http, PersistentMockEnvironment @@ -75,3 +77,33 @@ def test_ensure_quic_cache(remote_httpbin_secure, with_quic_cache_persistent): assert len(cache) == 1 assert "pie.dev" in list(cache.keys())[0] + + +def test_h3_not_compatible_anymore(remote_httpbin_secure, with_quic_cache_persistent): + tmp_path = with_quic_cache_persistent.config['quic_file'] + + cache = QuicCapabilityCache(tmp_path) + + # doing a __setitem__ should trigger save automatically! + cache[("pie.dev", 443)] = ("pie.dev", 61443) # nothing listen on 61443! + + # without timeout + r = http( + "--verify=no", + remote_httpbin_secure + '/get', + env=with_quic_cache_persistent, + tolerate_error_exit_status=True + ) + + assert "Unable to connect. Was the remote specified HTTP/3 compatible but is not anymore?" in r.stderr + + # with timeout + r = http( + "--verify=no", + "--timeout=1", + remote_httpbin_secure + '/get', + env=with_quic_cache_persistent, + tolerate_error_exit_status=True + ) + + assert "Unable to connect. Was the remote specified HTTP/3 compatible but is not anymore?" in r.stderr From 5c497fd151887584ec643258b7c5b557c148f7b8 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Thu, 4 Apr 2024 07:20:36 +0200 Subject: [PATCH 22/63] use compat for create_urllib3_context, resolve_ssl_version in httpie.ssl_ submodule --- httpie/compat.py | 8 ++++++++ httpie/ssl_.py | 6 +----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/httpie/compat.py b/httpie/compat.py index 7afc490279..203549bfe5 100644 --- a/httpie/compat.py +++ b/httpie/compat.py @@ -16,11 +16,19 @@ import urllib3 # noqa: F401 from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS, parse_url, Timeout # noqa: F401 from urllib3.fields import RequestField # noqa: F401 + from urllib3.util.ssl_ import ( # noqa: F401 + create_urllib3_context, + resolve_ssl_version, + ) else: # noinspection PyPackageRequirements import urllib3_future as urllib3 # noqa: F401 from urllib3_future.util import SKIP_HEADER, SKIPPABLE_HEADERS, parse_url, Timeout # noqa: F401 from urllib3_future.fields import RequestField # noqa: F401 + from urllib3_future.util.ssl_ import ( # noqa: F401 + create_urllib3_context, + resolve_ssl_version, + ) # Request does not carry the original policy attached to the # cookie jar, so until it is resolved we change the global cookie diff --git a/httpie/ssl_.py b/httpie/ssl_.py index 715a203ff0..888be5540d 100644 --- a/httpie/ssl_.py +++ b/httpie/ssl_.py @@ -6,11 +6,7 @@ import os.path from httpie.adapters import HTTPAdapter -# noinspection PyPackageRequirements -from urllib3.util.ssl_ import ( - create_urllib3_context, - resolve_ssl_version, -) +from .compat import create_urllib3_context, resolve_ssl_version # the minimum one may hope to negotiate with Python 3.7+ is tls1+ # anything else would be unsupported. From ff371668234bcb41040e4ea09be360e250333ef0 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Thu, 4 Apr 2024 07:21:16 +0200 Subject: [PATCH 23/63] inject QuicCapabilityCache is a path is given (and not None) --- httpie/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/httpie/client.py b/httpie/client.py index 397de49b5c..b04d78515e 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -210,7 +210,9 @@ def build_requests_session( quic_cache: typing.Optional[Path] = None, ) -> niquests.Session: requests_session = niquests.Session() - requests_session.quic_cache_layer = QuicCapabilityCache(quic_cache) + + if quic_cache is not None: + requests_session.quic_cache_layer = QuicCapabilityCache(quic_cache) if resolver: resolver_rebuilt = [] From e19eb259ec928678bed54c420f75aabf210f6c68 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Thu, 4 Apr 2024 07:47:27 +0200 Subject: [PATCH 24/63] fix multipart form data having filename with non ascii characters linked issue https://github.com/httpie/cli/issues/1401 --- CHANGELOG.md | 1 + httpie/compat.py | 4 ++-- httpie/internal/encoder.py | 5 +++-- tests/fixtures/__init__.py | 1 + "tests/fixtures/\345\244\251\347\213\227.txt" | 0 tests/test_uploads.py | 14 +++++++++++++- 6 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 "tests/fixtures/\345\244\251\347\213\227.txt" diff --git a/CHANGELOG.md b/CHANGELOG.md index c4b6f8aae2..eb856a4eb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Added support for using the system trust store to retrieve root CAs for verifying TLS certificates. ([#1531](https://github.com/httpie/cli/pull/1531)) - Removed support for keeping the original casing of HTTP headers. This come from an outer constraint by newer protocols, namely HTTP/2+ that normalize header keys by default. From the HTTPie user perspective, they are "prettified" on the output by default. e.g. "x-hello-world" is displayed as "X-Hello-World". +- Fixed multipart form data having filename not rfc2231 compliant when name contain non-ascii characters. ([#1401](https://github.com/httpie/cli/issues/1401)) The plugins are expected to work without any changes. The only caveat would be that certain plugin explicitly require `requests`. Future contributions may be made in order to relax the constraints where applicable. diff --git a/httpie/compat.py b/httpie/compat.py index 203549bfe5..0aa36e4410 100644 --- a/httpie/compat.py +++ b/httpie/compat.py @@ -15,7 +15,7 @@ # noinspection PyPackageRequirements import urllib3 # noqa: F401 from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS, parse_url, Timeout # noqa: F401 - from urllib3.fields import RequestField # noqa: F401 + from urllib3.fields import RequestField, format_header_param_rfc2231 # noqa: F401 from urllib3.util.ssl_ import ( # noqa: F401 create_urllib3_context, resolve_ssl_version, @@ -24,7 +24,7 @@ # noinspection PyPackageRequirements import urllib3_future as urllib3 # noqa: F401 from urllib3_future.util import SKIP_HEADER, SKIPPABLE_HEADERS, parse_url, Timeout # noqa: F401 - from urllib3_future.fields import RequestField # noqa: F401 + from urllib3_future.fields import RequestField, format_header_param_rfc2231 # noqa: F401 from urllib3_future.util.ssl_ import ( # noqa: F401 create_urllib3_context, resolve_ssl_version, diff --git a/httpie/internal/encoder.py b/httpie/internal/encoder.py index a6867223df..3a8d73657b 100644 --- a/httpie/internal/encoder.py +++ b/httpie/internal/encoder.py @@ -21,7 +21,7 @@ import os from uuid import uuid4 -from ..compat import RequestField +from ..compat import RequestField, format_header_param_rfc2231 class MultipartEncoder(object): @@ -239,7 +239,8 @@ def _iter_fields(self): name=k, data=file_pointer, filename=file_name, - headers=file_headers + headers=file_headers, + header_formatter=format_header_param_rfc2231 ) field.make_multipart(content_type=file_type) diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index 6e6e73676e..69f55eb292 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -22,6 +22,7 @@ def patharg(path): JSON_FILE_PATH = FIXTURES_ROOT / 'test.json' JSON_WITH_DUPE_KEYS_FILE_PATH = FIXTURES_ROOT / 'test_with_dupe_keys.json' BIN_FILE_PATH = FIXTURES_ROOT / 'test.bin' +UTF8_IN_NAME_FILE_PATH = FIXTURES_ROOT / '天狗.txt' XML_FILES_PATH = FIXTURES_ROOT / 'xmldata' XML_FILES_VALID = list((XML_FILES_PATH / 'valid').glob('*_raw.xml')) diff --git "a/tests/fixtures/\345\244\251\347\213\227.txt" "b/tests/fixtures/\345\244\251\347\213\227.txt" new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_uploads.py b/tests/test_uploads.py index 9258f6e7ca..0ac7365e69 100644 --- a/tests/test_uploads.py +++ b/tests/test_uploads.py @@ -18,7 +18,7 @@ MockEnvironment, StdinBytesIO, http, HTTP_OK, ) -from .fixtures import FILE_PATH_ARG, FILE_PATH, FILE_CONTENT +from .fixtures import FILE_PATH_ARG, FILE_PATH, FILE_CONTENT, UTF8_IN_NAME_FILE_PATH MAX_RESPONSE_WAIT_TIME = 5 @@ -263,6 +263,18 @@ def test_multipart(self, httpbin): assert FORM_CONTENT_TYPE not in r assert 'multipart/form-data' in r + def test_multipart_with_rfc2231(self, httpbin): + """Non ascii filename should be encoded properly, following RFC2231, even if it's said + to be half obsolete. HTTP headers don't support officially UTF-8! In 2024...""" + r = http( + '--verbose', + '--multipart', + httpbin + '/post', + f'my_file@{UTF8_IN_NAME_FILE_PATH}', + ) + assert HTTP_OK in r + assert "filename*=utf-8\'\'%E5%A4%A9%E7%8B%97.txt" in r + def test_form_multipart_custom_boundary(self, httpbin): boundary = 'HTTPIE_FTW' r = http( From bb5a57f2c6133836bc92a83f3cf405f02e77318d Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Thu, 4 Apr 2024 07:58:00 +0200 Subject: [PATCH 25/63] ignore "subprocess" is still running and this process pid=x is multi-threaded warnings --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 5f2933e0cd..0d13bdaaa7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,8 @@ filterwarnings = # Happen in Windows. Oppose no threats to our test suite. # "An operation was attempted on something that is not a socket" during shutdown ignore:Exception in thread:pytest.PytestUnhandledThreadExceptionWarning - + ignore:subprocess [0-9]+ is still running:ResourceWarning + ignore:This process \(pid=[0-9]+\) is multi\-threaded:DeprecationWarning [metadata] name = httpie From ea2d675c21656812dfdfc768377f97b22ac57ed9 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Thu, 4 Apr 2024 08:21:43 +0200 Subject: [PATCH 26/63] remove 'verify=False' when requesting packages.httpie.io for latest versions published --- httpie/internal/update_warnings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/httpie/internal/update_warnings.py b/httpie/internal/update_warnings.py index c684bb80ad..1d44bffe6b 100644 --- a/httpie/internal/update_warnings.py +++ b/httpie/internal/update_warnings.py @@ -41,8 +41,7 @@ def _fetch_updates(env: Environment) -> str: file = env.config.version_info_file data = _read_data_error_free(file) - response = niquests.get(PACKAGE_INDEX_LINK, verify=False) - response.raise_for_status() + response = niquests.get(PACKAGE_INDEX_LINK).raise_for_status() data.setdefault('last_warned_date', None) data['last_fetched_date'] = datetime.now().isoformat() From e181193e53e9053c2bbef898987a405307555312 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Thu, 4 Apr 2024 08:36:18 +0200 Subject: [PATCH 27/63] don't chain call niquests.get and raise_for_status due to mock in test_update_warnings.py --- httpie/internal/update_warnings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/httpie/internal/update_warnings.py b/httpie/internal/update_warnings.py index 1d44bffe6b..5150d0c87f 100644 --- a/httpie/internal/update_warnings.py +++ b/httpie/internal/update_warnings.py @@ -41,7 +41,8 @@ def _fetch_updates(env: Environment) -> str: file = env.config.version_info_file data = _read_data_error_free(file) - response = niquests.get(PACKAGE_INDEX_LINK).raise_for_status() + response = niquests.get(PACKAGE_INDEX_LINK) + response.raise_for_status() data.setdefault('last_warned_date', None) data['last_fetched_date'] = datetime.now().isoformat() From c859191480b9ba3a291759d01d437172a9c6b2bd Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Thu, 4 Apr 2024 08:41:36 +0200 Subject: [PATCH 28/63] add note for why 'Host' header cannot be unset --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb856a4eb2..78747e7e55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [4.0.0.b1](https://github.com/httpie/cli/compare/3.2.2...master) (unreleased) - Make it possible to [unset](https://httpie.io/docs/cli/default-request-headers) the `User-Agent`, and `Accept-Encoding` headers. ([#1502](https://github.com/httpie/cli/issues/1502)) + The `Host` header cannot be unset due to support for HTTP/2+ (internally translated into `:authority`). - Dependency on requests was changed in favor of compatible niquests. ([#1531](https://github.com/httpie/cli/pull/1531)) - Added support for HTTP/2, and HTTP/3 protocols. ([#523](https://github.com/httpie/cli/issues/523)) ([#692](https://github.com/httpie/cli/issues/692)) ([#1531](https://github.com/httpie/cli/pull/1531)) - Added request metadata for the TLS certificate, negotiated version with cipher, the revocation status and the remote peer IP address. ([#1495](https://github.com/httpie/cli/issues/1495)) ([#1023](https://github.com/httpie/cli/issues/1023)) ([#826](https://github.com/httpie/cli/issues/826)) ([#1531](https://github.com/httpie/cli/pull/1531)) From eb93eabac90292d0aa124e6a69c75fc3e285678c Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Thu, 4 Apr 2024 16:00:41 +0200 Subject: [PATCH 29/63] fix infinite loop update checker in a specific condition close issue https://github.com/httpie/cli/issues/1527 --- CHANGELOG.md | 1 + httpie/config.py | 4 ++++ httpie/internal/daemons.py | 4 ++-- httpie/internal/update_warnings.py | 28 +++++++++++++++++++++++----- tests/test_config.py | 15 +++++++++++++++ 5 files changed, 45 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78747e7e55..ef3b884a6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Removed support for keeping the original casing of HTTP headers. This come from an outer constraint by newer protocols, namely HTTP/2+ that normalize header keys by default. From the HTTPie user perspective, they are "prettified" on the output by default. e.g. "x-hello-world" is displayed as "X-Hello-World". - Fixed multipart form data having filename not rfc2231 compliant when name contain non-ascii characters. ([#1401](https://github.com/httpie/cli/issues/1401)) +- Fixed issue where the configuration directory was not created at runtime that made the update fetcher run everytime. ([#1527](https://github.com/httpie/cli/issues/1527)) The plugins are expected to work without any changes. The only caveat would be that certain plugin explicitly require `requests`. Future contributions may be made in order to relax the constraints where applicable. diff --git a/httpie/config.py b/httpie/config.py index 5e6fc1245e..34ae72ca4c 100644 --- a/httpie/config.py +++ b/httpie/config.py @@ -143,6 +143,10 @@ class Config(BaseConfigDict): def __init__(self, directory: Union[str, Path] = DEFAULT_CONFIG_DIR): self.directory = Path(directory) super().__init__(path=self.directory / self.FILENAME) + # this one ensure we do not init HTTPie without the proper config directory + # there's an issue where the fetch_update daemon run without having the directory present. that induce a + # loop trying to fetch latest versions information. + self.ensure_directory() self.update(self.DEFAULTS) @property diff --git a/httpie/internal/daemons.py b/httpie/internal/daemons.py index 929f960ca0..4289ec5553 100644 --- a/httpie/internal/daemons.py +++ b/httpie/internal/daemons.py @@ -109,8 +109,8 @@ def _spawn(args: List[str], process_context: ProcessContext) -> None: _spawn_posix(args, process_context) -def spawn_daemon(task: str) -> None: - args = [task, '--daemon'] +def spawn_daemon(task: str, *args: str) -> None: + args = [task, '--daemon', *args] process_context = os.environ.copy() if not is_frozen: file_path = os.path.abspath(inspect.stack()[0][1]) diff --git a/httpie/internal/update_warnings.py b/httpie/internal/update_warnings.py index 5150d0c87f..fb4df97d6c 100644 --- a/httpie/internal/update_warnings.py +++ b/httpie/internal/update_warnings.py @@ -37,16 +37,34 @@ def _read_data_error_free(file: Path) -> Any: return {} -def _fetch_updates(env: Environment) -> str: +def _fetch_updates(env: Environment) -> None: file = env.config.version_info_file data = _read_data_error_free(file) - response = niquests.get(PACKAGE_INDEX_LINK) - response.raise_for_status() + try: + # HTTPie have a server that can return latest versions for various + # package channels, we shall attempt to retrieve this information once in a while + if hasattr(env.args, "verify"): + if env.args.verify.lower() in {"yes", "true", "no", "false"}: + verify = env.args.verify.lower() in {"yes", "true"} + else: + verify = env.args.verify + else: + verify = True + + response = niquests.get(PACKAGE_INDEX_LINK, verify=verify) + response.raise_for_status() + versions = response.json() + except (niquests.exceptions.ConnectionError, niquests.exceptions.HTTPError): + # in case of an error, let's ignore to avoid looping indefinitely. + # (spawn daemon background task maybe_fetch_update) + versions = { + BUILD_CHANNEL: httpie.__version__ + } data.setdefault('last_warned_date', None) data['last_fetched_date'] = datetime.now().isoformat() - data['last_released_versions'] = response.json() + data['last_released_versions'] = versions with open_with_lockfile(file, 'w') as stream: json.dump(data, stream) @@ -54,7 +72,7 @@ def _fetch_updates(env: Environment) -> str: def fetch_updates(env: Environment, lazy: bool = True): if lazy: - spawn_daemon('fetch_updates') + spawn_daemon('fetch_updates', f'--verify={env.args.verify}') else: _fetch_updates(env) diff --git a/tests/test_config.py b/tests/test_config.py index 1d2eea0750..d987b7eca9 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,3 +1,4 @@ +import os.path from pathlib import Path import pytest @@ -23,6 +24,20 @@ def test_default_options(httpbin): } +def test_config_dir_is_created(): + dir_path = str(get_default_config_dir()) + "--fake" + + try: + os.rmdir(dir_path) + except FileNotFoundError: + pass + + assert not os.path.exists(dir_path) + Config(dir_path) + assert os.path.exists(dir_path) + os.rmdir(dir_path) + + def test_config_file_not_valid(httpbin): env = MockEnvironment() env.create_temp_config_dir() From 4ae49f181330afcea16d38b40f8976b2f9ea89c9 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 24 Apr 2024 08:06:04 +0200 Subject: [PATCH 30/63] add section for protocol combinations and http/3 support note --- docs/README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/README.md b/docs/README.md index 0432b66f8b..1afdcbc9ee 100644 --- a/docs/README.md +++ b/docs/README.md @@ -283,6 +283,13 @@ $ http --version Note that on your machine, the version name will have the `.dev0` suffix. +### HTTP/3 support + +Support for HTTP/3 is available by default if both your interpreter and architecture are served by `qh3` published pre-built wheels. +The underlying library **Niquests** does not enforce its installation in order to avoid friction for most users. + +See https://urllib3future.readthedocs.io/en/latest/user-guide.html#http-2-and-http-3-support to learn more. + ## Usage Hello World: @@ -1884,6 +1891,21 @@ The remote server yield its support for HTTP/3 in the Alt-Svc header, if present the successive requests via HTTP/3. You may use that argument in case the remote peer does not support either HTTP/1.1 or HTTP/2. +## Protocol combinations + +Following `Force HTTP/3` and `Disable HTTP/2, or HTTP/3`, you may find a summary on how to make HTTPie negotiate a +specific protocol. + +| Argument(s) | Enabled protocol(s) | +|-----------------------------------|---------------------| +| `--disable-http2` | HTTP/1.1 or HTTP/3 | +| `--disable-http2 --disable-http3` | HTTP/1.1 | +| `--disable-http3` | HTTP/1.1 or HTTP/2 | +| `--http3` | HTTP/3 | + +You cannot enforce HTTP/2 without prior knowledge nor can you negotiate it without TLS and ALPN. +Also, you may not disable HTTP/1.1 as it is ultimately used as a fallback in case HTTP/2 and HTTP/3 are not supported. + ## Custom DNS resolver ### Using DNS url From 6c55c94cf390bd4fb660c019881ed8414f5ead79 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 24 Apr 2024 08:11:11 +0200 Subject: [PATCH 31/63] downgrade macos-14 to macos-13 due to missing brotlicffi->cffi wheels --- .github/workflows/release-brew.yml | 2 +- .github/workflows/test-package-mac-brew.yml | 2 +- .github/workflows/tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-brew.yml b/.github/workflows/release-brew.yml index d58e6b6e3d..3ececcd303 100644 --- a/.github/workflows/release-brew.yml +++ b/.github/workflows/release-brew.yml @@ -11,7 +11,7 @@ on: jobs: brew-release: name: Release the Homebrew Package - runs-on: macos-latest + runs-on: macos-13 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test-package-mac-brew.yml b/.github/workflows/test-package-mac-brew.yml index babdaa5def..38b42aa301 100644 --- a/.github/workflows/test-package-mac-brew.yml +++ b/.github/workflows/test-package-mac-brew.yml @@ -9,7 +9,7 @@ on: jobs: brew: - runs-on: macos-latest + runs-on: macos-13 steps: - uses: actions/checkout@v3 - name: Setup brew diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 021eb0e530..787867928f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-13, windows-latest] python-version: - '3.12' - '3.11' From 007234ff53675360e32181b3db695d101f10784f Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Mon, 20 May 2024 17:21:57 +0200 Subject: [PATCH 32/63] Tweak --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef3b884a6a..f677b07358 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Fixed multipart form data having filename not rfc2231 compliant when name contain non-ascii characters. ([#1401](https://github.com/httpie/cli/issues/1401)) - Fixed issue where the configuration directory was not created at runtime that made the update fetcher run everytime. ([#1527](https://github.com/httpie/cli/issues/1527)) -The plugins are expected to work without any changes. The only caveat would be that certain plugin explicitly require `requests`. +Existing plugins are expected to work without any changes. The only caveat would be that certain plugin explicitly require `requests`. Future contributions may be made in order to relax the constraints where applicable. ## [3.2.2](https://github.com/httpie/cli/compare/3.2.1...3.2.2) (2022-05-19) From 71ab43e5cb00b92b8940e239673d01f366dc41cc Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 22 May 2024 07:56:45 +0200 Subject: [PATCH 33/63] fix test_secure_cookies_on_localhost case non https server were expected to receive "secure" cookies...? this seems to be a bug that lied in Requests for quite some time. --- CHANGELOG.md | 1 + tests/test_sessions.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f677b07358..cd0368dca2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). From the HTTPie user perspective, they are "prettified" on the output by default. e.g. "x-hello-world" is displayed as "X-Hello-World". - Fixed multipart form data having filename not rfc2231 compliant when name contain non-ascii characters. ([#1401](https://github.com/httpie/cli/issues/1401)) - Fixed issue where the configuration directory was not created at runtime that made the update fetcher run everytime. ([#1527](https://github.com/httpie/cli/issues/1527)) +- Fixed cookie persistence in HTTPie session when targeting localhost. They were dropped due to the standard library. ([#1527](https://github.com/httpie/cli/issues/1527)) Existing plugins are expected to work without any changes. The only caveat would be that certain plugin explicitly require `requests`. Future contributions may be made in order to relax the constraints where applicable. diff --git a/tests/test_sessions.py b/tests/test_sessions.py index aa5243487d..83cc5385a8 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -822,19 +822,27 @@ def test_session_multiple_headers_with_same_name(basic_session, httpbin): [ ( 'localhost_http_server', - {'secure_cookie': 'foo', 'insecure_cookie': 'bar'} + {'insecure_cookie': 'bar'} ), ( 'remote_httpbin', {'insecure_cookie': 'bar'} + ), + ( + 'httpbin_secure_untrusted', + {'secure_cookie': 'foo', 'insecure_cookie': 'bar'} ) ] ) def test_secure_cookies_on_localhost(mock_env, tmp_path, server, expected_cookies, request): server = request.getfixturevalue(server) session_path = tmp_path / 'session.json' + server = str(server).replace('127.0.0.1', 'localhost') + additional_args = ['--verify=no'] if "https" in server else [] + http( '--session', str(session_path), + *additional_args, server + '/cookies/set', 'secure_cookie==foo', 'insecure_cookie==bar' @@ -847,6 +855,8 @@ def test_secure_cookies_on_localhost(mock_env, tmp_path, server, expected_cookie r = http( '--session', str(session_path), + *additional_args, server + '/cookies' ) + assert r.json == {'cookies': expected_cookies} From d9db5a2b32b2dbfba5fb1a2bb630c008c6ee333f Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Thu, 23 May 2024 06:09:57 +0200 Subject: [PATCH 34/63] fix downloader with compressed content #1554 #423 --- CHANGELOG.md | 1 + httpie/downloads.py | 34 ++++++++++++++++++++++++++++++---- tests/test_downloads.py | 4 ++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd0368dca2..47f2248ee4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Fixed multipart form data having filename not rfc2231 compliant when name contain non-ascii characters. ([#1401](https://github.com/httpie/cli/issues/1401)) - Fixed issue where the configuration directory was not created at runtime that made the update fetcher run everytime. ([#1527](https://github.com/httpie/cli/issues/1527)) - Fixed cookie persistence in HTTPie session when targeting localhost. They were dropped due to the standard library. ([#1527](https://github.com/httpie/cli/issues/1527)) +- Fixed downloader when trying to fetch compressed content. The process will no longer exit with the "Incomplete download" error. ([#1554](https://github.com/httpie/cli/issues/1554)) ([#423](https://github.com/httpie/cli/issues/423)) ([#1527](https://github.com/httpie/cli/issues/1527)) Existing plugins are expected to work without any changes. The only caveat would be that certain plugin explicitly require `requests`. Future contributions may be made in order to relax the constraints where applicable. diff --git a/httpie/downloads.py b/httpie/downloads.py index d987b0c989..d9c711c5fe 100644 --- a/httpie/downloads.py +++ b/httpie/downloads.py @@ -217,11 +217,37 @@ def start( """ assert not self.status.time_started - # FIXME: some servers still might sent Content-Encoding: gzip - # try: - total_size = int(final_response.headers['Content-Length']) - except (KeyError, ValueError, TypeError): + supported_decoders = final_response.raw.CONTENT_DECODERS + except AttributeError: + supported_decoders = ["gzip", "deflate"] + + use_content_length = True + + # If the content is actually compressed, the http client will automatically + # stream decompressed content. This ultimately means that the server send the content-length + # that is related to the compressed body. this might fool the downloader. + # but... there's a catch, we don't decompress everything, everytime. It depends on the + # Content-Encoding. + if 'Content-Encoding' in final_response.headers: + will_decompress = True + + encoding_list = final_response.headers['Content-Encoding'].replace(' ', '').lower().split(',') + + for encoding in encoding_list: + if encoding not in supported_decoders: + will_decompress = False + break + + if will_decompress: + use_content_length = False + + if use_content_length: + try: + total_size = int(final_response.headers['Content-Length']) + except (KeyError, ValueError, TypeError): + total_size = None + else: total_size = None if not self._output_file: diff --git a/tests/test_downloads.py b/tests/test_downloads.py index 6bd8dcc609..f63273aebf 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -259,3 +259,7 @@ def test_download_with_redirect_original_url_used_for_filename(self, httpbin): assert os.listdir('.') == [expected_filename] finally: os.chdir(orig_cwd) + + def test_download_gzip_content_encoding(self, httpbin): + r = http('--download', httpbin + '/gzip') + assert r.exit_status == 0 From ec02b1c74ef0e7ac31ec06d0d2a559a0beed88fb Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 24 May 2024 06:49:11 +0200 Subject: [PATCH 35/63] add support for resolving ".localhost" domains close https://github.com/httpie/cli/issues/1458 --- CHANGELOG.md | 1 + httpie/client.py | 16 +++++++++++++--- tests/test_regressions.py | 10 ++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47f2248ee4..f9b26d3efc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Fixed issue where the configuration directory was not created at runtime that made the update fetcher run everytime. ([#1527](https://github.com/httpie/cli/issues/1527)) - Fixed cookie persistence in HTTPie session when targeting localhost. They were dropped due to the standard library. ([#1527](https://github.com/httpie/cli/issues/1527)) - Fixed downloader when trying to fetch compressed content. The process will no longer exit with the "Incomplete download" error. ([#1554](https://github.com/httpie/cli/issues/1554)) ([#423](https://github.com/httpie/cli/issues/423)) ([#1527](https://github.com/httpie/cli/issues/1527)) +- Added automated resolution of hosts ending with `.localhost` to the default loopback address. ([#1458](https://github.com/httpie/cli/issues/1458)) ([#1527](https://github.com/httpie/cli/issues/1527)) Existing plugins are expected to work without any changes. The only caveat would be that certain plugin explicitly require `requests`. Future contributions may be made in order to relax the constraints where applicable. diff --git a/httpie/client.py b/httpie/client.py index b04d78515e..eea5d6a00b 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -77,21 +77,31 @@ def collect_messages( min_port, max_port = args.local_port.split('-', 1) source_address = (args.interface or "0.0.0.0", randint(int(min_port), int(max_port))) + parsed_url = parse_url(args.url) + resolver = args.resolver or None + + # we want to make sure every ".localhost" host resolve to loopback + if parsed_url.host and parsed_url.host.endswith(".localhost"): + ensure_resolver = f"in-memory://default/?hosts={parsed_url.host}:127.0.0.1&hosts={parsed_url.host}:[::1]" + + if resolver and isinstance(resolver, list): + resolver.append(ensure_resolver) + else: + resolver = [ensure_resolver, "system://"] + requests_session = build_requests_session( ssl_version=args.ssl_version, ciphers=args.ciphers, verify=bool(send_kwargs_mergeable_from_env['verify']), disable_http2=args.disable_http2, disable_http3=args.disable_http3, - resolver=args.resolver or None, + resolver=resolver, disable_ipv6=args.ipv4, disable_ipv4=args.ipv6, source_address=source_address, quic_cache=env.config.quic_file, ) - parsed_url = parse_url(args.url) - if args.disable_http3 is False and args.force_http3 is True: requests_session.quic_cache_layer[(parsed_url.host, parsed_url.port or 443)] = (parsed_url.host, parsed_url.port or 443) # well, this one is tricky. If we allow HTTP/3, and remote host was marked as QUIC capable diff --git a/tests/test_regressions.py b/tests/test_regressions.py index a5cd00a076..44c675c777 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -46,3 +46,13 @@ def test_verbose_redirected_stdout_separator(httpbin): Expect.RESPONSE_HEADERS, Expect.BODY, ]) + + +def test_every_localhost_resolve(httpbin): + """ + https://github.com/httpie/cli/issues/1458 + + """ + new_target = str(httpbin).replace('127.0.0.1', 'example.localhost') + assert 'example.localhost' in new_target + http(str(httpbin).replace('127.0.0.1', 'example.localhost') + '/get') From 2583671e038a993be407e64b06c651d8d7e535df Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Fri, 24 May 2024 18:32:46 +0200 Subject: [PATCH 36/63] Changelog tweaks --- CHANGELOG.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9b26d3efc..64c3ccc68c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,33 +5,33 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [4.0.0.b1](https://github.com/httpie/cli/compare/3.2.2...master) (unreleased) -- Make it possible to [unset](https://httpie.io/docs/cli/default-request-headers) the `User-Agent`, and `Accept-Encoding` headers. ([#1502](https://github.com/httpie/cli/issues/1502)) +- Switched from [`requests`](https://github.com/psf/requests) to the compatible [`niquests`](https://github.com/jawah/niquests). ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for HTTP/2, and HTTP/3 protocols. ([#523](https://github.com/httpie/cli/issues/523), [#692](https://github.com/httpie/cli/issues/692), [#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for IPv4/IPv6 enforcement with `-6` and `-4`. ([#94](https://github.com/httpie/cli/issues/94), [#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for alternative DNS resolvers via `--resolver`. DNS over HTTPS, DNS over TLS, DNS over QUIC, and DNS over UDP are accepted. ([#99](https://github.com/httpie/cli/issues/99), [#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for binding to a specific network adapter with `--interface`. ([#1422](https://github.com/httpie/cli/issues/1422), [#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for specifying the local port with `--local-port`. ([#1456](https://github.com/httpie/cli/issues/1456), [#1531](https://github.com/httpie/cli/pull/1531)) +- Added the ability to [unset](https://httpie.io/docs/cli/default-request-headers) the `User-Agent`, and `Accept-Encoding` headers. ([#1502](https://github.com/httpie/cli/issues/1502)) The `Host` header cannot be unset due to support for HTTP/2+ (internally translated into `:authority`). -- Dependency on requests was changed in favor of compatible niquests. ([#1531](https://github.com/httpie/cli/pull/1531)) -- Added support for HTTP/2, and HTTP/3 protocols. ([#523](https://github.com/httpie/cli/issues/523)) ([#692](https://github.com/httpie/cli/issues/692)) ([#1531](https://github.com/httpie/cli/pull/1531)) - Added request metadata for the TLS certificate, negotiated version with cipher, the revocation status and the remote peer IP address. ([#1495](https://github.com/httpie/cli/issues/1495)) ([#1023](https://github.com/httpie/cli/issues/1023)) ([#826](https://github.com/httpie/cli/issues/826)) ([#1531](https://github.com/httpie/cli/pull/1531)) -- Added support to load the operating system trust store for the peer certificate validation. ([#480](https://github.com/httpie/cli/issues/480)) ([#1531](https://github.com/httpie/cli/pull/1531)) -- Added detailed timings in response metadata with DNS resolution, established, TLS handshake, and request sending delays. ([#1023](https://github.com/httpie/cli/issues/1023)) ([#1531](https://github.com/httpie/cli/pull/1531)) -- Added support for using alternative DNS resolver using `--resolver`. DNS over HTTPS, DNS over TLS, DNS over QUIC, and DNS over UDP are accepted. ([#99](https://github.com/httpie/cli/issues/99)) ([#1531](https://github.com/httpie/cli/pull/1531)) -- Added support for binding to a specific network adapter with `--interface`. ([#1422](https://github.com/httpie/cli/issues/1422)) ([#1531](https://github.com/httpie/cli/pull/1531)) -- Added support for specifying the local port with `--local-port`. ([#1456](https://github.com/httpie/cli/issues/1456)) ([#1531](https://github.com/httpie/cli/pull/1531)) -- Added support for forcing either IPv4 or IPv6 to reach the remote HTTP server with `-6` or `-4`. ([#94](https://github.com/httpie/cli/issues/94)) ([#1531](https://github.com/httpie/cli/pull/1531)) -- Removed support for pyopenssl. ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added support to load the operating system trust store for the peer certificate validation. ([#480](https://github.com/httpie/cli/issues/480), [#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for using the system trust store to retrieve root CAs for verifying TLS certificates. ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added detailed timings in response metadata with DNS resolution, established, TLS handshake, and request sending delays. ([#1023](https://github.com/httpie/cli/issues/1023), [#1531](https://github.com/httpie/cli/pull/1531)) +- Added automated resolution of hosts ending with `.localhost` to the default loopback address. ([#1458](https://github.com/httpie/cli/issues/1458), [#1527](https://github.com/httpie/cli/issues/1527)) +- Removed support for `pyopenssl`. ([#1531](https://github.com/httpie/cli/pull/1531)) - Removed support for dead SSL protocols < TLS 1.0 (e.g. sslv3) as per pyopenssl removal. ([#1531](https://github.com/httpie/cli/pull/1531)) -- Dropped dependency on `requests_toolbelt` in favor of directly including `MultipartEncoder` into HTTPie due to its direct dependency to requests. ([#1531](https://github.com/httpie/cli/pull/1531)) -- Dropped dependency on `multidict` in favor of implementing an internal one due to often missing pre-built wheels. ([#1522](https://github.com/httpie/cli/issues/1522)) ([#1531](https://github.com/httpie/cli/pull/1531)) -- Fixed the case when multiple headers where concatenated in the response output. ([#1413](https://github.com/httpie/cli/issues/1413)) ([#1531](https://github.com/httpie/cli/pull/1531)) -- Fixed an edge case where HTTPie could be lead to believe data was passed in stdin, thus sending a POST by default. ([#1551](https://github.com/httpie/cli/issues/1551)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Removed dependency on `requests_toolbelt` in favor of directly including `MultipartEncoder` into HTTPie due to its direct dependency to requests. ([#1531](https://github.com/httpie/cli/pull/1531)) +- Removed dependency on `multidict` in favor of implementing an internal one due to often missing pre-built wheels. ([#1522](https://github.com/httpie/cli/issues/1522), [#1531](https://github.com/httpie/cli/pull/1531)) +- Fixed the case when multiple headers where concatenated in the response output. ([#1413](https://github.com/httpie/cli/issues/1413), [#1531](https://github.com/httpie/cli/pull/1531)) +- Fixed an edge case where HTTPie could be lead to believe data was passed in stdin, thus sending a POST by default. ([#1551](https://github.com/httpie/cli/issues/1551), [#1531](https://github.com/httpie/cli/pull/1531)) This fix has the particularity to consider 0 byte long stdin buffer as absent stdin. Empty stdin buffer will be ignored. - Slightly improved performance while downloading by setting chunk size to `-1` to retrieve packets as they arrive. ([#1531](https://github.com/httpie/cli/pull/1531)) -- Added support for using the system trust store to retrieve root CAs for verifying TLS certificates. ([#1531](https://github.com/httpie/cli/pull/1531)) - Removed support for keeping the original casing of HTTP headers. This come from an outer constraint by newer protocols, namely HTTP/2+ that normalize header keys by default. From the HTTPie user perspective, they are "prettified" on the output by default. e.g. "x-hello-world" is displayed as "X-Hello-World". - Fixed multipart form data having filename not rfc2231 compliant when name contain non-ascii characters. ([#1401](https://github.com/httpie/cli/issues/1401)) - Fixed issue where the configuration directory was not created at runtime that made the update fetcher run everytime. ([#1527](https://github.com/httpie/cli/issues/1527)) - Fixed cookie persistence in HTTPie session when targeting localhost. They were dropped due to the standard library. ([#1527](https://github.com/httpie/cli/issues/1527)) -- Fixed downloader when trying to fetch compressed content. The process will no longer exit with the "Incomplete download" error. ([#1554](https://github.com/httpie/cli/issues/1554)) ([#423](https://github.com/httpie/cli/issues/423)) ([#1527](https://github.com/httpie/cli/issues/1527)) -- Added automated resolution of hosts ending with `.localhost` to the default loopback address. ([#1458](https://github.com/httpie/cli/issues/1458)) ([#1527](https://github.com/httpie/cli/issues/1527)) +- Fixed downloader when trying to fetch compressed content. The process will no longer exit with the "Incomplete download" error. ([#1554](https://github.com/httpie/cli/issues/1554), [#423](https://github.com/httpie/cli/issues/423), [#1527](https://github.com/httpie/cli/issues/1527)) Existing plugins are expected to work without any changes. The only caveat would be that certain plugin explicitly require `requests`. Future contributions may be made in order to relax the constraints where applicable. From 523faa674d23e618f6359b0acc1e4411802f018d Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Sun, 26 May 2024 05:52:01 +0200 Subject: [PATCH 37/63] adjust docs around tls version cli command --- docs/README.md | 2 +- extras/man/http.1 | 9 ++++----- extras/man/https.1 | 9 ++++----- extras/profiling/benchmarks.py | 4 ++-- httpie/cli/definition.py | 9 ++++----- 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/docs/README.md b/docs/README.md index 1afdcbc9ee..a979f1c0bd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1537,7 +1537,7 @@ $ http --cert=client.pem --cert-key=client.key --cert-key-pass=my_password https ### SSL version Use the `--ssl=` option to specify the desired protocol version to use. -This will default to TLS v1.0 which will negotiate the highest protocol that both the server and your installation of OpenSSL support. +If not specified, it tries to negotiate the highest protocol that both the server and your installation of OpenSSL support. The available protocols are `tls1`, `tls1.1`, `tls1.2`, `tls1.3`. (The actually available set of protocols may vary depending on your OpenSSL installation.) diff --git a/extras/man/http.1 b/extras/man/http.1 index 634cb84372..fd5a674828 100644 --- a/extras/man/http.1 +++ b/extras/man/http.1 @@ -558,11 +558,10 @@ variable instead.) .IP "\fB\,--ssl\/\fR" -The desired protocol version to use. This will default to -SSL v2.3 which will negotiate the highest protocol that both -the server and your installation of OpenSSL support. Available protocols -may vary depending on OpenSSL installation (only the supported ones -are shown here). +The desired protocol version to use. If not specified, it tries to +negotiate the highest protocol that both the server and your installation +of OpenSSL support. Available protocols may vary depending on OpenSSL +installation (only the supported ones are shown here). diff --git a/extras/man/https.1 b/extras/man/https.1 index 6a7370e071..ba3399d718 100644 --- a/extras/man/https.1 +++ b/extras/man/https.1 @@ -558,11 +558,10 @@ variable instead.) .IP "\fB\,--ssl\/\fR" -The desired protocol version to use. This will default to -SSL v2.3 which will negotiate the highest protocol that both -the server and your installation of OpenSSL support. Available protocols -may vary depending on OpenSSL installation (only the supported ones -are shown here). +The desired protocol version to use. If not specified, it tries to +negotiate the highest protocol that both the server and your installation +of OpenSSL support. Available protocols may vary depending on OpenSSL +installation (only the supported ones are shown here). diff --git a/extras/profiling/benchmarks.py b/extras/profiling/benchmarks.py index 9d409debbe..04ea37a203 100644 --- a/extras/profiling/benchmarks.py +++ b/extras/profiling/benchmarks.py @@ -175,11 +175,11 @@ def run(self, context: Context) -> pyperf.Benchmark: for pretty in ['all', 'none']: CommandRunner( 'startup', - f'`http --pretty={pretty} pie.dev/stream/1000`', + f'`http --pretty={pretty} httpbin.local:8888/stream/1000`', [ '--print=HBhb', f'--pretty={pretty}', - 'httpbin.org/stream/1000' + 'httpbin.local:8888/stream/1000' ] ) DownloadRunner('download', '`http --download :/big_file.txt` (3GB)', '3G') diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 1a412dc50b..2a1a6c7072 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -896,11 +896,10 @@ def format_auth_help(auth_plugins_mapping, *, isolation_mode: bool = False): choices=sorted(AVAILABLE_SSL_VERSION_ARG_MAPPING.keys()), short_help='The desired protocol version to used.', help=""" - The desired protocol version to use. This will default to - SSL v2.3 which will negotiate the highest protocol that both - the server and your installation of OpenSSL support. Available protocols - may vary depending on OpenSSL installation (only the supported ones - are shown here). + The desired protocol version to use. If not specified, it tries to + negotiate the highest protocol that both the server and your installation + of OpenSSL support. Available protocols may vary depending on OpenSSL + installation (only the supported ones are shown here). """, ) From e3cbc7ff286c3b5a362a310a028a7eb3badcc14e Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Sun, 26 May 2024 07:58:18 +0200 Subject: [PATCH 38/63] improve support --ssl=[...] setting --- extras/man/http.1 | 3 ++- extras/man/https.1 | 3 ++- extras/profiling/benchmarks.py | 4 ++-- httpie/cli/definition.py | 5 +++-- httpie/ssl_.py | 27 ++++++++++++++++++++++++++- 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/extras/man/http.1 b/extras/man/http.1 index fd5a674828..ed69895f8a 100644 --- a/extras/man/http.1 +++ b/extras/man/http.1 @@ -570,9 +570,10 @@ installation (only the supported ones are shown here). A string in the OpenSSL cipher list format. +tls1.3 ciphers are always present regardless of your cipher list. -See `http \fB\,--help\/\fR` for the default ciphers list on you system. +See `http \fB\,--help\/\fR` for the default ciphers list. diff --git a/extras/man/https.1 b/extras/man/https.1 index ba3399d718..bdb0ef82a6 100644 --- a/extras/man/https.1 +++ b/extras/man/https.1 @@ -570,9 +570,10 @@ installation (only the supported ones are shown here). A string in the OpenSSL cipher list format. +tls1.3 ciphers are always present regardless of your cipher list. -See `http \fB\,--help\/\fR` for the default ciphers list on you system. +See `http \fB\,--help\/\fR` for the default ciphers list. diff --git a/extras/profiling/benchmarks.py b/extras/profiling/benchmarks.py index 04ea37a203..c262d46977 100644 --- a/extras/profiling/benchmarks.py +++ b/extras/profiling/benchmarks.py @@ -175,11 +175,11 @@ def run(self, context: Context) -> pyperf.Benchmark: for pretty in ['all', 'none']: CommandRunner( 'startup', - f'`http --pretty={pretty} httpbin.local:8888/stream/1000`', + f'`http --pretty={pretty} pie.dev/stream/1000`', [ '--print=HBhb', f'--pretty={pretty}', - 'httpbin.local:8888/stream/1000' + 'pie.dev/stream/1000' ] ) DownloadRunner('download', '`http --download :/big_file.txt` (3GB)', '3G') diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 2a1a6c7072..7287eca6bb 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -906,12 +906,12 @@ def format_auth_help(auth_plugins_mapping, *, isolation_mode: bool = False): CIPHERS_CURRENT_DEFAULTS = ( """ - See `http --help` for the default ciphers list on you system. + See `http --help` for the default ciphers list. """ if IS_MAN_PAGE else f""" - By default, the following ciphers are used on your system: + By default, the following ciphers are used: {DEFAULT_SSL_CIPHERS_STRING} @@ -923,6 +923,7 @@ def format_auth_help(auth_plugins_mapping, *, isolation_mode: bool = False): help=f""" A string in the OpenSSL cipher list format. + tls1.3 ciphers are always present regardless of your cipher list. {CIPHERS_CURRENT_DEFAULTS} diff --git a/httpie/ssl_.py b/httpie/ssl_.py index 888be5540d..9dbd27005f 100644 --- a/httpie/ssl_.py +++ b/httpie/ssl_.py @@ -14,7 +14,7 @@ 'tls1': 'PROTOCOL_TLSv1', 'tls1.1': 'PROTOCOL_TLSv1_1', 'tls1.2': 'PROTOCOL_TLSv1_2', - 'tls1.3': 'PROTOCOL_TLSv1_3', + 'tls1.3': 'PROTOCOL_TLS_CLIENT', # CPython does not have a "PROTOCOL_TLSv1_3" constant, so, we'll improvise. } # todo: we'll need to update this in preparation for Python 3.13+ # could be a removal (after a long deprecation about constants @@ -104,6 +104,18 @@ def __init__( self._verify = None if ssl_version or ciphers: + # By default, almost all installed CPython have modern OpenSSL backends + # This actively prevent folks to negotiate "almost" dead TLS protocols + # HTTPie wants to help users when they explicitly expect "old" TLS support + # Common errors for user if not set: + # >- [SSL: NO_CIPHERS_AVAILABLE] no ciphers available + # >- [SSL: LEGACY_SIGALG_DISALLOWED_OR_UNSUPPORTED] legacy sigalg disallowed or unsupported + if ssl_version in {ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1} and ciphers is None: + # Please do not raise a "security" concern for that line. + # If the interpreter reach that line, it means that the user willingly set + # an unsafe TLS protocol. + ciphers = "DEFAULT:@SECLEVEL=0" + # Only set the custom context if user supplied one. # Because urllib3-future set his own secure ctx with a set of # ciphers (moz recommended list). thus avoiding excluding QUIC @@ -142,6 +154,19 @@ def _create_ssl_context( ssl_version: str = None, ciphers: str = None, ) -> 'ssl.SSLContext': + # HTTPie will take `ssl.PROTOCOL_TLS_CLIENT` as TLS 1.3 enforced! + # This piece of code is only triggered if user supplied --ssl=tls1.3 + if ssl_version is ssl.PROTOCOL_TLS_CLIENT: + return create_urllib3_context( + ciphers=ciphers, + ssl_minimum_version=ssl.TLSVersion.TLSv1_3, + ssl_maximum_version=ssl.TLSVersion.TLSv1_3, + # Since we are using a custom SSL context, we need to pass this + # here manually, even though it’s also passed to the connection + # in `super().cert_verify()`. + cert_reqs=ssl.CERT_REQUIRED if verify else ssl.CERT_NONE + ) + return create_urllib3_context( ciphers=ciphers, ssl_version=resolve_ssl_version(ssl_version), From 78aa25d51a0b74b8b99801158b309bac4651f36b Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Mon, 3 Jun 2024 13:49:36 +0200 Subject: [PATCH 39/63] Docs wip --- CHANGELOG.md | 18 ++++++++---------- docs/README.md | 45 ++++++++++++++++++++++++++------------------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64c3ccc68c..70009dd48f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,33 +5,31 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [4.0.0.b1](https://github.com/httpie/cli/compare/3.2.2...master) (unreleased) -- Switched from [`requests`](https://github.com/psf/requests) to the compatible [`niquests`](https://github.com/jawah/niquests). ([#1531](https://github.com/httpie/cli/pull/1531)) +- Switched from the [`requests`](https://github.com/psf/requests) library to the compatible [`niquests`](https://github.com/jawah/niquests). ([#1531](https://github.com/httpie/cli/pull/1531)) - Added support for HTTP/2, and HTTP/3 protocols. ([#523](https://github.com/httpie/cli/issues/523), [#692](https://github.com/httpie/cli/issues/692), [#1531](https://github.com/httpie/cli/pull/1531)) - Added support for IPv4/IPv6 enforcement with `-6` and `-4`. ([#94](https://github.com/httpie/cli/issues/94), [#1531](https://github.com/httpie/cli/pull/1531)) - Added support for alternative DNS resolvers via `--resolver`. DNS over HTTPS, DNS over TLS, DNS over QUIC, and DNS over UDP are accepted. ([#99](https://github.com/httpie/cli/issues/99), [#1531](https://github.com/httpie/cli/pull/1531)) - Added support for binding to a specific network adapter with `--interface`. ([#1422](https://github.com/httpie/cli/issues/1422), [#1531](https://github.com/httpie/cli/pull/1531)) - Added support for specifying the local port with `--local-port`. ([#1456](https://github.com/httpie/cli/issues/1456), [#1531](https://github.com/httpie/cli/pull/1531)) - Added the ability to [unset](https://httpie.io/docs/cli/default-request-headers) the `User-Agent`, and `Accept-Encoding` headers. ([#1502](https://github.com/httpie/cli/issues/1502)) - The `Host` header cannot be unset due to support for HTTP/2+ (internally translated into `:authority`). -- Added request metadata for the TLS certificate, negotiated version with cipher, the revocation status and the remote peer IP address. ([#1495](https://github.com/httpie/cli/issues/1495)) ([#1023](https://github.com/httpie/cli/issues/1023)) ([#826](https://github.com/httpie/cli/issues/826)) ([#1531](https://github.com/httpie/cli/pull/1531)) +- Added request metadata for the TLS certificate, negotiated version with cipher, the revocation status and the remote peer IP address. ([#1495](https://github.com/httpie/cli/issues/1495), [#1023](https://github.com/httpie/cli/issues/1023), [#826](https://github.com/httpie/cli/issues/826), [#1531](https://github.com/httpie/cli/pull/1531)) - Added support to load the operating system trust store for the peer certificate validation. ([#480](https://github.com/httpie/cli/issues/480), [#1531](https://github.com/httpie/cli/pull/1531)) - Added support for using the system trust store to retrieve root CAs for verifying TLS certificates. ([#1531](https://github.com/httpie/cli/pull/1531)) - Added detailed timings in response metadata with DNS resolution, established, TLS handshake, and request sending delays. ([#1023](https://github.com/httpie/cli/issues/1023), [#1531](https://github.com/httpie/cli/pull/1531)) - Added automated resolution of hosts ending with `.localhost` to the default loopback address. ([#1458](https://github.com/httpie/cli/issues/1458), [#1527](https://github.com/httpie/cli/issues/1527)) -- Removed support for `pyopenssl`. ([#1531](https://github.com/httpie/cli/pull/1531)) -- Removed support for dead SSL protocols < TLS 1.0 (e.g. sslv3) as per pyopenssl removal. ([#1531](https://github.com/httpie/cli/pull/1531)) -- Removed dependency on `requests_toolbelt` in favor of directly including `MultipartEncoder` into HTTPie due to its direct dependency to requests. ([#1531](https://github.com/httpie/cli/pull/1531)) -- Removed dependency on `multidict` in favor of implementing an internal one due to often missing pre-built wheels. ([#1522](https://github.com/httpie/cli/issues/1522), [#1531](https://github.com/httpie/cli/pull/1531)) - Fixed the case when multiple headers where concatenated in the response output. ([#1413](https://github.com/httpie/cli/issues/1413), [#1531](https://github.com/httpie/cli/pull/1531)) - Fixed an edge case where HTTPie could be lead to believe data was passed in stdin, thus sending a POST by default. ([#1551](https://github.com/httpie/cli/issues/1551), [#1531](https://github.com/httpie/cli/pull/1531)) This fix has the particularity to consider 0 byte long stdin buffer as absent stdin. Empty stdin buffer will be ignored. -- Slightly improved performance while downloading by setting chunk size to `-1` to retrieve packets as they arrive. ([#1531](https://github.com/httpie/cli/pull/1531)) -- Removed support for keeping the original casing of HTTP headers. This come from an outer constraint by newer protocols, namely HTTP/2+ that normalize header keys by default. - From the HTTPie user perspective, they are "prettified" on the output by default. e.g. "x-hello-world" is displayed as "X-Hello-World". +- Improved performance while downloading by setting chunk size to `-1` to retrieve packets as they arrive. ([#1531](https://github.com/httpie/cli/pull/1531)) - Fixed multipart form data having filename not rfc2231 compliant when name contain non-ascii characters. ([#1401](https://github.com/httpie/cli/issues/1401)) - Fixed issue where the configuration directory was not created at runtime that made the update fetcher run everytime. ([#1527](https://github.com/httpie/cli/issues/1527)) - Fixed cookie persistence in HTTPie session when targeting localhost. They were dropped due to the standard library. ([#1527](https://github.com/httpie/cli/issues/1527)) - Fixed downloader when trying to fetch compressed content. The process will no longer exit with the "Incomplete download" error. ([#1554](https://github.com/httpie/cli/issues/1554), [#423](https://github.com/httpie/cli/issues/423), [#1527](https://github.com/httpie/cli/issues/1527)) +- Removed support for preserving the original casing of HTTP headers. This comes as a constraint of newer protocols, namely HTTP/2+ that normalize header keys by default. From the HTTPie user perspective, they are "prettified" in the output by default. e.g. `x-hello-world` is displayed as `X-Hello-World`. +- Removed support for `pyopenssl`. ([#1531](https://github.com/httpie/cli/pull/1531)) +- Removed support for dead SSL protocols < TLS 1.0 (e.g. sslv3) as per pyopenssl removal. ([#1531](https://github.com/httpie/cli/pull/1531)) +- Removed dependency on `requests_toolbelt` in favor of directly including `MultipartEncoder` into HTTPie due to its direct dependency to requests. ([#1531](https://github.com/httpie/cli/pull/1531)) +- Removed dependency on `multidict` in favor aof an internal one due to often missing pre-built wheels. ([#1522](https://github.com/httpie/cli/issues/1522), [#1531](https://github.com/httpie/cli/pull/1531)) Existing plugins are expected to work without any changes. The only caveat would be that certain plugin explicitly require `requests`. Future contributions may be made in order to relax the constraints where applicable. diff --git a/docs/README.md b/docs/README.md index a979f1c0bd..d0e7b022cd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -286,7 +286,7 @@ Note that on your machine, the version name will have the `.dev0` suffix. ### HTTP/3 support Support for HTTP/3 is available by default if both your interpreter and architecture are served by `qh3` published pre-built wheels. -The underlying library **Niquests** does not enforce its installation in order to avoid friction for most users. +The underlying library Niquests does not enforce its installation in order to avoid friction for most users. See https://urllib3future.readthedocs.io/en/latest/user-guide.html#http-2-and-http-3-support to learn more. @@ -1193,22 +1193,24 @@ You can read headers from a file by using the `:@` operator. This would also eff $ http pie.dev/headers X-Data:@files/text.txt ``` -### Empty headers and header un-setting +### Empty request headers -To unset a previously specified header (such a one of the default headers), use `Header:`: +To send a header with an empty value, use `Header;`, with a semicolon: ```bash -$ http pie.dev/headers Accept: User-Agent: +$ http pie.dev/headers 'Header;' ``` -To send a header with an empty value, use `Header;`, with a semicolon: +### Header un-setting + +To unset a previously specified header or one of the [default headers](#default-request-headers), use the `Header:` notation: ```bash -$ http pie.dev/headers 'Header;' +$ http pie.dev/headers Accept: User-Agent: ``` -Please note that some internal headers, such as `Content-Length`, can’t be unset if -they are automatically added by the client itself. +Please note that some internal headers, such as `Content-Length`, can’t be unset if they are automatically added by the client itself. +Also, the `Host` header cannot be unset due to support for HTTP/2+ (internally translated into `:authority`) ### Multiple header values with the same name @@ -1864,7 +1866,11 @@ $ http --chunked pie.dev/post @files/data.xml $ cat files/data.xml | http --chunked pie.dev/post ``` -## Disable HTTP/2, or HTTP/3 +## Supported HTTP versions + +HTTPie has full support for HTTP/1.1, HTTP/2, and HTTP/3. + +### Disable HTTP/2, or HTTP/3 You can at your own discretion toggle on and off HTTP/2, or/and HTTP/3. @@ -1876,7 +1882,7 @@ $ https --disable-http2 PUT pie.dev/put hello=world $ https --disable-http3 PUT pie.dev/put hello=world ``` -## Force HTTP/3 +### Force HTTP/3 By opposition to the previous section, you can force the HTTP/3 negotiation. @@ -1887,21 +1893,22 @@ $ https --http3 pie.dev/get By default, HTTPie cannot negotiate HTTP/3 without a first HTTP/1.1, or HTTP/2 successful response unless the remote host specified a DNS HTTPS record that indicate its support (and by using a custom DNS resolver, see bellow section). -The remote server yield its support for HTTP/3 in the Alt-Svc header, if present HTTPie will issue +The remote server yield its support for HTTP/3 in the `Alt-Svc` header, if present HTTPie will issue the successive requests via HTTP/3. You may use that argument in case the remote peer does not support either HTTP/1.1 or HTTP/2. -## Protocol combinations +### Protocol combinations Following `Force HTTP/3` and `Disable HTTP/2, or HTTP/3`, you may find a summary on how to make HTTPie negotiate a specific protocol. -| Argument(s) | Enabled protocol(s) | -|-----------------------------------|---------------------| -| `--disable-http2` | HTTP/1.1 or HTTP/3 | -| `--disable-http2 --disable-http3` | HTTP/1.1 | -| `--disable-http3` | HTTP/1.1 or HTTP/2 | -| `--http3` | HTTP/3 | +| Arguments | HTTP/1.1
enabled | HTTP/2
enabled | HTTP/3
enabled | +|----------------------------------:|:--------------------:|:------------------:|:------------------:| +| (Default) | ✔ | ✔ | ✔ | +| `--disable-http2` | ✔ | ✗ | ✔ | +| `--disable-http3` | ✔ | ✔ | ✗ | +| `--disable-http2 --disable-http3` | ✔ | ✗ | ✗ | +| `--http3` | ✗ | ✗ | ✔ | You cannot enforce HTTP/2 without prior knowledge nor can you negotiate it without TLS and ALPN. Also, you may not disable HTTP/1.1 as it is ultimately used as a fallback in case HTTP/2 and HTTP/3 are not supported. @@ -1942,7 +1949,7 @@ You can specify multiple entries, concatenated with a comma: $ https --resolver "pie.dev:10.10.4.1,re.pie.dev:10.10.8.1" pie.dev/get ``` -## Attach to a specific network adapter +## Network interface In order to bind emitted request from a specific network adapter you can use the `--interface` flag. From d238725106f044d5cccdeec930c2646ebd7de978 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Mon, 3 Jun 2024 14:35:54 +0200 Subject: [PATCH 40/63] Add `cd_clean_tmp_dir()` test utility context manager --- tests/test_downloads.py | 66 ++++++++++++++++++----------------------- tests/test_output.py | 12 ++------ tests/test_sessions.py | 8 ++--- tests/utils/__init__.py | 18 +++++++++++ 4 files changed, 52 insertions(+), 52 deletions(-) diff --git a/tests/test_downloads.py b/tests/test_downloads.py index f63273aebf..8d4166a5ef 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -12,7 +12,7 @@ parse_content_range, filename_from_content_disposition, filename_from_url, get_unique_filename, ContentRangeError, Downloader, PARTIAL_CONTENT ) -from .utils import http, MockEnvironment +from .utils import http, MockEnvironment, cd_clean_tmp_dir class Response(niquests.Response): @@ -160,32 +160,26 @@ def test_download_no_Content_Length(self, mock_env, httpbin_both): assert not downloader.interrupted def test_download_output_from_content_disposition(self, mock_env, httpbin_both): - with tempfile.TemporaryDirectory() as tmp_dirname: - orig_cwd = os.getcwd() - os.chdir(tmp_dirname) - try: - assert not os.path.isfile('filename.bin') - downloader = Downloader(mock_env) - downloader.start( - final_response=Response( - url=httpbin_both.url + '/', - headers={ - 'Content-Length': 5, - 'Content-Disposition': 'attachment; filename="filename.bin"', - } - ), - initial_url='/' - ) - downloader.chunk_downloaded(b'12345') - downloader.finish() - downloader.failed() # Stop the reporter - assert not downloader.interrupted + output_file_name = 'filename.bin' + with cd_clean_tmp_dir(assert_filenames_after=[output_file_name]): + downloader = Downloader(mock_env) + downloader.start( + final_response=Response( + url=httpbin_both.url + '/', + headers={ + 'Content-Length': 5, + 'Content-Disposition': f'attachment; filename="{output_file_name}"', + } + ), + initial_url='/' + ) + downloader.chunk_downloaded(b'12345') + downloader.finish() + downloader.failed() # Stop the reporter + assert not downloader.interrupted - # TODO: Auto-close the file in that case? - downloader._output_file.close() - assert os.path.isfile('filename.bin') - finally: - os.chdir(orig_cwd) + # TODO: Auto-close the file in that case? + downloader._output_file.close() def test_download_interrupted(self, mock_env, httpbin_both): with open(os.devnull, 'w') as devnull: @@ -239,7 +233,10 @@ def test_download_resumed(self, mock_env, httpbin_both): downloader.start( final_response=Response( url=httpbin_both.url + '/', - headers={'Content-Length': 5, 'Content-Range': 'bytes 3-4/5'}, + headers={ + 'Content-Length': 5, + 'Content-Range': 'bytes 3-4/5', + }, status_code=PARTIAL_CONTENT ), initial_url='/' @@ -250,16 +247,11 @@ def test_download_resumed(self, mock_env, httpbin_both): def test_download_with_redirect_original_url_used_for_filename(self, httpbin): # Redirect from `/redirect/1` to `/get`. expected_filename = '1.json' - orig_cwd = os.getcwd() - with tempfile.TemporaryDirectory() as tmp_dirname: - os.chdir(tmp_dirname) - try: - assert os.listdir('.') == [] - http('--download', httpbin + '/redirect/1') - assert os.listdir('.') == [expected_filename] - finally: - os.chdir(orig_cwd) + with cd_clean_tmp_dir(assert_filenames_after=[expected_filename]): + http('--download', httpbin + '/redirect/1') def test_download_gzip_content_encoding(self, httpbin): - r = http('--download', httpbin + '/gzip') + expected_filename = 'gzip.json' + with cd_clean_tmp_dir(assert_filenames_after=[expected_filename]): + r = http('--download', httpbin + '/gzip') assert r.exit_status == 0 diff --git a/tests/test_output.py b/tests/test_output.py index 1e01de4afd..6fb2cdd812 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -21,7 +21,7 @@ from httpie.output.formatters.colors import get_lexer, PIE_STYLE_NAMES, BUNDLED_STYLES from httpie.status import ExitStatus from .fixtures import XML_DATA_RAW, XML_DATA_FORMATTED -from .utils import COLOR, CRLF, HTTP_OK, MockEnvironment, http, DUMMY_URL, strip_colors +from .utils import COLOR, CRLF, HTTP_OK, MockEnvironment, http, DUMMY_URL, strip_colors, cd_clean_tmp_dir # For ensuring test reproducibility, avoid using the unsorted @@ -159,16 +159,13 @@ def test_quiet_with_explicit_output_options(self, httpbin, quiet_flags, output_o @pytest.mark.parametrize('quiet_flags', QUIET_SCENARIOS) @pytest.mark.parametrize('with_download', [True, False]) - def test_quiet_with_output_redirection(self, tmp_path, httpbin, quiet_flags, with_download): + def test_quiet_with_output_redirection(self, httpbin, quiet_flags, with_download): url = httpbin + '/robots.txt' output_path = Path('output.txt') env = MockEnvironment() - orig_cwd = os.getcwd() output = niquests.get(url).text extra_args = ['--download'] if with_download else [] - os.chdir(tmp_path) - try: - assert os.listdir('.') == [] + with cd_clean_tmp_dir(assert_filenames_after=[str(output_path)]): r = http( *quiet_flags, '--output', str(output_path), @@ -176,7 +173,6 @@ def test_quiet_with_output_redirection(self, tmp_path, httpbin, quiet_flags, wit url, env=env ) - assert os.listdir('.') == [str(output_path)] assert r == '' assert r.stderr == '' assert env.stderr is env.devnull @@ -185,8 +181,6 @@ def test_quiet_with_output_redirection(self, tmp_path, httpbin, quiet_flags, wit else: assert env.stdout is not env.devnull # --output swaps stdout. assert output_path.read_text(encoding=UTF8) == output - finally: - os.chdir(orig_cwd) class TestVerboseFlag: diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 83cc5385a8..e233fa9dfa 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -18,7 +18,7 @@ from httpie.sessions import Session from httpie.utils import get_expired_cookies from .test_auth_plugins import basic_auth -from .utils import DUMMY_HOST, HTTP_OK, MockEnvironment, http, mk_config_dir +from .utils import DUMMY_HOST, HTTP_OK, MockEnvironment, http, mk_config_dir, cd_clean_tmp_dir from base64 import b64encode @@ -253,13 +253,9 @@ def test_session_default_header_value_overwritten(self, httpbin): def test_download_in_session(self, tmp_path, httpbin): # https://github.com/httpie/cli/issues/412 self.start_session(httpbin) - cwd = os.getcwd() - os.chdir(tmp_path) - try: + with cd_clean_tmp_dir(): http('--session=test', '--download', httpbin + '/get', env=self.env()) - finally: - os.chdir(cwd) @pytest.mark.parametrize( 'auth_require_param, auth_parse_param', diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 962dfc5aec..e61f82719e 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,4 +1,6 @@ """Utilities for HTTPie test suite.""" +import contextlib +import os import re import shlex import sys @@ -407,6 +409,7 @@ def http( add_to_args.append('--timeout=3') complete_args = [program_name, *add_to_args, *args] + # print(' '.join(complete_args)) def dump_stderr(): @@ -468,3 +471,18 @@ def dump_stderr(): finally: env.cleanup() + + +@contextlib.contextmanager +def cd_clean_tmp_dir(assert_filenames_after: list = None): + """Run commands inside a clean temporary directory, optionally checking for created file names.""" + orig_cwd = os.getcwd() + with tempfile.TemporaryDirectory() as tmp_dirname: + os.chdir(tmp_dirname) + assert os.listdir('.') == [] + try: + yield tmp_dirname + if assert_filenames_after is not None: + assert os.listdir('.') == assert_filenames_after + finally: + os.chdir(orig_cwd) From 474766964b653d3754aab90e726190647d7fc4fe Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Mon, 3 Jun 2024 14:38:12 +0200 Subject: [PATCH 41/63] Cleanup imports --- tests/test_output.py | 11 ++++------- tests/test_sessions.py | 8 +++----- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/tests/test_output.py b/tests/test_output.py index 6fb2cdd812..998de117c9 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -1,17 +1,14 @@ import argparse -from pathlib import Path -from unittest import mock - -import json -import os import io +import json import warnings +from pathlib import Path +from unittest import mock from urllib.request import urlopen -import pytest import niquests +import pytest import responses - from httpie.cli.argtypes import ( PARSED_DEFAULT_FORMAT_OPTIONS, parse_format_options, diff --git a/tests/test_sessions.py b/tests/test_sessions.py index e233fa9dfa..d97b230f5b 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -1,15 +1,13 @@ import json -import os import shutil +from base64 import b64encode from contextlib import contextmanager from datetime import datetime -from unittest import mock from pathlib import Path from typing import Iterator +from unittest import mock import pytest - -from .fixtures import FILE_PATH_ARG, UNICODE from httpie.context import Environment from httpie.encoding import UTF8 from httpie.plugins import AuthPlugin @@ -17,9 +15,9 @@ from httpie.plugins.registry import plugin_manager from httpie.sessions import Session from httpie.utils import get_expired_cookies +from .fixtures import FILE_PATH_ARG, UNICODE from .test_auth_plugins import basic_auth from .utils import DUMMY_HOST, HTTP_OK, MockEnvironment, http, mk_config_dir, cd_clean_tmp_dir -from base64 import b64encode class SessionTestBase: From b3f9e01af951ebad8fb432ef6674f0a45376e5cb Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Thu, 6 Jun 2024 11:20:09 +0200 Subject: [PATCH 42/63] Tests --- tests/test_sessions.py | 2 +- tests/utils/__init__.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_sessions.py b/tests/test_sessions.py index d97b230f5b..97d7744102 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -251,7 +251,7 @@ def test_session_default_header_value_overwritten(self, httpbin): def test_download_in_session(self, tmp_path, httpbin): # https://github.com/httpie/cli/issues/412 self.start_session(httpbin) - with cd_clean_tmp_dir(): + with cd_clean_tmp_dir(assert_filenames_after=['get.json']): http('--session=test', '--download', httpbin + '/get', env=self.env()) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index e61f82719e..1f8229bec6 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -474,15 +474,15 @@ def dump_stderr(): @contextlib.contextmanager -def cd_clean_tmp_dir(assert_filenames_after: list = None): - """Run commands inside a clean temporary directory, optionally checking for created file names.""" +def cd_clean_tmp_dir(assert_filenames_after = []): + """Run commands inside a clean temporary directory, and verify created file names.""" orig_cwd = os.getcwd() with tempfile.TemporaryDirectory() as tmp_dirname: os.chdir(tmp_dirname) assert os.listdir('.') == [] try: yield tmp_dirname - if assert_filenames_after is not None: - assert os.listdir('.') == assert_filenames_after + actual_filenames = os.listdir('.') + assert actual_filenames == assert_filenames_after, (actual_filenames, assert_filenames_after) finally: os.chdir(orig_cwd) From bb3583ceacd027d90a12314c8d204bdbef90150e Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Thu, 6 Jun 2024 11:23:09 +0200 Subject: [PATCH 43/63] Clean --- tests/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 1f8229bec6..c3a29ba684 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -474,7 +474,7 @@ def dump_stderr(): @contextlib.contextmanager -def cd_clean_tmp_dir(assert_filenames_after = []): +def cd_clean_tmp_dir(assert_filenames_after=[]): """Run commands inside a clean temporary directory, and verify created file names.""" orig_cwd = os.getcwd() with tempfile.TemporaryDirectory() as tmp_dirname: From 77bbf8cad4a66750227e3cbceb8807215644086c Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Thu, 6 Jun 2024 06:08:55 -0700 Subject: [PATCH 44/63] Update CHANGELOG.md Co-authored-by: Jan Brasna <1784648+janbrasna@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70009dd48f..2e0e476283 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Removed support for `pyopenssl`. ([#1531](https://github.com/httpie/cli/pull/1531)) - Removed support for dead SSL protocols < TLS 1.0 (e.g. sslv3) as per pyopenssl removal. ([#1531](https://github.com/httpie/cli/pull/1531)) - Removed dependency on `requests_toolbelt` in favor of directly including `MultipartEncoder` into HTTPie due to its direct dependency to requests. ([#1531](https://github.com/httpie/cli/pull/1531)) -- Removed dependency on `multidict` in favor aof an internal one due to often missing pre-built wheels. ([#1522](https://github.com/httpie/cli/issues/1522), [#1531](https://github.com/httpie/cli/pull/1531)) +- Removed dependency on `multidict` in favor of an internal one due to often missing pre-built wheels. ([#1522](https://github.com/httpie/cli/issues/1522), [#1531](https://github.com/httpie/cli/pull/1531)) Existing plugins are expected to work without any changes. The only caveat would be that certain plugin explicitly require `requests`. Future contributions may be made in order to relax the constraints where applicable. From 128d9528bdd77b8169b4543d0381bae633a33b79 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Thu, 6 Jun 2024 06:09:07 -0700 Subject: [PATCH 45/63] Update CHANGELOG.md Co-authored-by: Jan Brasna <1784648+janbrasna@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e0e476283..4d85c51a1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Removed dependency on `requests_toolbelt` in favor of directly including `MultipartEncoder` into HTTPie due to its direct dependency to requests. ([#1531](https://github.com/httpie/cli/pull/1531)) - Removed dependency on `multidict` in favor of an internal one due to often missing pre-built wheels. ([#1522](https://github.com/httpie/cli/issues/1522), [#1531](https://github.com/httpie/cli/pull/1531)) -Existing plugins are expected to work without any changes. The only caveat would be that certain plugin explicitly require `requests`. +Existing plugins are expected to work without any changes. The only caveat would be that certain plugins explicitly require `requests`. Future contributions may be made in order to relax the constraints where applicable. ## [3.2.2](https://github.com/httpie/cli/compare/3.2.1...3.2.2) (2022-05-19) From 540d3f3230df7df53d6ef4defca1f543226283a4 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Fri, 14 Jun 2024 13:30:34 +0200 Subject: [PATCH 46/63] Decoded downloads: refactor, add decoded size notice, tests, etc. Related to #423 --- httpie/core.py | 11 ++- httpie/downloads.py | 141 +++++++++++++++++------------- httpie/output/ui/rich_progress.py | 84 +++++++++--------- httpie/utils.py | 4 + tests/test_downloads.py | 133 +++++++++++++++++++++++++--- tests/utils/__init__.py | 2 +- 6 files changed, 253 insertions(+), 122 deletions(-) diff --git a/httpie/core.py b/httpie/core.py index 3193d0190b..28b0af2867 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -10,6 +10,7 @@ from niquests import __version__ as requests_version from . import __version__ as httpie_version +from .cli.argparser import HTTPieArgumentParser from .cli.constants import OUT_REQ_BODY from .cli.nested_json import NestedJSONSyntaxError from .client import collect_messages @@ -30,7 +31,7 @@ # noinspection PyDefaultArgument def raw_main( - parser: argparse.ArgumentParser, + parser: HTTPieArgumentParser, main_program: Callable[[argparse.Namespace, Environment], ExitStatus], args: List[Union[str, bytes]] = sys.argv, env: Environment = Environment(), @@ -97,10 +98,7 @@ def handle_generic_error(e, annotation=None): else: check_updates(env) try: - exit_status = main_program( - args=parsed_args, - env=env, - ) + exit_status = main_program(parsed_args, env) except KeyboardInterrupt: env.stderr.write('\n') if include_traceback: @@ -150,6 +148,7 @@ def handle_generic_error(e, annotation=None): return exit_status +# noinspection PyDefaultArgument def main( args: List[Union[str, bytes]] = sys.argv, env: Environment = Environment() @@ -295,7 +294,7 @@ def prepared_request_readiness(pr): ) write_stream(stream=download_stream, outfile=download_to, flush=False) downloader.finish() - if downloader.interrupted: + if downloader.is_interrupted: exit_status = ExitStatus.ERROR env.log_error( f'Incomplete download: size={downloader.status.total_size};' diff --git a/httpie/downloads.py b/httpie/downloads.py index d9c711c5fe..8e25974ddd 100644 --- a/httpie/downloads.py +++ b/httpie/downloads.py @@ -7,7 +7,7 @@ import re from mailbox import Message from time import monotonic -from typing import IO, Optional, Tuple +from typing import IO, Optional, Tuple, List from urllib.parse import urlsplit import niquests @@ -15,6 +15,7 @@ from .models import HTTPResponse, OutputOptions from .output.streams import RawStream from .context import Environment +from .utils import split_header_values PARTIAL_CONTENT = 206 @@ -159,6 +160,29 @@ def get_unique_filename(filename: str, exists=os.path.exists) -> str: attempt += 1 +def get_content_length(response: niquests.Response) -> Optional[int]: + try: + return int(response.headers['Content-Length']) + except (KeyError, ValueError, TypeError): + pass + + + +def get_decodeable_content_encodings(encoded_response: niquests.Response) -> Optional[List[str]]: + content_encoding = encoded_response.headers.get('Content-Encoding') + if not content_encoding: + return None + applied_encodings = split_header_values(content_encoding) + try: + supported_decoders = encoded_response.raw.CONTENT_DECODERS + except AttributeError: + supported_decoders = ['gzip', 'deflate'] + for encoding in applied_encodings: + if encoding not in supported_decoders: + return None + return applied_encodings + + class Downloader: def __init__( @@ -174,8 +198,6 @@ def __init__( :param output_file: The file to store response body in. If not provided, it will be guessed from the response. - :param progress_file: Where to report download progress. - """ self.finished = False self.status = DownloadStatus(env=env) @@ -191,7 +213,11 @@ def pre_request(self, request_headers: dict): """ # Ask the server not to encode the content so that we can resume, etc. + # TODO: Reconsider this once the underlying library can report raw download size (i.e., not decoded). + # Then it might still be needed when resuming. But in the default case, it won’t probably be necessary. + # request_headers['Accept-Encoding'] = 'identity' + if self._resume: bytes_have = os.path.getsize(self._output_file.name) if bytes_have: @@ -217,38 +243,11 @@ def start( """ assert not self.status.time_started - try: - supported_decoders = final_response.raw.CONTENT_DECODERS - except AttributeError: - supported_decoders = ["gzip", "deflate"] - - use_content_length = True - - # If the content is actually compressed, the http client will automatically - # stream decompressed content. This ultimately means that the server send the content-length - # that is related to the compressed body. this might fool the downloader. - # but... there's a catch, we don't decompress everything, everytime. It depends on the - # Content-Encoding. - if 'Content-Encoding' in final_response.headers: - will_decompress = True - - encoding_list = final_response.headers['Content-Encoding'].replace(' ', '').lower().split(',') - - for encoding in encoding_list: - if encoding not in supported_decoders: - will_decompress = False - break - - if will_decompress: - use_content_length = False - - if use_content_length: - try: - total_size = int(final_response.headers['Content-Length']) - except (KeyError, ValueError, TypeError): - total_size = None - else: - total_size = None + # Even though we specify `Accept-Encoding: identity`, the server might still encode the response. + # In such cases, the reported size will be of the decoded content, not the downloaded bytes. + # This is a limitation of the underlying Niquests library . + decoded_from = get_decodeable_content_encodings(final_response) + total_size = get_content_length(final_response) if not self._output_file: self._output_file = self._get_output_file_from_response( @@ -282,7 +281,8 @@ def start( self.status.started( output_file=self._output_file, resumed_from=self._resumed_from, - total_size=total_size + total_size=total_size, + decoded_from=decoded_from, ) return stream, self._output_file @@ -299,12 +299,8 @@ def failed(self): self.status.terminate() @property - def interrupted(self) -> bool: - return ( - self.finished - and self.status.total_size - and self.status.total_size != self.status.downloaded - ) + def is_interrupted(self) -> bool: + return self.status.is_interrupted def chunk_downloaded(self, chunk: bytes): """ @@ -334,6 +330,9 @@ def _get_output_file_from_response( unique_filename = get_unique_filename(filename) return open(unique_filename, buffering=0, mode='a+b') +DECODED_FROM_SUFFIX = ' - decoded from {encodings}' +DECODED_SIZE_NOTE_SUFFIX = ' - decoded size' + class DownloadStatus: """Holds details about the download status.""" @@ -342,40 +341,49 @@ def __init__(self, env): self.env = env self.downloaded = 0 self.total_size = None + self.decoded_from = [] self.resumed_from = 0 self.time_started = None self.time_finished = None + self.display = None - def started(self, output_file, resumed_from=0, total_size=None): + def started(self, output_file, resumed_from=0, total_size=None, decoded_from: List[str]=None): assert self.time_started is None self.total_size = total_size + self.decoded_from = decoded_from self.downloaded = self.resumed_from = resumed_from self.time_started = monotonic() self.start_display(output_file=output_file) def start_display(self, output_file): from httpie.output.ui.rich_progress import ( - DummyDisplay, - StatusDisplay, - ProgressDisplay + DummyProgressDisplay, + ProgressDisplayNoTotal, + ProgressDisplayFull ) - message = f'Downloading to {output_file.name}' - if self.env.show_displays: - if self.total_size is None: - # Rich does not support progress bars without a total - # size given. Instead we use status objects. - self.display = StatusDisplay(self.env) - else: - self.display = ProgressDisplay(self.env) + message_suffix = '' + summary_suffix = '' + if not self.env.show_displays: + progress_display_class = DummyProgressDisplay else: - self.display = DummyDisplay(self.env) - - self.display.start( - total=self.total_size, - at=self.downloaded, - description=message + has_reliable_total = self.total_size is not None and not self.decoded_from + if has_reliable_total: + progress_display_class = ProgressDisplayFull + else: + if self.decoded_from: + encodings = ', '.join(f'`{enc}`' for enc in self.decoded_from) + message_suffix = DECODED_FROM_SUFFIX.format(encodings=encodings) + summary_suffix = DECODED_SIZE_NOTE_SUFFIX + progress_display_class = ProgressDisplayNoTotal + self.display = progress_display_class( + env=self.env, + total_size=self.total_size, + resumed_from=self.resumed_from, + description=message + message_suffix, + summary_suffix=summary_suffix, ) + self.display.start() def chunk_downloaded(self, size): assert self.time_finished is None @@ -386,6 +394,15 @@ def chunk_downloaded(self, size): def has_finished(self): return self.time_finished is not None + @property + def is_interrupted(self): + return ( + self.has_finished + and self.total_size is not None + and not self.decoded_from + and self.total_size != self.downloaded + ) + @property def time_spent(self): if ( @@ -400,9 +417,9 @@ def finished(self): assert self.time_started is not None assert self.time_finished is None self.time_finished = monotonic() - if hasattr(self, 'display'): + if self.display: self.display.stop(self.time_spent) def terminate(self): - if hasattr(self, 'display'): + if self.display: self.display.stop(self.time_spent) diff --git a/httpie/output/ui/rich_progress.py b/httpie/output/ui/rich_progress.py index d2cfd38c70..41a2893d53 100644 --- a/httpie/output/ui/rich_progress.py +++ b/httpie/output/ui/rich_progress.py @@ -3,24 +3,27 @@ from httpie.context import Environment + if TYPE_CHECKING: from rich.console import Console @dataclass -class BaseDisplay: +class BaseProgressDisplay: env: Environment + total_size: Optional[float] + resumed_from: int + description: str + summary_suffix: str - def start( - self, *, total: Optional[float], at: float, description: str - ) -> None: - ... + def start(self): + raise NotImplementedError - def update(self, steps: float) -> None: - ... + def update(self, steps: float): + raise NotImplementedError - def stop(self, time_spent: float) -> None: - ... + def stop(self, time_spent: float): + raise NotImplementedError @property def console(self) -> 'Console': @@ -31,57 +34,58 @@ def _print_summary( self, is_finished: bool, observed_steps: int, time_spent: float ): from rich import filesize - if is_finished: verb = 'Done' else: verb = 'Interrupted' - total_size = filesize.decimal(observed_steps) + # noinspection PyTypeChecker avg_speed = filesize.decimal(observed_steps / time_spent) - minutes, seconds = divmod(time_spent, 60) hours, minutes = divmod(int(minutes), 60) if hours: total_time = f'{hours:d}:{minutes:02d}:{seconds:0.5f}' else: total_time = f'{minutes:02d}:{seconds:0.5f}' - self.console.print( - f'[progress.description]{verb}. {total_size} in {total_time} ({avg_speed}/s)' + f'[progress.description]{verb}. {total_size} in {total_time} ({avg_speed}/s){self.summary_suffix}' ) -class DummyDisplay(BaseDisplay): +class DummyProgressDisplay(BaseProgressDisplay): """ A dummy display object to be used when the progress bars, spinners etc. are disabled globally (or during tests). """ + def start(self): + pass + + def update(self, steps: float): + pass + + def stop(self, time_spent: float): + pass -class StatusDisplay(BaseDisplay): - def start( - self, *, total: Optional[float], at: float, description: str - ) -> None: - self.observed = at + +class ProgressDisplayNoTotal(BaseProgressDisplay): + observed = 0 + status = None + + def start(self) -> None: + self.observed = self.resumed_from self.description = ( - f'[progress.description]{description}[/progress.description]' + f'[progress.description]{self.description}[/progress.description]' ) - self.status = self.console.status(self.description, spinner='line') self.status.start() - def update(self, steps: float) -> None: + def update(self, steps: int) -> None: from rich import filesize - self.observed += steps - - observed_amount, observed_unit = filesize.decimal( - self.observed - ).split() - self.status.update( - status=f'{self.description} [progress.download]{observed_amount}/? {observed_unit}[/progress.download]' - ) + observed_amount, observed_unit = filesize.decimal(self.observed).split() + msg = f'{self.description} [progress.download]{observed_amount}/? {observed_unit}[/progress.download]' + self.status.update(status=msg) def stop(self, time_spent: float) -> None: self.status.stop() @@ -94,10 +98,11 @@ def stop(self, time_spent: float) -> None: ) -class ProgressDisplay(BaseDisplay): - def start( - self, *, total: Optional[float], at: float, description: str - ) -> None: +class ProgressDisplayFull(BaseProgressDisplay): + progress_bar = None + transfer_task = None + + def start(self) -> None: from rich.progress import ( Progress, BarColumn, @@ -105,9 +110,8 @@ def start( TimeRemainingColumn, TransferSpeedColumn, ) - - assert total is not None - self.console.print(f'[progress.description]{description}') + assert self.total_size is not None + self.console.print(f'[progress.description]{self.description}') self.progress_bar = Progress( '[', BarColumn(), @@ -123,7 +127,9 @@ def start( ) self.progress_bar.start() self.transfer_task = self.progress_bar.add_task( - description, completed=at, total=total + description=self.description, + completed=self.resumed_from, + total=self.total_size, ) def update(self, steps: float) -> None: diff --git a/httpie/utils.py b/httpie/utils.py index 33d8158568..e021f1a349 100644 --- a/httpie/utils.py +++ b/httpie/utils.py @@ -307,3 +307,7 @@ def split_version(version: str) -> Tuple[int, ...]: return tuple(parts) return split_version(version_1) > split_version(version_2) + + +def split_header_values(header: str) -> List[str]: + return [value.strip() for value in header.split(',')] diff --git a/tests/test_downloads.py b/tests/test_downloads.py index 8d4166a5ef..516342d45c 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -1,18 +1,26 @@ import os import tempfile import time -import niquests from unittest import mock from urllib.request import urlopen +import niquests import pytest -from niquests.structures import CaseInsensitiveDict - +import responses from httpie.downloads import ( - parse_content_range, filename_from_content_disposition, filename_from_url, - get_unique_filename, ContentRangeError, Downloader, PARTIAL_CONTENT + parse_content_range, + filename_from_content_disposition, + filename_from_url, + get_unique_filename, + ContentRangeError, + Downloader, + PARTIAL_CONTENT, + DECODED_SIZE_NOTE_SUFFIX, + DECODED_FROM_SUFFIX, ) -from .utils import http, MockEnvironment, cd_clean_tmp_dir +from niquests.exceptions import ChunkedEncodingError +from niquests.structures import CaseInsensitiveDict +from .utils import http, MockEnvironment, cd_clean_tmp_dir, DUMMY_URL class Response(niquests.Response): @@ -102,7 +110,6 @@ def test_filename_from_url(self): def test_unique_filename(self, get_filename_max_length, orig_name, unique_on_attempt, expected): - def attempts(unique_on_attempt=0): # noinspection PyUnresolvedReferences,PyUnusedLocal def exists(filename): @@ -120,7 +127,7 @@ def exists(filename): assert expected == actual -class TestDownloads: +class TestDownloader: def test_actual_download(self, httpbin_both, httpbin): robots_txt = '/robots.txt' @@ -145,7 +152,7 @@ def test_download_with_Content_Length(self, mock_env, httpbin_both): time.sleep(1.1) downloader.chunk_downloaded(b'12345') downloader.finish() - assert not downloader.interrupted + assert not downloader.is_interrupted def test_download_no_Content_Length(self, mock_env, httpbin_both): with open(os.devnull, 'w') as devnull: @@ -157,7 +164,7 @@ def test_download_no_Content_Length(self, mock_env, httpbin_both): time.sleep(1.1) downloader.chunk_downloaded(b'12345') downloader.finish() - assert not downloader.interrupted + assert not downloader.is_interrupted def test_download_output_from_content_disposition(self, mock_env, httpbin_both): output_file_name = 'filename.bin' @@ -176,12 +183,12 @@ def test_download_output_from_content_disposition(self, mock_env, httpbin_both): downloader.chunk_downloaded(b'12345') downloader.finish() downloader.failed() # Stop the reporter - assert not downloader.interrupted + assert not downloader.is_interrupted # TODO: Auto-close the file in that case? downloader._output_file.close() - def test_download_interrupted(self, mock_env, httpbin_both): + def test_downloader_is_interrupted(self, mock_env, httpbin_both): with open(os.devnull, 'w') as devnull: downloader = Downloader(mock_env, output_file=devnull) downloader.start( @@ -193,7 +200,7 @@ def test_download_interrupted(self, mock_env, httpbin_both): ) downloader.chunk_downloaded(b'1234') downloader.finish() - assert downloader.interrupted + assert downloader.is_interrupted def test_download_resumed(self, mock_env, httpbin_both): with tempfile.TemporaryDirectory() as tmp_dirname: @@ -214,7 +221,7 @@ def test_download_resumed(self, mock_env, httpbin_both): downloader.chunk_downloaded(b'123') downloader.finish() downloader.failed() - assert downloader.interrupted + assert downloader.is_interrupted # Write bytes with open(file, 'wb') as fh: @@ -255,3 +262,101 @@ def test_download_gzip_content_encoding(self, httpbin): with cd_clean_tmp_dir(assert_filenames_after=[expected_filename]): r = http('--download', httpbin + '/gzip') assert r.exit_status == 0 + + @responses.activate + def test_incomplete_response(self): + # We have incompleteness checks in the downloader, but it might not be needed as it’s built into (ni|req)uests. + error_msg = 'peer closed connection without sending complete message body (received 2 bytes, expected 1 more)' + responses.add( + method=responses.GET, + url=DUMMY_URL, + headers={ + 'Content-Length': '3', + }, + body='12', + ) + with cd_clean_tmp_dir(), pytest.raises(ChunkedEncodingError) as exc_info: + http('--download', DUMMY_URL) + assert error_msg in str(exc_info.value) + + +class TestDecodedDownloads: + """Test downloading responses with `Content-Encoding`""" + + @responses.activate + def test_decoded_response_no_content_length(self): + responses.add( + method=responses.GET, + url=DUMMY_URL, + headers={ + 'Content-Encoding': 'gzip, br', + }, + body='123', + ) + with cd_clean_tmp_dir(): + r = http('--download', '--headers', DUMMY_URL) + assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr + assert DECODED_SIZE_NOTE_SUFFIX in r.stderr + print(r.stderr) + + @responses.activate + def test_decoded_response_with_content_length(self): + responses.add( + method=responses.GET, + url=DUMMY_URL, + headers={ + 'Content-Encoding': 'gzip, br', + 'Content-Length': '3', + }, + body='123', + ) + with cd_clean_tmp_dir(): + r = http('--download', DUMMY_URL) + assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr + assert DECODED_SIZE_NOTE_SUFFIX in r.stderr + print(r.stderr) + + @responses.activate + def test_decoded_response_without_content_length(self): + responses.add( + method=responses.GET, + url=DUMMY_URL, + headers={ + 'Content-Encoding': 'gzip, br', + }, + body='123', + ) + with cd_clean_tmp_dir(): + r = http('--download', DUMMY_URL) + assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr + assert DECODED_SIZE_NOTE_SUFFIX in r.stderr + print(r.stderr) + + @responses.activate + def test_non_decoded_response_without_content_length(self): + responses.add( + method=responses.GET, + url=DUMMY_URL, + headers={ + 'Content-Length': '3', + }, + body='123', + ) + with cd_clean_tmp_dir(): + r = http('--download', DUMMY_URL) + assert DECODED_SIZE_NOTE_SUFFIX not in r.stderr + print(r.stderr) + + @responses.activate + def test_non_decoded_response_with_content_length(self): + responses.add( + method=responses.GET, + url=DUMMY_URL, + headers={ + }, + body='123', + ) + with cd_clean_tmp_dir(): + r = http('--download', DUMMY_URL) + assert DECODED_SIZE_NOTE_SUFFIX not in r.stderr + print(r.stderr) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index c3a29ba684..a00d586fe1 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -136,7 +136,7 @@ class MockEnvironment(Environment): stdin_isatty = True stdout_isatty = True is_windows = False - show_displays = False + show_displays = True def __init__(self, create_temp_config_dir=True, **kwargs): self._encoder = Encoder() From 121cba16230c93694de947bb603d3524e3ac1304 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Fri, 14 Jun 2024 13:36:26 +0200 Subject: [PATCH 47/63] Clean --- httpie/downloads.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/httpie/downloads.py b/httpie/downloads.py index 8e25974ddd..c11ef423ad 100644 --- a/httpie/downloads.py +++ b/httpie/downloads.py @@ -167,7 +167,6 @@ def get_content_length(response: niquests.Response) -> Optional[int]: pass - def get_decodeable_content_encodings(encoded_response: niquests.Response) -> Optional[List[str]]: content_encoding = encoded_response.headers.get('Content-Encoding') if not content_encoding: @@ -330,6 +329,7 @@ def _get_output_file_from_response( unique_filename = get_unique_filename(filename) return open(unique_filename, buffering=0, mode='a+b') + DECODED_FROM_SUFFIX = ' - decoded from {encodings}' DECODED_SIZE_NOTE_SUFFIX = ' - decoded size' @@ -347,7 +347,7 @@ def __init__(self, env): self.time_finished = None self.display = None - def started(self, output_file, resumed_from=0, total_size=None, decoded_from: List[str]=None): + def started(self, output_file, resumed_from=0, total_size=None, decoded_from: List[str] = None): assert self.time_started is None self.total_size = total_size self.decoded_from = decoded_from From 5dedead64b9a066a99b4fc2372f01efb4ebe4e99 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Fri, 14 Jun 2024 14:08:06 +0200 Subject: [PATCH 48/63] Fix --- tests/utils/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index a00d586fe1..9e38e87bc2 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -474,7 +474,7 @@ def dump_stderr(): @contextlib.contextmanager -def cd_clean_tmp_dir(assert_filenames_after=[]): +def cd_clean_tmp_dir(assert_filenames_after=None): """Run commands inside a clean temporary directory, and verify created file names.""" orig_cwd = os.getcwd() with tempfile.TemporaryDirectory() as tmp_dirname: @@ -483,6 +483,7 @@ def cd_clean_tmp_dir(assert_filenames_after=[]): try: yield tmp_dirname actual_filenames = os.listdir('.') - assert actual_filenames == assert_filenames_after, (actual_filenames, assert_filenames_after) + if assert_filenames_after is not None: + assert actual_filenames == assert_filenames_after, (actual_filenames, assert_filenames_after) finally: os.chdir(orig_cwd) From f7cbd64eb8fc250544d22a6b41e2328468bc2983 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Fri, 14 Jun 2024 16:33:00 +0200 Subject: [PATCH 49/63] Fix --- tests/test_downloads.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_downloads.py b/tests/test_downloads.py index 516342d45c..9845e3d3cd 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -18,7 +18,6 @@ DECODED_SIZE_NOTE_SUFFIX, DECODED_FROM_SUFFIX, ) -from niquests.exceptions import ChunkedEncodingError from niquests.structures import CaseInsensitiveDict from .utils import http, MockEnvironment, cd_clean_tmp_dir, DUMMY_URL @@ -275,7 +274,7 @@ def test_incomplete_response(self): }, body='12', ) - with cd_clean_tmp_dir(), pytest.raises(ChunkedEncodingError) as exc_info: + with cd_clean_tmp_dir(), pytest.raises(Exception) as exc_info: http('--download', DUMMY_URL) assert error_msg in str(exc_info.value) From f989e5ecad1cadf0bc9943edd34cd8d613159efc Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Mon, 24 Jun 2024 21:57:00 +0100 Subject: [PATCH 50/63] support for force http2, http1 or http3 and fix real download speed when body is compressed --- CHANGELOG.md | 1 + docs/README.md | 23 ++++++--- httpie/__init__.py | 4 +- httpie/cli/definition.py | 20 ++++++++ httpie/client.py | 20 ++++++++ httpie/downloads.py | 18 +++++-- httpie/output/streams.py | 8 ++- setup.cfg | 2 +- tests/test_downloads.py | 102 +++++++++++++++++++-------------------- tests/utils/__init__.py | 23 +++++---- 10 files changed, 145 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d85c51a1c..37a59b4a8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Fixed issue where the configuration directory was not created at runtime that made the update fetcher run everytime. ([#1527](https://github.com/httpie/cli/issues/1527)) - Fixed cookie persistence in HTTPie session when targeting localhost. They were dropped due to the standard library. ([#1527](https://github.com/httpie/cli/issues/1527)) - Fixed downloader when trying to fetch compressed content. The process will no longer exit with the "Incomplete download" error. ([#1554](https://github.com/httpie/cli/issues/1554), [#423](https://github.com/httpie/cli/issues/423), [#1527](https://github.com/httpie/cli/issues/1527)) +- Fixed downloader yielding an incorrect speed when the remote is using `Content-Encoding` aka. compressed body. ([#1554](https://github.com/httpie/cli/issues/1554), [#423](https://github.com/httpie/cli/issues/423), [#1527](https://github.com/httpie/cli/issues/1527)) - Removed support for preserving the original casing of HTTP headers. This comes as a constraint of newer protocols, namely HTTP/2+ that normalize header keys by default. From the HTTPie user perspective, they are "prettified" in the output by default. e.g. `x-hello-world` is displayed as `X-Hello-World`. - Removed support for `pyopenssl`. ([#1531](https://github.com/httpie/cli/pull/1531)) - Removed support for dead SSL protocols < TLS 1.0 (e.g. sslv3) as per pyopenssl removal. ([#1531](https://github.com/httpie/cli/pull/1531)) diff --git a/docs/README.md b/docs/README.md index d0e7b022cd..c52cdded9a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1872,19 +1872,19 @@ HTTPie has full support for HTTP/1.1, HTTP/2, and HTTP/3. ### Disable HTTP/2, or HTTP/3 -You can at your own discretion toggle on and off HTTP/2, or/and HTTP/3. +You can at your own discretion toggle on and off HTTP/1, HTTP/2, or/and HTTP/3. ```bash $ https --disable-http2 PUT pie.dev/put hello=world ``` ```bash -$ https --disable-http3 PUT pie.dev/put hello=world +$ https --disable-http3 --disable-http1 PUT pie.dev/put hello=world ``` -### Force HTTP/3 +### Force HTTP/3, HTTP/2 or HTTP/1.1 -By opposition to the previous section, you can force the HTTP/3 negotiation. +By opposition to the previous section, you can force the HTTP/3, HTTP/2 or HTTP/1.1 negotiation. ```bash $ https --http3 pie.dev/get @@ -1899,19 +1899,28 @@ either HTTP/1.1 or HTTP/2. ### Protocol combinations -Following `Force HTTP/3` and `Disable HTTP/2, or HTTP/3`, you may find a summary on how to make HTTPie negotiate a +Following `Force HTTP/3, HTTP/2 and HTTP/1` and `Disable HTTP/1, HTTP/2, or HTTP/3`, you may find a summary on how to make HTTPie negotiate a specific protocol. | Arguments | HTTP/1.1
enabled | HTTP/2
enabled | HTTP/3
enabled | |----------------------------------:|:--------------------:|:------------------:|:------------------:| | (Default) | ✔ | ✔ | ✔ | +| `--disable-http1` | ✗ | ✔ | ✔ | | `--disable-http2` | ✔ | ✗ | ✔ | | `--disable-http3` | ✔ | ✔ | ✗ | | `--disable-http2 --disable-http3` | ✔ | ✗ | ✗ | +| `--disable-http1 --disable-http2` | ✗ | ✗ | ✔ | +| `--http1` | ✔ | ✗ | ✗ | +| `--http2` | ✗ | ✔ | ✗ | | `--http3` | ✗ | ✗ | ✔ | -You cannot enforce HTTP/2 without prior knowledge nor can you negotiate it without TLS and ALPN. -Also, you may not disable HTTP/1.1 as it is ultimately used as a fallback in case HTTP/2 and HTTP/3 are not supported. +Some specifics, through: + +- You cannot enforce HTTP/3 over non HTTPS URLs. +- You cannot disable both HTTP/1.1 and HTTP/2 for non HTTPS URLs. +- Of course, you cannot disable all three protocols. +- Those toggles do not apply to the DNS-over-HTTPS custom resolver. You will have to specify it within the resolver URL. +- When reaching a HTTPS URL, the ALPN extension sent during SSL/TLS handshake is affected. ## Custom DNS resolver diff --git a/httpie/__init__.py b/httpie/__init__.py index b1c1a48bcc..d3eb07335b 100644 --- a/httpie/__init__.py +++ b/httpie/__init__.py @@ -3,7 +3,7 @@ """ -__version__ = '4.0.0.b1' -__date__ = '2024-01-01' +__version__ = '4.0.0' +__date__ = '2024-06-25' __author__ = 'Jakub Roztocil' __licence__ = 'BSD' diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 7287eca6bb..2f6bf55c4e 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -816,12 +816,32 @@ def format_auth_help(auth_plugins_mapping, *, isolation_mode: bool = False): 'The Transfer-Encoding header is set to chunked.' ) ) +network.add_argument( + "--disable-http1", + default=False, + action="store_true", + short_help="Disable the HTTP/1 protocol." +) +network.add_argument( + "--http1", + default=False, + action="store_true", + dest="force_http1", + short_help="Use the HTTP/1 protocol for the request." +) network.add_argument( "--disable-http2", default=False, action="store_true", short_help="Disable the HTTP/2 protocol." ) +network.add_argument( + "--http2", + default=False, + action="store_true", + dest="force_http2", + short_help="Use the HTTP/2 protocol for the request." +) network.add_argument( "--disable-http3", default=False, diff --git a/httpie/client.py b/httpie/client.py index eea5d6a00b..27811b4e46 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -89,10 +89,26 @@ def collect_messages( else: resolver = [ensure_resolver, "system://"] + if args.force_http1: + args.disable_http1 = False + args.disable_http2 = True + args.disable_http3 = True + + if args.force_http2: + args.disable_http1 = True + args.disable_http2 = False + args.disable_http3 = True + + if args.force_http3: + args.disable_http1 = True + args.disable_http2 = True + args.disable_http3 = False + requests_session = build_requests_session( ssl_version=args.ssl_version, ciphers=args.ciphers, verify=bool(send_kwargs_mergeable_from_env['verify']), + disable_http1=args.disable_http1, disable_http2=args.disable_http2, disable_http3=args.disable_http3, resolver=resolver, @@ -211,6 +227,7 @@ def build_requests_session( verify: bool, ssl_version: str = None, ciphers: str = None, + disable_http1: bool = False, disable_http2: bool = False, disable_http3: bool = False, resolver: typing.List[str] = None, @@ -239,6 +256,8 @@ def build_requests_session( disable_ipv4=disable_ipv4, disable_ipv6=disable_ipv6, source_address=source_address, + disable_http1=disable_http1, + disable_http2=disable_http2, ) https_adapter = HTTPieHTTPSAdapter( ciphers=ciphers, @@ -247,6 +266,7 @@ def build_requests_session( AVAILABLE_SSL_VERSION_ARG_MAPPING[ssl_version] if ssl_version else None ), + disable_http1=disable_http1, disable_http2=disable_http2, disable_http3=disable_http3, resolver=resolver, diff --git a/httpie/downloads.py b/httpie/downloads.py index c11ef423ad..4f6f75eed6 100644 --- a/httpie/downloads.py +++ b/httpie/downloads.py @@ -7,7 +7,7 @@ import re from mailbox import Message from time import monotonic -from typing import IO, Optional, Tuple, List +from typing import IO, Optional, Tuple, List, Union from urllib.parse import urlsplit import niquests @@ -301,7 +301,7 @@ def failed(self): def is_interrupted(self) -> bool: return self.status.is_interrupted - def chunk_downloaded(self, chunk: bytes): + def chunk_downloaded(self, chunk_or_new_total: Union[bytes, int]): """ A download progress callback. @@ -309,7 +309,10 @@ def chunk_downloaded(self, chunk: bytes): been downloaded and written to the output. """ - self.status.chunk_downloaded(len(chunk)) + if isinstance(chunk_or_new_total, int): + self.status.set_total(chunk_or_new_total) + else: + self.status.chunk_downloaded(len(chunk_or_new_total)) @staticmethod def _get_output_file_from_response( @@ -367,7 +370,8 @@ def start_display(self, output_file): if not self.env.show_displays: progress_display_class = DummyProgressDisplay else: - has_reliable_total = self.total_size is not None and not self.decoded_from + has_reliable_total = self.total_size is not None + if has_reliable_total: progress_display_class = ProgressDisplayFull else: @@ -390,6 +394,12 @@ def chunk_downloaded(self, size): self.downloaded += size self.display.update(size) + def set_total(self, total: int) -> None: + assert self.time_finished is None + prev_value = self.downloaded + self.downloaded = total + self.display.update(total - prev_value) + @property def has_finished(self): return self.time_finished is not None diff --git a/httpie/output/streams.py b/httpie/output/streams.py index 1686a97913..83d1c5673e 100644 --- a/httpie/output/streams.py +++ b/httpie/output/streams.py @@ -75,7 +75,13 @@ def __iter__(self) -> Iterable[bytes]: for chunk in self.iter_body(): yield chunk if self.on_body_chunk_downloaded: - self.on_body_chunk_downloaded(chunk) + # Niquests 3.7+ have a way to determine the "real" amt of raw data collected + # Useful when the remote compress the body. We use the "untouched" amt of data to determine + # the download speed. + if hasattr(self.msg, "_orig") and hasattr(self.msg._orig, "download_progress") and self.msg._orig.download_progress: + self.on_body_chunk_downloaded(self.msg._orig.download_progress.total) + else: + self.on_body_chunk_downloaded(chunk) except DataSuppressedError as e: if self.output_options.headers: yield b'\n' diff --git a/setup.cfg b/setup.cfg index 0d13bdaaa7..cc71785d25 100644 --- a/setup.cfg +++ b/setup.cfg @@ -71,7 +71,7 @@ install_requires = pip charset_normalizer>=2.0.0 defusedxml>=0.6.0 - niquests[socks]>=3 + niquests[socks]>=3.7 Pygments>=2.5.2 setuptools importlib-metadata>=1.4.0; python_version<"3.8" diff --git a/tests/test_downloads.py b/tests/test_downloads.py index 9845e3d3cd..4747fa399b 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -16,7 +16,6 @@ Downloader, PARTIAL_CONTENT, DECODED_SIZE_NOTE_SUFFIX, - DECODED_FROM_SUFFIX, ) from niquests.structures import CaseInsensitiveDict from .utils import http, MockEnvironment, cd_clean_tmp_dir, DUMMY_URL @@ -282,54 +281,55 @@ def test_incomplete_response(self): class TestDecodedDownloads: """Test downloading responses with `Content-Encoding`""" - @responses.activate - def test_decoded_response_no_content_length(self): - responses.add( - method=responses.GET, - url=DUMMY_URL, - headers={ - 'Content-Encoding': 'gzip, br', - }, - body='123', - ) - with cd_clean_tmp_dir(): - r = http('--download', '--headers', DUMMY_URL) - assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr - assert DECODED_SIZE_NOTE_SUFFIX in r.stderr - print(r.stderr) - - @responses.activate - def test_decoded_response_with_content_length(self): - responses.add( - method=responses.GET, - url=DUMMY_URL, - headers={ - 'Content-Encoding': 'gzip, br', - 'Content-Length': '3', - }, - body='123', - ) - with cd_clean_tmp_dir(): - r = http('--download', DUMMY_URL) - assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr - assert DECODED_SIZE_NOTE_SUFFIX in r.stderr - print(r.stderr) - - @responses.activate - def test_decoded_response_without_content_length(self): - responses.add( - method=responses.GET, - url=DUMMY_URL, - headers={ - 'Content-Encoding': 'gzip, br', - }, - body='123', - ) - with cd_clean_tmp_dir(): - r = http('--download', DUMMY_URL) - assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr - assert DECODED_SIZE_NOTE_SUFFIX in r.stderr - print(r.stderr) + # todo: find an appropriate way to mock compressed bodies within those tests. + # @responses.activate + # def test_decoded_response_no_content_length(self): + # responses.add( + # method=responses.GET, + # url=DUMMY_URL, + # headers={ + # 'Content-Encoding': 'gzip, br', + # }, + # body='123', + # ) + # with cd_clean_tmp_dir(): + # r = http('--download', '--headers', DUMMY_URL) + # print(r.stderr) + # assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr + # assert DECODED_SIZE_NOTE_SUFFIX in r.stderr + # + # @responses.activate + # def test_decoded_response_with_content_length(self): + # responses.add( + # method=responses.GET, + # url=DUMMY_URL, + # headers={ + # 'Content-Encoding': 'gzip, br', + # 'Content-Length': '3', + # }, + # body='123', + # ) + # with cd_clean_tmp_dir(): + # r = http('--download', DUMMY_URL) + # print(r.stderr) + # assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr + # assert DECODED_SIZE_NOTE_SUFFIX in r.stderr + # + # @responses.activate + # def test_decoded_response_without_content_length(self): + # responses.add( + # method=responses.GET, + # url=DUMMY_URL, + # headers={ + # 'Content-Encoding': 'gzip, br', + # }, + # body='123', + # ) + # with cd_clean_tmp_dir(): + # r = http('--download', DUMMY_URL) + # print(r.stderr) + # assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr + # assert DECODED_SIZE_NOTE_SUFFIX in r.stderr @responses.activate def test_non_decoded_response_without_content_length(self): @@ -343,8 +343,8 @@ def test_non_decoded_response_without_content_length(self): ) with cd_clean_tmp_dir(): r = http('--download', DUMMY_URL) - assert DECODED_SIZE_NOTE_SUFFIX not in r.stderr print(r.stderr) + assert DECODED_SIZE_NOTE_SUFFIX not in r.stderr @responses.activate def test_non_decoded_response_with_content_length(self): @@ -357,5 +357,5 @@ def test_non_decoded_response_with_content_length(self): ) with cd_clean_tmp_dir(): r = http('--download', DUMMY_URL) - assert DECODED_SIZE_NOTE_SUFFIX not in r.stderr print(r.stderr) + assert DECODED_SIZE_NOTE_SUFFIX not in r.stderr diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 9e38e87bc2..8cfbd941e8 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -477,13 +477,16 @@ def dump_stderr(): def cd_clean_tmp_dir(assert_filenames_after=None): """Run commands inside a clean temporary directory, and verify created file names.""" orig_cwd = os.getcwd() - with tempfile.TemporaryDirectory() as tmp_dirname: - os.chdir(tmp_dirname) - assert os.listdir('.') == [] - try: - yield tmp_dirname - actual_filenames = os.listdir('.') - if assert_filenames_after is not None: - assert actual_filenames == assert_filenames_after, (actual_filenames, assert_filenames_after) - finally: - os.chdir(orig_cwd) + try: + with tempfile.TemporaryDirectory() as tmp_dirname: + os.chdir(tmp_dirname) + assert os.listdir('.') == [] + try: + yield tmp_dirname + actual_filenames = os.listdir('.') + if assert_filenames_after is not None: + assert actual_filenames == assert_filenames_after, (actual_filenames, assert_filenames_after) + finally: + os.chdir(orig_cwd) + except (PermissionError, NotADirectoryError): + pass From d02b88254d852c0ea7a175a70e4d5a182d28a6f6 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 26 Jun 2024 16:09:58 +0100 Subject: [PATCH 51/63] update docs for protocol negotiation and gen man --- docs/README.md | 58 +++++++++++++++++++++++++++------------------ extras/man/http.1 | 20 +++++++++++++++- extras/man/httpie.1 | 2 +- extras/man/https.1 | 20 +++++++++++++++- 4 files changed, 74 insertions(+), 26 deletions(-) diff --git a/docs/README.md b/docs/README.md index c52cdded9a..cddb8621a6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1890,37 +1890,49 @@ By opposition to the previous section, you can force the HTTP/3, HTTP/2 or HTTP/ $ https --http3 pie.dev/get ``` -By default, HTTPie cannot negotiate HTTP/3 without a first HTTP/1.1, or HTTP/2 successful response unless the -remote host specified a DNS HTTPS record that indicate its support (and by using a custom DNS resolver, see bellow section). +For HTTP (unencrypted) URLs, you can enforce HTTP 1 or HTTP 2 but not HTTP 3. +You cannot enforce multiple protocols like `--http2 --http3`, they (toggles) are mutually exclusive. -The remote server yield its support for HTTP/3 in the `Alt-Svc` header, if present HTTPie will issue -the successive requests via HTTP/3. You may use that argument in case the remote peer does not support -either HTTP/1.1 or HTTP/2. +### Protocol selection + +By default, HTTPie follows what modern browser do to choose a protocol. + +#### For HTTP URLs + +HTTP/1.1 will always be chosen unless you specified `--http2` to enforce HTTP/2 with prior knowledge (also known as h2c). + +Notes: + +- You cannot enforce HTTP/3. +- You cannot disable both HTTP/1.1 and HTTP/2. -### Protocol combinations +#### For HTTPS URLs -Following `Force HTTP/3, HTTP/2 and HTTP/1` and `Disable HTTP/1, HTTP/2, or HTTP/3`, you may find a summary on how to make HTTPie negotiate a -specific protocol. +When reaching to an SSL/TLS server, HTTPie negotiate the protocol through what is called the ALPN extension during +the handshake. -| Arguments | HTTP/1.1
enabled | HTTP/2
enabled | HTTP/3
enabled | -|----------------------------------:|:--------------------:|:------------------:|:------------------:| -| (Default) | ✔ | ✔ | ✔ | -| `--disable-http1` | ✗ | ✔ | ✔ | -| `--disable-http2` | ✔ | ✗ | ✔ | -| `--disable-http3` | ✔ | ✔ | ✗ | -| `--disable-http2 --disable-http3` | ✔ | ✗ | ✗ | -| `--disable-http1 --disable-http2` | ✗ | ✗ | ✔ | -| `--http1` | ✔ | ✗ | ✗ | -| `--http2` | ✗ | ✔ | ✗ | -| `--http3` | ✗ | ✗ | ✔ | +Basically, HTTPie says during the "Hello" phase: "I can speak HTTP/1.1 and HTTP/2 over TCP, and you?". +Depending on what the server respond to us, we will choose a mutual supported protocols. -Some specifics, through: +Nowadays, it is most certainly be HTTP/2 by default. -- You cannot enforce HTTP/3 over non HTTPS URLs. -- You cannot disable both HTTP/1.1 and HTTP/2 for non HTTPS URLs. -- Of course, you cannot disable all three protocols. +Some specifics: + +- You cannot disable all three protocols. - Those toggles do not apply to the DNS-over-HTTPS custom resolver. You will have to specify it within the resolver URL. - When reaching a HTTPS URL, the ALPN extension sent during SSL/TLS handshake is affected. +- HTTPie never tries HTTP/3 by default unless something hints us that it is possible. + +##### HTTP 3 Negotiation + +By default, HTTPie cannot negotiate HTTP/3 without a first HTTP/1.1, or HTTP/2 successful response unless the +remote host specified a DNS HTTPS record that indicate its support (and by using a custom DNS resolver, see bellow section). + +The remote server yield its support for HTTP/3 in the `Alt-Svc` header, if present HTTPie will issue +the successive requests via HTTP/3. You may use that argument in case the remote peer does not support +either HTTP/1.1 or HTTP/2. + +Note: HTTPie caches what server are QUIC compatible in the `config` directory so that we can remember. ## Custom DNS resolver diff --git a/extras/man/http.1 b/extras/man/http.1 index ed69895f8a..e13ccf6be3 100644 --- a/extras/man/http.1 +++ b/extras/man/http.1 @@ -1,5 +1,5 @@ .\" This file is auto-generated from the parser declaration in httpie/cli/definition.py by extras/scripts/generate_man_pages.py. -.TH http 1 "2024-01-01" "HTTPie 4.0.0.b1" "HTTPie Manual" +.TH http 1 "2024-06-25" "HTTPie 4.0.0" "HTTPie Manual" .SH NAME http .SH SYNOPSIS @@ -496,12 +496,30 @@ Bypass dot segment (/../ or /./) URL squashing. Enable streaming via chunked transfer encoding. The Transfer-Encoding header is set to chunked. +.IP "\fB\,--disable-http1\/\fR" + + +Disable the HTTP/1 protocol. + + +.IP "\fB\,--http1\/\fR" + + +Use the HTTP/1 protocol for the request. + + .IP "\fB\,--disable-http2\/\fR" Disable the HTTP/2 protocol. +.IP "\fB\,--http2\/\fR" + + +Use the HTTP/2 protocol for the request. + + .IP "\fB\,--disable-http3\/\fR" diff --git a/extras/man/httpie.1 b/extras/man/httpie.1 index 69bd76265f..7f396ef9f9 100644 --- a/extras/man/httpie.1 +++ b/extras/man/httpie.1 @@ -1,5 +1,5 @@ .\" This file is auto-generated from the parser declaration in httpie/manager/cli.py by extras/scripts/generate_man_pages.py. -.TH httpie 1 "2024-01-01" "HTTPie 4.0.0.b1" "HTTPie Manual" +.TH httpie 1 "2024-06-25" "HTTPie 4.0.0" "HTTPie Manual" .SH NAME httpie .SH SYNOPSIS diff --git a/extras/man/https.1 b/extras/man/https.1 index bdb0ef82a6..c3b44d056a 100644 --- a/extras/man/https.1 +++ b/extras/man/https.1 @@ -1,5 +1,5 @@ .\" This file is auto-generated from the parser declaration in httpie/cli/definition.py by extras/scripts/generate_man_pages.py. -.TH https 1 "2024-01-01" "HTTPie 4.0.0.b1" "HTTPie Manual" +.TH https 1 "2024-06-25" "HTTPie 4.0.0" "HTTPie Manual" .SH NAME https .SH SYNOPSIS @@ -496,12 +496,30 @@ Bypass dot segment (/../ or /./) URL squashing. Enable streaming via chunked transfer encoding. The Transfer-Encoding header is set to chunked. +.IP "\fB\,--disable-http1\/\fR" + + +Disable the HTTP/1 protocol. + + +.IP "\fB\,--http1\/\fR" + + +Use the HTTP/1 protocol for the request. + + .IP "\fB\,--disable-http2\/\fR" Disable the HTTP/2 protocol. +.IP "\fB\,--http2\/\fR" + + +Use the HTTP/2 protocol for the request. + + .IP "\fB\,--disable-http3\/\fR" From 55aea9cce5fcc2d29e4a9941d27bb7483eb1c5a5 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 28 Jun 2024 00:24:53 +0100 Subject: [PATCH 52/63] fixed+enabled recently disabled tests for download + compressed bodies --- httpie/downloads.py | 26 +++++----- httpie/output/streams.py | 7 +++ tests/test_downloads.py | 104 +++++++++++++++++++-------------------- 3 files changed, 68 insertions(+), 69 deletions(-) diff --git a/httpie/downloads.py b/httpie/downloads.py index 4f6f75eed6..a07718cb21 100644 --- a/httpie/downloads.py +++ b/httpie/downloads.py @@ -211,12 +211,6 @@ def pre_request(self, request_headers: dict): Might alter `request_headers`. """ - # Ask the server not to encode the content so that we can resume, etc. - # TODO: Reconsider this once the underlying library can report raw download size (i.e., not decoded). - # Then it might still be needed when resuming. But in the default case, it won’t probably be necessary. - # - request_headers['Accept-Encoding'] = 'identity' - if self._resume: bytes_have = os.path.getsize(self._output_file.name) if bytes_have: @@ -301,11 +295,11 @@ def failed(self): def is_interrupted(self) -> bool: return self.status.is_interrupted - def chunk_downloaded(self, chunk_or_new_total: Union[bytes, int]): + def chunk_downloaded(self, chunk_or_new_total: Union[bytes, int]) -> None: """ A download progress callback. - :param chunk: A chunk of response body data that has just + :param chunk_or_new_total: A chunk of response body data that has just been downloaded and written to the output. """ @@ -333,8 +327,7 @@ def _get_output_file_from_response( return open(unique_filename, buffering=0, mode='a+b') -DECODED_FROM_SUFFIX = ' - decoded from {encodings}' -DECODED_SIZE_NOTE_SUFFIX = ' - decoded size' +DECODED_FROM_SUFFIX = ' - decoded using {encodings}' class DownloadStatus: @@ -365,8 +358,14 @@ def start_display(self, output_file): ProgressDisplayFull ) message = f'Downloading to {output_file.name}' - message_suffix = '' summary_suffix = '' + + if self.decoded_from: + encodings = ', '.join(f'`{enc}`' for enc in self.decoded_from) + message_suffix = DECODED_FROM_SUFFIX.format(encodings=encodings) + else: + message_suffix = '' + if not self.env.show_displays: progress_display_class = DummyProgressDisplay else: @@ -375,11 +374,8 @@ def start_display(self, output_file): if has_reliable_total: progress_display_class = ProgressDisplayFull else: - if self.decoded_from: - encodings = ', '.join(f'`{enc}`' for enc in self.decoded_from) - message_suffix = DECODED_FROM_SUFFIX.format(encodings=encodings) - summary_suffix = DECODED_SIZE_NOTE_SUFFIX progress_display_class = ProgressDisplayNoTotal + self.display = progress_display_class( env=self.env, total_size=self.total_size, diff --git a/httpie/output/streams.py b/httpie/output/streams.py index 83d1c5673e..5c1171336a 100644 --- a/httpie/output/streams.py +++ b/httpie/output/streams.py @@ -79,8 +79,15 @@ def __iter__(self) -> Iterable[bytes]: # Useful when the remote compress the body. We use the "untouched" amt of data to determine # the download speed. if hasattr(self.msg, "_orig") and hasattr(self.msg._orig, "download_progress") and self.msg._orig.download_progress: + # this is plan A: using public interfaces! self.on_body_chunk_downloaded(self.msg._orig.download_progress.total) + elif hasattr(self.msg, "_orig") and hasattr(self.msg._orig, "raw") and hasattr(self.msg._orig.raw, "_fp_bytes_read"): + # plan B, falling back on a private property that may disapear from urllib3-future... + # this case is mandatory due to how the mocking library works. it does not use any "socket" but + # rather a simple io.BytesIO. + self.on_body_chunk_downloaded(self.msg._orig.raw._fp_bytes_read) else: + # well. this case will certainly cause issues if the body is compressed. self.on_body_chunk_downloaded(chunk) except DataSuppressedError as e: if self.output_options.headers: diff --git a/tests/test_downloads.py b/tests/test_downloads.py index 4747fa399b..63acdc7d6e 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -1,6 +1,7 @@ import os import tempfile import time +import zlib from unittest import mock from urllib.request import urlopen @@ -15,7 +16,7 @@ ContentRangeError, Downloader, PARTIAL_CONTENT, - DECODED_SIZE_NOTE_SUFFIX, + DECODED_FROM_SUFFIX, ) from niquests.structures import CaseInsensitiveDict from .utils import http, MockEnvironment, cd_clean_tmp_dir, DUMMY_URL @@ -232,7 +233,6 @@ def test_download_resumed(self, mock_env, httpbin_both): # Ensure `pre_request()` is working as expected too headers = {} downloader.pre_request(headers) - assert headers['Accept-Encoding'] == 'identity' assert headers['Range'] == 'bytes=3-' downloader.start( @@ -264,7 +264,7 @@ def test_download_gzip_content_encoding(self, httpbin): @responses.activate def test_incomplete_response(self): # We have incompleteness checks in the downloader, but it might not be needed as it’s built into (ni|req)uests. - error_msg = 'peer closed connection without sending complete message body (received 2 bytes, expected 1 more)' + error_msg = 'IncompleteRead(2 bytes read, 1 more expected)' responses.add( method=responses.GET, url=DUMMY_URL, @@ -281,55 +281,53 @@ def test_incomplete_response(self): class TestDecodedDownloads: """Test downloading responses with `Content-Encoding`""" - # todo: find an appropriate way to mock compressed bodies within those tests. - # @responses.activate - # def test_decoded_response_no_content_length(self): - # responses.add( - # method=responses.GET, - # url=DUMMY_URL, - # headers={ - # 'Content-Encoding': 'gzip, br', - # }, - # body='123', - # ) - # with cd_clean_tmp_dir(): - # r = http('--download', '--headers', DUMMY_URL) - # print(r.stderr) - # assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr - # assert DECODED_SIZE_NOTE_SUFFIX in r.stderr - # - # @responses.activate - # def test_decoded_response_with_content_length(self): - # responses.add( - # method=responses.GET, - # url=DUMMY_URL, - # headers={ - # 'Content-Encoding': 'gzip, br', - # 'Content-Length': '3', - # }, - # body='123', - # ) - # with cd_clean_tmp_dir(): - # r = http('--download', DUMMY_URL) - # print(r.stderr) - # assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr - # assert DECODED_SIZE_NOTE_SUFFIX in r.stderr - # - # @responses.activate - # def test_decoded_response_without_content_length(self): - # responses.add( - # method=responses.GET, - # url=DUMMY_URL, - # headers={ - # 'Content-Encoding': 'gzip, br', - # }, - # body='123', - # ) - # with cd_clean_tmp_dir(): - # r = http('--download', DUMMY_URL) - # print(r.stderr) - # assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr - # assert DECODED_SIZE_NOTE_SUFFIX in r.stderr + @responses.activate + def test_decoded_response_no_content_length(self): + responses.add( + method=responses.GET, + url=DUMMY_URL, + headers={ + 'Content-Encoding': 'deflate', + }, + body=zlib.compress(b"foobar"), + ) + with cd_clean_tmp_dir(): + r = http('--download', '--headers', DUMMY_URL) + print(r.stderr) + assert DECODED_FROM_SUFFIX.format(encodings='`deflate`') in r.stderr + + @responses.activate + def test_decoded_response_with_content_length(self): + payload = zlib.compress(b"foobar") + + responses.add( + method=responses.GET, + url=DUMMY_URL, + headers={ + 'Content-Encoding': 'deflate', + 'Content-Length': str(len(payload)), + }, + body=payload, + ) + with cd_clean_tmp_dir(): + r = http('--download', DUMMY_URL) + print(r.stderr) + assert DECODED_FROM_SUFFIX.format(encodings='`deflate`') in r.stderr + + @responses.activate + def test_decoded_response_without_content_length(self): + responses.add( + method=responses.GET, + url=DUMMY_URL, + headers={ + 'Content-Encoding': 'deflate', + }, + body=zlib.compress(b'foobar'), + ) + with cd_clean_tmp_dir(): + r = http('--download', DUMMY_URL) + print(r.stderr) + assert DECODED_FROM_SUFFIX.format(encodings='`deflate`') in r.stderr @responses.activate def test_non_decoded_response_without_content_length(self): @@ -344,7 +342,6 @@ def test_non_decoded_response_without_content_length(self): with cd_clean_tmp_dir(): r = http('--download', DUMMY_URL) print(r.stderr) - assert DECODED_SIZE_NOTE_SUFFIX not in r.stderr @responses.activate def test_non_decoded_response_with_content_length(self): @@ -358,4 +355,3 @@ def test_non_decoded_response_with_content_length(self): with cd_clean_tmp_dir(): r = http('--download', DUMMY_URL) print(r.stderr) - assert DECODED_SIZE_NOTE_SUFFIX not in r.stderr From 1379a2e2819ec89b7df5192c1d772648c82fc3f9 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Sun, 13 Oct 2024 20:09:35 +0200 Subject: [PATCH 53/63] add support for early responses 1xx (informational) --- CHANGELOG.md | 3 ++- httpie/client.py | 8 +++++--- httpie/core.py | 24 +++++++++++++++--------- setup.cfg | 2 +- tests/test_early_response.py | 11 +++++++++++ 5 files changed, 34 insertions(+), 14 deletions(-) create mode 100644 tests/test_early_response.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ffdd6915d..dd20033204 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,11 @@ This document records all notable changes to [HTTPie](https://httpie.io). This project adheres to [Semantic Versioning](https://semver.org/). -## [4.0.0.b1](https://github.com/httpie/cli/compare/3.2.2...master) (unreleased) +## [4.0.0](https://github.com/httpie/cli/compare/3.2.3...master) (unreleased) - Switched from the [`requests`](https://github.com/psf/requests) library to the compatible [`niquests`](https://github.com/jawah/niquests). ([#1531](https://github.com/httpie/cli/pull/1531)) - Added support for HTTP/2, and HTTP/3 protocols. ([#523](https://github.com/httpie/cli/issues/523), [#692](https://github.com/httpie/cli/issues/692), [#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for early (informational) responses. ([#752](https://github.com/httpie/cli/issues/752)) ([#1531](https://github.com/httpie/cli/pull/1531)) - Added support for IPv4/IPv6 enforcement with `-6` and `-4`. ([#94](https://github.com/httpie/cli/issues/94), [#1531](https://github.com/httpie/cli/pull/1531)) - Added support for alternative DNS resolvers via `--resolver`. DNS over HTTPS, DNS over TLS, DNS over QUIC, and DNS over UDP are accepted. ([#99](https://github.com/httpie/cli/issues/99), [#1531](https://github.com/httpie/cli/pull/1531)) - Added support for binding to a specific network adapter with `--interface`. ([#1422](https://github.com/httpie/cli/issues/1422), [#1531](https://github.com/httpie/cli/pull/1531)) diff --git a/httpie/client.py b/httpie/client.py index 27811b4e46..a771ac5419 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import json import sys @@ -43,7 +45,7 @@ def collect_messages( env: Environment, args: argparse.Namespace, request_body_read_callback: Callable[[bytes], None] = None, - prepared_request_readiness: Callable[[niquests.PreparedRequest], None] = None, + request_or_response_callback: Callable[[niquests.PreparedRequest | niquests.Response], None] = None, ) -> Iterable[RequestsMessage]: httpie_session = None httpie_session_headers = None @@ -155,8 +157,8 @@ def collect_messages( # It will help us yield the request before it is # actually sent. This will permit us to know about # the connection information for example. - if prepared_request_readiness: - hooks = {"pre_send": [prepared_request_readiness]} + if request_or_response_callback: + hooks = {"pre_send": [request_or_response_callback], "early_response": [request_or_response_callback]} request = niquests.Request(**request_kwargs, hooks=hooks) prepared_request = requests_session.prepare_request(request) diff --git a/httpie/core.py b/httpie/core.py index 28b0af2867..ccd4477967 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -3,6 +3,7 @@ import platform import sys import socket +from time import monotonic from typing import List, Optional, Union, Callable import niquests @@ -211,21 +212,26 @@ def request_body_read_callback(chunk: bytes): downloader = Downloader(env, output_file=args.output_file, resume=args.download_resume) downloader.pre_request(args.headers) - def prepared_request_readiness(pr): - """This callback is meant to output the request part. It is triggered by - the underlying Niquests library just after establishing the connection.""" + def request_or_response_callback(delayed_message): + """This callback is called in two scenario: + + (i) just after initializing a connection to remote host + (ii) an early response has been received (1xx responses)""" oo = OutputOptions.from_message( - pr, + delayed_message, args.output_options ) - oo = oo._replace( - body=isinstance(pr.body, (str, bytes)) and (args.verbose or oo.body) - ) + if hasattr(delayed_message, "body"): + oo = oo._replace( + body=isinstance(delayed_message.body, (str, bytes)) and (args.verbose or oo.body) + ) + else: + delayed_message._httpie_headers_parsed_at = monotonic() write_message( - requests_message=pr, + requests_message=delayed_message, env=env, output_options=oo, processing_options=processing_options @@ -238,7 +244,7 @@ def prepared_request_readiness(pr): env, args=args, request_body_read_callback=request_body_read_callback, - prepared_request_readiness=prepared_request_readiness + request_or_response_callback=request_or_response_callback ) force_separator = False diff --git a/setup.cfg b/setup.cfg index cc71785d25..9d7c0e9ca4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -71,7 +71,7 @@ install_requires = pip charset_normalizer>=2.0.0 defusedxml>=0.6.0 - niquests[socks]>=3.7 + niquests[socks]>=3.9 Pygments>=2.5.2 setuptools importlib-metadata>=1.4.0; python_version<"3.8" diff --git a/tests/test_early_response.py b/tests/test_early_response.py new file mode 100644 index 0000000000..809c098bfa --- /dev/null +++ b/tests/test_early_response.py @@ -0,0 +1,11 @@ +from .utils import http + + +def test_early_response_show(remote_httpbin_secure): + r = http( + "--verify=no", + 'https://early-hints.fastlylabs.com/' + ) + + assert "103 Early Hints" in r + assert "200 OK" in r From 03bac4d848fba853f446c04068db1ff356ed32b0 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Sun, 13 Oct 2024 20:10:27 +0200 Subject: [PATCH 54/63] use ubuntu-22.04 to preserve python 3.7 and add python 3.13 in tests --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 787867928f..a8ac8aaa3d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,8 +24,9 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-13, windows-latest] + os: [ubuntu-22.04, macos-13, windows-latest] python-version: + - '3.13' - '3.12' - '3.11' - '3.10' From 7f5c8a10e69b6a08344d57783e6a39fd5cafccc1 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Sun, 13 Oct 2024 20:17:09 +0200 Subject: [PATCH 55/63] fix test_h3_not_compatible_anymore as niquests automatically downgrade in case of failure --- tests/test_h2n3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_h2n3.py b/tests/test_h2n3.py index 3f3ddd82b4..53d5cd448a 100644 --- a/tests/test_h2n3.py +++ b/tests/test_h2n3.py @@ -95,7 +95,7 @@ def test_h3_not_compatible_anymore(remote_httpbin_secure, with_quic_cache_persis tolerate_error_exit_status=True ) - assert "Unable to connect. Was the remote specified HTTP/3 compatible but is not anymore?" in r.stderr + assert "HTTP/2 200 OK" in r # with timeout r = http( @@ -106,4 +106,4 @@ def test_h3_not_compatible_anymore(remote_httpbin_secure, with_quic_cache_persis tolerate_error_exit_status=True ) - assert "Unable to connect. Was the remote specified HTTP/3 compatible but is not anymore?" in r.stderr + assert "HTTP/2 200 OK" in r From 46ca048a3a6d3817b23a88cb3537642d41e6eee2 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Sun, 13 Oct 2024 20:32:40 +0200 Subject: [PATCH 56/63] add note on changed test_h3_not_compatible_anymore --- tests/test_h2n3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_h2n3.py b/tests/test_h2n3.py index 53d5cd448a..4a9a3304ae 100644 --- a/tests/test_h2n3.py +++ b/tests/test_h2n3.py @@ -80,6 +80,7 @@ def test_ensure_quic_cache(remote_httpbin_secure, with_quic_cache_persistent): def test_h3_not_compatible_anymore(remote_httpbin_secure, with_quic_cache_persistent): + """verify that we can handle failures and fallback appropriately.""" tmp_path = with_quic_cache_persistent.config['quic_file'] cache = QuicCapabilityCache(tmp_path) From f6f6619140f1e756dbc1cfd95297b01639871a51 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Sun, 13 Oct 2024 20:41:56 +0200 Subject: [PATCH 57/63] fix test when http3 support is unavailable (windows+py3.7) --- tests/test_h2n3.py | 8 ++++++++ tests/test_meta.py | 6 ++++++ tests/test_resolver.py | 8 ++++++++ 3 files changed, 22 insertions(+) diff --git a/tests/test_h2n3.py b/tests/test_h2n3.py index 4a9a3304ae..44d8444650 100644 --- a/tests/test_h2n3.py +++ b/tests/test_h2n3.py @@ -5,6 +5,11 @@ from .utils import HTTP_OK, http, PersistentMockEnvironment +try: + import qh3 +except ImportError: + qh3 = None + def test_should_not_do_http1_by_default(remote_httpbin_secure): r = http( @@ -29,6 +34,7 @@ def test_disable_http2n3(remote_httpbin_secure): assert HTTP_OK in r +@pytest.mark.skipif(qh3 is None, reason="test require HTTP/3 support") def test_force_http3(remote_httpbin_secure): r = http( "--verify=no", @@ -48,6 +54,7 @@ def with_quic_cache_persistent(tmp_path): env.cleanup(force=True) +@pytest.mark.skipif(qh3 is None, reason="test require HTTP/3 support") def test_ensure_quic_cache(remote_httpbin_secure, with_quic_cache_persistent): """ This test aim to verify that the QuicCapabilityCache work as intended. @@ -79,6 +86,7 @@ def test_ensure_quic_cache(remote_httpbin_secure, with_quic_cache_persistent): assert "pie.dev" in list(cache.keys())[0] +@pytest.mark.skipif(qh3 is None, reason="test require HTTP/3 support") def test_h3_not_compatible_anymore(remote_httpbin_secure, with_quic_cache_persistent): """verify that we can handle failures and fallback appropriately.""" tmp_path = with_quic_cache_persistent.config['quic_file'] diff --git a/tests/test_meta.py b/tests/test_meta.py index fa8a7e4d20..1e47834e2c 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -4,12 +4,18 @@ from httpie.output.formatters.colors import PIE_STYLE_NAMES from .utils import http, MockEnvironment, COLOR +try: + import qh3 +except ImportError: + qh3 = None + def test_meta_elapsed_time(httpbin): r = http('--meta', httpbin + '/delay/1') assert f'{ELAPSED_TIME_LABEL}: 1.' in r +@pytest.mark.skipif(qh3 is None, reason="test require HTTP/3 support") def test_meta_extended_tls(remote_httpbin_secure): # using --verify=no may cause the certificate information not to display with Python < 3.10 # it is guaranteed to be there when using HTTP/3 over QUIC. That's why we set the '--http3' flag. diff --git a/tests/test_resolver.py b/tests/test_resolver.py index b9c6c8b690..eebd0cf0ca 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -1,6 +1,14 @@ +import pytest + from .utils import http +try: + import qh3 +except ImportError: + qh3 = None + +@pytest.mark.skipif(qh3 is None, reason="test require HTTP/3 support") def test_ensure_resolver_used(remote_httpbin_secure): """This test ensure we're using specified resolver to get into pie.dev. Using a custom resolver with Niquests enable direct HTTP/3 negotiation and pie.dev From 2ac67d59ff71c0311b97944241a5ae03b788e569 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Tue, 15 Oct 2024 09:44:37 +0200 Subject: [PATCH 58/63] fix unclosed file in case of failed download --- httpie/downloads.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/httpie/downloads.py b/httpie/downloads.py index a07718cb21..bfd98eaffd 100644 --- a/httpie/downloads.py +++ b/httpie/downloads.py @@ -289,6 +289,8 @@ def finish(self): self._output_file.close() def failed(self): + if self._output_file_created: + self._output_file.close() self.status.terminate() @property From 7537b40bfcd602120871f09f60d5568299dcd390 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Tue, 15 Oct 2024 09:45:05 +0200 Subject: [PATCH 59/63] add support for happy eyeballs --- CHANGELOG.md | 1 + docs/README.md | 11 +++++++++++ httpie/cli/definition.py | 14 ++++++++++++++ httpie/client.py | 5 +++++ tests/test_network.py | 10 ++++++++++ 5 files changed, 41 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd20033204..bb7b879701 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Added support for HTTP/2, and HTTP/3 protocols. ([#523](https://github.com/httpie/cli/issues/523), [#692](https://github.com/httpie/cli/issues/692), [#1531](https://github.com/httpie/cli/pull/1531)) - Added support for early (informational) responses. ([#752](https://github.com/httpie/cli/issues/752)) ([#1531](https://github.com/httpie/cli/pull/1531)) - Added support for IPv4/IPv6 enforcement with `-6` and `-4`. ([#94](https://github.com/httpie/cli/issues/94), [#1531](https://github.com/httpie/cli/pull/1531)) +- Added support for Happy Eyeballs algorithm via `--heb` flag (disabled by default). [#1599](https://github.com/httpie/cli/issues/1599) [#1531](https://github.com/httpie/cli/pull/1531) - Added support for alternative DNS resolvers via `--resolver`. DNS over HTTPS, DNS over TLS, DNS over QUIC, and DNS over UDP are accepted. ([#99](https://github.com/httpie/cli/issues/99), [#1531](https://github.com/httpie/cli/pull/1531)) - Added support for binding to a specific network adapter with `--interface`. ([#1422](https://github.com/httpie/cli/issues/1422), [#1531](https://github.com/httpie/cli/pull/1531)) - Added support for specifying the local port with `--local-port`. ([#1456](https://github.com/httpie/cli/issues/1456), [#1531](https://github.com/httpie/cli/pull/1531)) diff --git a/docs/README.md b/docs/README.md index cddb8621a6..8d339be817 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1970,6 +1970,17 @@ You can specify multiple entries, concatenated with a comma: $ https --resolver "pie.dev:10.10.4.1,re.pie.dev:10.10.8.1" pie.dev/get ``` +## Happy Eyeballs + +By default, when HTTPie establish the connection it asks for the IP(v4 or v6) records of +the requested domain and then tries them sequentially preferring IPv6 by default. This +may induce longer connection delays and in some case hangs due to an unresponsive endpoint. +To concurrently try to connect to available IP(v4 or v6), set the following flag: + +```bash +$ https --heb pie.dev/get +``` + ## Network interface In order to bind emitted request from a specific network adapter you can use the `--interface` flag. diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 2f6bf55c4e..44061593a4 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -864,6 +864,20 @@ def format_auth_help(auth_plugins_mapping, *, isolation_mode: bool = False): """ ) +network.add_argument( + "--heb", + default=False, + dest="happy_eyeballs", + action="store_true", + short_help="Establish the connection using IETF Happy Eyeballs algorithm", + help=""" + By default, when HTTPie establish the connection it asks for the IP(v4 or v6) records of + the requested domain and then tries them sequentially preferring IPv6 by default. This + may induce longer connection delays and in some case hangs due to an unresponsive endpoint. + To concurrently try to connect to available IP(v4 or v6), set this flag. + + """ +) network.add_argument( "--resolver", default=[], diff --git a/httpie/client.py b/httpie/client.py index a771ac5419..63c5bd7262 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -118,6 +118,7 @@ def collect_messages( disable_ipv4=args.ipv6, source_address=source_address, quic_cache=env.config.quic_file, + happy_eyeballs=args.happy_eyeballs, ) if args.disable_http3 is False and args.force_http3 is True: @@ -237,6 +238,7 @@ def build_requests_session( disable_ipv6: bool = False, source_address: typing.Tuple[str, int] = None, quic_cache: typing.Optional[Path] = None, + happy_eyeballs: bool = False, ) -> niquests.Session: requests_session = niquests.Session() @@ -260,6 +262,8 @@ def build_requests_session( source_address=source_address, disable_http1=disable_http1, disable_http2=disable_http2, + disable_http3=disable_http3, + happy_eyeballs=happy_eyeballs, ) https_adapter = HTTPieHTTPSAdapter( ciphers=ciphers, @@ -276,6 +280,7 @@ def build_requests_session( disable_ipv6=disable_ipv6, source_address=source_address, quic_cache_layer=requests_session.quic_cache_layer, + happy_eyeballs=happy_eyeballs, ) requests_session.mount('http://', http_adapter) requests_session.mount('https://', https_adapter) diff --git a/tests/test_network.py b/tests/test_network.py index 0be34d04d5..4480470081 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -36,3 +36,13 @@ def test_ensure_interface_and_port_parameters(httpbin): assert r.exit_status == 0 assert HTTP_OK in r + + +def test_happy_eyeballs(remote_httpbin_secure): + r = http( + "--heb", # this will automatically and concurrently try IPv6 and IPv4 endpoints + remote_httpbin_secure + "/get", + ) + + assert r.exit_status == 0 + assert HTTP_OK in r From dd60adb0c6377c3f48d9a8dd4d07b96250f3b200 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Tue, 15 Oct 2024 09:45:43 +0200 Subject: [PATCH 60/63] ensure all plugins are migrated to Niquests prior to importing them --- httpie/plugins/manager.py | 13 ++++++++++++- tests/conftest.py | 13 ------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/httpie/plugins/manager.py b/httpie/plugins/manager.py index 27af6eedac..1cba1f66d3 100644 --- a/httpie/plugins/manager.py +++ b/httpie/plugins/manager.py @@ -8,12 +8,23 @@ from pathlib import Path from contextlib import contextmanager, nullcontext -from ..compat import importlib_metadata, find_entry_points, get_dist_name +import niquests + +from ..compat import importlib_metadata, find_entry_points, get_dist_name, urllib3 from ..utils import repr_dict, get_site_paths from . import AuthPlugin, ConverterPlugin, FormatterPlugin, TransportPlugin from .base import BasePlugin +# make sure all the plugins are forced to migrate to Niquests +# as it is a drop in replacement for Requests, everything should +# run smooth. +sys.modules["requests"] = niquests +sys.modules["requests.adapters"] = niquests.adapters +sys.modules["requests.sessions"] = niquests.sessions +sys.modules["requests.exceptions"] = niquests.exceptions +sys.modules["requests.packages.urllib3"] = urllib3 + ENTRY_POINT_CLASSES = { 'httpie.plugins.auth.v1': AuthPlugin, diff --git a/tests/conftest.py b/tests/conftest.py index aac94e3269..31ef202a61 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,10 +3,6 @@ import pytest from pytest_httpbin import certs from pytest_httpbin.serve import Server as PyTestHttpBinServer -from sys import modules - -import niquests -import urllib3 from .utils import ( # noqa HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, @@ -28,15 +24,6 @@ # Patch to support `url = str(server)` in addition to `url = server + '/foo'`. PyTestHttpBinServer.__str__ = lambda self: self.url -# the mock utility 'response' only works with 'requests' -# we're trying to fool it, thinking requests is there. -# to remove when a similar (or same, but compatible) -# utility emerge for Niquests. -modules["requests"] = niquests -modules["requests.adapters"] = niquests.adapters -modules["requests.exceptions"] = niquests.exceptions -modules["requests.packages.urllib3"] = urllib3 - @pytest.fixture(scope='function', autouse=True) def httpbin_add_ca_bundle(monkeypatch): From 7a1c3419ef69294ab2bc71ccd5724c5a9288d230 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Tue, 15 Oct 2024 09:49:51 +0200 Subject: [PATCH 61/63] update man pages --- extras/man/http.1 | 10 ++++++++++ extras/man/https.1 | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/extras/man/http.1 b/extras/man/http.1 index e13ccf6be3..fe6acba454 100644 --- a/extras/man/http.1 +++ b/extras/man/http.1 @@ -538,6 +538,16 @@ either HTTP/1.1 or HTTP/2. +.IP "\fB\,--heb\/\fR" + + +By default, when HTTPie establish the connection it asks for the IP(v4 or v6) records of +the requested domain and then tries them sequentially preferring IPv6 by default. This +may induce longer connection delays and in some case hangs due to an unresponsive endpoint. +To concurrently try to connect to available IP(v4 or v6), set this flag. + + + .IP "\fB\,--resolver\/\fR" diff --git a/extras/man/https.1 b/extras/man/https.1 index c3b44d056a..c6e450f484 100644 --- a/extras/man/https.1 +++ b/extras/man/https.1 @@ -538,6 +538,16 @@ either HTTP/1.1 or HTTP/2. +.IP "\fB\,--heb\/\fR" + + +By default, when HTTPie establish the connection it asks for the IP(v4 or v6) records of +the requested domain and then tries them sequentially preferring IPv6 by default. This +may induce longer connection delays and in some case hangs due to an unresponsive endpoint. +To concurrently try to connect to available IP(v4 or v6), set this flag. + + + .IP "\fB\,--resolver\/\fR" From 5e87d1b0574e962ff3779f4613df692847d8d020 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Tue, 15 Oct 2024 09:52:13 +0200 Subject: [PATCH 62/63] update test_network.py --- tests/test_network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_network.py b/tests/test_network.py index 4480470081..0b5624d4e0 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -41,6 +41,7 @@ def test_ensure_interface_and_port_parameters(httpbin): def test_happy_eyeballs(remote_httpbin_secure): r = http( "--heb", # this will automatically and concurrently try IPv6 and IPv4 endpoints + "--verify=no", remote_httpbin_secure + "/get", ) From da6cc13b8b775834edbde68836c826390da3dbc3 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Wed, 16 Oct 2024 08:19:42 +0200 Subject: [PATCH 63/63] unpin werkzeug, allow <4 fix https://github.com/httpie/cli/issues/1530 --- setup.cfg | 4 ++-- tests/test_binary.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9d7c0e9ca4..647fdc4b42 100644 --- a/setup.cfg +++ b/setup.cfg @@ -103,7 +103,7 @@ dev = pytest-httpbin>=0.0.6 responses pytest-mock - werkzeug<2.1.0 + werkzeug<4 flake8 flake8-comprehensions flake8-deprecated @@ -119,7 +119,7 @@ test = pytest-httpbin>=0.0.6 responses pytest-mock - werkzeug<2.1.0 + werkzeug<4 [options.data_files] share/man/man1 = diff --git a/tests/test_binary.py b/tests/test_binary.py index 2203c71d63..a699fcc9d9 100644 --- a/tests/test_binary.py +++ b/tests/test_binary.py @@ -32,19 +32,22 @@ def test_binary_file_form(self, httpbin): class TestBinaryResponseData: + """local httpbin crash due to an unfixed bug. + See https://github.com/psf/httpbin/pull/41 + It is merged but not yet released.""" - def test_binary_suppresses_when_terminal(self, httpbin): - r = http('GET', httpbin + '/bytes/1024?seed=1') + def test_binary_suppresses_when_terminal(self, remote_httpbin): + r = http('GET', remote_httpbin + '/bytes/1024?seed=1') assert BINARY_SUPPRESSED_NOTICE.decode() in r - def test_binary_suppresses_when_not_terminal_but_pretty(self, httpbin): + def test_binary_suppresses_when_not_terminal_but_pretty(self, remote_httpbin): env = MockEnvironment(stdin_isatty=True, stdout_isatty=False) - r = http('--pretty=all', 'GET', httpbin + '/bytes/1024?seed=1', env=env) + r = http('--pretty=all', 'GET', remote_httpbin + '/bytes/1024?seed=1', env=env) assert BINARY_SUPPRESSED_NOTICE.decode() in r - def test_binary_included_and_correct_when_suitable(self, httpbin): + def test_binary_included_and_correct_when_suitable(self, remote_httpbin): env = MockEnvironment(stdin_isatty=True, stdout_isatty=False) - url = httpbin + '/bytes/1024?seed=1' + url = remote_httpbin + '/bytes/1024?seed=1' r = http('GET', url, env=env) expected = niquests.get(url).content assert r == expected