diff --git a/Pipfile.lock b/Pipfile.lock index 8a6bb0d..aaf0c06 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -16,11 +16,11 @@ "default": { "awscli": { "hashes": [ - "sha256:1888b771ce0b175f927209ae9f3e7282e54162b185182b22927f5e32094cd9c6", - "sha256:91279fefc5fc8a39799da23e6facda2767679ab5338d6122f2152cc9f8db1587" + "sha256:2b0ab35ac6adbc65f00125830089602e83b175a80f53c467d5c7b149924a56bc", + "sha256:2fb44ff10ede8cc18376407719777c5d052d7c4039e09395f3ece75402222219" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.19.36" + "version": "==1.19.64" }, "bagit": { "hashes": [ @@ -43,23 +43,23 @@ "globus" ], "path": ".", - "version": "==1.6.1" + "version": "==1.6.2" }, "boto3": { "hashes": [ - "sha256:16ca7a34eb88138e0d1ae2532e17975eef578aa1754e2d209ad41a8dfce059ce", - "sha256:75f59fb3d764a381bb0108cb5036b398d0c8a1cf719e6b5aadbc5a53a1fd735e" + "sha256:ac10d832ad716281da6ca77cea824d723af479f8611087dee4b0489c48c32fd9", + "sha256:e2ef25afc36a301199bfbd662aef46dd11ed0db9baf96fce111db4043928065b" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.17.36" + "version": "==1.17.64" }, "botocore": { "hashes": [ - "sha256:148f5d7d48c54ed450831e5dd4d13284b2418955b6d99db23a3d9c4c6cb515c8", - "sha256:9ce33bd4175d58c5fdeb8e35052aa370aff74b347227e65ddc3f4fa01ef0686f" + "sha256:42dde7c699b3710e5c3a944cd8ce8b7a80b9f610d8857a0ad36bdc9743cc3375", + "sha256:ec418c273c37efd33d39bb4559f7df09de46df1f87fdbb064d8ebb281849a625" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.20.36" + "version": "==1.20.64" }, "certifi": { "hashes": [ @@ -128,20 +128,20 @@ }, "cryptography": { "hashes": [ - "sha256:066bc53f052dfeda2f2d7c195cf16fb3e5ff13e1b6b7415b468514b40b381a5b", - "sha256:0923ba600d00718d63a3976f23cab19aef10c1765038945628cd9be047ad0336", - "sha256:2d32223e5b0ee02943f32b19245b61a62db83a882f0e76cc564e1cec60d48f87", - "sha256:4169a27b818de4a1860720108b55a2801f32b6ae79e7f99c00d79f2a2822eeb7", - "sha256:57ad77d32917bc55299b16d3b996ffa42a1c73c6cfa829b14043c561288d2799", - "sha256:5ecf2bcb34d17415e89b546dbb44e73080f747e504273e4d4987630493cded1b", - "sha256:600cf9bfe75e96d965509a4c0b2b183f74a4fa6f5331dcb40fb7b77b7c2484df", - "sha256:66b57a9ca4b3221d51b237094b0303843b914b7d5afd4349970bb26518e350b0", - "sha256:93cfe5b7ff006de13e1e89830810ecbd014791b042cbe5eec253be11ac2b28f3", - "sha256:9e98b452132963678e3ac6c73f7010fe53adf72209a32854d55690acac3f6724", - "sha256:df186fcbf86dc1ce56305becb8434e4b6b7504bc724b71ad7a3239e0c9d14ef2", - "sha256:fec7fb46b10da10d9e1d078d1ff8ed9e05ae14f431fdbd11145edd0550b9a964" - ], - "version": "==3.4.6" + "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d", + "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959", + "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6", + "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873", + "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2", + "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713", + "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1", + "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177", + "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250", + "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca", + "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d", + "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9" + ], + "version": "==3.4.7" }, "docutils": { "hashes": [ @@ -272,18 +272,18 @@ }, "rsa": { "hashes": [ - "sha256:35c5b5f6675ac02120036d97cf96f1fde4d49670543db2822ba5015e21a18032", - "sha256:4d409f5a7d78530a4a2062574c7bd80311bc3af29b364e293aa9b03eea77714f" + "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", + "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==4.5" + "markers": "python_version > '2.7'", + "version": "==4.7.2" }, "s3transfer": { "hashes": [ - "sha256:5d48b1fd2232141a9d5fb279709117aaba506cacea7f86f11bc392f06bfa8fc2", - "sha256:c5dadf598762899d8cfaecf68eba649cd25b0ce93b6c954b156aaa3eed160547" + "sha256:9b3752887a2880690ce628bc263d6d13a3864083aeacff4890c1c9839a5eb0bc", + "sha256:cb022f4b16551edebbb31a377d3f09600dbada7363d8c5db7976e7f47732e1b2" ], - "version": "==0.3.6" + "version": "==0.4.2" }, "setuptools-scm": { "hashes": [ @@ -384,19 +384,19 @@ }, "flake8": { "hashes": [ - "sha256:12d05ab02614b6aee8df7c36b97d1a3b2372761222b19b58621355e82acddcff", - "sha256:78873e372b12b093da7b5e5ed302e8ad9e988b38b063b61ad937f26ca58fc5f0" + "sha256:1aa8990be1e689d96c745c5682b687ea49f2e05a443aff1f8251092b0014e378", + "sha256:3b9f848952dddccf635be78098ca75010f073bfe14d2c6bda867154bea728d2a" ], "index": "pypi", - "version": "==3.9.0" + "version": "==3.9.1" }, "importlib-metadata": { "hashes": [ - "sha256:742add720a20d0467df2f444ae41704000f50e1234f46174b51f9c6031a1bd71", - "sha256:b74159469b464a99cb8cc3e21973e4d96e05d3024d337313fedb618a6e86e6f4" + "sha256:8c501196e49fb9df5df43833bdb1e4328f64847763ec8a50703148b73784d581", + "sha256:d7eb1dea6d6a6086f8be21784cc9e3bcfa55872b52309bc5fad53a8ea444465d" ], "markers": "python_version < '3.8'", - "version": "==3.7.3" + "version": "==4.0.1" }, "mccabe": { "hashes": [ @@ -431,12 +431,12 @@ }, "typing-extensions": { "hashes": [ - "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", - "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", - "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497", + "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", + "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" ], "markers": "python_version < '3.8'", - "version": "==3.7.4.3" + "version": "==3.10.0.0" }, "zipp": { "hashes": [ diff --git a/bdbag/__init__.py b/bdbag/__init__.py index 5f87d77..be96de2 100644 --- a/bdbag/__init__.py +++ b/bdbag/__init__.py @@ -25,7 +25,7 @@ from distutils.util import strtobool from pkg_resources import parse_version, get_distribution, DistributionNotFound -__version__ = "1.6.1" +__version__ = "1.6.2" __bagit_version__ = "1.8.1" __bagit_profile_version__ = "1.3.1" diff --git a/bdbag/fetch/transports/fetch_http.py b/bdbag/fetch/transports/fetch_http.py index 35a9a4a..333c892 100644 --- a/bdbag/fetch/transports/fetch_http.py +++ b/bdbag/fetch/transports/fetch_http.py @@ -18,9 +18,10 @@ import logging import certifi import requests +from requests.utils import default_user_agent from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry -from bdbag import urlsplit, stob, get_typed_exception +from bdbag import urlsplit, stob, get_typed_exception, VERSION from bdbag.bdbag_config import DEFAULT_CONFIG, DEFAULT_FETCH_CONFIG, FETCH_CONFIG_TAG, \ FETCH_HTTP_REDIRECT_STATUS_CODES_TAG, DEFAULT_FETCH_HTTP_SESSION_CONFIG, DEFAULT_FETCH_HTTP_REDIRECT_STATUS_CODES from bdbag.fetch import * @@ -31,7 +32,7 @@ logger = logging.getLogger(__name__) CHUNK_SIZE = 10 * Megabyte -HEADERS = {"Connection": "keep-alive"} +HEADERS = {"User-Agent": "bdbag/%s (%s)" % (VERSION, default_user_agent())} class HTTPFetchTransport(BaseFetchTransport): @@ -178,7 +179,8 @@ def get_session(self, url): def fetch(self, url, output_path, **kwargs): try: - headers = kwargs.get("headers", HEADERS) + headers = kwargs.get("headers", {"Connection": "keep-alive"}) + headers.update(HEADERS) redirect_status_codes = self.config.get( FETCH_HTTP_REDIRECT_STATUS_CODES_TAG, DEFAULT_FETCH_HTTP_REDIRECT_STATUS_CODES) diff --git a/setup.py b/setup.py index ca9c59d..eac61a2 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ long_description=readme, long_description_content_type='text/markdown', url='https://github.com/fair-research/bdbag/', + author="Mike D'Arcy", maintainer='USC Information Sciences Institute, Informatics Systems Research Division', maintainer_email='isrd-support@isi.edu', version=__version__,