From 801847c6be981eb83b1bd450dca753764608716f Mon Sep 17 00:00:00 2001 From: "Michael A. Smith" Date: Mon, 12 Jul 2021 15:18:11 -0400 Subject: [PATCH] Load Session from Zipfile When attempting to use botocore / boto3 in AWS EMR or PySpark, it's necessary to load from a zipfile. However, botocore does not support this. ``` + PYTHONPATH=botocore.zip:boto3.zip + .venv/bin/python -c 'import boto3; client=boto3.client("rds")' Traceback (most recent call last): File "", line 1, in File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/boto3.zip/boto3/__init__.py", line 91, in client File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/boto3.zip/boto3/session.py", line 258, in client File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/session.py", line 827, in create_client File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/session.py", line 700, in _get_internal_component File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/session.py", line 924, in get_component File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/session.py", line 163, in create_default_resolver File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/loaders.py", line 132, in _wrapper File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/loaders.py", line 424, in load_data botocore.exceptions.DataNotFoundError: Unable to load data for: endpoints ``` This is a re-submission of #1969 [with permission from @gliptak](https://github.com/boto/botocore/pull/1969#issuecomment-878509145), the original author. --- botocore/__init__.py | 2 -- botocore/loaders.py | 5 +++-- tests/functional/test_zip.py | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 tests/functional/test_zip.py diff --git a/botocore/__init__.py b/botocore/__init__.py index 3d7482d70d..a4a23710ff 100644 --- a/botocore/__init__.py +++ b/botocore/__init__.py @@ -60,8 +60,6 @@ def emit(self, record): # individual case. ScalarTypes = ('string', 'integer', 'boolean', 'timestamp', 'float', 'double') -BOTOCORE_ROOT = os.path.dirname(os.path.abspath(__file__)) - # Used to specify anonymous (unsigned) request signature class UNSIGNED(object): diff --git a/botocore/loaders.py b/botocore/loaders.py index dbb83d5110..ac3cfe5f9b 100644 --- a/botocore/loaders.py +++ b/botocore/loaders.py @@ -104,7 +104,8 @@ import logging import os -from botocore import BOTOCORE_ROOT +import pkg_resources + from botocore.compat import OrderedDict, json from botocore.exceptions import DataNotFoundError, UnknownServiceError from botocore.utils import deep_merge @@ -209,7 +210,7 @@ class Loader(object): """ FILE_LOADER_CLASS = JSONFileLoader # The included models in botocore/data/ that we ship with botocore. - BUILTIN_DATA_PATH = os.path.join(BOTOCORE_ROOT, 'data') + BUILTIN_DATA_PATH = pkg_resources.resource_filename('botocore', 'data') # For convenience we automatically add ~/.aws/models to the data path. CUSTOMER_DATA_PATH = os.path.join(os.path.expanduser('~'), '.aws', 'models') diff --git a/tests/functional/test_zip.py b/tests/functional/test_zip.py new file mode 100644 index 0000000000..d3517f6146 --- /dev/null +++ b/tests/functional/test_zip.py @@ -0,0 +1,20 @@ +import shutil +import tempfile +import os +import nose +import sys +from tests import BaseSessionTest + +class ZipTest(BaseSessionTest): + def test_load_from_zip(self): + fd, temp_path = tempfile.mkstemp(suffix='.zip') + root, ext = os.path.splitext(temp_path) + shutil.make_archive(root, 'zip', '../..') + sys.path.insert(0, temp_path) + super(ZipTest, self).setUp() + self.region = 'us-west-2' + self.client = self.session.create_client( + 's3', self.region) + sys.path.remove(temp_path) + os.close(fd) + os.remove(temp_path)