How do I get STACReader to recognize an s3 URI?

[tylere]

I am trying to adapt the STACReader example code to work with a public STAC Item file stored on AWS S3 using the s3 URI scheme.

Using the Object URL works:

from rio_tiler.io import STACReader
s3_object_url = "https://test-stac-data-catalog-public.s3.us-west-2.amazonaws.com/stac-spec/examples/simple-item.json"
with STACReader(s3_object_url) as stac:
    pass

However, using the s3 URI fails:

s3_uri = 's3://test-stac-data-catalog-public/stac-spec/examples/simple-item.json'
with STACReader(s3_uri) as stac:
    pass

with the error

ValueError: Invalid endpoint: s3.amazonaws.com

stack trace

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[5], line 3
      1 s3_uri = 's3://test-stac-data-catalog-public/stac-spec/examples/simple-item.json'
----> 3 with STACReader(s3_uri) as stac:
      4     pass
File :28, in init(self, input, item, tms, minzoom, maxzoom, geographic_crs, include_assets, exclude_assets, include_asset_types, exclude_asset_types, reader, reader_options, fetch_options, ctx)

26     self.fetch_options = __attr_factory_fetch_options()

27 self.ctx = ctx

---> 28 self.attrs_post_init()
File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/rio_tiler/io/stac.py:253, in STACReader.attrs_post_init(self)

250 def attrs_post_init(self):

251     """Fetch STAC Item and get list of valid assets."""

252     self.item = self.item or pystac.Item.from_dict(

--> 253         fetch(self.input, **self.fetch_options), self.input

254     )

256     # TODO: get bounds/crs using PROJ extension if available

257     self.bounds = self.item.bbox
File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/cachetools/init.py:737, in cached..decorator..wrapper(*args, **kwargs)

735 except KeyError:

736     pass  # key not found

--> 737 v = func(*args, **kwargs)

738 try:

739     cache[k] = v
File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/rio_tiler/io/stac.py:119, in fetch(filepath, **kwargs)

117     bucket = parsed.netloc

118     key = parsed.path.strip("/")

--> 119     return json.loads(aws_get_object(bucket, key, **kwargs))

121 elif parsed.scheme in ["https", "http", "ftp"]:

122     resp = httpx.get(filepath, **kwargs)
File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/rio_tiler/io/stac.py:85, in aws_get_object(bucket, key, request_pays, client)

82         else:

83             endpoint_url = "http://" + endpoint_url

---> 85     client = session.client(

86         "s3",

87         endpoint_url=endpoint_url or "s3.amazonaws.com",

88     )

90 params = {"Bucket": bucket, "Key": key}

91 if request_pays or os.environ.get("AWS_REQUEST_PAYER", "").lower() == "requester":
File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/boto3/session.py:299, in Session.client(self, service_name, region_name, api_version, use_ssl, verify, endpoint_url, aws_access_key_id, aws_secret_access_key, aws_session_token, config)

217 def client(

218     self,

219     service_name,

(...)

228     config=None,

229 ):

230     """

231     Create a low-level service client by name.

232

(...)

297

298     """

--> 299     return self._session.create_client(

300         service_name,

301         region_name=region_name,

302         api_version=api_version,

303         use_ssl=use_ssl,

304         verify=verify,

305         endpoint_url=endpoint_url,

306         aws_access_key_id=aws_access_key_id,

307         aws_secret_access_key=aws_secret_access_key,

308         aws_session_token=aws_session_token,

309         config=config,

310     )
File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/botocore/session.py:997, in Session.create_client(self, service_name, region_name, api_version, use_ssl, verify, endpoint_url, aws_access_key_id, aws_secret_access_key, aws_session_token, config)

980 self._add_configured_endpoint_provider(

981     client_name=service_name,

982     config_store=config_store,

983 )

985 client_creator = botocore.client.ClientCreator(

986     loader,

987     endpoint_resolver,

(...)

995     user_agent_creator=user_agent_creator,

996 )

--> 997 client = client_creator.create_client(

998     service_name=service_name,

999     region_name=region_name,

1000     is_secure=use_ssl,

1001     endpoint_url=endpoint_url,

1002     verify=verify,

1003     credentials=credentials,

1004     scoped_config=self.get_scoped_config(),

1005     client_config=config,

1006     api_version=api_version,

1007     auth_token=auth_token,

1008 )

1009 monitor = self._get_internal_component('monitor')

1010 if monitor is not None:
File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/botocore/client.py:161, in ClientCreator.create_client(self, service_name, region_name, is_secure, endpoint_url, verify, credentials, scoped_config, api_version, client_config, auth_token)

148 region_name, client_config = self._normalize_fips_region(

149     region_name, client_config

150 )

151 endpoint_bridge = ClientEndpointBridge(

152     self._endpoint_resolver,

153     scoped_config,

(...)

159     ),

160 )

--> 161 client_args = self._get_client_args(

162     service_model,

163     region_name,

164     is_secure,

165     endpoint_url,

166     verify,

167     credentials,

168     scoped_config,

169     client_config,

170     endpoint_bridge,

171     auth_token,

172     endpoints_ruleset_data,

173     partition_data,

174 )

175 service_client = cls(**client_args)

176 self._register_retries(service_client)
File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/botocore/client.py:508, in ClientCreator._get_client_args(self, service_model, region_name, is_secure, endpoint_url, verify, credentials, scoped_config, client_config, endpoint_bridge, auth_token, endpoints_ruleset_data, partition_data)

484 def _get_client_args(

485     self,

486     service_model,

(...)

497     partition_data,

498 ):

499     args_creator = ClientArgsCreator(

500         self._event_emitter,

501         self._user_agent,

(...)

506         user_agent_creator=self._user_agent_creator,

507     )

--> 508     return args_creator.get_client_args(

509         service_model,

510         region_name,

511         is_secure,

512         endpoint_url,

513         verify,

514         credentials,

515         scoped_config,

516         client_config,

517         endpoint_bridge,

518         auth_token,

519         endpoints_ruleset_data,

520         partition_data,

521     )
File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/botocore/args.py:137, in ClientArgsCreator.get_client_args(self, service_model, region_name, is_secure, endpoint_url, verify, credentials, scoped_config, client_config, endpoint_bridge, auth_token, endpoints_ruleset_data, partition_data)

134 new_config = Config(**config_kwargs)

135 endpoint_creator = EndpointCreator(event_emitter)

--> 137 endpoint = endpoint_creator.create_endpoint(

138     service_model,

139     region_name=endpoint_region_name,

140     endpoint_url=endpoint_config['endpoint_url'],

141     verify=verify,

142     response_parser_factory=self._response_parser_factory,

143     max_pool_connections=new_config.max_pool_connections,

144     proxies=new_config.proxies,

145     timeout=(new_config.connect_timeout, new_config.read_timeout),

146     socket_options=socket_options,

147     client_cert=new_config.client_cert,

148     proxies_config=new_config.proxies_config,

149 )

151 serializer = botocore.serialize.create_serializer(

152     protocol, parameter_validation

153 )

154 response_parser = botocore.parsers.create_parser(protocol)
File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/botocore/endpoint.py:402, in EndpointCreator.create_endpoint(self, service_model, region_name, endpoint_url, verify, response_parser_factory, timeout, max_pool_connections, http_session_cls, proxies, socket_options, client_cert, proxies_config)

384 def create_endpoint(

385     self,

386     service_model,

(...)

397     proxies_config=None,

398 ):

399     if not is_valid_endpoint_url(

400         endpoint_url

401     ) and not is_valid_ipv6_endpoint_url(endpoint_url):

--> 402         raise ValueError("Invalid endpoint: %s" % endpoint_url)

404     if proxies is None:

405         proxies = self._get_proxies(endpoint_url)
ValueError: Invalid endpoint: s3.amazonaws.com

To get STACReader to recognize s3 URIs, do you need to setup a custom StacIO class?

[vincentsarago]

👋
I think I've seen that recently

it should work natively with S3 URL (as long as you have credential set in your environment) https://github.com/cogeotiff/rio-tiler/blob/main/rio_tiler/io/stac.py#L54-L88

let me have a look

[tylere]

I'm using a shared credentials file and a default profile. Here is my AWS config:

% aws configure list
      Name                    Value             Type    Location
      ----                    -----             ----    --------
   profile                <not set>             None    None
access_key     ****************VDRQ shared-credentials-file    
secret_key     ****************hdp9 shared-credentials-file    
    region                us-west-2      config-file    ~/.aws/config

Fwiw, with this configuration I am able to successfully access to the STAC Item's asset href using rio_tiler.io.Reader. The asset href uses the s3:// scheme.

import rio_tiler
from IPython.display import Image

stac_item_asset_url = stac_item.assets['asset'].href

def get_tile(_stac_item_asset_url, tile_x, tile_y, tile_z):
    with rasterio.Env(rasterio_aws_session):
        with rio_tiler.io.Reader(_stac_item_asset_url) as image:
            img = image.tile(tile_x, tile_y, tile_z)
            img.rescale(
                in_range=((0, 600),),
                out_range=((0, 255),)
            )
    cm = rio_tiler.colormap.cmap.get("greens")
    return Image(img.render(colormap=cm))

[tylere]

@vincentsarago given that it is working fine for you, could you check how your execution path and/or variables differ from what I am seeing in the traceback? Below I have included the values of the local variables at each level of the traceback.

Environment variables

AWS envars: [{'AWS_REGION': 'us-west-2'}, {'AWS_DEFAULT_REGION': 'us-west-2'}]

---------------------------------------------------------------------------
ClientError                               Traceback (most recent call last)
Cell In[2], line 11
      8 print('AWS envars:', [{k:v} for k,v in os.environ.items() if 'AWS' in k])
     10 s3_uri = 's3://test-stac-data-catalog-public/stac-spec/examples/simple-item.json'
---> 11 with STACReader(s3_uri) as stac:
     12     pass

File <attrs generated init rio_tiler.io.stac.STACReader>:28, in __init__(self, input, item, tms, minzoom, maxzoom, geographic_crs, include_assets, exclude_assets, include_asset_types, exclude_asset_types, reader, reader_options, fetch_options, ctx)
     26     self.fetch_options = __attr_factory_fetch_options()
     27 self.ctx = ctx
---> 28 self.__attrs_post_init__()

locals() @ <attrs generated init rio_tiler.io.stac.STACReader>:28

{
    "self": STACReader(
        bounds=NOTHING,
        crs=NOTHING,
        assets=NOTHING,
        input="s3://test-stac-data-catalog-public/stac-spec/examples/simple-item.json",
        item=None,
        tms=,
        reader_options={},
        fetch_options={},
        ctx=
    ), 
    "input": 
        "s3: //test-stac-data-catalog-public/stac-spec/examples/simple-item.json", 
        "item": None, 
        "tms": ,
            "reader_options": NOTHING, 
            "fetch_options": NOTHING, 
            "ctx": 
}

File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/rio_tiler/io/stac.py:250, in STACReader.__attrs_post_init__(self)
    247 def __attrs_post_init__(self):
    248     """Fetch STAC Item and get list of valid assets."""
    249     self.item = self.item or pystac.Item.from_dict(
--> 250         fetch(self.input, **self.fetch_options), self.input
    251     )
    253     # TODO: get bounds/crs using PROJ extension if available
    254     self.bounds = self.item.bbox

locals() @ rio_tiler/io/stac.py:250

{
    "self": STACReader(
        bounds=NOTHING, 
        crs=NOTHING, 
        assets=NOTHING, 
        input="s3://test-stac-data-catalog-public/stac-spec/examples/simple-item.json", 
        item=None, 
        tms=,
        reader_options={},
        fetch_options={},
        ctx=
    )
}

File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/cachetools/__init__.py:737, in cached.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
    735 except KeyError:
    736     pass  # key not found
--> 737 v = func(*args, **kwargs)
    738 try:
    739     cache[k] = v

locals() @ cachetools/__init__.py:737

{
    "args": ("s3://test-stac-data-catalog-public/stac-spec/examples/simple-item.json",),
    "kwargs": {},
    "k": ("s3://test-stac-data-catalog-public/stac-spec/examples/simple-item.json",
    "{}"),
    "cache": LRUCache({}, maxsize=512, currsize=0),
    "func": ,
    "key":  at 0x1567dfce0>
}

File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/rio_tiler/io/stac.py:116, in fetch(filepath, **kwargs)
    114     bucket = parsed.netloc
    115     key = parsed.path.strip("/")
--> 116     return json.loads(aws_get_object(bucket, key, **kwargs))
    118 elif parsed.scheme in ["https", "http", "ftp"]:
    119     resp = httpx.get(filepath, **kwargs)

locals() @ rio_tiler/io/stac.py:116

{
    "filepath": "s3://test-stac-data-catalog-public/stac-spec/examples/simple-item.json",
    "kwargs": {},
    "parsed": ParseResult(scheme="s3", netloc="test-stac-data-catalog-public", path="/stac-spec/examples/simple-item.json", params="", query="", fragment=""),
    "bucket": "test-stac-data-catalog-public",
    "key": "stac-spec/examples/simple-item.json"
}

File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/rio_tiler/io/stac.py:91, in aws_get_object(bucket, key, request_pays, client)
     88 if request_pays or os.environ.get("AWS_REQUEST_PAYER", "").lower() == "requester":
     89     params["RequestPayer"] = "requester"
---> 91 response = client.get_object(**params)
     92 return response["Body"].read()

locals() @ rio_tiler/io/stac.py:91

{
    "bucket": "test-stac-data-catalog-public",
    "key": "stac-spec/examples/simple-item.json",
    "request_pays": False,
    "client": ,
    "profile_name": None,
    "session": Session(region_name="us-west-2"),
    "access_key": None,
    "secret_access_key": None,
    "access_token": None,
    "region_name": "us-west-2",
    "endpoint_url": "https://s3.amazonaws.com",
    "use_https": "YES",
    "params": {
        "Bucket": "test-stac-data-catalog-public",
        "Key": "stac-spec/examples/simple-item.json"
    }
}

File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/botocore/client.py:553, in ClientCreator._create_api_method.<locals>._api_call(self, *args, **kwargs)
    549     raise TypeError(
    550         f"{py_operation_name}() only accepts keyword arguments."
    551     )
    552 # The "self" in this scope is referring to the BaseClient.
--> 553 return self._make_api_call(operation_name, kwargs)

locals() @ botocore/client.py:553

{
    "self": "",
    "args": (),
    "kwargs": {
        "Bucket": "test-stac-data-catalog-public",
        "Key": "stac-spec/examples/simple-item.json"
    },
    "operation_name": "GetObject",
    "py_operation_name": "get_object"
}

File ~/Documents/GitHub/ctrees-products/stac-catalog/.pixi/env/lib/python3.11/site-packages/botocore/client.py:1009, in BaseClient._make_api_call(self, operation_name, api_params)
   1005     error_code = error_info.get("QueryErrorCode") or error_info.get(
   1006         "Code"
   1007     )
   1008     error_class = self.exceptions.from_code(error_code)
-> 1009     raise error_class(parsed_response, operation_name)
   1010 else:
   1011     return parsed_response

ClientError: An error occurred (PermanentRedirect) when calling the GetObject operation: The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.

locals() @ botocore/client.py:1009

{
    "self": ,
    "operation_name": "GetObject",
    "api_params": {
        "Bucket": "test-stac-data-catalog-public",
        "Key": "stac-spec/examples/simple-item.json"
    },
    "operation_model": OperationModel(name=GetObject),
    "service_name": "s3",
    "request_context": {
        "client_region": "us-west-2",
        "client_config": ,
        "has_streaming_input": False,
        "auth_type": "v4",
        "s3_redirect": {
            "redirected": True,
            "bucket": "test-stac-data-catalog-public",
            "params": {
                "Bucket": "test-stac-data-catalog-public",
                "Key": "stac-spec/examples/simple-item.json"
            }
        },
        "S3Express": {
            "bucket_name": "test-stac-data-catalog-public"
        },
        "signing": {
            "region": "us-west-2",
            "signing_name": "s3",
            "disableDoubleEncoding": True
        },
        "endpoint_properties": {
            "authSchemes": [
                {
                    "disableDoubleEncoding": True,
                    "name": "sigv4",
                    "signingName": "s3",
                    "signingRegion": "us-west-2"
                }
            ]
        },
        "retries": {
            "attempt": 2,
            "invocation-id": "711fa417-0da7-472f-b261-05e5e5ef785f",
            "max": 5,
            "ttl": "20231230T161138Z"
        },
        "timestamp": "20231230T161038Z"
    },
    "endpoint_url": "https://s3.amazonaws.com/test-stac-data-catalog-public",
    "additional_headers": {},
    "properties": {
        "authSchemes": [
            {
                "disableDoubleEncoding": True,
                "name": "sigv4",
                "signingName": "s3",
                "signingRegion": "us-west-2"
            }
        ]
    },
    "request_dict": {
        "url_path": "/stac-spec/examples/simple-item.json",
        "query_string": {},
        "method": "GET",
        "headers": {
            "User-Agent": "Boto3/1.33.6 md/Botocore#1.33.6 ua/2.0 os/macos#22.4.0 md/arch#arm64 lang/python#3.11.0 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.33.6"
        },
        "body": b"",
        "auth_path": "/test-stac-data-catalog-public/stac-spec/examples/simple-item.json",
        "url": "https://s3.amazonaws.com/test-stac-data-catalog-public/stac-spec/examples/simple-item.json",
        "context": {
            "client_region": "us-west-2",
            "client_config": ,
            "has_streaming_input": False,
            "auth_type": "v4",
            "s3_redirect": {
                "redirected": True,
                "bucket": "test-stac-data-catalog-public",
                "params": {
                    "Bucket": "test-stac-data-catalog-public",
                    "Key": "stac-spec/examples/simple-item.json"
                }
            },
            "S3Express": {
                "bucket_name": "test-stac-data-catalog-public"
            },
            "signing": {
                "region": "us-west-2",
                "signing_name": "s3",
                "disableDoubleEncoding": True
            },
            "endpoint_properties": {
                "authSchemes": [
                    {
                        "disableDoubleEncoding": True,
                        "name": "sigv4",
                        "signingName": "s3",
                        "signingRegion": "us-west-2"
                    }
                ]
            },
            "retries": {
                "attempt": 2,
                "invocation-id": "711fa417-0da7-472f-b261-05e5e5ef785f",
                "max": 5,
                "ttl": "20231230T161138Z"
            },
            "timestamp": "20231230T161038Z"
        }
    },
    "service_id": "s3",
    "handler": ,
    "event_response": None,
    "http": ,
    "parsed_response": {
        "Error": {
            "Code": "PermanentRedirect",
            "Message": "The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.",
            "Endpoint": "test-stac-data-catalog-public.s3.amazonaws.com",
            "Bucket": "test-stac-data-catalog-public"
        },
        "ResponseMetadata": {
            "RequestId": "VQ5AMCE9CY3V3YYV",
            "HostId": "SL24MtrWWU1nknxe74Hp4e1BerT9Dw5GMovhBEKSyEG2A6LHE4+/uRBpKY5OUZ6OPwN+vdFAQBs=",
            "HTTPStatusCode": 301,
            "HTTPHeaders": {
                "x-amz-bucket-region": "us-west-2",
                "x-amz-request-id": "VQ5AMCE9CY3V3YYV",
                "x-amz-id-2": "SL24MtrWWU1nknxe74Hp4e1BerT9Dw5GMovhBEKSyEG2A6LHE4+/uRBpKY5OUZ6OPwN+vdFAQBs=",
                "content-type": "application/xml",
                "transfer-encoding": "chunked",
                "date": "Sat, 30 Dec 2023 16:10:38 GMT",
                "server": "AmazonS3"
            },
            "RetryAttempts": 1
        }
    },
    "error_info": {
        "Code": "PermanentRedirect",
        "Message": "The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.",
        "Endpoint": "test-stac-data-catalog-public.s3.amazonaws.com",
        "Bucket": "test-stac-data-catalog-public"
    },
    "error_code": "PermanentRedirect",
    "error_class": 
}

[tylere]

The boto3 docs say that the endpoint_url parameter is normally not specified, but automatically constructed:
https://github.com/boto/boto3/blob/master/boto3/session.py#L265

    :param endpoint_url: The complete URL to use for the constructed
       client. Normally, botocore will automatically construct the
       appropriate URL to use when communicating with a service.  You
       can specify a complete URL (including the "http/https" scheme)
       to override this behavior.  If this value is provided,
       then ``use_ssl`` is ignored.

I find that if I do not pass the endpoint_url (i.e. client = session.client("s3")), the error is avoided.

Is there a reason that rio-tiler constructs and passes a value for endpoint_url?
https://github.com/cogeotiff/rio-tiler/blob/main/rio_tiler/io/stac.py#L85

[vincentsarago]

ok so I think we should do something like

        # AWS_S3_ENDPOINT and AWS_HTTPS are GDAL config options of vsis3 driver
        # https://gdal.org/user/virtual_file_systems.html#vsis3-aws-s3-files
        endpoint_url = os.environ.get("AWS_S3_ENDPOINT", None)
        if endpoint_url:
            use_https = os.environ.get("AWS_HTTPS", "YES")
            if use_https.upper() in ["YES", "TRUE", "ON"]:
                endpoint_url = "https://" + endpoint_url

            else:
                endpoint_url = "http://" + endpoint_url

        client = session.client("s3", endpoint_url=endpoint_url)

[tylere]

Confirming that your suggested change will likely work for my use case.

from rio_tiler.io import STACReader
import boto3

boto3.set_stream_logger('botocore', level='INFO')

s3_uri = 's3://test-stac-data-catalog-public/stac-spec/examples/simple-item.json'
with STACReader(s3_uri) as stac:
    pass

results in

2024-01-05 09:38:16,332 botocore.credentials [INFO] Found credentials in shared credentials file: ~/.aws/credentials

[tylere]

Can rio-tiler make use of a rasterio AWS session? Something like:

with rasterio.Env(rasterio_aws_session):
    with rio_tiler.io.STACReader(s3_uri) as stac:
        pass

[vincentsarago]

you can do, but I still don't get why the snippet doesn't work :-(

client = rasterio_aws_session.client("s3")
with rio_tiler.io.STACReader(s3_uri, fetch_option={"client": client}) as stac:
        pass