archive-gh-me/rasterio
1
0
Fork 0
mirror of https://github.com/rasterio/rasterio.git synced 2025-12-08 17:36:12 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'maint-1.1'

Browse Source
This commit is contained in:
Sean Gillies 2020-02-13 17:28:53 -07:00
commit ef8b9c9680
4 changed files with 93 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
rasterio/env.py
View File

@ -8,19 +8,13 @@ import re
import threading
import warnings
try:
import boto3
except ImportError:
log.info("failed to import boto3, continuing.")
boto3 = None
from rasterio._env import (
GDALEnv, get_gdal_config, set_gdal_config,
GDALDataFinder, PROJDataFinder, set_proj_data_search_path)
from rasterio.compat import string_types, getargspec
from rasterio.errors import (
EnvError, GDALVersionError, RasterioDeprecationWarning)
from rasterio.session import Session, AWSSession, DummySession
from rasterio.session import Session, DummySession
class ThreadEnv(threading.local):
@ -110,7 +104,7 @@ class Env(object):
}
def __init__(self, session=None, aws_unsigned=False, profile_name=None,
session_class=AWSSession, **options):
session_class=Session.aws_or_dummy, **options):
"""Create a new GDAL/AWS environment.
Note: this class is a context manager. GDAL isn't configured
@ -187,12 +181,12 @@ class Env(object):
"AWSSession object instead.",
RasterioDeprecationWarning
)
session = AWSSession(session=session)
session = Session.aws_or_dummy(session=session)
self.session = session
elif aws_access_key_id or profile_name or aws_unsigned:
self.session = AWSSession(
self.session = Session.aws_or_dummy(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
@ -200,13 +194,8 @@ class Env(object):
profile_name=profile_name,
aws_unsigned=aws_unsigned)
elif 'AWS_ACCESS_KEY_ID' in os.environ and 'AWS_SECRET_ACCESS_KEY' in os.environ and boto3 is not None:
try:
self.session = AWSSession()
self.session.credentials
except RuntimeError as exc:
log.info("No AWS session created. Credentials in environment have expired.")
self.session = DummySession()
elif 'AWS_ACCESS_KEY_ID' in os.environ and 'AWS_SECRET_ACCESS_KEY' in os.environ:
self.session = Session.from_environ()
else:
self.session = DummySession()

86
rasterio/session.py
View File

@ -2,17 +2,17 @@
import logging
try:
import boto3
except ImportError:
log.info("failed to import boto3, continuing.")
boto3 = None
from rasterio.path import parse_path, UnparsedPath
log = logging.getLogger(__name__)
try:
import boto3
except ImportError:
log.debug("Could not import boto3, continuing with reduced functionality.")
boto3 = None
class Session(object):
"""Base for classes that configure access to secured resources.
@ -100,7 +100,7 @@ class Session(object):
return DummySession
elif path.scheme == "s3" or "amazonaws.com" in path.path:
if boto3:
if boto3 is not None:
return AWSSession
else:
log.info("boto3 not available, falling back to a DummySession.")
@ -109,7 +109,6 @@ class Session(object):
elif path.scheme == "oss" or "aliyuncs.com" in path.path:
return OSSSession
elif path.path.startswith("/vsiswift/"):
return SwiftSession
@ -140,6 +139,55 @@ class Session(object):
"""
return Session.cls_from_path(path)(*args, **kwargs)
@staticmethod
def aws_or_dummy(*args, **kwargs):
"""Create an AWSSession if boto3 is available, else DummySession
Parameters
----------
path : str
A dataset path or identifier.
args : sequence
Positional arguments for the foreign session constructor.
kwargs : dict
Keyword arguments for the foreign session constructor.
Returns
-------
Session
"""
if boto3 is not None:
return AWSSession(*args, **kwargs)
else:
return DummySession(*args, **kwargs)
@staticmethod
def from_environ(*args, **kwargs):
"""Create a session object suited to the environment.
Parameters
----------
path : str
A dataset path or identifier.
args : sequence
Positional arguments for the foreign session constructor.
kwargs : dict
Keyword arguments for the foreign session constructor.
Returns
-------
Session
"""
try:
session = Session.aws_or_dummy(*args, **kwargs)
session.credentials
except RuntimeError as exc:
log.warn("Credentials in environment have expired. Creating a DummySession.")
session = DummySession(*args, **kwargs)
return session
class DummySession(Session):
"""A dummy session.
@ -193,7 +241,7 @@ class AWSSession(Session):
aws_secret_access_key=None, aws_session_token=None,
region_name=None, profile_name=None, endpoint_url=None,
requester_pays=False):
"""Create a new boto3 session
"""Create a new AWS session
Parameters
----------
@ -254,7 +302,7 @@ class AWSSession(Session):
def credentials(self):
"""The session credentials as a dict"""
res = {}
if self._creds: # pragma: no branch
if self._creds: # pragma: no branch
frozen_creds = self._creds.get_frozen_credentials()
if frozen_creds.access_key: # pragma: no branch
res['aws_access_key_id'] = frozen_creds.access_key
@ -308,7 +356,7 @@ class OSSSession(Session):
"oss_secret_access_key": oss_secret_access_key,
"oss_endpoint": oss_endpoint
}
@classmethod
def hascreds(cls, config):
"""Determine if the given configuration has proper credentials
@ -397,14 +445,15 @@ class GSSession(Session):
class SwiftSession(Session):
"""Configures access to secured resources stored in OpenStack Swift Object Storage.
"""
def __init__(self, session=None,
swift_storage_url=None, swift_auth_token=None,
def __init__(self, session=None,
swift_storage_url=None, swift_auth_token=None,
swift_auth_v1_url=None, swift_user=None, swift_key=None):
"""Create new OpenStack Swift Object Storage Session.
Three methods are possible:
"""Create new OpenStack Swift Object Storage Session.
Three methods are possible:
1. Create session by the swiftclient library.
2. The SWIFT_STORAGE_URL and SWIFT_AUTH_TOKEN (this method is recommended by GDAL docs).
3. The SWIFT_AUTH_V1_URL, SWIFT_USER and SWIFT_KEY (This depends on the swiftclient library).
2. The SWIFT_STORAGE_URL and SWIFT_AUTH_TOKEN (this method is recommended by GDAL docs).
3. The SWIFT_AUTH_V1_URL, SWIFT_USER and SWIFT_KEY (This depends on the swiftclient library).
Parameters
----------
@ -453,7 +502,7 @@ class SwiftSession(Session):
"swift_storage_url": self._session.get_auth()[0],
"swift_auth_token": self._session.get_auth()[1]
}
@classmethod
def hascreds(cls, config):
"""Determine if the given configuration has proper credentials
@ -481,4 +530,3 @@ class SwiftSession(Session):
dict
"""
return {k.upper(): v for k, v in self.credentials.items()}

14
tests/test_env.py
View File

@ -329,7 +329,6 @@ def test_s3_open_with_implicit_env(gdalenv):
def test_s3_open_with_implicit_env_no_boto3(monkeypatch, gdalenv):
"""Read from S3 using default env."""
with monkeypatch.context() as mpctx:
mpctx.setattr("rasterio.env.boto3", None)
mpctx.setattr("rasterio.session.boto3", None)
with rasterio.open(L8TIF) as dataset:
assert dataset.count == 1
@ -843,11 +842,12 @@ def test_swift_session_credentials(gdalenv):
def test_swift_session_by_user_key():
def mock_init(self, session=None,
swift_storage_url=None, swift_auth_token=None,
swift_auth_v1_url=None, swift_user=None, swift_key=None):
self._creds = {'SWIFT_STORAGE_URL':'foo',
'SWIFT_AUTH_TOKEN':'bar'}
def mock_init(
self, session=None,
swift_storage_url=None, swift_auth_token=None,
swift_auth_v1_url=None, swift_user=None, swift_key=None):
self._creds = {'SWIFT_STORAGE_URL': 'foo',
'SWIFT_AUTH_TOKEN': 'bar'}
with mock.patch('rasterio.session.SwiftSession.__init__', new=mock_init):
swift_session = SwiftSession(
@ -864,7 +864,7 @@ def test_dummy_session_without_boto3(monkeypatch, caplog):
"""Without boto3, always revert to dummy session"""
# Confirm fix of #1708.
with monkeypatch.context() as mpctx:
mpctx.setattr("rasterio.env.boto3", None)
mpctx.setattr("rasterio.session.boto3", None)
mpctx.setenv('AWS_ACCESS_KEY_ID', 'lol')
mpctx.setenv('AWS_SECRET_ACCESS_KEY', 'wut')
assert isinstance(rasterio.env.Env().session, DummySession)

14
tests/test_session.py
View File

@ -211,4 +211,16 @@ def test_session_factory_swift_kwargs():
sesh = Session.from_path("/vsiswift/lol/wut", swift_storage_url='foo', swift_auth_token='bar')
assert isinstance(sesh, DummySession)
def test_session_aws_or_dummy_aws():
"""Get an AWSSession when boto3 is available"""
boto3 = pytest.importorskip("boto3")
assert isinstance(Session.aws_or_dummy(), AWSSession)
def test_session_aws_or_dummy_dummy(monkeypatch):
"""Get a DummySession when boto3 is not available"""
boto3 = pytest.importorskip("boto3")
with monkeypatch.context() as mpctx:
mpctx.setattr("rasterio.session.boto3", None)
assert isinstance(Session.aws_or_dummy(), DummySession)