Merge branch 'maint-1.3' into main

.github/workflows/tests.yaml

@@ -37,7 +37,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ['3.9', '3.10']
-        gdal-version: ['3.5.0', '3.4.3']
+        gdal-version: ['3.5.2', '3.4.3']
         include:
           - python-version: '3.8'
             gdal-version: '3.3.3'

.gitignore

@@ -88,3 +88,4 @@ MANIFEST
 .ipynb_checkpoints
 .pytest_cache
 *.ipynb
+mprofile*.dat

CHANGES.txt

@@ -1,6 +1,30 @@
 Changes
 =======
 
+1.3.3 (2022-10-19)
+------------------
+
+Packaging:
+
+The rasterio._loading module, which supports DLL loading on Windows,
+has been moved into __init__.py and is no longer used anywhere else
+(#2594).
+
+Bug fixes:
+
+- GeoTIFF profiles combining tiled=False and a defined blockysize are now
+  supported (#2599).
+- Rasterio's Python file VSI plugin now sets the position of the underlying
+  stream to 0 on close, resolving #2550.
+
+1.3.2 (2022-08-19)
+------------------
+
+Packaging:
+
+1.3.1 was released from a branch lacking the advertised bug fixes. 1.3.2
+contains the fixes listed under 1.3.1.
+
 1.3.1 (2022-08-17)
 ------------------
 

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools", "wheel", "cython~=0.29.24", "oldest-supported-numpy"]
+requires = ["setuptools", "wheel", "cython>=0.29.29", "oldest-supported-numpy"]
 
 [tool.pytest.ini_options]
 markers = [

rasterio/__init__.py

@@ -1,58 +1,83 @@
 """Rasterio"""
 
 from collections import namedtuple
-from contextlib import ExitStack
+import glob
 import logging
 from logging import NullHandler
 import os
+import platform
 import warnings
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    from rasterio._show_versions import show_versions
-    from rasterio._version import gdal_version, get_geos_version, get_proj_version
-    from rasterio.crs import CRS
-    from rasterio.drivers import driver_from_extension, is_blacklisted
-    from rasterio.dtypes import (
-        bool_,
-        ubyte,
-        sbyte,
-        uint8,
-        int8,
-        uint16,
-        int16,
-        uint32,
-        int32,
-        int64,
-        uint64,
-        float32,
-        float64,
-        complex_,
-        check_dtype,
-        complex_int16,
-    )
-    from rasterio.env import ensure_env_with_credentials, Env, env_ctx_if_needed
-    from rasterio.errors import RasterioIOError, DriverCapabilityError, RasterioDeprecationWarning
-    from rasterio.io import (
-        DatasetReader, get_writer_for_path, get_writer_for_driver, MemoryFile)
-    from rasterio.profiles import default_gtiff_profile
-    from rasterio.transform import Affine, guard_transform
-    from rasterio._path import _parse_path
+# On Windows we must explicitly register the directories that contain
+# the GDAL and supporting DLLs starting with Python 3.8. Presently, we
+# support the rasterio-wheels location or directories on the system's
+# executable path.
+if platform.system() == "Windows":
+    _whl_dir = os.path.join(os.path.dirname(__file__), ".libs")
+    if os.path.exists(_whl_dir):
+        os.add_dll_directory(_whl_dir)
+    else:
+        if "PATH" in os.environ:
+            for p in os.environ["PATH"].split(os.pathsep):
+                if glob.glob(os.path.join(p, "gdal*.dll")):
+                    os.add_dll_directory(p)
 
-    # These modules are imported from the Cython extensions, but are also import
-    # here to help tools like cx_Freeze find them automatically
-    import rasterio._err
-    import rasterio.coords
-    import rasterio.enums
-    import rasterio._path
 
-    try:
-        from rasterio.io import FilePath
-        have_vsi_plugin = True
-    except ImportError:
-        class FilePath:
-            pass
-        have_vsi_plugin = False
+from rasterio._show_versions import show_versions
+from rasterio._version import gdal_version, get_geos_version, get_proj_version
+from rasterio.crs import CRS
+from rasterio.drivers import driver_from_extension, is_blacklisted
+from rasterio.dtypes import (
+    bool_,
+    ubyte,
+    sbyte,
+    uint8,
+    int8,
+    uint16,
+    int16,
+    uint32,
+    int32,
+    int64,
+    uint64,
+    float32,
+    float64,
+    complex_,
+    check_dtype,
+    complex_int16,
+)
+from rasterio.env import ensure_env_with_credentials, Env
+from rasterio.errors import (
+    RasterioIOError,
+    DriverCapabilityError,
+    RasterioDeprecationWarning,
+)
+from rasterio.io import (
+    DatasetReader,
+    get_writer_for_path,
+    get_writer_for_driver,
+    MemoryFile,
+)
+from rasterio.profiles import default_gtiff_profile
+from rasterio.transform import Affine, guard_transform
+from rasterio._path import _parse_path
+
+# These modules are imported from the Cython extensions, but are also import
+# here to help tools like cx_Freeze find them automatically
+import rasterio._err
+import rasterio.coords
+import rasterio.enums
+import rasterio._path
+
+try:
+    from rasterio.io import FilePath
+
+    have_vsi_plugin = True
+except ImportError:
+
+    class FilePath:
+        pass
+
+    have_vsi_plugin = False
 
 __all__ = ['band', 'open', 'pad', 'Band', 'Env', 'CRS']
 __version__ = "1.4dev"
@@ -68,6 +93,7 @@ __geos_version__ = ".".join([str(version) for version in get_geos_version()])
 log = logging.getLogger(__name__)
 log.addHandler(NullHandler())
 
+
 # Remove this in 1.4.0 (see comment on gh-2423).
 def parse_path(path):
     warnings.warn(

rasterio/_base.pyx

@@ -1023,7 +1023,7 @@ cdef class DatasetBase:
                 blockysize=self.block_shapes[0][0],
                 tiled=True)
         else:
-            m.update(tiled=False)
+            m.update(blockysize=self.block_shapes[0][0], tiled=False)
         if self.compression:
             m['compress'] = self.compression.name
         if self.interleaving:

rasterio/_filepath.pyx

@@ -155,7 +155,9 @@ cdef size_t filepath_read(void *pFile, void *pBuffer, size_t nSize, size_t nCoun
 cdef int filepath_close(void *pFile) except -1 with gil:
     # Optional
     cdef object file_wrapper = <object>pFile
-    del _FILESYSTEM_INFO[file_wrapper._filepath_path]
+    cdef object file_obj = file_wrapper._file_obj
+    file_obj.seek(0)
+    _ = _FILESYSTEM_INFO.pop(file_wrapper._filepath_path, None)
     return 0
 
 

rasterio/_io.pyx

@@ -356,7 +356,7 @@ cdef char **convert_options(kwargs):
     for k, v in kwargs.items():
         if k.lower() in ['affine']:
             continue
-        elif k in ['BLOCKXSIZE', 'BLOCKYSIZE'] and not tiled:
+        elif k in ['BLOCKXSIZE'] and not tiled:
             continue
 
         # Special cases for enums and tuples.

rasterio/_loading.py

@@ -1,32 +0,0 @@
-import glob
-import os
-import logging
-import contextlib
-import platform
-import sys
-
-log = logging.getLogger(__name__)
-log.addHandler(logging.NullHandler())
-
-# With Python >= 3.8 on Windows directories in PATH are not automatically
-# searched for DLL dependencies and must be added manually with
-# os.add_dll_directory.
-# see https://github.com/Toblerity/Fiona/issues/851
-
-@contextlib.contextmanager
-def add_gdal_dll_directories():
-    dll_dirs = []
-    if platform.system() == 'Windows' and sys.version_info >= (3, 8):
-        dll_directory = os.path.join(os.path.dirname(__file__), '.libs')
-        if os.path.exists(dll_directory):
-            dll_dirs.append(os.add_dll_directory(dll_directory))
-        else:
-            if 'PATH' in os.environ:
-                for p in os.environ['PATH'].split(os.pathsep):
-                    if glob.glob(os.path.join(p, 'gdal*.dll')):
-                        os.add_dll_directory(p)
-    try:
-        yield None
-    finally:
-        for dll_dir in dll_dirs:
-            dll_dir.close()

rasterio/_path.py

@@ -64,7 +64,7 @@ class _ParsedPath(_Path):
     @classmethod
     def from_uri(cls, uri):
         parts = urlparse(uri)
-        path = parts.path
+        path = pathlib.Path(parts.path).as_posix() if parts.path else parts.path
         scheme = parts.scheme or None
 
         if parts.query:
@@ -76,7 +76,7 @@ class _ParsedPath(_Path):
         parts = path.split('!')
         path = parts.pop() if parts else None
         archive = parts.pop() if parts else None
-        return _ParsedPath(pathlib.Path(path).as_posix(), archive, scheme)
+        return _ParsedPath(path, archive, scheme)
 
     @property
     def name(self):

rasterio/drivers.py

@@ -11,10 +11,8 @@ http://unidata.github.io/netcdf4-python/.
 """
 import os
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    from rasterio._base import _raster_driver_extensions
-    from rasterio.env import GDALVersion, ensure_env
+from rasterio._base import _raster_driver_extensions
+from rasterio.env import GDALVersion, ensure_env
 
 # Methods like `rasterio.open()` may use this blacklist to preempt
 # combinations of drivers and file modes.

rasterio/env.py

@@ -11,15 +11,17 @@ import warnings
 
 import attr
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    from rasterio._env import (
-            GDALEnv, get_gdal_config, set_gdal_config,
-            GDALDataFinder, PROJDataFinder, set_proj_data_search_path)
-    from rasterio._version import gdal_version
-    from rasterio.errors import (
-        EnvError, GDALVersionError, RasterioDeprecationWarning)
-    from rasterio.session import Session, DummySession
+from rasterio._env import (
+    GDALEnv,
+    get_gdal_config,
+    set_gdal_config,
+    GDALDataFinder,
+    PROJDataFinder,
+    set_proj_data_search_path,
+)
+from rasterio._version import gdal_version
+from rasterio.errors import EnvError, GDALVersionError, RasterioDeprecationWarning
+from rasterio.session import Session, DummySession
 
 
 class ThreadEnv(threading.local):

rasterio/features.py

@@ -7,20 +7,22 @@ import warnings
 
 import numpy as np
 
-import rasterio._loading
-
-with rasterio._loading.add_gdal_dll_directories():
-    import rasterio
-    from rasterio.dtypes import validate_dtype, can_cast_dtype, get_minimum_dtype, _getnpdtype
-    from rasterio.enums import MergeAlg
-    from rasterio.env import ensure_env, GDALVersion
-    from rasterio.errors import ShapeSkipWarning
-    from rasterio._features import _shapes, _sieve, _rasterize, _bounds
-    from rasterio import warp
-    from rasterio.rio.helpers import coords
-    from rasterio.transform import Affine
-    from rasterio.transform import IDENTITY, guard_transform, rowcol
-    from rasterio.windows import Window
+import rasterio
+from rasterio.dtypes import (
+    validate_dtype,
+    can_cast_dtype,
+    get_minimum_dtype,
+    _getnpdtype,
+)
+from rasterio.enums import MergeAlg
+from rasterio.env import ensure_env, GDALVersion
+from rasterio.errors import ShapeSkipWarning
+from rasterio._features import _shapes, _sieve, _rasterize, _bounds
+from rasterio import warp
+from rasterio.rio.helpers import coords
+from rasterio.transform import Affine
+from rasterio.transform import IDENTITY, guard_transform, rowcol
+from rasterio.windows import Window
 
 log = logging.getLogger(__name__)
 

rasterio/fill.py

@@ -1,15 +1,12 @@
 """Fill holes in raster dataset by interpolation from the edges."""
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-
-    import rasterio
-    from rasterio._fill import _fillnodata
-    from rasterio.env import ensure_env
-    from rasterio import dtypes
-
 from numpy.ma import MaskedArray
 
+import rasterio
+from rasterio._fill import _fillnodata
+from rasterio.env import ensure_env
+from rasterio import dtypes
+
 
 @ensure_env
 def fillnodata(

rasterio/io.py

@@ -5,21 +5,22 @@ Instances of these classes are called dataset objects.
 
 import logging
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    from rasterio._base import (
-        get_dataset_driver, driver_can_create, driver_can_create_copy)
-    from rasterio._io import (
-        DatasetReaderBase, DatasetWriterBase, BufferedDatasetWriterBase,
-        MemoryFileBase)
-    from rasterio.windows import WindowMethodsMixin
-    from rasterio.env import ensure_env
-    from rasterio.transform import TransformMethodsMixin
-    from rasterio._path import _UnparsedPath
-    try:
-        from rasterio._filepath import FilePathBase
-    except ImportError:
-        FilePathBase = object
+from rasterio._base import get_dataset_driver, driver_can_create, driver_can_create_copy
+from rasterio._io import (
+    DatasetReaderBase,
+    DatasetWriterBase,
+    BufferedDatasetWriterBase,
+    MemoryFileBase,
+)
+from rasterio.windows import WindowMethodsMixin
+from rasterio.env import ensure_env
+from rasterio.transform import TransformMethodsMixin
+from rasterio._path import _UnparsedPath
+
+try:
+    from rasterio._filepath import FilePathBase
+except ImportError:
+    FilePathBase = object
 
 
 log = logging.getLogger(__name__)

rasterio/mask.py

@@ -5,10 +5,8 @@ import warnings
 
 import numpy
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    from rasterio.errors import WindowError
-    from rasterio.features import geometry_mask, geometry_window
+from rasterio.errors import WindowError
+from rasterio.features import geometry_mask, geometry_window
 
 
 logger = logging.getLogger(__name__)

rasterio/merge.py

@@ -9,15 +9,12 @@ import warnings
 
 import numpy as np
 
-import rasterio._loading
-
-with rasterio._loading.add_gdal_dll_directories():
-    import rasterio
-    from rasterio.coords import disjoint_bounds
-    from rasterio.enums import Resampling
-    from rasterio.errors import RasterioDeprecationWarning
-    from rasterio import windows
-    from rasterio.transform import Affine
+import rasterio
+from rasterio.coords import disjoint_bounds
+from rasterio.enums import Resampling
+from rasterio.errors import RasterioDeprecationWarning
+from rasterio import windows
+from rasterio.transform import Affine
 
 logger = logging.getLogger(__name__)
 

rasterio/plot.py

@@ -12,11 +12,9 @@ import warnings
 
 import numpy as np
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    import rasterio
-    from rasterio.io import DatasetReader
-    from rasterio.transform import guard_transform
+import rasterio
+from rasterio.io import DatasetReader
+from rasterio.transform import guard_transform
 
 logger = logging.getLogger(__name__)
 

rasterio/sample.py

@@ -1,13 +1,13 @@
 # Workaround for issue #378. A pure Python generator.
 
+from itertools import zip_longest
+
 import numpy as np
 from itertools import islice
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    from rasterio.enums import MaskFlags
-    from rasterio.windows import Window
-    from rasterio.transform import rowcol
+from rasterio.enums import MaskFlags
+from rasterio.windows import Window
+from rasterio.transform import rowcol
 
 
 def _transform_xy(dataset, xy):

rasterio/session.py

@@ -3,9 +3,7 @@
 import logging
 import os
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    from rasterio._path import _parse_path, _UnparsedPath
+from rasterio._path import _parse_path, _UnparsedPath
 
 
 log = logging.getLogger(__name__)

rasterio/tools.py

@@ -6,10 +6,8 @@ https://github.com/rasterio/rasterio/issues/1300.
 
 import json
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    import rasterio
-    from rasterio.features import dataset_features
+import rasterio
+from rasterio.features import dataset_features
 
 
 class JSONSequenceTool:

rasterio/transform.py

@@ -6,22 +6,21 @@ from functools import partial
 import math
 import numpy as np
 import sys
+import warnings
 
 from affine import Affine
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    import rasterio
-    from rasterio.env import env_ctx_if_needed
-    from rasterio._transform import (
-        _transform_from_gcps,
-        RPCTransformerBase,
-        GCPTransformerBase
-    )
-    from rasterio.enums import TransformDirection, TransformMethod
-    from rasterio.control import GroundControlPoint
-    from rasterio.rpc import RPC
-    from rasterio.errors import TransformError, RasterioDeprecationWarning
+import rasterio
+from rasterio.env import env_ctx_if_needed
+from rasterio._transform import (
+    _transform_from_gcps,
+    RPCTransformerBase,
+    GCPTransformerBase,
+)
+from rasterio.enums import TransformDirection, TransformMethod
+from rasterio.control import GroundControlPoint
+from rasterio.rpc import RPC
+from rasterio.errors import TransformError, RasterioDeprecationWarning
 
 IDENTITY = Affine.identity()
 GDAL_IDENTITY = IDENTITY.to_gdal()

rasterio/vrt.py

@@ -2,15 +2,13 @@
 
 import xml.etree.ElementTree as ET
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    import rasterio
-    from rasterio._warp import WarpedVRTReaderBase
-    from rasterio.dtypes import _gdal_typename
-    from rasterio.enums import MaskFlags
-    from rasterio._path import _parse_path
-    from rasterio.transform import TransformMethodsMixin
-    from rasterio.windows import WindowMethodsMixin
+import rasterio
+from rasterio._warp import WarpedVRTReaderBase
+from rasterio.dtypes import _gdal_typename
+from rasterio.enums import MaskFlags
+from rasterio._path import _parse_path
+from rasterio.transform import TransformMethodsMixin
+from rasterio.windows import WindowMethodsMixin
 
 
 class WarpedVRT(WarpedVRTReaderBase, WindowMethodsMixin,

rasterio/warp.py

@@ -5,23 +5,21 @@ from math import ceil, floor
 from affine import Affine
 import numpy as np
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    import rasterio
+import rasterio
 
-    from rasterio._base import _transform
-    from rasterio.crs import CRS
-    from rasterio.enums import Resampling
-    from rasterio.env import ensure_env, require_gdal_version
-    from rasterio.errors import TransformError, RPCError
-    from rasterio.transform import array_bounds
-    from rasterio._warp import (
-        _calculate_default_transform,
-        _reproject,
-        _transform_bounds,
-        _transform_geom,
-        SUPPORTED_RESAMPLING
-    )
+from rasterio._base import _transform
+from rasterio.crs import CRS
+from rasterio.enums import Resampling
+from rasterio.env import ensure_env, require_gdal_version
+from rasterio.errors import TransformError, RPCError
+from rasterio.transform import array_bounds
+from rasterio._warp import (
+    _calculate_default_transform,
+    _reproject,
+    _transform_bounds,
+    _transform_geom,
+    SUPPORTED_RESAMPLING,
+)
 
 @ensure_env
 def transform(src_crs, dst_crs, xs, ys, zs=None):

rasterio/windows.py

@@ -27,10 +27,8 @@ from affine import Affine
 import attr
 import numpy as np
 
-import rasterio._loading
-with rasterio._loading.add_gdal_dll_directories():
-    from rasterio.errors import WindowError, RasterioDeprecationWarning
-    from rasterio.transform import rowcol, guard_transform
+from rasterio.errors import WindowError, RasterioDeprecationWarning
+from rasterio.transform import rowcol, guard_transform
 
 
 class WindowMethodsMixin:

tests/test__env.py

@@ -43,6 +43,7 @@ def mock_debian(tmpdir):
     tmpdir.ensure("share/gdal/3.3/header.dxf")
     tmpdir.ensure("share/gdal/3.4/header.dxf")
     tmpdir.ensure("share/gdal/3.5/header.dxf")
+    tmpdir.ensure("share/gdal/3.6/header.dxf")
     tmpdir.ensure("share/proj/epsg")
     return tmpdir
 

tests/test_dataset.py

@@ -80,3 +80,25 @@ def test_statistics(path_rgb_byte_tif):
         assert stats.max == 255
         assert_almost_equal(stats.mean, 29.947726688477)
         assert_almost_equal(stats.std, 52.340921626611)
+
+
+@pytest.mark.parametrize("blockysize", [1, 2, 3, 7, 61, 62])
+def test_creation_untiled_blockysize(tmp_path, blockysize):
+    """Check for fix of gh-2599"""
+    tmpfile = tmp_path / "test.tif"
+    with rasterio.open(
+        tmpfile,
+        "w",
+        count=1,
+        height=61,
+        width=37,
+        dtype="uint8",
+        blockysize=blockysize,
+        tiled=False,
+    ) as dataset:
+        pass
+
+    with rasterio.open(tmpfile) as dataset:
+        assert not dataset.is_tiled
+        assert dataset.profile["blockysize"] == min(blockysize, 61)
+        assert dataset.block_shapes[0][0] == min(blockysize, 61)

tests/test_filepath.py

@@ -159,3 +159,26 @@ def test_concurrent(path_rgb_byte_tif, path_rgb_lzw_byte_tif, path_cogeo_tif, pa
     tifs = [path_rgb_byte_tif, path_rgb_lzw_byte_tif, path_cogeo_tif, path_alpha_tif] * 4
     with ThreadPoolExecutor(max_workers=8) as exe:
         list(exe.map(_open_geotiff, tifs, timeout=5))
+
+
+def test_python_file_reuse():
+    """Test that we can reuse a Python file, see gh-2550."""
+    ascii_raster_string = """ncols        5
+    nrows        5
+    xllcorner    440720.000000000000
+    yllcorner    3750120.000000000000
+    cellsize     60.000000000000
+    nodata_value -99999
+        107    123    132    115    132
+        115    132    107    123    148
+        115    132    140    132    123
+        148    132    123    123    115
+        132    156    132    140    132
+    """
+    ascii_raster_io = BytesIO(ascii_raster_string.encode("utf-8"))
+
+    with rasterio.open(ascii_raster_io) as rds:
+        _ = rds.bounds
+
+    with rasterio.open(ascii_raster_io) as rds:
+        _ = rds.bounds

tests/test_path.py

@@ -99,6 +99,14 @@ def test_parse_gdal():
     assert _parse_path('GDAL:filepath:varname').path == 'GDAL:filepath:varname'
 
 
+def test_parse_http_password():
+    """Make sure password unmodified GH2602"""
+    parsed = _parse_path('https://foo.tif?bar=a//b')
+    assert parsed.path == 'foo.tif?bar=a//b'
+    assert parsed.archive is None
+    assert parsed.scheme == 'https'
+
+
 @pytest.mark.skipif(sys.platform == 'win32', reason="Checking behavior on posix, not win32")
 def test_parse_windows_path(monkeypatch):
     """Return Windows paths unparsed"""

tests/test_profile.py

@@ -80,10 +80,10 @@ def test_dataset_profile_property_tiled(data):
 
 
 def test_dataset_profile_property_untiled(data, path_rgb_byte_tif):
-    """An untiled dataset's profile has no block sizes"""
+    """An untiled dataset's profile has block y sizes"""
     with rasterio.open(path_rgb_byte_tif) as src:
         assert 'blockxsize' not in src.profile
-        assert 'blockysize' not in src.profile
+        assert src.profile['blockysize'] == 3
         assert src.profile['tiled'] is False
 
 

tests/test_warp.py

@@ -2125,7 +2125,7 @@ def test_coordinate_pipeline(tmp_path):
 
 
 @pytest.mark.skipif(
-    not gdal_version.at_least('3.4'),
+    not gdal_version.at_least('3.4') or gdal_version.at_least("3.5"),
     reason="Requires GDAL 3.4.x")
 def test_issue2353bis(caplog):
     """Errors left by a successful transformation are cleaned up."""