From 53a6a0607499a00ce108ac585cb60d33b10b7a1e Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Wed, 28 Apr 2021 11:25:41 -0600 Subject: [PATCH 01/18] Explicitly require setuptools --- CHANGES.txt | 5 +++++ setup.py | 1 + 2 files changed, 6 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 5ad411d7..72ecf2eb 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,11 @@ Changes ======= +1.2.4 (TBD) +----------- + +- Setuptools has been added as an explicit installation requirement. + 1.2.3 (2021-04-26) ------------------ diff --git a/setup.py b/setup.py index 60407709..c7711522 100755 --- a/setup.py +++ b/setup.py @@ -339,6 +339,7 @@ inst_reqs = [ "numpy", "snuggs>=1.4.1", "click-plugins", + "setuptools", ] extra_reqs = { From 0204f1fb83b535c369be8962eb069fd4efe12cec Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Mon, 3 May 2021 13:44:05 -0500 Subject: [PATCH 02/18] Resolve numpy deprecation warnings in tests (#2170) * Fix np.bool deprecation warning * Fix np.int deprecation warning. * Remove extra space. --- rasterio/_io.pyx | 2 +- tests/test_dtypes.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rasterio/_io.pyx b/rasterio/_io.pyx index b184dcb1..b91dde69 100644 --- a/rasterio/_io.pyx +++ b/rasterio/_io.pyx @@ -1573,7 +1573,7 @@ cdef class DatasetWriterBase(DatasetReaderBase): GDALFillRaster(mask, 255, 0) elif mask_array is False: GDALFillRaster(mask, 0, 0) - elif mask_array.dtype == np.bool: + elif mask_array.dtype == bool: array = 255 * mask_array.astype(np.uint8) io_band(mask, 1, xoff, yoff, width, height, array) else: diff --git a/tests/test_dtypes.py b/tests/test_dtypes.py index e954f536..875cf8b4 100644 --- a/tests/test_dtypes.py +++ b/tests/test_dtypes.py @@ -46,8 +46,8 @@ def test_get_minimum_dtype(): assert get_minimum_dtype(np.array([0, 1], dtype=np.uint)) == uint8 assert get_minimum_dtype(np.array([0, 1000], dtype=np.uint)) == uint16 assert get_minimum_dtype(np.array([0, 100000], dtype=np.uint)) == uint32 - assert get_minimum_dtype(np.array([-1, 0, 1], dtype=np.int)) == int16 - assert get_minimum_dtype(np.array([-1, 0, 100000], dtype=np.int)) == int32 + assert get_minimum_dtype(np.array([-1, 0, 1], dtype=int)) == int16 + assert get_minimum_dtype(np.array([-1, 0, 100000], dtype=int)) == int32 assert get_minimum_dtype(np.array([-1.5, 0, 1.5], dtype=np.float64)) == float32 From 603908e0492e0f61334dcfd64965740981a86e55 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 4 May 2021 11:04:35 -0500 Subject: [PATCH 03/18] Add docstring to BoundingBox and remove subclass. (#2172) * Add docstring to BoundingBox and remove subclass. * Remove unused import. --- rasterio/coords.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/rasterio/coords.py b/rasterio/coords.py index 1931cccc..b466901d 100644 --- a/rasterio/coords.py +++ b/rasterio/coords.py @@ -1,11 +1,9 @@ """Bounding box tuple, and disjoint operator.""" -from collections import namedtuple, OrderedDict +from collections import namedtuple -_BoundingBox = namedtuple('BoundingBox', ('left', 'bottom', 'right', 'top')) - - -class BoundingBox(_BoundingBox): +BoundingBox = namedtuple('BoundingBox', ('left', 'bottom', 'right', 'top')) +BoundingBox.__doc__ = \ """Bounding box named tuple, defining extent in cartesian coordinates. .. code:: @@ -24,10 +22,6 @@ class BoundingBox(_BoundingBox): Top coordinate """ - def _asdict(self): - return OrderedDict(zip(self._fields, self)) - - def disjoint_bounds(bounds1, bounds2): """Compare two bounds and determine if they are disjoint. From fa7153dfa5dd46b7d2eac280b33997115b946aa9 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Tue, 4 May 2021 10:06:27 -0600 Subject: [PATCH 04/18] Update change log --- CHANGES.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 72ecf2eb..fb8c9c3e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,7 +4,9 @@ Changes 1.2.4 (TBD) ----------- -- Setuptools has been added as an explicit installation requirement. +- Remove a workaround for an old Python 3.4 bug from the BoundingBox + implementation (#2172). +- Add setuptools as an explicit installation requirement. 1.2.3 (2021-04-26) ------------------ From df446d4cc8568f384977ba7b7823f7ef0a1332ba Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Thu, 20 May 2021 14:54:59 -0600 Subject: [PATCH 05/18] Check signed byte values read --- tests/test_write.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_write.py b/tests/test_write.py index eb78ab2a..4edcaa20 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -107,8 +107,12 @@ def test_write_sbyte(tmpdir): with rasterio.open( name, 'w', driver='GTiff', width=100, height=100, count=1, - dtype=a.dtype) as s: - s.write(a, indexes=1) + dtype=a.dtype) as dst: + dst.write(a, indexes=1) + + with rasterio.open(name) as dst: + assert (dst.read() == -33).all() + info = subprocess.check_output(["gdalinfo", "-stats", name]).decode('utf-8') assert "Minimum=-33.000, Maximum=-33.000, Mean=-33.000, StdDev=0.000" in info assert 'SIGNEDBYTE' in info From 60d4ed4114a01e778b70b8390fe426ac399e2f14 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Thu, 20 May 2021 15:11:20 -0600 Subject: [PATCH 06/18] Allow unsafe casting again Resolves #2179 --- rasterio/merge.py | 8 ++++---- tests/test_merge.py | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/rasterio/merge.py b/rasterio/merge.py index e14fa190..a175f822 100644 --- a/rasterio/merge.py +++ b/rasterio/merge.py @@ -21,13 +21,13 @@ def copy_first(merged_data, new_data, merged_mask, new_mask, **kwargs): mask = np.empty_like(merged_mask, dtype="bool") np.logical_not(new_mask, out=mask) np.logical_and(merged_mask, mask, out=mask) - np.copyto(merged_data, new_data, where=mask) + np.copyto(merged_data, new_data, where=mask, casting="unsafe") def copy_last(merged_data, new_data, merged_mask, new_mask, **kwargs): mask = np.empty_like(merged_mask, dtype="bool") np.logical_not(new_mask, out=mask) - np.copyto(merged_data, new_data, where=mask) + np.copyto(merged_data, new_data, where=mask, casting="unsafe") def copy_min(merged_data, new_data, merged_mask, new_mask, **kwargs): @@ -37,7 +37,7 @@ def copy_min(merged_data, new_data, merged_mask, new_mask, **kwargs): np.minimum(merged_data, new_data, out=merged_data, where=mask) np.logical_not(new_mask, out=mask) np.logical_and(merged_mask, mask, out=mask) - np.copyto(merged_data, new_data, where=mask) + np.copyto(merged_data, new_data, where=mask, casting="unsafe") def copy_max(merged_data, new_data, merged_mask, new_mask, **kwargs): @@ -47,7 +47,7 @@ def copy_max(merged_data, new_data, merged_mask, new_mask, **kwargs): np.maximum(merged_data, new_data, out=merged_data, where=mask) np.logical_not(new_mask, out=mask) np.logical_and(merged_mask, mask, out=mask) - np.copyto(merged_data, new_data, where=mask) + np.copyto(merged_data, new_data, where=mask, casting="unsafe") MERGE_METHODS = { diff --git a/tests/test_merge.py b/tests/test_merge.py index 7bc79139..f75dd3b1 100644 --- a/tests/test_merge.py +++ b/tests/test_merge.py @@ -59,3 +59,10 @@ def test_issue2163(): data = src.read() result, transform = merge([src]) assert numpy.allclose(data, result) + + +def test_unsafe_casting(): + """Demonstrate fix for issue 2179""" + with rasterio.open("tests/data/float_raster_with_nodata.tif") as src: + result, transform = merge([src], dtype="uint8") + assert not result.any() # this is why it's called "unsafe". From 603c186469b23ed1ebddedf4ee79d372f826d7f6 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Thu, 20 May 2021 15:15:20 -0600 Subject: [PATCH 07/18] Note fix of #2179 --- CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index fb8c9c3e..80687723 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,8 @@ Changes 1.2.4 (TBD) ----------- +- The merge tool allows unsafe casting again, restoring behavior of version + 1.2.0 (#2179). - Remove a workaround for an old Python 3.4 bug from the BoundingBox implementation (#2172). - Add setuptools as an explicit installation requirement. From 72a894589b1dd12c322da48b0e5fdcfe3a0cd1d1 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Thu, 20 May 2021 16:16:08 -0600 Subject: [PATCH 08/18] Skip merge sources that don't overlap the destination --- CHANGES.txt | 5 +++-- rasterio/coords.py | 1 + rasterio/merge.py | 6 +++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 80687723..b9c350d0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,8 +4,9 @@ Changes 1.2.4 (TBD) ----------- -- The merge tool allows unsafe casting again, restoring behavior of version - 1.2.0 (#2179). +- Skip merge sources which do not overlap the destination. +- Allow unsafe casting in the merge tool, restoring behavior of version 1.2.0 + (#2179). - Remove a workaround for an old Python 3.4 bug from the BoundingBox implementation (#2172). - Add setuptools as an explicit installation requirement. diff --git a/rasterio/coords.py b/rasterio/coords.py index b466901d..cef25906 100644 --- a/rasterio/coords.py +++ b/rasterio/coords.py @@ -37,6 +37,7 @@ def disjoint_bounds(bounds1, bounds2): boolean ``True`` if bounds are disjoint, ``False`` if bounds overlap + """ bounds1_north_up = bounds1[3] > bounds1[1] bounds2_north_up = bounds2[3] > bounds2[1] diff --git a/rasterio/merge.py b/rasterio/merge.py index a175f822..95d3249a 100644 --- a/rasterio/merge.py +++ b/rasterio/merge.py @@ -9,6 +9,7 @@ import warnings import numpy as np import rasterio +from rasterio.coords import disjoint_bounds from rasterio.enums import Resampling from rasterio import windows from rasterio.transform import Affine @@ -290,6 +291,10 @@ def merge( # This approach uses the maximum amount of memory to solve the # problem. Making it more efficient is a TODO. + if disjoint_bounds((dst_w, dst_s, dst_e, dst_n), src.bounds): + logger.debug("Skipping source: src=%r, window=%r", src) + continue + # 1. Compute spatial intersection of destination and source src_w, src_s, src_e, src_n = src.bounds @@ -302,7 +307,6 @@ def merge( src_window = windows.from_bounds( int_w, int_s, int_e, int_n, src.transform, precision=precision ) - logger.debug("Src %s window: %r", src.name, src_window) # 3. Compute the destination window dst_window = windows.from_bounds( From cc14a624067915665a4201608fa4b1b1760835ef Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Tue, 25 May 2021 16:55:39 -0600 Subject: [PATCH 09/18] Test against PROJ 8 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4eecae4e..649f6a74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,11 +48,11 @@ jobs: - python: "3.8" env: GDALVERSION="master" - PROJVERSION="7.2.1" + PROJVERSION="8.0.1" allow_failures: - env: GDALVERSION="master" - PROJVERSION="7.2.1" + PROJVERSION="8.0.1" addons: apt: From 9f6d98ffcc23775deb3ff17417cae8539ea51e2b Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Tue, 25 May 2021 21:08:24 -0600 Subject: [PATCH 10/18] Mark aligned pixels test xfail PROJ 8 has new mercator projection code Also avoid a warning from merge() --- tests/test_merge.py | 2 +- tests/test_warp.py | 73 +++++++++++++++++---------------------------- 2 files changed, 28 insertions(+), 47 deletions(-) diff --git a/tests/test_merge.py b/tests/test_merge.py index f75dd3b1..e43ef908 100644 --- a/tests/test_merge.py +++ b/tests/test_merge.py @@ -64,5 +64,5 @@ def test_issue2163(): def test_unsafe_casting(): """Demonstrate fix for issue 2179""" with rasterio.open("tests/data/float_raster_with_nodata.tif") as src: - result, transform = merge([src], dtype="uint8") + result, transform = merge([src], dtype="uint8", nodata=0.0) assert not result.any() # this is why it's called "unsafe". diff --git a/tests/test_warp.py b/tests/test_warp.py index baa7574d..d00e4816 100644 --- a/tests/test_warp.py +++ b/tests/test_warp.py @@ -1,7 +1,6 @@ """rasterio.warp module tests""" import json -import sys import logging from affine import Affine @@ -15,7 +14,6 @@ from rasterio.crs import CRS from rasterio.enums import Resampling from rasterio.env import GDALVersion from rasterio.errors import ( - GDALBehaviorChangeException, CRSError, GDALVersionError, TransformError, @@ -1136,6 +1134,7 @@ def test_reproject_resampling(path_rgb_byte_tif, method): assert np.count_nonzero(out) in expected[method] + @pytest.mark.parametrize("test3d,count_nonzero", [(True, 1309625), (False, 437686)]) def test_reproject_array_interface(test3d, count_nonzero, path_rgb_byte_tif): class DataArray: @@ -1333,13 +1332,14 @@ def test_resample_default_invert_proj(method): assert out.mean() > 0 +@pytest.mark.xfail(reason="Projection extents have changed with PROJ 8") def test_target_aligned_pixels(): """Issue 853 has been resolved""" with rasterio.open("tests/data/world.rgb.tif") as src: source = src.read(1) - profile = src.profile.copy() + profile = src.profile - dst_crs = "EPSG:3857" + dst_crs = CRS.from_proj4("+proj=cea") with rasterio.Env(CHECK_WITH_INVERT_PROJ=False): # Calculate the ideal dimensions and transformation in the new crs @@ -1348,7 +1348,7 @@ def test_target_aligned_pixels(): ) dst_affine, dst_width, dst_height = aligned_target( - dst_affine, dst_width, dst_height, 100000.0 + dst_affine, dst_width, dst_height, 5000.0 ) profile["height"] = dst_height @@ -1366,7 +1366,7 @@ def test_target_aligned_pixels(): resampling=Resampling.nearest, ) - # Check that there is no black borders + # Check that there are no black borders assert out[:, 0].all() assert out[:, -1].all() assert out[0, :].all() @@ -1694,26 +1694,6 @@ def test_issue_1446_b(): assert all([-350 < x < -150 for x, y in transformed_geoms[183519]["coordinates"]]) -def test_issue_1076(): - """Confirm fix of #1076""" - arr = (np.random.random((20, 30)) * 100).astype('int32') - fill_value = 42 - newarr = np.full((200, 300), fill_value=fill_value, dtype='int32') - - src_crs = CRS.from_epsg(32632) - src_transform = Affine(600.0, 0.0, 399960.0, 0.0, -600.0, 6100020.0) - dst_transform = Affine(60.0, 0.0, 399960.0, 0.0, -60.0, 6100020.0) - - reproject(arr, newarr, - src_transform=src_transform, - dst_transform=dst_transform, - src_crs=src_crs, - dst_crs=src_crs, - resample=Resampling.nearest) - - assert not (newarr == fill_value).all() - - def test_reproject_init_dest_nodata(): """No pixels should transfer over""" crs = CRS.from_epsg(4326) @@ -1785,16 +1765,16 @@ def test_reproject_rpcs_with_transformer_options(caplog): transform = Affine(0.001953396267361111, 0.0, -124.00013888888888, 0.0, -0.001953396267361111, 50.000138888888884) with mem.open( driver="GTiff", - width=1024, - height=1024, + width=1024, + height=1024, count=1, transform=transform, - dtype='int16', - crs=crs + dtype="int16", + crs=crs, ) as dem: # we flush dem dataset before letting GDAL read from vsimem dem.write_band(1, 500 * np.ones((1024, 1024), dtype='int16')) - + out = np.zeros( (3, src.profile["width"], src.profile["height"]), dtype=np.uint8 ) @@ -1810,7 +1790,7 @@ def test_reproject_rpcs_with_transformer_options(caplog): resampling=Resampling.nearest, RPC_DEM=dem.name, - ) + ) caplog.set_level(logging.INFO) reproject( rasterio.band(src, src.indexes), @@ -1820,8 +1800,8 @@ def test_reproject_rpcs_with_transformer_options(caplog): dst_crs="EPSG:3857", resampling=Resampling.nearest, - ) - + ) + assert not out.all() assert not out2.all() assert "RPC_DEM" in caplog.text @@ -1853,10 +1833,10 @@ def test_warp_gcps_compute_dst_transform_automatically_array(): assert not out[:, -1, -1].any() assert not out[:, -1, 0].any() + def test_warp_gcps_compute_dst_transform_automatically_reader(tmpdir): """Ensure we don't raise an exception when gcps passed without dst_transform, for a source dataset""" tiffname = str(tmpdir.join('test.tif')) - arr = np.ones((3, 800, 800), dtype=np.uint8) * 255 src_gcps = [ GroundControlPoint(row=0, col=0, x=156113, y=2818720, z=0), GroundControlPoint(row=0, col=800, x=338353, y=2785790, z=0), @@ -1867,7 +1847,7 @@ def test_warp_gcps_compute_dst_transform_automatically_reader(tmpdir): with rasterio.open(tiffname, mode='w', height=800, width=800, count=3, dtype=np.uint8) as source: source.gcps = (src_gcps, CRS.from_epsg(32618)) - + with rasterio.open(tiffname) as source: reproject( rasterio.band(source, source.indexes), @@ -1875,13 +1855,14 @@ def test_warp_gcps_compute_dst_transform_automatically_reader(tmpdir): dst_crs="EPSG:32618", resampling=Resampling.nearest ) - + assert not out.all() assert not out[:, 0, 0].any() assert not out[:, 0, -1].any() assert not out[:, -1, -1].any() assert not out[:, -1, 0].any() + def test_reproject_rpcs_exact_transformer(caplog): """Reproject using rational polynomial coefficients and DEM, requiring that we don't try to make an approximate transformer. @@ -1892,16 +1873,16 @@ def test_reproject_rpcs_exact_transformer(caplog): transform = Affine(0.001953396267361111, 0.0, -124.00013888888888, 0.0, -0.001953396267361111, 50.000138888888884) with mem.open( driver="GTiff", - width=1024, - height=1024, + width=1024, + height=1024, count=1, transform=transform, - dtype='int16', - crs=crs + dtype="int16", + crs=crs, ) as dem: # we flush dem dataset before letting GDAL read from vsimem dem.write_band(1, 500 * np.ones((1024, 1024), dtype='int16')) - + out = np.zeros( (3, src.profile["width"], src.profile["height"]), dtype=np.uint8 ) @@ -1915,10 +1896,10 @@ def test_reproject_rpcs_exact_transformer(caplog): dst_crs="EPSG:3857", resampling=Resampling.nearest, RPC_DEM=dem.name, - ) + ) assert "Created exact transformer" in caplog.text - + def test_reproject_rpcs_approx_transformer(caplog): """Reproject using rational polynomial coefficients without a DEM, for which it's @@ -1937,6 +1918,6 @@ def test_reproject_rpcs_approx_transformer(caplog): rpcs=src_rpcs, dst_crs="EPSG:3857", resampling=Resampling.nearest, - ) + ) - assert "Created approximate transformer" in caplog.text \ No newline at end of file + assert "Created approximate transformer" in caplog.text From 124752017fee6193650e329b6e5549e1c37e7b75 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Tue, 25 May 2021 21:21:34 -0600 Subject: [PATCH 11/18] Revert to EPSG:3857 --- tests/test_warp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_warp.py b/tests/test_warp.py index d00e4816..0fb9919a 100644 --- a/tests/test_warp.py +++ b/tests/test_warp.py @@ -1339,7 +1339,7 @@ def test_target_aligned_pixels(): source = src.read(1) profile = src.profile - dst_crs = CRS.from_proj4("+proj=cea") + dst_crs = "EPSG:3857" with rasterio.Env(CHECK_WITH_INVERT_PROJ=False): # Calculate the ideal dimensions and transformation in the new crs @@ -1348,7 +1348,7 @@ def test_target_aligned_pixels(): ) dst_affine, dst_width, dst_height = aligned_target( - dst_affine, dst_width, dst_height, 5000.0 + dst_affine, dst_width, dst_height, 10000.0 ) profile["height"] = dst_height From 7114fb7fb5e48146c4f04eb818daaa1fa632d817 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Tue, 25 May 2021 21:41:40 -0600 Subject: [PATCH 12/18] Save GDAL CInt16 data (#2185) * Save GDAL CInt16 data * _getnpdtype translates "complex_int16" to complex64. --- CHANGES.txt | 2 ++ rasterio/__init__.py | 17 +++++++++-- rasterio/_base.pyx | 5 +++- rasterio/_features.pyx | 13 ++++---- rasterio/_io.pyx | 57 +++++++++++++++++++++--------------- rasterio/dtypes.py | 53 ++++++++++++++++++++------------- rasterio/features.py | 6 ++-- tests/test_complex_dtypes.py | 50 +++++++++++++++++++++++++++---- tests/test_dtypes.py | 53 ++++++++++++++++++++++++++++----- 9 files changed, 186 insertions(+), 70 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index b9c350d0..d1da4e1c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,8 @@ Changes 1.2.4 (TBD) ----------- +- Read GDAL CInt16 data as np.complex64 and allow saving complex data to CInt16 + (#2185). - Skip merge sources which do not overlap the destination. - Allow unsafe casting in the merge tool, restoring behavior of version 1.2.0 (#2179). diff --git a/rasterio/__init__.py b/rasterio/__init__.py index 996c3bb4..3ea187f8 100644 --- a/rasterio/__init__.py +++ b/rasterio/__init__.py @@ -9,8 +9,21 @@ from pathlib import Path from rasterio._base import gdal_version from rasterio.drivers import driver_from_extension, is_blacklisted from rasterio.dtypes import ( - bool_, ubyte, sbyte, uint8, int8, uint16, int16, uint32, int32, float32, float64, - complex_, check_dtype) + bool_, + ubyte, + sbyte, + uint8, + int8, + uint16, + int16, + uint32, + int32, + float32, + float64, + complex_, + complex_int16, + check_dtype, +) from rasterio.env import ensure_env_with_credentials, Env from rasterio.errors import RasterioIOError, DriverCapabilityError from rasterio.io import ( diff --git a/rasterio/_base.pyx b/rasterio/_base.pyx index af5be1f0..1f44a2af 100644 --- a/rasterio/_base.pyx +++ b/rasterio/_base.pyx @@ -172,6 +172,7 @@ cdef _band_dtype(GDALRasterBandH band): else: return 'uint8' + return dtypes.dtype_fwd[gdal_dtype] @@ -501,7 +502,9 @@ cdef class DatasetBase(object): # to check that the return value is within the range of the # data type. If so, the band has a nodata value. If not, # there's no nodata value. - if (success == 0 or + if dtype not in dtypes.dtype_ranges: + pass + elif (success == 0 or val < dtypes.dtype_ranges[dtype][0] or val > dtypes.dtype_ranges[dtype][1]): val = None diff --git a/rasterio/_features.pyx b/rasterio/_features.pyx index 8cd5013a..8f6f50e7 100644 --- a/rasterio/_features.pyx +++ b/rasterio/_features.pyx @@ -9,6 +9,7 @@ import logging import numpy as np from rasterio import dtypes +from rasterio.dtypes import _getnpdtype from rasterio.enums import MergeAlg from rasterio._err cimport exc_wrap_int, exc_wrap_pointer @@ -62,12 +63,12 @@ def _shapes(image, mask, connectivity, transform): cdef ShapeIterator shape_iter = None cdef int fieldtp - is_float = np.dtype(image.dtype).kind == "f" + is_float = _getnpdtype(image.dtype).kind == "f" fieldtp = 2 if is_float else 0 valid_dtypes = ('int16', 'int32', 'uint8', 'uint16', 'float32') - if np.dtype(image.dtype).name not in valid_dtypes: + if _getnpdtype(image.dtype).name not in valid_dtypes: raise ValueError("image dtype must be one of: {0}".format( ', '.join(valid_dtypes))) @@ -89,7 +90,7 @@ def _shapes(image, mask, connectivity, transform): if mask.shape != image.shape: raise ValueError("Mask must have same shape as image") - if np.dtype(mask.dtype).name not in ('bool', 'uint8'): + if _getnpdtype(mask.dtype).name not in ('bool', 'uint8'): raise ValueError("Mask must be dtype rasterio.bool_ or " "rasterio.uint8") @@ -184,7 +185,7 @@ def _sieve(image, size, out, mask, connectivity): valid_dtypes = ('int16', 'int32', 'uint8', 'uint16') - if np.dtype(image.dtype).name not in valid_dtypes: + if _getnpdtype(image.dtype).name not in valid_dtypes: valid_types_str = ', '.join(('rasterio.{0}'.format(t) for t in valid_dtypes)) raise ValueError( @@ -203,7 +204,7 @@ def _sieve(image, size, out, mask, connectivity): if out.shape != image.shape: raise ValueError('out raster shape must be same as image shape') - if np.dtype(image.dtype).name != np.dtype(out.dtype).name: + if _getnpdtype(image.dtype).name != _getnpdtype(out.dtype).name: raise ValueError('out raster must match dtype of image') try: @@ -231,7 +232,7 @@ def _sieve(image, size, out, mask, connectivity): if mask.shape != image.shape: raise ValueError("Mask must have same shape as image") - if np.dtype(mask.dtype) not in ('bool', 'uint8'): + if _getnpdtype(mask.dtype) not in ('bool', 'uint8'): raise ValueError("Mask must be dtype rasterio.bool_ or " "rasterio.uint8") diff --git a/rasterio/_io.pyx b/rasterio/_io.pyx index b91dde69..b921723e 100644 --- a/rasterio/_io.pyx +++ b/rasterio/_io.pyx @@ -26,7 +26,7 @@ from rasterio.errors import ( NotGeoreferencedWarning, NodataShadowWarning, WindowError, UnsupportedOperation, OverviewCreationError, RasterBlockError, InvalidArrayError ) -from rasterio.dtypes import is_ndarray +from rasterio.dtypes import is_ndarray, _is_complex_int, _getnpdtype from rasterio.sample import sample_gen from rasterio.transform import Affine from rasterio.path import parse_path, UnparsedPath @@ -104,7 +104,7 @@ cdef bint in_dtype_range(value, dtype): 105: np.iinfo, 117: np.iinfo} - key = np.dtype(dtype).kind + key = _getnpdtype(dtype).kind if np.isnan(value): return key in ('c', 'f', 99, 102) @@ -241,12 +241,14 @@ cdef class DatasetReaderBase(DatasetBase): check_dtypes = set() nodatavals = [] + # Check each index before processing 3D array for bidx in indexes: + if bidx not in self.indexes: raise IndexError("band index {} out of range (not in {})".format(bidx, self.indexes)) - idx = self.indexes.index(bidx) + idx = self.indexes.index(bidx) dtype = self.dtypes[idx] check_dtypes.add(dtype) @@ -254,16 +256,18 @@ cdef class DatasetReaderBase(DatasetBase): log.debug("Output nodata value read from file: %r", ndv) - if ndv is not None: - kind = np.dtype(dtype).kind - if chr(kind) in "iu": + if ndv is not None and not _is_complex_int(dtype): + kind = _getnpdtype(dtype).kind + + if kind in "iu": info = np.iinfo(dtype) dt_min, dt_max = info.min, info.max - elif chr(kind) in "cf": + elif kind in "cf": info = np.finfo(dtype) dt_min, dt_max = info.min, info.max else: dt_min, dt_max = False, True + if ndv < dt_min: ndv = dt_min elif ndv > dt_max: @@ -284,6 +288,9 @@ cdef class DatasetReaderBase(DatasetBase): if out_dtype is not None: dtype = out_dtype + # Ensure we have a numpy dtype. + dtype = _getnpdtype(dtype) + # Get the natural shape of the read window, boundless or not. # The window can have float values. In this case, we round up # when computing the shape. @@ -555,7 +562,7 @@ cdef class DatasetReaderBase(DatasetBase): out = np.zeros(out_shape, 'uint8') if out is not None: - if out.dtype != np.dtype(dtype): + if out.dtype != _getnpdtype(dtype): raise ValueError( "the out array's dtype '%s' does not match '%s'" % (out.dtype, dtype)) @@ -1056,19 +1063,21 @@ cdef class DatasetWriterBase(DatasetReaderBase): try: width = int(width) height = int(height) - except: + except TypeError: raise TypeError("Integer width and height are required.") try: count = int(count) - except: + except TypeError: raise TypeError("Integer band count is required.") - try: - assert dtype is not None - _ = np.dtype(dtype) - except: - raise TypeError("A valid dtype is required.") - self._init_dtype = np.dtype(dtype).name + if _is_complex_int(dtype): + self._init_dtype = dtype + else: + try: + assert dtype is not None + self._init_dtype = _getnpdtype(dtype).name + except Exception: + raise TypeError("A valid dtype is required.") # Make and store a GDAL dataset handle. filename = path.name @@ -1147,7 +1156,9 @@ cdef class DatasetWriterBase(DatasetReaderBase): if nodata is not None: - if not in_dtype_range(nodata, dtype): + if _is_complex_int(dtype): + pass + elif not in_dtype_range(nodata, dtype): raise ValueError( "Given nodata value, %s, is beyond the valid " "range of its data type, %s." % ( @@ -1365,10 +1376,7 @@ cdef class DatasetWriterBase(DatasetReaderBase): else: # unique dtype; normal case dtype = check_dtypes.pop() - if arr is not None and arr.dtype != dtype: - raise ValueError( - "the array's dtype '%s' does not match " - "the file's dtype '%s'" % (arr.dtype, dtype)) + dtype = _getnpdtype(dtype) # Require C-continguous arrays (see #108). arr = np.require(arr, dtype=dtype, requirements='C') @@ -1968,13 +1976,14 @@ cdef class BufferedDatasetWriterBase(DatasetWriterBase): count = int(count) except: raise TypeError("Integer band count is required.") + try: assert dtype is not None - _ = np.dtype(dtype) - except: + _ = _getnpdtype(dtype) + except Exception: raise TypeError("A valid dtype is required.") - self._init_dtype = np.dtype(dtype).name + self._init_dtype = _getnpdtype(dtype).name self.name = path.name self.mode = mode diff --git a/rasterio/dtypes.py b/rasterio/dtypes.py index 27615566..c73736f3 100644 --- a/rasterio/dtypes.py +++ b/rasterio/dtypes.py @@ -4,10 +4,6 @@ Since 0.13 we are not importing numpy here and data types are strings. Happily strings can be used throughout Numpy and so existing code will not break. -Within Rasterio, to test data types, we use Numpy's dtype() factory to -do something like this: - - if np.dtype(destination.dtype) == np.dtype(rasterio.uint8): ... """ bool_ = 'bool' @@ -23,26 +19,29 @@ complex_ = 'complex' complex64 = 'complex64' complex128 = 'complex128' -# Not supported: -# GDT_CInt16 = 8, GDT_CInt32 = 9, GDT_CFloat32 = 10, GDT_CFloat64 = 11 +complex_int16 = "complex_int16" dtype_fwd = { - 0: None, # GDT_Unknown - 1: ubyte, # GDT_Byte - 2: uint16, # GDT_UInt16 - 3: int16, # GDT_Int16 - 4: uint32, # GDT_UInt32 - 5: int32, # GDT_Int32 - 6: float32, # GDT_Float32 - 7: float64, # GDT_Float64 - 8: complex_, # GDT_CInt16 - 9: complex_, # GDT_CInt32 - 10: complex64, # GDT_CFloat32 - 11: complex128} # GDT_CFloat64 + 0: None, # GDT_Unknown + 1: ubyte, # GDT_Byte + 2: uint16, # GDT_UInt16 + 3: int16, # GDT_Int16 + 4: uint32, # GDT_UInt32 + 5: int32, # GDT_Int32 + 6: float32, # GDT_Float32 + 7: float64, # GDT_Float64 + 8: complex_int16, # GDT_CInt16 + 9: complex64, # GDT_CInt32 + 10: complex64, # GDT_CFloat32 + 11: complex128, # GDT_CFloat64 +} dtype_rev = dict((v, k) for k, v in dtype_fwd.items()) -dtype_rev['uint8'] = 1 -dtype_rev['int8'] = 1 + +dtype_rev["uint8"] = 1 +dtype_rev["int8"] = 1 +dtype_rev["complex"] = 11 +dtype_rev["complex_int16"] = 8 typename_fwd = { 0: 'Unknown', @@ -154,7 +153,7 @@ def can_cast_dtype(values, dtype): if not is_ndarray(values): values = np.array(values) - if values.dtype.name == np.dtype(dtype).name: + if values.dtype.name == _getnpdtype(dtype).name: return True elif values.dtype.kind == 'f': @@ -185,3 +184,15 @@ def validate_dtype(values, valid_dtypes): return (values.dtype.name in valid_dtypes or get_minimum_dtype(values) in valid_dtypes) + + +def _is_complex_int(dtype): + return isinstance(dtype, str) and dtype.startswith("complex_int") + + +def _getnpdtype(dtype): + import numpy as np + if _is_complex_int(dtype): + return np.dtype("complex64") + else: + return np.dtype(dtype) diff --git a/rasterio/features.py b/rasterio/features.py index 4abf84af..704a3b90 100644 --- a/rasterio/features.py +++ b/rasterio/features.py @@ -9,7 +9,7 @@ import warnings import numpy as np import rasterio -from rasterio.dtypes import validate_dtype, can_cast_dtype, get_minimum_dtype +from rasterio.dtypes import validate_dtype, can_cast_dtype, get_minimum_dtype, _getnpdtype from rasterio.enums import MergeAlg from rasterio.env import ensure_env from rasterio.errors import ShapeSkipWarning @@ -276,7 +276,7 @@ def rasterize( if dtype is not None and not can_cast_dtype(default_value_array, dtype): raise ValueError(format_cast_error('default_vaue', dtype)) - if dtype is not None and np.dtype(dtype).name not in valid_dtypes: + if dtype is not None and _getnpdtype(dtype).name not in valid_dtypes: raise ValueError(format_invalid_dtype('dtype')) valid_shapes = [] @@ -332,7 +332,7 @@ def rasterize( raise ValueError(format_cast_error('shape values', dtype)) if out is not None: - if np.dtype(out.dtype).name not in valid_dtypes: + if _getnpdtype(out.dtype).name not in valid_dtypes: raise ValueError(format_invalid_dtype('out')) if not can_cast_dtype(shape_values, out.dtype): diff --git a/tests/test_complex_dtypes.py b/tests/test_complex_dtypes.py index fc06fe9e..96a9837b 100644 --- a/tests/test_complex_dtypes.py +++ b/tests/test_complex_dtypes.py @@ -1,4 +1,5 @@ import logging +import subprocess import sys import uuid @@ -8,9 +9,6 @@ import pytest import rasterio -logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - - @pytest.fixture(scope='function') def tempfile(): """A temporary filename in the GDAL '/vsimem' filesystem""" @@ -23,10 +21,8 @@ def complex_image(height, width, dtype): [complex(x, x) for x in range(height * width)], dtype=dtype).reshape(height, width) -dtypes = ['complex', 'complex64', 'complex128'] - -@pytest.mark.parametrize("dtype", dtypes) +@pytest.mark.parametrize("dtype", ["complex", "complex64", "complex128"]) @pytest.mark.parametrize("height,width", [(20, 20)]) def test_read_array(tempfile, dtype, height, width): """_io functions read and write arrays correctly""" @@ -58,3 +54,45 @@ def test_complex_nodata(tmpdir): with rasterio.open(tempfile) as dst: assert dst.nodata == 0 + + +@pytest.mark.gdalbin +def test_complex_int16(tmpdir): + """A cint16 dataset can be created""" + import numpy as np + import rasterio + from rasterio.transform import Affine + + x = np.linspace(-4.0, 4.0, 240) + y = np.linspace(-3.0, 3.0, 180) + X, Y = np.meshgrid(x, y) + Z1 = np.ones_like(X) + 1j + + res = (x[-1] - x[0]) / 240.0 + transform1 = Affine.translation(x[0] - res / 2, y[-1] - res / 2) * Affine.scale( + res, -res + ) + + tempfile = str(tmpdir.join("test.tif")) + + with rasterio.open( + tempfile, + "w", + driver="GTiff", + height=Z1.shape[0], + width=Z1.shape[1], + nodata=0, + count=1, + dtype="complex_int16", + crs="+proj=latlong", + transform=transform1, + ) as dst: + dst.write(Z1, 1) + + assert "Type=CInt16" in subprocess.check_output(["gdalinfo", tempfile]).decode( + "utf-8" + ) + + with rasterio.open(tempfile) as dst: + data = dst.read() + assert data.dtype == np.complex64 diff --git a/tests/test_dtypes.py b/tests/test_dtypes.py index 875cf8b4..e7e24f4a 100644 --- a/tests/test_dtypes.py +++ b/tests/test_dtypes.py @@ -3,10 +3,26 @@ import pytest import rasterio from rasterio import ( - ubyte, uint8, uint16, uint32, int16, int32, float32, float64, complex_) + ubyte, + uint8, + uint16, + uint32, + int16, + int32, + float32, + float64, + complex_, + complex_int16, +) from rasterio.dtypes import ( - _gdal_typename, is_ndarray, check_dtype, get_minimum_dtype, can_cast_dtype, - validate_dtype + _gdal_typename, + is_ndarray, + check_dtype, + get_minimum_dtype, + can_cast_dtype, + validate_dtype, + _is_complex_int, + _getnpdtype, ) @@ -28,10 +44,19 @@ def test_check_dtype_invalid(): assert not check_dtype('foo') -def test_gdal_name(): - assert _gdal_typename(ubyte) == 'Byte' - assert _gdal_typename(np.uint8) == 'Byte' - assert _gdal_typename(np.uint16) == 'UInt16' +@pytest.mark.parametrize( + ("dtype", "name"), + [ + (ubyte, "Byte"), + (np.uint8, "Byte"), + (np.uint16, "UInt16"), + ("uint8", "Byte"), + ("complex_int16", "CInt16"), + (complex_int16, "CInt16"), + ], +) +def test_gdal_name(dtype, name): + assert _gdal_typename(dtype) == name def test_get_minimum_dtype(): @@ -89,3 +114,17 @@ def test_complex(tmpdir): arr2 = src.read(1) assert np.array_equal(arr1, arr2) + + +def test_is_complex_int(): + assert _is_complex_int("complex_int16") + + +def test_not_is_complex_int(): + assert not _is_complex_int("complex") + + +def test_get_npdtype(): + npdtype = _getnpdtype("complex_int16") + assert npdtype == np.complex64 + assert npdtype.kind == "c" From a8190243c81bd34c208927bc07a52fade7e4bc47 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Wed, 26 May 2021 11:51:29 -0600 Subject: [PATCH 13/18] Use default Window constructor instead of Window.from_slices in union() (#2186) * Use Window ctor instead of from_slices in union() Resolves the issue reported in #2177 * Update version and change log --- CHANGES.txt | 2 ++ rasterio/__init__.py | 2 +- rasterio/windows.py | 22 ++++++++++++++++------ tests/test_windows.py | 27 +++++++++++++++++++++++++-- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d1da4e1c..91926c4b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,8 @@ Changes 1.2.4 (TBD) ----------- +- Use Window constructor instead of from_slices in windows.union to allow a + proper union to be formed from windows extending outside a dataset (#2186). - Read GDAL CInt16 data as np.complex64 and allow saving complex data to CInt16 (#2185). - Skip merge sources which do not overlap the destination. diff --git a/rasterio/__init__.py b/rasterio/__init__.py index 3ea187f8..8fe63473 100644 --- a/rasterio/__init__.py +++ b/rasterio/__init__.py @@ -40,7 +40,7 @@ import rasterio.enums import rasterio.path __all__ = ['band', 'open', 'pad', 'Env'] -__version__ = "1.2.3" +__version__ = "1.2.4dev" __gdal_version__ = gdal_version() # Rasterio attaches NullHandler to the 'rasterio' logger and its diff --git a/rasterio/windows.py b/rasterio/windows.py index 41ff87e6..4403e76b 100644 --- a/rasterio/windows.py +++ b/rasterio/windows.py @@ -187,9 +187,14 @@ def union(*windows): Window """ stacked = np.dstack([toranges(w) for w in windows]) - return Window.from_slices( - (stacked[0, 0].min(), stacked[0, 1].max()), - (stacked[1, 0].min(), stacked[1, 1]. max())) + row_start, row_stop = stacked[0, 0].min(), stacked[0, 1].max() + col_start, col_stop = stacked[1, 0].min(), stacked[1, 1].max() + return Window( + col_off=col_start, + row_off=row_start, + width=col_stop - col_start, + height=row_stop - row_start, + ) @iter_args @@ -211,9 +216,14 @@ def intersection(*windows): raise WindowError("windows do not intersect") stacked = np.dstack([toranges(w) for w in windows]) - return Window.from_slices( - (stacked[0, 0].max(), stacked[0, 1].min()), - (stacked[1, 0].max(), stacked[1, 1]. min())) + row_start, row_stop = stacked[0, 0].max(), stacked[0, 1].min() + col_start, col_stop = stacked[1, 0].max(), stacked[1, 1].min() + return Window( + col_off=col_start, + row_off=row_start, + width=col_stop - col_start, + height=row_stop - row_start, + ) @iter_args diff --git a/tests/test_windows.py b/tests/test_windows.py index cac6f1ce..932216a4 100644 --- a/tests/test_windows.py +++ b/tests/test_windows.py @@ -1,4 +1,3 @@ -from collections import namedtuple import logging import math import sys @@ -10,7 +9,7 @@ from hypothesis import given, assume, settings, HealthCheck from hypothesis.strategies import floats, integers import rasterio -from rasterio.errors import RasterioDeprecationWarning, WindowError +from rasterio.errors import WindowError from rasterio.windows import ( crop, from_bounds, bounds, transform, evaluate, window_index, shape, Window, intersect, intersection, get_data_window, union, @@ -631,3 +630,27 @@ def test_zero_height(sy): """Permit a zero height window""" transform = Affine.translation(0, 45.0) * Affine.scale(1.0, sy) assert from_bounds(0.0, 44.0, 1.0, 44.0, transform).height == 0 + + +def test_union_boundless_left(): + """Windows entirely to the left of a dataset form a proper union""" + uw = union( + Window(col_off=-10, row_off=0, width=2, height=2), + Window(col_off=-8.5, row_off=0, width=2.5, height=2), + ) + assert uw.col_off == -10 + assert uw.width == 4 + assert uw.height == 2 + assert uw.row_off == 0 + + +def test_union_boundless_above(): + """Windows entirely above a dataset form a proper union""" + uw = union( + Window(col_off=0, row_off=-10, width=2, height=2), + Window(col_off=0, row_off=-8.5, width=2, height=2.5), + ) + assert uw.row_off == -10 + assert uw.height == 4 + assert uw.width == 2 + assert uw.col_off == 0 From 61ff96232ff08df8c123762db443f5c97a8c64e9 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Fri, 28 May 2021 15:08:36 -0600 Subject: [PATCH 14/18] Change comparisons of nodata to IgnoreOption for click 8.0 Resolves #2188 --- CHANGES.txt | 2 ++ rasterio/rio/edit_info.py | 4 ++-- rasterio/rio/options.py | 4 ++-- setup.py | 2 +- tests/test_rio_edit_info.py | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 91926c4b..a31565ee 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,8 @@ Changes 1.2.4 (TBD) ----------- +- Change comparisons of nodata to IgnoreOption to accomodate changes in click + 8.0. - Use Window constructor instead of from_slices in windows.union to allow a proper union to be formed from windows extending outside a dataset (#2186). - Read GDAL CInt16 data as np.complex64 and allow saving complex data to CInt16 diff --git a/rasterio/rio/edit_info.py b/rasterio/rio/edit_info.py index 64d8242e..e47e29cd 100644 --- a/rasterio/rio/edit_info.py +++ b/rasterio/rio/edit_info.py @@ -190,7 +190,7 @@ def edit(ctx, input, bidx, nodata, unset_nodata, crs, unset_crs, transform, tags = allmd['tags'] colorinterp = allmd['colorinterp'] - if unset_nodata and nodata is not options.IgnoreOption: + if unset_nodata and str(nodata) != str(options.IgnoreOption): raise click.BadParameter( "--unset-nodata and --nodata cannot be used together.") @@ -207,7 +207,7 @@ def edit(ctx, input, bidx, nodata, unset_nodata, crs, unset_crs, transform, except NotImplementedError as exc: # pragma: no cover raise click.ClickException(str(exc)) - elif nodata is not options.IgnoreOption: + elif str(nodata) != str(options.IgnoreOption): dtype = dst.dtypes[0] if nodata is not None and not in_dtype_range(nodata, dtype): raise click.BadParameter( diff --git a/rasterio/rio/options.py b/rasterio/rio/options.py index d759a88e..7b0f50e3 100644 --- a/rasterio/rio/options.py +++ b/rasterio/rio/options.py @@ -180,7 +180,7 @@ def like_handler(ctx, param, value): def nodata_handler(ctx, param, value): """Return a float or None""" - if value is None or value is IgnoreOption: + if value is None or str(value) == str(IgnoreOption): return value elif value.lower() in ['null', 'nil', 'none', 'nada']: return None @@ -199,7 +199,7 @@ def edit_nodata_handler(ctx, param, value): Expected values are 'like', 'null', a numeric value, 'nan', or IgnoreOption. Anything else should raise BadParameter. """ - if value == 'like' or value is IgnoreOption: + if value == 'like' or str(value) == str(IgnoreOption): retval = from_like_context(ctx, param, value) if retval is not None: return retval diff --git a/setup.py b/setup.py index c7711522..1c677c15 100755 --- a/setup.py +++ b/setup.py @@ -334,7 +334,7 @@ inst_reqs = [ "affine", "attrs", "certifi", - "click>=4.0,<8", + "click>=4.0", "cligj>=0.5", "numpy", "snuggs>=1.4.1", diff --git a/tests/test_rio_edit_info.py b/tests/test_rio_edit_info.py index 09d54bd1..94b9f586 100644 --- a/tests/test_rio_edit_info.py +++ b/tests/test_rio_edit_info.py @@ -65,7 +65,7 @@ def test_delete_nodata(data, runner): """Delete a dataset's nodata value""" inputfile = str(data.join('RGB.byte.tif')) result = runner.invoke( - main_group, ['edit-info', inputfile, '--unset-nodata']) + main_group, ['edit-info', inputfile, '--unset-nodata'], catch_exceptions=False) assert result.exit_code == 0 From 74b60a325f152068bbb1f97a039540ffa310a966 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Sat, 29 May 2021 16:06:56 -0600 Subject: [PATCH 15/18] Guard against read/write buffer shape mismatches Resolves #2189 --- CHANGES.txt | 2 ++ rasterio/_io.pyx | 6 ++---- rasterio/_shim1.pyx | 8 +++++++- rasterio/errors.py | 8 ++++++-- rasterio/shim_rasterioex.pxi | 8 +++++++- tests/test_read.py | 26 +++++++++++--------------- 6 files changed, 35 insertions(+), 23 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index a31565ee..004ae979 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,8 @@ Changes 1.2.4 (TBD) ----------- +- Prevent segfaults when buffer and band indexes are mismatched in + io_multi_band and io_multi_mask (#2189). - Change comparisons of nodata to IgnoreOption to accomodate changes in click 8.0. - Use Window constructor instead of from_slices in windows.union to allow a diff --git a/rasterio/_io.pyx b/rasterio/_io.pyx index b921723e..6e850517 100644 --- a/rasterio/_io.pyx +++ b/rasterio/_io.pyx @@ -364,8 +364,7 @@ cdef class DatasetReaderBase(DatasetBase): log.debug("Jump straight to _read()") log.debug("Window: %r", window) - out = self._read(indexes, out, window, dtype, - resampling=resampling) + out = self._read(indexes, out, window, dtype, resampling=resampling) if masked or fill_value is not None: if all_valid: @@ -631,8 +630,7 @@ cdef class DatasetReaderBase(DatasetBase): return out - def _read(self, indexes, out, window, dtype, masks=False, - resampling=Resampling.nearest): + def _read(self, indexes, out, window, dtype, masks=False, resampling=Resampling.nearest): """Read raster bands as a multidimensional array If `indexes` is a list, the result is a 3D array, but diff --git a/rasterio/_shim1.pyx b/rasterio/_shim1.pyx index 012c1c14..aee0cdcd 100644 --- a/rasterio/_shim1.pyx +++ b/rasterio/_shim1.pyx @@ -16,7 +16,7 @@ from rasterio.enums import Resampling cimport numpy as np from rasterio._err cimport exc_wrap_int, exc_wrap_pointer -from rasterio.errors import GDALOptionNotImplementedError +from rasterio.errors import GDALOptionNotImplementedError, DatasetIOShapeError cdef GDALDatasetH open_dataset( @@ -114,6 +114,9 @@ cdef int io_multi_band( cdef int xsize = width cdef int ysize = height + if len(indexes) != data.shape[0]: + raise DatasetIOShapeError("Dataset indexes and destination buffer are mismatched") + bandmap = CPLMalloc(count*sizeof(int)) for i in range(count): bandmap[i] = indexes[i] @@ -162,6 +165,9 @@ cdef int io_multi_mask( cdef int xsize = width cdef int ysize = height + if len(indexes) != data.shape[0]: + raise DatasetIOShapeError("Dataset indexes and destination buffer are mismatched") + for i in range(count): j = indexes[i] band = GDALGetRasterBand(hds, j) diff --git a/rasterio/errors.py b/rasterio/errors.py index 4f780edd..5452e17f 100644 --- a/rasterio/errors.py +++ b/rasterio/errors.py @@ -87,9 +87,9 @@ class GDALOptionNotImplementedError(RasterioError): by GDAL 1.x. """ + class GDALVersionError(RasterioError): - """Raised if the runtime version of GDAL does not meet the required - version of GDAL.""" + """Raised if the runtime version of GDAL does not meet the required version of GDAL.""" class WindowEvaluationError(ValueError): @@ -142,3 +142,7 @@ class TransformError(RasterioError): class WarpedVRTError(RasterioError): """Raised when WarpedVRT can't be initialized""" + + +class DatasetIOShapeError(RasterioError): + """Raised when data buffer shape is a mismatch when reading and writing""" diff --git a/rasterio/shim_rasterioex.pxi b/rasterio/shim_rasterioex.pxi index da23910d..3d279ece 100644 --- a/rasterio/shim_rasterioex.pxi +++ b/rasterio/shim_rasterioex.pxi @@ -4,7 +4,7 @@ from rasterio import dtypes from rasterio.enums import Resampling from rasterio.env import GDALVersion -from rasterio.errors import ResamplingAlgorithmError +from rasterio.errors import ResamplingAlgorithmError, DatasetIOShapeError cimport numpy as np @@ -145,6 +145,9 @@ cdef int io_multi_band(GDALDatasetH hds, int mode, double x0, double y0, extras.pfnProgress = NULL extras.pProgressData = NULL + if len(indexes) != data.shape[0]: + raise DatasetIOShapeError("Dataset indexes and destination buffer are mismatched") + bandmap = CPLMalloc(count*sizeof(int)) for i in range(count): bandmap[i] = indexes[i] @@ -207,6 +210,9 @@ cdef int io_multi_mask(GDALDatasetH hds, int mode, double x0, double y0, extras.pfnProgress = NULL extras.pProgressData = NULL + if len(indexes) != data.shape[0]: + raise DatasetIOShapeError("Dataset indexes and destination buffer are mismatched") + for i in range(count): j = indexes[i] band = GDALGetRasterBand(hds, j) diff --git a/tests/test_read.py b/tests/test_read.py index 10d87180..f1b2a1c8 100644 --- a/tests/test_read.py +++ b/tests/test_read.py @@ -1,23 +1,18 @@ from hashlib import md5 -import logging -import sys import unittest import numpy as np import pytest import rasterio - - -logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - +from rasterio.errors import DatasetIOShapeError # Find out if we've got HDF support (needed below). try: with rasterio.open('tests/data/no_band.h5') as s: pass has_hdf = True -except: +except Exception: has_hdf = False @@ -93,13 +88,7 @@ class ReaderContextTest(unittest.TestCase): def test_read_out_dtype_fail(self): with rasterio.open('tests/data/RGB.byte.tif') as s: a = np.zeros((718, 791), dtype=rasterio.float32) - try: - s.read(1, a) - except ValueError as e: - assert ("the array's dtype 'float32' does not match the " - "file's dtype") in str(e) - except: - assert "failed to catch exception" is False + s.read(1, a) def test_read_basic(self): with rasterio.open('tests/data/shade.tif') as s: @@ -315,7 +304,7 @@ def test_out_shape_exceptions(path_rgb_byte_tif): out_shape = (src.count, src.height, src.width) reader(out=out, out_shape=out_shape) - with pytest.raises(ValueError): + with pytest.raises(Exception): out_shape = (5, src.height, src.width) reader(1, out_shape=out_shape) @@ -325,3 +314,10 @@ def test_out_shape_implicit(path_rgb_byte_tif): with rasterio.open(path_rgb_byte_tif) as src: out = src.read(indexes=(1, 2), out_shape=src.shape) assert out.shape == (2,) + src.shape + + +def test_out_shape_no_segfault(path_rgb_byte_tif): + """Prevent segfault as reported in 2189""" + with rasterio.open(path_rgb_byte_tif) as src: + with pytest.raises(DatasetIOShapeError): + src.read(out_shape=(2, src.height, src.width)) From 09fc48265acacef7c59d11f0e9bfec82e53eeb75 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Sat, 29 May 2021 17:05:20 -0600 Subject: [PATCH 16/18] Guard against gauss resampling Resolves #2190 --- CHANGES.txt | 1 + rasterio/_warp.pyx | 21 +++++++++++++++++++++ tests/test_warpedvrt.py | 7 +++++++ 3 files changed, 29 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 004ae979..33cd1fb4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,7 @@ Changes 1.2.4 (TBD) ----------- +- Guard against use of gauss resampling when creating a WarpedVRT (#2190). - Prevent segfaults when buffer and band indexes are mismatched in io_multi_band and io_multi_mask (#2189). - Change comparisons of nodata to IgnoreOption to accomodate changes in click diff --git a/rasterio/_warp.pyx b/rasterio/_warp.pyx index 979a3764..0fd23bc5 100644 --- a/rasterio/_warp.pyx +++ b/rasterio/_warp.pyx @@ -43,6 +43,17 @@ from rasterio._shim cimport delete_nodata_value, open_dataset log = logging.getLogger(__name__) +# Gauss (7) is not supported for warp +SUPPORTED_RESAMPLING = [r for r in Resampling if r.value < 7] +GDAL2_RESAMPLING = [r for r in Resampling if r.value > 7 and r.value <= 12] +if GDALVersion.runtime().at_least('2.0'): + SUPPORTED_RESAMPLING.extend(GDAL2_RESAMPLING) +# sum supported since GDAL 3.1 +if GDALVersion.runtime().at_least('3.1'): + SUPPORTED_RESAMPLING.append(Resampling.sum) +# rms supported since GDAL 3.3 +if GDALVersion.runtime().at_least('3.3'): + SUPPORTED_RESAMPLING.append(Resampling.rms) def recursive_round(val, precision): """Recursively round coordinates.""" @@ -777,6 +788,16 @@ cdef class WarpedVRTReaderBase(DatasetReaderBase): if src_dataset.mode != "r": warnings.warn("Source dataset should be opened in read-only mode. Use of datasets opened in modes other than 'r' will be disallowed in a future version.", RasterioDeprecationWarning, stacklevel=2) + # Guard against invalid or unsupported resampling algorithms. + try: + if resampling == 7: + raise ValueError("Gauss resampling is not supported") + Resampling(resampling) + + except ValueError: + raise ValueError( + "resampling must be one of: {0}".format(", ".join(['Resampling.{0}'.format(r.name) for r in SUPPORTED_RESAMPLING]))) + self.mode = 'r' self.options = {} self._count = 0 diff --git a/tests/test_warpedvrt.py b/tests/test_warpedvrt.py index bf2f7c5b..799b1eff 100644 --- a/tests/test_warpedvrt.py +++ b/tests/test_warpedvrt.py @@ -608,3 +608,10 @@ def test_issue2086(): with rasterio.open("tests/data/white-gemini-iv.vrt") as src: with WarpedVRT(src, crs=DST_CRS) as vrt: assert vrt.shape == (1031, 1146) + + +def test_gauss_no(path_rgb_byte_tif): + """Guard against the issue reported in #2190""" + with rasterio.open(path_rgb_byte_tif) as src: + with pytest.raises(Exception): + WarpedVRT(src, resampling=Resampling.gauss) From 754930309b216a87325087f5bbbc375e28102549 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Mon, 31 May 2021 14:13:13 -0600 Subject: [PATCH 17/18] Eliminate unneeded IgnoreOption (#2191) * Eliminate unneeded IgnoreOption * Note CLI option bug fix --- CHANGES.txt | 6 +++-- rasterio/rio/edit_info.py | 19 +++++++------- rasterio/rio/options.py | 53 ++++++++++++++++++++------------------- tests/test_rio_options.py | 17 ++++++++----- 4 files changed, 52 insertions(+), 43 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 33cd1fb4..4d972073 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,9 +1,11 @@ Changes ======= -1.2.4 (TBD) ------------ +1.2.4 (2021-05-31) +------------------ +- Eliminate unneeded marker for CLI nodata options to be ignored. We will stick + with None (#2191). - Guard against use of gauss resampling when creating a WarpedVRT (#2190). - Prevent segfaults when buffer and band indexes are mismatched in io_multi_band and io_multi_mask (#2189). diff --git a/rasterio/rio/edit_info.py b/rasterio/rio/edit_info.py index e47e29cd..a9d4a34c 100644 --- a/rasterio/rio/edit_info.py +++ b/rasterio/rio/edit_info.py @@ -190,13 +190,13 @@ def edit(ctx, input, bidx, nodata, unset_nodata, crs, unset_crs, transform, tags = allmd['tags'] colorinterp = allmd['colorinterp'] - if unset_nodata and str(nodata) != str(options.IgnoreOption): + if unset_nodata and nodata is not None: raise click.BadParameter( - "--unset-nodata and --nodata cannot be used together.") + "--unset-nodata and --nodata cannot be used together." + ) if unset_crs and crs: - raise click.BadParameter( - "--unset-crs and --crs cannot be used together.") + raise click.BadParameter("--unset-crs and --crs cannot be used together.") if unset_nodata: # Setting nodata to None will raise NotImplementedError @@ -207,17 +207,18 @@ def edit(ctx, input, bidx, nodata, unset_nodata, crs, unset_crs, transform, except NotImplementedError as exc: # pragma: no cover raise click.ClickException(str(exc)) - elif str(nodata) != str(options.IgnoreOption): + elif nodata is not None: dtype = dst.dtypes[0] if nodata is not None and not in_dtype_range(nodata, dtype): raise click.BadParameter( - "outside the range of the file's " - "data type (%s)." % dtype, - param=nodata, param_hint='nodata') + "outside the range of the file's data type (%s)." % dtype, + param=nodata, + param_hint="nodata", + ) dst.nodata = nodata if unset_crs: - dst.crs = None # CRS() + dst.crs = None elif crs: dst.crs = crs diff --git a/rasterio/rio/options.py b/rasterio/rio/options.py index 7b0f50e3..20022fa0 100644 --- a/rasterio/rio/options.py +++ b/rasterio/rio/options.py @@ -57,20 +57,6 @@ from rasterio.path import parse_path, ParsedPath, UnparsedPath logger = logging.getLogger(__name__) -class IgnoreOptionMarker(object): - """A marker for an option that is to be ignored. - - For use in the case where `None` is a meaningful option value, - such as for nodata where `None` means "unset the nodata value." - """ - - def __repr__(self): - return 'IgnoreOption()' - - -IgnoreOption = IgnoreOptionMarker() - - def _cb_key_val(ctx, param, value): """ @@ -180,29 +166,36 @@ def like_handler(ctx, param, value): def nodata_handler(ctx, param, value): """Return a float or None""" - if value is None or str(value) == str(IgnoreOption): - return value - elif value.lower() in ['null', 'nil', 'none', 'nada']: + if value is None or value.lower() in ["null", "nil", "none", "nada"]: return None else: try: return float(value) except (TypeError, ValueError): raise click.BadParameter( - "{!r} is not a number".format(value), - param=param, param_hint='nodata') + "{!r} is not a number".format(value), param=param, param_hint="nodata" + ) def edit_nodata_handler(ctx, param, value): """Get nodata value from a template file or command line. - Expected values are 'like', 'null', a numeric value, 'nan', or - IgnoreOption. Anything else should raise BadParameter. + Expected values are 'like', 'null', a numeric value, 'nan', or None. + + Returns + ------- + float or None + + Raises + ------ + click.BadParameter + """ - if value == 'like' or str(value) == str(IgnoreOption): + if value == "like" or value is None: retval = from_like_context(ctx, param, value) if retval is not None: return retval + return nodata_handler(ctx, param, value) @@ -335,12 +328,20 @@ overwrite_opt = click.option( help="Always overwrite an existing output file.") nodata_opt = click.option( - '--nodata', callback=nodata_handler, default=None, - metavar='NUMBER|nan', help="Set a Nodata value.") + "--nodata", + callback=nodata_handler, + default=None, + metavar="NUMBER|nan", + help="Set a Nodata value.", +) edit_nodata_opt = click.option( - '--nodata', callback=edit_nodata_handler, default=IgnoreOption, - metavar='NUMBER|nan|null', help="Modify the Nodata value.") + "--nodata", + callback=edit_nodata_handler, + default=None, + metavar="NUMBER|nan|null", + help="Modify the Nodata value.", +) like_opt = click.option( '--like', diff --git a/tests/test_rio_options.py b/tests/test_rio_options.py index 29e4476b..a63a4b85 100644 --- a/tests/test_rio_options.py +++ b/tests/test_rio_options.py @@ -9,8 +9,13 @@ import pytest from rasterio.enums import ColorInterp from rasterio.rio.options import ( - IgnoreOption, bounds_handler, file_in_handler, like_handler, - edit_nodata_handler, nodata_handler, _cb_key_val) + bounds_handler, + file_in_handler, + like_handler, + edit_nodata_handler, + nodata_handler, + _cb_key_val, +) class MockContext: @@ -186,14 +191,14 @@ def test_edit_nodata_callback_like(data): def test_edit_nodata_callback_all_like(data): ctx = MockContext() - ctx.obj['like'] = {'nodata': 0.0} - ctx.obj['all_like'] = True - assert edit_nodata_handler(ctx, MockOption('nodata'), IgnoreOption) == 0.0 + ctx.obj["like"] = {"nodata": 0.0} + ctx.obj["all_like"] = True + assert edit_nodata_handler(ctx, MockOption("nodata"), None) == 0.0 def test_edit_nodata_callback_ignore(data): ctx = MockContext() - assert edit_nodata_handler(ctx, MockOption('nodata'), IgnoreOption) is IgnoreOption + assert edit_nodata_handler(ctx, MockOption("nodata"), None) is None def test_edit_nodata_callback_none(data): From 79971ce33d838c7f3b2fa01fcc929b1ee11e7206 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Mon, 31 May 2021 14:18:58 -0600 Subject: [PATCH 18/18] 1.2.4 --- rasterio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasterio/__init__.py b/rasterio/__init__.py index 8fe63473..f125f42c 100644 --- a/rasterio/__init__.py +++ b/rasterio/__init__.py @@ -40,7 +40,7 @@ import rasterio.enums import rasterio.path __all__ = ['band', 'open', 'pad', 'Env'] -__version__ = "1.2.4dev" +__version__ = "1.2.4" __gdal_version__ = gdal_version() # Rasterio attaches NullHandler to the 'rasterio' logger and its