GDAL 3.0.x support in code but not wheels (#1729)

* Added support for Proj6 and GDAL 3

* install proj6

* build proj with local installation dir

* conditional proj dependency

* added proj4 flags for all gdal versions

* Restore debian data tests

* Version specific proj configuration

* Fix string comp expression

* Explicit matrix

* Restructure build matrix

* Remove wait from proj build script

* Call script with bash

* Fix version text

* Bash syntax

* Correct PROJOPTs

* Use xvfb service

* Back to proj 4.8 for older GDAL

* Fix syntax error

* Add verbosity to tests

* Take proj off library path, remove gdal-bin package

* Install PROJ and GDAL to same prefix

* Set up runtime env

* Try 4.9.3

* Source build scripts

* Switch to case for versions

* set trace

* Test existence of share/gdal directory

The proj install script created the gdal install directory so a
test for just that is not sufficient.

* Test for share/proj

* Run tests under gdb

* Forgot the run command

* Wait 20 for GDAL

* travis_wait

* Restructure build script, less boilerplate

* Back to trusty

* dist: trusty

* Remove gdb

* Allow 3.0.1/6.1.1 to fail, some changes since 3.0.0/6.1.0
This commit is contained in:
Sean Gillies 2019-07-26 13:46:55 -06:00 committed by GitHub
parent 32faa236cf
commit 36786d5886
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 259 additions and 76 deletions

View File

@ -1,53 +1,53 @@
dist: trusty
language: python
sudo: false
cache:
directories:
- $GDALINST
- ~/.cache/pip
python:
- "2.7"
- "3.6"
env:
global:
- PIP_WHEEL_DIR=$HOME/.cache/pip/wheels
- PIP_FIND_LINKS=file://$HOME/.cache/pip/wheels
- GDALINST=$HOME/gdalinstall
- GDALBUILD=$HOME/gdalbuild
- PROJINST=$HOME/gdalinstall
- PROJBUILD=$HOME/projbuild
matrix:
- GDALVERSION="1.11.5"
- GDALVERSION="2.0.3"
- GDALVERSION="2.1.4"
- GDALVERSION="2.2.4"
- GDALVERSION="2.3.3"
- GDALVERSION="2.4.1"
- GDALVERSION="3.0.0"
- GDALVERSION="master"
- GDALVERSION="1.11.5" PROJVERSION="4.8.0"
- GDALVERSION="2.0.3" PROJVERSION="4.9.3"
- GDALVERSION="2.1.4" PROJVERSION="4.9.3"
- GDALVERSION="2.2.4" PROJVERSION="4.9.3"
- GDALVERSION="2.3.3" PROJVERSION="4.9.3"
- GDALVERSION="2.4.2" PROJVERSION="4.9.3"
- GDALVERSION="3.0.1" PROJVERSION="6.1.1"
- GDALVERSION="master" PROJVERSION="6.1.1"
matrix:
allow_failures:
- env: GDALVERSION="3.0.0"
- env: GDALVERSION="master"
- env: GDALVERSION="3.0.1" PROJVERSION="6.1.1"
- env: GDALVERSION="master" PROJVERSION="6.1.1"
addons:
apt:
packages:
- gdal-bin
- libproj-dev
- libhdf5-serial-dev
- libgdal-dev
- libatlas-dev
- libatlas-base-dev
- gfortran
python:
- "2.7"
- "3.6"
before_script: # configure a headless display to test matplotlib
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
- "sleep 3"
before_install:
- python -m pip install -U pip
- python -m pip install wheel
- travis_wait 20 bash ./scripts/travis_gdal_install.sh
- export PATH=$GDALINST/gdal-$GDALVERSION/bin:$PATH
- export LD_LIBRARY_PATH=$GDALINST/gdal-$GDALVERSION/lib:$LD_LIBRARY_PATH
- . ./scripts/travis_proj_install.sh
- travis_wait 20 . ./scripts/travis_gdal_install.sh
- export GDAL_DATA=$GDALINST/gdal-$GDALVERSION/share/gdal
- export PROJ_LIB=$GDALINST/gdal-$GDALVERSION/share/proj
install:
- "if [ \"$GDALVERSION\" == \"master\" -o $(gdal-config --version) == \"$GDALVERSION\" ]; then echo \"Using gdal $GDALVERSION\"; else echo \"NOT using gdal $GDALVERSION as expected; aborting\"; exit 1; fi"
- "python -m pip wheel -r requirements-dev.txt"
@ -57,8 +57,20 @@ install:
- "rio --version"
- "rio --gdal-version"
- "python -m pip list"
before_script:
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
- "sleep 3"
script:
- "if [[ $TRAVIS_PYTHON_VERSION == 3.5 && $GDALVERSION == 2.1.0 ]]; then python -m pytest --doctest-ignore-import-errors --doctest-glob='*.rst' docs/*.rst -k 'not index and not quickstart and not switch' ; fi"
- python -m pytest -m "not wheel" -rxXs --cov rasterio --cov-report term-missing
- python -m pytest -v -m "not wheel" -rxXs --cov rasterio --cov-report term-missing
after_success:
- coveralls || echo "!! intermittent coveralls failure"
cache:
directories:
- $GDALINST
- ~/.cache/pip

View File

@ -4,6 +4,7 @@ Changes
1.0.25 (TBD)
------------
- Add support for GDAL 3.x
- We were using pytest incorrectly and pytest 5 caught us doing it. This is
now fixed in commit b9f34ee.
- A bug preventing creation of Env instances, and thus dataset opening, when

View File

@ -15,7 +15,7 @@ from rasterio._err import (
GDALError, CPLE_BaseError, CPLE_IllegalArgError, CPLE_OpenFailedError,
CPLE_NotSupportedError)
from rasterio._err cimport exc_wrap_pointer, exc_wrap_int
from rasterio._shim cimport open_dataset
from rasterio._shim cimport open_dataset, osr_get_name
from rasterio.compat import string_types
from rasterio.control import GroundControlPoint

View File

@ -11,6 +11,7 @@ from rasterio.errors import CRSError
from rasterio._base cimport _osr_from_crs as osr_from_crs
from rasterio._base cimport _safe_osr_release
from rasterio._err cimport exc_wrap_ogrerr, exc_wrap_int, exc_wrap_pointer
from rasterio._shim cimport osr_get_name
log = logging.getLogger(__name__)
@ -112,20 +113,23 @@ cdef class _CRS(object):
cdef char *conv_wkt = NULL
try:
if morph_to_esri_dialect:
exc_wrap_ogrerr(OSRMorphToESRI(self._osr))
exc_wrap_ogrerr(OSRExportToWkt(self._osr, &conv_wkt))
if osr_get_name(self._osr) != NULL:
if morph_to_esri_dialect:
exc_wrap_ogrerr(OSRMorphToESRI(self._osr))
exc_wrap_ogrerr(OSRExportToWkt(self._osr, &conv_wkt))
except CPLE_BaseError as exc:
raise CRSError("Cannot convert to WKT. {}".format(exc))
else:
return conv_wkt.decode('utf-8')
if conv_wkt != NULL:
return conv_wkt.decode('utf-8')
else:
return ''
finally:
CPLFree(conv_wkt)
def to_epsg(self):
"""The epsg code of the CRS

View File

@ -5,3 +5,5 @@ cdef int delete_nodata_value(GDALRasterBandH hBand) except 3
cdef int io_band(GDALRasterBandH band, int mode, float xoff, float yoff, float width, float height, object data, int resampling=*) except -1
cdef int io_multi_band(GDALDatasetH hds, int mode, float xoff, float yoff, float width, float height, object data, Py_ssize_t[:] indexes, int resampling=*) except -1
cdef int io_multi_mask(GDALDatasetH hds, int mode, float xoff, float yoff, float width, float height, object data, Py_ssize_t[:] indexes, int resampling=*) except -1
cdef const char* osr_get_name(OGRSpatialReferenceH hSrs)

View File

@ -188,3 +188,8 @@ cdef int io_multi_mask(
break
return exc_wrap_int(retval)
cdef const char* osr_get_name(OGRSpatialReferenceH hSrs):
return ''

View File

@ -68,3 +68,7 @@ cdef GDALDatasetH open_dataset(
cdef int delete_nodata_value(GDALRasterBandH hBand) except 3:
raise NotImplementedError(
"GDAL versions < 2.1 do not support nodata deletion")
cdef const char* osr_get_name(OGRSpatialReferenceH hSrs):
return ''

View File

@ -69,3 +69,7 @@ cdef GDALDatasetH open_dataset(
cdef int delete_nodata_value(GDALRasterBandH hBand) except 3:
return GDALDeleteRasterNoDataValue(hBand)
cdef const char* osr_get_name(OGRSpatialReferenceH hSrs):
return ''

80
rasterio/_shim30.pyx Normal file
View File

@ -0,0 +1,80 @@
"""Rasterio shims for GDAL 3.x"""
# cython: boundscheck=False
include "directives.pxi"
# The baseline GDAL API.
include "gdal.pxi"
# Shim API for GDAL >= 3.0
include "shim_rasterioex.pxi"
# Declarations and implementations specific for GDAL >= 3.x
cdef extern from "gdal.h" nogil:
cdef CPLErr GDALDeleteRasterNoDataValue(GDALRasterBandH hBand)
GDALDatasetH GDALOpenEx(const char *filename, int flags, const char **allowed_drivers, const char **options, const char **siblings) # except -1
cdef extern from "ogr_srs_api.h" nogil:
const char* OSRGetName(OGRSpatialReferenceH hSRS)
from rasterio._err cimport exc_wrap_pointer
cdef GDALDatasetH open_dataset(
object filename, int flags, object allowed_drivers,
object open_options, object siblings) except NULL:
"""Wrapper for GDALOpen and GDALOpenShared"""
cdef char **drivers = NULL
cdef char **options = NULL
cdef GDALDatasetH hds = NULL
cdef const char *fname = NULL
filename = filename.encode('utf-8')
fname = filename
# Construct a null terminated C list of driver
# names for GDALOpenEx.
if allowed_drivers:
for name in allowed_drivers:
name = name.encode('utf-8')
drivers = CSLAddString(drivers, <const char *>name)
for k, v in open_options.items():
k = k.upper().encode('utf-8')
# Normalize values consistent with code in _env module.
if isinstance(v, bool):
v = ('ON' if v else 'OFF').encode('utf-8')
else:
v = str(v).encode('utf-8')
options = CSLAddNameValue(options, <const char *>k, <const char *>v)
# Support for sibling files is not yet implemented.
if siblings:
raise NotImplementedError(
"Sibling files are not implemented")
# Ensure raster flags
flags = flags | 0x02
with nogil:
hds = GDALOpenEx(fname, flags, drivers, options, NULL)
try:
return exc_wrap_pointer(hds)
finally:
CSLDestroy(drivers)
CSLDestroy(options)
cdef int delete_nodata_value(GDALRasterBandH hBand) except 3:
return GDALDeleteRasterNoDataValue(hBand)
cdef const char* osr_get_name(OGRSpatialReferenceH hSrs):
return OSRGetName(hSrs)

View File

@ -63,7 +63,6 @@ class CRS(collections.Mapping):
self._crs = None
if initialdata or kwargs:
data = dict(initialdata or {})
data.update(**kwargs)
data = {k: v for k, v in data.items() if k in all_proj_keys}

View File

@ -1,4 +1,4 @@
#!/bin/sh
#!/bin/bash
#
# originally contributed by @rbuffat to Toblerity/Fiona
set -e
@ -41,12 +41,6 @@ GDALOPTS=" --with-geos \
--without-perl \
--without-python"
if [ "${GDALVERSION::1}" = "1" ]; then
PROJOPT="--with-static-proj4=/usr/lib";
else
PROJOPT="--with-proj";
fi
# Create build dir if not exists
if [ ! -d "$GDALBUILD" ]; then
mkdir $GDALBUILD;
@ -59,35 +53,61 @@ fi
ls -l $GDALINST
if [ "$GDALVERSION" = "master" ]; then
cd $GDALBUILD
git clone --depth 1 https://github.com/OSGeo/gdal gdal-$GDALVERSION
cd gdal-$GDALVERSION/gdal
git rev-parse HEAD > newrev.txt
BUILD=no
# Only build if nothing cached or if the GDAL revision changed
if test ! -f $GDALINST/gdal-$GDALVERSION/rev.txt; then
BUILD=yes
elif ! diff newrev.txt $GDALINST/gdal-$GDALVERSION/rev.txt >/dev/null; then
BUILD=yes
fi
if test "$BUILD" = "yes"; then
mkdir -p $GDALINST/gdal-$GDALVERSION
cp newrev.txt $GDALINST/gdal-$GDALVERSION/rev.txt
./configure --prefix=$GDALINST/gdal-$GDALVERSION $GDALOPTS $PROJOPT
make -s -j 2
make install
fi
fi
PROJOPT="--with-proj=$GDALINST/gdal-$GDALVERSION"
cd $GDALBUILD
git clone --depth 1 https://github.com/OSGeo/gdal gdal-$GDALVERSION
cd gdal-$GDALVERSION/gdal
git rev-parse HEAD > newrev.txt
BUILD=no
# Only build if nothing cached or if the GDAL revision changed
if test ! -f $GDALINST/gdal-$GDALVERSION/rev.txt; then
BUILD=yes
elif ! diff newrev.txt $GDALINST/gdal-$GDALVERSION/rev.txt >/dev/null; then
BUILD=yes
fi
if test "$BUILD" = "yes"; then
mkdir -p $GDALINST/gdal-$GDALVERSION
cp newrev.txt $GDALINST/gdal-$GDALVERSION/rev.txt
./configure --prefix=$GDALINST/gdal-$GDALVERSION $GDALOPTS $PROJOPT
make -s -j 2
make install
fi
if [ "$GDALVERSION" != "master" -a ! -d "$GDALINST/gdal-$GDALVERSION" ]; then
cd $GDALBUILD
gdalver=$(expr "$GDALVERSION" : '\([0-9]*.[0-9]*.[0-9]*\)')
wget -q http://download.osgeo.org/gdal/$gdalver/gdal-$GDALVERSION.tar.gz
tar -xzf gdal-$GDALVERSION.tar.gz
cd gdal-$gdalver
./configure --prefix=$GDALINST/gdal-$GDALVERSION $GDALOPTS $PROJOPT
make -s -j 2
make install
else
case "$GDALVERSION" in
3*)
PROJOPT="--with-proj=$GDALINST/gdal-$GDALVERSION"
;;
2.4*)
PROJOPT="--with-proj=$GDALINST/gdal-$GDALVERSION"
;;
2.3*)
PROJOPT="--with-proj=$GDALINST/gdal-$GDALVERSION"
;;
2.2*)
PROJOPT="--with-static-proj4=$GDALINST/gdal-$GDALVERSION"
;;
2.1*)
PROJOPT="--with-static-proj4=$GDALINST/gdal-$GDALVERSION"
;;
2.0*)
PROJOPT="--with-static-proj4=$GDALINST/gdal-$GDALVERSION"
;;
1*)
PROJOPT="--with-static-proj4=$GDALINST/gdal-$GDALVERSION"
;;
esac
if [ ! -d "$GDALINST/gdal-$GDALVERSION/share/gdal" ]; then
cd $GDALBUILD
gdalver=$(expr "$GDALVERSION" : '\([0-9]*.[0-9]*.[0-9]*\)')
wget -q http://download.osgeo.org/gdal/$gdalver/gdal-$GDALVERSION.tar.gz
tar -xzf gdal-$GDALVERSION.tar.gz
cd gdal-$gdalver
./configure --prefix=$GDALINST/gdal-$GDALVERSION $GDALOPTS $PROJOPT
make -s -j 2
make install
fi
fi
# change back to travis build dir

View File

@ -0,0 +1,28 @@
#!/bin/sh
set -e
# Create build dir if not exists
if [ ! -d "$PROJBUILD" ]; then
mkdir $PROJBUILD;
fi
if [ ! -d "$PROJINST" ]; then
mkdir $PROJINST;
fi
ls -l $PROJINST
echo "PROJ VERSION: $PROJVERSION"
if [ ! -d "$PROJINST/gdal-$GDALVERSION/share/proj" ]; then
cd $PROJBUILD
wget -q https://download.osgeo.org/proj/proj-$PROJVERSION.tar.gz
tar -xzf proj-$PROJVERSION.tar.gz
cd proj-$PROJVERSION
./configure --prefix=$PROJINST/gdal-$GDALVERSION
make -s -j 2
make install
fi
# change back to travis build dir
cd $TRAVIS_BUILD_DIR

View File

@ -183,6 +183,12 @@ class sdist_multi_gdal(sdist):
shutil.copy('rasterio/_shim21.pyx', 'rasterio/_shim.pyx')
_ = check_output(['cython', '-v', '-f', 'rasterio/_shim.pyx',
'-o', 'rasterio/_shim21.c'])
print(_)
shutil.copy('rasterio/_shim30.pyx', 'rasterio/_shim.pyx')
_ = check_output(['cython', '-v', '-f', 'rasterio/_shim.pyx',
'-o', 'rasterio/_shim30.c'])
print(_)
sdist.run(self)
@ -259,7 +265,9 @@ if os.path.exists("MANIFEST.in") and "clean" not in sys.argv:
"Cython is required to build from a repo.")
# Copy the GDAL version-specific shim module to _shim.pyx.
if gdal_major_version == 2 and gdal_minor_version >= 1:
if gdal_major_version == 3 and gdal_minor_version >= 0:
shutil.copy('rasterio/_shim30.pyx', 'rasterio/_shim.pyx')
elif gdal_major_version == 2 and gdal_minor_version >= 1:
shutil.copy('rasterio/_shim21.pyx', 'rasterio/_shim.pyx')
elif gdal_major_version == 2 and gdal_minor_version == 0:
shutil.copy('rasterio/_shim20.pyx', 'rasterio/_shim.pyx')
@ -317,7 +325,10 @@ else:
'rasterio.shutil', ['rasterio/shutil.c'], **ext_options)]
# Copy the GDAL version-specific shim module to _shim.pyx.
if gdal_major_version == 2 and gdal_minor_version >= 1:
if gdal_major_version == 3 and gdal_minor_version >= 0:
ext_modules.append(
Extension('rasterio._shim', ['rasterio/_shim30.c'], **ext_options))
elif gdal_major_version == 2 and gdal_minor_version >= 1:
ext_modules.append(
Extension('rasterio._shim', ['rasterio/_shim21.c'], **ext_options))
elif gdal_major_version == 2 and gdal_minor_version == 0:

View File

@ -600,3 +600,7 @@ requires_gdal21 = pytest.mark.skipif(
requires_gdal22 = pytest.mark.skipif(
not gdal_version.at_least('2.2'),
reason="Requires GDAL 2.2.x")
requires_gdal_lt_3 = pytest.mark.skipif(
gdal_version.__lt__('3.0'),
reason="Requires GDAL 1.x/2.x")

View File

@ -12,7 +12,7 @@ from rasterio._crs import _CRS
from rasterio.env import env_ctx_if_needed, Env
from rasterio.errors import CRSError
from .conftest import requires_gdal21, requires_gdal22
from .conftest import requires_gdal21, requires_gdal22, requires_gdal_lt_3
# Items like "D_North_American_1983" characterize the Esri dialect
@ -110,6 +110,7 @@ def test_from_wkt_invalid():
_CRS.from_wkt('bogus')
@requires_gdal_lt_3
@pytest.mark.parametrize('projection_string', [ESRI_PROJECTION_STRING])
def test_from_esri_wkt_no_fix(projection_string):
"""Test ESRI CRS morphing with no datum fixing"""

View File

@ -4,7 +4,7 @@ import pytest
from rasterio._env import GDALDataFinder, PROJDataFinder
from .conftest import gdal_version
from .conftest import gdal_version, requires_gdal_lt_3
@pytest.fixture
@ -35,7 +35,7 @@ def mock_debian(tmpdir):
tmpdir.ensure("share/gdal/2.2/pcs.csv")
tmpdir.ensure("share/gdal/2.3/pcs.csv")
tmpdir.ensure("share/gdal/2.4/pcs.csv")
tmpdir.ensure("share/gdal/2.5/pcs.csv")
tmpdir.ensure("share/gdal/3.0/pcs.csv")
tmpdir.ensure("share/proj/epsg")
return tmpdir

View File

@ -15,7 +15,7 @@ from rasterio.crs import CRS
from rasterio.env import env_ctx_if_needed, Env
from rasterio.errors import CRSError
from .conftest import requires_gdal21, requires_gdal22
from .conftest import requires_gdal21, requires_gdal22, requires_gdal_lt_3
# Items like "D_North_American_1983" characterize the Esri dialect
@ -72,6 +72,7 @@ def test_read_epsg():
assert src.crs.to_epsg() == 32618
@requires_gdal_lt_3
def test_read_esri_wkt():
with rasterio.open('tests/data/test_esri_wkt.tif') as src:
assert 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",' in src.crs.wkt
@ -110,7 +111,7 @@ def test_write_3857(tmpdir):
info = subprocess.check_output([
'gdalinfo', dst_path])
# WKT string may vary a bit w.r.t GDAL versions
assert 'PROJCS["WGS 84 / Pseudo-Mercator"' in info.decode('utf-8')
assert '"WGS 84 / Pseudo-Mercator"' in info.decode('utf-8')
def test_write_bogus_fails(tmpdir, profile_rgb_byte_tif):
@ -326,6 +327,7 @@ def test_from_user_input_epsg():
assert 'init' in CRS.from_user_input('epsg:4326')
@requires_gdal_lt_3
@pytest.mark.parametrize('projection_string', [ESRI_PROJECTION_STRING])
def test_from_esri_wkt_no_fix(projection_string):
"""Test ESRI CRS morphing with no datum fixing"""
@ -410,6 +412,7 @@ def test_issue1609_wktext_a():
assert 'PARAMETER["latitude_of_origin",-70]' in wkt
@requires_gdal_lt_3
def test_issue1609_wktext_b():
"""Check on fix of issue 1609"""
dst_proj = {'ellps': 'WGS84',

View File

@ -7,9 +7,11 @@ import pytest
import rasterio
from rasterio._env import GDALDataFinder, PROJDataFinder
from .conftest import requires_gdal_lt_3
from rasterio.rio.main import main_group
@requires_gdal_lt_3
@pytest.mark.wheel
def test_gdal_data():
"""Get GDAL data path from a wheel"""
@ -21,6 +23,7 @@ def test_gdal_data_find_file():
GDALDataFinder().find_file("header.dxf")
@requires_gdal_lt_3
@pytest.mark.wheel
def test_proj_data():
"""Get GDAL data path from a wheel"""
@ -32,6 +35,7 @@ def test_proj_data_has_data():
PROJDataFinder().has_data()
@requires_gdal_lt_3
@pytest.mark.wheel
def test_env_gdal_data():
runner = CliRunner()
@ -40,6 +44,7 @@ def test_env_gdal_data():
assert result.output.strip() == os.path.join(os.path.dirname(rasterio.__file__), 'gdal_data')
@requires_gdal_lt_3
@pytest.mark.wheel
def test_env_proj_data():
runner = CliRunner()