diff --git a/CHANGES.txt b/CHANGES.txt index c6e2f3a4..c78d6625 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,6 +18,8 @@ Deprecations: Changes: +- The Python file VSI plugin is now created and installed at the same time + other GDAL drivers are registered (#2435). - Various optimizations for sample_gen() (#2338). - Update Cython property declarations to modern style (#2316). - The rasterio.path module has been moved to rasterio._path and all its member diff --git a/rasterio/_env.pyx b/rasterio/_env.pyx index c0b95bd8..f77efb21 100644 --- a/rasterio/_env.pyx +++ b/rasterio/_env.pyx @@ -20,6 +20,7 @@ import threading from rasterio._base cimport _safe_osr_release from rasterio._err import CPLE_BaseError from rasterio._err cimport exc_wrap_ogrerr, exc_wrap_int +from rasterio._filepath cimport install_filepath_plugin, uninstall_filepath_plugin from libc.stdio cimport stderr @@ -61,6 +62,8 @@ except ImportError: cdef bint is_64bit = sys.maxsize > 2 ** 32 +cdef VSIFilesystemPluginCallbacksStruct* filepath_plugin = NULL + cdef void log_error(CPLErr err_class, int err_no, const char* msg) with gil: """Send CPL errors to Python's logger. @@ -351,11 +354,13 @@ cdef class GDALEnv(ConfigEnv): # lock when the environment starts, and the inner avoids a # potential race condition. if not self._have_registered_drivers: - with threading.Lock(): - if not self._have_registered_drivers: + with threading.Lock(): + + if not self._have_registered_drivers: GDALAllRegister() OGRRegisterAll() + install_filepath_plugin(filepath_plugin) if 'GDAL_DATA' in os.environ: log.debug("GDAL_DATA found in environment.") diff --git a/rasterio/_filepath.pxd b/rasterio/_filepath.pxd new file mode 100644 index 00000000..8d0f3c70 --- /dev/null +++ b/rasterio/_filepath.pxd @@ -0,0 +1,6 @@ +include "gdal.pxi" + +cdef int install_filepath_plugin(VSIFilesystemPluginCallbacksStruct *callbacks_struct) +cdef void uninstall_filepath_plugin(VSIFilesystemPluginCallbacksStruct *callbacks_struct) + + diff --git a/rasterio/_filepath.pyx b/rasterio/_filepath.pyx index 4363302c..3b317890 100644 --- a/rasterio/_filepath.pyx +++ b/rasterio/_filepath.pyx @@ -70,43 +70,33 @@ cdef bytes FILESYSTEM_PREFIX_BYTES = FILESYSTEM_PREFIX.encode("ascii") # an entry to this dictionary. GDAL will then Open the path later. cdef _FILESYSTEM_INFO = {} + +cdef int install_filepath_plugin(VSIFilesystemPluginCallbacksStruct *callbacks_struct): + """Install handlers for python file-like objects if it isn't already installed.""" + callbacks_struct = VSIAllocFilesystemPluginCallbacksStruct() + callbacks_struct.open = filepath_open + callbacks_struct.tell = filepath_tell + callbacks_struct.seek = filepath_seek + callbacks_struct.read = filepath_read + callbacks_struct.close = filepath_close + callbacks_struct.pUserData = _FILESYSTEM_INFO + + if VSIFileManager.GetHandler("") == VSIFileManager.GetHandler(FILESYSTEM_PREFIX_BYTES): + log.debug("Installing FilePath filesystem handler plugin...") + return VSIInstallPluginHandler(FILESYSTEM_PREFIX_BYTES, callbacks_struct) + else: + return 0 + + +cdef void uninstall_filepath_plugin(VSIFilesystemPluginCallbacksStruct *callbacks_struct): + if callbacks_struct is not NULL: + callbacks_struct.pUserData = NULL + VSIFreeFilesystemPluginCallbacksStruct(callbacks_struct) + callbacks_struct = NULL + + ## Filesystem Functions -# cdef int filepath_stat(void *pUserData, const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags) with gil: -# # Optional -# printf("stat\n") -# -# -# cdef int filepath_unlink(void *pUserData, const char *pszFilename) with gil: -# # Optional -# printf("unlink\n") -# -# -# cdef int filepath_rename(void *pUserData, const char *oldpath, const char *newpath) with gil: -# # Optional -# printf("rename\n") -# -# -# cdef int filepath_mkdir(void *pUserData, const char *pszDirname, long nMode) with gil: -# # Optional -# printf("mkdir\n") -# -# -# cdef int filepath_rmdir(void *pUserData, const char *pszDirname) with gil: -# # Optional -# printf("rmdir\n") -# -# -# cdef char** filepath_read_dir(void *pUserData, const char *pszDirname, int nMaxFiles) with gil: -# # Optional -# printf("read_dir\n") -# -# -# cdef char** filepath_siblings_files(void *pUserData, const char *pszDirname) with gil: -# # Optional (GDAL 3.2+) -# printf("siblings_files\n") - - cdef void* filepath_open(void *pUserData, const char *pszFilename, const char *pszAccess) with gil: """Access existing open file-like object in the virtual filesystem. @@ -137,7 +127,6 @@ cdef void* filepath_open(void *pUserData, const char *pszFilename, const char *p ## File functions - cdef vsi_l_offset filepath_tell(void *pFile) with gil: cdef object file_wrapper = pFile cdef object file_obj = file_wrapper._file_obj @@ -163,34 +152,6 @@ cdef size_t filepath_read(void *pFile, void *pBuffer, size_t nSize, size_t nCoun return (num_bytes / nSize) -# cdef int filepath_read_multi_range(void *pFile, int, void **ppData, const vsi_l_offset *panOffsets, const size_t *panSizes) with gil: -# # Optional -# print("read_multi_range") -# -# -# cdef VSIRangeStatus filepath_get_range_status(void *pFile, vsi_l_offset nOffset, vsi_l_offset nLength) with gil: -# # Optional -# print("get_range_status") -# -# -# cdef int filepath_eof(void *pFile) with gil: -# # Mandatory? -# print("eof") -# -# -# cdef size_t filepath_write(void *pFile, const void *pBuffer, size_t nSize, size_t nCount) with gil: -# print("write") -# -# -# cdef int filepath_flush(void *pFile) with gil: -# # Optional -# print("flush") -# -# -# cdef int filepath_truncate(void *pFile, vsi_l_offset nNewSize) with gil: -# print("truncate") - - cdef int filepath_close(void *pFile) except -1 with gil: # Optional cdef object file_wrapper = pFile @@ -198,38 +159,9 @@ cdef int filepath_close(void *pFile) except -1 with gil: return 0 -cdef int install_rasterio_filepath_plugin(VSIFilesystemPluginCallbacksStruct *callbacks_struct): - """Install handlers for python file-like objects if it isn't already installed.""" - cdef int install_status - if VSIFileManager.GetHandler("") == VSIFileManager.GetHandler(FILESYSTEM_PREFIX_BYTES): - log.debug("Installing FilePath filesystem handler plugin...") - install_status = VSIInstallPluginHandler(FILESYSTEM_PREFIX_BYTES, callbacks_struct) - return install_status - return 0 - - cdef class FilePathBase: """Base for a BytesIO-like class backed by a Python file-like object.""" - cdef VSIFilesystemPluginCallbacksStruct* _vsif - - def __cinit__(self, file_or_bytes, *args, **kwargs): - self._vsif = VSIAllocFilesystemPluginCallbacksStruct() - # pUserData will be set later - self._vsif.open = filepath_open - - self._vsif.tell = filepath_tell - self._vsif.seek = filepath_seek - self._vsif.read = filepath_read - # self._vsif.eof = filepath_eof - self._vsif.close = filepath_close - - def __dealloc__(self): - if self._vsif is not NULL: - self._vsif.pUserData = NULL - VSIFreeFilesystemPluginCallbacksStruct(self._vsif) - self._vsif = NULL - def __init__(self, filelike_obj, dirname=None, filename=None): """A file in an in-memory filesystem. @@ -261,11 +193,7 @@ cdef class FilePathBase: self._file_obj = filelike_obj self.mode = "r" self.closed = False - - # TODO: Error checking _FILESYSTEM_INFO[self._filepath_path] = self - self._vsif.pUserData = _FILESYSTEM_INFO - install_rasterio_filepath_plugin(self._vsif) def exists(self): """Test if the in-memory file exists.