diff --git a/include/mapnik/mapped_memory_cache.hpp b/include/mapnik/mapped_memory_cache.hpp new file mode 100644 index 000000000..b143a6f30 --- /dev/null +++ b/include/mapnik/mapped_memory_cache.hpp @@ -0,0 +1,65 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2011 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +//$Id$ + +#ifndef MEMORY_REGION_CACHE_HPP +#define MEMORY_REGION_CACHE_HPP + +// mapnik +#include +#include +// boost +#include +#include +#include +#include +#include + +#ifdef MAPNIK_THREADSAFE +#include +#endif + +namespace mapnik +{ + +using namespace boost::interprocess; + +typedef boost::shared_ptr mapped_region_ptr; + +struct MAPNIK_DECL mapped_memory_cache : + public singleton , + private boost::noncopyable +{ + friend class CreateStatic; +#ifdef MAPNIK_THREADSAFE + static boost::mutex mutex_; +#endif + static boost::unordered_map cache_; + static bool insert(std::string const& key, mapped_region_ptr); + static boost::optional find(std::string const& key, bool update_cache = false); +}; + +} + +#endif // MAPNIK_MARKER_CACHE_HPP + diff --git a/plugins/input/shape/dbffile.cpp b/plugins/input/shape/dbffile.cpp index e37afb630..90e0fd16e 100644 --- a/plugins/input/shape/dbffile.cpp +++ b/plugins/input/shape/dbffile.cpp @@ -27,7 +27,9 @@ // boost #include #include - +//#include +//#include +#include // stl #include @@ -43,13 +45,21 @@ dbf_file::dbf_file(std::string const& file_name) num_fields_(0), record_length_(0), #ifdef SHAPE_MEMORY_MAPPED_FILE - file_(file_name), + file_(), #else file_(file_name,std::ios::in | std::ios::binary), #endif record_(0) { - if (file_.is_open()) + +#ifdef SHAPE_MEMORY_MAPPED_FILE + boost::optional memory = mapnik::mapped_memory_cache::find(file_name.c_str(),true); + if (memory) + { + file_.buffer(static_cast((*memory)->get_address()),(*memory)->get_size()); + } +#endif + if (file_) { read_header(); } @@ -64,17 +74,13 @@ dbf_file::~dbf_file() bool dbf_file::is_open() { - return file_.is_open(); +#ifdef SHAPE_MEMORY_MAPPED_FILE + return (file_.buffer().second > 0); +#else + return file_.is_open(); +#endif } - -void dbf_file::close() -{ - if (file_ && file_.is_open()) - file_.close(); -} - - int dbf_file::num_records() const { return num_records_; diff --git a/plugins/input/shape/dbffile.hpp b/plugins/input/shape/dbffile.hpp index b6c1f51c2..b3120692b 100644 --- a/plugins/input/shape/dbffile.hpp +++ b/plugins/input/shape/dbffile.hpp @@ -25,9 +25,13 @@ #include // boost +#include #include #include -#include +// +#include +//#include + // stl #include #include @@ -49,7 +53,7 @@ struct field_descriptor }; -class dbf_file +class dbf_file : private boost::noncopyable { private: int num_records_; @@ -57,7 +61,8 @@ private: stream_offset record_length_; std::vector fields_; #ifdef SHAPE_MEMORY_MAPPED_FILE - stream file_; + //stream file_; + boost::interprocess::ibufferstream file_; #else stream file_; #endif @@ -67,7 +72,6 @@ public: dbf_file(const std::string& file_name); ~dbf_file(); bool is_open(); - void close(); int num_records() const; int num_fields() const; field_descriptor const& descriptor(int col) const; @@ -75,8 +79,6 @@ public: std::string string_value(int col) const; void add_attribute(int col, transcoder const& tr, Feature const& f) const throw(); private: - dbf_file(const dbf_file&); - dbf_file& operator=(const dbf_file&); void read_header(); int read_short(); int read_int(); diff --git a/plugins/input/shape/shape_index_featureset.cpp b/plugins/input/shape/shape_index_featureset.cpp index 8177d1d0b..7c735fc32 100644 --- a/plugins/input/shape/shape_index_featureset.cpp +++ b/plugins/input/shape/shape_index_featureset.cpp @@ -24,7 +24,7 @@ #include // boost - +#include #include "shape_index_featureset.hpp" @@ -47,7 +47,8 @@ shape_index_featureset::shape_index_featureset(const filterT& filter, if (index) { #ifdef SHAPE_MEMORY_MAPPED_FILE - shp_index >::query(filter,index->file(),ids_); + //shp_index >::query(filter,index->file(),ids_); + shp_index::query(filter,index->file(),ids_); #else shp_index >::query(filter,index->file(),ids_); #endif diff --git a/plugins/input/shape/shape_io.cpp b/plugins/input/shape/shape_io.cpp index 6f3aa7d6b..69890434f 100644 --- a/plugins/input/shape/shape_io.cpp +++ b/plugins/input/shape/shape_io.cpp @@ -61,12 +61,7 @@ shape_io::shape_io(const std::string& shape_name, bool open_index) } } -shape_io::~shape_io() -{ - shp_.close(); - dbf_.close(); - if (index_) (*index_).close(); -} +shape_io::~shape_io() {} void shape_io::move_to (int pos) { diff --git a/plugins/input/shape/shapefile.hpp b/plugins/input/shape/shapefile.hpp index 8756f638c..0c28170d6 100644 --- a/plugins/input/shape/shapefile.hpp +++ b/plugins/input/shape/shapefile.hpp @@ -27,13 +27,19 @@ #include #include +#include // boost #include #include -#include +#include + +//#include +//#include + +#include #include -#include +//#include #include @@ -128,14 +134,18 @@ struct shape_record }; using namespace boost::iostreams; +using namespace boost::interprocess; class shape_file : boost::noncopyable { public: #ifdef SHAPE_MEMORY_MAPPED_FILE - typedef stream file_source_type; + //typedef stream file_source_type; + typedef ibufferstream file_source_type; typedef shape_record record_type; + //typedef boost::shared_ptr mapped_region_ptr; + //mapped_region_ptr region_; #else typedef stream file_source_type; typedef shape_record record_type; @@ -145,13 +155,23 @@ public: shape_file() {} shape_file(std::string const& file_name) - : + : #ifdef SHAPE_MEMORY_MAPPED_FILE - file_(file_name) -#else + file_() +#else file_(file_name,std::ios::in | std::ios::binary) #endif - {} + { +#ifdef SHAPE_MEMORY_MAPPED_FILE + //file_mapping mapping(file_name.c_str(),read_only); + //region_ = mapped_region_ptr(new mapped_region(mapping, read_only)); + boost::optional memory = mapnik::mapped_memory_cache::find(file_name.c_str(),true); + if (memory) + { + file_.buffer(static_cast((*memory)->get_address()),(*memory)->get_size()); + } +#endif + } ~shape_file() {} @@ -162,19 +182,17 @@ public: inline bool is_open() { +#ifdef SHAPE_MEMORY_MAPPED_FILE + return (file_.buffer().second > 0); +#else return file_.is_open(); - } - - inline void close() - { - if (file_ && file_.is_open()) - file_.close(); +#endif } inline void read_record(record_type& rec) { #ifdef SHAPE_MEMORY_MAPPED_FILE - rec.set_data(file_->data() + file_.tellg()); + rec.set_data(file_.buffer().first + file_.tellg()); file_.seekg(rec.size,std::ios::cur); #else file_.read(rec.get_data(),rec.size); diff --git a/src/SConscript b/src/SConscript index 813d9b777..6a3dae9b4 100644 --- a/src/SConscript +++ b/src/SConscript @@ -143,6 +143,7 @@ source = Split( wkt/wkt_factory.cpp metawriter_inmem.cpp metawriter_factory.cpp + mapped_memory_cache.cpp """ ) diff --git a/src/mapped_memory_cache.cpp b/src/mapped_memory_cache.cpp new file mode 100644 index 000000000..799f2873e --- /dev/null +++ b/src/mapped_memory_cache.cpp @@ -0,0 +1,92 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2011 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +//$Id$ + +// mapnik +#include + +// boost +#include +#include +#include + +namespace mapnik +{ + +boost::unordered_map mapped_memory_cache::cache_; + +bool mapped_memory_cache::insert (std::string const& uri, mapped_region_ptr mem) +{ +#ifdef MAPNIK_THREADSAFE + mutex::scoped_lock lock(mutex_); +#endif + return cache_.insert(std::make_pair(uri,mem)).second; +} + +boost::optional mapped_memory_cache::find(std::string const& uri, bool update_cache) +{ +#ifdef MAPNIK_THREADSAFE + mutex::scoped_lock lock(mutex_); +#endif + typedef boost::unordered_map::const_iterator iterator_type; + boost::optional result; + iterator_type itr = cache_.find(uri); + if (itr != cache_.end()) + { + result.reset(itr->second); + return result; + } + + boost::filesystem::path path(uri); + if (exists(path)) + { + try + { + file_mapping mapping(uri.c_str(),read_only); + mapped_region_ptr region(new mapped_region(mapping,read_only)); + + result.reset(region); + + if (update_cache) + { + cache_.insert(std::make_pair(uri,*result)); + } + return result; + } + catch (...) + { + std::cerr << "Exception caught while loading mapping memory file: " << uri << std::endl; + } + } + else + { + std::cerr << "### WARNING Memory region does not exist file:" << uri << std::endl; + } + return result; +} + +#ifdef MAPNIK_THREADSAFE +boost::mutex mapped_memory_cache::mutex_; +#endif + +} diff --git a/utils/shapeindex/shapeindex.cpp b/utils/shapeindex/shapeindex.cpp index 6e9e9b13f..51e1a5e46 100644 --- a/utils/shapeindex/shapeindex.cpp +++ b/utils/shapeindex/shapeindex.cpp @@ -217,8 +217,7 @@ int main (int argc,char** argv) break; } } - shp.close(); - + clog << " number shapes=" << count << endl; std::fstream file((shapename+".index").c_str(),