+ implement memory mapping using boost::interprocess

+ cache mapped memory regions for re-use

include/mapnik/mapped_memory_cache.hpp

@@ -0,0 +1,65 @@
+/*****************************************************************************
+ * 
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2011 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+//$Id$
+
+#ifndef MEMORY_REGION_CACHE_HPP
+#define MEMORY_REGION_CACHE_HPP
+
+// mapnik
+#include <mapnik/config.hpp>
+#include <mapnik/utils.hpp>
+// boost
+#include <boost/utility.hpp>
+#include <boost/unordered_map.hpp>
+#include <boost/interprocess/mapped_region.hpp>
+#include <boost/shared_ptr.hpp>
+#include <boost/optional.hpp>
+
+#ifdef MAPNIK_THREADSAFE
+#include <boost/thread/mutex.hpp>
+#endif
+
+namespace mapnik
+{
+
+using namespace boost::interprocess;
+
+typedef boost::shared_ptr<mapped_region> mapped_region_ptr;
+
+struct MAPNIK_DECL mapped_memory_cache :
+        public singleton <mapped_memory_cache, CreateStatic>,
+        private boost::noncopyable
+{
+    friend class CreateStatic<mapped_memory_cache>;
+#ifdef MAPNIK_THREADSAFE
+    static boost::mutex mutex_;
+#endif
+    static boost::unordered_map<std::string,mapped_region_ptr> cache_;
+    static bool insert(std::string const& key, mapped_region_ptr);
+    static boost::optional<mapped_region_ptr> find(std::string const& key, bool update_cache = false);
+};
+
+}
+
+#endif // MAPNIK_MARKER_CACHE_HPP
+

plugins/input/shape/dbffile.cpp

@@ -27,7 +27,9 @@
 // boost
 #include <boost/algorithm/string.hpp>
 #include <boost/spirit/include/qi.hpp>
-
+//#include <boost/interprocess/file_mapping.hpp>
+//#include <boost/interprocess/mapped_region.hpp>
+#include <mapnik/mapped_memory_cache.hpp>
 // stl
 #include <string>
 
@@ -43,13 +45,21 @@ dbf_file::dbf_file(std::string const& file_name)
      num_fields_(0),
      record_length_(0),
 #ifdef SHAPE_MEMORY_MAPPED_FILE
-     file_(file_name),
+     file_(),
 #else
      file_(file_name,std::ios::in | std::ios::binary),
 #endif
      record_(0)
 {
-    if (file_.is_open())
+
+#ifdef SHAPE_MEMORY_MAPPED_FILE
+    boost::optional<mapnik::mapped_region_ptr> memory = mapnik::mapped_memory_cache::find(file_name.c_str(),true);
+    if (memory)
+    {
+        file_.buffer(static_cast<char*>((*memory)->get_address()),(*memory)->get_size());
+    }
+#endif 
+    if (file_)
     {
         read_header();
     }
@@ -64,17 +74,13 @@ dbf_file::~dbf_file()
 
 bool dbf_file::is_open()
 {
-    return file_.is_open();
+#ifdef SHAPE_MEMORY_MAPPED_FILE
+        return (file_.buffer().second > 0);
+#else
+        return file_.is_open();
+#endif
 }
 
-
-void dbf_file::close()
-{
-    if (file_ && file_.is_open())
-        file_.close();
-}
-
-
 int dbf_file::num_records() const
 {
     return num_records_;

plugins/input/shape/dbffile.hpp

@@ -25,9 +25,13 @@
 
 #include <mapnik/feature.hpp>
 // boost
+#include <boost/utility.hpp>
 #include <boost/iostreams/stream.hpp>
 #include <boost/iostreams/device/file.hpp>
-#include <boost/iostreams/device/mapped_file.hpp>
+//
+#include <boost/interprocess/streams/bufferstream.hpp>
+//#include <boost/iostreams/device/mapped_file.hpp>
+
 // stl
 #include <vector>
 #include <string>
@@ -49,7 +53,7 @@ struct field_descriptor
 };
 
 
-class dbf_file
+class dbf_file : private boost::noncopyable
 {
 private:
     int num_records_;
@@ -57,7 +61,8 @@ private:
     stream_offset record_length_;
     std::vector<field_descriptor> fields_;
 #ifdef SHAPE_MEMORY_MAPPED_FILE
-    stream<mapped_file_source> file_;
+    //stream<mapped_file_source> file_;
+    boost::interprocess::ibufferstream file_;
 #else
     stream<file_source> file_;
 #endif
@@ -67,7 +72,6 @@ public:
     dbf_file(const std::string& file_name);
     ~dbf_file();
     bool is_open();
-    void close();
     int num_records() const;
     int num_fields() const;
     field_descriptor const& descriptor(int col) const;
@@ -75,8 +79,6 @@ public:
     std::string string_value(int col) const;
     void add_attribute(int col, transcoder const& tr, Feature const& f) const throw();
 private:
-    dbf_file(const dbf_file&);
-    dbf_file& operator=(const dbf_file&);
     void read_header();
     int read_short();
     int read_int();

plugins/input/shape/shape_index_featureset.cpp

@@ -24,7 +24,7 @@
 
 #include <mapnik/feature_factory.hpp>
 // boost
-
+#include <boost/interprocess/streams/bufferstream.hpp>
 
 #include "shape_index_featureset.hpp"
 
@@ -47,7 +47,8 @@ shape_index_featureset<filterT>::shape_index_featureset(const filterT& filter,
     if (index)
     {
 #ifdef SHAPE_MEMORY_MAPPED_FILE
-        shp_index<filterT,stream<mapped_file_source> >::query(filter,index->file(),ids_);
+        //shp_index<filterT,stream<mapped_file_source> >::query(filter,index->file(),ids_);
+        shp_index<filterT,boost::interprocess::ibufferstream>::query(filter,index->file(),ids_);
 #else
         shp_index<filterT,stream<file_source> >::query(filter,index->file(),ids_);
 #endif

plugins/input/shape/shape_io.cpp

@@ -61,12 +61,7 @@ shape_io::shape_io(const std::string& shape_name, bool open_index)
     }
 }
 
-shape_io::~shape_io()
-{
-   shp_.close();
-   dbf_.close();
-   if (index_) (*index_).close();
-}
+shape_io::~shape_io() {}
 
 void shape_io::move_to (int pos)
 {

plugins/input/shape/shapefile.hpp

@@ -27,13 +27,19 @@
 
 #include <mapnik/global.hpp>
 #include <mapnik/box2d.hpp>
+#include <mapnik/mapped_memory_cache.hpp>
 // boost
 #include <boost/utility.hpp>
 #include <boost/cstdint.hpp>
-#include <boost/iostreams/stream.hpp>
 
+#include <boost/interprocess/streams/bufferstream.hpp>
+
+//#include <boost/interprocess/file_mapping.hpp>
+//#include <boost/interprocess/mapped_region.hpp>
+
+#include <boost/iostreams/stream.hpp>
 #include <boost/iostreams/device/file.hpp>
-#include <boost/iostreams/device/mapped_file.hpp>
+//#include <boost/iostreams/device/mapped_file.hpp>
 
 #include <cstring>
 
@@ -128,14 +134,18 @@ struct shape_record
 };
 
 using namespace boost::iostreams;
+using namespace boost::interprocess;
 
 class shape_file : boost::noncopyable
 {  
 public:
 
 #ifdef SHAPE_MEMORY_MAPPED_FILE
-    typedef stream<mapped_file_source> file_source_type;
+    //typedef stream<mapped_file_source> file_source_type;
+    typedef ibufferstream file_source_type;
     typedef shape_record<MappedRecordTag> record_type;
+    //typedef boost::shared_ptr<mapped_region> mapped_region_ptr;    
+    //mapped_region_ptr region_;
 #else
     typedef stream<file_source> file_source_type;
     typedef shape_record<RecordTag> record_type;
@@ -145,13 +155,23 @@ public:
     shape_file() {}
     
     shape_file(std::string  const& file_name)
-        : 
+        :
 #ifdef SHAPE_MEMORY_MAPPED_FILE
-        file_(file_name)
-#else
+        file_()
+#else  
         file_(file_name,std::ios::in | std::ios::binary)
 #endif
-    {}
+    {
+#ifdef SHAPE_MEMORY_MAPPED_FILE
+        //file_mapping mapping(file_name.c_str(),read_only);
+        //region_ = mapped_region_ptr(new mapped_region(mapping, read_only));
+        boost::optional<mapnik::mapped_region_ptr> memory = mapnik::mapped_memory_cache::find(file_name.c_str(),true);
+        if (memory)
+        {
+            file_.buffer(static_cast<char*>((*memory)->get_address()),(*memory)->get_size());
+        }
+#endif        
+    }
     
     ~shape_file() {}
 
@@ -162,19 +182,17 @@ public:
     
     inline bool is_open()
     {
+#ifdef SHAPE_MEMORY_MAPPED_FILE
+        return (file_.buffer().second > 0);
+#else
         return file_.is_open();
-    }
- 
-    inline void close()
-    {
-        if (file_ && file_.is_open())
-            file_.close();
+#endif
     }
     
     inline void read_record(record_type& rec)
     {
 #ifdef SHAPE_MEMORY_MAPPED_FILE
-        rec.set_data(file_->data() + file_.tellg());
+        rec.set_data(file_.buffer().first + file_.tellg());
         file_.seekg(rec.size,std::ios::cur);
 #else
         file_.read(rec.get_data(),rec.size);

src/SConscript

@@ -143,6 +143,7 @@ source = Split(
     wkt/wkt_factory.cpp
     metawriter_inmem.cpp
     metawriter_factory.cpp
+    mapped_memory_cache.cpp
     """   
     )
 

src/mapped_memory_cache.cpp

@@ -0,0 +1,92 @@
+/*****************************************************************************
+ * 
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2011 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+//$Id$
+
+// mapnik
+#include <mapnik/mapped_memory_cache.hpp>
+
+// boost
+#include <boost/assert.hpp>
+#include <boost/interprocess/file_mapping.hpp>
+#include <boost/filesystem/operations.hpp>
+
+namespace mapnik 
+{
+
+boost::unordered_map<std::string, mapped_region_ptr> mapped_memory_cache::cache_;
+
+bool mapped_memory_cache::insert (std::string const& uri, mapped_region_ptr mem)
+{
+#ifdef MAPNIK_THREADSAFE
+    mutex::scoped_lock lock(mutex_);
+#endif
+    return cache_.insert(std::make_pair(uri,mem)).second;
+}
+
+boost::optional<mapped_region_ptr> mapped_memory_cache::find(std::string const& uri, bool update_cache)
+{
+#ifdef MAPNIK_THREADSAFE
+    mutex::scoped_lock lock(mutex_);
+#endif
+    typedef boost::unordered_map<std::string, mapped_region_ptr>::const_iterator iterator_type;
+    boost::optional<mapped_region_ptr> result;
+    iterator_type itr = cache_.find(uri);
+    if (itr != cache_.end())
+    {
+        result.reset(itr->second);
+        return result;
+    }
+    
+    boost::filesystem::path path(uri);
+    if (exists(path))
+    {
+        try
+        {
+            file_mapping mapping(uri.c_str(),read_only);
+            mapped_region_ptr region(new mapped_region(mapping,read_only));
+            
+            result.reset(region);
+            
+            if (update_cache)
+            {
+                cache_.insert(std::make_pair(uri,*result));
+            }
+            return result;
+        }
+        catch (...)
+        {
+            std::cerr << "Exception caught while loading mapping memory file: " << uri << std::endl;
+        }
+    }
+    else
+    {
+        std::cerr << "### WARNING Memory region does not exist file:" << uri << std::endl;
+    }
+    return result;
+}
+
+#ifdef MAPNIK_THREADSAFE
+boost::mutex mapped_memory_cache::mutex_;
+#endif
+
+}

utils/shapeindex/shapeindex.cpp

@@ -217,8 +217,7 @@ int main (int argc,char** argv)
                 break;
             }  
         } 
-        shp.close();
-  
+        
         clog << " number shapes=" << count << endl;  
     
         std::fstream file((shapename+".index").c_str(),