Merge branch 'master' of github.com:mapnik/mapnik into expr-v2

Makefile

@@ -14,10 +14,10 @@ endif
 all: mapnik
 
 install:
-	@python scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1 install
+	python scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1 install
 
 mapnik:
-	@python scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1
+	python scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1
 
 clean:
 	@python scons/scons.py -j$(JOBS) -c --config=cache --implicit-cache --max-drift=1

SConstruct

@@ -767,16 +767,20 @@ def FindBoost(context, prefixes, thread_flag):
     if BOOST_LIB_DIR:
         msg += '\nFound boost libs: %s' % BOOST_LIB_DIR
         env['BOOST_LIBS'] = BOOST_LIB_DIR
-    else:
+    elif not env['BOOST_LIBS']:
         env['BOOST_LIBS'] = '/usr/' + env['LIBDIR_SCHEMA']
         msg += '\nUsing default boost lib dir: %s' % env['BOOST_LIBS']
+    else:
+        msg += '\nUsing boost lib dir: %s' % env['BOOST_LIBS']
 
     if BOOST_INCLUDE_DIR:
         msg += '\nFound boost headers: %s' % BOOST_INCLUDE_DIR
         env['BOOST_INCLUDES'] = BOOST_INCLUDE_DIR
-    else:
+    elif not env['BOOST_INCLUDES']:
         env['BOOST_INCLUDES'] = '/usr/include'
         msg += '\nUsing default boost include dir: %s' % env['BOOST_INCLUDES']
+    else:
+        msg += '\nUsing boost include dir: %s' % env['BOOST_INCLUDES']
 
     if not env['BOOST_TOOLKIT'] and not env['BOOST_ABI'] and not env['BOOST_VERSION']:
         if BOOST_APPEND:

benchmark/run

@@ -10,6 +10,7 @@ function run {
     ${BASE}/$1 --threads $2 --iterations $(expr $3 / $2);
 }
 
+run test_array_allocation 20 100000
 run test_png_encoding1 10 1000
 run test_png_encoding2 10 50
 run test_to_string1 10 100000

benchmark/test_array_allocation.cpp

@@ -0,0 +1,310 @@
+#include "bench_framework.hpp"
+#include <cstring>
+#include <cstdlib>
+#include <stdexcept>
+#include <array>
+#include <valarray>
+
+// http://stackoverflow.com/questions/17347254/why-is-allocation-and-deallocation-of-stdvector-slower-than-dynamic-array-on-m
+
+#define FULL_ZERO_CHECK
+
+inline void ensure_zero(uint8_t * data, uint32_t size) {
+#ifdef FULL_ZERO_CHECK
+    for (std::size_t i=0;i<size;++i) {
+        if (data[i] != 0) {
+            throw std::runtime_error("found non zero value");
+        }
+    }
+#else
+    if (data[0] != 0) {
+        throw std::runtime_error("found non zero value");
+    }
+#endif
+}
+
+class test1 : public benchmark::test_case
+{
+public:
+    uint32_t size_;
+    std::vector<uint8_t> array_;
+    test1(mapnik::parameters const& params)
+     : test_case(params),
+       size_(*params.get<mapnik::value_integer>("size",256*256)),
+       array_(size_,0) { }
+    bool validate() const
+    {
+        return true;
+    }
+    void operator()() const
+    {
+         for (std::size_t i=0;i<iterations_;++i) {
+             // NOTE: sizeof(uint8_t) == 1
+             uint8_t *data = (uint8_t *)malloc(sizeof(uint8_t)*size_);
+             memcpy(data, &array_[0], size_);
+             ensure_zero(data,size_);
+             free(data);
+         }
+    }
+};
+
+class test1b : public benchmark::test_case
+{
+public:
+    uint32_t size_;
+    std::vector<uint8_t> array_;
+    test1b(mapnik::parameters const& params)
+     : test_case(params),
+       size_(*params.get<mapnik::value_integer>("size",256*256)),
+       array_(size_,0) { }
+    bool validate() const
+    {
+        return true;
+    }
+    void operator()() const
+    {
+         for (std::size_t i=0;i<iterations_;++i) {
+             // NOTE: sizeof(uint8_t) == 1
+             uint8_t *data = (uint8_t *)malloc(sizeof(uint8_t)*size_);
+             memset(data, 0, sizeof(uint8_t)*size_);
+             ensure_zero(data,size_);
+             free(data);
+         }
+    }
+};
+
+class test2 : public benchmark::test_case
+{
+public:
+    uint32_t size_;
+    std::vector<uint8_t> array_;
+    test2(mapnik::parameters const& params)
+     : test_case(params),
+       size_(*params.get<mapnik::value_integer>("size",256*256)),
+       array_(size_,0) { }
+    bool validate() const
+    {
+        return true;
+    }
+    void operator()() const
+    {
+         for (std::size_t i=0;i<iterations_;++i) {
+             uint8_t * data = static_cast<uint8_t*>(::operator new(sizeof(uint8_t)*size_));
+             memcpy(data, &array_[0], size_);
+             ensure_zero(data,size_);
+             ::operator delete(data),data=0;
+         }
+    }
+};
+
+class test3 : public benchmark::test_case
+{
+public:
+    uint32_t size_;
+    std::vector<uint8_t> array_;
+    test3(mapnik::parameters const& params)
+     : test_case(params),
+       size_(*params.get<mapnik::value_integer>("size",256*256)),
+       array_(size_,0) { }
+    bool validate() const
+    {
+        return true;
+    }
+    void operator()() const
+    {
+         for (std::size_t i=0;i<iterations_;++i) {
+             std::vector<uint8_t> data(size_);
+             ensure_zero(&data[0],data.size());
+         }
+    }
+};
+
+
+class test3b : public benchmark::test_case
+{
+public:
+    uint32_t size_;
+    std::vector<uint8_t> array_;
+    test3b(mapnik::parameters const& params)
+     : test_case(params),
+       size_(*params.get<mapnik::value_integer>("size",256*256)),
+       array_(size_,0) { }
+    bool validate() const
+    {
+        return true;
+    }
+    void operator()() const
+    {
+         for (std::size_t i=0;i<iterations_;++i) {
+             std::vector<uint8_t> data(0);
+             data.resize(size_,0);
+             ensure_zero(&data[0],data.size());
+         }
+    }
+};
+
+
+class test3c : public benchmark::test_case
+{
+public:
+    uint32_t size_;
+    std::vector<uint8_t> array_;
+    test3c(mapnik::parameters const& params)
+     : test_case(params),
+       size_(*params.get<mapnik::value_integer>("size",256*256)),
+       array_(size_,0) { }
+    bool validate() const
+    {
+        return true;
+    }
+    void operator()() const
+    {
+         for (std::size_t i=0;i<iterations_;++i) {
+             std::vector<uint8_t> data(0);
+             data.assign(size_,0);
+             ensure_zero(&data[0],data.size());
+         }
+    }
+};
+
+class test4 : public benchmark::test_case
+{
+public:
+    uint32_t size_;
+    std::vector<uint8_t> array_;
+    test4(mapnik::parameters const& params)
+     : test_case(params),
+       size_(*params.get<mapnik::value_integer>("size",256*256)),
+       array_(size_,0) { }
+    bool validate() const
+    {
+        return true;
+    }
+    void operator()() const
+    {
+         for (std::size_t i=0;i<iterations_;++i) {
+             uint8_t *data = (uint8_t *)calloc(size_,sizeof(uint8_t));
+             ensure_zero(data,size_);
+             free(data);
+         }
+    }
+};
+
+class test5 : public benchmark::test_case
+{
+public:
+    uint32_t size_;
+    std::vector<uint8_t> array_;
+    test5(mapnik::parameters const& params)
+     : test_case(params),
+       size_(*params.get<mapnik::value_integer>("size",256*256)),
+       array_(size_,0) { }
+    bool validate() const
+    {
+        return true;
+    }
+    void operator()() const
+    {
+         for (std::size_t i=0;i<iterations_;++i) {
+             std::string data(array_.begin(),array_.end());
+             ensure_zero((uint8_t *)&data[0],size_);
+         }
+    }
+};
+
+class test5b : public benchmark::test_case
+{
+public:
+    uint32_t size_;
+    std::vector<char> array_;
+    test5b(mapnik::parameters const& params)
+     : test_case(params),
+       size_(*params.get<mapnik::value_integer>("size",256*256)),
+       array_(size_,0) { }
+    bool validate() const
+    {
+        return true;
+    }
+    void operator()() const
+    {
+         for (std::size_t i=0;i<iterations_;++i) {
+             std::string data(&array_[0]);
+             ensure_zero((uint8_t *)&data[0],size_);
+         }
+    }
+};
+
+// C++14 dynarray<T>
+// http://isocpp.org/blog/2013/04/trip-report-iso-c-spring-2013-meeting
+// http://lists.cs.uiuc.edu/pipermail/cfe-commits/Week-of-Mon-20130909/088700.html
+// http://stackoverflow.com/questions/17303902/any-alternative-to-stddynarray-presently-available
+
+class test6 : public benchmark::test_case
+{
+public:
+    uint32_t size_;
+    std::vector<uint8_t> array_;
+    test6(mapnik::parameters const& params)
+     : test_case(params),
+       size_(*params.get<mapnik::value_integer>("size",256*256)),
+       array_(size_,0) { }
+    bool validate() const
+    {
+        return true;
+    }
+    void operator()() const
+    {
+         for (std::size_t i=0;i<iterations_;++i) {
+             std::valarray<uint8_t> data(size_,0);
+             ensure_zero(&data[0],size_);
+         }
+    }
+};
+
+int main(int argc, char** argv)
+{
+    mapnik::parameters params;
+    benchmark::handle_args(argc,argv,params);
+    {
+        test4 test_runner4(params);
+        run(test_runner4,"calloc");
+    }
+    {
+        test1 test_runner(params);
+        run(test_runner,"malloc/memcpy");
+    }
+    {
+        test1b test_runner(params);
+        run(test_runner,"malloc/memset");
+
+    }
+    {
+        test2 test_runner(params);
+        run(test_runner,"new");
+    }
+    {
+        test3 test_runner(params);
+        run(test_runner,"vector(N)");
+    }
+    {
+        test3b test_runner(params);
+        run(test_runner,"vector/resize");
+    }
+    {
+        test3c test_runner(params);
+        run(test_runner,"vector/assign");
+    }
+    {
+        test5 test_runner(params);
+        run(test_runner,"std::string range");
+    }
+    {
+        test5 test_runner(params);
+        run(test_runner,"std::string &[0]");
+    }
+    {
+        test5 test_runner(params);
+        run(test_runner,"valarray");
+    }
+    return 0;
+}

include/mapnik/datasource_cache.hpp

@@ -35,6 +35,7 @@
 
 // stl
 #include <map>
+#include <set>
 
 namespace mapnik {
 
@@ -56,7 +57,7 @@ private:
     ~datasource_cache();
     std::map<std::string,std::shared_ptr<PluginInfo> > plugins_;
     bool registered_;
-    std::vector<std::string> plugin_directories_;
+    std::set<std::string> plugin_directories_;
 };
 }
 

include/mapnik/hextree.hpp

@@ -33,6 +33,7 @@
 
 // stl
 #include <vector>
+#include <cstring>
 #include <set>
 #include <algorithm>
 #include <cmath>
@@ -68,7 +69,7 @@ class hextree : private mapnik::noncopyable
               pixel_count(0),
               children_count(0)
         {
-            memset(&children_[0],0,sizeof(children_));
+            std::memset(&children_[0],0,sizeof(children_));
         }
 
         ~node ()

include/mapnik/octree.hpp

@@ -30,6 +30,7 @@
 
 // stl
 #include <vector>
+#include <cstring>
 #include <deque>
 #include <algorithm>
 
@@ -62,7 +63,7 @@ class octree : private mapnik::noncopyable
                children_count(0),
                index(0)
         {
-            memset(&children_[0],0,sizeof(children_));
+            std::memset(&children_[0],0,sizeof(children_));
         }
 
         ~node()

include/mapnik/webp_io.hpp

@@ -24,6 +24,7 @@
 #define MAPNIK_WEBP_IO_HPP
 
 // mapnik
+#include <mapnik/image_data.hpp>
 #include <mapnik/util/conversions.hpp>
 
 // webp

plugins/input/ogr/ogr_featureset.cpp

@@ -68,7 +68,7 @@ ogr_featureset::ogr_featureset(mapnik::context_ptr const& ctx,
       layer_(layer),
       layerdef_(layer.GetLayerDefn()),
       tr_(new transcoder(encoding)),
-      fidcolumn_(layer_.GetFIDColumn()),
+      fidcolumn_(layer_.GetFIDColumn()), // TODO - unused
       count_(0)
 {
     layer_.SetSpatialFilterRect (extent.minx(),
@@ -83,6 +83,13 @@ ogr_featureset::~ogr_featureset()
 
 feature_ptr ogr_featureset::next()
 {
+    if (count_ == 0)
+    {
+        // Reset the layer reading on the first feature read
+        // this is a hack, but needed due to https://github.com/mapnik/mapnik/issues/2048
+        // Proper solution is to avoid storing layer state in featureset
+        layer_.ResetReading();
+    }
     OGRFeature *poFeature;
     while ((poFeature = layer_.GetNextFeature()) != nullptr)
     {

plugins/input/rasterlite/rasterlite_featureset.cpp

@@ -31,8 +31,7 @@
 #include <mapnik/feature.hpp>
 #include <mapnik/feature_factory.hpp>
 
-// boost
-
+#include <cstring>
 
 using mapnik::coord2d;
 using mapnik::box2d;
@@ -122,7 +121,7 @@ feature_ptr rasterlite_featureset::get_feature(mapnik::query const& q)
                 unsigned char* raster_data = static_cast<unsigned char*>(raster);
                 unsigned char* image_data = image.getBytes();
 
-                memcpy (image_data, raster_data, size);
+                std::memcpy(image_data, raster_data, size);
 
                 feature->set_raster(rasterp);
 

plugins/input/shape/dbfile.cpp

@@ -38,6 +38,7 @@
 
 // stl
 #include <string>
+#include <cstring>
 #include <stdexcept>
 
 dbf_file::dbf_file()
@@ -223,7 +224,7 @@ void dbf_file::read_header()
         skip(22);
         std::streampos offset=0;
         char name[11];
-        memset(&name,0,11);
+        std::memset(&name,0,11);
         fields_.reserve(num_fields_);
         for (int i=0;i<num_fields_;++i)
         {

src/datasource_cache.cpp

@@ -165,8 +165,7 @@ void datasource_cache::register_datasources(std::string const& str)
 #ifdef MAPNIK_THREADSAFE
     mapnik::scoped_lock lock(mutex_);
 #endif
-    // TODO - only push unique paths
-    plugin_directories_.push_back(str);
+    plugin_directories_.insert(str);
     if (mapnik::util::exists(str) && mapnik::util::is_directory(str))
     {
         boost::filesystem::directory_iterator end_itr;
@@ -194,9 +193,15 @@ void datasource_cache::register_datasources(std::string const& str)
 
 bool datasource_cache::register_datasource(std::string const& filename)
 {
-    bool success = false;
     try
     {
+        if (!mapnik::util::exists(filename))
+        {
+            MAPNIK_LOG_ERROR(datasource_cache)
+                    << "Cannot load '"
+                    << filename << "' (plugin does not exist)";
+            return false;
+        }
         std::shared_ptr<PluginInfo> plugin = std::make_shared<PluginInfo>(filename,"datasource_name");
         if (plugin->valid())
         {
@@ -208,11 +213,13 @@ bool datasource_cache::register_datasource(std::string const& filename)
             }
             else
             {
-                plugins_.insert(std::make_pair(plugin->name(),plugin));
-                MAPNIK_LOG_DEBUG(datasource_cache)
-                        << "datasource_cache: Registered="
-                        << plugin->name();
-                success = true;
+                if (plugins_.insert(std::make_pair(plugin->name(),plugin)).second)
+                {
+                    MAPNIK_LOG_ERROR(datasource_cache)
+                            << "datasource_cache: Registered="
+                            << plugin->name();
+                    return true;
+                }
             }
         }
         else
@@ -228,7 +235,7 @@ bool datasource_cache::register_datasource(std::string const& filename)
                 << "Exception caught while loading plugin library: "
                 << filename << " (" << ex.what() << ")";
     }
-    return success;
+    return false;
 }
 
 }

src/png_reader.cpp

@@ -35,6 +35,7 @@ extern "C"
 #include <boost/iostreams/stream.hpp>
 
 // stl
+#include <cstring>
 #include <memory>
 
 namespace mapnik
@@ -158,7 +159,7 @@ template <typename T>
 void png_reader<T>::init()
 {
     png_byte header[8];
-    memset(header,0,8);
+    std::memset(header,0,8);
     stream_.read(reinterpret_cast<char*>(header),8);
     if ( stream_.gcount() != 8)
     {

tests/cpp_tests/agg_blend_src_over_test.cpp

@@ -1,6 +1,7 @@
 #include <boost/detail/lightweight_test.hpp>
 #include <iostream>
 #include <cstdio>
+#include <cstring>
 #include <sstream>
 #include <string>
 #include <vector>
@@ -32,7 +33,7 @@ color blend(color const& source, color const& dest, unsigned cover=255)
     dest_pre.premultiply();
 
     unsigned char* buffer = new unsigned char[size*size*stride];
-    memset(buffer, 0, size*size*stride);
+    std::memset(buffer, 0, size*size*stride);
     buffer[0] = dest_pre.r;
     buffer[1] = dest_pre.g;
     buffer[2] = dest_pre.b;
@@ -62,7 +63,7 @@ color normal_blend(color const& source, color const& dest, unsigned cover=255)
     dest_pre.premultiply();
     // source buffer
     unsigned char* source_buffer = new unsigned char[size*size*stride];
-    memset(source_buffer, 0, size*size*stride);
+    std::memset(source_buffer, 0, size*size*stride);
     source_buffer[0] = source_pre.r;
     source_buffer[1] = source_pre.g;
     source_buffer[2] = source_pre.b;
@@ -72,7 +73,7 @@ color normal_blend(color const& source, color const& dest, unsigned cover=255)
 
     // destination buffer
     unsigned char* dest_buffer = new unsigned char[size*size*stride];
-    memset(dest_buffer, 0, size*size*stride);
+    std::memset(dest_buffer, 0, size*size*stride);
     dest_buffer[0] = dest_pre.r;
     dest_buffer[1] = dest_pre.g;
     dest_buffer[2] = dest_pre.b;

utils/shapeindex/quadtree.hpp

@@ -42,7 +42,7 @@ struct quadtree_node
     quadtree_node(const box2d<double>& ext)
         : ext_(ext),data_()
     {
-        memset(children_,0,sizeof(quadtree_node<T>*)*4);
+        std::memset(children_,0,sizeof(quadtree_node<T>*)*4);
     }
 
     ~quadtree_node()
@@ -118,7 +118,7 @@ public:
     void write(std::ostream& out)
     {
         char header[16];
-        memset(header,0,16);
+        std::memset(header,0,16);
         header[0]='m';
         header[1]='a';
         header[2]='p';
@@ -205,10 +205,10 @@ private:
             int shape_count=node->data_.size();
             int recsize=sizeof(box2d<double>) + 3 * sizeof(int) + shape_count * sizeof(T);
             char* node_record=new char[recsize];
-            memset(node_record,0,recsize);
-            memcpy(node_record,&offset,4);
-            memcpy(node_record+4,&node->ext_,sizeof(box2d<double>));
-            memcpy(node_record+36,&shape_count,4);
+            std::memset(node_record,0,recsize);
+            std::memcpy(node_record,&offset,4);
+            std::memcpy(node_record+4,&node->ext_,sizeof(box2d<double>));
+            std::memcpy(node_record+36,&shape_count,4);
             for (int i=0;i<shape_count;++i)
             {
                 memcpy(node_record + 40 + i * sizeof(T),&(node->data_[i]),sizeof(T));
@@ -221,7 +221,7 @@ private:
                     ++num_subnodes;
                 }
             }
-            memcpy(node_record + 40 + shape_count * sizeof(T),&num_subnodes,4);
+            std::memcpy(node_record + 40 + shape_count * sizeof(T),&num_subnodes,4);
             out.write(node_record,recsize);
             delete [] node_record;