From ae96fffee0f9eab5a3df1f75526be5afbbc18656 Mon Sep 17 00:00:00 2001 From: Artem Pavlenko Date: Tue, 7 Jul 2009 23:56:01 +0000 Subject: [PATCH] + use MAPNIK_BIG_ENDIAN + support for reading int16_t + allow compiling shape.input with or without memory mapped files support ( define SHAPE_MEMORY_MAPPED_FILE to enable) --- include/mapnik/global.hpp | 43 +++++++++--- plugins/input/postgis/postgis.cpp | 6 +- plugins/input/shape/dbffile.cpp | 23 ++++--- plugins/input/shape/dbffile.hpp | 16 +++-- plugins/input/shape/shape_io.cpp | 12 ++-- plugins/input/shape/shapefile.cpp | 21 ------ plugins/input/shape/shapefile.hpp | 107 ++++++++++++++++++++++-------- src/wkb.cpp | 6 +- 8 files changed, 152 insertions(+), 82 deletions(-) diff --git a/include/mapnik/global.hpp b/include/mapnik/global.hpp index 65dd636db..99bb7d273 100644 --- a/include/mapnik/global.hpp +++ b/include/mapnik/global.hpp @@ -34,6 +34,10 @@ namespace mapnik { +#ifdef BOOST_BIG_ENDIAN +#define MAPNUK_BIG_ENDIAN +#endif + #define int2net(A) (int16_t) (((boost::uint16_t) ((boost::uint8_t) (A)[1])) | \ (((boost::uint16_t) ((boost::uint8_t) (A)[0])) << 8)) @@ -62,10 +66,22 @@ namespace mapnik (V)=def_temp; } while(0) - // read int NDR (little endian) - inline int& read_int_ndr(const char* data, int & val) + // read int16_t NDR (little endian) + inline boost::int16_t& read_int16_ndr(const char* data, boost::int16_t & val) { -#ifndef BOOST_BIG_ENDIAN +#ifndef MAPNIK_BIG_ENDIAN + memcpy(&val,data,2); +#else + val = (data[0]&0xff) | + ((data[1]&0xff)<<8); +#endif + return val; + } + + // read int32_t NDR (little endian) + inline boost::int32_t& read_int32_ndr(const char* data, boost::int32_t & val) + { +#ifndef MAPNIK_BIG_ENDIAN memcpy(&val,data,4); #else val = (data[0]&0xff) | @@ -79,7 +95,7 @@ namespace mapnik // read double NDR (little endian) inline double& read_double_ndr(const char* data, double & val) { -#ifndef BOOST_BIG_ENDIAN +#ifndef MAPNIK_BIG_ENDIAN std::memcpy(&val,&data[0],8); #else boost::int64_t bits = ((boost::int64_t)data[0] & 0xff) | @@ -95,10 +111,21 @@ namespace mapnik return val; } - // read int XDR (big endian) - inline int& read_int_xdr(const char* data, int & val) + // read int16_t XDR (big endian) + inline boost::int16_t& read_int16_xdr(const char* data, boost::int16_t & val) { -#ifndef BOOST_BIG_ENDIAN +#ifndef MAPNIK_BIG_ENDIAN + val = (data[3]&0xff) | ((data[2]&0xff)<<8); +#else + memcpy(&val,data,2); +#endif + return val; + } + + // read int32_t XDR (big endian) + inline boost::int32_t& read_int32_xdr(const char* data, boost::int32_t & val) + { +#ifndef MAPNIK_BIG_ENDIAN val = (data[3]&0xff) | ((data[2]&0xff)<<8) | ((data[1]&0xff)<<16) | ((data[0]&0xff)<<24); #else memcpy(&val,data,4); @@ -109,7 +136,7 @@ namespace mapnik // read double XDR (big endian) inline double& read_double_xdr(const char* data, double & val) { -#ifndef BOOST_BIG_ENDIAN +#ifndef MAPNIK_BIG_ENDIAN boost::int64_t bits = ((boost::int64_t)data[7] & 0xff) | ((boost::int64_t)data[6] & 0xff) << 8 | ((boost::int64_t)data[5] & 0xff) << 16 | diff --git a/plugins/input/postgis/postgis.cpp b/plugins/input/postgis/postgis.cpp index 8ef459317..c4af23b19 100644 --- a/plugins/input/postgis/postgis.cpp +++ b/plugins/input/postgis/postgis.cpp @@ -23,15 +23,15 @@ //$Id: postgis.cc 44 2005-04-22 18:53:54Z pavlenko $ // mapnik +#include +#include #include "connection_manager.hpp" #include "postgis.hpp" -#include // boost #include #include #include -#include // stl #include @@ -40,7 +40,7 @@ #include #include -#ifndef BOOST_BIG_ENDIAN +#ifndef MAPNIK_BIG_ENDIAN #define WKB_ENCODING "NDR" #else #define WKB_ENCODING "XDR" diff --git a/plugins/input/shape/dbffile.cpp b/plugins/input/shape/dbffile.cpp index 327f50f75..97aaaf945 100644 --- a/plugins/input/shape/dbffile.cpp +++ b/plugins/input/shape/dbffile.cpp @@ -19,15 +19,18 @@ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * *****************************************************************************/ -// stl -#include -// boost -#include -#include // mapnik +#include #include #include #include "dbffile.hpp" +// boost +#include +#include + +// stl +#include + dbf_file::dbf_file() : num_records_(0), @@ -197,7 +200,9 @@ int dbf_file::read_short() { char b[2]; file_.read(b,2); - return (b[0] & 0xff) | (b[1] & 0xff) << 8; + boost::int16_t val; + mapnik::read_int16_ndr(b,val); + return val; } @@ -205,11 +210,11 @@ int dbf_file::read_int() { char b[4]; file_.read(b,4); - return (b[0] & 0xff) | (b[1] & 0xff) << 8 | - (b[2] & 0xff) << 16 | (b[3] & 0xff) <<24; + boost::int32_t val; + mapnik::read_int32_ndr(b,val); + return val; } - void dbf_file::skip(int bytes) { file_.seekg(bytes,std::ios::cur); diff --git a/plugins/input/shape/dbffile.hpp b/plugins/input/shape/dbffile.hpp index 53ce2f9e3..cca766a7f 100644 --- a/plugins/input/shape/dbffile.hpp +++ b/plugins/input/shape/dbffile.hpp @@ -51,12 +51,16 @@ using namespace boost::iostreams; class dbf_file { - private: - int num_records_; - int num_fields_; - int record_length_; - std::vector fields_; - stream file_; +private: + int num_records_; + int num_fields_; + int record_length_; + std::vector fields_; +#ifdef SHAPE_MEMORY_MAPPED_FILE + stream file_; +#else + stream file_; +#endif char* record_; public: dbf_file(); diff --git a/plugins/input/shape/shape_io.cpp b/plugins/input/shape/shape_io.cpp index 7f3f31c52..630e4e09e 100644 --- a/plugins/input/shape/shape_io.cpp +++ b/plugins/input/shape/shape_io.cpp @@ -98,7 +98,7 @@ dbf_file& shape_io::dbf() geometry2d * shape_io::read_polyline() { using mapnik::line_string_impl; - shape_record record(reclength_*2-36); + shape_file::record_type record(reclength_*2-36); shp_.read_record(record); int num_parts=record.read_ndr_integer(); int num_points=record.read_ndr_integer(); @@ -153,7 +153,7 @@ geometry2d * shape_io::read_polyline() geometry2d * shape_io::read_polylinem() { using mapnik::line_string_impl; - shape_record record(reclength_*2-36); + shape_file::record_type record(reclength_*2-36); shp_.read_record(record); int num_parts=record.read_ndr_integer(); int num_points=record.read_ndr_integer(); @@ -216,7 +216,7 @@ geometry2d * shape_io::read_polylinem() geometry2d * shape_io::read_polylinez() { using mapnik::line_string_impl; - shape_record record(reclength_*2-36); + shape_file::record_type record(reclength_*2-36); shp_.read_record(record); int num_parts=record.read_ndr_integer(); int num_points=record.read_ndr_integer(); @@ -286,7 +286,7 @@ geometry2d * shape_io::read_polylinez() geometry2d * shape_io::read_polygon() { using mapnik::polygon_impl; - shape_record record(reclength_*2-36); + shape_file::record_type record(reclength_*2-36); shp_.read_record(record); int num_parts=record.read_ndr_integer(); int num_points=record.read_ndr_integer(); @@ -327,7 +327,7 @@ geometry2d * shape_io::read_polygon() geometry2d * shape_io::read_polygonm() { using mapnik::polygon_impl; - shape_record record(reclength_*2-36); + shape_file::record_type record(reclength_*2-36); shp_.read_record(record); int num_parts=record.read_ndr_integer(); int num_points=record.read_ndr_integer(); @@ -376,7 +376,7 @@ geometry2d * shape_io::read_polygonm() geometry2d * shape_io::read_polygonz() { using mapnik::polygon_impl; - shape_record record(reclength_*2-36); + shape_file::record_type record(reclength_*2-36); shp_.read_record(record); int num_parts=record.read_ndr_integer(); int num_points=record.read_ndr_integer(); diff --git a/plugins/input/shape/shapefile.cpp b/plugins/input/shape/shapefile.cpp index a0838dfc7..d85355de6 100644 --- a/plugins/input/shape/shapefile.cpp +++ b/plugins/input/shape/shapefile.cpp @@ -22,24 +22,3 @@ #include "shapefile.hpp" -shape_file::shape_file() {} - -shape_file::shape_file(const std::string& file_name) - : file_(file_name) {} - -shape_file::~shape_file() {} - -bool shape_file::is_open() -{ - return file_.is_open(); -} - - -void shape_file::close() -{ - if (file_ && file_.is_open()) - file_.close(); -} - - - diff --git a/plugins/input/shape/shapefile.hpp b/plugins/input/shape/shapefile.hpp index 509835e00..ff1934558 100644 --- a/plugins/input/shape/shapefile.hpp +++ b/plugins/input/shape/shapefile.hpp @@ -34,31 +34,59 @@ #include #include -//#include #include using mapnik::Envelope; -using mapnik::read_int_ndr; -using mapnik::read_int_xdr; +using mapnik::read_int32_ndr; +using mapnik::read_int32_xdr; using mapnik::read_double_ndr; using mapnik::read_double_xdr; + +struct RecordTag +{ + typedef char* data_type; + static data_type alloc(unsigned size) + { + return static_cast(::operator new(sizeof(char)*size)); + } + + static void dealloc(data_type data) + { + ::operator delete(data); + } +}; + +struct MappedRecordTag +{ + typedef const char* data_type; + static data_type alloc(unsigned) { return 0;} + static void dealloc(data_type ) {} +}; + +template struct shape_record { - const char* data; + typename Tag::data_type data; size_t size; mutable size_t pos; explicit shape_record(size_t size) - : //data(static_cast(::operator new(sizeof(char)*size))), + : + data(Tag::alloc(size)), size(size), pos(0) {} - void set_data(const char * data_) + void set_data(typename Tag::data_type data_) { data = data_; } - + + typename Tag::data_type get_data() + { + return data; + } + void skip(unsigned n) { pos+=n; @@ -67,7 +95,7 @@ struct shape_record int read_ndr_integer() { int val; - read_int_ndr(&data[pos],val); + read_int32_ndr(&data[pos],val); pos+=4; return val; } @@ -75,7 +103,7 @@ struct shape_record int read_xdr_integer() { int val; - read_int_xdr(&data[pos],val); + read_int32_xdr(&data[pos],val); pos+=4; return val; } @@ -94,7 +122,7 @@ struct shape_record ~shape_record() { - //::operator delete(data); + Tag::dealloc(data); } }; @@ -103,19 +131,46 @@ using namespace boost::iostreams; class shape_file : boost::noncopyable { +#ifdef SHAPE_MEMORY_MAPPED_FILE stream file_; - //stream file_; +#else + stream file_; +#endif + public: - shape_file(); - shape_file(std::string const& file_name); - ~shape_file(); - bool is_open(); - void close(); - - inline void read_record(shape_record& rec) +#ifdef SHAPE_MEMORY_MAPPED_FILE + typedef shape_record record_type; +#else + typedef shape_record record_type; +#endif + + shape_file() {} + + shape_file(std::string const& file_name) + : file_(file_name) {} + + ~shape_file() {} + + inline bool is_open() { + return file_.is_open(); + } + + + inline void close() + { + if (file_ && file_.is_open()) + file_.close(); + } + + inline void read_record(record_type& rec) + { +#ifdef SHAPE_MEMORY_MAPPED_FILE rec.set_data(file_->data() + file_.tellg()); file_.seekg(rec.size,std::ios::cur); +#else + file_.read(rec.get_data(),rec.size); +#endif } inline int read_xdr_integer() @@ -123,7 +178,7 @@ public: char b[4]; file_.read(b, 4); int val; - read_int_xdr(b,val); + read_int32_xdr(b,val); return val; } @@ -132,14 +187,14 @@ public: char b[4]; file_.read(b,4); int val; - read_int_ndr(b,val); + read_int32_ndr(b,val); return val; } inline double read_double() { double val; -#ifndef WORDS_BIGENDIAN +#ifndef MAPNIK_BIG_ENDIAN file_.read(reinterpret_cast(&val),8); #else char b[8]; @@ -151,16 +206,16 @@ public: inline void read_envelope(Envelope& envelope) { -#ifndef WORDS_BIGENDIAN +#ifndef MAPNIK_BIG_ENDIAN file_.read(reinterpret_cast(&envelope),sizeof(envelope)); #else char data[4*8]; file_.read(data,4*8); double minx,miny,maxx,maxy; - read_double_ndr(data,minx); - read_double_ndr(data,miny); - read_double_ndr(data,maxx); - read_double_ndr(data,maxy); + read_double_ndr(data + 0*8,minx); + read_double_ndr(data + 1*8,miny); + read_double_ndr(data + 2*8,maxx); + read_double_ndr(data + 3*8,maxy); envelope.init(minx,miny,maxx,maxy); #endif } diff --git a/src/wkb.cpp b/src/wkb.cpp index 7e92642bc..d6307ebef 100644 --- a/src/wkb.cpp +++ b/src/wkb.cpp @@ -79,7 +79,7 @@ namespace mapnik break; } -#ifndef BOOST_BIG_ENDIAN +#ifndef MAPNIK_BIG_ENDIAN needSwap_=byteOrder_?wkbXDR:wkbNDR; #else needSwap_=byteOrder_?wkbNDR:wkbXDR; @@ -155,11 +155,11 @@ namespace mapnik int n; if (needSwap_) { - read_int_xdr(wkb_+pos_,n); + read_int32_xdr(wkb_+pos_,n); } else { - read_int_ndr(wkb_+pos_,n); + read_int32_ndr(wkb_+pos_,n); } pos_+=4;