diff --git a/README.md b/README.md index 81368c4..28137ad 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,8 @@ Patch Binary (Dimensional) byte: endianness (1 = NDR, 0 = XDR) uint32: pcid (key to POINTCLOUD_SCHEMAS) uint32: 1 = dimensional compression - dimensional[]: dimensionally compressed data for each dimension + uint32: npoints + dimensions[]: dimensionally compressed data for each dimension /* dimensional compression */ byte: dimensional compression type (0 = none, 1 = significant bits, 2 = deflate, 3 = run-length) diff --git a/TODO.md b/TODO.md deleted file mode 100644 index ec85708..0000000 --- a/TODO.md +++ /dev/null @@ -1,5 +0,0 @@ -To Do List ----------- - -- Update the patch serialization/deserialization code to handle compression types (compression implies smaller memory sizes) -- Think through compression implications in general \ No newline at end of file diff --git a/lib/Makefile b/lib/Makefile index 623d9ff..41c5798 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -6,10 +6,11 @@ LDFLAGS = $(XML2_LDFLAGS) $(ZLIB_LDFLAGS) OBJS = \ pc_bytes.o \ - pc_dimlist.o \ pc_dimstats.o \ pc_mem.o \ pc_patch.o \ + pc_patch_dimensional.o \ + pc_patch_uncompressed.o \ pc_point.o \ pc_pointlist.o \ pc_schema.o \ diff --git a/lib/TODO.md b/lib/TODO.md new file mode 100644 index 0000000..35b8bdf --- /dev/null +++ b/lib/TODO.md @@ -0,0 +1,6 @@ +To Do +===== + +- convert PCBYTES to use PCDIMENSION* instead of holding all values as dupes +- make dimensional min/max generation respect scale/offset +? convert PCBYTES handlign to pass-by-reference instead of pass-by-value diff --git a/lib/cunit/cu_pc_patch.c b/lib/cunit/cu_pc_patch.c index 3f04529..4ef6f34 100644 --- a/lib/cunit/cu_pc_patch.c +++ b/lib/cunit/cu_pc_patch.c @@ -97,7 +97,7 @@ test_endian_flip() static void test_patch_hex_in() { - // 00 endian + // 00 endian (big) // 00000000 pcid // 00000000 compression // 00000002 npoints @@ -110,15 +110,15 @@ test_patch_hex_in() size_t hexsize = strlen(hexbuf); uint8_t *wkb = bytes_from_hexbytes(hexbuf, hexsize); PCPATCH *pa = pc_patch_from_wkb(simpleschema, wkb, hexsize/2); - PCPOINTLIST *pl = pc_patch_to_points(pa); - pc_point_get_double_by_name(pl->points[0], "X", &d); + PCPOINTLIST *pl = pc_patch_to_pointlist(pa); + pc_point_get_double_by_name(pc_pointlist_get_point(pl, 0), "X", &d); CU_ASSERT_DOUBLE_EQUAL(d, 0.02, 0.000001); - pc_point_get_double_by_name(pl->points[1], "Intensity", &d); + pc_point_get_double_by_name(pc_pointlist_get_point(pl, 1), "Intensity", &d); CU_ASSERT_DOUBLE_EQUAL(d, 8, 0.000001); str = pc_patch_to_string(pa); - CU_ASSERT_STRING_EQUAL(str, "[ 0 : (0.02, 0.03, 0.05, 6), (0.02, 0.03, 0.05, 8) ]"); - // printf("\n%s\n",str); + CU_ASSERT_STRING_EQUAL(str, "{\"pcid\":0,\"pts\":[[0.02,0.03,0.05,6],[0.02,0.03,0.05,8]]}"); + // printf("\n%s\n",str); pc_pointlist_free(pl); pc_patch_free(pa); @@ -138,7 +138,7 @@ test_patch_hex_out() // 0000000200000003000000050006 pt1 (XYZi) // 0000000200000003000000050008 pt2 (XYZi) - static char *wkt_result = "[ 0 : (0.02, 0.03, 0.05, 6), (0.02, 0.03, 0.05, 8) ]"; + static char *wkt_result = "{\"pcid\":0,\"pts\":[[0.02,0.03,0.05,6],[0.02,0.03,0.05,8]]}"; static char *hexresult_xdr = "0000000000000000000000000200000002000000030000000500060000000200000003000000050008"; static char *hexresult_ndr = @@ -150,7 +150,7 @@ test_patch_hex_out() PCPOINT *pt0 = pc_point_from_double_array(simpleschema, d0, 4); PCPOINT *pt1 = pc_point_from_double_array(simpleschema, d1, 4); - PCPATCH *pa; + PCPATCH_UNCOMPRESSED *pa; uint8_t *wkb; size_t wkbsize; char *hexwkb; @@ -160,8 +160,8 @@ test_patch_hex_out() pc_pointlist_add_point(pl, pt0); pc_pointlist_add_point(pl, pt1); - pa = pc_patch_from_points(pl); - wkb = pc_patch_to_wkb(pa, &wkbsize); + pa = pc_patch_uncompressed_from_pointlist(pl); + wkb = pc_patch_uncompressed_to_wkb(pa, &wkbsize); // printf("wkbsize %zu\n", wkbsize); hexwkb = hexbytes_from_bytes(wkb, wkbsize); @@ -177,11 +177,12 @@ test_patch_hex_out() CU_ASSERT_STRING_EQUAL(hexwkb, hexresult_xdr); } - wkt = pc_patch_to_string(pa); + wkt = pc_patch_uncompressed_to_string(pa); + // printf("wkt %s\n", wkt); CU_ASSERT_STRING_EQUAL(wkt, wkt_result); pc_pointlist_free(pl); - pc_patch_free(pa); + pc_patch_uncompressed_free(pa); pcfree(hexwkb); pcfree(wkb); pcfree(wkt); @@ -219,7 +220,7 @@ static PCBYTES initbytes(uint8_t *bytes, size_t size, uint32_t interp) pcb.bytes = bytes; pcb.size = size; pcb.interpretation = interp; - pcb.npoints = pcb.size / INTERPRETATION_SIZES[pcb.interpretation]; + pcb.npoints = pcb.size / pc_interpretation_size(pcb.interpretation); pcb.compression = PC_DIM_NONE; return pcb; } @@ -536,13 +537,13 @@ test_zlib_encoding() * Test for data loss or alteration. */ static void -test_dimlist() +test_patch_dimensional() { PCPOINT *pt; int i; int npts = 10; PCPOINTLIST *pl1, *pl2; - PCDIMLIST *pdl; + PCPATCH_DIMENSIONAL *pdl; PCDIMSTATS *pds; pl1 = pc_pointlist_make(npts); @@ -557,12 +558,12 @@ test_dimlist() pc_pointlist_add_point(pl1, pt); } - pdl = pc_dimlist_from_pointlist(pl1); - pl2 = pc_pointlist_from_dimlist(pdl); + pdl = pc_patch_dimensional_from_pointlist(pl1); + pl2 = pc_pointlist_from_dimensional(pdl); for ( i = 0; i < npts; i++ ) { - pt = pl2->points[i]; + pt = pc_pointlist_get_point(pl2, i); double v1, v2, v3, v4; pc_point_get_double_by_name(pt, "x", &v1); pc_point_get_double_by_name(pt, "y", &v2); @@ -580,12 +581,78 @@ test_dimlist() pc_dimstats_update(pds, pdl); - pc_dimlist_free(pdl); + pc_patch_dimensional_free(pdl); pc_pointlist_free(pl1); pc_pointlist_free(pl2); pc_dimstats_free(pds); } + +static void +test_patch_dimensional_compression() +{ + PCPOINT *pt; + int i; + int npts = 400; + PCPOINTLIST *pl1, *pl2; + PCPATCH_DIMENSIONAL *pch1, *pch2; + PCDIMSTATS *pds = NULL; + size_t z1, z2; + char *str; + + pl1 = pc_pointlist_make(npts); + + for ( i = 0; i < npts; i++ ) + { + pt = pc_point_make(simpleschema); + pc_point_set_double_by_name(pt, "x", i*2.0); + pc_point_set_double_by_name(pt, "y", i*1.9); + pc_point_set_double_by_name(pt, "Z", i*0.34); + pc_point_set_double_by_name(pt, "intensity", 10); + pc_pointlist_add_point(pl1, pt); + } + + pch1 = pc_patch_dimensional_from_pointlist(pl1); + z1 = pc_patch_dimensional_serialized_size((PCPATCH*)pch1); + // printf("z1 %ld\n", z1); + + pds = pc_dimstats_make(simpleschema); + pc_dimstats_update(pds, pch1); + pc_dimstats_update(pds, pch1); + pch2 = pc_patch_dimensional_compress(pch1, pds); + z2 = pc_patch_dimensional_serialized_size((PCPATCH*)pch2); + // printf("z2 %ld\n", z2); + + str = pc_dimstats_to_string(pds); + CU_ASSERT_STRING_EQUAL(str, "{\"ndims\":4,\"total_points\":1200,\"total_patches\":3,\"dims\":[{\"total_runs\":1200,\"total_commonbits\":45,\"recommended_compression\":2},{\"total_runs\":1200,\"total_commonbits\":45,\"recommended_compression\":2},{\"total_runs\":1200,\"total_commonbits\":54,\"recommended_compression\":2},{\"total_runs\":3,\"total_commonbits\":48,\"recommended_compression\":1}]}"); + // printf("%s\n", str); + pcfree(str); + + pl2 = pc_pointlist_from_dimensional(pch2); + + for ( i = 0; i < npts; i++ ) + { + pt = pc_pointlist_get_point(pl2, i); + double v1, v2, v3, v4; + pc_point_get_double_by_name(pt, "x", &v1); + pc_point_get_double_by_name(pt, "y", &v2); + pc_point_get_double_by_name(pt, "Z", &v3); + pc_point_get_double_by_name(pt, "intensity", &v4); + // printf("%g\n", v4); + CU_ASSERT_DOUBLE_EQUAL(v1, i*2.0, 0.001); + CU_ASSERT_DOUBLE_EQUAL(v2, i*1.9, 0.001); + CU_ASSERT_DOUBLE_EQUAL(v3, i*0.34, 0.001); + CU_ASSERT_DOUBLE_EQUAL(v4, 10, 0.001); + } + + pc_patch_dimensional_free(pch1); + pc_patch_dimensional_free(pch2); +// pc_patch_dimensional_free(pch3); + pc_pointlist_free(pl1); + pc_pointlist_free(pl2); + if ( pds ) pc_dimstats_free(pds); +} + /* REGISTER ***********************************************************/ CU_TestInfo patch_tests[] = { @@ -596,7 +663,8 @@ CU_TestInfo patch_tests[] = { PC_TEST(test_run_length_encoding), PC_TEST(test_sigbits_encoding), PC_TEST(test_zlib_encoding), - PC_TEST(test_dimlist), + PC_TEST(test_patch_dimensional), + PC_TEST(test_patch_dimensional_compression), CU_TEST_INFO_NULL }; diff --git a/lib/cunit/cu_pc_schema.c b/lib/cunit/cu_pc_schema.c index e427bca..810f6a5 100644 --- a/lib/cunit/cu_pc_schema.c +++ b/lib/cunit/cu_pc_schema.c @@ -19,8 +19,10 @@ static const char *xmlfile = "data/pdal-schema.xml"; static int init_suite(void) { - schema = NULL; - return 0; + char *xmlstr = file_to_str(xmlfile); + int rv = pc_schema_from_xml(xmlstr, &schema); + pcfree(xmlstr); + return rv == PC_FAILURE; } static int @@ -36,8 +38,9 @@ clean_suite(void) static void test_schema_from_xml() { + static PCSCHEMA *myschema = NULL; char *xmlstr = file_to_str(xmlfile); - int rv = pc_schema_from_xml(xmlstr, &schema); + int rv = pc_schema_from_xml(xmlstr, &myschema); pcfree(xmlstr); // char *schemastr = pc_schema_to_json(schema); @@ -45,7 +48,8 @@ test_schema_from_xml() // printf("name0 %s\n", schema->dims[0]->name); // printf("%s\n", schemastr); - CU_ASSERT(schema != NULL); + CU_ASSERT(myschema != NULL); + pc_schema_free(myschema); } static void diff --git a/lib/pc_api.h b/lib/pc_api.h index 6a6f85f..7761217 100644 --- a/lib/pc_api.h +++ b/lib/pc_api.h @@ -77,6 +77,21 @@ typedef struct } PCSCHEMA; +typedef struct +{ + uint32_t total_runs; + uint32_t total_commonbits; + uint32_t recommended_compression; +} PCDIMSTAT; + +typedef struct +{ + int32_t ndims; + uint32_t total_points; + uint32_t total_patches; + PCDIMSTAT *stats; +} PCDIMSTATS; + /** * Uncompressed structure for in-memory handling * of points. A read-only PgSQL point can be wrapped in @@ -99,6 +114,16 @@ typedef struct PCPOINT **points; } PCPOINTLIST; +typedef struct +{ + size_t size; + uint32_t npoints; + uint32_t interpretation; + uint32_t compression; + uint32_t readonly; + uint8_t *bytes; +} PCBYTES; + /** * Uncompressed Structure for in-memory handling * of patches. A read-only PgSQL patch can be wrapped in @@ -106,6 +131,15 @@ typedef struct * PgSQL memory and setting the capacity to 0 * to indicate it is read-only. */ +typedef struct +{ + int type; + int8_t readonly; + const PCSCHEMA *schema; + uint32_t npoints; /* How many points we have */ + double xmin, xmax, ymin, ymax; +} PCPATCH; + typedef struct { int type; @@ -114,10 +148,30 @@ typedef struct uint32_t npoints; /* How many points we have */ double xmin, xmax, ymin, ymax; uint32_t maxpoints; /* How man points we can hold (or 0 for read-only) */ - uint8_t compressed; /* Has compression been applied to the data buffer? */ size_t datasize; uint8_t *data; /* A serialized version of the data */ -} PCPATCH; +} PCPATCH_UNCOMPRESSED; + +typedef struct +{ + int type; + int8_t readonly; + const PCSCHEMA *schema; + uint32_t npoints; /* How many points we have */ + double xmin, xmax, ymin, ymax; + PCBYTES *bytes; +} PCPATCH_DIMENSIONAL; + +typedef struct +{ + int type; + int8_t readonly; + const PCSCHEMA *schema; + uint32_t npoints; /* How many points we have */ + double xmin, xmax, ymin, ymax; + uint8_t *data; +} PCPATCH_GHT; + /* Global function signatures for memory/logging handlers. */ @@ -158,16 +212,15 @@ void pc_install_default_handlers(void); * UTILITY */ -/** Read the the PCID from WKB form of a POINT/PATCH */ -uint32_t wkb_get_pcid(const uint8_t *wkb); -/** Read the the npoints from WKB form of a PATCH */ -uint32_t wkb_get_compression(const uint8_t *wkb); -/** Flips the bytes of an int32_t */ -int32_t int32_flip_endian(int32_t val); /** Convert binary to hex */ uint8_t* bytes_from_hexbytes(const char *hexbuf, size_t hexsize); /** Convert hex to binary */ char* hexbytes_from_bytes(const uint8_t *bytebuf, size_t bytesize); +/** Read the the PCID from WKB form of a POINT/PATCH */ +uint32_t wkb_get_pcid(const uint8_t *wkb); +/** Build an empty #PCDIMSTATS based on the schema */ +PCDIMSTATS* pc_dimstats_make(const PCSCHEMA *schema); + /********************************************************************** @@ -203,6 +256,9 @@ void pc_pointlist_free(PCPOINTLIST *pl); /** Add a point to the list, expanding buffer as necessary */ void pc_pointlist_add_point(PCPOINTLIST *pl, PCPOINT *pt); +/** Get a point from the list */ +PCPOINT* pc_pointlist_get_point(const PCPOINTLIST *pl, int i); + /********************************************************************** * PCPOINT @@ -226,12 +282,6 @@ int pc_point_get_double_by_name(const PCPOINT *pt, const char *name, double *d); /** Casts dimension value to double and scale/offset appropriately before returning */ int pc_point_get_double_by_index(const PCPOINT *pt, uint32_t idx, double *d); -/** Scales/offsets double, casts to appropriate dimension type, and writes into point */ -int pc_point_set_double_by_index(PCPOINT *pt, uint32_t idx, double val); - -/** Scales/offsets double, casts to appropriate dimension type, and writes into point */ -int pc_point_set_double_by_name(PCPOINT *pt, const char *name, double val); - /** Returns X coordinate */ double pc_point_get_x(const PCPOINT *pt); @@ -255,26 +305,17 @@ uint8_t* pc_point_to_geometry_wkb(const PCPOINT *pt, size_t *wkbsize); * PCPATCH */ -/** Create new empty PCPATCH */ -PCPATCH* pc_patch_make(const PCSCHEMA *s); - /** Create new PCPATCH from a PCPOINT set. Copies data, doesn't take ownership of points */ -PCPATCH* pc_patch_from_points(const PCPOINTLIST *pl); +PCPATCH* pc_patch_from_pointlist(const PCPOINTLIST *ptl); /** Returns a list of points extracted from patch */ -PCPOINTLIST* pc_patch_to_points(const PCPATCH *patch); +PCPOINTLIST* pc_patch_to_pointlist(const PCPATCH *patch); /** Free patch memory, respecting read-only status. Does not free referenced schema */ void pc_patch_free(PCPATCH *patch); -/** Add a point to read/write PCPATCH */ -int pc_patch_add_point(PCPATCH *c, const PCPOINT *p); - /** Create a compressed copy, using the compression schema referenced in the PCSCHEMA */ -PCPATCH* pc_patch_compress(const PCPATCH *patch); - -/** Create a full copy of the patch, including data, but not including PCSCHEMA */ -PCPATCH* pc_patch_clone(const PCPATCH *patch); +PCPATCH* pc_patch_compress(const PCPATCH *patch, void *userdata); /** Create a new readwrite PCPOINT from a byte array */ PCPATCH* pc_patch_from_wkb(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize); @@ -288,5 +329,14 @@ char* pc_patch_to_string(const PCPATCH *patch); /** Returns OGC WKB for envelope of PCPATCH */ uint8_t* pc_patch_to_geometry_wkb_envelope(const PCPATCH *pa, size_t *wkbsize); +/** */ +size_t pc_patch_dimensional_serialized_size(const PCPATCH *patch); + +/** Write the representation down to a buffer */ +int pc_bytes_serialize(const PCBYTES *pcb, uint8_t *buf, size_t *size); + +/** Read a buffer up into a bytes structure */ +int pc_bytes_deserialize(const uint8_t *buf, const PCDIMENSION *dim, PCBYTES *pcb, int readonly, int flip_endian); + #endif /* _PC_API_H */ \ No newline at end of file diff --git a/lib/pc_api_internal.h b/lib/pc_api_internal.h index 60a698a..426dd8b 100644 --- a/lib/pc_api_internal.h +++ b/lib/pc_api_internal.h @@ -79,43 +79,25 @@ enum DIMCOMPRESSIONS { PC_DIM_ZLIB = 3 }; -typedef struct -{ - size_t size; - uint32_t npoints; - uint32_t interpretation; - uint32_t compression; - uint32_t read_only; - uint8_t *bytes; -} PCBYTES; -typedef struct -{ - uint32_t npoints; - const PCSCHEMA *schema; - PCBYTES *bytes; -} PCDIMLIST; - -typedef struct -{ - uint32_t total_runs; - uint32_t total_commonbits; - uint32_t recommended_compression; -} PCDIMSTAT; - -typedef struct -{ - int32_t ndims; - uint32_t total_points; - uint32_t total_patches; - PCDIMSTAT *stats; -} PCDIMSTATS; /** What is the endianness of this system? */ char machine_endian(void); +/** Flips the bytes of an int32_t */ +int32_t int32_flip_endian(int32_t val); + +/** Read the the npoints from WKB form of a PATCH */ +uint32_t wkb_get_compression(const uint8_t *wkb); + +/** Read an int32 from a byte array, flipping if requested */ +int32_t wkb_get_int32(const uint8_t *wkb, int flip_endian); + +/** Read an int16 from a byte array, flipping if requested */ +int16_t wkb_get_int16(const uint8_t *wkb, int flip_endian); + /** Force a byte array into the machine endianness */ uint8_t* uncompressed_bytes_flip_endian(const uint8_t *bytebuf, const PCSCHEMA *schema, uint32_t npoints); @@ -128,12 +110,72 @@ int pc_double_to_ptr(uint8_t *ptr, uint32_t interpretation, double val); /** Return number of bytes in a given interpretation */ size_t pc_interpretation_size(uint32_t interp); -/** True if there is a dimension of that name */ -int pc_schema_has_name(const PCSCHEMA *s, const char *name); - /** Copy a string within the global memory management context */ char* pcstrdup(const char *str); +/** Scales/offsets double, casts to appropriate dimension type, and writes into point */ +int pc_point_set_double_by_index(PCPOINT *pt, uint32_t idx, double val); + +/** Scales/offsets double, casts to appropriate dimension type, and writes into point */ +int pc_point_set_double_by_name(PCPOINT *pt, const char *name, double val); + + +/**************************************************************************** +* DIMENSION STATISTICS +*/ + +/** Analyze the bytes in the #PCPATCH_DIMENSIONAL and update the #PCDIMSTATS */ +int pc_dimstats_update(PCDIMSTATS *pds, const PCPATCH_DIMENSIONAL *pdl); +/** Free the PCDIMSTATS memory */ +void pc_dimstats_free(PCDIMSTATS *pds); +char* pc_dimstats_to_string(const PCDIMSTATS *pds); + + +/**************************************************************************** +* PATCHES +*/ + +/* DIMENSIONAL PATCHES */ +char* pc_patch_dimensional_to_string(const PCPATCH_DIMENSIONAL *pa); +PCPATCH_DIMENSIONAL* pc_patch_dimensional_from_uncompressed(const PCPATCH_UNCOMPRESSED *pa); +PCPATCH_DIMENSIONAL* pc_patch_dimensional_compress(const PCPATCH_DIMENSIONAL *pdl, PCDIMSTATS *pds); +PCPATCH_DIMENSIONAL* pc_patch_dimensional_decompress(const PCPATCH_DIMENSIONAL *pdl); +void pc_patch_dimensional_free(PCPATCH_DIMENSIONAL *pdl); +int pc_patch_dimensional_compute_extent(PCPATCH_DIMENSIONAL *pdl); +uint8_t* pc_patch_dimensional_to_wkb(const PCPATCH_DIMENSIONAL *patch, size_t *wkbsize); +PCPATCH* pc_patch_dimensional_from_wkb(const PCSCHEMA *schema, const uint8_t *wkb, size_t wkbsize); +PCPATCH_DIMENSIONAL* pc_patch_dimensional_from_pointlist(const PCPOINTLIST *pdl); +PCPOINTLIST* pc_pointlist_from_dimensional(const PCPATCH_DIMENSIONAL *pdl); + +/* UNCOMPRESSED PATCHES */ +char* pc_patch_uncompressed_to_string(const PCPATCH_UNCOMPRESSED *patch); +uint8_t* pc_patch_uncompressed_to_wkb(const PCPATCH_UNCOMPRESSED *patch, size_t *wkbsize); +PCPATCH* pc_patch_uncompressed_from_wkb(const PCSCHEMA *s, const uint8_t *wkb, size_t wkbsize); +PCPATCH_UNCOMPRESSED* pc_patch_uncompressed_make(const PCSCHEMA *s); +int pc_patch_uncompressed_compute_extent(PCPATCH_UNCOMPRESSED *patch); +void pc_patch_uncompressed_free(PCPATCH_UNCOMPRESSED *patch); +PCPOINTLIST* pc_patch_uncompressed_to_pointlist(const PCPATCH_UNCOMPRESSED *patch); +PCPATCH_UNCOMPRESSED* pc_patch_uncompressed_from_pointlist(const PCPOINTLIST *pl); +PCPATCH_UNCOMPRESSED* pc_patch_uncompressed_from_dimensional(const PCPATCH_DIMENSIONAL *pdl); +int pc_patch_uncompressed_add_point(PCPATCH_UNCOMPRESSED *c, const PCPOINT *p); + + + +/**************************************************************************** +* BYTES +*/ + +/** Construct empty byte array (zero out attribute and allocate byte buffer) */ +PCBYTES pc_bytes_make(const PCDIMENSION *dim, uint32_t npoints); +/** Empty the byte array (free the byte buffer) */ +void pc_bytes_free(PCBYTES bytes); +/** Apply the compresstion to the byte array in place, freeing the original byte buffer */ +PCBYTES pc_bytes_encode(PCBYTES pcb, int compression); +/** Convert the bytes in #PCBYTES to PC_DIM_NONE compression */ +PCBYTES pc_bytes_decode(PCBYTES epcb); +/** How big will the serialization be? */ +size_t pc_bytes_serialized_size(const PCBYTES *pcb); + /** Convert value bytes to RLE bytes */ PCBYTES pc_bytes_run_length_encode(const PCBYTES pcb); /** Convert RLE bytes to value bytes */ @@ -159,42 +201,6 @@ uint16_t pc_bytes_sigbits_count_16(const PCBYTES *pcb, uint32_t *nsigbits); uint32_t pc_bytes_sigbits_count_32(const PCBYTES *pcb, uint32_t *nsigbits); /** Using an 64-bit word, what is the common word and number of bits in common? */ uint64_t pc_bytes_sigbits_count_64(const PCBYTES *pcb, uint32_t *nsigbits); -/** Pivot a #PCPOINTLIST to a #PCDIMLIST, taking copies of data. */ -PCDIMLIST* pc_dimlist_from_pointlist(const PCPOINTLIST *pl); -/** Pivot a #PCDIMLIST to a #PCPOINTLIST, taking copies of data. */ -PCPOINTLIST* pc_pointlist_from_dimlist(PCDIMLIST *pdl); -/** Compress #DIMLIST, using the suggestions from the #PCDIMSTATS. Compresses in place and frees existing data. */ -int pc_dimlist_encode(PCDIMLIST *pdl, PCDIMSTATS **pdsptr); -/** Decompress #DIMLIST. Decompresses in place and frees existing data. */ -int pc_dimlist_decode(PCDIMLIST *pdl); - -/** Build an empty #PCDIMSTATS based on the schema */ -PCDIMSTATS* pc_dimstats_make(const PCSCHEMA *schema); -/** Analyze the bytes in the #PCDIMLIST and update the #PCDIMSTATS */ -int pc_dimstats_update(PCDIMSTATS *pds, const PCDIMLIST *pdl); -/** Free the PCDIMSTATS memory */ -void pc_dimstats_free(PCDIMSTATS *pds); - -/** Construct empty byte array (zero out attribute and allocate byte buffer) */ -PCBYTES pc_bytes_make(const PCDIMENSION *dim, uint32_t npoints); -/** Empty the byte array (free the byte buffer) */ -void pc_bytes_free(PCBYTES bytes); -/** Apply the compresstion to the byte array in place, freeing the original byte buffer */ -PCBYTES pc_bytes_encode(PCBYTES pcb, int compression); -/** Convert the bytes in #PCBYTES to PC_DIM_NONE compression */ -PCBYTES pc_bytes_decode(PCBYTES epcb); - -/** How big will the serialization be? */ -size_t pc_bytes_get_serialized_size(const PCBYTES *pcb); -/** Write the representation down to a buffer */ -int pc_bytes_serialize(const PCBYTES *pcb, uint8_t *buf, size_t *size); -/** Read a buffer up into a bytes structure */ -int pc_bytes_deserialize(uint8_t *buf, const PCDIMENSION *dim, PCBYTES *pcb, int read_only, int flip_endian); - -/** Read an int32 from a byte array, flipping if requested */ -int32_t wkb_get_int32(const uint8_t *wkb, int flip_endian); -/** Read an int16 from a byte array, flipping if requested */ -int16_t wkb_get_int16(const uint8_t *wkb, int flip_endian); #endif /* _PC_API_INTERNAL_H */ \ No newline at end of file diff --git a/lib/pc_bytes.c b/lib/pc_bytes.c index 62aafc6..835e7a2 100644 --- a/lib/pc_bytes.c +++ b/lib/pc_bytes.c @@ -18,6 +18,7 @@ #include #include +#include #include "pc_api_internal.h" #include "zlib.h" @@ -26,7 +27,7 @@ void pc_bytes_free(PCBYTES pcb) { - if ( ! pcb.read_only ) + if ( ! pcb.readonly ) pcfree(pcb.bytes); } @@ -39,7 +40,7 @@ pc_bytes_make(const PCDIMENSION *dim, uint32_t npoints) pcb.npoints = npoints; pcb.interpretation = dim->interpretation; pcb.compression = PC_DIM_NONE; - pcb.read_only = PC_FALSE; + pcb.readonly = PC_FALSE; return pcb; } @@ -75,14 +76,14 @@ pc_bytes_encode(PCBYTES pcb, int compression) } case PC_DIM_NONE: { - return pcb; + epcb = pc_bytes_clone(pcb); + break; } default: { pcerror("pc_bytes_encode: Uh oh"); } } - pc_bytes_free(pcb); return epcb; } @@ -94,29 +95,29 @@ pc_bytes_decode(PCBYTES epcb) { case PC_DIM_RLE: { - pcb = pc_bytes_run_length_decode(pcb); + pcb = pc_bytes_run_length_decode(epcb); break; } case PC_DIM_SIGBITS: { - pcb = pc_bytes_sigbits_decode(pcb); + pcb = pc_bytes_sigbits_decode(epcb); break; } case PC_DIM_ZLIB: { - pcb = pc_bytes_zlib_decode(pcb); + pcb = pc_bytes_zlib_decode(epcb); break; } case PC_DIM_NONE: { - return epcb; + pcb = pc_bytes_clone(epcb); + break; } default: { pcerror("pc_bytes_decode: Uh oh"); } } - pc_bytes_free(epcb); return pcb; } @@ -132,7 +133,7 @@ pc_bytes_run_count(const PCBYTES *pcb) int i; const uint8_t *ptr0; const uint8_t *ptr1; - size_t size = INTERPRETATION_SIZES[pcb->interpretation]; + size_t size = pc_interpretation_size(pcb->interpretation); uint32_t runcount = 1; for ( i = 1; i < pcb->npoints; i++ ) @@ -162,7 +163,7 @@ pc_bytes_run_length_encode(const PCBYTES pcb) const uint8_t *bytesptr; const uint8_t *runstart; uint8_t *bytes_rle; - size_t size = INTERPRETATION_SIZES[pcb.interpretation]; + size_t size = pc_interpretation_size(pcb.interpretation); uint8_t runlength = 1; PCBYTES pcbout = pcb; @@ -203,7 +204,7 @@ pc_bytes_run_length_encode(const PCBYTES pcb) /* We're going to replace the current buffer */ pcbout.bytes = bytes_rle; pcbout.compression = PC_DIM_RLE; - pcbout.read_only = PC_FALSE; + pcbout.readonly = PC_FALSE; return pcbout; } @@ -223,7 +224,7 @@ pc_bytes_run_length_decode(const PCBYTES pcb) const uint8_t *bytes_rle_ptr = pcb.bytes; const uint8_t *bytes_rle_end = pcb.bytes + pcb.size; - size_t size = INTERPRETATION_SIZES[pcb.interpretation]; + size_t size = pc_interpretation_size(pcb.interpretation); size_t size_out; uint8_t runlength; uint32_t npoints = 0; @@ -259,7 +260,7 @@ pc_bytes_run_length_decode(const PCBYTES pcb) pcbout.compression = PC_DIM_NONE; pcbout.size = size_out; pcbout.bytes = bytes; - pcbout.read_only = PC_FALSE; + pcbout.readonly = PC_FALSE; return pcbout; } @@ -273,8 +274,9 @@ pc_bytes_run_length_flip_endian(PCBYTES pcb) { int i, n; uint8_t *bytes_ptr = pcb.bytes; + uint8_t *end_ptr = pcb.bytes + pcb.size; uint8_t tmp; - size_t size = INTERPRETATION_SIZES[pcb.interpretation]; + size_t size = pc_interpretation_size(pcb.interpretation); assert(pcb.compression == PC_DIM_RLE); assert(pcb.npoints > 0); @@ -284,20 +286,20 @@ pc_bytes_run_length_flip_endian(PCBYTES pcb) return pcb; /* Don't try to modify read-only memory, make some fresh memory */ - if ( pcb.read_only == PC_TRUE ) + if ( pcb.readonly == PC_TRUE ) { uint8_t *oldbytes = pcb.bytes; pcb.bytes = pcalloc(pcb.size); memcpy(pcb.bytes, oldbytes, pcb.size); - pcb.read_only = PC_FALSE; + pcb.readonly = PC_FALSE; } - /* Move to start of first word */ - bytes_ptr++; + bytes_ptr++; /* Advance past count */ /* Visit each entry and flip the word, skip the count */ - for ( i = 0; i < pcb.npoints; i++ ) + while( bytes_ptr < end_ptr ) { + /* Swap the bytes in a way that makes sense for this word size */ for ( n = 0; n < size/2; n++ ) { @@ -306,8 +308,10 @@ pc_bytes_run_length_flip_endian(PCBYTES pcb) bytes_ptr[size-n-1] = tmp; } - /* Move forward one word and one counter */ - bytes_ptr += size+1; + /* Move past this word */ + bytes_ptr += size; + /* Advance past next count */ + bytes_ptr++; } return pcb; @@ -428,7 +432,7 @@ pc_bytes_sigbits_count_64(const PCBYTES *pcb, uint32_t *nsigbits) uint32_t pc_bytes_sigbits_count(const PCBYTES *pcb) { - size_t size = INTERPRETATION_SIZES[pcb->interpretation]; + size_t size = pc_interpretation_size(pcb->interpretation); uint32_t nbits = -1; switch ( size ) { @@ -569,7 +573,9 @@ pc_bytes_sigbits_encode_16(const PCBYTES pcb, uint16_t commonvalue, uint8_t comm /* How wide are our unique values? */ int nbits = bitwidth - commonbits; /* Size of output buffer (#bits/8+1remainder+4metadata) */ - size_t size_out = (nbits * pcb.npoints / 8) + 5; + size_t size_out_raw = (nbits * pcb.npoints / 8) + 5; + /* Make sure buffer is size to hold all our words */ + size_t size_out = size_out_raw + (size_out_raw % 2); uint8_t *bytes_out = pcalloc(size_out); /* Use this to zero out the parts that are common */ uint16_t mask = (0xFFFF >> commonbits); @@ -658,7 +664,8 @@ pc_bytes_sigbits_encode_32(const PCBYTES pcb, uint32_t commonvalue, uint8_t comm /* How wide are our unique values? */ int nbits = bitwidth - commonbits; /* Size of output buffer (#bits/8+1remainder+8metadata) */ - size_t size_out = (nbits * pcb.npoints / 8) + 9; + size_t size_out_raw = (nbits * pcb.npoints / 8) + 9; + size_t size_out = size_out_raw + (4 - (size_out_raw % 4)); uint8_t *bytes_out = pcalloc(size_out); /* Use this to zero out the parts that are common */ uint32_t mask = (0xFFFFFFFF >> commonbits); @@ -739,7 +746,7 @@ pc_bytes_sigbits_encode_32(const PCBYTES pcb, uint32_t commonvalue, uint8_t comm PCBYTES pc_bytes_sigbits_encode(const PCBYTES pcb) { - size_t size = INTERPRETATION_SIZES[pcb.interpretation]; + size_t size = pc_interpretation_size(pcb.interpretation); uint32_t nbits; switch ( size ) { @@ -772,7 +779,7 @@ pc_bytes_sigbits_flip_endian(const PCBYTES pcb) { int n; uint8_t tmp1, tmp2; - size_t size = INTERPRETATION_SIZES[pcb.interpretation]; + size_t size = pc_interpretation_size(pcb.interpretation); uint8_t *b1 = pcb.bytes; uint8_t *b2 = pcb.bytes + size; @@ -870,7 +877,7 @@ pc_bytes_sigbits_decode_16(const PCBYTES pcb) uint16_t commonvalue; uint16_t mask; int bit = 16; - size_t outbytes_size = sizeof(uint8_t) * pcb.npoints; + size_t outbytes_size = sizeof(uint16_t) * pcb.npoints; uint8_t *outbytes = pcalloc(outbytes_size); uint16_t *obytes = (uint16_t*)outbytes; PCBYTES pcbout = pcb; @@ -926,7 +933,7 @@ pc_bytes_sigbits_decode_32(const PCBYTES pcb) uint32_t commonvalue; uint32_t mask; int bit = 32; - size_t outbytes_size = sizeof(uint8_t) * pcb.npoints; + size_t outbytes_size = sizeof(uint32_t) * pcb.npoints; uint8_t *outbytes = pcalloc(outbytes_size); uint32_t *obytes = (uint32_t*)outbytes; PCBYTES pcbout = pcb; @@ -978,7 +985,7 @@ pc_bytes_sigbits_decode_32(const PCBYTES pcb) PCBYTES pc_bytes_sigbits_decode(const PCBYTES pcb) { - size_t size = INTERPRETATION_SIZES[pcb.interpretation]; + size_t size = pc_interpretation_size(pcb.interpretation); switch ( size ) { case 1: @@ -1070,7 +1077,7 @@ pc_bytes_zlib_decode(const PCBYTES pcb) int ret; PCBYTES pcbout = pcb; - pcbout.size = INTERPRETATION_SIZES[pcb.interpretation] * pcb.npoints; + pcbout.size = pc_interpretation_size(pcb.interpretation) * pcb.npoints; /* Set up output memory */ pcbout.bytes = pcalloc(pcbout.size); @@ -1095,11 +1102,14 @@ pc_bytes_zlib_decode(const PCBYTES pcb) } /** -* This flips bytes in-place, so won't work on read_only bytes +* This flips bytes in-place, so won't work on readonly bytes */ PCBYTES pc_bytes_flip_endian(PCBYTES pcb) { + if ( pcb.readonly ) + pcerror("pc_bytes_flip_endian: cannot flip readonly bytes"); + switch(pcb.compression) { case PC_DIM_NONE: @@ -1113,11 +1123,12 @@ pc_bytes_flip_endian(PCBYTES pcb) default: pcerror("pc_bytes_flip_endian: unknown compression"); } + return pcb; } size_t -pc_bytes_get_serialized_size(const PCBYTES *pcb) +pc_bytes_serialized_size(const PCBYTES *pcb) { /* compression type (1) + size of data (4) + data */ return 1 + 4 + pcb->size; @@ -1135,24 +1146,115 @@ pc_bytes_serialize(const PCBYTES *pcb, uint8_t *buf, size_t *size) } int -pc_bytes_deserialize(uint8_t *buf, const PCDIMENSION *dim, PCBYTES *pcb, int read_only, int flip_endian) +pc_bytes_deserialize(const uint8_t *buf, const PCDIMENSION *dim, PCBYTES *pcb, int readonly, int flip_endian) { pcb->compression = buf[0]; pcb->size = wkb_get_int32(buf+1, flip_endian); - pcb->read_only = read_only; - if ( read_only && flip_endian ) + pcb->readonly = readonly; + if ( readonly && flip_endian ) pcerror("pc_bytes_deserialize: cannot create a read-only buffer on byteswapped input"); - if ( read_only ) + if ( readonly ) { - pcb->bytes = buf + 5; + pcb->bytes = (uint8_t*)(buf+5); } else { pcb->bytes = pcalloc(pcb->size); memcpy(pcb->bytes, buf+5, pcb->size); + if ( flip_endian ) + { + *pcb = pc_bytes_flip_endian(*pcb); + } } + pcb->interpretation = dim->interpretation; /* WARNING, still need to set externally */ - /* pcb.interpretation */ /* pcb.npoints */ return PC_SUCCESS; } + + +static int +pc_bytes_uncompressed_minmax(const PCBYTES *pcb, double *min, double *max) +{ + int i; + int element_size = pc_interpretation_size(pcb->interpretation); + double d; + double mn = MAXFLOAT; + double mx = -1*MAXFLOAT; + for ( i = 0; i < pcb->npoints; i++ ) + { + d = pc_double_from_ptr(pcb->bytes + i*element_size, pcb->interpretation); + if ( d < mn ) + mn = d; + if ( d > mx ) + mx = d; + } + *min = mn; + *max = mx; +} + +static int +pc_bytes_run_length_minmax(const PCBYTES *pcb, double *min, double *max) +{ + int element_size = pc_interpretation_size(pcb->interpretation); + double mn = MAXFLOAT; + double mx = -1*MAXFLOAT; + double d; + uint8_t *ptr = pcb->bytes; + uint8_t *ptr_end = pcb->bytes + pcb->size; + + /* Move past first count */ + ptr++; + + while( ptr < ptr_end ) + { + d = pc_double_from_ptr(ptr, pcb->interpretation); + /* Calc min/max */ + if ( d < mn ) + mn = d; + if ( d > mx ) + mx = d; + + ptr += element_size; + } + + *min = mn; + *max = mx; +} + + +static int +pc_bytes_zlib_minmax(const PCBYTES *pcb, double *min, double *max) +{ + PCBYTES zcb = pc_bytes_zlib_decode(*pcb); + int rv = pc_bytes_uncompressed_minmax(&zcb, min, max); + pc_bytes_free(zcb); + return rv; +} + +static int +pc_bytes_sigbits_minmax(const PCBYTES *pcb, double *min, double *max) +{ + PCBYTES zcb = pc_bytes_sigbits_decode(*pcb); + int rv = pc_bytes_uncompressed_minmax(&zcb, min, max); + pc_bytes_free(zcb); + return rv; +} + +int pc_bytes_minmax(const PCBYTES *pcb, double *min, double *max) +{ + switch(pcb->compression) + { + case PC_DIM_NONE: + return pc_bytes_uncompressed_minmax(pcb, min, max); + case PC_DIM_SIGBITS: + return pc_bytes_sigbits_minmax(pcb, min, max); + case PC_DIM_ZLIB: + return pc_bytes_zlib_minmax(pcb, min, max); + case PC_DIM_RLE: + return pc_bytes_run_length_minmax(pcb, min, max); + default: + pcerror("pc_bytes_minmax: unknown compression"); + } + return PC_FAILURE; +} diff --git a/lib/pc_dimlist.c b/lib/pc_dimlist.c deleted file mode 100644 index 8f5bc4e..0000000 --- a/lib/pc_dimlist.c +++ /dev/null @@ -1,223 +0,0 @@ -/*********************************************************************** -* pc_dimlist.c -* -* Support for "dimensional compression", which is a catch-all -* term for applying compression separately on each dimension -* of a PCPATCH collection of PCPOINTS. -* -* Depending on the character of the data, one of these schemes -* will be used: -* -* - run-length encoding -* - significant-bit removal -* - deflate -* -* Portions Copyright (c) 2012, OpenGeo -* -***********************************************************************/ - -#include -#include "pc_api_internal.h" - - - -/** -* Converts a list of I N-dimensional points into a -* list of N I-valued dimensions. Precursor to running -* compression on each dimension separately. -*/ -PCDIMLIST * -pc_dimlist_from_pointlist(const PCPOINTLIST *pl) -{ - PCDIMLIST *pdl; - int i, j, ndims, npoints; - assert(pl); - - if ( pl->npoints == 0 ) return NULL; - - pdl = pcalloc(sizeof(PCDIMLIST)); - pdl->schema = pl->points[0]->schema; - ndims = pdl->schema->ndims; - npoints = pl->npoints; - pdl->npoints = npoints; - pdl->bytes = pcalloc(ndims * sizeof(PCBYTES)); - - for ( i = 0; i < ndims; i++ ) - { - PCDIMENSION *dim = pc_schema_get_dimension(pdl->schema, i); - pdl->bytes[i] = pc_bytes_make(dim, npoints); - for ( j = 0; j < npoints; j++ ) - { - uint8_t *to = pdl->bytes[i].bytes + dim->size * j; - uint8_t *from = pl->points[j]->data + dim->byteoffset; - memcpy(to, from, dim->size); - } - } - return pdl; -} - -void -pc_dimlist_free(PCDIMLIST *pdl) -{ - int i; - assert(pdl); - assert(pdl->schema); - - if ( pdl->bytes ) - { - for ( i = 0; i < pdl->schema->ndims; i++ ) - pc_bytes_free(pdl->bytes[i]); - - pcfree(pdl->bytes); - } - - pcfree(pdl); -} - -int -pc_dimlist_encode(PCDIMLIST *pdl, PCDIMSTATS **pdsptr) -{ - int i; - PCDIMSTATS *pds; - - assert(pdl); - assert(pdl->schema); - assert(pdsptr); - - /* Maybe we have stats passed in */ - pds = *pdsptr; - - /* No stats at all, make a new one */ - if ( ! pds ) - pds = pc_dimstats_make(pdl->schema); - - /* Still sampling, update stats */ - if ( pds->total_points < PCDIMSTATS_MIN_SAMPLE ) - pc_dimstats_update(pds, pdl); - - /* Compress each dimension as dictated by stats */ - for ( i = 0; i < pdl->schema->ndims; i++ ) - { - pdl->bytes[i] = pc_bytes_encode(pdl->bytes[i], pds->stats[i].recommended_compression); - } - - return PC_SUCCESS; -} - -int -pc_dimlist_decode(PCDIMLIST *pdl) -{ - int i; - int ndims; - assert(pdl); - assert(pdl->schema); - - /* Compress each dimension as dictated by stats */ - for ( i = 0; i < pdl->schema->ndims; i++ ) - { - pdl->bytes[i] = pc_bytes_decode(pdl->bytes[i]); - } - - return PC_SUCCESS; -} - -PCDIMLIST * -pc_dimlist_from_patch(const PCPATCH *pa) -{ - PCDIMLIST *pdl; - const PCSCHEMA *schema; - int i, j, ndims, npoints; - - assert(pa); - npoints = pa->npoints; - schema = pa->schema; - ndims = schema->ndims; - - /* Uncompressed patches only! */ - if ( pa->compressed ) - pcerror("pc_dimlist_from_patch: cannot operate on compressed patch"); - - /* Cannot handle empty patches */ - if ( npoints == 0 ) return NULL; - - /* Initialize list */ - pdl = pcalloc(sizeof(PCDIMLIST)); - pdl->schema = schema; - pdl->npoints = npoints; - pdl->bytes = pcalloc(ndims * sizeof(PCBYTES)); - - for ( i = 0; i < ndims; i++ ) - { - PCDIMENSION *dim = pc_schema_get_dimension(schema, i); - pdl->bytes[i] = pc_bytes_make(dim, npoints); - for ( j = 0; j < npoints; j++ ) - { - uint8_t *to = pdl->bytes[i].bytes + dim->size * j; - uint8_t *from = pa->data + schema->size * j + dim->byteoffset; - memcpy(to, from, dim->size); - } - } - return pdl; -} - - -uint8_t * -pc_dimlist_serialize(const PCDIMLIST *pdl, size_t *size) -{ - int i; - int ndims = pdl->schema->ndims; - size_t dlsize = 0; - uint8_t *buf; - - for ( i = 0; i < ndims; i++ ) - dlsize += pc_bytes_get_serialized_size(&(pdl->bytes[i])); - - buf = pcalloc(dlsize); - - for ( i = 0; i < ndims ; i++ ) - { - size_t bsize = 0; - pc_bytes_serialize(&(pdl->bytes[i]), buf, &bsize); - buf += bsize; - } - - return buf; -} - - -PCDIMLIST * -pc_dimlist_deserialize(const PCSCHEMA *schema, int npoints, uint8_t *buf, int read_only, int flip_endian) -{ - int i; - size_t size = 0; - int ndims = schema->ndims; - PCDIMLIST *pdl = pcalloc(sizeof(PCDIMLIST)); - pdl->schema = schema; - pdl->npoints = npoints; - pdl->bytes = pcalloc(ndims * sizeof(PCBYTES)); - - for ( i = 0; i < ndims; i++ ) - { - PCDIMENSION *dim = schema->dims[i]; - PCBYTES pcb = pdl->bytes[i]; - pc_bytes_deserialize(buf, dim, &pcb, read_only, flip_endian); - /* pc_bytes_deserialize can't fill in npoints and interpretation */ - pcb.npoints = npoints; - pcb.interpretation = dim->interpretation; - /* move forward to next data area */ - buf += pcb.size; - pdl->bytes[i] = pcb; - } - - return pdl; -} - - - - - - - - - - diff --git a/lib/pc_dimstats.c b/lib/pc_dimstats.c index 9d1a972..134e2ee 100644 --- a/lib/pc_dimstats.c +++ b/lib/pc_dimstats.c @@ -19,6 +19,7 @@ #include #include #include "pc_api_internal.h" +#include "stringbuffer.h" PCDIMSTATS * @@ -38,9 +39,54 @@ pc_dimstats_free(PCDIMSTATS *pds) pcfree(pds->stats); pcfree(pds); } +/* +typedef struct +{ + uint32_t total_runs; + uint32_t total_commonbits; + uint32_t recommended_compression; +} PCDIMSTAT; + +typedef struct +{ + int32_t ndims; + uint32_t total_points; + uint32_t total_patches; + PCDIMSTAT *stats; +} PCDIMSTATS; +*/ + +char * +pc_dimstats_to_string(const PCDIMSTATS *pds) +{ + int i; + stringbuffer_t *sb = stringbuffer_create(); + char *str; + + stringbuffer_aprintf(sb,"{\"ndims\":%d,\"total_points\":%d,\"total_patches\":%d,\"dims\":[", + pds->ndims, + pds->total_points, + pds->total_patches + ); + + for ( i = 0; i < pds->ndims; i++ ) + { + if ( i ) stringbuffer_append(sb, ","); + stringbuffer_aprintf(sb, "{\"total_runs\":%d,\"total_commonbits\":%d,\"recommended_compression\":%d}", + pds->stats[i].total_runs, + pds->stats[i].total_commonbits, + pds->stats[i].recommended_compression + ); + } + stringbuffer_append(sb, "]}"); + + str = stringbuffer_getstringcopy(sb); + stringbuffer_destroy(sb); + return str; +} int -pc_dimstats_update(PCDIMSTATS *pds, const PCDIMLIST *pdl) +pc_dimstats_update(PCDIMSTATS *pds, const PCPATCH_DIMENSIONAL *pdl) { int i, j; uint32_t nelems = pdl->npoints; @@ -77,16 +123,16 @@ pc_dimstats_update(PCDIMSTATS *pds, const PCDIMLIST *pdl) if ( dim->interpretation != PC_DOUBLE ) { /* If sigbits is better than 4:1, use that */ - if ( raw_size/sigbits_size > 4.0 ) + if ( raw_size/sigbits_size > 1.6 ) { pds->stats[i].recommended_compression = PC_DIM_SIGBITS; } /* If RLE size is even better, use that. */ - else if ( raw_size/rle_size > 4.0 ) + if ( raw_size/rle_size > 4.0 ) { pds->stats[i].recommended_compression = PC_DIM_RLE; } } } return PC_SUCCESS; -} +} \ No newline at end of file diff --git a/lib/pc_patch.c b/lib/pc_patch.c index 581e867..be059b1 100644 --- a/lib/pc_patch.c +++ b/lib/pc_patch.c @@ -9,86 +9,25 @@ ***********************************************************************/ #include +#include #include "pc_api_internal.h" #include "stringbuffer.h" -PCPATCH * -pc_patch_make(const PCSCHEMA *s) -{ - PCPATCH *pch; - uint32_t maxpoints = PCPATCH_DEFAULT_MAXPOINTS; - size_t datasize; - - if ( ! s ) - { - pcerror("null schema passed into pc_patch_make"); - return NULL; - } - - /* Width of the data area */ - if ( ! s->size ) - { - pcerror("invalid size calculation in pc_patch_make"); - return NULL; - } - - /* Make our own data area */ - pch = pcalloc(sizeof(PCPATCH)); - pch->compressed = PC_FALSE; - datasize = s->size * maxpoints; - pch->data = pcalloc(datasize); - pch->datasize = datasize; - /* Initialize bounds */ - pch->xmin = pch->ymin = MAXFLOAT; - pch->xmax = pch->ymax = -1 * MAXFLOAT; - - /* Set up basic info */ - pch->readonly = PC_FALSE; - pch->npoints = 0; - pch->maxpoints = maxpoints; - pch->schema = s; - return pch; -} -static int -pc_patch_compute_extent_uncompressed(PCPATCH *patch) -{ - int i; - PCPOINT *pt = pc_point_from_data(patch->schema, patch->data); - /* Initialize bounds */ - patch->xmin = patch->ymin = MAXFLOAT; - patch->xmax = patch->ymax = -1 * MAXFLOAT; - - /* Calculate bounds */ - for ( i = 0; i < patch->npoints; i++ ) - { - double x, y; - /* Just push the data buffer forward by one point at a time */ - pt->data = patch->data + i * patch->schema->size; - x = pc_point_get_x(pt); - y = pc_point_get_y(pt); - if ( patch->xmin > x ) patch->xmin = x; - if ( patch->ymin > y ) patch->ymin = y; - if ( patch->xmax < x ) patch->xmax = x; - if ( patch->ymax < y ) patch->ymax = y; - } - return PC_SUCCESS; -} - -static int +int pc_patch_compute_extent(PCPATCH *patch) { - switch( patch->schema->compression ) + switch( patch->type ) { case PC_NONE: - return pc_patch_compute_extent_uncompressed(patch); + return pc_patch_uncompressed_compute_extent((PCPATCH_UNCOMPRESSED*)patch); case PC_GHT: return PC_FAILURE; case PC_DIMENSIONAL: - return PC_FAILURE; + return pc_patch_dimensional_compute_extent((PCPATCH_DIMENSIONAL*)patch); } return PC_FAILURE; } @@ -96,397 +35,97 @@ pc_patch_compute_extent(PCPATCH *patch) void pc_patch_free(PCPATCH *patch) { - if ( ! patch->readonly ) + switch( patch->type ) { - pcfree(patch->data); + case PC_NONE: + { + pc_patch_uncompressed_free((PCPATCH_UNCOMPRESSED*)patch); + break; + } + case PC_GHT: + { + pcerror("pc_patch_free: GHT not supported"); + break; + } + case PC_DIMENSIONAL: + { + pc_patch_dimensional_free((PCPATCH_DIMENSIONAL*)patch); + break; + } + default: + { + pcerror("pc_patch_free: unknown compression type %d", patch->type); + break; + } } - pcfree(patch); } -int -pc_patch_add_point(PCPATCH *c, const PCPOINT *p) -{ - size_t sz; - uint8_t *ptr; - double x, y; - - if ( ! ( c && p ) ) - { - pcerror("pc_patch_add_point: null point or patch argument"); - return PC_FAILURE; - } - - if ( c->schema->pcid != p->schema->pcid ) - { - pcerror("pc_patch_add_point: pcids of point (%d) and patch (%d) not equal", c->schema->pcid, p->schema->pcid); - return PC_FAILURE; - } - - if ( c->readonly ) - { - pcerror("pc_patch_add_point: cannot add point to readonly patch"); - return PC_FAILURE; - } - - if ( c->compressed && c->schema->compression != PC_NONE ) - { - pcerror("pc_patch_add_point: cannot add point to compressed patch"); - return PC_FAILURE; - } - - sz = c->schema->size; - - /* Double the data buffer if it's already full */ - if ( c->npoints == c->maxpoints ) - { - c->maxpoints *= 2; - c->datasize = c->maxpoints * sz; - c->data = pcrealloc(c->data, c->datasize); - } - - /* Copy the data buffer from point to patch */ - ptr = c->data + sz * c->npoints; - memcpy(ptr, p->data, sz); - c->npoints += 1; - - /* Update bounding box */ - x = pc_point_get_x(p); - y = pc_point_get_y(p); - if ( c->xmin > x ) c->xmin = x; - if ( c->ymin > y ) c->ymin = y; - if ( c->xmax < x ) c->xmax = x; - if ( c->ymax < y ) c->ymax = y; - - return PC_SUCCESS; -} - - PCPATCH * -pc_patch_from_points(const PCPOINTLIST *pl) +pc_patch_from_pointlist(const PCPOINTLIST *ptl) { - PCPATCH *pch; - const PCSCHEMA *s; - uint8_t *ptr; - int i; - uint32_t numpts; - - if ( ! pl ) - { - pcerror("null PCPOINTLIST passed into pc_patch_from_points"); - return NULL; - } - - numpts = pl->npoints; - if ( ! numpts ) - { - pcerror("zero size PCPOINTLIST passed into pc_patch_from_points"); - return NULL; - } - - /* Assume the first PCSCHEMA is the same as the rest for now */ - /* We will check this as we go along */ - s = pl->points[0]->schema; - - /* Confirm we have a schema pointer */ - if ( ! s ) - { - pcerror("pc_patch_from_points: null schema encountered"); - return NULL; - } - - /* Confirm width of a point data buffer */ - if ( ! s->size ) - { - pcerror("pc_patch_from_points: invalid point size"); - return NULL; - } - - /* Make our own data area */ - pch = pcalloc(sizeof(PCPATCH)); - pch->datasize = s->size * numpts; - pch->data = pcalloc(pch->datasize); - ptr = pch->data; - - /* Initialize bounds */ - pch->xmin = pch->ymin = MAXFLOAT; - pch->xmax = pch->ymax = -1 * MAXFLOAT; - - /* Set up basic info */ - pch->readonly = PC_FALSE; - pch->compressed = PC_FALSE; - pch->maxpoints = numpts; - pch->schema = s; - pch->npoints = 0; - - for ( i = 0; i < numpts; i++ ) - { - if ( pl->points[i] ) - { - if ( pl->points[i]->schema->pcid != s->pcid ) - { - pcerror("pc_patch_from_points: points do not share a schema"); - return NULL; - } - memcpy(ptr, pl->points[i]->data, s->size); - pch->npoints++; - ptr += s->size; - } - else - { - pcwarn("pc_patch_from_points: encountered null point"); - } - } - - if ( ! pc_patch_compute_extent(pch) ) - { - pcerror("pc_patch_from_points: failed to compute patch extent"); - return NULL; - } - - return pch; -} - - - -PCPOINTLIST * -pc_patch_to_points_uncompressed(const PCPATCH *patch) -{ - int i; - PCPOINTLIST *pl; - size_t pt_size = patch->schema->size; - uint32_t npoints = patch->npoints; - - pl = pc_pointlist_make(npoints); - for ( i = 0; i < npoints; i++ ) - { - pc_pointlist_add_point(pl, pc_point_from_data(patch->schema, patch->data + i*pt_size)); - } - return pl; + return (PCPATCH*)pc_patch_uncompressed_from_pointlist(ptl); } PCPOINTLIST * -pc_patch_to_points(const PCPATCH *patch) +pc_patch_to_pointlist(const PCPATCH *patch) { - uint32_t compression = patch->schema->compression; - - if ( ! patch->compressed ) - return pc_patch_to_points_uncompressed(patch); - - switch ( compression ) + switch ( patch->type ) { case PC_NONE: { - return pc_patch_to_points_uncompressed(patch); + return pc_patch_uncompressed_to_pointlist((PCPATCH_UNCOMPRESSED*)patch); } case PC_GHT: { - // return pc_patch_to_points_ght(patch); + // return pc_patch_to_pointlist_ght(patch); } case PC_DIMENSIONAL: { - // return pc_patch_to_points_dimensional(patch); + PCPATCH_UNCOMPRESSED *pch = pc_patch_uncompressed_from_dimensional((PCPATCH_DIMENSIONAL*)patch); + PCPOINTLIST *ptl = pc_patch_uncompressed_to_pointlist((PCPATCH_UNCOMPRESSED*)patch); + pc_patch_uncompressed_free(pch); + return ptl; } } /* Don't get here */ - pcerror("pc_patch_to_points: unsupported compression type %d", compression); - return NULL; -} - -#if 0 -static PCPATCH * -pc_patch_compress_dimensional(const PCPATCH *patch) -{ - PCDIMLIST *pdl; - PCDIMSTATS *pds = NULL; - int rv; - - if ( patch->compressed ) - pcerror("pc_patch_compress_dimensional cannot work on previously compressed data"); - - pdl = pc_dimlist_from_patch(patch); - rv = pc_dimlist_encode(pdl, &pds); - - - return NULL; -} -#endif - -static PCPATCH * -pc_patch_compress_ght(const PCPATCH *patch) -{ - pcerror("pc_patch_compress_ght unimplemented"); - return NULL; -} - - -PCPATCH * -pc_patch_compress(const PCPATCH *patch) -{ - uint32_t compression = patch->schema->compression; - - if ( patch->compressed ) - return pc_patch_clone(patch); - - switch ( compression ) - { - case PC_NONE: - { - PCPATCH *newpatch = pc_patch_clone(patch); - newpatch->compressed = PC_TRUE; - return newpatch; - } - case PC_GHT: - { - return pc_patch_compress_ght(patch); - } - case PC_DIMENSIONAL: - { - // return pc_patch_compress_dimensional(patch); - } - } - - /* Don't get here */ - pcerror("pc_patch_compress: unknown compression type %d", compression); + pcerror("pc_patch_to_pointlist: unsupported compression type %d", patch->type); return NULL; } PCPATCH * -pc_patch_clone(const PCPATCH *patch) +pc_patch_compress(const PCPATCH *patch, void *userdata) { - PCPATCH *newpatch = pcalloc(sizeof(PCPATCH)); - memcpy(newpatch, patch, sizeof(PCPATCH)); - newpatch->data = pcalloc(newpatch->datasize); - memcpy(newpatch->data, patch->data, newpatch->datasize); - return newpatch; + uint32_t schema_compression = patch->schema->compression; + uint32_t patch_compression = patch->type; + + if ( schema_compression == PC_DIMENSIONAL && + patch_compression == PC_NONE ) + { + PCPATCH_DIMENSIONAL *pcdu = pc_patch_dimensional_from_uncompressed((PCPATCH_UNCOMPRESSED*)patch); + PCPATCH_DIMENSIONAL *pcdd = pc_patch_dimensional_compress(pcdu, (PCDIMSTATS*)userdata); + pc_patch_dimensional_free(pcdu); + return (PCPATCH*)pcdd; + } + + if ( schema_compression == PC_DIMENSIONAL && + patch_compression == PC_DIMENSIONAL ) + { + return (PCPATCH*)pc_patch_dimensional_compress((PCPATCH_DIMENSIONAL*)patch, (PCDIMSTATS*)userdata); + } + + if ( schema_compression == PC_NONE && + patch_compression == PC_NONE ) + { + return (PCPATCH*)patch; + } + + pcerror("pc_patch_compress: cannot convert patch compressed %d to compressed %d", patch_compression, schema_compression); + return NULL; } -static PCPATCH * -pc_patch_from_wkb_uncompressed(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize) -{ - /* - byte: endianness (1 = NDR, 0 = XDR) - uint32: pcid (key to POINTCLOUD_SCHEMAS) - uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT) - uint32: npoints - pcpoint[]: data (interpret relative to pcid) - */ - static size_t hdrsz = 1+4+4+4; /* endian + pcid + compression + npoints */ - PCPATCH *patch; - uint8_t *data; - uint8_t swap_endian = (wkb[0] != machine_endian()); - uint32_t npoints; - - if ( wkb_get_compression(wkb) != PC_NONE ) - { - pcerror("pc_patch_from_wkb_uncompressed: call with wkb that is not uncompressed"); - return NULL; - } - - npoints = wkb_get_npoints(wkb); - if ( (wkbsize - hdrsz) != (s->size * npoints) ) - { - pcerror("pc_patch_from_wkb_uncompressed: wkb size and expected data size do not match"); - return NULL; - } - - if ( swap_endian ) - { - data = uncompressed_bytes_flip_endian(wkb+hdrsz, s, npoints); - } - else - { - data = pcalloc(npoints * s->size); - memcpy(data, wkb+hdrsz, npoints*s->size); - } - - patch = pcalloc(sizeof(PCPATCH)); - patch->npoints = npoints; - patch->maxpoints = npoints; - patch->schema = s; - patch->compressed = PC_TRUE; /* It's in whatever compression it came in */ - patch->datasize = (wkbsize - hdrsz); - patch->data = data; - patch->readonly = PC_FALSE; - - if ( PC_FAILURE == pc_patch_compute_extent(patch) ) - { - pcerror("pc_patch_compute_extent failed"); - } - - return patch; -} - -#if 0 -static PCPATCH * -pc_patch_from_wkb_dimensional(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize) -{ - /* - byte: endianness (1 = NDR, 0 = XDR) - uint32: pcid (key to POINTCLOUD_SCHEMAS) - uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT) - uint32: npoints - pcpoint[]: data (interpret relative to pcid) - */ - static size_t hdrsz = 1+4+4+4; /* endian + pcid + compression + npoints */ - PCPATCH *patch; - uint8_t *data; - uint8_t swap_endian = (wkb[0] != machine_endian()); - uint32_t npoints; - size_t datasize = wkbsize - hdrsz; - - if ( wkb_get_compression(wkb) != PC_DIMENSIONAL ) - { - pcerror("pc_patch_from_wkb_dimensional: call with wkb that is not dimensionally compressed"); - return NULL; - } - - npoints = wkb_get_npoints(wkb); - -#if 0 -typedef struct -{ - int8_t readonly; - uint32_t npoints; /* How many points we have */ - uint32_t maxpoints; /* How man points we can hold (or 0 for read-only) */ - const PCSCHEMA *schema; - double xmin, xmax, ymin, ymax; - uint8_t compressed; /* Has compression been applied to the data buffer? */ - size_t datasize; - uint8_t *data; /* A serialized version of the data */ -} PCPATCH; -#endif - - - if ( swap_endian ) - { - data = uncompressed_bytes_flip_endian(wkb+hdrsz, s, npoints); - } - else - { - data = pcalloc(datasize); - memcpy(data, wkb+hdrsz, datasize); - } - - patch = pcalloc(sizeof(PCPATCH)); - patch->npoints = npoints; - patch->maxpoints = 0; - patch->schema = s; - patch->compressed = PC_TRUE; /* It's in whatever compression it came in */ - patch->datasize = datasize; - patch->data = data; - - if ( PC_FAILURE == pc_patch_compute_extent(patch) ) - { - pcerror("pc_patch_compute_extent failed"); - } - - return patch; -} -#endif - PCPATCH * pc_patch_from_wkb(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize) { @@ -503,13 +142,14 @@ pc_patch_from_wkb(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize) pcerror("pc_patch_from_wkb: zero length wkb"); } + /* + * It is possible for the WKB compression to be different from the + * schema compression at this point. The schema compression is only + * forced at serialization time. + */ pcid = wkb_get_pcid(wkb); compression = wkb_get_compression(wkb); - // if ( compression != s->compression ) - // { - // pcerror("pc_patch_from_wkb: wkb compression (%d) not consistent with schema compression (%d)", compression, s->compression); - // } if ( pcid != s->pcid ) { pcerror("pc_patch_from_wkb: wkb pcid (%d) not consistent with schema pcid (%d)", pcid, s->pcid); @@ -519,18 +159,17 @@ pc_patch_from_wkb(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize) { case PC_NONE: { - return pc_patch_from_wkb_uncompressed(s, wkb, wkbsize); + return pc_patch_uncompressed_from_wkb(s, wkb, wkbsize); + } + case PC_DIMENSIONAL: + { + return pc_patch_dimensional_from_wkb(s, wkb, wkbsize); } case PC_GHT: { pcerror("pc_patch_from_wkb: GHT compression not yet supported"); return NULL; } - case PC_DIMENSIONAL: - { - pcerror("pc_patch_from_wkb: Dimensional compression not yet supported"); - return NULL; - } } /* Don't get here */ @@ -538,31 +177,7 @@ pc_patch_from_wkb(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize) return NULL; } -static uint8_t * -pc_patch_to_wkb_uncompressed(const PCPATCH *patch, size_t *wkbsize) -{ - /* - byte: endianness (1 = NDR, 0 = XDR) - uint32: pcid (key to POINTCLOUD_SCHEMAS) - uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT) - uint32: npoints - uchar[]: data (interpret relative to pcid) - */ - char endian = machine_endian(); - /* endian + pcid + compression + npoints + datasize */ - size_t size = 1 + 4 + 4 + 4 + patch->datasize; - uint8_t *wkb = pcalloc(size); - uint32_t compression = patch->schema->compression; - uint32_t npoints = patch->npoints; - uint32_t pcid = patch->schema->pcid; - wkb[0] = endian; /* Write endian flag */ - memcpy(wkb + 1, &pcid, 4); /* Write PCID */ - memcpy(wkb + 5, &compression, 4); /* Write compression */ - memcpy(wkb + 9, &npoints, 4); /* Write npoints */ - memcpy(wkb + 13, patch->data, patch->datasize); /* Write data */ - if ( wkbsize ) *wkbsize = size; - return wkb; -} + uint8_t * pc_patch_to_wkb(const PCPATCH *patch, size_t *wkbsize) @@ -573,23 +188,21 @@ pc_patch_to_wkb(const PCPATCH *patch, size_t *wkbsize) uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT) uchar[]: data (interpret relative to pcid and compression) */ - - switch ( patch->schema->compression ) + switch ( patch->type ) { case PC_NONE: { - return pc_patch_to_wkb_uncompressed(patch, wkbsize); + return pc_patch_uncompressed_to_wkb((PCPATCH_UNCOMPRESSED*)patch, wkbsize); + } + case PC_DIMENSIONAL: + { + return pc_patch_dimensional_to_wkb((PCPATCH_DIMENSIONAL*)patch, wkbsize); } case PC_GHT: { pcerror("pc_patch_to_wkb: GHT compression not yet supported"); return NULL; } - case PC_DIMENSIONAL: - { - pcerror("pc_patch_to_wkb: Dimensional compression not yet supported"); - return NULL; - } } pcerror("pc_patch_to_wkb: unknown compression requested '%d'", patch->schema->compression); return NULL; @@ -598,45 +211,14 @@ pc_patch_to_wkb(const PCPATCH *patch, size_t *wkbsize) char * pc_patch_to_string(const PCPATCH *patch) { - /* ( : (, , , ), (, , , ))*/ - stringbuffer_t *sb = stringbuffer_create(); - PCPOINTLIST *pl; - char *str; - int i, j; - - pl = pc_patch_to_points(patch); - stringbuffer_aprintf(sb, "[ %d : ", patch->schema->pcid); - for ( i = 0; i < pl->npoints; i++ ) - { - PCPOINT *pt = pl->points[i]; - if ( i ) - { - stringbuffer_append(sb, ", "); - } - stringbuffer_append(sb, "("); - for ( j = 0; j < pt->schema->ndims; j++ ) - { - double d; - if ( ! pc_point_get_double_by_index(pt, j, &d)) - { - pcerror("pc_patch_to_string: unable to read double at index %d", j); - } - if ( j ) - { - stringbuffer_append(sb, ", "); - } - stringbuffer_aprintf(sb, "%g", d); - } - stringbuffer_append(sb, ")"); - } - stringbuffer_append(sb, " ]"); - - /* All done, copy and clean up */ - pc_pointlist_free(pl); - str = stringbuffer_getstringcopy(sb); - stringbuffer_destroy(sb); - - return str; + switch( patch->type ) + { + case PC_NONE: + return pc_patch_uncompressed_to_string((PCPATCH_UNCOMPRESSED*)patch); + case PC_DIMENSIONAL: + return pc_patch_dimensional_to_string((PCPATCH_DIMENSIONAL*)patch); + } + pcerror("pc_patch_to_string: unsupported compression %d requested", patch->type); } static uint8_t * diff --git a/lib/pc_patch_dimensional.c b/lib/pc_patch_dimensional.c new file mode 100644 index 0000000..182edaa --- /dev/null +++ b/lib/pc_patch_dimensional.c @@ -0,0 +1,283 @@ +/*********************************************************************** +* pc_patch_dimensional.c +* +* Pointclound patch handling. Create, get and set values from the +* basic PCPATCH structure. +* +* Portions Copyright (c) 2012, OpenGeo +* +***********************************************************************/ + +#include +#include +#include "pc_api_internal.h" + +/* +typedef struct +{ + int type; + int8_t readonly; + const PCSCHEMA *schema; + uint32_t npoints; + double xmin, xmax, ymin, ymax; + PCBYTES *bytes; +} PCPATCH_DIMENSIONAL; +*/ + + +size_t +pc_patch_dimensional_serialized_size(const PCPATCH *patch) +{ + PCPATCH_DIMENSIONAL *p = (PCPATCH_DIMENSIONAL*)patch; + int i; + size_t size = 0; + for ( i = 0; i < p->schema->ndims; i++ ) + { + size += pc_bytes_serialized_size(&(p->bytes[i])); + } + return size; +} + + +char * +pc_patch_dimensional_to_string(const PCPATCH_DIMENSIONAL *pa) +{ + PCPATCH_UNCOMPRESSED *patch = pc_patch_uncompressed_from_dimensional(pa); + char *str = pc_patch_uncompressed_to_string(patch); + pc_patch_uncompressed_free(patch); + return str; +} + +PCPATCH_DIMENSIONAL * +pc_patch_dimensional_from_uncompressed(const PCPATCH_UNCOMPRESSED *pa) +{ + PCPATCH_DIMENSIONAL *pdl; + const PCSCHEMA *schema; + int i, j, ndims, npoints; + + assert(pa); + npoints = pa->npoints; + schema = pa->schema; + ndims = schema->ndims; + + /* Cannot handle empty patches */ + if ( npoints == 0 ) return NULL; + + /* Initialize list */ + pdl = pcalloc(sizeof(PCPATCH_DIMENSIONAL)); + pdl->schema = schema; + pdl->npoints = npoints; + pdl->bytes = pcalloc(ndims * sizeof(PCBYTES)); + pdl->readonly = PC_FALSE; + pdl->type = PC_DIMENSIONAL; + pdl->xmin = pa->xmin; + pdl->xmax = pa->xmax; + pdl->ymin = pa->ymin; + pdl->ymax = pa->ymax; + + for ( i = 0; i < ndims; i++ ) + { + PCDIMENSION *dim = pc_schema_get_dimension(schema, i); + pdl->bytes[i] = pc_bytes_make(dim, npoints); + for ( j = 0; j < npoints; j++ ) + { + uint8_t *to = pdl->bytes[i].bytes + dim->size * j; + uint8_t *from = pa->data + schema->size * j + dim->byteoffset; + memcpy(to, from, dim->size); + } + } + return pdl; +} + +PCPATCH_DIMENSIONAL * +pc_patch_dimensional_compress(const PCPATCH_DIMENSIONAL *pdl, PCDIMSTATS *pds) +{ + int i; + int ndims = pdl->schema->ndims; + PCPATCH_DIMENSIONAL *pdl_compressed; + + assert(pdl); + assert(pdl->schema); + + if ( ! pds ) + pds = pc_dimstats_make(pdl->schema); + + /* Still sampling, update stats */ + if ( pds->total_points < PCDIMSTATS_MIN_SAMPLE ) + pc_dimstats_update(pds, pdl); + + pdl_compressed = pcalloc(sizeof(PCPATCH_DIMENSIONAL)); + memcpy(pdl_compressed, pdl, sizeof(PCPATCH_DIMENSIONAL)); + pdl_compressed->bytes = pcalloc(ndims*sizeof(PCBYTES)); + + /* Compress each dimension as dictated by stats */ + for ( i = 0; i < ndims; i++ ) + { + pdl_compressed->bytes[i] = pc_bytes_encode(pdl->bytes[i], pds->stats[i].recommended_compression); + } + + return pdl_compressed; +} + +PCPATCH_DIMENSIONAL * +pc_patch_dimensional_decompress(const PCPATCH_DIMENSIONAL *pdl) +{ + int i; + int ndims = pdl->schema->ndims; + PCPATCH_DIMENSIONAL *pdl_decompressed; + + assert(pdl); + assert(pdl->schema); + + pdl_decompressed = pcalloc(sizeof(PCPATCH_DIMENSIONAL)); + memcpy(pdl_decompressed, pdl, sizeof(PCPATCH_DIMENSIONAL)); + pdl_decompressed->bytes = pcalloc(ndims*sizeof(PCBYTES)); + + /* Compress each dimension as dictated by stats */ + for ( i = 0; i < ndims; i++ ) + { + pdl_decompressed->bytes[i] = pc_bytes_decode(pdl->bytes[i]); + } + + return pdl_decompressed; +} + +void +pc_patch_dimensional_free(PCPATCH_DIMENSIONAL *pdl) +{ + int i; + assert(pdl); + assert(pdl->schema); + + if ( pdl->bytes ) + { + for ( i = 0; i < pdl->schema->ndims; i++ ) + pc_bytes_free(pdl->bytes[i]); + + pcfree(pdl->bytes); + } + + pcfree(pdl); +} + +int +pc_patch_dimensional_compute_extent(PCPATCH_DIMENSIONAL *pdl) +{ + int i; + double xmin, xmax, ymin, ymax; + int rv; + PCBYTES *pcb; + + assert(pdl); + assert(pdl->schema); + + /* Get x extremes */ + pcb = &(pdl->bytes[pdl->schema->x_position]); + rv = pc_bytes_minmax(pcb, xmin, xmax); + pdl->xmin = xmin; + pdl->xmax = xmax; + + /* Get y extremes */ + pcb = &(pdl->bytes[pdl->schema->y_position]); + rv = pc_bytes_minmax(pcb, ymin, ymax); + pdl->ymin = ymin; + pdl->ymax = ymax; + + return PC_SUCCESS; +} + +uint8_t * +pc_patch_dimensional_to_wkb(const PCPATCH_DIMENSIONAL *patch, size_t *wkbsize) +{ + /* + byte: endianness (1 = NDR, 0 = XDR) + uint32: pcid (key to POINTCLOUD_SCHEMAS) + uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT) + uint32: npoints + dimensions[]: pcbytes (interpret relative to pcid and compressions) + */ + int ndims = patch->schema->ndims; + int i; + uint8_t *buf; + char endian = machine_endian(); + /* endian + pcid + compression + npoints + datasize */ + size_t size = 1 + 4 + 4 + 4 + pc_patch_dimensional_serialized_size((PCPATCH*)patch); + uint8_t *wkb = pcalloc(size); + uint32_t compression = patch->type; + uint32_t npoints = patch->npoints; + uint32_t pcid = patch->schema->pcid; + wkb[0] = endian; /* Write endian flag */ + memcpy(wkb + 1, &pcid, 4); /* Write PCID */ + memcpy(wkb + 5, &compression, 4); /* Write compression */ + memcpy(wkb + 9, &npoints, 4); /* Write npoints */ + + buf = wkb + 13; + for ( i = 0; i < ndims; i++ ) + { + size_t bsz; + PCBYTES *pcb = &(patch->bytes[i]); + pc_bytes_serialize(pcb, buf, &bsz); + buf += bsz; + } + + if ( wkbsize ) *wkbsize = size; + return wkb; +} + + +PCPATCH * +pc_patch_dimensional_from_wkb(const PCSCHEMA *schema, const uint8_t *wkb, size_t wkbsize) +{ + /* + byte: endianness (1 = NDR, 0 = XDR) + uint32: pcid (key to POINTCLOUD_SCHEMAS) + uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT) + uint32: npoints + dimensions[]: dims (interpret relative to pcid and compressions) + */ + static size_t hdrsz = 1+4+4+4; /* endian + pcid + compression + npoints */ + PCPATCH_DIMENSIONAL *patch; + uint8_t swap_endian = (wkb[0] != machine_endian()); + uint32_t npoints, ndims; + const uint8_t *buf; + int i; + + if ( wkb_get_compression(wkb) != PC_DIMENSIONAL ) + { + pcerror("pc_patch_dimensional_from_wkb: call with wkb that is not dimensionally compressed"); + return NULL; + } + + npoints = wkb_get_npoints(wkb); + ndims = schema->ndims; + + patch = pcalloc(sizeof(PCPATCH_DIMENSIONAL)); + patch->npoints = npoints; + patch->type = PC_DIMENSIONAL; + patch->schema = schema; + patch->readonly = PC_FALSE; + patch->bytes = pcalloc(ndims*sizeof(PCBYTES)); + + buf = wkb+hdrsz; + for ( i = 0; i < ndims; i++ ) + { + PCBYTES *pcb = &(patch->bytes[i]); + pc_bytes_deserialize(buf, schema->dims[i], pcb, PC_FALSE /*readonly*/, swap_endian); + pcb->npoints = npoints; + buf += pcb->size; + } + + if ( PC_FAILURE == pc_patch_dimensional_compute_extent(patch) ) + pcerror("pc_patch_dimensional_compute_extent failed"); + + return (PCPATCH*)patch; +} + +PCPATCH_DIMENSIONAL * +pc_patch_dimensional_from_pointlist(const PCPOINTLIST *pdl) +{ + PCPATCH_UNCOMPRESSED *patch = pc_patch_uncompressed_from_pointlist(pdl); + PCPATCH_DIMENSIONAL *dimpatch = pc_patch_dimensional_from_uncompressed(patch); + pc_patch_uncompressed_free(patch); + return dimpatch; +} \ No newline at end of file diff --git a/lib/pc_patch_uncompressed.c b/lib/pc_patch_uncompressed.c new file mode 100644 index 0000000..1fc8d6e --- /dev/null +++ b/lib/pc_patch_uncompressed.c @@ -0,0 +1,420 @@ +/*********************************************************************** +* pc_patch_uncompressed.c +* +* Portions Copyright (c) 2012, OpenGeo +* +***********************************************************************/ + +#include +#include "pc_api_internal.h" +#include "stringbuffer.h" + + +char * +pc_patch_uncompressed_to_string(const PCPATCH_UNCOMPRESSED *patch) +{ + /* { "pcid":1, "points":[[, , , ],[, , , ]] }*/ + stringbuffer_t *sb = stringbuffer_create(); + PCPOINTLIST *pl; + char *str; + int i, j; + + pl = pc_patch_uncompressed_to_pointlist(patch); + stringbuffer_aprintf(sb, "{\"pcid\":%d,\"pts\":[", patch->schema->pcid); + for ( i = 0; i < pl->npoints; i++ ) + { + PCPOINT *pt = pc_pointlist_get_point(pl, i); + if ( i ) + { + stringbuffer_append(sb, ","); + } + stringbuffer_append(sb, "["); + for ( j = 0; j < pt->schema->ndims; j++ ) + { + double d; + if ( ! pc_point_get_double_by_index(pt, j, &d)) + { + pcerror("pc_patch_to_string: unable to read double at index %d", j); + } + if ( j ) + { + stringbuffer_append(sb, ","); + } + stringbuffer_aprintf(sb, "%g", d); + } + stringbuffer_append(sb, "]"); + } + stringbuffer_append(sb, "]}"); + + /* All done, copy and clean up */ + pc_pointlist_free(pl); + str = stringbuffer_getstringcopy(sb); + stringbuffer_destroy(sb); + + return str; +} + +uint8_t * +pc_patch_uncompressed_to_wkb(const PCPATCH_UNCOMPRESSED *patch, size_t *wkbsize) +{ + /* + byte: endianness (1 = NDR, 0 = XDR) + uint32: pcid (key to POINTCLOUD_SCHEMAS) + uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT) + uint32: npoints + uchar[]: data (interpret relative to pcid) + */ + char endian = machine_endian(); + /* endian + pcid + compression + npoints + datasize */ + size_t size = 1 + 4 + 4 + 4 + patch->datasize; + uint8_t *wkb = pcalloc(size); + uint32_t compression = patch->type; + uint32_t npoints = patch->npoints; + uint32_t pcid = patch->schema->pcid; + wkb[0] = endian; /* Write endian flag */ + memcpy(wkb + 1, &pcid, 4); /* Write PCID */ + memcpy(wkb + 5, &compression, 4); /* Write compression */ + memcpy(wkb + 9, &npoints, 4); /* Write npoints */ + memcpy(wkb + 13, patch->data, patch->datasize); /* Write data */ + if ( wkbsize ) *wkbsize = size; + return wkb; +} + + +PCPATCH * +pc_patch_uncompressed_from_wkb(const PCSCHEMA *s, const uint8_t *wkb, size_t wkbsize) +{ + /* + byte: endianness (1 = NDR, 0 = XDR) + uint32: pcid (key to POINTCLOUD_SCHEMAS) + uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT) + uint32: npoints + pcpoint[]: data (interpret relative to pcid) + */ + static size_t hdrsz = 1+4+4+4; /* endian + pcid + compression + npoints */ + PCPATCH_UNCOMPRESSED *patch; + uint8_t *data; + uint8_t swap_endian = (wkb[0] != machine_endian()); + uint32_t npoints; + + if ( wkb_get_compression(wkb) != PC_NONE ) + { + pcerror("pc_patch_uncompressed_from_wkb: call with wkb that is not uncompressed"); + return NULL; + } + + npoints = wkb_get_npoints(wkb); + if ( (wkbsize - hdrsz) != (s->size * npoints) ) + { + pcerror("pc_patch_uncompressed_from_wkb: wkb size and expected data size do not match"); + return NULL; + } + + if ( swap_endian ) + { + data = uncompressed_bytes_flip_endian(wkb+hdrsz, s, npoints); + } + else + { + data = pcalloc(npoints * s->size); + memcpy(data, wkb+hdrsz, npoints*s->size); + } + + patch = pcalloc(sizeof(PCPATCH_UNCOMPRESSED)); + patch->npoints = npoints; + patch->type = PC_NONE; + patch->maxpoints = npoints; + patch->schema = s; + patch->datasize = (wkbsize - hdrsz); + patch->data = data; + patch->readonly = PC_FALSE; + + if ( PC_FAILURE == pc_patch_uncompressed_compute_extent(patch) ) + pcerror("pc_patch_uncompressed_compute_extent failed"); + + return (PCPATCH*)patch; +} + +PCPATCH_UNCOMPRESSED * +pc_patch_uncompressed_make(const PCSCHEMA *s) +{ + PCPATCH_UNCOMPRESSED *pch; + uint32_t maxpoints = PCPATCH_DEFAULT_MAXPOINTS; + size_t datasize; + + if ( ! s ) + { + pcerror("null schema passed into pc_patch_uncompressed_make"); + return NULL; + } + + /* Width of the data area */ + if ( ! s->size ) + { + pcerror("invalid size calculation in pc_patch_uncompressed_make"); + return NULL; + } + + /* Make our own data area */ + pch = pcalloc(sizeof(PCPATCH)); + datasize = s->size * maxpoints; + pch->data = pcalloc(datasize); + pch->datasize = datasize; + + /* Initialize bounds */ + pch->xmin = pch->ymin = MAXFLOAT; + pch->xmax = pch->ymax = -1 * MAXFLOAT; + + /* Set up basic info */ + pch->readonly = PC_FALSE; + pch->npoints = 0; + pch->type = PC_NONE; + pch->maxpoints = maxpoints; + pch->schema = s; + return pch; +} + +int +pc_patch_uncompressed_compute_extent(PCPATCH_UNCOMPRESSED *patch) +{ + int i; + PCPOINT *pt = pc_point_from_data(patch->schema, patch->data); + + /* Initialize bounds */ + patch->xmin = patch->ymin = MAXFLOAT; + patch->xmax = patch->ymax = -1 * MAXFLOAT; + + /* Calculate bounds */ + for ( i = 0; i < patch->npoints; i++ ) + { + double x, y; + /* Just push the data buffer forward by one point at a time */ + pt->data = patch->data + i * patch->schema->size; + x = pc_point_get_x(pt); + y = pc_point_get_y(pt); + if ( patch->xmin > x ) patch->xmin = x; + if ( patch->ymin > y ) patch->ymin = y; + if ( patch->xmax < x ) patch->xmax = x; + if ( patch->ymax < y ) patch->ymax = y; + } + + return PC_SUCCESS; +} + +void +pc_patch_uncompressed_free(PCPATCH_UNCOMPRESSED *patch) +{ + if ( ! patch->readonly ) + { + pcfree(patch->data); + } + pcfree(patch); +} + +PCPOINTLIST * +pc_patch_uncompressed_to_pointlist(const PCPATCH_UNCOMPRESSED *patch) +{ + int i; + PCPOINTLIST *pl; + size_t pt_size = patch->schema->size; + uint32_t npoints = patch->npoints; + + pl = pc_pointlist_make(npoints); + for ( i = 0; i < npoints; i++ ) + { + pc_pointlist_add_point(pl, pc_point_from_data(patch->schema, patch->data + i*pt_size)); + } + return pl; +} + +PCPATCH_UNCOMPRESSED * +pc_patch_uncompressed_from_pointlist(const PCPOINTLIST *pl) +{ + PCPATCH_UNCOMPRESSED *pch; + const PCSCHEMA *s; + PCPOINT *pt; + uint8_t *ptr; + int i; + uint32_t numpts; + + if ( ! pl ) + { + pcerror("null PCPOINTLIST passed into pc_patch_uncompressed_from_pointlist"); + return NULL; + } + + numpts = pl->npoints; + if ( ! numpts ) + { + pcerror("zero size PCPOINTLIST passed into pc_patch_uncompressed_from_pointlist"); + return NULL; + } + + /* Assume the first PCSCHEMA is the same as the rest for now */ + /* We will check this as we go along */ + pt = pc_pointlist_get_point(pl, 0); + s = pt->schema; + + /* Confirm we have a schema pointer */ + if ( ! s ) + { + pcerror("pc_patch_uncompressed_from_pointlist: null schema encountered"); + return NULL; + } + + /* Confirm width of a point data buffer */ + if ( ! s->size ) + { + pcerror("pc_patch_uncompressed_from_pointlist: invalid point size"); + return NULL; + } + + /* Make our own data area */ + pch = pcalloc(sizeof(PCPATCH_UNCOMPRESSED)); + pch->datasize = s->size * numpts; + pch->data = pcalloc(pch->datasize); + ptr = pch->data; + + /* Initialize bounds */ + pch->xmin = pch->ymin = MAXFLOAT; + pch->xmax = pch->ymax = -1 * MAXFLOAT; + + /* Set up basic info */ + pch->readonly = PC_FALSE; + pch->maxpoints = numpts; + pch->type = PC_NONE; + pch->schema = s; + pch->npoints = 0; + + for ( i = 0; i < numpts; i++ ) + { + pt = pc_pointlist_get_point(pl, i); + if ( pt ) + { + if ( pt->schema->pcid != s->pcid ) + { + pcerror("pc_patch_uncompressed_from_pointlist: points do not share a schema"); + return NULL; + } + memcpy(ptr, pt->data, s->size); + pch->npoints++; + ptr += s->size; + } + else + { + pcwarn("pc_patch_uncompressed_from_pointlist: encountered null point"); + } + } + + if ( ! pc_patch_compute_extent(pch) ) + { + pcerror("pc_patch_uncompressed_from_pointlist: failed to compute patch extent"); + return NULL; + } + + return pch; +} + + +PCPATCH_UNCOMPRESSED * +pc_patch_uncompressed_from_dimensional(const PCPATCH_DIMENSIONAL *pdl) +{ + int i, j, npoints; + PCPATCH_UNCOMPRESSED *patch; + PCPATCH_DIMENSIONAL *pdl_uncompressed; + const PCSCHEMA *schema; + uint8_t *buf; + + npoints = pdl->npoints; + schema = pdl->schema; + patch = pcalloc(sizeof(PCPATCH_UNCOMPRESSED)); + patch->schema = schema; + patch->npoints = npoints; + patch->maxpoints = npoints; + patch->readonly = PC_FALSE; + patch->type = PC_NONE; + patch->xmin = pdl->xmin; + patch->xmax = pdl->xmax; + patch->ymin = pdl->ymin; + patch->ymax = pdl->ymax; + patch->datasize = schema->size * pdl->npoints; + patch->data = pcalloc(patch->datasize); + buf = patch->data; + + /* Can only read from uncompressed dimensions */ + pdl_uncompressed = pc_patch_dimensional_decompress(pdl); + + for ( i = 0; i < npoints; i++ ) + { + for ( j = 0; j < schema->ndims; j++ ) + { + PCDIMENSION *dim = pc_schema_get_dimension(schema, j); + uint8_t *in = pdl_uncompressed->bytes[j].bytes + dim->size * i; + uint8_t *out = buf + dim->byteoffset; + memcpy(out, in, dim->size); + } + buf += schema->size; + } + + pc_patch_dimensional_free(pdl_uncompressed); + + return patch; +} + + +int +pc_patch_uncompressed_add_point(PCPATCH_UNCOMPRESSED *c, const PCPOINT *p) +{ + size_t sz; + uint8_t *ptr; + double x, y; + + if ( ! ( c && p ) ) + { + pcerror("pc_patch_uncompressed_add_point: null point or patch argument"); + return PC_FAILURE; + } + + if ( c->schema->pcid != p->schema->pcid ) + { + pcerror("pc_patch_uncompressed_add_point: pcids of point (%d) and patch (%d) not equal", c->schema->pcid, p->schema->pcid); + return PC_FAILURE; + } + + if ( c->readonly ) + { + pcerror("pc_patch_uncompressed_add_point: cannot add point to readonly patch"); + return PC_FAILURE; + } + + if ( c->type != PC_NONE ) + { + pcerror("pc_patch_uncompressed_add_point: cannot add point to compressed patch"); + return PC_FAILURE; + } + + sz = c->schema->size; + + /* Double the data buffer if it's already full */ + if ( c->npoints == c->maxpoints ) + { + c->maxpoints *= 2; + c->datasize = c->maxpoints * sz; + c->data = pcrealloc(c->data, c->datasize); + } + + /* Copy the data buffer from point to patch */ + ptr = c->data + sz * c->npoints; + memcpy(ptr, p->data, sz); + c->npoints += 1; + + /* Update bounding box */ + x = pc_point_get_x(p); + y = pc_point_get_y(p); + if ( c->xmin > x ) c->xmin = x; + if ( c->ymin > y ) c->ymin = y; + if ( c->xmax < x ) c->xmax = x; + if ( c->ymax < y ) c->ymax = y; + + return PC_SUCCESS; +} diff --git a/lib/pc_point.c b/lib/pc_point.c index d4cc05e..55df51a 100644 --- a/lib/pc_point.c +++ b/lib/pc_point.c @@ -169,12 +169,12 @@ pc_point_get_y(const PCPOINT *pt) char * pc_point_to_string(const PCPOINT *pt) { - /* ( : , , , )*/ + /* { "pcid":1, "values":[, , , ] }*/ stringbuffer_t *sb = stringbuffer_create(); char *str; int i; - stringbuffer_aprintf(sb, "( %d : ", pt->schema->pcid); + stringbuffer_aprintf(sb, "{\"pcid\":%d,\"pt\":[", pt->schema->pcid); for ( i = 0; i < pt->schema->ndims; i++ ) { double d; @@ -184,11 +184,11 @@ pc_point_to_string(const PCPOINT *pt) } if ( i ) { - stringbuffer_append(sb, ", "); + stringbuffer_append(sb, ","); } stringbuffer_aprintf(sb, "%g", d); } - stringbuffer_append(sb, " )"); + stringbuffer_append(sb, "]}"); str = stringbuffer_getstringcopy(sb); stringbuffer_destroy(sb); return str; diff --git a/lib/pc_pointlist.c b/lib/pc_pointlist.c index 2910f8c..59d6fda 100644 --- a/lib/pc_pointlist.c +++ b/lib/pc_pointlist.c @@ -50,34 +50,41 @@ pc_pointlist_add_point(PCPOINTLIST *pl, PCPOINT *pt) return; } +PCPOINT * +pc_pointlist_get_point(const PCPOINTLIST *pl, int i) +{ + return pl->points[i]; +} PCPOINTLIST * -pc_pointlist_from_dimlist(PCDIMLIST *pdl) +pc_pointlist_from_dimensional(const PCPATCH_DIMENSIONAL *pdl) { PCPOINTLIST *pl; + PCPATCH_DIMENSIONAL *pdl_uncompressed; + const PCSCHEMA *schema = pdl->schema; int i, j, ndims, npoints; assert(pdl); - /* We can only pull off this trick on uncompressed data */ - if ( PC_FAILURE == pc_dimlist_decode(pdl) ) - return NULL; + pdl_uncompressed = pc_patch_dimensional_decompress(pdl); - ndims = pdl->schema->ndims; + ndims = schema->ndims; npoints = pdl->npoints; pl = pc_pointlist_make(npoints); for ( i = 0; i < npoints; i++ ) { - PCPOINT *pt = pc_point_make(pdl->schema); + PCPOINT *pt = pc_point_make(schema); for ( j = 0; j < ndims; j++ ) { - PCDIMENSION *dim = pc_schema_get_dimension(pdl->schema, j); + PCDIMENSION *dim = pc_schema_get_dimension(schema, j); - uint8_t *in = pdl->bytes[j].bytes + dim->size * i; + uint8_t *in = pdl_uncompressed->bytes[j].bytes + dim->size * i; uint8_t *out = pt->data + dim->byteoffset; memcpy(out, in, dim->size); } pc_pointlist_add_point(pl, pt); } + pc_patch_dimensional_free(pdl_uncompressed); + return pl; -} +} \ No newline at end of file diff --git a/lib/pc_schema.c b/lib/pc_schema.c index 3d90da4..864744e 100644 --- a/lib/pc_schema.c +++ b/lib/pc_schema.c @@ -69,9 +69,14 @@ size_t pc_interpretation_size(uint32_t interp) { if ( interp >= 0 && interp < NUM_INTERPRETATIONS ) + { return INTERPRETATION_SIZES[interp]; + } else + { + pcerror("pc_interpretation_size: invalid interpretation"); return 0; + } } /** Allocate clean memory for a PCDIMENSION struct */ @@ -140,7 +145,7 @@ static int pc_schema_set_dimension(PCSCHEMA *s, PCDIMENSION *d) { s->dims[d->position] = d; - hashtable_insert(s->namehash, pcstrdup(d->name), d); + hashtable_insert(s->namehash, d->name, d); } @@ -480,13 +485,3 @@ pc_schema_get_dimension_by_name(const PCSCHEMA *s, const char *name) return hashtable_search(s->namehash, name); } -int -pc_schema_has_name(const PCSCHEMA *s, const char *name) -{ - if ( hashtable_search(s->namehash, name) ) - { - return PC_TRUE; - } - return PC_FALSE; -} - diff --git a/lib/pc_val.c b/lib/pc_val.c index c2b2ade..30b038b 100644 --- a/lib/pc_val.c +++ b/lib/pc_val.c @@ -10,6 +10,24 @@ #include #include "pc_api_internal.h" + + +double +pc_value_from_ptr(const uint8_t *ptr, const PCDIMENSION *dim) +{ + double val = pc_double_from_ptr(ptr, dim->interpretation); + + /* Scale value */ + if ( dim->scale ) + val *= dim->scale; + + /* Offset value */ + if ( dim->offset ) + val += dim->offset; + + return val; +} + double pc_double_from_ptr(const uint8_t *ptr, uint32_t interpretation) { diff --git a/pgsql/expected/pointcloud.out b/pgsql/expected/pointcloud.out index 4218ac0..b1c434d 100644 --- a/pgsql/expected/pointcloud.out +++ b/pgsql/expected/pointcloud.out @@ -79,11 +79,11 @@ SELECT Sum(PC_Get(pt, 'y')) FROM pt_test; (1 row) SELECT PC_AsText(pt) FROM pt_test; - pc_astext ------------------------------ - ( 1 : 0.01, 0.02, 0.03, 4 ) - ( 1 : 0.02, 0.03, 0.03, 5 ) - ( 1 : 0.03, 0.04, 0.03, 6 ) + pc_astext +------------------------------------ + {"pcid":1,"pt":[0.01,0.02,0.03,4]} + {"pcid":1,"pt":[0.02,0.03,0.03,5]} + {"pcid":1,"pt":[0.03,0.04,0.03,6]} (3 rows) CREATE TABLE IF NOT EXISTS pa_test ( @@ -103,10 +103,10 @@ LINE 1: INSERT INTO pa_test (pa) VALUES ('00000000020000000000000002... INSERT INTO pa_test (pa) VALUES ('0000000001000000000000000200000002000000030000000500060000000200000003000000050008'); INSERT INTO pa_test (pa) VALUES ('000000000100000000000000020000000600000007000000050006000000090000000A00000005000A'); SELECT PC_AsText(pa) FROM pa_test; - pc_astext ------------------------------------------------------- - [ 1 : (0.02, 0.03, 0.05, 6), (0.02, 0.03, 0.05, 8) ] - [ 1 : (0.06, 0.07, 0.05, 6), (0.09, 0.1, 0.05, 10) ] + pc_astext +---------------------------------------------------------- + {"pcid":1,"pts":[[0.02,0.03,0.05,6],[0.02,0.03,0.05,8]]} + {"pcid":1,"pts":[[0.06,0.07,0.05,6],[0.09,0.1,0.05,10]]} (2 rows) SELECT PC_Envelope(pa) from pa_test; @@ -119,16 +119,17 @@ SELECT PC_Envelope(pa) from pa_test; SELECT PC_AsText(PC_Patch(pt)) FROM pt_test; pc_astext ----------------------------------------------------------------------------- - [ 1 : (0.01, 0.02, 0.03, 4), (0.02, 0.03, 0.03, 5), (0.03, 0.04, 0.03, 6) ] + {"pcid":1,"pts":[[0.01,0.02,0.03,4],[0.02,0.03,0.03,5],[0.03,0.04,0.03,6]]} (1 row) SELECT PC_AsText(PC_Explode(PC_Patch(pt))) FROM pt_test; - pc_astext ------------------------------ - ( 1 : 0.01, 0.02, 0.03, 4 ) - ( 1 : 0.02, 0.03, 0.03, 5 ) - ( 1 : 0.03, 0.04, 0.03, 6 ) + pc_astext +------------------------------------ + {"pcid":1,"pt":[0.01,0.02,0.03,4]} + {"pcid":1,"pt":[0.02,0.03,0.03,5]} + {"pcid":1,"pt":[0.03,0.04,0.03,6]} (3 rows) +--DROP TABLE pts_collection; --DROP TABLE pt_test; --DROP TABLE pa_test; diff --git a/pgsql/pc_access.c b/pgsql/pc_access.c index 991357c..82e0416 100644 --- a/pgsql/pc_access.c +++ b/pgsql/pc_access.c @@ -134,7 +134,7 @@ pcpatch_from_array(ArrayType *array, FunctionCallInfoData *fcinfo) if ( pl->npoints == 0 ) return NULL; - pa = pc_patch_from_points(pl); + pa = pc_patch_from_pointlist(pl); pc_pointlist_free(pl); return pa; } @@ -154,7 +154,7 @@ Datum pcpatch_from_pcpoint_array(PG_FUNCTION_ARGS) if ( ! pa ) PG_RETURN_NULL(); - serpa = pc_patch_serialize(pa); + serpa = pc_patch_serialize(pa, NULL); pc_patch_free(pa); PG_RETURN_POINTER(serpa); } @@ -281,7 +281,7 @@ Datum pcpoint_agg_final_pcpatch(PG_FUNCTION_ARGS) if ( ! pa ) PG_RETURN_NULL(); - serpa = pc_patch_serialize(pa); + serpa = pc_patch_serialize(pa, NULL); pc_patch_free(pa); PG_RETURN_POINTER(serpa); } @@ -330,7 +330,7 @@ Datum pcpatch_unnest(PG_FUNCTION_ARGS) /* initialize state */ fctx->nextelem = 0; fctx->numelems = patch->npoints; - fctx->pointlist = pc_patch_to_points(patch); + fctx->pointlist = pc_patch_to_pointlist(patch); /* save user context, switch back to function context */ funcctx->user_fctx = fctx; @@ -344,7 +344,7 @@ Datum pcpatch_unnest(PG_FUNCTION_ARGS) if (fctx->nextelem < fctx->numelems) { Datum elem; - PCPOINT *pt = fctx->pointlist->points[fctx->nextelem]; + PCPOINT *pt = pc_pointlist_get_point(fctx->pointlist, fctx->nextelem); SERIALIZED_POINT *serpt = pc_point_serialize(pt); fctx->nextelem++; elem = PointerGetDatum(serpt); diff --git a/pgsql/pc_inout.c b/pgsql/pc_inout.c index cecfb79..dddcb4f 100644 --- a/pgsql/pc_inout.c +++ b/pgsql/pc_inout.c @@ -125,7 +125,7 @@ Datum pcpatch_in(PG_FUNCTION_ARGS) /* Hex-encoded binary */ patch = pc_patch_from_hexwkb(str, strlen(str), fcinfo); pcid_consistent(patch->schema->pcid, pcid); - serpatch = pc_patch_serialize(patch); + serpatch = pc_patch_serialize(patch, NULL); pc_patch_free(patch); } else @@ -196,7 +196,6 @@ Datum pcpoint_from_double_array(PG_FUNCTION_ARGS) uint32 pcid = PG_GETARG_INT32(0); ArrayType *arrptr = PG_GETARG_ARRAYTYPE_P(1); int nelems; - int i; float8 *vals; PCPOINT *pt; PCSCHEMA *schema = pc_schema_from_pcid(pcid, fcinfo); @@ -218,17 +217,9 @@ Datum pcpoint_from_double_array(PG_FUNCTION_ARGS) if ( nelems != schema->ndims || ARR_LBOUND(arrptr)[0] > 1 ) elog(ERROR, "array dimenensions do not match schema dimensions of pcid = %d", pcid); - pt = pc_point_make(schema); vals = (float8*) ARR_DATA_PTR(arrptr); + pt = pc_point_from_double_array(schema, vals, nelems); - for ( i = 0; i < nelems; i++ ) - { - float8 val = vals[i]; - int err = pc_point_set_double_by_index(pt, i, val); - if ( ! err ) - elog(ERROR, "failed to set value into point"); - } - serpt = pc_point_serialize(pt); pc_point_free(pt); pc_schema_free(schema); diff --git a/pgsql/pc_pgsql.c b/pgsql/pc_pgsql.c index ab6027e..31eb397 100644 --- a/pgsql/pc_pgsql.c +++ b/pgsql/pc_pgsql.c @@ -431,19 +431,41 @@ pc_point_deserialize(const SERIALIZED_POINT *serpt, const PCSCHEMA *schema) } -SERIALIZED_PATCH * -pc_patch_serialize(const PCPATCH *pcpch) +size_t +pc_patch_serialized_size(const PCPATCH *patch) +{ + switch( patch->type ) + { + case PC_NONE: + { + PCPATCH_UNCOMPRESSED *upatch = (PCPATCH_UNCOMPRESSED*)patch; + return sizeof(SERIALIZED_PATCH) - 1 + upatch->datasize; + } + case PC_GHT: + { + pcerror("pc_patch_serialized_size: GHT format not yet supported"); + } + case PC_DIMENSIONAL: + { + return sizeof(SERIALIZED_PATCH) - 1 + pc_patch_dimensional_serialized_size(patch); + } + default: + { + pcerror("pc_patch_serialized_size: unknown compresed %d", patch->type); + } + } + return -1; +} + +static SERIALIZED_PATCH * +pc_patch_uncompressed_serialize(const PCPATCH *patch_in) { size_t serpch_size; - PCPATCH *patch; SERIALIZED_PATCH *serpch; + const PCPATCH_UNCOMPRESSED *patch = (PCPATCH_UNCOMPRESSED *)patch_in; - /* Compress uncompressed patches before saving */ - patch = pc_patch_compress(pcpch); - - /* Allocate: size(int4) + pcid(int4) + npoints(int4) + box(4*float8) + data(?) */ - serpch_size = sizeof(SERIALIZED_PATCH) - 1 + patch->datasize; - serpch = palloc(serpch_size); + serpch_size = pc_patch_serialized_size(patch_in); + serpch = pcalloc(serpch_size); /* Copy */ serpch->pcid = patch->schema->pcid; @@ -457,20 +479,103 @@ pc_patch_serialize(const PCPATCH *pcpch) return serpch; } -PCPATCH * -pc_patch_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema) + +static SERIALIZED_PATCH * +pc_patch_dimensional_serialize(const PCPATCH *patch_in) { - PCPATCH *patch; + size_t serpch_size; + SERIALIZED_PATCH *serpch; + int i; + uint8_t *buf; + const PCPATCH_DIMENSIONAL *patch = (PCPATCH_DIMENSIONAL*)patch_in; + + serpch_size = pc_patch_serialized_size((PCPATCH*)patch); + serpch = pcalloc(serpch_size); + + /* Copy basics */ + serpch->pcid = patch->schema->pcid; + serpch->npoints = patch->npoints; + serpch->xmin = patch->xmin; + serpch->ymin = patch->ymin; + serpch->xmax = patch->xmax; + serpch->ymax = patch->ymax; + + /* Copy byte buffers, one by one */ + buf = serpch->data; + for ( i = 0; i < patch->schema->ndims; i++ ) + { + size_t bsize = 0; + PCBYTES *pcb = &(patch->bytes[i]); + pc_bytes_serialize(pcb, buf, &bsize); + buf += bsize; + } + + SET_VARSIZE(serpch, serpch_size); + return serpch; +} + + +/** +* Convert struct to byte array. +* Userdata is currently only PCDIMSTATS, hopefully updated across +* a number of iterations and saved. +*/ +SERIALIZED_PATCH * +pc_patch_serialize(const PCPATCH *patch_in, void *userdata) +{ + PCPATCH *patch = (PCPATCH*)patch_in; + SERIALIZED_PATCH *serpatch; + /* + * Convert the patch to the final target compression, + * which is the one in the schema. + */ + if ( patch->type != patch->schema->compression ) + { + patch = pc_patch_compress(patch_in, userdata); + } + + switch( patch->type ) + { + case PC_NONE: + serpatch = pc_patch_uncompressed_serialize(patch); + break; + case PC_DIMENSIONAL: + { + serpatch = pc_patch_dimensional_serialize(patch); + break; + } + case PC_GHT: + { + pcerror("pc_patch_serialize: GHT compression currently unsupported"); + break; + } + default: + { + pcerror("pc_patch_serialize: unsupported compression type %d", patch->type); + } + } + + if ( patch != patch_in ) + pc_patch_free(patch); + + return serpatch; +} + + + +static PCPATCH * +pc_patch_uncompressed_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema) +{ + PCPATCH_UNCOMPRESSED *patch; /* Reference the external data */ - patch = pcalloc(sizeof(PCPATCH)); + patch = pcalloc(sizeof(PCPATCH_UNCOMPRESSED)); patch->data = (uint8*)serpatch->data; patch->datasize = VARSIZE(serpatch) - sizeof(SERIALIZED_PATCH) + 1; /* Set up basic info */ patch->schema = schema; patch->readonly = true; - patch->compressed = true; patch->npoints = serpatch->npoints; patch->maxpoints = 0; patch->xmin = serpatch->xmin; @@ -478,6 +583,58 @@ pc_patch_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema) patch->xmax = serpatch->xmax; patch->ymax = serpatch->ymax; - return patch; + return (PCPATCH*)patch; } +static PCPATCH * +pc_patch_dimensional_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema) +{ + PCPATCH_DIMENSIONAL *patch; + int i; + const uint8_t *buf; + int ndims = schema->ndims; + int npoints = serpatch->npoints; + + /* Reference the external data */ + patch = pcalloc(sizeof(PCPATCH_DIMENSIONAL)); + + /* Set up basic info */ + patch->schema = schema; + patch->readonly = true; + patch->npoints = npoints; + patch->xmin = serpatch->xmin; + patch->ymin = serpatch->ymin; + patch->xmax = serpatch->xmax; + patch->ymax = serpatch->ymax; + + /* Set up dimensions */ + patch->bytes = pcalloc(ndims * sizeof(PCBYTES)); + buf = serpatch->data; + + for ( i = 0; i < ndims; i++ ) + { + PCBYTES *pcb = &(patch->bytes[i]); + PCDIMENSION *dim = schema->dims[i]; + pc_bytes_deserialize(buf, dim, pcb, true /*readonly*/, false /*flipendian*/); + pcb->npoints = npoints; + buf += pcb->size; + } + + return (PCPATCH*)patch; +} + +PCPATCH * +pc_patch_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema) +{ + switch(schema->compression) + { + case PC_NONE: + return pc_patch_uncompressed_deserialize(serpatch, schema); + case PC_DIMENSIONAL: + return pc_patch_dimensional_deserialize(serpatch, schema); + case PC_GHT: + pcerror("pc_patch_deserialize: GHT compression currently unsupported"); + } + pcerror("pc_patch_deserialize: unsupported compression type"); + return NULL; +} diff --git a/pgsql/pc_pgsql.h b/pgsql/pc_pgsql.h index d9b74f8..c42d2f1 100644 --- a/pgsql/pc_pgsql.h +++ b/pgsql/pc_pgsql.h @@ -83,8 +83,11 @@ PCPOINT* pc_point_from_hexwkb(const char *hexwkb, size_t hexlen, FunctionCallInf /** Create a hex representation of a PCPOINT */ char* pc_point_to_hexwkb(const PCPOINT *pt); +/** How big will this thing be on disk? */ +size_t pc_patch_serialized_size(const PCPATCH *patch); + /** Turn a PCPATCH into a byte buffer suitable for saving in PgSQL */ -SERIALIZED_PATCH* pc_patch_serialize(const PCPATCH *patch); +SERIALIZED_PATCH* pc_patch_serialize(const PCPATCH *patch, void *userdata); /** Turn a byte buffer into a PCPATCH for processing */ PCPATCH* pc_patch_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema);