archive-gh-me/pointcloud
1
0
Fork 0
mirror of https://github.com/pgpointcloud/pointcloud.git synced 2025-12-08 20:36:04 +00:00
Code Issues Projects Releases Wiki Activity

Add dimensional compression routines, right out to the

Browse Source 
patch level. Now to add them to SQL.
This commit is contained in:
Paul Ramsey 2013-03-01 16:42:15 -08:00
parent 716fdd2f50
commit c7060031b5
23 changed files with 1483 additions and 970 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -64,7 +64,8 @@ Patch Binary (Dimensional)
byte: endianness (1 = NDR, 0 = XDR)
uint32: pcid (key to POINTCLOUD_SCHEMAS)
uint32: 1 = dimensional compression
dimensional[]: dimensionally compressed data for each dimension
uint32: npoints
dimensions[]: dimensionally compressed data for each dimension
/* dimensional compression */
byte: dimensional compression type (0 = none, 1 = significant bits, 2 = deflate, 3 = run-length)

5
TODO.md
View File

@ -1,5 +0,0 @@
To Do List
----------
- Update the patch serialization/deserialization code to handle compression types (compression implies smaller memory sizes)
- Think through compression implications in general

3
lib/Makefile
View File

@ -6,10 +6,11 @@ LDFLAGS = $(XML2_LDFLAGS) $(ZLIB_LDFLAGS)
OBJS = \
pc_bytes.o \
pc_dimlist.o \
pc_dimstats.o \
pc_mem.o \
pc_patch.o \
pc_patch_dimensional.o \
pc_patch_uncompressed.o \
pc_point.o \
pc_pointlist.o \
pc_schema.o \

6
lib/TODO.md Normal file
View File

@ -0,0 +1,6 @@
To Do
=====
- convert PCBYTES to use PCDIMENSION* instead of holding all values as dupes
- make dimensional min/max generation respect scale/offset
? convert PCBYTES handlign to pass-by-reference instead of pass-by-value

108
lib/cunit/cu_pc_patch.c
View File

@ -97,7 +97,7 @@ test_endian_flip()
static void
test_patch_hex_in()
{
// 00 endian
// 00 endian (big)
// 00000000 pcid
// 00000000 compression
// 00000002 npoints
@ -110,15 +110,15 @@ test_patch_hex_in()
size_t hexsize = strlen(hexbuf);
uint8_t *wkb = bytes_from_hexbytes(hexbuf, hexsize);
PCPATCH *pa = pc_patch_from_wkb(simpleschema, wkb, hexsize/2);
PCPOINTLIST *pl = pc_patch_to_points(pa);
pc_point_get_double_by_name(pl->points[0], "X", &d);
PCPOINTLIST *pl = pc_patch_to_pointlist(pa);
pc_point_get_double_by_name(pc_pointlist_get_point(pl, 0), "X", &d);
CU_ASSERT_DOUBLE_EQUAL(d, 0.02, 0.000001);
pc_point_get_double_by_name(pl->points[1], "Intensity", &d);
pc_point_get_double_by_name(pc_pointlist_get_point(pl, 1), "Intensity", &d);
CU_ASSERT_DOUBLE_EQUAL(d, 8, 0.000001);
str = pc_patch_to_string(pa);
CU_ASSERT_STRING_EQUAL(str, "[ 0 : (0.02, 0.03, 0.05, 6), (0.02, 0.03, 0.05, 8) ]");
// printf("\n%s\n",str);
CU_ASSERT_STRING_EQUAL(str, "{\"pcid\":0,\"pts\":[[0.02,0.03,0.05,6],[0.02,0.03,0.05,8]]}");
// printf("\n%s\n",str);
pc_pointlist_free(pl);
pc_patch_free(pa);
@ -138,7 +138,7 @@ test_patch_hex_out()
// 0000000200000003000000050006 pt1 (XYZi)
// 0000000200000003000000050008 pt2 (XYZi)
static char *wkt_result = "[ 0 : (0.02, 0.03, 0.05, 6), (0.02, 0.03, 0.05, 8) ]";
static char *wkt_result = "{\"pcid\":0,\"pts\":[[0.02,0.03,0.05,6],[0.02,0.03,0.05,8]]}";
static char *hexresult_xdr =
"0000000000000000000000000200000002000000030000000500060000000200000003000000050008";
static char *hexresult_ndr =
@ -150,7 +150,7 @@ test_patch_hex_out()
PCPOINT *pt0 = pc_point_from_double_array(simpleschema, d0, 4);
PCPOINT *pt1 = pc_point_from_double_array(simpleschema, d1, 4);
PCPATCH *pa;
PCPATCH_UNCOMPRESSED *pa;
uint8_t *wkb;
size_t wkbsize;
char *hexwkb;
@ -160,8 +160,8 @@ test_patch_hex_out()
pc_pointlist_add_point(pl, pt0);
pc_pointlist_add_point(pl, pt1);
pa = pc_patch_from_points(pl);
wkb = pc_patch_to_wkb(pa, &wkbsize);
pa = pc_patch_uncompressed_from_pointlist(pl);
wkb = pc_patch_uncompressed_to_wkb(pa, &wkbsize);
// printf("wkbsize %zu\n", wkbsize);
hexwkb = hexbytes_from_bytes(wkb, wkbsize);
@ -177,11 +177,12 @@ test_patch_hex_out()
CU_ASSERT_STRING_EQUAL(hexwkb, hexresult_xdr);
}
wkt = pc_patch_to_string(pa);
wkt = pc_patch_uncompressed_to_string(pa);
// printf("wkt %s\n", wkt);
CU_ASSERT_STRING_EQUAL(wkt, wkt_result);
pc_pointlist_free(pl);
pc_patch_free(pa);
pc_patch_uncompressed_free(pa);
pcfree(hexwkb);
pcfree(wkb);
pcfree(wkt);
@ -219,7 +220,7 @@ static PCBYTES initbytes(uint8_t *bytes, size_t size, uint32_t interp)
pcb.bytes = bytes;
pcb.size = size;
pcb.interpretation = interp;
pcb.npoints = pcb.size / INTERPRETATION_SIZES[pcb.interpretation];
pcb.npoints = pcb.size / pc_interpretation_size(pcb.interpretation);
pcb.compression = PC_DIM_NONE;
return pcb;
}
@ -536,13 +537,13 @@ test_zlib_encoding()
* Test for data loss or alteration.
*/
static void
test_dimlist()
test_patch_dimensional()
{
PCPOINT *pt;
int i;
int npts = 10;
PCPOINTLIST *pl1, *pl2;
PCDIMLIST *pdl;
PCPATCH_DIMENSIONAL *pdl;
PCDIMSTATS *pds;
pl1 = pc_pointlist_make(npts);
@ -557,12 +558,12 @@ test_dimlist()
pc_pointlist_add_point(pl1, pt);
}
pdl = pc_dimlist_from_pointlist(pl1);
pl2 = pc_pointlist_from_dimlist(pdl);
pdl = pc_patch_dimensional_from_pointlist(pl1);
pl2 = pc_pointlist_from_dimensional(pdl);
for ( i = 0; i < npts; i++ )
{
pt = pl2->points[i];
pt = pc_pointlist_get_point(pl2, i);
double v1, v2, v3, v4;
pc_point_get_double_by_name(pt, "x", &v1);
pc_point_get_double_by_name(pt, "y", &v2);
@ -580,12 +581,78 @@ test_dimlist()
pc_dimstats_update(pds, pdl);
pc_dimlist_free(pdl);
pc_patch_dimensional_free(pdl);
pc_pointlist_free(pl1);
pc_pointlist_free(pl2);
pc_dimstats_free(pds);
}
static void
test_patch_dimensional_compression()
{
PCPOINT *pt;
int i;
int npts = 400;
PCPOINTLIST *pl1, *pl2;
PCPATCH_DIMENSIONAL *pch1, *pch2;
PCDIMSTATS *pds = NULL;
size_t z1, z2;
char *str;
pl1 = pc_pointlist_make(npts);
for ( i = 0; i < npts; i++ )
{
pt = pc_point_make(simpleschema);
pc_point_set_double_by_name(pt, "x", i*2.0);
pc_point_set_double_by_name(pt, "y", i*1.9);
pc_point_set_double_by_name(pt, "Z", i*0.34);
pc_point_set_double_by_name(pt, "intensity", 10);
pc_pointlist_add_point(pl1, pt);
}
pch1 = pc_patch_dimensional_from_pointlist(pl1);
z1 = pc_patch_dimensional_serialized_size((PCPATCH*)pch1);
// printf("z1 %ld\n", z1);
pds = pc_dimstats_make(simpleschema);
pc_dimstats_update(pds, pch1);
pc_dimstats_update(pds, pch1);
pch2 = pc_patch_dimensional_compress(pch1, pds);
z2 = pc_patch_dimensional_serialized_size((PCPATCH*)pch2);
// printf("z2 %ld\n", z2);
str = pc_dimstats_to_string(pds);
CU_ASSERT_STRING_EQUAL(str, "{\"ndims\":4,\"total_points\":1200,\"total_patches\":3,\"dims\":[{\"total_runs\":1200,\"total_commonbits\":45,\"recommended_compression\":2},{\"total_runs\":1200,\"total_commonbits\":45,\"recommended_compression\":2},{\"total_runs\":1200,\"total_commonbits\":54,\"recommended_compression\":2},{\"total_runs\":3,\"total_commonbits\":48,\"recommended_compression\":1}]}");
// printf("%s\n", str);
pcfree(str);
pl2 = pc_pointlist_from_dimensional(pch2);
for ( i = 0; i < npts; i++ )
{
pt = pc_pointlist_get_point(pl2, i);
double v1, v2, v3, v4;
pc_point_get_double_by_name(pt, "x", &v1);
pc_point_get_double_by_name(pt, "y", &v2);
pc_point_get_double_by_name(pt, "Z", &v3);
pc_point_get_double_by_name(pt, "intensity", &v4);
// printf("%g\n", v4);
CU_ASSERT_DOUBLE_EQUAL(v1, i*2.0, 0.001);
CU_ASSERT_DOUBLE_EQUAL(v2, i*1.9, 0.001);
CU_ASSERT_DOUBLE_EQUAL(v3, i*0.34, 0.001);
CU_ASSERT_DOUBLE_EQUAL(v4, 10, 0.001);
}
pc_patch_dimensional_free(pch1);
pc_patch_dimensional_free(pch2);
// pc_patch_dimensional_free(pch3);
pc_pointlist_free(pl1);
pc_pointlist_free(pl2);
if ( pds ) pc_dimstats_free(pds);
}
/* REGISTER ***********************************************************/
CU_TestInfo patch_tests[] = {
@ -596,7 +663,8 @@ CU_TestInfo patch_tests[] = {
PC_TEST(test_run_length_encoding),
PC_TEST(test_sigbits_encoding),
PC_TEST(test_zlib_encoding),
PC_TEST(test_dimlist),
PC_TEST(test_patch_dimensional),
PC_TEST(test_patch_dimensional_compression),
CU_TEST_INFO_NULL
};

12
lib/cunit/cu_pc_schema.c
View File

@ -19,8 +19,10 @@ static const char *xmlfile = "data/pdal-schema.xml";
static int
init_suite(void)
{
schema = NULL;
return 0;
char *xmlstr = file_to_str(xmlfile);
int rv = pc_schema_from_xml(xmlstr, &schema);
pcfree(xmlstr);
return rv == PC_FAILURE;
}
static int
@ -36,8 +38,9 @@ clean_suite(void)
static void
test_schema_from_xml()
{
static PCSCHEMA *myschema = NULL;
char *xmlstr = file_to_str(xmlfile);
int rv = pc_schema_from_xml(xmlstr, &schema);
int rv = pc_schema_from_xml(xmlstr, &myschema);
pcfree(xmlstr);
// char *schemastr = pc_schema_to_json(schema);
@ -45,7 +48,8 @@ test_schema_from_xml()
// printf("name0 %s\n", schema->dims[0]->name);
// printf("%s\n", schemastr);
CU_ASSERT(schema != NULL);
CU_ASSERT(myschema != NULL);
pc_schema_free(myschema);
}
static void

102
lib/pc_api.h
View File

@ -77,6 +77,21 @@ typedef struct
} PCSCHEMA;
typedef struct
{
uint32_t total_runs;
uint32_t total_commonbits;
uint32_t recommended_compression;
} PCDIMSTAT;
typedef struct
{
int32_t ndims;
uint32_t total_points;
uint32_t total_patches;
PCDIMSTAT *stats;
} PCDIMSTATS;
/**
* Uncompressed structure for in-memory handling
* of points. A read-only PgSQL point can be wrapped in
@ -99,6 +114,16 @@ typedef struct
PCPOINT **points;
} PCPOINTLIST;
typedef struct
{
size_t size;
uint32_t npoints;
uint32_t interpretation;
uint32_t compression;
uint32_t readonly;
uint8_t *bytes;
} PCBYTES;
/**
* Uncompressed Structure for in-memory handling
* of patches. A read-only PgSQL patch can be wrapped in
@ -106,6 +131,15 @@ typedef struct
* PgSQL memory and setting the capacity to 0
* to indicate it is read-only.
*/
typedef struct
{
int type;
int8_t readonly;
const PCSCHEMA *schema;
uint32_t npoints; /* How many points we have */
double xmin, xmax, ymin, ymax;
} PCPATCH;
typedef struct
{
int type;
@ -114,10 +148,30 @@ typedef struct
uint32_t npoints; /* How many points we have */
double xmin, xmax, ymin, ymax;
uint32_t maxpoints; /* How man points we can hold (or 0 for read-only) */
uint8_t compressed; /* Has compression been applied to the data buffer? */
size_t datasize;
uint8_t *data; /* A serialized version of the data */
} PCPATCH;
} PCPATCH_UNCOMPRESSED;
typedef struct
{
int type;
int8_t readonly;
const PCSCHEMA *schema;
uint32_t npoints; /* How many points we have */
double xmin, xmax, ymin, ymax;
PCBYTES *bytes;
} PCPATCH_DIMENSIONAL;
typedef struct
{
int type;
int8_t readonly;
const PCSCHEMA *schema;
uint32_t npoints; /* How many points we have */
double xmin, xmax, ymin, ymax;
uint8_t *data;
} PCPATCH_GHT;
/* Global function signatures for memory/logging handlers. */
@ -158,16 +212,15 @@ void pc_install_default_handlers(void);
* UTILITY
*/
/** Read the the PCID from WKB form of a POINT/PATCH */
uint32_t wkb_get_pcid(const uint8_t *wkb);
/** Read the the npoints from WKB form of a PATCH */
uint32_t wkb_get_compression(const uint8_t *wkb);
/** Flips the bytes of an int32_t */
int32_t int32_flip_endian(int32_t val);
/** Convert binary to hex */
uint8_t* bytes_from_hexbytes(const char *hexbuf, size_t hexsize);
/** Convert hex to binary */
char* hexbytes_from_bytes(const uint8_t *bytebuf, size_t bytesize);
/** Read the the PCID from WKB form of a POINT/PATCH */
uint32_t wkb_get_pcid(const uint8_t *wkb);
/** Build an empty #PCDIMSTATS based on the schema */
PCDIMSTATS* pc_dimstats_make(const PCSCHEMA *schema);
/**********************************************************************
@ -203,6 +256,9 @@ void pc_pointlist_free(PCPOINTLIST *pl);
/** Add a point to the list, expanding buffer as necessary */
void pc_pointlist_add_point(PCPOINTLIST *pl, PCPOINT *pt);
/** Get a point from the list */
PCPOINT* pc_pointlist_get_point(const PCPOINTLIST *pl, int i);
/**********************************************************************
* PCPOINT
@ -226,12 +282,6 @@ int pc_point_get_double_by_name(const PCPOINT *pt, const char *name, double *d);
/** Casts dimension value to double and scale/offset appropriately before returning */
int pc_point_get_double_by_index(const PCPOINT *pt, uint32_t idx, double *d);
/** Scales/offsets double, casts to appropriate dimension type, and writes into point */
int pc_point_set_double_by_index(PCPOINT *pt, uint32_t idx, double val);
/** Scales/offsets double, casts to appropriate dimension type, and writes into point */
int pc_point_set_double_by_name(PCPOINT *pt, const char *name, double val);
/** Returns X coordinate */
double pc_point_get_x(const PCPOINT *pt);
@ -255,26 +305,17 @@ uint8_t* pc_point_to_geometry_wkb(const PCPOINT *pt, size_t *wkbsize);
* PCPATCH
*/
/** Create new empty PCPATCH */
PCPATCH* pc_patch_make(const PCSCHEMA *s);
/** Create new PCPATCH from a PCPOINT set. Copies data, doesn't take ownership of points */
PCPATCH* pc_patch_from_points(const PCPOINTLIST *pl);
PCPATCH* pc_patch_from_pointlist(const PCPOINTLIST *ptl);
/** Returns a list of points extracted from patch */
PCPOINTLIST* pc_patch_to_points(const PCPATCH *patch);
PCPOINTLIST* pc_patch_to_pointlist(const PCPATCH *patch);
/** Free patch memory, respecting read-only status. Does not free referenced schema */
void pc_patch_free(PCPATCH *patch);
/** Add a point to read/write PCPATCH */
int pc_patch_add_point(PCPATCH *c, const PCPOINT *p);
/** Create a compressed copy, using the compression schema referenced in the PCSCHEMA */
PCPATCH* pc_patch_compress(const PCPATCH *patch);
/** Create a full copy of the patch, including data, but not including PCSCHEMA */
PCPATCH* pc_patch_clone(const PCPATCH *patch);
PCPATCH* pc_patch_compress(const PCPATCH *patch, void *userdata);
/** Create a new readwrite PCPOINT from a byte array */
PCPATCH* pc_patch_from_wkb(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize);
@ -288,5 +329,14 @@ char* pc_patch_to_string(const PCPATCH *patch);
/** Returns OGC WKB for envelope of PCPATCH */
uint8_t* pc_patch_to_geometry_wkb_envelope(const PCPATCH *pa, size_t *wkbsize);
/** */
size_t pc_patch_dimensional_serialized_size(const PCPATCH *patch);
/** Write the representation down to a buffer */
int pc_bytes_serialize(const PCBYTES *pcb, uint8_t *buf, size_t *size);
/** Read a buffer up into a bytes structure */
int pc_bytes_deserialize(const uint8_t *buf, const PCDIMENSION *dim, PCBYTES *pcb, int readonly, int flip_endian);
#endif /* _PC_API_H */

144
lib/pc_api_internal.h
View File

@ -79,43 +79,25 @@ enum DIMCOMPRESSIONS {
PC_DIM_ZLIB = 3
};
typedef struct
{
size_t size;
uint32_t npoints;
uint32_t interpretation;
uint32_t compression;
uint32_t read_only;
uint8_t *bytes;
} PCBYTES;
typedef struct
{
uint32_t npoints;
const PCSCHEMA *schema;
PCBYTES *bytes;
} PCDIMLIST;
typedef struct
{
uint32_t total_runs;
uint32_t total_commonbits;
uint32_t recommended_compression;
} PCDIMSTAT;
typedef struct
{
int32_t ndims;
uint32_t total_points;
uint32_t total_patches;
PCDIMSTAT *stats;
} PCDIMSTATS;
/** What is the endianness of this system? */
char machine_endian(void);
/** Flips the bytes of an int32_t */
int32_t int32_flip_endian(int32_t val);
/** Read the the npoints from WKB form of a PATCH */
uint32_t wkb_get_compression(const uint8_t *wkb);
/** Read an int32 from a byte array, flipping if requested */
int32_t wkb_get_int32(const uint8_t *wkb, int flip_endian);
/** Read an int16 from a byte array, flipping if requested */
int16_t wkb_get_int16(const uint8_t *wkb, int flip_endian);
/** Force a byte array into the machine endianness */
uint8_t* uncompressed_bytes_flip_endian(const uint8_t *bytebuf, const PCSCHEMA *schema, uint32_t npoints);
@ -128,12 +110,72 @@ int pc_double_to_ptr(uint8_t *ptr, uint32_t interpretation, double val);
/** Return number of bytes in a given interpretation */
size_t pc_interpretation_size(uint32_t interp);
/** True if there is a dimension of that name */
int pc_schema_has_name(const PCSCHEMA *s, const char *name);
/** Copy a string within the global memory management context */
char* pcstrdup(const char *str);
/** Scales/offsets double, casts to appropriate dimension type, and writes into point */
int pc_point_set_double_by_index(PCPOINT *pt, uint32_t idx, double val);
/** Scales/offsets double, casts to appropriate dimension type, and writes into point */
int pc_point_set_double_by_name(PCPOINT *pt, const char *name, double val);
/****************************************************************************
* DIMENSION STATISTICS
*/
/** Analyze the bytes in the #PCPATCH_DIMENSIONAL and update the #PCDIMSTATS */
int pc_dimstats_update(PCDIMSTATS *pds, const PCPATCH_DIMENSIONAL *pdl);
/** Free the PCDIMSTATS memory */
void pc_dimstats_free(PCDIMSTATS *pds);
char* pc_dimstats_to_string(const PCDIMSTATS *pds);
/****************************************************************************
* PATCHES
*/
/* DIMENSIONAL PATCHES */
char* pc_patch_dimensional_to_string(const PCPATCH_DIMENSIONAL *pa);
PCPATCH_DIMENSIONAL* pc_patch_dimensional_from_uncompressed(const PCPATCH_UNCOMPRESSED *pa);
PCPATCH_DIMENSIONAL* pc_patch_dimensional_compress(const PCPATCH_DIMENSIONAL *pdl, PCDIMSTATS *pds);
PCPATCH_DIMENSIONAL* pc_patch_dimensional_decompress(const PCPATCH_DIMENSIONAL *pdl);
void pc_patch_dimensional_free(PCPATCH_DIMENSIONAL *pdl);
int pc_patch_dimensional_compute_extent(PCPATCH_DIMENSIONAL *pdl);
uint8_t* pc_patch_dimensional_to_wkb(const PCPATCH_DIMENSIONAL *patch, size_t *wkbsize);
PCPATCH* pc_patch_dimensional_from_wkb(const PCSCHEMA *schema, const uint8_t *wkb, size_t wkbsize);
PCPATCH_DIMENSIONAL* pc_patch_dimensional_from_pointlist(const PCPOINTLIST *pdl);
PCPOINTLIST* pc_pointlist_from_dimensional(const PCPATCH_DIMENSIONAL *pdl);
/* UNCOMPRESSED PATCHES */
char* pc_patch_uncompressed_to_string(const PCPATCH_UNCOMPRESSED *patch);
uint8_t* pc_patch_uncompressed_to_wkb(const PCPATCH_UNCOMPRESSED *patch, size_t *wkbsize);
PCPATCH* pc_patch_uncompressed_from_wkb(const PCSCHEMA *s, const uint8_t *wkb, size_t wkbsize);
PCPATCH_UNCOMPRESSED* pc_patch_uncompressed_make(const PCSCHEMA *s);
int pc_patch_uncompressed_compute_extent(PCPATCH_UNCOMPRESSED *patch);
void pc_patch_uncompressed_free(PCPATCH_UNCOMPRESSED *patch);
PCPOINTLIST* pc_patch_uncompressed_to_pointlist(const PCPATCH_UNCOMPRESSED *patch);
PCPATCH_UNCOMPRESSED* pc_patch_uncompressed_from_pointlist(const PCPOINTLIST *pl);
PCPATCH_UNCOMPRESSED* pc_patch_uncompressed_from_dimensional(const PCPATCH_DIMENSIONAL *pdl);
int pc_patch_uncompressed_add_point(PCPATCH_UNCOMPRESSED *c, const PCPOINT *p);
/****************************************************************************
* BYTES
*/
/** Construct empty byte array (zero out attribute and allocate byte buffer) */
PCBYTES pc_bytes_make(const PCDIMENSION *dim, uint32_t npoints);
/** Empty the byte array (free the byte buffer) */
void pc_bytes_free(PCBYTES bytes);
/** Apply the compresstion to the byte array in place, freeing the original byte buffer */
PCBYTES pc_bytes_encode(PCBYTES pcb, int compression);
/** Convert the bytes in #PCBYTES to PC_DIM_NONE compression */
PCBYTES pc_bytes_decode(PCBYTES epcb);
/** How big will the serialization be? */
size_t pc_bytes_serialized_size(const PCBYTES *pcb);
/** Convert value bytes to RLE bytes */
PCBYTES pc_bytes_run_length_encode(const PCBYTES pcb);
/** Convert RLE bytes to value bytes */
@ -159,42 +201,6 @@ uint16_t pc_bytes_sigbits_count_16(const PCBYTES *pcb, uint32_t *nsigbits);
uint32_t pc_bytes_sigbits_count_32(const PCBYTES *pcb, uint32_t *nsigbits);
/** Using an 64-bit word, what is the common word and number of bits in common? */
uint64_t pc_bytes_sigbits_count_64(const PCBYTES *pcb, uint32_t *nsigbits);
/** Pivot a #PCPOINTLIST to a #PCDIMLIST, taking copies of data. */
PCDIMLIST* pc_dimlist_from_pointlist(const PCPOINTLIST *pl);
/** Pivot a #PCDIMLIST to a #PCPOINTLIST, taking copies of data. */
PCPOINTLIST* pc_pointlist_from_dimlist(PCDIMLIST *pdl);
/** Compress #DIMLIST, using the suggestions from the #PCDIMSTATS. Compresses in place and frees existing data. */
int pc_dimlist_encode(PCDIMLIST *pdl, PCDIMSTATS **pdsptr);
/** Decompress #DIMLIST. Decompresses in place and frees existing data. */
int pc_dimlist_decode(PCDIMLIST *pdl);
/** Build an empty #PCDIMSTATS based on the schema */
PCDIMSTATS* pc_dimstats_make(const PCSCHEMA *schema);
/** Analyze the bytes in the #PCDIMLIST and update the #PCDIMSTATS */
int pc_dimstats_update(PCDIMSTATS *pds, const PCDIMLIST *pdl);
/** Free the PCDIMSTATS memory */
void pc_dimstats_free(PCDIMSTATS *pds);
/** Construct empty byte array (zero out attribute and allocate byte buffer) */
PCBYTES pc_bytes_make(const PCDIMENSION *dim, uint32_t npoints);
/** Empty the byte array (free the byte buffer) */
void pc_bytes_free(PCBYTES bytes);
/** Apply the compresstion to the byte array in place, freeing the original byte buffer */
PCBYTES pc_bytes_encode(PCBYTES pcb, int compression);
/** Convert the bytes in #PCBYTES to PC_DIM_NONE compression */
PCBYTES pc_bytes_decode(PCBYTES epcb);
/** How big will the serialization be? */
size_t pc_bytes_get_serialized_size(const PCBYTES *pcb);
/** Write the representation down to a buffer */
int pc_bytes_serialize(const PCBYTES *pcb, uint8_t *buf, size_t *size);
/** Read a buffer up into a bytes structure */
int pc_bytes_deserialize(uint8_t *buf, const PCDIMENSION *dim, PCBYTES *pcb, int read_only, int flip_endian);
/** Read an int32 from a byte array, flipping if requested */
int32_t wkb_get_int32(const uint8_t *wkb, int flip_endian);
/** Read an int16 from a byte array, flipping if requested */
int16_t wkb_get_int16(const uint8_t *wkb, int flip_endian);
#endif /* _PC_API_INTERNAL_H */

180
lib/pc_bytes.c
View File

@ -18,6 +18,7 @@
#include <stdarg.h>
#include <assert.h>
#include <math.h>
#include "pc_api_internal.h"
#include "zlib.h"
@ -26,7 +27,7 @@
void
pc_bytes_free(PCBYTES pcb)
{
if ( ! pcb.read_only )
if ( ! pcb.readonly )
pcfree(pcb.bytes);
}
@ -39,7 +40,7 @@ pc_bytes_make(const PCDIMENSION *dim, uint32_t npoints)
pcb.npoints = npoints;
pcb.interpretation = dim->interpretation;
pcb.compression = PC_DIM_NONE;
pcb.read_only = PC_FALSE;
pcb.readonly = PC_FALSE;
return pcb;
}
@ -75,14 +76,14 @@ pc_bytes_encode(PCBYTES pcb, int compression)
}
case PC_DIM_NONE:
{
return pcb;
epcb = pc_bytes_clone(pcb);
break;
}
default:
{
pcerror("pc_bytes_encode: Uh oh");
}
}
pc_bytes_free(pcb);
return epcb;
}
@ -94,29 +95,29 @@ pc_bytes_decode(PCBYTES epcb)
{
case PC_DIM_RLE:
{
pcb = pc_bytes_run_length_decode(pcb);
pcb = pc_bytes_run_length_decode(epcb);
break;
}
case PC_DIM_SIGBITS:
{
pcb = pc_bytes_sigbits_decode(pcb);
pcb = pc_bytes_sigbits_decode(epcb);
break;
}
case PC_DIM_ZLIB:
{
pcb = pc_bytes_zlib_decode(pcb);
pcb = pc_bytes_zlib_decode(epcb);
break;
}
case PC_DIM_NONE:
{
return epcb;
pcb = pc_bytes_clone(epcb);
break;
}
default:
{
pcerror("pc_bytes_decode: Uh oh");
}
}
pc_bytes_free(epcb);
return pcb;
}
@ -132,7 +133,7 @@ pc_bytes_run_count(const PCBYTES *pcb)
int i;
const uint8_t *ptr0;
const uint8_t *ptr1;
size_t size = INTERPRETATION_SIZES[pcb->interpretation];
size_t size = pc_interpretation_size(pcb->interpretation);
uint32_t runcount = 1;
for ( i = 1; i < pcb->npoints; i++ )
@ -162,7 +163,7 @@ pc_bytes_run_length_encode(const PCBYTES pcb)
const uint8_t *bytesptr;
const uint8_t *runstart;
uint8_t *bytes_rle;
size_t size = INTERPRETATION_SIZES[pcb.interpretation];
size_t size = pc_interpretation_size(pcb.interpretation);
uint8_t runlength = 1;
PCBYTES pcbout = pcb;
@ -203,7 +204,7 @@ pc_bytes_run_length_encode(const PCBYTES pcb)
/* We're going to replace the current buffer */
pcbout.bytes = bytes_rle;
pcbout.compression = PC_DIM_RLE;
pcbout.read_only = PC_FALSE;
pcbout.readonly = PC_FALSE;
return pcbout;
}
@ -223,7 +224,7 @@ pc_bytes_run_length_decode(const PCBYTES pcb)
const uint8_t *bytes_rle_ptr = pcb.bytes;
const uint8_t *bytes_rle_end = pcb.bytes + pcb.size;
size_t size = INTERPRETATION_SIZES[pcb.interpretation];
size_t size = pc_interpretation_size(pcb.interpretation);
size_t size_out;
uint8_t runlength;
uint32_t npoints = 0;
@ -259,7 +260,7 @@ pc_bytes_run_length_decode(const PCBYTES pcb)
pcbout.compression = PC_DIM_NONE;
pcbout.size = size_out;
pcbout.bytes = bytes;
pcbout.read_only = PC_FALSE;
pcbout.readonly = PC_FALSE;
return pcbout;
}
@ -273,8 +274,9 @@ pc_bytes_run_length_flip_endian(PCBYTES pcb)
{
int i, n;
uint8_t *bytes_ptr = pcb.bytes;
uint8_t *end_ptr = pcb.bytes + pcb.size;
uint8_t tmp;
size_t size = INTERPRETATION_SIZES[pcb.interpretation];
size_t size = pc_interpretation_size(pcb.interpretation);
assert(pcb.compression == PC_DIM_RLE);
assert(pcb.npoints > 0);
@ -284,20 +286,20 @@ pc_bytes_run_length_flip_endian(PCBYTES pcb)
return pcb;
/* Don't try to modify read-only memory, make some fresh memory */
if ( pcb.read_only == PC_TRUE )
if ( pcb.readonly == PC_TRUE )
{
uint8_t *oldbytes = pcb.bytes;
pcb.bytes = pcalloc(pcb.size);
memcpy(pcb.bytes, oldbytes, pcb.size);
pcb.read_only = PC_FALSE;
pcb.readonly = PC_FALSE;
}
/* Move to start of first word */
bytes_ptr++;
bytes_ptr++; /* Advance past count */
/* Visit each entry and flip the word, skip the count */
for ( i = 0; i < pcb.npoints; i++ )
while( bytes_ptr < end_ptr )
{
/* Swap the bytes in a way that makes sense for this word size */
for ( n = 0; n < size/2; n++ )
{
@ -306,8 +308,10 @@ pc_bytes_run_length_flip_endian(PCBYTES pcb)
bytes_ptr[size-n-1] = tmp;
}
/* Move forward one word and one counter */
bytes_ptr += size+1;
/* Move past this word */
bytes_ptr += size;
/* Advance past next count */
bytes_ptr++;
}
return pcb;
@ -428,7 +432,7 @@ pc_bytes_sigbits_count_64(const PCBYTES *pcb, uint32_t *nsigbits)
uint32_t
pc_bytes_sigbits_count(const PCBYTES *pcb)
{
size_t size = INTERPRETATION_SIZES[pcb->interpretation];
size_t size = pc_interpretation_size(pcb->interpretation);
uint32_t nbits = -1;
switch ( size )
{
@ -569,7 +573,9 @@ pc_bytes_sigbits_encode_16(const PCBYTES pcb, uint16_t commonvalue, uint8_t comm
/* How wide are our unique values? */
int nbits = bitwidth - commonbits;
/* Size of output buffer (#bits/8+1remainder+4metadata) */
size_t size_out = (nbits * pcb.npoints / 8) + 5;
size_t size_out_raw = (nbits * pcb.npoints / 8) + 5;
/* Make sure buffer is size to hold all our words */
size_t size_out = size_out_raw + (size_out_raw % 2);
uint8_t *bytes_out = pcalloc(size_out);
/* Use this to zero out the parts that are common */
uint16_t mask = (0xFFFF >> commonbits);
@ -658,7 +664,8 @@ pc_bytes_sigbits_encode_32(const PCBYTES pcb, uint32_t commonvalue, uint8_t comm
/* How wide are our unique values? */
int nbits = bitwidth - commonbits;
/* Size of output buffer (#bits/8+1remainder+8metadata) */
size_t size_out = (nbits * pcb.npoints / 8) + 9;
size_t size_out_raw = (nbits * pcb.npoints / 8) + 9;
size_t size_out = size_out_raw + (4 - (size_out_raw % 4));
uint8_t *bytes_out = pcalloc(size_out);
/* Use this to zero out the parts that are common */
uint32_t mask = (0xFFFFFFFF >> commonbits);
@ -739,7 +746,7 @@ pc_bytes_sigbits_encode_32(const PCBYTES pcb, uint32_t commonvalue, uint8_t comm
PCBYTES
pc_bytes_sigbits_encode(const PCBYTES pcb)
{
size_t size = INTERPRETATION_SIZES[pcb.interpretation];
size_t size = pc_interpretation_size(pcb.interpretation);
uint32_t nbits;
switch ( size )
{
@ -772,7 +779,7 @@ pc_bytes_sigbits_flip_endian(const PCBYTES pcb)
{
int n;
uint8_t tmp1, tmp2;
size_t size = INTERPRETATION_SIZES[pcb.interpretation];
size_t size = pc_interpretation_size(pcb.interpretation);
uint8_t *b1 = pcb.bytes;
uint8_t *b2 = pcb.bytes + size;
@ -870,7 +877,7 @@ pc_bytes_sigbits_decode_16(const PCBYTES pcb)
uint16_t commonvalue;
uint16_t mask;
int bit = 16;
size_t outbytes_size = sizeof(uint8_t) * pcb.npoints;
size_t outbytes_size = sizeof(uint16_t) * pcb.npoints;
uint8_t *outbytes = pcalloc(outbytes_size);
uint16_t *obytes = (uint16_t*)outbytes;
PCBYTES pcbout = pcb;
@ -926,7 +933,7 @@ pc_bytes_sigbits_decode_32(const PCBYTES pcb)
uint32_t commonvalue;
uint32_t mask;
int bit = 32;
size_t outbytes_size = sizeof(uint8_t) * pcb.npoints;
size_t outbytes_size = sizeof(uint32_t) * pcb.npoints;
uint8_t *outbytes = pcalloc(outbytes_size);
uint32_t *obytes = (uint32_t*)outbytes;
PCBYTES pcbout = pcb;
@ -978,7 +985,7 @@ pc_bytes_sigbits_decode_32(const PCBYTES pcb)
PCBYTES
pc_bytes_sigbits_decode(const PCBYTES pcb)
{
size_t size = INTERPRETATION_SIZES[pcb.interpretation];
size_t size = pc_interpretation_size(pcb.interpretation);
switch ( size )
{
case 1:
@ -1070,7 +1077,7 @@ pc_bytes_zlib_decode(const PCBYTES pcb)
int ret;
PCBYTES pcbout = pcb;
pcbout.size = INTERPRETATION_SIZES[pcb.interpretation] * pcb.npoints;
pcbout.size = pc_interpretation_size(pcb.interpretation) * pcb.npoints;
/* Set up output memory */
pcbout.bytes = pcalloc(pcbout.size);
@ -1095,11 +1102,14 @@ pc_bytes_zlib_decode(const PCBYTES pcb)
}
/**
* This flips bytes in-place, so won't work on read_only bytes
* This flips bytes in-place, so won't work on readonly bytes
*/
PCBYTES
pc_bytes_flip_endian(PCBYTES pcb)
{
if ( pcb.readonly )
pcerror("pc_bytes_flip_endian: cannot flip readonly bytes");
switch(pcb.compression)
{
case PC_DIM_NONE:
@ -1113,11 +1123,12 @@ pc_bytes_flip_endian(PCBYTES pcb)
default:
pcerror("pc_bytes_flip_endian: unknown compression");
}
return pcb;
}
size_t
pc_bytes_get_serialized_size(const PCBYTES *pcb)
pc_bytes_serialized_size(const PCBYTES *pcb)
{
/* compression type (1) + size of data (4) + data */
return 1 + 4 + pcb->size;
@ -1135,24 +1146,115 @@ pc_bytes_serialize(const PCBYTES *pcb, uint8_t *buf, size_t *size)
}
int
pc_bytes_deserialize(uint8_t *buf, const PCDIMENSION *dim, PCBYTES *pcb, int read_only, int flip_endian)
pc_bytes_deserialize(const uint8_t *buf, const PCDIMENSION *dim, PCBYTES *pcb, int readonly, int flip_endian)
{
pcb->compression = buf[0];
pcb->size = wkb_get_int32(buf+1, flip_endian);
pcb->read_only = read_only;
if ( read_only && flip_endian )
pcb->readonly = readonly;
if ( readonly && flip_endian )
pcerror("pc_bytes_deserialize: cannot create a read-only buffer on byteswapped input");
if ( read_only )
if ( readonly )
{
pcb->bytes = buf + 5;
pcb->bytes = (uint8_t*)(buf+5);
}
else
{
pcb->bytes = pcalloc(pcb->size);
memcpy(pcb->bytes, buf+5, pcb->size);
if ( flip_endian )
{
*pcb = pc_bytes_flip_endian(*pcb);
}
}
pcb->interpretation = dim->interpretation;
/* WARNING, still need to set externally */
/* pcb.interpretation */
/* pcb.npoints */
return PC_SUCCESS;
}
static int
pc_bytes_uncompressed_minmax(const PCBYTES *pcb, double *min, double *max)
{
int i;
int element_size = pc_interpretation_size(pcb->interpretation);
double d;
double mn = MAXFLOAT;
double mx = -1*MAXFLOAT;
for ( i = 0; i < pcb->npoints; i++ )
{
d = pc_double_from_ptr(pcb->bytes + i*element_size, pcb->interpretation);
if ( d < mn )
mn = d;
if ( d > mx )
mx = d;
}
*min = mn;
*max = mx;
}
static int
pc_bytes_run_length_minmax(const PCBYTES *pcb, double *min, double *max)
{
int element_size = pc_interpretation_size(pcb->interpretation);
double mn = MAXFLOAT;
double mx = -1*MAXFLOAT;
double d;
uint8_t *ptr = pcb->bytes;
uint8_t *ptr_end = pcb->bytes + pcb->size;
/* Move past first count */
ptr++;
while( ptr < ptr_end )
{
d = pc_double_from_ptr(ptr, pcb->interpretation);
/* Calc min/max */
if ( d < mn )
mn = d;
if ( d > mx )
mx = d;
ptr += element_size;
}
*min = mn;
*max = mx;
}
static int
pc_bytes_zlib_minmax(const PCBYTES *pcb, double *min, double *max)
{
PCBYTES zcb = pc_bytes_zlib_decode(*pcb);
int rv = pc_bytes_uncompressed_minmax(&zcb, min, max);
pc_bytes_free(zcb);
return rv;
}
static int
pc_bytes_sigbits_minmax(const PCBYTES *pcb, double *min, double *max)
{
PCBYTES zcb = pc_bytes_sigbits_decode(*pcb);
int rv = pc_bytes_uncompressed_minmax(&zcb, min, max);
pc_bytes_free(zcb);
return rv;
}
int pc_bytes_minmax(const PCBYTES *pcb, double *min, double *max)
{
switch(pcb->compression)
{
case PC_DIM_NONE:
return pc_bytes_uncompressed_minmax(pcb, min, max);
case PC_DIM_SIGBITS:
return pc_bytes_sigbits_minmax(pcb, min, max);
case PC_DIM_ZLIB:
return pc_bytes_zlib_minmax(pcb, min, max);
case PC_DIM_RLE:
return pc_bytes_run_length_minmax(pcb, min, max);
default:
pcerror("pc_bytes_minmax: unknown compression");
}
return PC_FAILURE;
}

223
lib/pc_dimlist.c
View File

@ -1,223 +0,0 @@
/***********************************************************************
* pc_dimlist.c
*
* Support for "dimensional compression", which is a catch-all
* term for applying compression separately on each dimension
* of a PCPATCH collection of PCPOINTS.
*
* Depending on the character of the data, one of these schemes
* will be used:
*
* - run-length encoding
* - significant-bit removal
* - deflate
*
* Portions Copyright (c) 2012, OpenGeo
*
***********************************************************************/
#include <assert.h>
#include "pc_api_internal.h"
/**
* Converts a list of I N-dimensional points into a
* list of N I-valued dimensions. Precursor to running
* compression on each dimension separately.
*/
PCDIMLIST *
pc_dimlist_from_pointlist(const PCPOINTLIST *pl)
{
PCDIMLIST *pdl;
int i, j, ndims, npoints;
assert(pl);
if ( pl->npoints == 0 ) return NULL;
pdl = pcalloc(sizeof(PCDIMLIST));
pdl->schema = pl->points[0]->schema;
ndims = pdl->schema->ndims;
npoints = pl->npoints;
pdl->npoints = npoints;
pdl->bytes = pcalloc(ndims * sizeof(PCBYTES));
for ( i = 0; i < ndims; i++ )
{
PCDIMENSION *dim = pc_schema_get_dimension(pdl->schema, i);
pdl->bytes[i] = pc_bytes_make(dim, npoints);
for ( j = 0; j < npoints; j++ )
{
uint8_t *to = pdl->bytes[i].bytes + dim->size * j;
uint8_t *from = pl->points[j]->data + dim->byteoffset;
memcpy(to, from, dim->size);
}
}
return pdl;
}
void
pc_dimlist_free(PCDIMLIST *pdl)
{
int i;
assert(pdl);
assert(pdl->schema);
if ( pdl->bytes )
{
for ( i = 0; i < pdl->schema->ndims; i++ )
pc_bytes_free(pdl->bytes[i]);
pcfree(pdl->bytes);
}
pcfree(pdl);
}
int
pc_dimlist_encode(PCDIMLIST *pdl, PCDIMSTATS **pdsptr)
{
int i;
PCDIMSTATS *pds;
assert(pdl);
assert(pdl->schema);
assert(pdsptr);
/* Maybe we have stats passed in */
pds = *pdsptr;
/* No stats at all, make a new one */
if ( ! pds )
pds = pc_dimstats_make(pdl->schema);
/* Still sampling, update stats */
if ( pds->total_points < PCDIMSTATS_MIN_SAMPLE )
pc_dimstats_update(pds, pdl);
/* Compress each dimension as dictated by stats */
for ( i = 0; i < pdl->schema->ndims; i++ )
{
pdl->bytes[i] = pc_bytes_encode(pdl->bytes[i], pds->stats[i].recommended_compression);
}
return PC_SUCCESS;
}
int
pc_dimlist_decode(PCDIMLIST *pdl)
{
int i;
int ndims;
assert(pdl);
assert(pdl->schema);
/* Compress each dimension as dictated by stats */
for ( i = 0; i < pdl->schema->ndims; i++ )
{
pdl->bytes[i] = pc_bytes_decode(pdl->bytes[i]);
}
return PC_SUCCESS;
}
PCDIMLIST *
pc_dimlist_from_patch(const PCPATCH *pa)
{
PCDIMLIST *pdl;
const PCSCHEMA *schema;
int i, j, ndims, npoints;
assert(pa);
npoints = pa->npoints;
schema = pa->schema;
ndims = schema->ndims;
/* Uncompressed patches only! */
if ( pa->compressed )
pcerror("pc_dimlist_from_patch: cannot operate on compressed patch");
/* Cannot handle empty patches */
if ( npoints == 0 ) return NULL;
/* Initialize list */
pdl = pcalloc(sizeof(PCDIMLIST));
pdl->schema = schema;
pdl->npoints = npoints;
pdl->bytes = pcalloc(ndims * sizeof(PCBYTES));
for ( i = 0; i < ndims; i++ )
{
PCDIMENSION *dim = pc_schema_get_dimension(schema, i);
pdl->bytes[i] = pc_bytes_make(dim, npoints);
for ( j = 0; j < npoints; j++ )
{
uint8_t *to = pdl->bytes[i].bytes + dim->size * j;
uint8_t *from = pa->data + schema->size * j + dim->byteoffset;
memcpy(to, from, dim->size);
}
}
return pdl;
}
uint8_t *
pc_dimlist_serialize(const PCDIMLIST *pdl, size_t *size)
{
int i;
int ndims = pdl->schema->ndims;
size_t dlsize = 0;
uint8_t *buf;
for ( i = 0; i < ndims; i++ )
dlsize += pc_bytes_get_serialized_size(&(pdl->bytes[i]));
buf = pcalloc(dlsize);
for ( i = 0; i < ndims ; i++ )
{
size_t bsize = 0;
pc_bytes_serialize(&(pdl->bytes[i]), buf, &bsize);
buf += bsize;
}
return buf;
}
PCDIMLIST *
pc_dimlist_deserialize(const PCSCHEMA *schema, int npoints, uint8_t *buf, int read_only, int flip_endian)
{
int i;
size_t size = 0;
int ndims = schema->ndims;
PCDIMLIST *pdl = pcalloc(sizeof(PCDIMLIST));
pdl->schema = schema;
pdl->npoints = npoints;
pdl->bytes = pcalloc(ndims * sizeof(PCBYTES));
for ( i = 0; i < ndims; i++ )
{
PCDIMENSION *dim = schema->dims[i];
PCBYTES pcb = pdl->bytes[i];
pc_bytes_deserialize(buf, dim, &pcb, read_only, flip_endian);
/* pc_bytes_deserialize can't fill in npoints and interpretation */
pcb.npoints = npoints;
pcb.interpretation = dim->interpretation;
/* move forward to next data area */
buf += pcb.size;
pdl->bytes[i] = pcb;
}
return pdl;
}

54
lib/pc_dimstats.c
View File

@ -19,6 +19,7 @@
#include <stdarg.h>
#include <assert.h>
#include "pc_api_internal.h"
#include "stringbuffer.h"
PCDIMSTATS *
@ -38,9 +39,54 @@ pc_dimstats_free(PCDIMSTATS *pds)
pcfree(pds->stats);
pcfree(pds);
}
/*
typedef struct
{
uint32_t total_runs;
uint32_t total_commonbits;
uint32_t recommended_compression;
} PCDIMSTAT;
typedef struct
{
int32_t ndims;
uint32_t total_points;
uint32_t total_patches;
PCDIMSTAT *stats;
} PCDIMSTATS;
*/
char *
pc_dimstats_to_string(const PCDIMSTATS *pds)
{
int i;
stringbuffer_t *sb = stringbuffer_create();
char *str;
stringbuffer_aprintf(sb,"{\"ndims\":%d,\"total_points\":%d,\"total_patches\":%d,\"dims\":[",
pds->ndims,
pds->total_points,
pds->total_patches
);
for ( i = 0; i < pds->ndims; i++ )
{
if ( i ) stringbuffer_append(sb, ",");
stringbuffer_aprintf(sb, "{\"total_runs\":%d,\"total_commonbits\":%d,\"recommended_compression\":%d}",
pds->stats[i].total_runs,
pds->stats[i].total_commonbits,
pds->stats[i].recommended_compression
);
}
stringbuffer_append(sb, "]}");
str = stringbuffer_getstringcopy(sb);
stringbuffer_destroy(sb);
return str;
}
int
pc_dimstats_update(PCDIMSTATS *pds, const PCDIMLIST *pdl)
pc_dimstats_update(PCDIMSTATS *pds, const PCPATCH_DIMENSIONAL *pdl)
{
int i, j;
uint32_t nelems = pdl->npoints;
@ -77,16 +123,16 @@ pc_dimstats_update(PCDIMSTATS *pds, const PCDIMLIST *pdl)
if ( dim->interpretation != PC_DOUBLE )
{
/* If sigbits is better than 4:1, use that */
if ( raw_size/sigbits_size > 4.0 )
if ( raw_size/sigbits_size > 1.6 )
{
pds->stats[i].recommended_compression = PC_DIM_SIGBITS;
}
/* If RLE size is even better, use that. */
else if ( raw_size/rle_size > 4.0 )
if ( raw_size/rle_size > 4.0 )
{
pds->stats[i].recommended_compression = PC_DIM_RLE;
}
}
}
return PC_SUCCESS;
}
}

596
lib/pc_patch.c
View File

@ -9,86 +9,25 @@
***********************************************************************/
#include <math.h>
#include <assert.h>
#include "pc_api_internal.h"
#include "stringbuffer.h"
PCPATCH *
pc_patch_make(const PCSCHEMA *s)
{
PCPATCH *pch;
uint32_t maxpoints = PCPATCH_DEFAULT_MAXPOINTS;
size_t datasize;
if ( ! s )
{
pcerror("null schema passed into pc_patch_make");
return NULL;
}
/* Width of the data area */
if ( ! s->size )
{
pcerror("invalid size calculation in pc_patch_make");
return NULL;
}
/* Make our own data area */
pch = pcalloc(sizeof(PCPATCH));
pch->compressed = PC_FALSE;
datasize = s->size * maxpoints;
pch->data = pcalloc(datasize);
pch->datasize = datasize;
/* Initialize bounds */
pch->xmin = pch->ymin = MAXFLOAT;
pch->xmax = pch->ymax = -1 * MAXFLOAT;
/* Set up basic info */
pch->readonly = PC_FALSE;
pch->npoints = 0;
pch->maxpoints = maxpoints;
pch->schema = s;
return pch;
}
static int
pc_patch_compute_extent_uncompressed(PCPATCH *patch)
{
int i;
PCPOINT *pt = pc_point_from_data(patch->schema, patch->data);
/* Initialize bounds */
patch->xmin = patch->ymin = MAXFLOAT;
patch->xmax = patch->ymax = -1 * MAXFLOAT;
/* Calculate bounds */
for ( i = 0; i < patch->npoints; i++ )
{
double x, y;
/* Just push the data buffer forward by one point at a time */
pt->data = patch->data + i * patch->schema->size;
x = pc_point_get_x(pt);
y = pc_point_get_y(pt);
if ( patch->xmin > x ) patch->xmin = x;
if ( patch->ymin > y ) patch->ymin = y;
if ( patch->xmax < x ) patch->xmax = x;
if ( patch->ymax < y ) patch->ymax = y;
}
return PC_SUCCESS;
}
static int
int
pc_patch_compute_extent(PCPATCH *patch)
{
switch( patch->schema->compression )
switch( patch->type )
{
case PC_NONE:
return pc_patch_compute_extent_uncompressed(patch);
return pc_patch_uncompressed_compute_extent((PCPATCH_UNCOMPRESSED*)patch);
case PC_GHT:
return PC_FAILURE;
case PC_DIMENSIONAL:
return PC_FAILURE;
return pc_patch_dimensional_compute_extent((PCPATCH_DIMENSIONAL*)patch);
}
return PC_FAILURE;
}
@ -96,397 +35,97 @@ pc_patch_compute_extent(PCPATCH *patch)
void
pc_patch_free(PCPATCH *patch)
{
if ( ! patch->readonly )
switch( patch->type )
{
pcfree(patch->data);
case PC_NONE:
{
pc_patch_uncompressed_free((PCPATCH_UNCOMPRESSED*)patch);
break;
}
case PC_GHT:
{
pcerror("pc_patch_free: GHT not supported");
break;
}
case PC_DIMENSIONAL:
{
pc_patch_dimensional_free((PCPATCH_DIMENSIONAL*)patch);
break;
}
default:
{
pcerror("pc_patch_free: unknown compression type %d", patch->type);
break;
}
}
pcfree(patch);
}
int
pc_patch_add_point(PCPATCH *c, const PCPOINT *p)
{
size_t sz;
uint8_t *ptr;
double x, y;
if ( ! ( c && p ) )
{
pcerror("pc_patch_add_point: null point or patch argument");
return PC_FAILURE;
}
if ( c->schema->pcid != p->schema->pcid )
{
pcerror("pc_patch_add_point: pcids of point (%d) and patch (%d) not equal", c->schema->pcid, p->schema->pcid);
return PC_FAILURE;
}
if ( c->readonly )
{
pcerror("pc_patch_add_point: cannot add point to readonly patch");
return PC_FAILURE;
}
if ( c->compressed && c->schema->compression != PC_NONE )
{
pcerror("pc_patch_add_point: cannot add point to compressed patch");
return PC_FAILURE;
}
sz = c->schema->size;
/* Double the data buffer if it's already full */
if ( c->npoints == c->maxpoints )
{
c->maxpoints *= 2;
c->datasize = c->maxpoints * sz;
c->data = pcrealloc(c->data, c->datasize);
}
/* Copy the data buffer from point to patch */
ptr = c->data + sz * c->npoints;
memcpy(ptr, p->data, sz);
c->npoints += 1;
/* Update bounding box */
x = pc_point_get_x(p);
y = pc_point_get_y(p);
if ( c->xmin > x ) c->xmin = x;
if ( c->ymin > y ) c->ymin = y;
if ( c->xmax < x ) c->xmax = x;
if ( c->ymax < y ) c->ymax = y;
return PC_SUCCESS;
}
PCPATCH *
pc_patch_from_points(const PCPOINTLIST *pl)
pc_patch_from_pointlist(const PCPOINTLIST *ptl)
{
PCPATCH *pch;
const PCSCHEMA *s;
uint8_t *ptr;
int i;
uint32_t numpts;
if ( ! pl )
{
pcerror("null PCPOINTLIST passed into pc_patch_from_points");
return NULL;
}
numpts = pl->npoints;
if ( ! numpts )
{
pcerror("zero size PCPOINTLIST passed into pc_patch_from_points");
return NULL;
}
/* Assume the first PCSCHEMA is the same as the rest for now */
/* We will check this as we go along */
s = pl->points[0]->schema;
/* Confirm we have a schema pointer */
if ( ! s )
{
pcerror("pc_patch_from_points: null schema encountered");
return NULL;
}
/* Confirm width of a point data buffer */
if ( ! s->size )
{
pcerror("pc_patch_from_points: invalid point size");
return NULL;
}
/* Make our own data area */
pch = pcalloc(sizeof(PCPATCH));
pch->datasize = s->size * numpts;
pch->data = pcalloc(pch->datasize);
ptr = pch->data;
/* Initialize bounds */
pch->xmin = pch->ymin = MAXFLOAT;
pch->xmax = pch->ymax = -1 * MAXFLOAT;
/* Set up basic info */
pch->readonly = PC_FALSE;
pch->compressed = PC_FALSE;
pch->maxpoints = numpts;
pch->schema = s;
pch->npoints = 0;
for ( i = 0; i < numpts; i++ )
{
if ( pl->points[i] )
{
if ( pl->points[i]->schema->pcid != s->pcid )
{
pcerror("pc_patch_from_points: points do not share a schema");
return NULL;
}
memcpy(ptr, pl->points[i]->data, s->size);
pch->npoints++;
ptr += s->size;
}
else
{
pcwarn("pc_patch_from_points: encountered null point");
}
}
if ( ! pc_patch_compute_extent(pch) )
{
pcerror("pc_patch_from_points: failed to compute patch extent");
return NULL;
}
return pch;
}
PCPOINTLIST *
pc_patch_to_points_uncompressed(const PCPATCH *patch)
{
int i;
PCPOINTLIST *pl;
size_t pt_size = patch->schema->size;
uint32_t npoints = patch->npoints;
pl = pc_pointlist_make(npoints);
for ( i = 0; i < npoints; i++ )
{
pc_pointlist_add_point(pl, pc_point_from_data(patch->schema, patch->data + i*pt_size));
}
return pl;
return (PCPATCH*)pc_patch_uncompressed_from_pointlist(ptl);
}
PCPOINTLIST *
pc_patch_to_points(const PCPATCH *patch)
pc_patch_to_pointlist(const PCPATCH *patch)
{
uint32_t compression = patch->schema->compression;
if ( ! patch->compressed )
return pc_patch_to_points_uncompressed(patch);
switch ( compression )
switch ( patch->type )
{
case PC_NONE:
{
return pc_patch_to_points_uncompressed(patch);
return pc_patch_uncompressed_to_pointlist((PCPATCH_UNCOMPRESSED*)patch);
}
case PC_GHT:
{
// return pc_patch_to_points_ght(patch);
// return pc_patch_to_pointlist_ght(patch);
}
case PC_DIMENSIONAL:
{
// return pc_patch_to_points_dimensional(patch);
PCPATCH_UNCOMPRESSED *pch = pc_patch_uncompressed_from_dimensional((PCPATCH_DIMENSIONAL*)patch);
PCPOINTLIST *ptl = pc_patch_uncompressed_to_pointlist((PCPATCH_UNCOMPRESSED*)patch);
pc_patch_uncompressed_free(pch);
return ptl;
}
}
/* Don't get here */
pcerror("pc_patch_to_points: unsupported compression type %d", compression);
return NULL;
}
#if 0
static PCPATCH *
pc_patch_compress_dimensional(const PCPATCH *patch)
{
PCDIMLIST *pdl;
PCDIMSTATS *pds = NULL;
int rv;
if ( patch->compressed )
pcerror("pc_patch_compress_dimensional cannot work on previously compressed data");
pdl = pc_dimlist_from_patch(patch);
rv = pc_dimlist_encode(pdl, &pds);
return NULL;
}
#endif
static PCPATCH *
pc_patch_compress_ght(const PCPATCH *patch)
{
pcerror("pc_patch_compress_ght unimplemented");
return NULL;
}
PCPATCH *
pc_patch_compress(const PCPATCH *patch)
{
uint32_t compression = patch->schema->compression;
if ( patch->compressed )
return pc_patch_clone(patch);
switch ( compression )
{
case PC_NONE:
{
PCPATCH *newpatch = pc_patch_clone(patch);
newpatch->compressed = PC_TRUE;
return newpatch;
}
case PC_GHT:
{
return pc_patch_compress_ght(patch);
}
case PC_DIMENSIONAL:
{
// return pc_patch_compress_dimensional(patch);
}
}
/* Don't get here */
pcerror("pc_patch_compress: unknown compression type %d", compression);
pcerror("pc_patch_to_pointlist: unsupported compression type %d", patch->type);
return NULL;
}
PCPATCH *
pc_patch_clone(const PCPATCH *patch)
pc_patch_compress(const PCPATCH *patch, void *userdata)
{
PCPATCH *newpatch = pcalloc(sizeof(PCPATCH));
memcpy(newpatch, patch, sizeof(PCPATCH));
newpatch->data = pcalloc(newpatch->datasize);
memcpy(newpatch->data, patch->data, newpatch->datasize);
return newpatch;
uint32_t schema_compression = patch->schema->compression;
uint32_t patch_compression = patch->type;
if ( schema_compression == PC_DIMENSIONAL &&
patch_compression == PC_NONE )
{
PCPATCH_DIMENSIONAL *pcdu = pc_patch_dimensional_from_uncompressed((PCPATCH_UNCOMPRESSED*)patch);
PCPATCH_DIMENSIONAL *pcdd = pc_patch_dimensional_compress(pcdu, (PCDIMSTATS*)userdata);
pc_patch_dimensional_free(pcdu);
return (PCPATCH*)pcdd;
}
if ( schema_compression == PC_DIMENSIONAL &&
patch_compression == PC_DIMENSIONAL )
{
return (PCPATCH*)pc_patch_dimensional_compress((PCPATCH_DIMENSIONAL*)patch, (PCDIMSTATS*)userdata);
}
if ( schema_compression == PC_NONE &&
patch_compression == PC_NONE )
{
return (PCPATCH*)patch;
}
pcerror("pc_patch_compress: cannot convert patch compressed %d to compressed %d", patch_compression, schema_compression);
return NULL;
}
static PCPATCH *
pc_patch_from_wkb_uncompressed(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize)
{
/*
byte: endianness (1 = NDR, 0 = XDR)
uint32: pcid (key to POINTCLOUD_SCHEMAS)
uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT)
uint32: npoints
pcpoint[]: data (interpret relative to pcid)
*/
static size_t hdrsz = 1+4+4+4; /* endian + pcid + compression + npoints */
PCPATCH *patch;
uint8_t *data;
uint8_t swap_endian = (wkb[0] != machine_endian());
uint32_t npoints;
if ( wkb_get_compression(wkb) != PC_NONE )
{
pcerror("pc_patch_from_wkb_uncompressed: call with wkb that is not uncompressed");
return NULL;
}
npoints = wkb_get_npoints(wkb);
if ( (wkbsize - hdrsz) != (s->size * npoints) )
{
pcerror("pc_patch_from_wkb_uncompressed: wkb size and expected data size do not match");
return NULL;
}
if ( swap_endian )
{
data = uncompressed_bytes_flip_endian(wkb+hdrsz, s, npoints);
}
else
{
data = pcalloc(npoints * s->size);
memcpy(data, wkb+hdrsz, npoints*s->size);
}
patch = pcalloc(sizeof(PCPATCH));
patch->npoints = npoints;
patch->maxpoints = npoints;
patch->schema = s;
patch->compressed = PC_TRUE; /* It's in whatever compression it came in */
patch->datasize = (wkbsize - hdrsz);
patch->data = data;
patch->readonly = PC_FALSE;
if ( PC_FAILURE == pc_patch_compute_extent(patch) )
{
pcerror("pc_patch_compute_extent failed");
}
return patch;
}
#if 0
static PCPATCH *
pc_patch_from_wkb_dimensional(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize)
{
/*
byte: endianness (1 = NDR, 0 = XDR)
uint32: pcid (key to POINTCLOUD_SCHEMAS)
uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT)
uint32: npoints
pcpoint[]: data (interpret relative to pcid)
*/
static size_t hdrsz = 1+4+4+4; /* endian + pcid + compression + npoints */
PCPATCH *patch;
uint8_t *data;
uint8_t swap_endian = (wkb[0] != machine_endian());
uint32_t npoints;
size_t datasize = wkbsize - hdrsz;
if ( wkb_get_compression(wkb) != PC_DIMENSIONAL )
{
pcerror("pc_patch_from_wkb_dimensional: call with wkb that is not dimensionally compressed");
return NULL;
}
npoints = wkb_get_npoints(wkb);
#if 0
typedef struct
{
int8_t readonly;
uint32_t npoints; /* How many points we have */
uint32_t maxpoints; /* How man points we can hold (or 0 for read-only) */
const PCSCHEMA *schema;
double xmin, xmax, ymin, ymax;
uint8_t compressed; /* Has compression been applied to the data buffer? */
size_t datasize;
uint8_t *data; /* A serialized version of the data */
} PCPATCH;
#endif
if ( swap_endian )
{
data = uncompressed_bytes_flip_endian(wkb+hdrsz, s, npoints);
}
else
{
data = pcalloc(datasize);
memcpy(data, wkb+hdrsz, datasize);
}
patch = pcalloc(sizeof(PCPATCH));
patch->npoints = npoints;
patch->maxpoints = 0;
patch->schema = s;
patch->compressed = PC_TRUE; /* It's in whatever compression it came in */
patch->datasize = datasize;
patch->data = data;
if ( PC_FAILURE == pc_patch_compute_extent(patch) )
{
pcerror("pc_patch_compute_extent failed");
}
return patch;
}
#endif
PCPATCH *
pc_patch_from_wkb(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize)
{
@ -503,13 +142,14 @@ pc_patch_from_wkb(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize)
pcerror("pc_patch_from_wkb: zero length wkb");
}
/*
* It is possible for the WKB compression to be different from the
* schema compression at this point. The schema compression is only
* forced at serialization time.
*/
pcid = wkb_get_pcid(wkb);
compression = wkb_get_compression(wkb);
// if ( compression != s->compression )
// {
// pcerror("pc_patch_from_wkb: wkb compression (%d) not consistent with schema compression (%d)", compression, s->compression);
// }
if ( pcid != s->pcid )
{
pcerror("pc_patch_from_wkb: wkb pcid (%d) not consistent with schema pcid (%d)", pcid, s->pcid);
@ -519,18 +159,17 @@ pc_patch_from_wkb(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize)
{
case PC_NONE:
{
return pc_patch_from_wkb_uncompressed(s, wkb, wkbsize);
return pc_patch_uncompressed_from_wkb(s, wkb, wkbsize);
}
case PC_DIMENSIONAL:
{
return pc_patch_dimensional_from_wkb(s, wkb, wkbsize);
}
case PC_GHT:
{
pcerror("pc_patch_from_wkb: GHT compression not yet supported");
return NULL;
}
case PC_DIMENSIONAL:
{
pcerror("pc_patch_from_wkb: Dimensional compression not yet supported");
return NULL;
}
}
/* Don't get here */
@ -538,31 +177,7 @@ pc_patch_from_wkb(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize)
return NULL;
}
static uint8_t *
pc_patch_to_wkb_uncompressed(const PCPATCH *patch, size_t *wkbsize)
{
/*
byte: endianness (1 = NDR, 0 = XDR)
uint32: pcid (key to POINTCLOUD_SCHEMAS)
uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT)
uint32: npoints
uchar[]: data (interpret relative to pcid)
*/
char endian = machine_endian();
/* endian + pcid + compression + npoints + datasize */
size_t size = 1 + 4 + 4 + 4 + patch->datasize;
uint8_t *wkb = pcalloc(size);
uint32_t compression = patch->schema->compression;
uint32_t npoints = patch->npoints;
uint32_t pcid = patch->schema->pcid;
wkb[0] = endian; /* Write endian flag */
memcpy(wkb + 1, &pcid, 4); /* Write PCID */
memcpy(wkb + 5, &compression, 4); /* Write compression */
memcpy(wkb + 9, &npoints, 4); /* Write npoints */
memcpy(wkb + 13, patch->data, patch->datasize); /* Write data */
if ( wkbsize ) *wkbsize = size;
return wkb;
}
uint8_t *
pc_patch_to_wkb(const PCPATCH *patch, size_t *wkbsize)
@ -573,23 +188,21 @@ pc_patch_to_wkb(const PCPATCH *patch, size_t *wkbsize)
uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT)
uchar[]: data (interpret relative to pcid and compression)
*/
switch ( patch->schema->compression )
switch ( patch->type )
{
case PC_NONE:
{
return pc_patch_to_wkb_uncompressed(patch, wkbsize);
return pc_patch_uncompressed_to_wkb((PCPATCH_UNCOMPRESSED*)patch, wkbsize);
}
case PC_DIMENSIONAL:
{
return pc_patch_dimensional_to_wkb((PCPATCH_DIMENSIONAL*)patch, wkbsize);
}
case PC_GHT:
{
pcerror("pc_patch_to_wkb: GHT compression not yet supported");
return NULL;
}
case PC_DIMENSIONAL:
{
pcerror("pc_patch_to_wkb: Dimensional compression not yet supported");
return NULL;
}
}
pcerror("pc_patch_to_wkb: unknown compression requested '%d'", patch->schema->compression);
return NULL;
@ -598,45 +211,14 @@ pc_patch_to_wkb(const PCPATCH *patch, size_t *wkbsize)
char *
pc_patch_to_string(const PCPATCH *patch)
{
/* ( <pcid> : (<dim1>, <dim2>, <dim3>, <dim4>), (<dim1>, <dim2>, <dim3>, <dim4>))*/
stringbuffer_t *sb = stringbuffer_create();
PCPOINTLIST *pl;
char *str;
int i, j;
pl = pc_patch_to_points(patch);
stringbuffer_aprintf(sb, "[ %d : ", patch->schema->pcid);
for ( i = 0; i < pl->npoints; i++ )
{
PCPOINT *pt = pl->points[i];
if ( i )
{
stringbuffer_append(sb, ", ");
}
stringbuffer_append(sb, "(");
for ( j = 0; j < pt->schema->ndims; j++ )
{
double d;
if ( ! pc_point_get_double_by_index(pt, j, &d))
{
pcerror("pc_patch_to_string: unable to read double at index %d", j);
}
if ( j )
{
stringbuffer_append(sb, ", ");
}
stringbuffer_aprintf(sb, "%g", d);
}
stringbuffer_append(sb, ")");
}
stringbuffer_append(sb, " ]");
/* All done, copy and clean up */
pc_pointlist_free(pl);
str = stringbuffer_getstringcopy(sb);
stringbuffer_destroy(sb);
return str;
switch( patch->type )
{
case PC_NONE:
return pc_patch_uncompressed_to_string((PCPATCH_UNCOMPRESSED*)patch);
case PC_DIMENSIONAL:
return pc_patch_dimensional_to_string((PCPATCH_DIMENSIONAL*)patch);
}
pcerror("pc_patch_to_string: unsupported compression %d requested", patch->type);
}
static uint8_t *

283
lib/pc_patch_dimensional.c Normal file
View File

@ -0,0 +1,283 @@
/***********************************************************************
* pc_patch_dimensional.c
*
* Pointclound patch handling. Create, get and set values from the
* basic PCPATCH structure.
*
* Portions Copyright (c) 2012, OpenGeo
*
***********************************************************************/
#include <math.h>
#include <assert.h>
#include "pc_api_internal.h"
/*
typedef struct
{
int type;
int8_t readonly;
const PCSCHEMA *schema;
uint32_t npoints;
double xmin, xmax, ymin, ymax;
PCBYTES *bytes;
} PCPATCH_DIMENSIONAL;
*/
size_t
pc_patch_dimensional_serialized_size(const PCPATCH *patch)
{
PCPATCH_DIMENSIONAL *p = (PCPATCH_DIMENSIONAL*)patch;
int i;
size_t size = 0;
for ( i = 0; i < p->schema->ndims; i++ )
{
size += pc_bytes_serialized_size(&(p->bytes[i]));
}
return size;
}
char *
pc_patch_dimensional_to_string(const PCPATCH_DIMENSIONAL *pa)
{
PCPATCH_UNCOMPRESSED *patch = pc_patch_uncompressed_from_dimensional(pa);
char *str = pc_patch_uncompressed_to_string(patch);
pc_patch_uncompressed_free(patch);
return str;
}
PCPATCH_DIMENSIONAL *
pc_patch_dimensional_from_uncompressed(const PCPATCH_UNCOMPRESSED *pa)
{
PCPATCH_DIMENSIONAL *pdl;
const PCSCHEMA *schema;
int i, j, ndims, npoints;
assert(pa);
npoints = pa->npoints;
schema = pa->schema;
ndims = schema->ndims;
/* Cannot handle empty patches */
if ( npoints == 0 ) return NULL;
/* Initialize list */
pdl = pcalloc(sizeof(PCPATCH_DIMENSIONAL));
pdl->schema = schema;
pdl->npoints = npoints;
pdl->bytes = pcalloc(ndims * sizeof(PCBYTES));
pdl->readonly = PC_FALSE;
pdl->type = PC_DIMENSIONAL;
pdl->xmin = pa->xmin;
pdl->xmax = pa->xmax;
pdl->ymin = pa->ymin;
pdl->ymax = pa->ymax;
for ( i = 0; i < ndims; i++ )
{
PCDIMENSION *dim = pc_schema_get_dimension(schema, i);
pdl->bytes[i] = pc_bytes_make(dim, npoints);
for ( j = 0; j < npoints; j++ )
{
uint8_t *to = pdl->bytes[i].bytes + dim->size * j;
uint8_t *from = pa->data + schema->size * j + dim->byteoffset;
memcpy(to, from, dim->size);
}
}
return pdl;
}
PCPATCH_DIMENSIONAL *
pc_patch_dimensional_compress(const PCPATCH_DIMENSIONAL *pdl, PCDIMSTATS *pds)
{
int i;
int ndims = pdl->schema->ndims;
PCPATCH_DIMENSIONAL *pdl_compressed;
assert(pdl);
assert(pdl->schema);
if ( ! pds )
pds = pc_dimstats_make(pdl->schema);
/* Still sampling, update stats */
if ( pds->total_points < PCDIMSTATS_MIN_SAMPLE )
pc_dimstats_update(pds, pdl);
pdl_compressed = pcalloc(sizeof(PCPATCH_DIMENSIONAL));
memcpy(pdl_compressed, pdl, sizeof(PCPATCH_DIMENSIONAL));
pdl_compressed->bytes = pcalloc(ndims*sizeof(PCBYTES));
/* Compress each dimension as dictated by stats */
for ( i = 0; i < ndims; i++ )
{
pdl_compressed->bytes[i] = pc_bytes_encode(pdl->bytes[i], pds->stats[i].recommended_compression);
}
return pdl_compressed;
}
PCPATCH_DIMENSIONAL *
pc_patch_dimensional_decompress(const PCPATCH_DIMENSIONAL *pdl)
{
int i;
int ndims = pdl->schema->ndims;
PCPATCH_DIMENSIONAL *pdl_decompressed;
assert(pdl);
assert(pdl->schema);
pdl_decompressed = pcalloc(sizeof(PCPATCH_DIMENSIONAL));
memcpy(pdl_decompressed, pdl, sizeof(PCPATCH_DIMENSIONAL));
pdl_decompressed->bytes = pcalloc(ndims*sizeof(PCBYTES));
/* Compress each dimension as dictated by stats */
for ( i = 0; i < ndims; i++ )
{
pdl_decompressed->bytes[i] = pc_bytes_decode(pdl->bytes[i]);
}
return pdl_decompressed;
}
void
pc_patch_dimensional_free(PCPATCH_DIMENSIONAL *pdl)
{
int i;
assert(pdl);
assert(pdl->schema);
if ( pdl->bytes )
{
for ( i = 0; i < pdl->schema->ndims; i++ )
pc_bytes_free(pdl->bytes[i]);
pcfree(pdl->bytes);
}
pcfree(pdl);
}
int
pc_patch_dimensional_compute_extent(PCPATCH_DIMENSIONAL *pdl)
{
int i;
double xmin, xmax, ymin, ymax;
int rv;
PCBYTES *pcb;
assert(pdl);
assert(pdl->schema);
/* Get x extremes */
pcb = &(pdl->bytes[pdl->schema->x_position]);
rv = pc_bytes_minmax(pcb, xmin, xmax);
pdl->xmin = xmin;
pdl->xmax = xmax;
/* Get y extremes */
pcb = &(pdl->bytes[pdl->schema->y_position]);
rv = pc_bytes_minmax(pcb, ymin, ymax);
pdl->ymin = ymin;
pdl->ymax = ymax;
return PC_SUCCESS;
}
uint8_t *
pc_patch_dimensional_to_wkb(const PCPATCH_DIMENSIONAL *patch, size_t *wkbsize)
{
/*
byte: endianness (1 = NDR, 0 = XDR)
uint32: pcid (key to POINTCLOUD_SCHEMAS)
uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT)
uint32: npoints
dimensions[]: pcbytes (interpret relative to pcid and compressions)
*/
int ndims = patch->schema->ndims;
int i;
uint8_t *buf;
char endian = machine_endian();
/* endian + pcid + compression + npoints + datasize */
size_t size = 1 + 4 + 4 + 4 + pc_patch_dimensional_serialized_size((PCPATCH*)patch);
uint8_t *wkb = pcalloc(size);
uint32_t compression = patch->type;
uint32_t npoints = patch->npoints;
uint32_t pcid = patch->schema->pcid;
wkb[0] = endian; /* Write endian flag */
memcpy(wkb + 1, &pcid, 4); /* Write PCID */
memcpy(wkb + 5, &compression, 4); /* Write compression */
memcpy(wkb + 9, &npoints, 4); /* Write npoints */
buf = wkb + 13;
for ( i = 0; i < ndims; i++ )
{
size_t bsz;
PCBYTES *pcb = &(patch->bytes[i]);
pc_bytes_serialize(pcb, buf, &bsz);
buf += bsz;
}
if ( wkbsize ) *wkbsize = size;
return wkb;
}
PCPATCH *
pc_patch_dimensional_from_wkb(const PCSCHEMA *schema, const uint8_t *wkb, size_t wkbsize)
{
/*
byte: endianness (1 = NDR, 0 = XDR)
uint32: pcid (key to POINTCLOUD_SCHEMAS)
uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT)
uint32: npoints
dimensions[]: dims (interpret relative to pcid and compressions)
*/
static size_t hdrsz = 1+4+4+4; /* endian + pcid + compression + npoints */
PCPATCH_DIMENSIONAL *patch;
uint8_t swap_endian = (wkb[0] != machine_endian());
uint32_t npoints, ndims;
const uint8_t *buf;
int i;
if ( wkb_get_compression(wkb) != PC_DIMENSIONAL )
{
pcerror("pc_patch_dimensional_from_wkb: call with wkb that is not dimensionally compressed");
return NULL;
}
npoints = wkb_get_npoints(wkb);
ndims = schema->ndims;
patch = pcalloc(sizeof(PCPATCH_DIMENSIONAL));
patch->npoints = npoints;
patch->type = PC_DIMENSIONAL;
patch->schema = schema;
patch->readonly = PC_FALSE;
patch->bytes = pcalloc(ndims*sizeof(PCBYTES));
buf = wkb+hdrsz;
for ( i = 0; i < ndims; i++ )
{
PCBYTES *pcb = &(patch->bytes[i]);
pc_bytes_deserialize(buf, schema->dims[i], pcb, PC_FALSE /*readonly*/, swap_endian);
pcb->npoints = npoints;
buf += pcb->size;
}
if ( PC_FAILURE == pc_patch_dimensional_compute_extent(patch) )
pcerror("pc_patch_dimensional_compute_extent failed");
return (PCPATCH*)patch;
}
PCPATCH_DIMENSIONAL *
pc_patch_dimensional_from_pointlist(const PCPOINTLIST *pdl)
{
PCPATCH_UNCOMPRESSED *patch = pc_patch_uncompressed_from_pointlist(pdl);
PCPATCH_DIMENSIONAL *dimpatch = pc_patch_dimensional_from_uncompressed(patch);
pc_patch_uncompressed_free(patch);
return dimpatch;
}

420
lib/pc_patch_uncompressed.c Normal file
View File

@ -0,0 +1,420 @@
/***********************************************************************
* pc_patch_uncompressed.c
*
* Portions Copyright (c) 2012, OpenGeo
*
***********************************************************************/
#include <math.h>
#include "pc_api_internal.h"
#include "stringbuffer.h"
char *
pc_patch_uncompressed_to_string(const PCPATCH_UNCOMPRESSED *patch)
{
/* { "pcid":1, "points":[[<dim1>, <dim2>, <dim3>, <dim4>],[<dim1>, <dim2>, <dim3>, <dim4>]] }*/
stringbuffer_t *sb = stringbuffer_create();
PCPOINTLIST *pl;
char *str;
int i, j;
pl = pc_patch_uncompressed_to_pointlist(patch);
stringbuffer_aprintf(sb, "{\"pcid\":%d,\"pts\":[", patch->schema->pcid);
for ( i = 0; i < pl->npoints; i++ )
{
PCPOINT *pt = pc_pointlist_get_point(pl, i);
if ( i )
{
stringbuffer_append(sb, ",");
}
stringbuffer_append(sb, "[");
for ( j = 0; j < pt->schema->ndims; j++ )
{
double d;
if ( ! pc_point_get_double_by_index(pt, j, &d))
{
pcerror("pc_patch_to_string: unable to read double at index %d", j);
}
if ( j )
{
stringbuffer_append(sb, ",");
}
stringbuffer_aprintf(sb, "%g", d);
}
stringbuffer_append(sb, "]");
}
stringbuffer_append(sb, "]}");
/* All done, copy and clean up */
pc_pointlist_free(pl);
str = stringbuffer_getstringcopy(sb);
stringbuffer_destroy(sb);
return str;
}
uint8_t *
pc_patch_uncompressed_to_wkb(const PCPATCH_UNCOMPRESSED *patch, size_t *wkbsize)
{
/*
byte: endianness (1 = NDR, 0 = XDR)
uint32: pcid (key to POINTCLOUD_SCHEMAS)
uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT)
uint32: npoints
uchar[]: data (interpret relative to pcid)
*/
char endian = machine_endian();
/* endian + pcid + compression + npoints + datasize */
size_t size = 1 + 4 + 4 + 4 + patch->datasize;
uint8_t *wkb = pcalloc(size);
uint32_t compression = patch->type;
uint32_t npoints = patch->npoints;
uint32_t pcid = patch->schema->pcid;
wkb[0] = endian; /* Write endian flag */
memcpy(wkb + 1, &pcid, 4); /* Write PCID */
memcpy(wkb + 5, &compression, 4); /* Write compression */
memcpy(wkb + 9, &npoints, 4); /* Write npoints */
memcpy(wkb + 13, patch->data, patch->datasize); /* Write data */
if ( wkbsize ) *wkbsize = size;
return wkb;
}
PCPATCH *
pc_patch_uncompressed_from_wkb(const PCSCHEMA *s, const uint8_t *wkb, size_t wkbsize)
{
/*
byte: endianness (1 = NDR, 0 = XDR)
uint32: pcid (key to POINTCLOUD_SCHEMAS)
uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT)
uint32: npoints
pcpoint[]: data (interpret relative to pcid)
*/
static size_t hdrsz = 1+4+4+4; /* endian + pcid + compression + npoints */
PCPATCH_UNCOMPRESSED *patch;
uint8_t *data;
uint8_t swap_endian = (wkb[0] != machine_endian());
uint32_t npoints;
if ( wkb_get_compression(wkb) != PC_NONE )
{
pcerror("pc_patch_uncompressed_from_wkb: call with wkb that is not uncompressed");
return NULL;
}
npoints = wkb_get_npoints(wkb);
if ( (wkbsize - hdrsz) != (s->size * npoints) )
{
pcerror("pc_patch_uncompressed_from_wkb: wkb size and expected data size do not match");
return NULL;
}
if ( swap_endian )
{
data = uncompressed_bytes_flip_endian(wkb+hdrsz, s, npoints);
}
else
{
data = pcalloc(npoints * s->size);
memcpy(data, wkb+hdrsz, npoints*s->size);
}
patch = pcalloc(sizeof(PCPATCH_UNCOMPRESSED));
patch->npoints = npoints;
patch->type = PC_NONE;
patch->maxpoints = npoints;
patch->schema = s;
patch->datasize = (wkbsize - hdrsz);
patch->data = data;
patch->readonly = PC_FALSE;
if ( PC_FAILURE == pc_patch_uncompressed_compute_extent(patch) )
pcerror("pc_patch_uncompressed_compute_extent failed");
return (PCPATCH*)patch;
}
PCPATCH_UNCOMPRESSED *
pc_patch_uncompressed_make(const PCSCHEMA *s)
{
PCPATCH_UNCOMPRESSED *pch;
uint32_t maxpoints = PCPATCH_DEFAULT_MAXPOINTS;
size_t datasize;
if ( ! s )
{
pcerror("null schema passed into pc_patch_uncompressed_make");
return NULL;
}
/* Width of the data area */
if ( ! s->size )
{
pcerror("invalid size calculation in pc_patch_uncompressed_make");
return NULL;
}
/* Make our own data area */
pch = pcalloc(sizeof(PCPATCH));
datasize = s->size * maxpoints;
pch->data = pcalloc(datasize);
pch->datasize = datasize;
/* Initialize bounds */
pch->xmin = pch->ymin = MAXFLOAT;
pch->xmax = pch->ymax = -1 * MAXFLOAT;
/* Set up basic info */
pch->readonly = PC_FALSE;
pch->npoints = 0;
pch->type = PC_NONE;
pch->maxpoints = maxpoints;
pch->schema = s;
return pch;
}
int
pc_patch_uncompressed_compute_extent(PCPATCH_UNCOMPRESSED *patch)
{
int i;
PCPOINT *pt = pc_point_from_data(patch->schema, patch->data);
/* Initialize bounds */
patch->xmin = patch->ymin = MAXFLOAT;
patch->xmax = patch->ymax = -1 * MAXFLOAT;
/* Calculate bounds */
for ( i = 0; i < patch->npoints; i++ )
{
double x, y;
/* Just push the data buffer forward by one point at a time */
pt->data = patch->data + i * patch->schema->size;
x = pc_point_get_x(pt);
y = pc_point_get_y(pt);
if ( patch->xmin > x ) patch->xmin = x;
if ( patch->ymin > y ) patch->ymin = y;
if ( patch->xmax < x ) patch->xmax = x;
if ( patch->ymax < y ) patch->ymax = y;
}
return PC_SUCCESS;
}
void
pc_patch_uncompressed_free(PCPATCH_UNCOMPRESSED *patch)
{
if ( ! patch->readonly )
{
pcfree(patch->data);
}
pcfree(patch);
}
PCPOINTLIST *
pc_patch_uncompressed_to_pointlist(const PCPATCH_UNCOMPRESSED *patch)
{
int i;
PCPOINTLIST *pl;
size_t pt_size = patch->schema->size;
uint32_t npoints = patch->npoints;
pl = pc_pointlist_make(npoints);
for ( i = 0; i < npoints; i++ )
{
pc_pointlist_add_point(pl, pc_point_from_data(patch->schema, patch->data + i*pt_size));
}
return pl;
}
PCPATCH_UNCOMPRESSED *
pc_patch_uncompressed_from_pointlist(const PCPOINTLIST *pl)
{
PCPATCH_UNCOMPRESSED *pch;
const PCSCHEMA *s;
PCPOINT *pt;
uint8_t *ptr;
int i;
uint32_t numpts;
if ( ! pl )
{
pcerror("null PCPOINTLIST passed into pc_patch_uncompressed_from_pointlist");
return NULL;
}
numpts = pl->npoints;
if ( ! numpts )
{
pcerror("zero size PCPOINTLIST passed into pc_patch_uncompressed_from_pointlist");
return NULL;
}
/* Assume the first PCSCHEMA is the same as the rest for now */
/* We will check this as we go along */
pt = pc_pointlist_get_point(pl, 0);
s = pt->schema;
/* Confirm we have a schema pointer */
if ( ! s )
{
pcerror("pc_patch_uncompressed_from_pointlist: null schema encountered");
return NULL;
}
/* Confirm width of a point data buffer */
if ( ! s->size )
{
pcerror("pc_patch_uncompressed_from_pointlist: invalid point size");
return NULL;
}
/* Make our own data area */
pch = pcalloc(sizeof(PCPATCH_UNCOMPRESSED));
pch->datasize = s->size * numpts;
pch->data = pcalloc(pch->datasize);
ptr = pch->data;
/* Initialize bounds */
pch->xmin = pch->ymin = MAXFLOAT;
pch->xmax = pch->ymax = -1 * MAXFLOAT;
/* Set up basic info */
pch->readonly = PC_FALSE;
pch->maxpoints = numpts;
pch->type = PC_NONE;
pch->schema = s;
pch->npoints = 0;
for ( i = 0; i < numpts; i++ )
{
pt = pc_pointlist_get_point(pl, i);
if ( pt )
{
if ( pt->schema->pcid != s->pcid )
{
pcerror("pc_patch_uncompressed_from_pointlist: points do not share a schema");
return NULL;
}
memcpy(ptr, pt->data, s->size);
pch->npoints++;
ptr += s->size;
}
else
{
pcwarn("pc_patch_uncompressed_from_pointlist: encountered null point");
}
}
if ( ! pc_patch_compute_extent(pch) )
{
pcerror("pc_patch_uncompressed_from_pointlist: failed to compute patch extent");
return NULL;
}
return pch;
}
PCPATCH_UNCOMPRESSED *
pc_patch_uncompressed_from_dimensional(const PCPATCH_DIMENSIONAL *pdl)
{
int i, j, npoints;
PCPATCH_UNCOMPRESSED *patch;
PCPATCH_DIMENSIONAL *pdl_uncompressed;
const PCSCHEMA *schema;
uint8_t *buf;
npoints = pdl->npoints;
schema = pdl->schema;
patch = pcalloc(sizeof(PCPATCH_UNCOMPRESSED));
patch->schema = schema;
patch->npoints = npoints;
patch->maxpoints = npoints;
patch->readonly = PC_FALSE;
patch->type = PC_NONE;
patch->xmin = pdl->xmin;
patch->xmax = pdl->xmax;
patch->ymin = pdl->ymin;
patch->ymax = pdl->ymax;
patch->datasize = schema->size * pdl->npoints;
patch->data = pcalloc(patch->datasize);
buf = patch->data;
/* Can only read from uncompressed dimensions */
pdl_uncompressed = pc_patch_dimensional_decompress(pdl);
for ( i = 0; i < npoints; i++ )
{
for ( j = 0; j < schema->ndims; j++ )
{
PCDIMENSION *dim = pc_schema_get_dimension(schema, j);
uint8_t *in = pdl_uncompressed->bytes[j].bytes + dim->size * i;
uint8_t *out = buf + dim->byteoffset;
memcpy(out, in, dim->size);
}
buf += schema->size;
}
pc_patch_dimensional_free(pdl_uncompressed);
return patch;
}
int
pc_patch_uncompressed_add_point(PCPATCH_UNCOMPRESSED *c, const PCPOINT *p)
{
size_t sz;
uint8_t *ptr;
double x, y;
if ( ! ( c && p ) )
{
pcerror("pc_patch_uncompressed_add_point: null point or patch argument");
return PC_FAILURE;
}
if ( c->schema->pcid != p->schema->pcid )
{
pcerror("pc_patch_uncompressed_add_point: pcids of point (%d) and patch (%d) not equal", c->schema->pcid, p->schema->pcid);
return PC_FAILURE;
}
if ( c->readonly )
{
pcerror("pc_patch_uncompressed_add_point: cannot add point to readonly patch");
return PC_FAILURE;
}
if ( c->type != PC_NONE )
{
pcerror("pc_patch_uncompressed_add_point: cannot add point to compressed patch");
return PC_FAILURE;
}
sz = c->schema->size;
/* Double the data buffer if it's already full */
if ( c->npoints == c->maxpoints )
{
c->maxpoints *= 2;
c->datasize = c->maxpoints * sz;
c->data = pcrealloc(c->data, c->datasize);
}
/* Copy the data buffer from point to patch */
ptr = c->data + sz * c->npoints;
memcpy(ptr, p->data, sz);
c->npoints += 1;
/* Update bounding box */
x = pc_point_get_x(p);
y = pc_point_get_y(p);
if ( c->xmin > x ) c->xmin = x;
if ( c->ymin > y ) c->ymin = y;
if ( c->xmax < x ) c->xmax = x;
if ( c->ymax < y ) c->ymax = y;
return PC_SUCCESS;
}

8
lib/pc_point.c
View File

@ -169,12 +169,12 @@ pc_point_get_y(const PCPOINT *pt)
char *
pc_point_to_string(const PCPOINT *pt)
{
/* ( <pcid> : <dim1>, <dim2>, <dim3>, <dim4> )*/
/* { "pcid":1, "values":[<dim1>, <dim2>, <dim3>, <dim4>] }*/
stringbuffer_t *sb = stringbuffer_create();
char *str;
int i;
stringbuffer_aprintf(sb, "( %d : ", pt->schema->pcid);
stringbuffer_aprintf(sb, "{\"pcid\":%d,\"pt\":[", pt->schema->pcid);
for ( i = 0; i < pt->schema->ndims; i++ )
{
double d;
@ -184,11 +184,11 @@ pc_point_to_string(const PCPOINT *pt)
}
if ( i )
{
stringbuffer_append(sb, ", ");
stringbuffer_append(sb, ",");
}
stringbuffer_aprintf(sb, "%g", d);
}
stringbuffer_append(sb, " )");
stringbuffer_append(sb, "]}");
str = stringbuffer_getstringcopy(sb);
stringbuffer_destroy(sb);
return str;

25
lib/pc_pointlist.c
View File

@ -50,34 +50,41 @@ pc_pointlist_add_point(PCPOINTLIST *pl, PCPOINT *pt)
return;
}
PCPOINT *
pc_pointlist_get_point(const PCPOINTLIST *pl, int i)
{
return pl->points[i];
}
PCPOINTLIST *
pc_pointlist_from_dimlist(PCDIMLIST *pdl)
pc_pointlist_from_dimensional(const PCPATCH_DIMENSIONAL *pdl)
{
PCPOINTLIST *pl;
PCPATCH_DIMENSIONAL *pdl_uncompressed;
const PCSCHEMA *schema = pdl->schema;
int i, j, ndims, npoints;
assert(pdl);
/* We can only pull off this trick on uncompressed data */
if ( PC_FAILURE == pc_dimlist_decode(pdl) )
return NULL;
pdl_uncompressed = pc_patch_dimensional_decompress(pdl);
ndims = pdl->schema->ndims;
ndims = schema->ndims;
npoints = pdl->npoints;
pl = pc_pointlist_make(npoints);
for ( i = 0; i < npoints; i++ )
{
PCPOINT *pt = pc_point_make(pdl->schema);
PCPOINT *pt = pc_point_make(schema);
for ( j = 0; j < ndims; j++ )
{
PCDIMENSION *dim = pc_schema_get_dimension(pdl->schema, j);
PCDIMENSION *dim = pc_schema_get_dimension(schema, j);
uint8_t *in = pdl->bytes[j].bytes + dim->size * i;
uint8_t *in = pdl_uncompressed->bytes[j].bytes + dim->size * i;
uint8_t *out = pt->data + dim->byteoffset;
memcpy(out, in, dim->size);
}
pc_pointlist_add_point(pl, pt);
}
pc_patch_dimensional_free(pdl_uncompressed);
return pl;
}
}

17
lib/pc_schema.c
View File

@ -69,9 +69,14 @@ size_t
pc_interpretation_size(uint32_t interp)
{
if ( interp >= 0 && interp < NUM_INTERPRETATIONS )
{
return INTERPRETATION_SIZES[interp];
}
else
{
pcerror("pc_interpretation_size: invalid interpretation");
return 0;
}
}
/** Allocate clean memory for a PCDIMENSION struct */
@ -140,7 +145,7 @@ static int
pc_schema_set_dimension(PCSCHEMA *s, PCDIMENSION *d)
{
s->dims[d->position] = d;
hashtable_insert(s->namehash, pcstrdup(d->name), d);
hashtable_insert(s->namehash, d->name, d);
}
@ -480,13 +485,3 @@ pc_schema_get_dimension_by_name(const PCSCHEMA *s, const char *name)
return hashtable_search(s->namehash, name);
}
int
pc_schema_has_name(const PCSCHEMA *s, const char *name)
{
if ( hashtable_search(s->namehash, name) )
{
return PC_TRUE;
}
return PC_FALSE;
}

18
lib/pc_val.c
View File

@ -10,6 +10,24 @@
#include <math.h>
#include "pc_api_internal.h"
double
pc_value_from_ptr(const uint8_t *ptr, const PCDIMENSION *dim)
{
double val = pc_double_from_ptr(ptr, dim->interpretation);
/* Scale value */
if ( dim->scale )
val *= dim->scale;
/* Offset value */
if ( dim->offset )
val += dim->offset;
return val;
}
double
pc_double_from_ptr(const uint8_t *ptr, uint32_t interpretation)
{

31
pgsql/expected/pointcloud.out
View File

@ -79,11 +79,11 @@ SELECT Sum(PC_Get(pt, 'y')) FROM pt_test;
(1 row)
SELECT PC_AsText(pt) FROM pt_test;
pc_astext
-----------------------------
( 1 : 0.01, 0.02, 0.03, 4 )
( 1 : 0.02, 0.03, 0.03, 5 )
( 1 : 0.03, 0.04, 0.03, 6 )
pc_astext
------------------------------------
{"pcid":1,"pt":[0.01,0.02,0.03,4]}
{"pcid":1,"pt":[0.02,0.03,0.03,5]}
{"pcid":1,"pt":[0.03,0.04,0.03,6]}
(3 rows)
CREATE TABLE IF NOT EXISTS pa_test (
@ -103,10 +103,10 @@ LINE 1: INSERT INTO pa_test (pa) VALUES ('00000000020000000000000002...
INSERT INTO pa_test (pa) VALUES ('0000000001000000000000000200000002000000030000000500060000000200000003000000050008');
INSERT INTO pa_test (pa) VALUES ('000000000100000000000000020000000600000007000000050006000000090000000A00000005000A');
SELECT PC_AsText(pa) FROM pa_test;
pc_astext
------------------------------------------------------
[ 1 : (0.02, 0.03, 0.05, 6), (0.02, 0.03, 0.05, 8) ]
[ 1 : (0.06, 0.07, 0.05, 6), (0.09, 0.1, 0.05, 10) ]
pc_astext
----------------------------------------------------------
{"pcid":1,"pts":[[0.02,0.03,0.05,6],[0.02,0.03,0.05,8]]}
{"pcid":1,"pts":[[0.06,0.07,0.05,6],[0.09,0.1,0.05,10]]}
(2 rows)
SELECT PC_Envelope(pa) from pa_test;
@ -119,16 +119,17 @@ SELECT PC_Envelope(pa) from pa_test;
SELECT PC_AsText(PC_Patch(pt)) FROM pt_test;
pc_astext
-----------------------------------------------------------------------------
[ 1 : (0.01, 0.02, 0.03, 4), (0.02, 0.03, 0.03, 5), (0.03, 0.04, 0.03, 6) ]
{"pcid":1,"pts":[[0.01,0.02,0.03,4],[0.02,0.03,0.03,5],[0.03,0.04,0.03,6]]}
(1 row)
SELECT PC_AsText(PC_Explode(PC_Patch(pt))) FROM pt_test;
pc_astext
-----------------------------
( 1 : 0.01, 0.02, 0.03, 4 )
( 1 : 0.02, 0.03, 0.03, 5 )
( 1 : 0.03, 0.04, 0.03, 6 )
pc_astext
------------------------------------
{"pcid":1,"pt":[0.01,0.02,0.03,4]}
{"pcid":1,"pt":[0.02,0.03,0.03,5]}
{"pcid":1,"pt":[0.03,0.04,0.03,6]}
(3 rows)
--DROP TABLE pts_collection;
--DROP TABLE pt_test;
--DROP TABLE pa_test;

10
pgsql/pc_access.c
View File

@ -134,7 +134,7 @@ pcpatch_from_array(ArrayType *array, FunctionCallInfoData *fcinfo)
if ( pl->npoints == 0 )
return NULL;
pa = pc_patch_from_points(pl);
pa = pc_patch_from_pointlist(pl);
pc_pointlist_free(pl);
return pa;
}
@ -154,7 +154,7 @@ Datum pcpatch_from_pcpoint_array(PG_FUNCTION_ARGS)
if ( ! pa )
PG_RETURN_NULL();
serpa = pc_patch_serialize(pa);
serpa = pc_patch_serialize(pa, NULL);
pc_patch_free(pa);
PG_RETURN_POINTER(serpa);
}
@ -281,7 +281,7 @@ Datum pcpoint_agg_final_pcpatch(PG_FUNCTION_ARGS)
if ( ! pa )
PG_RETURN_NULL();
serpa = pc_patch_serialize(pa);
serpa = pc_patch_serialize(pa, NULL);
pc_patch_free(pa);
PG_RETURN_POINTER(serpa);
}
@ -330,7 +330,7 @@ Datum pcpatch_unnest(PG_FUNCTION_ARGS)
/* initialize state */
fctx->nextelem = 0;
fctx->numelems = patch->npoints;
fctx->pointlist = pc_patch_to_points(patch);
fctx->pointlist = pc_patch_to_pointlist(patch);
/* save user context, switch back to function context */
funcctx->user_fctx = fctx;
@ -344,7 +344,7 @@ Datum pcpatch_unnest(PG_FUNCTION_ARGS)
if (fctx->nextelem < fctx->numelems)
{
Datum elem;
PCPOINT *pt = fctx->pointlist->points[fctx->nextelem];
PCPOINT *pt = pc_pointlist_get_point(fctx->pointlist, fctx->nextelem);
SERIALIZED_POINT *serpt = pc_point_serialize(pt);
fctx->nextelem++;
elem = PointerGetDatum(serpt);

13
pgsql/pc_inout.c
View File

@ -125,7 +125,7 @@ Datum pcpatch_in(PG_FUNCTION_ARGS)
/* Hex-encoded binary */
patch = pc_patch_from_hexwkb(str, strlen(str), fcinfo);
pcid_consistent(patch->schema->pcid, pcid);
serpatch = pc_patch_serialize(patch);
serpatch = pc_patch_serialize(patch, NULL);
pc_patch_free(patch);
}
else
@ -196,7 +196,6 @@ Datum pcpoint_from_double_array(PG_FUNCTION_ARGS)
uint32 pcid = PG_GETARG_INT32(0);
ArrayType *arrptr = PG_GETARG_ARRAYTYPE_P(1);
int nelems;
int i;
float8 *vals;
PCPOINT *pt;
PCSCHEMA *schema = pc_schema_from_pcid(pcid, fcinfo);
@ -218,17 +217,9 @@ Datum pcpoint_from_double_array(PG_FUNCTION_ARGS)
if ( nelems != schema->ndims || ARR_LBOUND(arrptr)[0] > 1 )
elog(ERROR, "array dimenensions do not match schema dimensions of pcid = %d", pcid);
pt = pc_point_make(schema);
vals = (float8*) ARR_DATA_PTR(arrptr);
pt = pc_point_from_double_array(schema, vals, nelems);
for ( i = 0; i < nelems; i++ )
{
float8 val = vals[i];
int err = pc_point_set_double_by_index(pt, i, val);
if ( ! err )
elog(ERROR, "failed to set value into point");
}
serpt = pc_point_serialize(pt);
pc_point_free(pt);
pc_schema_free(schema);

187
pgsql/pc_pgsql.c
View File

@ -431,19 +431,41 @@ pc_point_deserialize(const SERIALIZED_POINT *serpt, const PCSCHEMA *schema)
}
SERIALIZED_PATCH *
pc_patch_serialize(const PCPATCH *pcpch)
size_t
pc_patch_serialized_size(const PCPATCH *patch)
{
switch( patch->type )
{
case PC_NONE:
{
PCPATCH_UNCOMPRESSED *upatch = (PCPATCH_UNCOMPRESSED*)patch;
return sizeof(SERIALIZED_PATCH) - 1 + upatch->datasize;
}
case PC_GHT:
{
pcerror("pc_patch_serialized_size: GHT format not yet supported");
}
case PC_DIMENSIONAL:
{
return sizeof(SERIALIZED_PATCH) - 1 + pc_patch_dimensional_serialized_size(patch);
}
default:
{
pcerror("pc_patch_serialized_size: unknown compresed %d", patch->type);
}
}
return -1;
}
static SERIALIZED_PATCH *
pc_patch_uncompressed_serialize(const PCPATCH *patch_in)
{
size_t serpch_size;
PCPATCH *patch;
SERIALIZED_PATCH *serpch;
const PCPATCH_UNCOMPRESSED *patch = (PCPATCH_UNCOMPRESSED *)patch_in;
/* Compress uncompressed patches before saving */
patch = pc_patch_compress(pcpch);
/* Allocate: size(int4) + pcid(int4) + npoints(int4) + box(4*float8) + data(?) */
serpch_size = sizeof(SERIALIZED_PATCH) - 1 + patch->datasize;
serpch = palloc(serpch_size);
serpch_size = pc_patch_serialized_size(patch_in);
serpch = pcalloc(serpch_size);
/* Copy */
serpch->pcid = patch->schema->pcid;
@ -457,20 +479,103 @@ pc_patch_serialize(const PCPATCH *pcpch)
return serpch;
}
PCPATCH *
pc_patch_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema)
static SERIALIZED_PATCH *
pc_patch_dimensional_serialize(const PCPATCH *patch_in)
{
PCPATCH *patch;
size_t serpch_size;
SERIALIZED_PATCH *serpch;
int i;
uint8_t *buf;
const PCPATCH_DIMENSIONAL *patch = (PCPATCH_DIMENSIONAL*)patch_in;
serpch_size = pc_patch_serialized_size((PCPATCH*)patch);
serpch = pcalloc(serpch_size);
/* Copy basics */
serpch->pcid = patch->schema->pcid;
serpch->npoints = patch->npoints;
serpch->xmin = patch->xmin;
serpch->ymin = patch->ymin;
serpch->xmax = patch->xmax;
serpch->ymax = patch->ymax;
/* Copy byte buffers, one by one */
buf = serpch->data;
for ( i = 0; i < patch->schema->ndims; i++ )
{
size_t bsize = 0;
PCBYTES *pcb = &(patch->bytes[i]);
pc_bytes_serialize(pcb, buf, &bsize);
buf += bsize;
}
SET_VARSIZE(serpch, serpch_size);
return serpch;
}
/**
* Convert struct to byte array.
* Userdata is currently only PCDIMSTATS, hopefully updated across
* a number of iterations and saved.
*/
SERIALIZED_PATCH *
pc_patch_serialize(const PCPATCH *patch_in, void *userdata)
{
PCPATCH *patch = (PCPATCH*)patch_in;
SERIALIZED_PATCH *serpatch;
/*
* Convert the patch to the final target compression,
* which is the one in the schema.
*/
if ( patch->type != patch->schema->compression )
{
patch = pc_patch_compress(patch_in, userdata);
}
switch( patch->type )
{
case PC_NONE:
serpatch = pc_patch_uncompressed_serialize(patch);
break;
case PC_DIMENSIONAL:
{
serpatch = pc_patch_dimensional_serialize(patch);
break;
}
case PC_GHT:
{
pcerror("pc_patch_serialize: GHT compression currently unsupported");
break;
}
default:
{
pcerror("pc_patch_serialize: unsupported compression type %d", patch->type);
}
}
if ( patch != patch_in )
pc_patch_free(patch);
return serpatch;
}
static PCPATCH *
pc_patch_uncompressed_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema)
{
PCPATCH_UNCOMPRESSED *patch;
/* Reference the external data */
patch = pcalloc(sizeof(PCPATCH));
patch = pcalloc(sizeof(PCPATCH_UNCOMPRESSED));
patch->data = (uint8*)serpatch->data;
patch->datasize = VARSIZE(serpatch) - sizeof(SERIALIZED_PATCH) + 1;
/* Set up basic info */
patch->schema = schema;
patch->readonly = true;
patch->compressed = true;
patch->npoints = serpatch->npoints;
patch->maxpoints = 0;
patch->xmin = serpatch->xmin;
@ -478,6 +583,58 @@ pc_patch_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema)
patch->xmax = serpatch->xmax;
patch->ymax = serpatch->ymax;
return patch;
return (PCPATCH*)patch;
}
static PCPATCH *
pc_patch_dimensional_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema)
{
PCPATCH_DIMENSIONAL *patch;
int i;
const uint8_t *buf;
int ndims = schema->ndims;
int npoints = serpatch->npoints;
/* Reference the external data */
patch = pcalloc(sizeof(PCPATCH_DIMENSIONAL));
/* Set up basic info */
patch->schema = schema;
patch->readonly = true;
patch->npoints = npoints;
patch->xmin = serpatch->xmin;
patch->ymin = serpatch->ymin;
patch->xmax = serpatch->xmax;
patch->ymax = serpatch->ymax;
/* Set up dimensions */
patch->bytes = pcalloc(ndims * sizeof(PCBYTES));
buf = serpatch->data;
for ( i = 0; i < ndims; i++ )
{
PCBYTES *pcb = &(patch->bytes[i]);
PCDIMENSION *dim = schema->dims[i];
pc_bytes_deserialize(buf, dim, pcb, true /*readonly*/, false /*flipendian*/);
pcb->npoints = npoints;
buf += pcb->size;
}
return (PCPATCH*)patch;
}
PCPATCH *
pc_patch_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema)
{
switch(schema->compression)
{
case PC_NONE:
return pc_patch_uncompressed_deserialize(serpatch, schema);
case PC_DIMENSIONAL:
return pc_patch_dimensional_deserialize(serpatch, schema);
case PC_GHT:
pcerror("pc_patch_deserialize: GHT compression currently unsupported");
}
pcerror("pc_patch_deserialize: unsupported compression type");
return NULL;
}

5
pgsql/pc_pgsql.h
View File

@ -83,8 +83,11 @@ PCPOINT* pc_point_from_hexwkb(const char *hexwkb, size_t hexlen, FunctionCallInf
/** Create a hex representation of a PCPOINT */
char* pc_point_to_hexwkb(const PCPOINT *pt);
/** How big will this thing be on disk? */
size_t pc_patch_serialized_size(const PCPATCH *patch);
/** Turn a PCPATCH into a byte buffer suitable for saving in PgSQL */
SERIALIZED_PATCH* pc_patch_serialize(const PCPATCH *patch);
SERIALIZED_PATCH* pc_patch_serialize(const PCPATCH *patch, void *userdata);
/** Turn a byte buffer into a PCPATCH for processing */
PCPATCH* pc_patch_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema);