From ef3413c2ade222cf1266c11637d4d443957283b2 Mon Sep 17 00:00:00 2001 From: Blottiere Paul Date: Wed, 9 Feb 2022 13:56:01 +0100 Subject: [PATCH 1/4] Add update chapter --- doc/index.rst | 8 ++++++++ doc/update.rst | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 doc/update.rst diff --git a/doc/index.rst b/doc/index.rst index 530eddb..d18b99f 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -54,6 +54,14 @@ Functions functions/index +Update +-------------------------------------------------------------------------------- + +.. toctree:: + :maxdepth: 2 + + update + Tutorials -------------------------------------------------------------------------------- diff --git a/doc/update.rst b/doc/update.rst new file mode 100644 index 0000000..c2abf4e --- /dev/null +++ b/doc/update.rst @@ -0,0 +1,48 @@ +.. _update: + +****************************************************************************** +Update +****************************************************************************** + +pgPointcloud extension +------------------------------------------------------------------------------ + +Once a new version of pgPointcloud installed, you may want to update your +databases where the extension is already in use. The first thing to compare is +the version currently used with versions actually available on your system: + +.. code-block:: bash + + mydatabase=# SELECT pc_version(); + pc_version + ------------ + 1.1.1 + (1 row) + + mydatabase=# SELECT version FROM pg_available_extension_versions WHERE name ='pointcloud'; + version + ----------- + 1.1.1 + 1.2.1 + (2 rows) + + +Then you can update to the latest version with ``ALTER EXTENSION pointcloud +UPDATE`` or target a specific version: + +.. code-block:: bash + + mydatabase=# ALTER EXTENSION pointcloud UPDATE TO '1.2.1'; + ALTER EXTENSION + mydatabase=# SELECT pc_version(); + pc_version + ------------ + 1.2.1 + (1 row) + + +.. warning:: + + The GHT compression has been removed in the 1.2.0 version. Unfortunately, + you have to remove the compression before updating the extension from 1.1.x + to a higher version. From 6435405a583b21d50d79b044b0bb30a34aaed996 Mon Sep 17 00:00:00 2001 From: Blottiere Paul Date: Wed, 9 Feb 2022 22:47:58 +0100 Subject: [PATCH 2/4] Add compression chapter --- doc/tutorials/compression.rst | 83 +++++++++++++++++++++++++++++++++++ doc/tutorials/index.rst | 1 + doc/tutorials/storing.rst | 56 +++++++++++------------ 3 files changed, 112 insertions(+), 28 deletions(-) create mode 100644 doc/tutorials/compression.rst diff --git a/doc/tutorials/compression.rst b/doc/tutorials/compression.rst new file mode 100644 index 0000000..2ddc264 --- /dev/null +++ b/doc/tutorials/compression.rst @@ -0,0 +1,83 @@ +****************************************************************************** +Schema and compression +****************************************************************************** + +This tutorial is an introduction for investigating XML schemas and playing with +compression of patches. + +------------------------------------------------------------------------------ +Compression type +------------------------------------------------------------------------------ + +The compression of a patch may be retrieved through its XML schema but it's +also stored in the patch itself. Of course, both needs to be consistent so +updating an existing schema is hardly discouraged and may lead to errors. + +In the first case, the XML schema needs to be parsed with ``xpath`` function to +retrieve the ``pc:metadata`` tag of a specific patch: + +.. code-block:: sql + + pointclouds=# WITH tmp AS ( + SELECT pc_pcid(pa) + AS _pcid + FROM airport + LIMIT 1 + ) + SELECT unnest( + xpath( + '/pc:PointCloudSchema/pc:metadata/Metadata/text()', + schema::xml, + array[ + ['pc', 'http://pointcloud.org/schemas/PC/'], + ['xsi', 'http://www.w3.org/2001/XMLSchema-instance'] + ] + ) + ) + AS "compression" + FROM tmp,pointcloud_formats + WHERE pcid=tmp._pcid; + + metadata + --------------- + dimensional + (1 row) + + +A much easier way to retrieve the compression type is to take a look to the +JSON summary of the patch: + +.. code-block:: sql + + pointclouds=# SELECT pc_summary(pa)::json->'compr' + AS "compression" + FROM airport + LIMIT 1; + + compression + --------------- + "dimensional" + (1 row) + + +.. code-block:: sql + + pointclouds=# INSERT INTO pointcloud_formats (pcid, srid, schema) + VALUES ( + 34, + 4326, + ( + SELECT Regexp_replace(schema, 'dimensional', 'none', 'g') + FROM pointcloud_formats + WHERE  pcid = 3 + ) + );  + + + +$ create table pa_dim_to_none as select pc_transform(pa, 35) as pa from pa_dim; + + +------------------------------------------------------------------------------ +Update compression +------------------------------------------------------------------------------ diff --git a/doc/tutorials/index.rst b/doc/tutorials/index.rst index fd30446..18b56bb 100644 --- a/doc/tutorials/index.rst +++ b/doc/tutorials/index.rst @@ -11,3 +11,4 @@ pgPointcloud. :maxdepth: 1 storing + compression diff --git a/doc/tutorials/storing.rst b/doc/tutorials/storing.rst index 267a377..26dd8dd 100644 --- a/doc/tutorials/storing.rst +++ b/doc/tutorials/storing.rst @@ -11,7 +11,7 @@ Start Docker container First we download the latest tag of the pgPoincloud Docker image: -.. code-block:: +.. code-block:: bash $ docker pull pgpointcloud/pointcloud @@ -25,13 +25,13 @@ For a basic usage, we have to define two environment variables: Then we can start a new container: -.. code-block:: +.. code-block:: bash $ docker run --name pgpointcloud -e POSTGRES_DB=pointclouds -e POSTGRES_PASSWORD=mysecretpassword -d pgpointcloud/pointcloud Extensions are automatically created in the new database named ``pointclouds``: -.. code-block:: +.. code-block:: bash $ docker exec -it pgpointcloud psql -U postgres -d pointclouds -c "\dx" List of installed extensions @@ -55,9 +55,9 @@ Run PDAL pipeline For the need of the tutorial, we can download sample data from the `PDAL`_ organization: -.. code-block:: +.. code-block:: bash - $ wget https://github.com/PDAL/data/raw/master/liblas/LAS12_Sample_withRGB_Quick_Terrain_Modeler_fixed.laz /tmp + $ wget https://github.com/PDAL/data/raw/master/liblas/LAS12_Sample_withRGB_Quick_Terrain_Modeler_fixed.laz -P /tmp Thanks to the ``pdal info`` command, we can obtain some information on the dataset: @@ -68,7 +68,7 @@ To configure the json PDAL pipeline, we need to set up the ``connection`` parameter for the ``pgpointcloud`` writer. To do that, the Docker container IP adress on which the PostgreSQL database is running is necessary: -.. code-block:: +.. code-block:: bash $ docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgpointcloud 172.17.0.2 @@ -76,27 +76,27 @@ adress on which the PostgreSQL database is running is necessary: So the ``pipeline.json`` file looks like: -.. code-block:: +.. code-block:: json { - "pipeline":[ - { - "type":"readers.las", - "filename":"/tmp/LAS12_Sample_withRGB_Quick_Terrain_Modeler_fixed.laz" - }, - { - "type":"filters.chipper", - "capacity":"400" - }, - { - "type":"writers.pgpointcloud", - "connection":"host='172.17.0.2' dbname='pointclouds' user='postgres' password='mysecretpassword' port='5432'", - "table":"airport", - "compression":"none", - "srid":"32616" - } - ] -} + "pipeline":[ + { + "type":"readers.las", + "filename":"/tmp/LAS12_Sample_withRGB_Quick_Terrain_Modeler_fixed.laz" + }, + { + "type":"filters.chipper", + "capacity":"400" + }, + { + "type":"writers.pgpointcloud", + "connection":"host='172.17.0.2' dbname='pointclouds' user='postgres' password='mysecretpassword' port='5432'", + "table":"airport", + "compression":"dimensional", + "srid":"32616" + } + ] + } The PDAL pipeline can finally be execute with ``pdal pipeline pipeline.json`` and an ``airport`` table is created. @@ -111,7 +111,7 @@ Configure connection service file To facilitate the access to the database hosted on the Docker container, we can configure the PostgreSQL connection service file: -.. code-block:: +.. code-block:: bash [pgpointcloud] host=172.17.0.2 @@ -122,13 +122,13 @@ configure the PostgreSQL connection service file: Then we can explore the content of the new ``airport`` table: -.. code-block:: +.. code-block:: bash $ psql service=pgpointcloud psql (12.3) Type "help" for help. - pointclouds=# select count(*) from airport; + pointclouds=# SELECT COUNT(*) FROM airport; count ------- 9529 From 862748d5859c10ebaadcb9cf9e99daa08faa062a Mon Sep 17 00:00:00 2001 From: Blottiere Paul Date: Wed, 9 Feb 2022 23:39:45 +0100 Subject: [PATCH 3/4] Add transform chapter --- doc/tutorials/compression.rst | 111 +++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 48 deletions(-) diff --git a/doc/tutorials/compression.rst b/doc/tutorials/compression.rst index 2ddc264..7b07801 100644 --- a/doc/tutorials/compression.rst +++ b/doc/tutorials/compression.rst @@ -18,30 +18,25 @@ retrieve the ``pc:metadata`` tag of a specific patch: .. code-block:: sql - pointclouds=# WITH tmp AS ( - SELECT pc_pcid(pa) - AS _pcid - FROM airport - LIMIT 1 - ) - SELECT unnest( - xpath( - '/pc:PointCloudSchema/pc:metadata/Metadata/text()', - schema::xml, - array[ - ['pc', 'http://pointcloud.org/schemas/PC/'], - ['xsi', 'http://www.w3.org/2001/XMLSchema-instance'] - ] - ) - ) - AS "compression" - FROM tmp,pointcloud_formats - WHERE pcid=tmp._pcid; - - metadata - --------------- - dimensional - (1 row) + WITH tmp AS ( + SELECT pc_pcid(pa) + AS _pcid + FROM airport + LIMIT 1 + ) + SELECT unnest( + xpath( + '/pc:PointCloudSchema/pc:metadata/Metadata/text()', + schema::xml, + array[ + ['pc', 'http://pointcloud.org/schemas/PC/'], + ['xsi', 'http://www.w3.org/2001/XMLSchema-instance'] + ] + ) + ) + FROM tmp,pointcloud_formats + WHERE pcid=tmp._pcid; + --> dimensional A much easier way to retrieve the compression type is to take a look to the @@ -49,35 +44,55 @@ JSON summary of the patch: .. code-block:: sql - pointclouds=# SELECT pc_summary(pa)::json->'compr' - AS "compression" - FROM airport - LIMIT 1; + SELECT pc_summary(pa)::json->'compr' FROM airport LIMIT 1; + --> dimensional - compression - --------------- - "dimensional" - (1 row) +------------------------------------------------------------------------------ +Create a new schema +------------------------------------------------------------------------------ +A schema is just a XML document and may be manually inserted into the +``pointcloud_formats`` table directly from a file. We can also duplicate an +existing schema and tweak some parameters. + +For example, we can create a new schema without compression and based on the +schema ``pcid=1``: .. code-block:: sql - pointclouds=# INSERT INTO pointcloud_formats (pcid, srid, schema) - VALUES ( - 34, - 4326, - ( - SELECT Regexp_replace(schema, 'dimensional', 'none', 'g') - FROM pointcloud_formats - WHERE  pcid = 3 - ) - );  - - - -$ create table pa_dim_to_none as select pc_transform(pa, 35) as pa from pa_dim; - + INSERT INTO pointcloud_formats (pcid, srid, schema) + SELECT 2, srid, regexp_replace(schema, 'dimensional', 'none', 'g') + FROM pointcloud_formats + WHERE pcid=1; ------------------------------------------------------------------------------ -Update compression +Transform a patch ------------------------------------------------------------------------------ + +Thanks to the ``pc_transform`` function, we can transform the underlying data +of a patch to match a specific schema. So if we want to remove the dimensional +compression from an existing patch, we can use the schema with ``pcid=2`` +previously created. + +In this particular case, the transformed patch doesn't have compression +anymore: + +.. code-block:: sql + + SELECT pc_summary(pc_transform(pa, 2))::json->'compr' FROM airport LIMIT 1; + --> none + +So a new table of uncompressed patches may be easily created: + +.. code-block:: sql + + CREATE TABLE airport_uncompressed AS SELECT pc_transform(pa, 2) AS pa FROM airport; + + SELECT pc_summary(pa)::json->'compr' FROM airport_uncompressed LIMIT 1; + --> none + + SELECT pc_astext(pc_patchavg(pa)) FROM airport LIMIT 1; + --> {"pcid":1,"pt":[65535,0,0,0,0,0,0,0,0,30744,25999,17189,728265,4.67644e+06,299.08]} + + SELECT pc_astext(pc_patchavg(pa)) FROM airport_uncompressed LIMIT 1; + --> {"pcid":2,"pt":[65535,0,0,0,0,0,0,0,0,30744,25999,17189,728265,4.67644e+06,299.08]} From 360267d0bb44ba0f98faa8a24e719ea3d57aeacc Mon Sep 17 00:00:00 2001 From: Blottiere Paul Date: Wed, 9 Feb 2022 23:46:46 +0100 Subject: [PATCH 4/4] Link to the corresponding tutorial page --- doc/update.rst | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/doc/update.rst b/doc/update.rst index c2abf4e..50b1260 100644 --- a/doc/update.rst +++ b/doc/update.rst @@ -11,38 +11,32 @@ Once a new version of pgPointcloud installed, you may want to update your databases where the extension is already in use. The first thing to compare is the version currently used with versions actually available on your system: -.. code-block:: bash +.. code-block:: sql - mydatabase=# SELECT pc_version(); - pc_version - ------------ - 1.1.1 - (1 row) + SELECT pc_version(); + --> 1.1.1 - mydatabase=# SELECT version FROM pg_available_extension_versions WHERE name ='pointcloud'; - version - ----------- - 1.1.1 - 1.2.1 - (2 rows) + SELECT version FROM pg_available_extension_versions WHERE name ='pointcloud'; + --> 1.1.1 + --> 1.2.1 Then you can update to the latest version with ``ALTER EXTENSION pointcloud UPDATE`` or target a specific version: -.. code-block:: bash +.. code-block:: sql - mydatabase=# ALTER EXTENSION pointcloud UPDATE TO '1.2.1'; - ALTER EXTENSION - mydatabase=# SELECT pc_version(); - pc_version - ------------ - 1.2.1 - (1 row) + ALTER EXTENSION pointcloud UPDATE TO '1.2.1'; + + SELECT pc_version(); + --> 1.2.1 .. warning:: The GHT compression has been removed in the 1.2.0 version. Unfortunately, - you have to remove the compression before updating the extension from 1.1.x - to a higher version. + you have to remove the compression on your tables before updating the + extension from 1.1.x to a higher version. Some information are available in + the `Schema and compression`_ tutorial. + +.. _`Schema and compression`: /https://pgpointcloud.github.io/pointcloud/tutorials/compression.html#schema-and-compression