From 39ff306c55a60a0081985e6488ba027261a26d34 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Mon, 18 Apr 2022 16:42:49 +0200 Subject: [PATCH] Switch to geozero as mvt parser --- Cargo.lock | 3 +- Cargo.toml | 5 +- src/io/geometry_index.rs | 2 +- src/io/mod.rs | 7 +- src/io/shared_thread_state.rs | 234 ++++++++++++--------------- src/map_state.rs | 2 +- src/render/render_state.rs | 2 +- src/tessellation/layer.rs | 2 - src/tessellation/mod.rs | 2 +- src/tessellation/zero_tessellator.rs | 195 ++++++++++++++++++++++ 10 files changed, 309 insertions(+), 145 deletions(-) create mode 100644 src/tessellation/zero_tessellator.rs diff --git a/Cargo.lock b/Cargo.lock index 98f01593..d10664c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1219,7 +1219,7 @@ dependencies = [ [[package]] name = "geozero" version = "0.8.0" -source = "git+https://github.com/georust/geozero?rev=24973aa#24973aa93ca2b8783d0d89c2652de11fdfb569ae" +source = "git+https://github.com/georust/geozero?rev=373b731#373b731e3c3deb367ad13eb3babdbcee4b7f0010" dependencies = [ "geo-types", "prost", @@ -1785,7 +1785,6 @@ dependencies = [ "tracing-tracy", "tracing-wasm", "tracy-client", - "vector-tile", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", diff --git a/Cargo.toml b/Cargo.toml index c2a157d8..c2b6d724 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,9 +76,6 @@ raw-window-handle = "0.4" tracing = { version = "0.1" } tracing-subscriber = { version = "0.3", optional = true } - -# Vector riles -vector-tile = { path = "./libs/vector_tile" } style-spec = { path = "./libs/style_spec" } tilejson-spec = { path = "./libs/tilejson_spec" } @@ -88,7 +85,7 @@ geo = { version = "0.19" } geo-types = { version = "0.7", features = ["use-rstar_0_9"] } rstar = { version = "0.9" } prost = "0.9" -geozero = { git = "https://github.com/georust/geozero", rev = "24973aa", default-features = false, features = ["with-mvt", "with-geo"]} +geozero = { git = "https://github.com/georust/geozero", rev = "373b731", default-features = false, features = ["with-mvt", "with-geo"]} # Rendering wgpu = { version = "0.12" } diff --git a/src/io/geometry_index.rs b/src/io/geometry_index.rs index f91b328d..d0ac85a4 100644 --- a/src/io/geometry_index.rs +++ b/src/io/geometry_index.rs @@ -23,7 +23,7 @@ impl GeometryIndex { } } - pub fn index_tile(&mut self, coords: &&WorldTileCoords, tile_index: TileIndex) { + pub fn index_tile(&mut self, coords: &WorldTileCoords, tile_index: TileIndex) { coords .build_quad_key() .and_then(|key| self.index.insert(key, tile_index)); diff --git a/src/io/mod.rs b/src/io/mod.rs index 1d4d20dc..fd215494 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -5,11 +5,10 @@ use crate::coords::WorldTileCoords; use crate::render::ShaderVertex; use crate::tessellation::{IndexDataType, OverAlignedVertexBuffer}; +use geozero::mvt::tile; use std::collections::HashSet; use std::fmt; -use vector_tile::tile::Layer; - pub mod scheduler; pub mod source_client; pub mod static_tile_fetcher; @@ -62,7 +61,7 @@ pub enum LayerTessellateMessage { buffer: OverAlignedVertexBuffer, /// Holds for each feature the count of indices feature_indices: Vec, - layer_data: Layer, + layer_data: tile::Layer, }, } @@ -83,7 +82,7 @@ impl LayerTessellateMessage { pub fn layer_name(&self) -> &str { match self { LayerTessellateMessage::UnavailableLayer { layer_name, .. } => layer_name.as_str(), - LayerTessellateMessage::TessellatedLayer { layer_data, .. } => layer_data.name(), + LayerTessellateMessage::TessellatedLayer { layer_data, .. } => &layer_data.name, } } } diff --git a/src/io/shared_thread_state.rs b/src/io/shared_thread_state.rs index 15faff19..5c7b7f99 100644 --- a/src/io/shared_thread_state.rs +++ b/src/io/shared_thread_state.rs @@ -1,4 +1,4 @@ -use crate::coords::{WorldCoords, Zoom}; +use crate::coords::{TileCoords, WorldCoords, WorldTileCoords, Zoom}; use crate::error::Error; use crate::io::geometry_index::{GeometryIndex, IndexProcessor, IndexedGeometry, TileIndex}; use crate::io::tile_request_state::TileRequestState; @@ -7,8 +7,14 @@ use crate::io::{ TileTessellateMessage, }; use crate::tessellation::Tessellated; +use std::collections::HashSet; +use crate::tessellation::zero_tessellator::ZeroTessellator; +use geozero::mvt::tile; +use geozero::GeozeroDatasource; +use prost::Message; use std::sync::{mpsc, Arc, Mutex}; +use tracing_subscriber::fmt::layer; #[derive(Clone)] pub struct SharedThreadState { @@ -28,54 +34,115 @@ impl SharedThreadState { #[tracing::instrument(skip_all)] pub fn process_tile(&self, request_id: TileRequestID, data: Box<[u8]>) -> Result<(), Error> { if let Some(tile_request) = self.get_tile_request(request_id) { - let tile_result = TileFetchResult::Tile { - coords: tile_request.coords, - data, - }; + let coords = tile_request.coords; - self.tessellate_layers_with_request(&tile_result, &tile_request, request_id)?; - self.index_geometry(&tile_result); - } + tracing::info!("parsing tile {} with {}bytes", &coords, data.len()); - Ok(()) - } + let _span_ = tracing::span!(tracing::Level::TRACE, "parse_tile_bytes").entered(); - pub fn tile_unavailable(&self, request_id: TileRequestID) -> Result<(), Error> { - if let Some(tile_request) = self.get_tile_request(request_id) { - let tile_result = TileFetchResult::Unavailable { - coords: tile_request.coords, - }; - self.tessellate_layers_with_request(&tile_result, &tile_request, request_id)?; - } + let mut tile = geozero::mvt::Tile::decode(data.as_ref()).expect("failed to load tile"); - Ok(()) - } + let mut index = IndexProcessor::new(); - #[tracing::instrument(skip_all)] - fn index_geometry(&self, tile_result: &TileFetchResult) { - match tile_result { - TileFetchResult::Tile { data, coords } => { - use geozero::GeozeroDatasource; - use prost::Message; - - let tile = geozero::mvt::Tile::decode(data.as_ref()).unwrap(); - - let mut processor = IndexProcessor::new(); - for mut layer in tile.layers { - layer.process(&mut processor).unwrap(); + for mut layer in &mut tile.layers { + let cloned_layer = layer.clone(); + let layer_name: &str = &cloned_layer.name; + if !tile_request.layers.contains(layer_name) { + continue; } - if let Ok(mut geometry_index) = self.geometry_index.lock() { - geometry_index.index_tile( - &coords, - TileIndex::Linear { - list: processor.get_geometries(), + tracing::info!("layer {} at {} ready", layer_name, &coords); + + let mut tessellator = ZeroTessellator::default(); + if let Err(e) = layer.process(&mut tessellator) { + self.message_sender.send(TessellateMessage::Layer( + LayerTessellateMessage::UnavailableLayer { + coords, + layer_name: layer_name.to_owned(), }, - ) + ))?; + + tracing::error!( + "layer {} at {} tesselation failed {:?}", + layer_name, + &coords, + e + ); + } else { + self.message_sender.send(TessellateMessage::Layer( + LayerTessellateMessage::TessellatedLayer { + coords, + buffer: tessellator.buffer.into(), + feature_indices: tessellator.feature_indices, + layer_data: cloned_layer, + }, + ))?; } + + // TODO + // layer.process(&mut index).unwrap(); + } + + let available_layers: HashSet<_> = tile + .layers + .iter() + .map(|layer| layer.name.clone()) + .collect::>(); + + for missing_layer in tile_request.layers.difference(&available_layers) { + self.message_sender.send(TessellateMessage::Layer( + LayerTessellateMessage::UnavailableLayer { + coords, + layer_name: missing_layer.to_owned(), + }, + ))?; + + tracing::info!( + "requested layer {} at {} not found in tile", + missing_layer, + &coords + ); + } + + tracing::info!("tile tessellated at {} finished", &tile_request.coords); + + self.message_sender + .send(TessellateMessage::Tile(TileTessellateMessage { + request_id, + coords: tile_request.coords, + }))?; + + if let Ok(mut geometry_index) = self.geometry_index.lock() { + geometry_index.index_tile( + &coords, + TileIndex::Linear { + list: index.get_geometries(), + }, + ) } - _ => {} } + + Ok(()) + } + + pub fn tile_unavailable( + &self, + coords: &WorldTileCoords, + request_id: TileRequestID, + ) -> Result<(), Error> { + if let Some(tile_request) = self.get_tile_request(request_id) { + for to_load in &tile_request.layers { + tracing::warn!("layer {} at {} unavailable", to_load, coords); + self.message_sender.send(TessellateMessage::Layer( + LayerTessellateMessage::UnavailableLayer { + coords: tile_request.coords, + layer_name: to_load.to_string(), + }, + ))?; + } + } + + Ok(()) } #[tracing::instrument(skip_all)] @@ -99,95 +166,4 @@ impl SharedThreadState { unimplemented!() } } - - #[tracing::instrument(skip_all)] - fn tessellate_layers_with_request( - &self, - tile_result: &TileFetchResult, - tile_request: &TileRequest, - request_id: TileRequestID, - ) -> Result<(), Error> { - match tile_result { - TileFetchResult::Unavailable { coords } => { - for to_load in &tile_request.layers { - tracing::warn!("layer {} at {} unavailable", to_load, &coords); - self.message_sender.send(TessellateMessage::Layer( - LayerTessellateMessage::UnavailableLayer { - coords: *coords, - layer_name: to_load.to_string(), - }, - ))?; - } - } - TileFetchResult::Tile { data, coords } => { - tracing::info!("parsing tile {} with {}bytes", &coords, data.len()); - - let tile = { - let _span_ = - tracing::span!(tracing::Level::TRACE, "parse_tile_bytes").entered(); - vector_tile::parse_tile_bytes(data).expect("failed to load tile") - }; - - for to_load in &tile_request.layers { - if let Some(layer) = tile - .layers() - .iter() - .find(|layer| to_load.as_str() == layer.name()) - { - match layer.tessellate() { - Ok((buffer, feature_indices)) => { - tracing::info!("layer {} at {} ready", to_load, &coords); - self.message_sender.send(TessellateMessage::Layer( - LayerTessellateMessage::TessellatedLayer { - coords: *coords, - buffer: buffer.into(), - feature_indices, - layer_data: layer.clone(), - }, - ))?; - } - Err(e) => { - self.message_sender.send(TessellateMessage::Layer( - LayerTessellateMessage::UnavailableLayer { - coords: *coords, - layer_name: to_load.to_string(), - }, - ))?; - - tracing::error!( - "layer {} at {} tesselation failed {:?}", - to_load, - &coords, - e - ); - } - } - } else { - self.message_sender.send(TessellateMessage::Layer( - LayerTessellateMessage::UnavailableLayer { - coords: *coords, - layer_name: to_load.to_string(), - }, - ))?; - - tracing::info!( - "requested layer {} at {} not found in tile", - to_load, - &coords - ); - } - } - } - } - - tracing::info!("tile at {} finished", &tile_request.coords); - - self.message_sender - .send(TessellateMessage::Tile(TileTessellateMessage { - request_id, - coords: tile_request.coords, - }))?; - - Ok(()) - } } diff --git a/src/map_state.rs b/src/map_state.rs index e837228c..2912922e 100644 --- a/src/map_state.rs +++ b/src/map_state.rs @@ -235,7 +235,7 @@ impl MapState { .unwrap(), Err(e) => { log::error!("{:?}", &e); - state.tile_unavailable(request_id).unwrap() + state.tile_unavailable(&coords, request_id).unwrap() } } }, diff --git a/src/render/render_state.rs b/src/render/render_state.rs index 24283ff1..eeec912b 100644 --- a/src/render/render_state.rs +++ b/src/render/render_state.rs @@ -461,7 +461,7 @@ impl RenderState { let guard = allocate_feature_metadata.enter(); let feature_metadata = layer_data - .features() + .features .iter() .enumerate() .flat_map(|(i, _feature)| { diff --git a/src/tessellation/layer.rs b/src/tessellation/layer.rs index 9ab11fab..ae1c951f 100644 --- a/src/tessellation/layer.rs +++ b/src/tessellation/layer.rs @@ -11,8 +11,6 @@ use lyon::tessellation::{ }; use crate::error::Error; -use vector_tile::geometry::{Command, Geometry}; -use vector_tile::tile::Layer; use crate::render::ShaderVertex; use crate::tessellation::{Tessellated, VertexConstructor, DEFAULT_TOLERANCE}; diff --git a/src/tessellation/mod.rs b/src/tessellation/mod.rs index ff68a923..a1ba1f2b 100644 --- a/src/tessellation/mod.rs +++ b/src/tessellation/mod.rs @@ -11,7 +11,7 @@ use lyon::tessellation::{ use crate::error::Error; use wgpu::BufferAddress; -mod layer; +pub mod zero_tessellator; const DEFAULT_TOLERANCE: f32 = 0.02; diff --git a/src/tessellation/zero_tessellator.rs b/src/tessellation/zero_tessellator.rs new file mode 100644 index 00000000..9a947200 --- /dev/null +++ b/src/tessellation/zero_tessellator.rs @@ -0,0 +1,195 @@ +use bytemuck::Pod; +use geozero::{FeatureProcessor, GeomProcessor, PropertyProcessor}; +use lyon::geom; +use lyon::geom::point; +use lyon::lyon_tessellation::VertexBuffers; +use lyon::path::path::Builder; +use lyon::path::Path; +use lyon::tessellation::geometry_builder::MaxIndex; +use lyon::tessellation::{ + BuffersBuilder, FillOptions, FillRule, FillTessellator, StrokeOptions, StrokeTessellator, +}; +use std::cell::RefCell; + +use crate::render::ShaderVertex; +use crate::tessellation::{VertexConstructor, DEFAULT_TOLERANCE}; + +type GeoResult = geozero::error::Result; + +pub struct ZeroTessellator + MaxIndex> { + path_builder: RefCell, + path_open: bool, + is_point: bool, + + pub buffer: VertexBuffers, + + pub feature_indices: Vec, + current_index: usize, +} + +impl + MaxIndex> Default + for ZeroTessellator +{ + fn default() -> Self { + Self { + path_builder: RefCell::new(Path::builder()), + buffer: VertexBuffers::new(), + feature_indices: Vec::new(), + current_index: 0, + path_open: false, + is_point: false, + } + } +} + +impl + MaxIndex> ZeroTessellator { + fn update_feature_indices(&mut self) { + let next_index = self.buffer.indices.len(); + let indices = (next_index - self.current_index) as u32; + self.feature_indices.push(indices); + self.current_index = next_index; + } + + fn tessellate_strokes(&mut self) { + let path_builder = self.path_builder.replace(Path::builder()); + + StrokeTessellator::new() + .tessellate_path( + &path_builder.build(), + &StrokeOptions::tolerance(DEFAULT_TOLERANCE), + &mut BuffersBuilder::new(&mut self.buffer, VertexConstructor {}), + ) + .unwrap(); + } + + fn end(&mut self, close: bool) { + if self.path_open { + self.path_builder.borrow_mut().end(close); + self.path_open = false; + } + } + + fn tessellate_fill(&mut self) { + let path_builder = self.path_builder.replace(Path::builder()); + + FillTessellator::new() + .tessellate_path( + &path_builder.build(), + &FillOptions::tolerance(DEFAULT_TOLERANCE).with_fill_rule(FillRule::NonZero), + &mut BuffersBuilder::new(&mut self.buffer, VertexConstructor {}), + ) + .unwrap(); + } +} + +impl + MaxIndex> GeomProcessor + for ZeroTessellator +{ + fn xy(&mut self, x: f64, y: f64, idx: usize) -> GeoResult<()> { + // log::info!("xy"); + + if self.is_point { + // log::info!("point"); + } else if !self.path_open { + self.path_builder + .borrow_mut() + .begin(geom::point(x as f32, y as f32)); + self.path_open = true; + } else { + self.path_builder + .borrow_mut() + .line_to(geom::point(x as f32, y as f32)); + } + Ok(()) + } + + fn point_begin(&mut self, _idx: usize) -> GeoResult<()> { + // log::info!("point_begin"); + self.is_point = true; + Ok(()) + } + + fn point_end(&mut self, _idx: usize) -> GeoResult<()> { + // log::info!("point_end"); + self.is_point = false; + Ok(()) + } + + fn multipoint_begin(&mut self, _size: usize, _idx: usize) -> GeoResult<()> { + // log::info!("multipoint_begin"); + Ok(()) + } + + fn multipoint_end(&mut self, _idx: usize) -> GeoResult<()> { + // log::info!("multipoint_end"); + Ok(()) + } + + fn linestring_begin(&mut self, _tagged: bool, _size: usize, _idx: usize) -> GeoResult<()> { + // log::info!("linestring_begin"); + Ok(()) + } + + fn linestring_end(&mut self, tagged: bool, _idx: usize) -> GeoResult<()> { + // log::info!("linestring_end"); + + self.end(false); + + if tagged { + self.tessellate_strokes(); + } + Ok(()) + } + + fn multilinestring_begin(&mut self, _size: usize, _idx: usize) -> GeoResult<()> { + // log::info!("multilinestring_begin"); + Ok(()) + } + + fn multilinestring_end(&mut self, _idx: usize) -> GeoResult<()> { + // log::info!("multilinestring_end"); + self.tessellate_strokes(); + Ok(()) + } + + fn polygon_begin(&mut self, _tagged: bool, _size: usize, _idx: usize) -> GeoResult<()> { + // log::info!("polygon_begin"); + Ok(()) + } + + fn polygon_end(&mut self, tagged: bool, _idx: usize) -> GeoResult<()> { + // log::info!("polygon_end"); + + self.end(true); + if tagged { + self.tessellate_fill(); + } + Ok(()) + } + + fn multipolygon_begin(&mut self, size: usize, idx: usize) -> GeoResult<()> { + // log::info!("multipolygon_begin"); + Ok(()) + } + + fn multipolygon_end(&mut self, _idx: usize) -> GeoResult<()> { + // log::info!("multipolygon_end"); + + self.tessellate_fill(); + Ok(()) + } +} + +impl + MaxIndex> PropertyProcessor + for ZeroTessellator +{ +} + +impl + MaxIndex> FeatureProcessor + for ZeroTessellator +{ + fn feature_end(&mut self, _idx: u64) -> geozero::error::Result<()> { + self.update_feature_indices(); + Ok(()) + } +}