From 2040887f39395b59ffdd545d37de5094de9566ce Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Thu, 10 Mar 2022 16:20:02 +0100 Subject: [PATCH] Support TMS as well as XYZ addressing --- src/coords.rs | 72 ++++++++++++---------- src/io/tile_cache.rs | 47 +++++++------- src/io/workflow.rs | 120 +++++++++++++++++++++++++----------- src/platform/android/mod.rs | 4 +- src/platform/apple/mod.rs | 4 +- src/platform/generic.rs | 5 +- src/platform/web/mod.rs | 4 +- src/render/buffer_pool.rs | 10 +-- src/render/render_state.rs | 37 +++++------ 9 files changed, 178 insertions(+), 125 deletions(-) diff --git a/src/coords.rs b/src/coords.rs index b2c9f17a..3b826d70 100644 --- a/src/coords.rs +++ b/src/coords.rs @@ -1,10 +1,12 @@ //! File which exposes all kinds of coordinates used throughout mapr +use crate::io::tile_cache::TileCache; use crate::util::math::{div_away, div_floor, Aabb2}; use cgmath::num_traits::Pow; use cgmath::{Matrix4, Point3, SquareMatrix, Vector3}; use std::fmt; use std::ops::Mul; +use style_spec::source::TileAdressingScheme; pub const EXTENT_UINT: u32 = 4096; pub const EXTENT_SINT: i32 = EXTENT_UINT as i32; @@ -16,7 +18,7 @@ pub const EXTENT: f64 = EXTENT_UINT as f64; /// # Coordinate System Origin /// /// For Web Mercator the origin of the coordinate system is in the upper-left corner. -#[derive(Clone, Copy, Debug, Hash, std::cmp::Eq, std::cmp::PartialEq, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] pub struct TileCoords { pub x: u32, pub y: u32, @@ -24,13 +26,20 @@ pub struct TileCoords { } impl TileCoords { - /// Transforms the tile coordinates as defined by the tile grid into a representation which is + /// Transforms the tile coordinates as defined by the tile grid addressing scheme into a representation which is /// used in the 3d-world. - pub fn into_world_tile(self) -> WorldTileCoords { - WorldTileCoords { - x: self.x as i32, - y: self.y as i32, - z: self.z, + pub fn into_world_tile(self, scheme: &TileAdressingScheme) -> WorldTileCoords { + match scheme { + TileAdressingScheme::XYZ => WorldTileCoords { + x: self.x as i32, + y: self.y as i32, + z: self.z, + }, + TileAdressingScheme::TMS => WorldTileCoords { + x: self.x as i32, + y: (2u32.pow(self.z as u32) - 1 - self.y) as i32, + z: self.z, + }, } } } @@ -51,24 +60,6 @@ impl From<(u32, u32, u8)> for TileCoords { } } -impl From for TileCoords { - fn from(world_coords: WorldTileCoords) -> Self { - let mut tile_x = world_coords.x; - let mut tile_y = world_coords.y; - if tile_x < 0 { - tile_x = 0; - } - if tile_y < 0 { - tile_y = 0; - } - TileCoords { - x: tile_x as u32, - y: tile_y as u32, - z: world_coords.z, - } - } -} - /// Every tile has tile coordinates. Every tile coordinate can be mapped to a coordinate within /// the world. This provides the freedom to map from [TMS](https://wiki.openstreetmap.org/wiki/TMS) /// to [Slippy_map_tilenames](https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames). @@ -76,7 +67,7 @@ impl From for TileCoords { /// # Coordinate System Origin /// /// The origin of the coordinate system is in the upper-left corner. -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct WorldTileCoords { pub x: i32, pub y: i32, @@ -84,6 +75,21 @@ pub struct WorldTileCoords { } impl WorldTileCoords { + pub fn into_tile(self, scheme: &TileAdressingScheme) -> TileCoords { + match scheme { + TileAdressingScheme::XYZ => TileCoords { + x: self.x as u32, + y: self.y as u32, + z: self.z, + }, + TileAdressingScheme::TMS => TileCoords { + x: self.x as u32, + y: 2u32.pow(self.z as u32) - 1 - self.y as u32, + z: self.z, + }, + } + } + pub fn transform_for_zoom(&self, zoom: f64) -> Matrix4 { /* For tile.z = zoom: @@ -252,8 +258,8 @@ impl From> for WorldCoords { } pub struct ViewRegion { - min_tile: TileCoords, - max_tile: TileCoords, + min_tile: WorldTileCoords, + max_tile: WorldTileCoords, z: u8, } @@ -261,22 +267,20 @@ impl ViewRegion { pub fn new(view_region: Aabb2, zoom: f64, z: u8) -> Self { let min_world: WorldCoords = WorldCoords::at_ground(view_region.min.x, view_region.min.y); let min_world_tile: WorldTileCoords = min_world.into_world_tile(z, zoom); - let min_tile: TileCoords = min_world_tile.into(); let max_world: WorldCoords = WorldCoords::at_ground(view_region.max.x, view_region.max.y); let max_world_tile: WorldTileCoords = max_world.into_world_tile(z, zoom); - let max_tile: TileCoords = max_world_tile.into(); Self { - min_tile, - max_tile, + min_tile: min_world_tile, + max_tile: max_world_tile, z, } } - pub fn iter(&self) -> impl Iterator + '_ { + pub fn iter(&self) -> impl Iterator + '_ { (self.min_tile.x..self.max_tile.x + 1).flat_map(move |x| { (self.min_tile.y..self.max_tile.y + 1).map(move |y| { - let tile_coord: TileCoords = (x, y, self.z as u8).into(); + let tile_coord: WorldTileCoords = (x, y, self.z as u8).into(); tile_coord }) }) diff --git a/src/io/tile_cache.rs b/src/io/tile_cache.rs index 9d96dfc0..615ad525 100644 --- a/src/io/tile_cache.rs +++ b/src/io/tile_cache.rs @@ -1,11 +1,11 @@ -use crate::coords::TileCoords; +use crate::coords::{TileCoords, WorldTileCoords}; use crate::io::workflow::LayerResult; -use std::collections::{btree_map, BTreeMap}; +use std::collections::{btree_map, BTreeMap, HashSet}; use std::sync::{Arc, Mutex}; #[derive(Clone)] pub struct TileCache { - store: Arc>>>, + store: Arc>>>, } impl TileCache { @@ -15,9 +15,9 @@ impl TileCache { } } - pub(crate) fn push(&self, result: LayerResult) -> bool { + pub fn push(&self, result: LayerResult) -> bool { if let Ok(mut map) = self.store.lock() { - match map.entry(result.get_tile_coords()) { + match map.entry(result.get_coords()) { btree_map::Entry::Vacant(entry) => { entry.insert(vec![result]); } @@ -31,10 +31,10 @@ impl TileCache { } } - pub(crate) fn get_tessellated_layers_at( + pub fn get_tessellated_layers_at( &self, - coords: &TileCoords, - skip_layers: &Vec, + coords: &WorldTileCoords, + skip_layers: &HashSet, ) -> Vec { let mut ret = Vec::new(); if let Ok(map) = self.store.try_lock() { @@ -50,29 +50,26 @@ impl TileCache { ret } - pub(crate) fn get_missing_tessellated_layer_names_at( + pub fn get_missing_tessellated_layer_names_at( &self, - coords: &TileCoords, - layers: &Vec, - ) -> Vec { + coords: &WorldTileCoords, + mut layers: HashSet, + ) -> Option> { if let Ok(loaded) = self.store.try_lock() { if let Some(tessellated_layers) = loaded.get(coords) { - let mut result = Vec::new(); - for layer in layers { - if tessellated_layers - .iter() - .find(|tessellated_layer| tessellated_layer.layer_name() == layer) - .is_none() - { - result.push(layer.clone()); - } - } - result + let tessellated_set: HashSet = tessellated_layers + .iter() + .map(|tessellated_layer| tessellated_layer.layer_name().to_string()) + .collect(); + + layers.retain(|layer| !tessellated_set.contains(layer)); + + Some(layers) } else { - layers.clone() + Some(layers) } } else { - Vec::new() + None } } } diff --git a/src/io/workflow.rs b/src/io/workflow.rs index 1cf4aa66..27360ed5 100644 --- a/src/io/workflow.rs +++ b/src/io/workflow.rs @@ -1,22 +1,24 @@ /// Describes through which channels work-requests travel. It describes the flow of work. -use crate::coords::TileCoords; +use crate::coords::{TileCoords, WorldTileCoords}; use crate::io::tile_cache::TileCache; use crate::io::web_tile_fetcher::WebTileFetcher; use crate::io::{HttpFetcherConfig, TileFetcher}; use crate::render::ShaderVertex; use crate::tessellation::{IndexDataType, OverAlignedVertexBuffer, Tessellated}; -use crossbeam_channel::{unbounded as channel, Receiver, SendError, Sender}; +use crossbeam_channel::{unbounded as channel, Receiver, RecvError, SendError, Sender}; use log::{error, info, warn}; use std::collections::HashSet; +use std::fmt::{Debug, Formatter}; use std::sync::Mutex; +use style_spec::source::TileAdressingScheme; use vector_tile::parse_tile_bytes; use vector_tile::tile::Layer; pub struct Workflow { layer_result_receiver: Receiver, - pub tile_request_dispatcher: TileRequestDispatcher, - pub download_tessellate_loop: Option, - pub tile_cache: TileCache, + tile_request_dispatcher: TileRequestDispatcher, + download_tessellate_loop: Option, + tile_cache: TileCache, } impl Drop for Workflow { @@ -50,6 +52,23 @@ impl Workflow { } } + pub fn request_tile( + &mut self, + tile_request: TileRequest, + ) -> Result<(), SendError> { + self.tile_request_dispatcher + .request_tile(tile_request, &self.tile_cache) + } + + pub fn get_tessellated_layers_at( + &self, + coords: &WorldTileCoords, + skip_layers: &HashSet, + ) -> Vec { + self.tile_cache + .get_tessellated_layers_at(coords, skip_layers) + } + pub fn take_download_loop(&mut self) -> DownloadTessellateLoop { self.download_tessellate_loop.take().unwrap() } @@ -58,11 +77,11 @@ impl Workflow { #[derive(Clone)] pub enum LayerResult { EmptyLayer { - coords: TileCoords, + coords: WorldTileCoords, layer_name: String, }, TessellatedLayer { - coords: TileCoords, + coords: WorldTileCoords, buffer: OverAlignedVertexBuffer, /// Holds for each feature the count of indices feature_indices: Vec, @@ -70,8 +89,14 @@ pub enum LayerResult { }, } +impl Debug for LayerResult { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "LayerResult{}", self.get_coords()) + } +} + impl LayerResult { - pub fn get_tile_coords(&self) -> TileCoords { + pub fn get_coords(&self) -> WorldTileCoords { match self { LayerResult::EmptyLayer { coords, .. } => *coords, LayerResult::TessellatedLayer { coords, .. } => *coords, @@ -86,11 +111,14 @@ impl LayerResult { } } -pub struct TileRequest(pub TileCoords, pub Vec); +pub struct TileRequest { + pub coords: WorldTileCoords, + pub layers: HashSet, +} pub struct TileRequestDispatcher { request_sender: Sender, - pending_tiles: HashSet, + pending_tiles: HashSet, } impl TileRequestDispatcher { @@ -106,21 +134,21 @@ impl TileRequestDispatcher { tile_request: TileRequest, tile_cache: &TileCache, ) -> Result<(), SendError> { - let TileRequest(coords, layers) = &tile_request; + let TileRequest { coords, layers } = &tile_request; - let missing_layers = tile_cache.get_missing_tessellated_layer_names_at(&coords, &layers); + if let Some(missing_layers) = + tile_cache.get_missing_tessellated_layer_names_at(&coords, layers.clone()) + { + if self.pending_tiles.contains(&coords) { + return Ok(()); + } + self.pending_tiles.insert(*coords); - if missing_layers.is_empty() { + info!("new tile request: {}", &coords); + self.request_sender.send(tile_request) + } else { return Ok(()); } - - if self.pending_tiles.contains(&coords) { - return Ok(()); - } - self.pending_tiles.insert(*coords); - - info!("new tile request: {}", &coords); - self.request_sender.send(tile_request) } } @@ -129,6 +157,24 @@ pub struct DownloadTessellateLoop { result_sender: Sender, } +#[derive(Debug)] +pub enum SendReceiveError { + Send(SendError), + Receive(RecvError), +} + +impl From for SendReceiveError { + fn from(e: RecvError) -> Self { + SendReceiveError::Receive(e) + } +} + +impl From> for SendReceiveError { + fn from(e: SendError) -> Self { + SendReceiveError::Send(e) + } +} + impl DownloadTessellateLoop { pub fn new( request_receiver: Receiver, @@ -140,7 +186,7 @@ impl DownloadTessellateLoop { } } - pub async fn run_loop(&self) { + pub async fn run_loop(&self) -> Result<(), SendReceiveError> { let fetcher = WebTileFetcher::new(HttpFetcherConfig { cache_path: "/tmp/mapr-cache".to_string(), }); @@ -149,11 +195,15 @@ impl DownloadTessellateLoop { loop { // Internally uses Condvar probably: Condvar is also supported on WASM // see https://github.com/rust-lang/rust/blob/effea9a2a0d501db5722d507690a1a66236933bf/library/std/src/sys/wasm/atomics/condvar.rs - if let TileRequest(coords, layers_to_load) = self.request_receiver.recv().unwrap() { - // TODO remove unwrap - match fetcher.fetch_tile(&coords).await { + if let TileRequest { + coords, + layers: layers_to_load, + } = self.request_receiver.recv()? + { + let tile_coords = coords.into_tile(&TileAdressingScheme::TMS); + match fetcher.fetch_tile(&tile_coords).await { Ok(data) => { - info!("preparing tile {} with {}bytes", &coords, data.len()); + info!("preparing tile {} with {}bytes", &tile_coords, data.len()); let tile = parse_tile_bytes(data.as_slice()).expect("failed to load tile"); for to_load in layers_to_load { @@ -163,21 +213,19 @@ impl DownloadTessellateLoop { .find(|layer| to_load.as_str() == layer.name()) { if let Some((buffer, feature_indices)) = layer.tessellate() { - self.result_sender - .send(LayerResult::TessellatedLayer { - coords, - buffer: buffer.into(), - feature_indices, - layer_data: layer.clone(), - }) - .unwrap(); + self.result_sender.send(LayerResult::TessellatedLayer { + coords, + buffer: buffer.into(), + feature_indices, + layer_data: layer.clone(), + })?; } } } - info!("layer ready: {:?}", &coords); + info!("LayerResult ready: {}", &coords); } Err(err) => { - error!("layer failed: {:?}", &err); + error!("LayerResult failed: {:?}", &err); } } } diff --git a/src/platform/android/mod.rs b/src/platform/android/mod.rs index 3c4ff2ca..4d2d7743 100644 --- a/src/platform/android/mod.rs +++ b/src/platform/android/mod.rs @@ -24,7 +24,9 @@ pub async fn main() { let join_handle = task::spawn_blocking(move || { Handle::current().block_on(async move { - download_tessellate_loop.run_loop().await; + if let Err(e) = download_tessellate_loop.run_loop().await { + error!("Worker loop errored {:?}", e) + } }); }); diff --git a/src/platform/apple/mod.rs b/src/platform/apple/mod.rs index ea7bcaff..baf08e55 100644 --- a/src/platform/apple/mod.rs +++ b/src/platform/apple/mod.rs @@ -25,7 +25,9 @@ pub async fn mapr_apple_main() { let join_handle = task::spawn_blocking(move || { Handle::current().block_on(async move { - download_tessellate_loop.run_loop().await; + if let Err(e) = download_tessellate_loop.run_loop().await { + error!("Worker loop errored {:?}", e) + } }); }); diff --git a/src/platform/generic.rs b/src/platform/generic.rs index 881822fe..aeee1dad 100644 --- a/src/platform/generic.rs +++ b/src/platform/generic.rs @@ -3,6 +3,7 @@ use crate::io::tile_cache::TileCache; use crate::io::workflow::Workflow; use crate::main_loop; +use log::error; pub use std::time::Instant; use tokio::runtime::Handle; use tokio::task; @@ -27,7 +28,9 @@ pub async fn mapr_generic_main() { let join_handle = task::spawn_blocking(move || { Handle::current().block_on(async move { - download_tessellate_loop.run_loop().await; + if let Err(e) = download_tessellate_loop.run_loop().await { + error!("Worker loop errored {:?}", e) + } }); }); diff --git a/src/platform/web/mod.rs b/src/platform/web/mod.rs index 6ab807a3..8a8e9855 100644 --- a/src/platform/web/mod.rs +++ b/src/platform/web/mod.rs @@ -47,7 +47,9 @@ pub async fn run_worker_loop(workflow_ptr: *mut Workflow) { let mut workflow: Box = unsafe { Box::from_raw(workflow_ptr) }; // Either call forget or the worker loop to keep it alive - workflow.take_download_loop().run_loop().await; + if let Err(e) = workflow.take_download_loop().run_loop().await { + error!("Worker loop errored {:?}", e) + } //std::mem::forget(workflow); } diff --git a/src/render/buffer_pool.rs b/src/render/buffer_pool.rs index f3b1b096..56f54b34 100644 --- a/src/render/buffer_pool.rs +++ b/src/render/buffer_pool.rs @@ -1,5 +1,5 @@ use std::collections::vec_deque::Iter; -use std::collections::VecDeque; +use std::collections::{HashSet, VecDeque}; use std::fmt::Debug; use std::marker::PhantomData; use std::mem::size_of; @@ -7,7 +7,7 @@ use std::ops::Range; use wgpu::BufferAddress; -use crate::coords::TileCoords; +use crate::coords::{TileCoords, WorldTileCoords}; use crate::render::shaders::ShaderTileMetadata; use crate::tessellation::OverAlignedVertexBuffer; @@ -134,7 +134,7 @@ impl, B, V: bytemuck::Pod, I: bytemuck::Pod, TM: bytemuck::Pod, FM: } /// FIXME: use an id instead of layer_name to identify tiles - pub fn get_loaded_layers(&self, coords: &TileCoords) -> Vec { + pub fn get_loaded_layers(&self, coords: &WorldTileCoords) -> HashSet { self.index .iter() .filter(|entry| entry.coords == *coords) @@ -150,7 +150,7 @@ impl, B, V: bytemuck::Pod, I: bytemuck::Pod, TM: bytemuck::Pod, FM: pub fn allocate_tile_geometry( &mut self, queue: &Q, - coords: TileCoords, + coords: WorldTileCoords, layer_name: &str, geometry: &OverAlignedVertexBuffer, tile_metadata: TM, @@ -347,7 +347,7 @@ impl BackingBuffer { #[derive(Debug)] pub struct IndexEntry { - pub coords: TileCoords, + pub coords: WorldTileCoords, pub layer_name: String, // Range of bytes within the backing buffer for vertices buffer_vertices: Range, diff --git a/src/render/render_state.rs b/src/render/render_state.rs index 7626dcbf..ae5e20a7 100644 --- a/src/render/render_state.rs +++ b/src/render/render_state.rs @@ -1,4 +1,5 @@ use cgmath::Matrix4; +use std::collections::HashSet; use std::default::Default; use std::{cmp, iter}; @@ -354,20 +355,17 @@ impl RenderState { // Fetch tiles which are currently in view if let Some(view_region) = &view_region { - for tile_coords in view_region.iter() { - let tile_request = TileRequest( - tile_coords, - vec![ + for coords in view_region.iter() { + let tile_request = TileRequest { + coords, + layers: HashSet::from([ "transportation".to_string(), "building".to_string(), "boundary".to_string(), "water".to_string(), - ], - ); - workflow - .tile_request_dispatcher - .request_tile(tile_request, &workflow.tile_cache) - .unwrap(); + ]), + }; + workflow.request_tile(tile_request).unwrap(); } } @@ -377,7 +375,7 @@ impl RenderState { // We perform the update before uploading new tessellated tiles, such that each // tile metadata in the the `buffer_pool` gets updated exactly once and not twice. for entry in self.buffer_pool.index() { - let world_coords = entry.coords.into_world_tile(); + let world_coords = entry.coords; let transform: Matrix4 = (view_proj * world_coords.transform_for_zoom(self.zoom)) .cast() .unwrap(); @@ -391,12 +389,10 @@ impl RenderState { // Upload all tessellated layers which are in view if let Some(view_region) = &view_region { - for tile_coords in view_region.iter() { - let loaded_layers = self.buffer_pool.get_loaded_layers(&tile_coords); + for coords in view_region.iter() { + let loaded_layers = self.buffer_pool.get_loaded_layers(&coords); - let layers = workflow - .tile_cache - .get_tessellated_layers_at(&tile_coords, &loaded_layers); + let layers = workflow.get_tessellated_layers_at(&coords, &loaded_layers); for result in layers { match result { LayerResult::EmptyLayer { .. } => {} @@ -407,7 +403,7 @@ impl RenderState { buffer, .. } => { - let world_coords = coords.into_world_tile(); + let world_coords = coords; let feature_metadata = layer_data .features() @@ -516,10 +512,9 @@ impl RenderState { .index() .filter(|entry| entry.coords.z == self.visible_z()) { - let reference = self - .tile_mask_pattern - .stencil_reference_value(&entry.coords.into_world_tile()) - as u32; + let reference = + self.tile_mask_pattern + .stencil_reference_value(&entry.coords) as u32; // Draw mask {