From 1e63718dfcdef78d354eaabd210d38120e960436 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sun, 22 May 2022 12:46:33 +0200 Subject: [PATCH 01/31] Implement headless rendering --- maplibre-demo/Cargo.toml | 3 + maplibre-demo/src/main.rs | 74 ++++++++++- maplibre-winit/src/winit/mod.rs | 4 - maplibre-winit/src/winit/noweb.rs | 37 +++--- maplibre-winit/src/winit/web.rs | 46 +++++-- maplibre/Cargo.toml | 2 + maplibre/src/lib.rs | 34 ++++- .../src/render/copy_surface_to_buffer_node.rs | 65 ++++++++++ maplibre/src/render/mod.rs | 100 ++++++++++----- maplibre/src/render/resource/surface.rs | 116 ++++++++++++++---- .../src/render/stages/graph_runner_stage.rs | 11 +- maplibre/src/render/stages/mod.rs | 6 + maplibre/src/render/stages/resource_stage.rs | 3 +- maplibre/src/render/stages/upload_stage.rs | 10 +- .../stages/write_surface_buffer_stage.rs | 50 ++++++++ maplibre/src/window.rs | 30 ++++- 16 files changed, 478 insertions(+), 113 deletions(-) create mode 100644 maplibre/src/render/copy_surface_to_buffer_node.rs create mode 100644 maplibre/src/render/stages/write_surface_buffer_stage.rs diff --git a/maplibre-demo/Cargo.toml b/maplibre-demo/Cargo.toml index c05d63d9..abd35546 100644 --- a/maplibre-demo/Cargo.toml +++ b/maplibre-demo/Cargo.toml @@ -17,6 +17,9 @@ env_logger = "0.9" maplibre = { path = "../maplibre", version = "0.0.2" } maplibre-winit = { path = "../maplibre-winit", version = "0.0.1" } +tokio = "1.18" +wgpu = "0.12" + tracing = { version = "0.1" } tracing-subscriber = { version = "0.3", optional = true } tracing-tracy = { version = "0.8", optional = true } diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index 86e286d9..28759fb9 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -1,8 +1,15 @@ +use maplibre::error::Error; +use maplibre::io::scheduler::ScheduleMethod; +use maplibre::io::source_client::HTTPClient; +use maplibre::map_state::MapSchedule; use maplibre::platform::http_client::ReqwestHttpClient; use maplibre::platform::run_multithreaded; use maplibre::platform::schedule_method::TokioScheduleMethod; +use maplibre::render::settings::RendererSettings; +use maplibre::window::{MapWindow, MapWindowConfig, Runnable, WindowSize}; use maplibre::MapBuilder; use maplibre_winit::winit::{WinitEventLoop, WinitMapWindow, WinitMapWindowConfig, WinitWindow}; +use wgpu::TextureFormat; #[cfg(feature = "trace")] fn enable_tracing() { @@ -13,6 +20,54 @@ fn enable_tracing() { tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); } +pub struct HeadlessMapWindowConfig; + +impl MapWindowConfig for HeadlessMapWindowConfig { + type MapWindow = HeadlessMapWindow; + + fn create(&self) -> Self::MapWindow { + Self::MapWindow {} + } +} + +pub struct HeadlessMapWindow; + +impl MapWindow for HeadlessMapWindow { + type EventLoop = (); + type RawWindow = (); + + fn size(&self) -> WindowSize { + WindowSize::new(1920, 1080).unwrap() + } + + fn inner(&self) -> &Self::RawWindow { + &() + } +} + +impl Runnable for HeadlessMapWindow +where + MWC: MapWindowConfig, + SM: ScheduleMethod, + HC: HTTPClient, +{ + fn run(mut self, mut map_state: MapSchedule, max_frames: Option) { + let arc = map_state.map_context.renderer.device.clone(); + tokio::task::spawn_blocking(move || loop { + arc.poll(wgpu::Maintain::Wait); + }); + for i in 0..3 { + match map_state.update_and_redraw() { + Ok(_) => {} + Err(Error::Render(e)) => { + eprintln!("{}", e); + if e.should_exit() {} + } + e => eprintln!("{:?}", e), + }; + } + } +} fn run_in_window() { run_multithreaded(async { @@ -27,11 +82,28 @@ fn run_in_window() { }) } +fn run_headless() { + run_multithreaded(async { + MapBuilder::new() + .with_map_window_config(HeadlessMapWindowConfig) + .with_http_client(ReqwestHttpClient::new(None)) + .with_schedule_method(TokioScheduleMethod::new()) + .with_renderer_settings(RendererSettings { + texture_format: TextureFormat::Rgba8UnormSrgb, + ..RendererSettings::default() + }) + .build() + .initialize_headless() + .await + .run() + }) +} + fn main() { env_logger::init_from_env(env_logger::Env::default().default_filter_or("info")); #[cfg(feature = "trace")] enable_tracing(); - run_in_window() + run_headless() } diff --git a/maplibre-winit/src/winit/mod.rs b/maplibre-winit/src/winit/mod.rs index 564134dc..f566d9c2 100644 --- a/maplibre-winit/src/winit/mod.rs +++ b/maplibre-winit/src/winit/mod.rs @@ -47,10 +47,6 @@ impl WinitMapWindowConfig { } } -impl MapWindowConfig for WinitMapWindowConfig { - type MapWindow = WinitMapWindow; -} - pub struct WinitMapWindow { window: WinitWindow, event_loop: Option, diff --git a/maplibre-winit/src/winit/noweb.rs b/maplibre-winit/src/winit/noweb.rs index 4338aded..3b7337e1 100644 --- a/maplibre-winit/src/winit/noweb.rs +++ b/maplibre-winit/src/winit/noweb.rs @@ -10,25 +10,11 @@ use super::WinitMapWindow; use super::WinitWindow; use super::WinitMapWindowConfig; -use maplibre::window::{MapWindow, WindowSize}; +use maplibre::window::{MapWindow, MapWindowConfig, WindowSize}; impl MapWindow for WinitMapWindow { type EventLoop = WinitEventLoop; - type Window = WinitWindow; - type MapWindowConfig = WinitMapWindowConfig; - - fn create(map_window_config: &Self::MapWindowConfig) -> Self { - let event_loop = WinitEventLoop::new(); - let window = WindowBuilder::new() - .with_title(&map_window_config.title) - .build(&event_loop) - .unwrap(); - - Self { - window, - event_loop: Some(event_loop), - } - } + type RawWindow = WinitWindow; fn size(&self) -> WindowSize { let size = self.window.inner_size(); @@ -44,7 +30,24 @@ impl MapWindow for WinitMapWindow { window_size } - fn inner(&self) -> &Self::Window { + fn inner(&self) -> &Self::RawWindow { &self.window } } + +impl MapWindowConfig for WinitMapWindowConfig { + type MapWindow = WinitMapWindow; + + fn create(&self) -> Self::MapWindow { + let event_loop = WinitEventLoop::new(); + let window = WindowBuilder::new() + .with_title(&self.title) + .build(&event_loop) + .unwrap(); + + Self::MapWindow { + window, + event_loop: Some(event_loop), + } + } +} diff --git a/maplibre-winit/src/winit/web.rs b/maplibre-winit/src/winit/web.rs index 981c8787..73912b4b 100644 --- a/maplibre-winit/src/winit/web.rs +++ b/maplibre-winit/src/winit/web.rs @@ -10,10 +10,40 @@ use winit::platform::web::WindowBuilderExtWebSys; impl MapWindow for WinitMapWindow { type EventLoop = WinitEventLoop; - type Window = WinitWindow; + type RawWindow = WinitWindow; type MapWindowConfig = WinitMapWindowConfig; - fn create(map_window_config: &Self::MapWindowConfig) -> Self { + fn size(&self) -> WindowSize { + let size = self.window.inner_size(); + + WindowSize::new(size.width, size.height).expect("failed to get window dimensions.") + } + + fn inner(&self) -> &Self::RawWindow { + &self.window + } +} + +fn create(map_window_config: &Self::MapWindowConfig) -> Self { + let event_loop = WinitEventLoop::new(); + + let window: winit::window::Window = WindowBuilder::new() + .with_canvas(Some(get_canvas(&map_window_config.canvas_id))) + .build(&event_loop) + .unwrap(); + + let size = get_body_size().unwrap(); + window.set_inner_size(size); + Self { + window, + event_loop: Some(event_loop), + } +} + +impl MapWindowConfig for WinitMapWindowConfig { + type MapWindow = WinitMapWindow; + + fn create(&self) -> Self::MapWindow { let event_loop = WinitEventLoop::new(); let window: winit::window::Window = WindowBuilder::new() @@ -23,21 +53,11 @@ impl MapWindow for WinitMapWindow { let size = get_body_size().unwrap(); window.set_inner_size(size); - Self { + Self::MapWindow { window, event_loop: Some(event_loop), } } - - fn size(&self) -> WindowSize { - let size = self.window.inner_size(); - - WindowSize::new(size.width, size.height).expect("failed to get window dimensions.") - } - - fn inner(&self) -> &Self::Window { - &self.window - } } pub fn get_body_size() -> Option> { diff --git a/maplibre/Cargo.toml b/maplibre/Cargo.toml index 020bff51..a8605835 100644 --- a/maplibre/Cargo.toml +++ b/maplibre/Cargo.toml @@ -72,5 +72,7 @@ thiserror = "1" downcast-rs = "1.2" smallvec = "1.8" +png = "0.17" + [build-dependencies] maplibre-build-tools = { path = "../maplibre-build-tools", version = "0.1.0", features = ["sqlite"] } diff --git a/maplibre/src/lib.rs b/maplibre/src/lib.rs index 36fab66f..cbc657ff 100644 --- a/maplibre/src/lib.rs +++ b/maplibre/src/lib.rs @@ -22,7 +22,7 @@ use crate::map_schedule::MapSchedule; use crate::render::settings::{RendererSettings, WgpuSettings}; use crate::render::{RenderState, Renderer}; use crate::style::Style; -use crate::window::{MapWindow, MapWindowConfig, Runnable, WindowSize}; +use crate::window::{HasRawWindow, MapWindow, MapWindowConfig, Runnable, WindowSize}; pub mod context; pub mod coords; @@ -47,9 +47,9 @@ pub(crate) mod tessellation; pub(crate) mod util; /// Map's configuration and execution. -pub struct Map +pub struct Map where - W: MapWindow, + MWC: MapWindowConfig, SM: ScheduleMethod, HC: HTTPClient, { @@ -57,9 +57,10 @@ where window: W, } -impl Map +impl Map where - W: MapWindow + Runnable, + MWC: MapWindowConfig, + MWC::MapWindow: Runnable, SM: ScheduleMethod, HC: HTTPClient, { @@ -142,6 +143,29 @@ where window, } } + + pub async fn initialize_headless(self) -> Map { + let window = self.map_window_config.create(); + + let renderer = Renderer::initialize_headless::( + &window, + self.wgpu_settings, + self.renderer_settings, + ) + .await + .unwrap(); + Map { + map_state: MapSchedule::new( + window.size(), + self.map_window_config, + Some(renderer), + self.scheduler, + self.http_client, + self.style, + ), + window, + } + } } pub struct MapBuilder diff --git a/maplibre/src/render/copy_surface_to_buffer_node.rs b/maplibre/src/render/copy_surface_to_buffer_node.rs new file mode 100644 index 00000000..c38cc484 --- /dev/null +++ b/maplibre/src/render/copy_surface_to_buffer_node.rs @@ -0,0 +1,65 @@ +use crate::render::graph::{Node, NodeRunError, RenderContext, RenderGraphContext, SlotInfo}; +use crate::render::render_commands::{DrawMasks, DrawTiles}; +use crate::render::render_phase::{PhaseItem, RenderCommand}; +use crate::render::resource::{Head, TrackedRenderPass}; +use crate::render::stages::draw_graph; +use crate::render::util::FloatOrd; +use crate::render::Eventually::Initialized; +use crate::render::RenderState; +use std::ops::{Deref, Range}; + +#[derive(Default)] +pub struct CopySurfaceBufferNode {} + +impl CopySurfaceBufferNode { + pub fn new() -> Self { + Self {} + } +} + +impl Node for CopySurfaceBufferNode { + fn input(&self) -> Vec { + vec![] + } + + fn update(&mut self, _state: &mut RenderState) {} + + fn run( + &self, + _graph: &mut RenderGraphContext, + RenderContext { + command_encoder, .. + }: &mut RenderContext, + state: &RenderState, + ) -> Result<(), NodeRunError> { + match state.surface.head() { + Head::Headed(_) => {} + Head::Headless(buffered_texture) => { + let size = state.surface.size(); + command_encoder.copy_texture_to_buffer( + buffered_texture.texture.as_image_copy(), + wgpu::ImageCopyBuffer { + buffer: &buffered_texture.output_buffer, + layout: wgpu::ImageDataLayout { + offset: 0, + bytes_per_row: Some( + std::num::NonZeroU32::new( + buffered_texture.buffer_dimensions.padded_bytes_per_row as u32, + ) + .unwrap(), + ), + rows_per_image: None, + }, + }, + wgpu::Extent3d { + width: size.width() as u32, + height: size.height() as u32, + depth_or_array_layers: 1, + }, + ); + } + } + + Ok(()) + } +} diff --git a/maplibre/src/render/mod.rs b/maplibre/src/render/mod.rs index 46d9910d..e08e2962 100644 --- a/maplibre/src/render/mod.rs +++ b/maplibre/src/render/mod.rs @@ -27,10 +27,12 @@ use crate::render::shaders::{ShaderFeatureStyle, ShaderLayerMetadata}; use crate::render::tile_view_pattern::{TileInView, TileShape, TileViewPattern}; use crate::render::util::Eventually; use crate::tessellation::IndexDataType; -use crate::MapWindow; +use crate::{HasRawWindow, MapWindow, MapWindowConfig}; use log::info; +use std::sync::Arc; // Rendering internals +mod copy_surface_to_buffer_node; mod graph; mod graph_runner; mod main_pass; @@ -52,7 +54,6 @@ pub use stages::register_render_stages; pub const INDEX_FORMAT: wgpu::IndexFormat = wgpu::IndexFormat::Uint32; // Must match IndexDataType -#[derive(Default)] pub struct RenderState { render_target: Eventually, @@ -76,13 +77,33 @@ pub struct RenderState { depth_texture: Eventually, multisampling_texture: Eventually>, + surface: Surface, + mask_phase: RenderPhase, tile_phase: RenderPhase<(IndexEntry, TileShape)>, } +impl RenderState { + pub fn new(surface: Surface) -> Self { + Self { + render_target: Default::default(), + buffer_pool: Default::default(), + tile_view_pattern: Default::default(), + tile_pipeline: Default::default(), + mask_pipeline: Default::default(), + globals_bind_group: Default::default(), + depth_texture: Default::default(), + multisampling_texture: Default::default(), + surface, + mask_phase: Default::default(), + tile_phase: Default::default(), + } + } +} + pub struct Renderer { pub instance: wgpu::Instance, - pub device: wgpu::Device, + pub device: Arc, pub queue: wgpu::Queue, pub adapter_info: wgpu::AdapterInfo, @@ -90,34 +111,29 @@ pub struct Renderer { pub settings: RendererSettings, pub state: RenderState, - pub surface: Surface, } impl Renderer { /// Initializes the renderer by retrieving and preparing the GPU instance, device and queue /// for the specified backend. - pub async fn initialize( - window: &MW, + pub async fn initialize( + window: &MWC::MapWindow, wgpu_settings: WgpuSettings, settings: RendererSettings, ) -> Result where - MW: MapWindow, + MWC::MapWindow: HasRawWindow, + MWC: MapWindowConfig, + /* <::MapWindow as MapWindow>::Window: + raw_window_handle::HasRawWindowHandle,*/ { let instance = wgpu::Instance::new(wgpu_settings.backends.unwrap_or(wgpu::Backends::all())); - let maybe_surface = match &settings.surface_type { - SurfaceType::Headless => None, - SurfaceType::Headed => Some(Surface::from_window(&instance, window, &settings)), - }; + let surface = Surface::from_window::(&instance, window, &settings); - let compatible_surface = if let Some(surface) = &maybe_surface { - match &surface.head() { - Head::Headed(window_head) => Some(window_head.surface()), - Head::Headless(_) => None, - } - } else { - None + let compatible_surface = match &surface.head() { + Head::Headed(window_head) => Some(window_head.surface()), + Head::Headless(_) => None, }; let (device, queue, adapter_info) = Self::request_device( @@ -131,11 +147,6 @@ impl Renderer { ) .await?; - let surface = maybe_surface.unwrap_or_else(|| match &settings.surface_type { - SurfaceType::Headless => Surface::from_image(&device, window, &settings), - SurfaceType::Headed => Surface::from_window(&instance, window, &settings), - }); - match surface.head() { Head::Headed(window) => window.configure(&device), Head::Headless(_) => {} @@ -143,18 +154,51 @@ impl Renderer { Ok(Self { instance, - device, + device: Arc::new(device), queue, adapter_info, wgpu_settings, settings, - state: Default::default(), - surface, + state: RenderState::new(surface), + }) + } + + pub async fn initialize_headless( + window: &MWC::MapWindow, + wgpu_settings: WgpuSettings, + settings: RendererSettings, + ) -> Result + where + MWC: MapWindowConfig, + { + let instance = wgpu::Instance::new(wgpu_settings.backends.unwrap_or(wgpu::Backends::all())); + + let (device, queue, adapter_info) = Self::request_device( + &instance, + &wgpu_settings, + &wgpu::RequestAdapterOptions { + power_preference: wgpu_settings.power_preference, + force_fallback_adapter: false, + compatible_surface: None, + }, + ) + .await?; + + let surface = Surface::from_image::(&device, window, &settings); + + Ok(Self { + instance, + device: Arc::new(device), + queue, + adapter_info, + wgpu_settings, + settings, + state: RenderState::new(surface), }) } pub fn resize(&mut self, width: u32, height: u32) { - self.surface.resize(width, height) + self.state.surface.resize(width, height) } /// Requests a device @@ -323,7 +367,7 @@ impl Renderer { &self.state } pub fn surface(&self) -> &Surface { - &self.surface + &self.state.surface } } diff --git a/maplibre/src/render/resource/surface.rs b/maplibre/src/render/resource/surface.rs index f4c4b800..1feeb0b8 100644 --- a/maplibre/src/render/resource/surface.rs +++ b/maplibre/src/render/resource/surface.rs @@ -4,14 +4,17 @@ use crate::render::resource::texture::TextureView; use crate::render::settings::RendererSettings; use crate::render::util::HasChanged; -use crate::{MapWindow, WindowSize}; +use crate::{HasRawWindow, MapWindow, MapWindowConfig, WindowSize}; +use std::fs::File; +use std::io::Write; use std::mem::size_of; +use std::sync::Arc; -struct BufferDimensions { - width: usize, - height: usize, - unpadded_bytes_per_row: usize, - padded_bytes_per_row: usize, +pub struct BufferDimensions { + pub width: usize, + pub height: usize, + pub unpadded_bytes_per_row: usize, + pub padded_bytes_per_row: usize, } impl BufferDimensions { @@ -40,9 +43,11 @@ impl WindowHead { self.surface.configure(device, &self.surface_config); } - pub fn recreate_surface(&mut self, window: &MW, instance: &wgpu::Instance) + pub fn recreate_surface(&mut self, window: &MWC::MapWindow, instance: &wgpu::Instance) where - MW: MapWindow, + MWC: MapWindowConfig, + <::MapWindow as MapWindow>::RawWindow: + raw_window_handle::HasRawWindowHandle, { self.surface = unsafe { instance.create_surface(window.inner()) }; } @@ -52,14 +57,61 @@ impl WindowHead { } pub struct BufferedTextureHead { - texture: wgpu::Texture, - output_buffer: wgpu::Buffer, - buffer_dimensions: BufferDimensions, + pub texture: wgpu::Texture, + pub output_buffer: wgpu::Buffer, + pub buffer_dimensions: BufferDimensions, +} + +impl BufferedTextureHead { + pub async fn create_png<'a>( + &self, + png_output_path: &str, + // device: &wgpu::Device, + ) { + // Note that we're not calling `.await` here. + let buffer_slice = self.output_buffer.slice(..); + let buffer_future = buffer_slice.map_async(wgpu::MapMode::Read); + + // Poll the device in a blocking manner so that our future resolves. + // In an actual application, `device.poll(...)` should + // be called in an event loop or on another thread. + //device.poll(wgpu::Maintain::Wait); + if let Ok(()) = buffer_future.await { + let padded_buffer = buffer_slice.get_mapped_range(); + + let mut png_encoder = png::Encoder::new( + File::create(png_output_path).unwrap(), + self.buffer_dimensions.width as u32, + self.buffer_dimensions.height as u32, + ); + png_encoder.set_depth(png::BitDepth::Eight); + png_encoder.set_color(png::ColorType::Rgba); + let mut png_writer = png_encoder + .write_header() + .unwrap() + .into_stream_writer_with_size(self.buffer_dimensions.unpadded_bytes_per_row) + .unwrap(); + + // from the padded_buffer we write just the unpadded bytes into the image + for chunk in padded_buffer.chunks(self.buffer_dimensions.padded_bytes_per_row) { + png_writer + .write_all(&chunk[..self.buffer_dimensions.unpadded_bytes_per_row]) + .unwrap(); + } + png_writer.finish().unwrap(); + + // With the current interface, we have to make sure all mapped views are + // dropped before we unmap the buffer. + drop(padded_buffer); + + self.output_buffer.unmap(); + } + } } pub enum Head { Headed(WindowHead), - Headless(BufferedTextureHead), + Headless(Arc), } pub struct Surface { @@ -68,13 +120,16 @@ pub struct Surface { } impl Surface { - pub fn from_window( + pub fn from_window( instance: &wgpu::Instance, - window: &MW, + window: &MWC::MapWindow, settings: &RendererSettings, ) -> Self where - MW: MapWindow, + MWC: MapWindowConfig, + MWC::MapWindow: HasRawWindow, + /* <::MapWindow as MapWindow>::Window: + raw_window_handle::HasRawWindowHandle,*/ { let size = window.size(); let surface_config = wgpu::SurfaceConfiguration { @@ -86,7 +141,7 @@ impl Surface { present_mode: wgpu::PresentMode::Fifo, // VSync }; - let surface = unsafe { instance.create_surface(window.inner()) }; + let surface = unsafe { instance.create_surface(window.raw_window()) }; Self { size, @@ -97,9 +152,13 @@ impl Surface { } } - pub fn from_image(device: &wgpu::Device, window: &MW, settings: &RendererSettings) -> Self + pub fn from_image( + device: &wgpu::Device, + window: &MWC::MapWindow, + settings: &RendererSettings, + ) -> Self where - MW: MapWindow, + MWC: MapWindowConfig, { let size = window.size(); @@ -111,7 +170,7 @@ impl Surface { BufferDimensions::new(size.width() as usize, size.height() as usize); // The output buffer lets us retrieve the data as an array let output_buffer = device.create_buffer(&wgpu::BufferDescriptor { - label: None, + label: Some("BufferedTextureHead buffer"), size: (buffer_dimensions.padded_bytes_per_row * buffer_dimensions.height) as u64, usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST, mapped_at_creation: false, @@ -133,11 +192,11 @@ impl Surface { Self { size, - head: Head::Headless(BufferedTextureHead { + head: Head::Headless(Arc::new(BufferedTextureHead { texture, output_buffer, buffer_dimensions, - }), + })), } } @@ -159,7 +218,8 @@ impl Surface { }; frame.into() } - Head::Headless(BufferedTextureHead { texture, .. }) => texture + Head::Headless(arc) => arc + .texture .create_view(&wgpu::TextureViewDescriptor::default()) .into(), } @@ -191,13 +251,17 @@ impl Surface { } } - pub fn recreate(&mut self, window: &MW, instance: &wgpu::Instance) + pub fn recreate(&mut self, window: &MWC::MapWindow, instance: &wgpu::Instance) where - MW: MapWindow, + MWC: MapWindowConfig, + <::MapWindow as MapWindow>::RawWindow: + raw_window_handle::HasRawWindowHandle, { match &mut self.head { - Head::Headed(head) => { - head.recreate_surface(window, instance); + Head::Headed(window_head) => { + if window_head.has_changed(&(self.size.width(), self.size.height())) { + window_head.recreate_surface::(window, instance); + } } Head::Headless(_) => {} } diff --git a/maplibre/src/render/stages/graph_runner_stage.rs b/maplibre/src/render/stages/graph_runner_stage.rs index 56376208..9a9f7cb6 100644 --- a/maplibre/src/render/stages/graph_runner_stage.rs +++ b/maplibre/src/render/stages/graph_runner_stage.rs @@ -6,6 +6,7 @@ // 3. "sub graph" modules should be nested beneath their parent graph module use crate::context::MapContext; +use crate::render::copy_surface_to_buffer_node::CopySurfaceBufferNode; use crate::render::graph::{EmptyNode, RenderGraph}; use crate::render::graph_runner::RenderGraphRunner; use crate::render::main_pass::{MainPassDriverNode, MainPassNode}; @@ -24,6 +25,7 @@ pub mod draw_graph { pub mod input {} pub mod node { pub const MAIN_PASS: &str = "main_pass"; + pub const COPY: &str = "copy"; } } @@ -34,17 +36,20 @@ pub struct GraphRunnerStage { impl Default for GraphRunnerStage { fn default() -> Self { - let pass_node = MainPassNode::new(); let mut graph = RenderGraph::default(); let mut draw_graph = RenderGraph::default(); - draw_graph.add_node(draw_graph::node::MAIN_PASS, pass_node); + draw_graph.add_node(draw_graph::node::MAIN_PASS, MainPassNode::new()); let input_node_id = draw_graph.set_input(vec![]); draw_graph .add_node_edge(input_node_id, draw_graph::node::MAIN_PASS) .unwrap(); - graph.add_sub_graph(draw_graph::NAME, draw_graph); + draw_graph.add_node(draw_graph::node::COPY, CopySurfaceBufferNode::default()); + draw_graph + .add_node_edge(draw_graph::node::MAIN_PASS, draw_graph::node::COPY) + .unwrap(); + graph.add_sub_graph(draw_graph::NAME, draw_graph); graph.add_node(node::MAIN_PASS_DEPENDENCIES, EmptyNode); graph.add_node(node::MAIN_PASS_DRIVER, MainPassDriverNode); graph diff --git a/maplibre/src/render/stages/mod.rs b/maplibre/src/render/stages/mod.rs index c055df7d..55969992 100644 --- a/maplibre/src/render/stages/mod.rs +++ b/maplibre/src/render/stages/mod.rs @@ -11,10 +11,12 @@ mod phase_sort_stage; mod queue_stage; mod resource_stage; mod upload_stage; +mod write_surface_buffer_stage; use crate::multi_stage; use crate::render::stages::phase_sort_stage::PhaseSortStage; use crate::render::stages::queue_stage::QueueStage; +use crate::render::stages::write_surface_buffer_stage::WriteSurfaceBufferStage; pub use graph_runner_stage::{draw_graph, node}; /// The labels of the default App rendering stages. @@ -53,4 +55,8 @@ pub fn register_render_stages(schedule: &mut Schedule) { schedule.add_stage(RenderStageLabel::Queue, QueueStage::default()); schedule.add_stage(RenderStageLabel::PhaseSort, PhaseSortStage::default()); schedule.add_stage(RenderStageLabel::Render, GraphRunnerStage::default()); + schedule.add_stage( + RenderStageLabel::Cleanup, + WriteSurfaceBufferStage::default(), + ); } diff --git a/maplibre/src/render/stages/resource_stage.rs b/maplibre/src/render/stages/resource_stage.rs index d1cf9891..f0e81553 100644 --- a/maplibre/src/render/stages/resource_stage.rs +++ b/maplibre/src/render/stages/resource_stage.rs @@ -28,13 +28,14 @@ impl Stage for ResourceStage { Renderer { settings, device, - surface, state, .. }, .. }: &mut MapContext, ) { + let surface = &mut state.surface; + let size = surface.size(); surface.reconfigure(device); diff --git a/maplibre/src/render/stages/upload_stage.rs b/maplibre/src/render/stages/upload_stage.rs index ffff2e4f..72420496 100644 --- a/maplibre/src/render/stages/upload_stage.rs +++ b/maplibre/src/render/stages/upload_stage.rs @@ -27,15 +27,7 @@ impl Stage for UploadStage { view_state, style, tile_cache, - renderer: - Renderer { - settings: _, - device: _, - queue, - surface: _, - state, - .. - }, + renderer: Renderer { queue, state, .. }, .. }: &mut MapContext, ) { diff --git a/maplibre/src/render/stages/write_surface_buffer_stage.rs b/maplibre/src/render/stages/write_surface_buffer_stage.rs new file mode 100644 index 00000000..e0101b3b --- /dev/null +++ b/maplibre/src/render/stages/write_surface_buffer_stage.rs @@ -0,0 +1,50 @@ +//! Sorts items of the [RenderPhases](RenderPhase). + +use crate::context::MapContext; +use crate::coords::{ViewRegion, Zoom}; +use crate::io::tile_cache::TileCache; +use crate::io::LayerTessellateMessage; +use crate::render::camera::ViewProjection; +use crate::render::render_phase::RenderPhase; +use crate::render::resource::{BufferDimensions, BufferedTextureHead, Head, IndexEntry}; +use crate::render::shaders::{ + ShaderCamera, ShaderFeatureStyle, ShaderGlobals, ShaderLayerMetadata, Vec4f32, +}; +use crate::render::tile_view_pattern::TileInView; +use crate::render::util::Eventually::Initialized; +use crate::schedule::Stage; +use crate::{RenderState, Renderer, Style}; +use std::fs::File; +use std::future::Future; +use std::io::Write; +use std::iter; +use std::ops::Deref; +use tokio::runtime::Handle; +use tokio::task; +use wgpu::{BufferAsyncError, BufferSlice}; + +#[derive(Default)] +pub struct WriteSurfaceBufferStage; + +impl Stage for WriteSurfaceBufferStage { + fn run( + &mut self, + MapContext { + renderer: Renderer { state, .. }, + .. + }: &mut MapContext, + ) { + match state.surface.head() { + Head::Headed(_) => {} + Head::Headless(buffered_texture) => { + let buffered_texture = buffered_texture.clone(); + + task::block_in_place(|| { + Handle::current().block_on(async { + buffered_texture.create_png("test.png").await; + }) + }); + } + } + } +} diff --git a/maplibre/src/window.rs b/maplibre/src/window.rs index c6865635..ea339d4e 100644 --- a/maplibre/src/window.rs +++ b/maplibre/src/window.rs @@ -1,22 +1,40 @@ //! Utilities for the window system. use crate::{HTTPClient, MapSchedule, ScheduleMethod}; +use raw_window_handle::{HasRawWindowHandle, RawWindowHandle}; /// Window with an optional [carte::window::WindowSize]. pub trait MapWindow { type EventLoop; - type Window: raw_window_handle::HasRawWindowHandle; // FIXME: Not true for headless - type MapWindowConfig: MapWindowConfig; - - fn create(map_window_config: &Self::MapWindowConfig) -> Self; + type RawWindow; fn size(&self) -> WindowSize; - fn inner(&self) -> &Self::Window; + fn inner(&self) -> &Self::RawWindow; +} + +pub trait HasRawWindow { + type HRWH: HasRawWindowHandle; + + fn raw_window(&self) -> &Self::HRWH; +} + +impl HasRawWindow for MW +where + MW: MapWindow, + MW::RawWindow: HasRawWindowHandle, +{ + type HRWH = MW::RawWindow; + + fn raw_window(&self) -> &Self::HRWH { + self.inner() + } } pub trait MapWindowConfig: 'static { - type MapWindow: MapWindow; + type MapWindow: MapWindow; + + fn create(&self) -> Self::MapWindow; } pub trait Runnable From 355977e3b76fca02c94fcf79f72ae7df3875a82b Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Sun, 22 May 2022 19:20:14 +0200 Subject: [PATCH 02/31] Simplify window traits --- maplibre-demo/src/main.rs | 7 ------- maplibre-winit/src/winit/mod.rs | 2 +- maplibre-winit/src/winit/noweb.rs | 9 ++++---- maplibre/src/lib.rs | 2 +- maplibre/src/render/mod.rs | 6 ++---- maplibre/src/render/resource/surface.rs | 15 ++++++------- maplibre/src/window.rs | 28 ++++++------------------- 7 files changed, 21 insertions(+), 48 deletions(-) diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index 28759fb9..c765ac0c 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -33,16 +33,9 @@ impl MapWindowConfig for HeadlessMapWindowConfig { pub struct HeadlessMapWindow; impl MapWindow for HeadlessMapWindow { - type EventLoop = (); - type RawWindow = (); - fn size(&self) -> WindowSize { WindowSize::new(1920, 1080).unwrap() } - - fn inner(&self) -> &Self::RawWindow { - &() - } } impl Runnable for HeadlessMapWindow diff --git a/maplibre-winit/src/winit/mod.rs b/maplibre-winit/src/winit/mod.rs index f566d9c2..b3dbbf07 100644 --- a/maplibre-winit/src/winit/mod.rs +++ b/maplibre-winit/src/winit/mod.rs @@ -8,7 +8,7 @@ use winit::event_loop::ControlFlow; use crate::input::{InputController, UpdateState}; use maplibre::map_schedule::MapSchedule; -use maplibre::window::{MapWindow, MapWindowConfig, Runnable}; +use maplibre::window::{HeadedMapWindow, MapWindow, MapWindowConfig, Runnable}; use winit::event::Event; #[cfg(target_arch = "wasm32")] diff --git a/maplibre-winit/src/winit/noweb.rs b/maplibre-winit/src/winit/noweb.rs index 3b7337e1..9457745b 100644 --- a/maplibre-winit/src/winit/noweb.rs +++ b/maplibre-winit/src/winit/noweb.rs @@ -10,12 +10,9 @@ use super::WinitMapWindow; use super::WinitWindow; use super::WinitMapWindowConfig; -use maplibre::window::{MapWindow, MapWindowConfig, WindowSize}; +use maplibre::window::{HeadedMapWindow, MapWindow, MapWindowConfig, WindowSize}; impl MapWindow for WinitMapWindow { - type EventLoop = WinitEventLoop; - type RawWindow = WinitWindow; - fn size(&self) -> WindowSize { let size = self.window.inner_size(); #[cfg(target_os = "android")] @@ -29,6 +26,10 @@ impl MapWindow for WinitMapWindow { WindowSize::new(size.width, size.height).expect("failed to get window dimensions."); window_size } +} +impl HeadedMapWindow for WinitMapWindow { + type EventLoop = WinitEventLoop; + type RawWindow = WinitWindow; fn inner(&self) -> &Self::RawWindow { &self.window diff --git a/maplibre/src/lib.rs b/maplibre/src/lib.rs index cbc657ff..04b17893 100644 --- a/maplibre/src/lib.rs +++ b/maplibre/src/lib.rs @@ -22,7 +22,7 @@ use crate::map_schedule::MapSchedule; use crate::render::settings::{RendererSettings, WgpuSettings}; use crate::render::{RenderState, Renderer}; use crate::style::Style; -use crate::window::{HasRawWindow, MapWindow, MapWindowConfig, Runnable, WindowSize}; +use crate::window::{HeadedMapWindow, MapWindow, MapWindowConfig, Runnable, WindowSize}; pub mod context; pub mod coords; diff --git a/maplibre/src/render/mod.rs b/maplibre/src/render/mod.rs index e08e2962..b35dd765 100644 --- a/maplibre/src/render/mod.rs +++ b/maplibre/src/render/mod.rs @@ -27,7 +27,7 @@ use crate::render::shaders::{ShaderFeatureStyle, ShaderLayerMetadata}; use crate::render::tile_view_pattern::{TileInView, TileShape, TileViewPattern}; use crate::render::util::Eventually; use crate::tessellation::IndexDataType; -use crate::{HasRawWindow, MapWindow, MapWindowConfig}; +use crate::{HeadedMapWindow, MapWindow, MapWindowConfig}; use log::info; use std::sync::Arc; @@ -122,10 +122,8 @@ impl Renderer { settings: RendererSettings, ) -> Result where - MWC::MapWindow: HasRawWindow, MWC: MapWindowConfig, - /* <::MapWindow as MapWindow>::Window: - raw_window_handle::HasRawWindowHandle,*/ + ::MapWindow: HeadedMapWindow, { let instance = wgpu::Instance::new(wgpu_settings.backends.unwrap_or(wgpu::Backends::all())); diff --git a/maplibre/src/render/resource/surface.rs b/maplibre/src/render/resource/surface.rs index 1feeb0b8..f2547886 100644 --- a/maplibre/src/render/resource/surface.rs +++ b/maplibre/src/render/resource/surface.rs @@ -4,7 +4,8 @@ use crate::render::resource::texture::TextureView; use crate::render::settings::RendererSettings; use crate::render::util::HasChanged; -use crate::{HasRawWindow, MapWindow, MapWindowConfig, WindowSize}; +use crate::window::HeadedMapWindow; +use crate::{MapWindow, MapWindowConfig, WindowSize}; use std::fs::File; use std::io::Write; use std::mem::size_of; @@ -46,8 +47,7 @@ impl WindowHead { pub fn recreate_surface(&mut self, window: &MWC::MapWindow, instance: &wgpu::Instance) where MWC: MapWindowConfig, - <::MapWindow as MapWindow>::RawWindow: - raw_window_handle::HasRawWindowHandle, + ::MapWindow: HeadedMapWindow, { self.surface = unsafe { instance.create_surface(window.inner()) }; } @@ -127,9 +127,7 @@ impl Surface { ) -> Self where MWC: MapWindowConfig, - MWC::MapWindow: HasRawWindow, - /* <::MapWindow as MapWindow>::Window: - raw_window_handle::HasRawWindowHandle,*/ + ::MapWindow: HeadedMapWindow, { let size = window.size(); let surface_config = wgpu::SurfaceConfiguration { @@ -141,7 +139,7 @@ impl Surface { present_mode: wgpu::PresentMode::Fifo, // VSync }; - let surface = unsafe { instance.create_surface(window.raw_window()) }; + let surface = unsafe { instance.create_surface(window.inner()) }; Self { size, @@ -254,8 +252,7 @@ impl Surface { pub fn recreate(&mut self, window: &MWC::MapWindow, instance: &wgpu::Instance) where MWC: MapWindowConfig, - <::MapWindow as MapWindow>::RawWindow: - raw_window_handle::HasRawWindowHandle, + ::MapWindow: HeadedMapWindow, { match &mut self.head { Head::Headed(window_head) => { diff --git a/maplibre/src/window.rs b/maplibre/src/window.rs index ea339d4e..784859e4 100644 --- a/maplibre/src/window.rs +++ b/maplibre/src/window.rs @@ -3,34 +3,18 @@ use crate::{HTTPClient, MapSchedule, ScheduleMethod}; use raw_window_handle::{HasRawWindowHandle, RawWindowHandle}; -/// Window with an optional [carte::window::WindowSize]. +/// Window with a [carte::window::WindowSize]. pub trait MapWindow { - type EventLoop; - type RawWindow; - fn size(&self) -> WindowSize; +} + +pub trait HeadedMapWindow: MapWindow { + type EventLoop; + type RawWindow: HasRawWindowHandle; fn inner(&self) -> &Self::RawWindow; } -pub trait HasRawWindow { - type HRWH: HasRawWindowHandle; - - fn raw_window(&self) -> &Self::HRWH; -} - -impl HasRawWindow for MW -where - MW: MapWindow, - MW::RawWindow: HasRawWindowHandle, -{ - type HRWH = MW::RawWindow; - - fn raw_window(&self) -> &Self::HRWH { - self.inner() - } -} - pub trait MapWindowConfig: 'static { type MapWindow: MapWindow; From 03f38c272578b0fdbcfcf5094a4b99322fbcf9a6 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Mon, 23 May 2022 15:57:19 +0200 Subject: [PATCH 03/31] Simplify and fix headless rendering --- maplibre-demo/src/main.rs | 6 +-- maplibre/src/lib.rs | 27 ++++++---- maplibre/src/map_schedule.rs | 50 ++++++++++--------- maplibre/src/render/mod.rs | 2 +- maplibre/src/render/resource/surface.rs | 3 +- .../stages/write_surface_buffer_stage.rs | 15 ++++-- 6 files changed, 60 insertions(+), 43 deletions(-) diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index c765ac0c..0d14fe5a 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -1,7 +1,7 @@ use maplibre::error::Error; use maplibre::io::scheduler::ScheduleMethod; use maplibre::io::source_client::HTTPClient; -use maplibre::map_state::MapSchedule; +use maplibre::map_schedule::MapSchedule; use maplibre::platform::http_client::ReqwestHttpClient; use maplibre::platform::run_multithreaded; use maplibre::platform::schedule_method::TokioScheduleMethod; @@ -45,10 +45,6 @@ where HC: HTTPClient, { fn run(mut self, mut map_state: MapSchedule, max_frames: Option) { - let arc = map_state.map_context.renderer.device.clone(); - tokio::task::spawn_blocking(move || loop { - arc.poll(wgpu::Maintain::Wait); - }); for i in 0..3 { match map_state.update_and_redraw() { Ok(_) => {} diff --git a/maplibre/src/lib.rs b/maplibre/src/lib.rs index 04b17893..5a3349f4 100644 --- a/maplibre/src/lib.rs +++ b/maplibre/src/lib.rs @@ -53,8 +53,8 @@ where SM: ScheduleMethod, HC: HTTPClient, { - map_state: MapSchedule, - window: W, + map_state: MapSchedule, + window: MWC::MapWindow, } impl Map @@ -115,14 +115,18 @@ where { /// Initializes the whole rendering pipeline for the given configuration. /// Returns the initialized map, ready to be run. - pub async fn initialize(self) -> Map { - let window = MWC::MapWindow::create(&self.map_window_config); + pub async fn initialize(self) -> Map + where + MWC: MapWindowConfig, + ::MapWindow: HeadedMapWindow, + { + let window = self.map_window_config.create(); let window_size = window.size(); #[cfg(target_os = "android")] let renderer = None; #[cfg(not(target_os = "android"))] - let renderer = Renderer::initialize( + let renderer = Renderer::initialize::( &window, self.wgpu_settings.clone(), self.renderer_settings.clone(), @@ -146,22 +150,25 @@ where pub async fn initialize_headless(self) -> Map { let window = self.map_window_config.create(); + let window_size = window.size(); let renderer = Renderer::initialize_headless::( &window, - self.wgpu_settings, - self.renderer_settings, + self.wgpu_settings.clone(), + self.renderer_settings.clone(), ) .await - .unwrap(); + .ok(); Map { map_state: MapSchedule::new( - window.size(), self.map_window_config, - Some(renderer), + window_size, + renderer, self.scheduler, self.http_client, self.style, + self.wgpu_settings, + self.renderer_settings, ), window, } diff --git a/maplibre/src/map_schedule.rs b/maplibre/src/map_schedule.rs index ac123e4c..abeedb24 100644 --- a/maplibre/src/map_schedule.rs +++ b/maplibre/src/map_schedule.rs @@ -14,22 +14,22 @@ use crate::schedule::{Schedule, Stage}; use crate::stages::register_stages; use crate::style::Style; use crate::{ - MapWindow, MapWindowConfig, Renderer, RendererSettings, ScheduleMethod, WgpuSettings, - WindowSize, + HeadedMapWindow, MapWindow, MapWindowConfig, Renderer, RendererSettings, ScheduleMethod, + WgpuSettings, WindowSize, }; use std::marker::PhantomData; use std::mem; use std::sync::{mpsc, Arc, Mutex}; pub struct PrematureMapContext { - pub view_state: ViewState, - pub style: Style, + view_state: ViewState, + style: Style, - pub tile_cache: TileCache, - pub scheduler: Box, + tile_cache: TileCache, + scheduler: Box, - pub message_receiver: mpsc::Receiver, - pub shared_thread_state: SharedThreadState, + message_receiver: mpsc::Receiver, + shared_thread_state: SharedThreadState, wgpu_settings: WgpuSettings, renderer_settings: RendererSettings, @@ -183,13 +183,16 @@ where self.suspended = true; } - pub fn resume(&mut self, window: &MW) + pub fn resume(&mut self, window: &MWC::MapWindow) where - MW: MapWindow, + ::MapWindow: HeadedMapWindow, { if let EventuallyMapContext::Full(map_context) = &mut self.map_context { let mut renderer = &mut map_context.renderer; - renderer.surface.recreate(window, &renderer.instance); + renderer + .state + .surface + .recreate::(window, &renderer.instance); self.suspended = false; } } @@ -201,24 +204,25 @@ where } } - pub async fn late_init(&mut self) -> bool { + pub async fn late_init(&mut self) -> bool + where + ::MapWindow: HeadedMapWindow, + { match &self.map_context { EventuallyMapContext::Full(_) => false, EventuallyMapContext::Premature(PrematureMapContext { - view_state, - style, - tile_cache, - scheduler, - message_receiver, - shared_thread_state, wgpu_settings, renderer_settings, + .. }) => { - let window = MWC::MapWindow::create(&self.map_window_config); - let renderer = - Renderer::initialize(&window, wgpu_settings.clone(), renderer_settings.clone()) - .await - .unwrap(); + let window = self.map_window_config.create(); + let renderer = Renderer::initialize::( + &window, + wgpu_settings.clone(), + renderer_settings.clone(), + ) + .await + .unwrap(); &self.map_context.make_full(renderer); true } diff --git a/maplibre/src/render/mod.rs b/maplibre/src/render/mod.rs index b35dd765..e3328c44 100644 --- a/maplibre/src/render/mod.rs +++ b/maplibre/src/render/mod.rs @@ -77,7 +77,7 @@ pub struct RenderState { depth_texture: Eventually, multisampling_texture: Eventually>, - surface: Surface, + pub surface: Surface, mask_phase: RenderPhase, tile_phase: RenderPhase<(IndexEntry, TileShape)>, diff --git a/maplibre/src/render/resource/surface.rs b/maplibre/src/render/resource/surface.rs index f2547886..9e51f731 100644 --- a/maplibre/src/render/resource/surface.rs +++ b/maplibre/src/render/resource/surface.rs @@ -65,6 +65,7 @@ pub struct BufferedTextureHead { impl BufferedTextureHead { pub async fn create_png<'a>( &self, + device: &wgpu::Device, png_output_path: &str, // device: &wgpu::Device, ) { @@ -75,7 +76,7 @@ impl BufferedTextureHead { // Poll the device in a blocking manner so that our future resolves. // In an actual application, `device.poll(...)` should // be called in an event loop or on another thread. - //device.poll(wgpu::Maintain::Wait); + device.poll(wgpu::Maintain::Wait); if let Ok(()) = buffer_future.await { let padded_buffer = buffer_slice.get_mapped_range(); diff --git a/maplibre/src/render/stages/write_surface_buffer_stage.rs b/maplibre/src/render/stages/write_surface_buffer_stage.rs index e0101b3b..adcba764 100644 --- a/maplibre/src/render/stages/write_surface_buffer_stage.rs +++ b/maplibre/src/render/stages/write_surface_buffer_stage.rs @@ -24,13 +24,15 @@ use tokio::task; use wgpu::{BufferAsyncError, BufferSlice}; #[derive(Default)] -pub struct WriteSurfaceBufferStage; +pub struct WriteSurfaceBufferStage { + frame: u64, +} impl Stage for WriteSurfaceBufferStage { fn run( &mut self, MapContext { - renderer: Renderer { state, .. }, + renderer: Renderer { state, device, .. }, .. }: &mut MapContext, ) { @@ -39,11 +41,18 @@ impl Stage for WriteSurfaceBufferStage { Head::Headless(buffered_texture) => { let buffered_texture = buffered_texture.clone(); + let device = device.clone(); + let current_frame = self.frame; + task::block_in_place(|| { Handle::current().block_on(async { - buffered_texture.create_png("test.png").await; + buffered_texture + .create_png(&device, format!("frame_{}.png", current_frame).as_str()) + .await; }) }); + + self.frame += 1; } } } From ae36e4f7d420d539afd3063a8eba2ce06e5443fb Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Thu, 26 May 2022 14:27:27 +0200 Subject: [PATCH 04/31] Rename HttpClient --- maplibre-demo/src/main.rs | 4 ++-- maplibre-winit/src/winit/mod.rs | 4 ++-- maplibre/src/io/source_client.rs | 10 +++++----- maplibre/src/lib.rs | 6 +++--- maplibre/src/map_schedule.rs | 6 +++--- maplibre/src/platform/noweb/http_client.rs | 4 ++-- maplibre/src/window.rs | 4 ++-- web/src/platform/http_client.rs | 4 ++-- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index 0d14fe5a..1156786d 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -1,7 +1,7 @@ use maplibre::error::Error; use maplibre::io::scheduler::ScheduleMethod; -use maplibre::io::source_client::HTTPClient; -use maplibre::map_schedule::MapSchedule; +use maplibre::io::source_client::{HttpClient, HttpSourceClient}; +use maplibre::map_schedule::{EventuallyMapContext, MapSchedule}; use maplibre::platform::http_client::ReqwestHttpClient; use maplibre::platform::run_multithreaded; use maplibre::platform::schedule_method::TokioScheduleMethod; diff --git a/maplibre-winit/src/winit/mod.rs b/maplibre-winit/src/winit/mod.rs index b3dbbf07..48313d0e 100644 --- a/maplibre-winit/src/winit/mod.rs +++ b/maplibre-winit/src/winit/mod.rs @@ -1,7 +1,7 @@ use instant::Instant; use maplibre::error::Error; use maplibre::io::scheduler::ScheduleMethod; -use maplibre::io::source_client::HTTPClient; +use maplibre::io::source_client::HttpClient; use std::borrow::BorrowMut; use winit::event::{ElementState, KeyboardInput, VirtualKeyCode, WindowEvent}; use winit::event_loop::ControlFlow; @@ -69,7 +69,7 @@ impl Runnable for WinitMapWindow where MWC: MapWindowConfig, SM: ScheduleMethod, - HC: HTTPClient, + HC: HttpClient, { fn run(mut self, mut map_state: MapSchedule, max_frames: Option) { let mut last_render_time = Instant::now(); diff --git a/maplibre/src/io/source_client.rs b/maplibre/src/io/source_client.rs index 44662bf9..6b094938 100644 --- a/maplibre/src/io/source_client.rs +++ b/maplibre/src/io/source_client.rs @@ -16,7 +16,7 @@ pub type HTTPClientFactory = dyn Fn() -> HC; /// the future "no-thread-safe-futures". Tokio futures are thread-safe. #[cfg_attr(feature = "no-thread-safe-futures", async_trait(?Send))] #[cfg_attr(not(feature = "no-thread-safe-futures"), async_trait)] -pub trait HTTPClient: Clone + Sync + Send + 'static { +pub trait HttpClient: Clone + Sync + Send + 'static { async fn fetch(&self, url: &str) -> Result, Error>; } @@ -25,7 +25,7 @@ pub trait HTTPClient: Clone + Sync + Send + 'static { #[derive(Clone)] pub struct HttpSourceClient where - HC: HTTPClient, + HC: HttpClient, { inner_client: HC, } @@ -35,7 +35,7 @@ where #[derive(Clone)] pub enum SourceClient where - HC: HTTPClient, + HC: HttpClient, { Http(HttpSourceClient), Mbtiles { @@ -45,7 +45,7 @@ where impl SourceClient where - HC: HTTPClient, + HC: HttpClient, { pub async fn fetch(&self, coords: &WorldTileCoords) -> Result, Error> { match self { @@ -57,7 +57,7 @@ where impl HttpSourceClient where - HC: HTTPClient, + HC: HttpClient, { pub fn new(http_client: HC) -> Self { Self { diff --git a/maplibre/src/lib.rs b/maplibre/src/lib.rs index 5a3349f4..060e1733 100644 --- a/maplibre/src/lib.rs +++ b/maplibre/src/lib.rs @@ -17,7 +17,7 @@ //! ``` use crate::io::scheduler::{ScheduleMethod, Scheduler}; -use crate::io::source_client::HTTPClient; +use crate::io::source_client::HttpClient; use crate::map_schedule::MapSchedule; use crate::render::settings::{RendererSettings, WgpuSettings}; use crate::render::{RenderState, Renderer}; @@ -96,7 +96,7 @@ pub struct UninitializedMap where MWC: MapWindowConfig, SM: ScheduleMethod, - HC: HTTPClient, + HC: HttpClient, { scheduler: Scheduler, http_client: HC, @@ -111,7 +111,7 @@ impl UninitializedMap where MWC: MapWindowConfig, SM: ScheduleMethod, - HC: HTTPClient, + HC: HttpClient, { /// Initializes the whole rendering pipeline for the given configuration. /// Returns the initialized map, ready to be run. diff --git a/maplibre/src/map_schedule.rs b/maplibre/src/map_schedule.rs index abeedb24..0abf6b26 100644 --- a/maplibre/src/map_schedule.rs +++ b/maplibre/src/map_schedule.rs @@ -5,7 +5,7 @@ use crate::error::Error; use crate::io::geometry_index::GeometryIndex; use crate::io::scheduler::Scheduler; use crate::io::shared_thread_state::SharedThreadState; -use crate::io::source_client::{HTTPClient, HttpSourceClient, SourceClient}; +use crate::io::source_client::{HttpClient, HttpSourceClient, SourceClient}; use crate::io::tile_cache::TileCache; use crate::io::tile_request_state::TileRequestState; use crate::io::TessellateMessage; @@ -80,7 +80,7 @@ pub struct MapSchedule where MWC: MapWindowConfig, SM: ScheduleMethod, - HC: HTTPClient, + HC: HttpClient, { map_window_config: MWC, @@ -98,7 +98,7 @@ impl MapSchedule where MWC: MapWindowConfig, SM: ScheduleMethod, - HC: HTTPClient, + HC: HttpClient, { pub fn new( map_window_config: MWC, diff --git a/maplibre/src/platform/noweb/http_client.rs b/maplibre/src/platform/noweb/http_client.rs index aa2d43c6..d6ba09b7 100644 --- a/maplibre/src/platform/noweb/http_client.rs +++ b/maplibre/src/platform/noweb/http_client.rs @@ -1,5 +1,5 @@ use crate::error::Error; -use crate::HTTPClient; +use crate::HttpClient; use async_trait::async_trait; use reqwest::{Client, StatusCode}; use reqwest_middleware::ClientWithMiddleware; @@ -41,7 +41,7 @@ impl ReqwestHttpClient { } #[async_trait] -impl HTTPClient for ReqwestHttpClient { +impl HttpClient for ReqwestHttpClient { async fn fetch(&self, url: &str) -> Result, Error> { let response = self.client.get(url).send().await?; match response.error_for_status() { diff --git a/maplibre/src/window.rs b/maplibre/src/window.rs index 784859e4..9e467d7a 100644 --- a/maplibre/src/window.rs +++ b/maplibre/src/window.rs @@ -1,6 +1,6 @@ //! Utilities for the window system. -use crate::{HTTPClient, MapSchedule, ScheduleMethod}; +use crate::{HttpClient, MapSchedule, ScheduleMethod}; use raw_window_handle::{HasRawWindowHandle, RawWindowHandle}; /// Window with a [carte::window::WindowSize]. @@ -25,7 +25,7 @@ pub trait Runnable where MWC: MapWindowConfig, SM: ScheduleMethod, - HC: HTTPClient, + HC: HttpClient, { fn run(self, map_state: MapSchedule, max_frames: Option); } diff --git a/web/src/platform/http_client.rs b/web/src/platform/http_client.rs index 39ea12ea..d503e8ee 100644 --- a/web/src/platform/http_client.rs +++ b/web/src/platform/http_client.rs @@ -1,5 +1,5 @@ use js_sys::{ArrayBuffer, Uint8Array}; -use maplibre::io::source_client::HTTPClient; +use maplibre::io::source_client::HttpClient; use wasm_bindgen::prelude::*; use wasm_bindgen::JsCast; use wasm_bindgen_futures::JsFuture; @@ -61,7 +61,7 @@ impl Clone for WHATWGFetchHttpClient { } #[async_trait(?Send)] -impl HTTPClient for WHATWGFetchHttpClient { +impl HttpClient for WHATWGFetchHttpClient { async fn fetch(&self, url: &str) -> Result, Error> { self.fetch_bytes(url) .await From 41d0b43755659bcaff5201173b8dab28c312d13c Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Thu, 26 May 2022 14:27:38 +0200 Subject: [PATCH 05/31] Ignore wasm-pack output --- .idea/maplibre-rs.iml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.idea/maplibre-rs.iml b/.idea/maplibre-rs.iml index cea86611..339ac96a 100644 --- a/.idea/maplibre-rs.iml +++ b/.idea/maplibre-rs.iml @@ -20,9 +20,10 @@ + - \ No newline at end of file + From 675e62bae12b7ba7d44c36503e9d045b2c6fc32c Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Thu, 26 May 2022 14:28:03 +0200 Subject: [PATCH 06/31] Refactor stages and context --- maplibre/src/context.rs | 4 -- maplibre/src/map_schedule.rs | 44 ++++---------- maplibre/src/stages/mod.rs | 37 ++++++++++-- .../src/stages/populate_tile_store_stage.rs | 30 ++++++---- maplibre/src/stages/request_stage.rs | 58 ++++++++----------- 5 files changed, 84 insertions(+), 89 deletions(-) diff --git a/maplibre/src/context.rs b/maplibre/src/context.rs index 96b77630..06a2d545 100644 --- a/maplibre/src/context.rs +++ b/maplibre/src/context.rs @@ -63,8 +63,4 @@ pub struct MapContext { pub tile_cache: TileCache, pub renderer: Renderer, - pub scheduler: Box, - - pub message_receiver: mpsc::Receiver, - pub shared_thread_state: SharedThreadState, } diff --git a/maplibre/src/map_schedule.rs b/maplibre/src/map_schedule.rs index 0abf6b26..d5c18516 100644 --- a/maplibre/src/map_schedule.rs +++ b/maplibre/src/map_schedule.rs @@ -26,10 +26,6 @@ pub struct PrematureMapContext { style: Style, tile_cache: TileCache, - scheduler: Box, - - message_receiver: mpsc::Receiver, - shared_thread_state: SharedThreadState, wgpu_settings: WgpuSettings, renderer_settings: RendererSettings, @@ -38,12 +34,12 @@ pub struct PrematureMapContext { pub enum EventuallyMapContext { Full(MapContext), Premature(PrematureMapContext), - Empty, + _Uninitialized, } impl EventuallyMapContext { pub fn make_full(&mut self, renderer: Renderer) { - let context = mem::replace(self, EventuallyMapContext::Empty); + let context = mem::replace(self, EventuallyMapContext::_Uninitialized); match context { EventuallyMapContext::Full(_) => {} @@ -51,11 +47,7 @@ impl EventuallyMapContext { view_state, style, tile_cache, - scheduler, - message_receiver, - shared_thread_state, - wgpu_settings, - renderer_settings, + .. }) => { mem::replace( self, @@ -64,13 +56,10 @@ impl EventuallyMapContext { style, tile_cache, renderer, - scheduler, - message_receiver, - shared_thread_state, }), ); } - EventuallyMapContext::Empty => {} + EventuallyMapContext::_Uninitialized => {} } } } @@ -84,7 +73,7 @@ where { map_window_config: MWC, - map_context: EventuallyMapContext, + pub map_context: EventuallyMapContext, schedule: Schedule, @@ -112,20 +101,13 @@ where ) -> Self { let view_state = ViewState::new(&window_size); let tile_cache = TileCache::new(); - let mut schedule = Schedule::default(); - let client: SourceClient = SourceClient::Http(HttpSourceClient::new(http_client)); - register_stages(&mut schedule, client); + + let http_source_client: HttpSourceClient = HttpSourceClient::new(http_client); + //register_stages(&mut schedule, http_source_client, Box::new(scheduler)); + register_render_stages(&mut schedule); - let (message_sender, message_receiver) = mpsc::channel(); - - let scheduler = Box::new(scheduler.take()); - let shared_thread_state = SharedThreadState { - tile_request_state: Arc::new(Mutex::new(TileRequestState::new())), - message_sender, - geometry_index: Arc::new(Mutex::new(GeometryIndex::new())), - }; Self { map_window_config, map_context: match renderer { @@ -133,10 +115,7 @@ where view_state, style, tile_cache, - scheduler, - shared_thread_state, wgpu_settings, - message_receiver, renderer_settings, }), Some(renderer) => EventuallyMapContext::Full(MapContext { @@ -144,9 +123,6 @@ where style, tile_cache, renderer, - scheduler, - shared_thread_state, - message_receiver, }), }, schedule, @@ -226,7 +202,7 @@ where &self.map_context.make_full(renderer); true } - EventuallyMapContext::Empty => false, + EventuallyMapContext::_Uninitialized => false, } } diff --git a/maplibre/src/stages/mod.rs b/maplibre/src/stages/mod.rs index 03073823..56ad24a7 100644 --- a/maplibre/src/stages/mod.rs +++ b/maplibre/src/stages/mod.rs @@ -1,15 +1,42 @@ //! [Stages](Stage) for requesting and preparing data -use crate::io::source_client::SourceClient; +use crate::io::geometry_index::GeometryIndex; +use crate::io::shared_thread_state::SharedThreadState; +use crate::io::source_client::{HttpSourceClient, SourceClient}; +use crate::io::tile_request_state::TileRequestState; +use crate::io::TessellateMessage; use crate::schedule::Schedule; use crate::stages::populate_tile_store_stage::PopulateTileStore; -use crate::HTTPClient; +use crate::{HttpClient, ScheduleMethod, Scheduler}; use request_stage::RequestStage; +use std::sync::{mpsc, Arc, Mutex}; mod populate_tile_store_stage; mod request_stage; -pub fn register_stages(schedule: &mut Schedule, source_client: SourceClient) { - schedule.add_stage("request", RequestStage::new(source_client)); - schedule.add_stage("populate_tile_store", PopulateTileStore::default()); +pub type MessageSender = mpsc::Sender; +pub type MessageReceiver = mpsc::Receiver; + +pub fn register_stages( + schedule: &mut Schedule, + http_source_client: HttpSourceClient, + scheduler: Box>, +) { + let (message_sender, message_receiver): (MessageSender, MessageReceiver) = mpsc::channel(); + let shared_thread_state = SharedThreadState { + tile_request_state: Arc::new(Mutex::new(TileRequestState::new())), + message_sender, + geometry_index: Arc::new(Mutex::new(GeometryIndex::new())), + }; + + let scheduler = Box::new(scheduler.take()); + + schedule.add_stage( + "request", + RequestStage::new(shared_thread_state.clone(), http_source_client, scheduler), + ); + schedule.add_stage( + "populate_tile_store", + PopulateTileStore::new(shared_thread_state, message_receiver), + ); } diff --git a/maplibre/src/stages/populate_tile_store_stage.rs b/maplibre/src/stages/populate_tile_store_stage.rs index 86b73560..4f6df7f0 100644 --- a/maplibre/src/stages/populate_tile_store_stage.rs +++ b/maplibre/src/stages/populate_tile_store_stage.rs @@ -1,23 +1,29 @@ //! Receives data from async threads and populates the [`crate::io::tile_cache::TileCache`]. use crate::context::MapContext; +use crate::io::shared_thread_state::SharedThreadState; use crate::io::{TessellateMessage, TileTessellateMessage}; use crate::schedule::Stage; +use crate::stages::MessageReceiver; +use std::sync::mpsc; -#[derive(Default)] -pub struct PopulateTileStore {} +pub struct PopulateTileStore { + shared_thread_state: SharedThreadState, + message_receiver: MessageReceiver, +} -impl Stage for PopulateTileStore { - fn run( - &mut self, - MapContext { - tile_cache, +impl PopulateTileStore { + pub fn new(shared_thread_state: SharedThreadState, message_receiver: MessageReceiver) -> Self { + Self { shared_thread_state, message_receiver, - .. - }: &mut MapContext, - ) { - if let Ok(result) = message_receiver.try_recv() { + } + } +} + +impl Stage for PopulateTileStore { + fn run(&mut self, MapContext { tile_cache, .. }: &mut MapContext) { + if let Ok(result) = self.message_receiver.try_recv() { match result { TessellateMessage::Layer(layer_result) => { tracing::trace!( @@ -29,7 +35,7 @@ impl Stage for PopulateTileStore { } TessellateMessage::Tile(TileTessellateMessage { request_id, coords }) => loop { if let Ok(mut tile_request_state) = - shared_thread_state.tile_request_state.try_lock() + self.shared_thread_state.tile_request_state.try_lock() { tile_request_state.finish_tile_request(request_id); tracing::trace!("Tile at {} finished loading", coords); diff --git a/maplibre/src/stages/request_stage.rs b/maplibre/src/stages/request_stage.rs index 3f10855b..b64bca8f 100644 --- a/maplibre/src/stages/request_stage.rs +++ b/maplibre/src/stages/request_stage.rs @@ -4,28 +4,36 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, WorldTileCoords}; use crate::error::Error; use crate::io::shared_thread_state::SharedThreadState; -use crate::io::source_client::SourceClient; +use crate::io::source_client::{HttpSourceClient, SourceClient}; use crate::io::tile_cache::TileCache; use crate::io::TileRequest; use crate::schedule::Stage; -use crate::{HTTPClient, ScheduleMethod, Style}; +use crate::{HttpClient, ScheduleMethod, Style}; use std::collections::HashSet; pub struct RequestStage where - HC: HTTPClient, + HC: HttpClient, { - pub source_client: SourceClient, - pub try_failed: bool, + shared_thread_state: SharedThreadState, + scheduler: Box, + http_source_client: HttpSourceClient, + try_failed: bool, } impl RequestStage where - HC: HTTPClient, + HC: HttpClient, { - pub fn new(source_client: SourceClient) -> Self { + pub fn new( + shared_thread_state: SharedThreadState, + http_source_client: HttpSourceClient, + scheduler: Box, + ) -> Self { Self { - source_client, + shared_thread_state, + scheduler, + http_source_client, try_failed: false, } } @@ -33,7 +41,7 @@ where impl Stage for RequestStage where - HC: HTTPClient, + HC: HttpClient, { fn run( &mut self, @@ -41,8 +49,6 @@ where view_state, style, tile_cache, - scheduler, - shared_thread_state, .. }: &mut MapContext, ) { @@ -59,13 +65,7 @@ where { if let Some(view_region) = &view_region { // FIXME: We also need to request tiles from layers above if we are over the maximum zoom level - self.try_failed = self.request_tiles_in_view( - tile_cache, - style, - shared_thread_state, - scheduler, - view_region, - ); + self.try_failed = self.request_tiles_in_view(tile_cache, style, view_region); } } @@ -76,7 +76,7 @@ where impl RequestStage where - HC: HTTPClient, + HC: HttpClient, { /// Request tiles which are currently in view. #[tracing::instrument(skip_all)] @@ -84,8 +84,6 @@ where &self, tile_cache: &TileCache, style: &Style, - shared_thread_state: &SharedThreadState, - scheduler: &Box, view_region: &ViewRegion, ) -> bool { let mut try_failed = false; @@ -99,13 +97,7 @@ where if coords.build_quad_key().is_some() { // TODO: Make tesselation depend on style? try_failed = self - .try_request_tile( - tile_cache, - shared_thread_state, - scheduler, - &coords, - &source_layers, - ) + .try_request_tile(tile_cache, &coords, &source_layers) .unwrap(); } } @@ -115,8 +107,6 @@ where fn try_request_tile( &self, tile_cache: &TileCache, - shared_thread_state: &SharedThreadState, - scheduler: &Box, coords: &WorldTileCoords, layers: &HashSet, ) -> Result { @@ -124,7 +114,7 @@ where return Ok(false); } - if let Ok(mut tile_request_state) = shared_thread_state.tile_request_state.try_lock() { + if let Ok(mut tile_request_state) = self.shared_thread_state.tile_request_state.try_lock() { if let Some(request_id) = tile_request_state.start_tile_request(TileRequest { coords: *coords, layers: layers.clone(), @@ -141,12 +131,12 @@ where ); }*/ - let client = self.source_client.clone(); + let client = SourceClient::Http(self.http_source_client.clone()); let coords = *coords; - scheduler + self.scheduler .schedule( - shared_thread_state.clone(), + self.shared_thread_state.clone(), Box::new(move |state: SharedThreadState| { Box::pin(async move { match client.fetch(&coords).await { From 810954f49dadfeaba1139ca211292640b60c6233 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Thu, 26 May 2022 14:28:35 +0200 Subject: [PATCH 07/31] Rename runnable to eventloop and fix variable names --- maplibre-demo/src/main.rs | 58 ++++++++++++++++++------------- maplibre-winit/src/winit/mod.rs | 22 ++++++------ maplibre-winit/src/winit/noweb.rs | 1 - maplibre/src/lib.rs | 47 ++++++++++++++++--------- maplibre/src/window.rs | 5 ++- 5 files changed, 77 insertions(+), 56 deletions(-) diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index 1156786d..822e6b6e 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -1,3 +1,4 @@ +use maplibre::coords::WorldTileCoords; use maplibre::error::Error; use maplibre::io::scheduler::ScheduleMethod; use maplibre::io::source_client::{HttpClient, HttpSourceClient}; @@ -6,7 +7,7 @@ use maplibre::platform::http_client::ReqwestHttpClient; use maplibre::platform::run_multithreaded; use maplibre::platform::schedule_method::TokioScheduleMethod; use maplibre::render::settings::RendererSettings; -use maplibre::window::{MapWindow, MapWindowConfig, Runnable, WindowSize}; +use maplibre::window::{EventLoop, MapWindow, MapWindowConfig, WindowSize}; use maplibre::MapBuilder; use maplibre_winit::winit::{WinitEventLoop, WinitMapWindow, WinitMapWindowConfig, WinitWindow}; use wgpu::TextureFormat; @@ -38,26 +39,6 @@ impl MapWindow for HeadlessMapWindow { } } -impl Runnable for HeadlessMapWindow -where - MWC: MapWindowConfig, - SM: ScheduleMethod, - HC: HTTPClient, -{ - fn run(mut self, mut map_state: MapSchedule, max_frames: Option) { - for i in 0..3 { - match map_state.update_and_redraw() { - Ok(_) => {} - Err(Error::Render(e)) => { - eprintln!("{}", e); - if e.should_exit() {} - } - e => eprintln!("{:?}", e), - }; - } - } -} - fn run_in_window() { run_multithreaded(async { MapBuilder::new() @@ -73,7 +54,7 @@ fn run_in_window() { fn run_headless() { run_multithreaded(async { - MapBuilder::new() + let mut map = MapBuilder::new() .with_map_window_config(HeadlessMapWindowConfig) .with_http_client(ReqwestHttpClient::new(None)) .with_schedule_method(TokioScheduleMethod::new()) @@ -83,8 +64,37 @@ fn run_headless() { }) .build() .initialize_headless() - .await - .run() + .await; + + let http_source_client: HttpSourceClient = + HttpSourceClient::new(ReqwestHttpClient::new(None)); + + let coords = WorldTileCoords::from((0, 0, 0)); + let request_id = 0; + + let x = match http_source_client.fetch(&coords).await { + Ok(data) => state.process_tile(0, data.into_boxed_slice()).unwrap(), + Err(e) => { + log::error!("{:?}", &e); + + state.tile_unavailable(&coords, request_id).unwrap() + } + }; + + match map.map_schedule_mut().map_context { + EventuallyMapContext::Full(a) => a.tile_cache.put_tessellated_layer(), + EventuallyMapContext::Premature(_) => {} + EventuallyMapContext::_Uninitialized => {} + } + + match map.map_schedule_mut().update_and_redraw() { + Ok(_) => {} + Err(Error::Render(e)) => { + eprintln!("{}", e); + if e.should_exit() {} + } + e => eprintln!("{:?}", e), + }; }) } diff --git a/maplibre-winit/src/winit/mod.rs b/maplibre-winit/src/winit/mod.rs index 48313d0e..52c089b5 100644 --- a/maplibre-winit/src/winit/mod.rs +++ b/maplibre-winit/src/winit/mod.rs @@ -8,7 +8,7 @@ use winit::event_loop::ControlFlow; use crate::input::{InputController, UpdateState}; use maplibre::map_schedule::MapSchedule; -use maplibre::window::{HeadedMapWindow, MapWindow, MapWindowConfig, Runnable}; +use maplibre::window::{EventLoop, HeadedMapWindow, MapWindow, MapWindowConfig}; use winit::event::Event; #[cfg(target_arch = "wasm32")] @@ -65,13 +65,13 @@ impl WinitMapWindow { ///* Input (Mouse/Keyboard) ///* Platform Events like suspend/resume ///* Render a new frame -impl Runnable for WinitMapWindow +impl EventLoop for WinitMapWindow where MWC: MapWindowConfig, SM: ScheduleMethod, HC: HttpClient, { - fn run(mut self, mut map_state: MapSchedule, max_frames: Option) { + fn run(mut self, mut map_schedule: MapSchedule, max_frames: Option) { let mut last_render_time = Instant::now(); let mut current_frame: u64 = 0; @@ -81,13 +81,13 @@ where .unwrap() .run(move |event, _, control_flow| { #[cfg(target_os = "android")] - if !map_state.is_initialized() && event == Event::Resumed { + if !map_schedule.is_initialized() && event == Event::Resumed { use tokio::runtime::Handle; use tokio::task; let state = task::block_in_place(|| { Handle::current().block_on(async { - map_state.late_init().await; + map_schedule.late_init().await; }) }); return; @@ -118,10 +118,10 @@ where .. } => *control_flow = ControlFlow::Exit, WindowEvent::Resized(physical_size) => { - map_state.resize(physical_size.width, physical_size.height); + map_schedule.resize(physical_size.width, physical_size.height); } WindowEvent::ScaleFactorChanged { new_inner_size, .. } => { - map_state.resize(new_inner_size.width, new_inner_size.height); + map_schedule.resize(new_inner_size.width, new_inner_size.height); } _ => {} } @@ -133,10 +133,10 @@ where last_render_time = now; { - input_controller.update_state(map_state.view_state_mut(), dt); + input_controller.update_state(map_schedule.view_state_mut(), dt); } - match map_state.update_and_redraw() { + match map_schedule.update_and_redraw() { Ok(_) => {} Err(Error::Render(e)) => { eprintln!("{}", e); @@ -157,10 +157,10 @@ where } } Event::Suspended => { - map_state.suspend(); + map_schedule.suspend(); } Event::Resumed => { - map_state.resume(&self); + map_schedule.resume(&self); } Event::MainEventsCleared => { // RedrawRequested will only trigger once, unless we manually diff --git a/maplibre-winit/src/winit/noweb.rs b/maplibre-winit/src/winit/noweb.rs index 9457745b..d2f9471c 100644 --- a/maplibre-winit/src/winit/noweb.rs +++ b/maplibre-winit/src/winit/noweb.rs @@ -28,7 +28,6 @@ impl MapWindow for WinitMapWindow { } } impl HeadedMapWindow for WinitMapWindow { - type EventLoop = WinitEventLoop; type RawWindow = WinitWindow; fn inner(&self) -> &Self::RawWindow { diff --git a/maplibre/src/lib.rs b/maplibre/src/lib.rs index 060e1733..e5258bea 100644 --- a/maplibre/src/lib.rs +++ b/maplibre/src/lib.rs @@ -22,7 +22,7 @@ use crate::map_schedule::MapSchedule; use crate::render::settings::{RendererSettings, WgpuSettings}; use crate::render::{RenderState, Renderer}; use crate::style::Style; -use crate::window::{HeadedMapWindow, MapWindow, MapWindowConfig, Runnable, WindowSize}; +use crate::window::{EventLoop, HeadedMapWindow, MapWindow, MapWindowConfig, WindowSize}; pub mod context; pub mod coords; @@ -51,30 +51,35 @@ pub struct Map where MWC: MapWindowConfig, SM: ScheduleMethod, - HC: HTTPClient, + HC: HttpClient, { - map_state: MapSchedule, + map_schedule: MapSchedule, window: MWC::MapWindow, } impl Map where MWC: MapWindowConfig, - MWC::MapWindow: Runnable, SM: ScheduleMethod, - HC: HTTPClient, + HC: HttpClient, { - /// Starts the [`crate::map_state::MapState`] Runnable with the configured event loop. - pub fn run(self) { + /// Starts the [`crate::map_schedule::MapState`] Runnable with the configured event loop. + pub fn run(self) + where + MWC::MapWindow: EventLoop, + { self.run_with_optionally_max_frames(None); } - /// Starts the [`crate::map_state::MapState`] Runnable with the configured event loop. + /// Starts the [`crate::map_schedule::MapState`] Runnable with the configured event loop. /// /// # Arguments /// /// * `max_frames` - Maximum number of frames per second. - pub fn run_with_max_frames(self, max_frames: u64) { + pub fn run_with_max_frames(self, max_frames: u64) + where + MWC::MapWindow: EventLoop, + { self.run_with_optionally_max_frames(Some(max_frames)); } @@ -83,15 +88,23 @@ where /// # Arguments /// /// * `max_frames` - Optional maximum number of frames per second. - pub fn run_with_optionally_max_frames(self, max_frames: Option) { - self.window.run(self.map_state, max_frames); + pub fn run_with_optionally_max_frames(self, max_frames: Option) + where + MWC::MapWindow: EventLoop, + { + self.window.run(self.map_schedule, max_frames); + } + + pub fn map_schedule(&self) -> &MapSchedule { + &self.map_schedule + } + + pub fn map_schedule_mut(&mut self) -> &mut MapSchedule { + &mut self.map_schedule } } /// Stores the map configuration before the map's state has been fully initialized. -/// -/// FIXME: We could maybe remove this class, and store the render_state in an Optional in [`crate::map_state::MapState`]. -/// FIXME: I think we can find a workaround so that this class doesn't exist. pub struct UninitializedMap where MWC: MapWindowConfig, @@ -134,7 +147,7 @@ where .await .ok(); Map { - map_state: MapSchedule::new( + map_schedule: MapSchedule::new( self.map_window_config, window_size, renderer, @@ -160,7 +173,7 @@ where .await .ok(); Map { - map_state: MapSchedule::new( + map_schedule: MapSchedule::new( self.map_window_config, window_size, renderer, @@ -193,7 +206,7 @@ impl MapBuilder where MWC: MapWindowConfig, SM: ScheduleMethod, - HC: HTTPClient, + HC: HttpClient, { pub fn new() -> Self { Self { diff --git a/maplibre/src/window.rs b/maplibre/src/window.rs index 9e467d7a..867b8e45 100644 --- a/maplibre/src/window.rs +++ b/maplibre/src/window.rs @@ -9,7 +9,6 @@ pub trait MapWindow { } pub trait HeadedMapWindow: MapWindow { - type EventLoop; type RawWindow: HasRawWindowHandle; fn inner(&self) -> &Self::RawWindow; @@ -21,13 +20,13 @@ pub trait MapWindowConfig: 'static { fn create(&self) -> Self::MapWindow; } -pub trait Runnable +pub trait EventLoop where MWC: MapWindowConfig, SM: ScheduleMethod, HC: HttpClient, { - fn run(self, map_state: MapSchedule, max_frames: Option); + fn run(self, map_schedule: MapSchedule, max_frames: Option); } /// Window size with a width and an height in pixels. From 40db69c864a96e8ca501b8513515b4df6228742f Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 27 May 2022 11:22:14 +0200 Subject: [PATCH 08/31] Fix tests --- maplibre-demo/src/main.rs | 6 +++--- maplibre/src/render/graph/graph.rs | 21 ++++++++++----------- maplibre/src/render/mod.rs | 3 ++- maplibre/src/render/resource/buffer_pool.rs | 7 +++---- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index 822e6b6e..f5a43265 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -66,13 +66,13 @@ fn run_headless() { .initialize_headless() .await; - let http_source_client: HttpSourceClient = + let http_source_client: HttpSourceClient = HttpSourceClient::new(ReqwestHttpClient::new(None)); let coords = WorldTileCoords::from((0, 0, 0)); let request_id = 0; - let x = match http_source_client.fetch(&coords).await { + /* let x = match http_source_client.fetch(&coords).await { Ok(data) => state.process_tile(0, data.into_boxed_slice()).unwrap(), Err(e) => { log::error!("{:?}", &e); @@ -85,7 +85,7 @@ fn run_headless() { EventuallyMapContext::Full(a) => a.tile_cache.put_tessellated_layer(), EventuallyMapContext::Premature(_) => {} EventuallyMapContext::_Uninitialized => {} - } + }*/ match map.map_schedule_mut().update_and_redraw() { Ok(_) => {} diff --git a/maplibre/src/render/graph/graph.rs b/maplibre/src/render/graph/graph.rs index d674f09c..7e8ead00 100644 --- a/maplibre/src/render/graph/graph.rs +++ b/maplibre/src/render/graph/graph.rs @@ -27,7 +27,8 @@ use super::EdgeExistence; /// /// ## Example /// Here is a simple render graph example with two nodes connected by a node edge. -/// ```ignore +/// ``` +/// # /// # use maplibre::render::graph::{Node, NodeRunError, RenderContext, RenderGraph, RenderGraphContext}; /// # use maplibre::render::{RenderState}; /// # struct MyNode; @@ -574,10 +575,8 @@ mod tests { Edge, Node, NodeId, NodeRunError, RenderGraph, RenderGraphContext, RenderGraphError, SlotInfo, }; - use crate::render::{ - graph::{RenderContext, SlotType}, - RenderState, - }; + use crate::render::graph::{RenderContext, SlotType}; + use crate::RenderState; use std::collections::HashSet; #[derive(Debug)] @@ -610,9 +609,9 @@ mod tests { fn run( &self, - _: &mut RenderGraphContext, - _: &mut RenderContext, - _: &RenderState, + graph: &mut RenderGraphContext, + render_context: &mut RenderContext, + state: &RenderState, ) -> Result<(), NodeRunError> { Ok(()) } @@ -683,9 +682,9 @@ mod tests { impl Node for MyNode { fn run( &self, - _: &mut RenderGraphContext, - _: &mut RenderContext, - _: &RenderState, + graph: &mut RenderGraphContext, + render_context: &mut RenderContext, + state: &RenderState, ) -> Result<(), NodeRunError> { Ok(()) } diff --git a/maplibre/src/render/mod.rs b/maplibre/src/render/mod.rs index e3328c44..7628a673 100644 --- a/maplibre/src/render/mod.rs +++ b/maplibre/src/render/mod.rs @@ -373,7 +373,8 @@ impl Renderer { mod tests { use crate::render::graph::RenderGraph; use crate::render::graph_runner::RenderGraphRunner; - use crate::render::RenderState; + use crate::render::resource::Surface; + use crate::RenderState; #[cfg(not(target_arch = "wasm32"))] #[tokio::test] diff --git a/maplibre/src/render/resource/buffer_pool.rs b/maplibre/src/render/resource/buffer_pool.rs index c5d2624d..da4082fc 100644 --- a/maplibre/src/render/resource/buffer_pool.rs +++ b/maplibre/src/render/resource/buffer_pool.rs @@ -44,7 +44,7 @@ pub struct BufferPool { } #[derive(Debug)] -enum BackingBufferType { +pub enum BackingBufferType { Vertices, Indices, Metadata, @@ -581,9 +581,8 @@ mod tests { use crate::style::layer::StyleLayer; use lyon::tessellation::VertexBuffers; - use crate::render::resource::buffer_pool::{ - BackingBufferDescriptor, BackingBufferType, BufferPool, Queue, - }; + use crate::render::resource::buffer_pool::BackingBufferType; + use crate::render::resource::{BackingBufferDescriptor, BufferPool, Queue}; #[derive(Debug)] struct TestBuffer { From 1fd40c2fe9c82172a3b5a39f26bdf371053388d6 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 27 May 2022 11:22:25 +0200 Subject: [PATCH 09/31] Add pipeline draft --- maplibre/src/io/mod.rs | 1 + maplibre/src/io/pipeline.rs | 183 ++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 maplibre/src/io/pipeline.rs diff --git a/maplibre/src/io/mod.rs b/maplibre/src/io/mod.rs index d0836514..7310a9a1 100644 --- a/maplibre/src/io/mod.rs +++ b/maplibre/src/io/mod.rs @@ -14,6 +14,7 @@ pub mod source_client; pub mod static_tile_fetcher; pub mod geometry_index; +pub mod pipeline; pub mod shared_thread_state; pub mod tile_cache; pub mod tile_request_state; diff --git a/maplibre/src/io/pipeline.rs b/maplibre/src/io/pipeline.rs new file mode 100644 index 00000000..d812d119 --- /dev/null +++ b/maplibre/src/io/pipeline.rs @@ -0,0 +1,183 @@ +use std::marker::PhantomData; +use std::process::Output; + +pub trait Processable { + type Input; + type Output; + + fn process(&self, input: Self::Input) -> Self::Output; +} + +pub struct PipelineStep +where + P: Processable, + N: Processable, +{ + func: P, + next: N, +} + +impl Processable for PipelineStep +where + P: Processable, + N: Processable, +{ + type Input = P::Input; + type Output = N::Output; + + fn process(&self, input: Self::Input) -> Self::Output { + let output = self.func.process(input); + self.next.process(output) + } +} + +#[derive(Default)] +pub struct EndStep { + phantom: PhantomData, +} + +impl Processable for EndStep { + type Input = I; + type Output = I; + + fn process(&self, input: Self::Input) -> Self::Output { + input + } +} + +impl Processable for &fn(I) -> O { + type Input = I; + type Output = O; + + fn process(&self, input: Self::Input) -> Self::Output { + (self)(input) + } +} + +impl Processable for fn(I) -> O { + type Input = I; + type Output = O; + + fn process(&self, input: Self::Input) -> Self::Output { + (self)(input) + } +} + +pub struct FnProcessable { + func: &'static fn(I) -> O, +} + +impl Processable for FnProcessable { + type Input = I; + type Output = O; + + fn process(&self, input: Self::Input) -> Self::Output { + (self.func)(input) + } +} + +pub struct ClosureProcessable +where + F: Fn(I) -> O, +{ + func: F, + phantom_i: PhantomData, + phantom_o: PhantomData, +} + +impl Processable for ClosureProcessable +where + F: Fn(I) -> O, +{ + type Input = I; + type Output = O; + + fn process(&self, input: Self::Input) -> Self::Output { + (self.func)(input) + } +} + +pub struct Closure2Processable { + func: fn(I) -> O, +} + +impl Processable for Closure2Processable { + type Input = I; + type Output = O; + + fn process(&self, input: Self::Input) -> Self::Output { + (self.func)(input) + } +} + +impl Processable for dyn Fn(I) -> O { + type Input = I; + type Output = O; + + fn process(&self, input: Self::Input) -> Self::Output { + (self)(input) + } +} + +#[cfg(test)] +mod tests { + use crate::io::pipeline::{ + Closure2Processable, ClosureProcessable, EndStep, FnProcessable, PipelineStep, Processable, + }; + + fn add_one(input: u32) -> u8 { + input as u8 + 1 + } + + fn add_two(input: u8) -> u32 { + input as u32 + 2 + } + + #[test] + fn test() { + let output: u32 = PipelineStep { + func: FnProcessable { + func: &(add_two as fn(u8) -> u32), + }, + next: EndStep::default(), + } + .process(5u8); + + assert_eq!(output, 7); + + let output = PipelineStep { + func: &(add_one as fn(u32) -> u8), + next: PipelineStep { + func: &(add_two as fn(u8) -> u32), + next: EndStep::default(), + }, + } + .process(5); + + assert_eq!(output, 8); + + let output: u32 = PipelineStep { + func: ClosureProcessable { + func: |input: u8| -> u32 { + return input as u32 + 2; + }, + phantom_i: Default::default(), + phantom_o: Default::default(), + }, + next: EndStep::default(), + } + .process(5u8); + + assert_eq!(output, 7); + + let output: u32 = PipelineStep { + func: Closure2Processable { + func: |input: u8| -> u32 { input as u32 + 2 }, + }, + next: EndStep::default(), + } + .process(5u8); + + assert_eq!(output, 7); + } +} From cdb7f1395ecc3ad69909ce32a1ccdcce7098a118 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Fri, 27 May 2022 12:13:40 +0200 Subject: [PATCH 10/31] Start implementing pipeline steps --- maplibre/src/io/pipeline.rs | 235 ++++++++++++++++++++++++++++++------ 1 file changed, 197 insertions(+), 38 deletions(-) diff --git a/maplibre/src/io/pipeline.rs b/maplibre/src/io/pipeline.rs index d812d119..3bb416c7 100644 --- a/maplibre/src/io/pipeline.rs +++ b/maplibre/src/io/pipeline.rs @@ -1,11 +1,78 @@ +use crate::coords::WorldTileCoords; +use crate::io::{LayerTessellateMessage, TessellateMessage, TileRequestID, TileTessellateMessage}; +use crate::render::ShaderVertex; +use crate::tessellation::{IndexDataType, OverAlignedVertexBuffer}; +use geozero::mvt::tile; use std::marker::PhantomData; use std::process::Output; +use std::sync::mpsc; + +pub trait PipelineProcessor { + fn finished_tile_tesselation(&mut self, request_id: TileRequestID, coords: &WorldTileCoords); + fn unavailable_layer(&mut self, coords: &WorldTileCoords, layer_name: &str); + fn finished_layer_tesselation( + &mut self, + coords: &WorldTileCoords, + buffer: OverAlignedVertexBuffer, + // Holds for each feature the count of indices. + feature_indices: Vec, + layer_data: tile::Layer, + ); +} + +pub struct HeadedPipelineProcessor { + pub message_sender: mpsc::Sender, +} + +impl PipelineProcessor for HeadedPipelineProcessor { + fn finished_tile_tesselation(&mut self, request_id: TileRequestID, coords: &WorldTileCoords) { + self.message_sender + .send(TessellateMessage::Tile(TileTessellateMessage { + request_id, + coords: *coords, + })) + .unwrap(); + } + + fn unavailable_layer(&mut self, coords: &WorldTileCoords, layer_name: &str) { + self.message_sender.send(TessellateMessage::Layer( + LayerTessellateMessage::UnavailableLayer { + coords: *coords, + layer_name: layer_name.to_owned(), + }, + )); + } + fn finished_layer_tesselation( + &mut self, + coords: &WorldTileCoords, + buffer: OverAlignedVertexBuffer, + feature_indices: Vec, + layer_data: tile::Layer, + ) { + self.message_sender + .send(TessellateMessage::Layer( + LayerTessellateMessage::TessellatedLayer { + coords: *coords, + buffer, + feature_indices, + layer_data, + }, + )) + .unwrap(); + } +} + +pub struct PipelineContext { + pub processor: Box, +} + +impl PipelineContext {} pub trait Processable { type Input; type Output; - fn process(&self, input: Self::Input) -> Self::Output; + fn process(&self, input: Self::Input, context: &mut PipelineContext) -> Self::Output; } pub struct PipelineStep @@ -25,9 +92,9 @@ where type Input = P::Input; type Output = N::Output; - fn process(&self, input: Self::Input) -> Self::Output { - let output = self.func.process(input); - self.next.process(output) + fn process(&self, input: Self::Input, context: &mut PipelineContext) -> Self::Output { + let output = self.func.process(input, context); + self.next.process(output, context) } } @@ -40,45 +107,45 @@ impl Processable for EndStep { type Input = I; type Output = I; - fn process(&self, input: Self::Input) -> Self::Output { + fn process(&self, input: Self::Input, _context: &mut PipelineContext) -> Self::Output { input } } -impl Processable for &fn(I) -> O { +impl Processable for &fn(input: I, context: &mut PipelineContext) -> O { type Input = I; type Output = O; - fn process(&self, input: Self::Input) -> Self::Output { - (self)(input) + fn process(&self, input: Self::Input, context: &mut PipelineContext) -> Self::Output { + (self)(input, context) } } -impl Processable for fn(I) -> O { +impl Processable for fn(input: I, context: &mut PipelineContext) -> O { type Input = I; type Output = O; - fn process(&self, input: Self::Input) -> Self::Output { - (self)(input) + fn process(&self, input: Self::Input, context: &mut PipelineContext) -> Self::Output { + (self)(input, context) } } pub struct FnProcessable { - func: &'static fn(I) -> O, + func: &'static fn(I, context: &mut PipelineContext) -> O, } impl Processable for FnProcessable { type Input = I; type Output = O; - fn process(&self, input: Self::Input) -> Self::Output { - (self.func)(input) + fn process(&self, input: Self::Input, context: &mut PipelineContext) -> Self::Output { + (self.func)(input, context) } } pub struct ClosureProcessable where - F: Fn(I) -> O, + F: Fn(I, &mut PipelineContext) -> O, { func: F, phantom_i: PhantomData, @@ -87,78 +154,170 @@ where impl Processable for ClosureProcessable where - F: Fn(I) -> O, + F: Fn(I, &mut PipelineContext) -> O, { type Input = I; type Output = O; - fn process(&self, input: Self::Input) -> Self::Output { - (self.func)(input) + fn process(&self, input: Self::Input, context: &mut PipelineContext) -> Self::Output { + (self.func)(input, context) } } pub struct Closure2Processable { - func: fn(I) -> O, + func: fn(I, context: &mut PipelineContext) -> O, } impl Processable for Closure2Processable { type Input = I; type Output = O; - fn process(&self, input: Self::Input) -> Self::Output { - (self.func)(input) + fn process(&self, input: Self::Input, context: &mut PipelineContext) -> Self::Output { + (self.func)(input, context) } } -impl Processable for dyn Fn(I) -> O { - type Input = I; - type Output = O; +mod steps { + use crate::io::pipeline::{PipelineContext, Processable}; + use crate::io::{TileRequest, TileRequestID}; + use crate::tessellation::zero_tessellator::ZeroTessellator; + use crate::tessellation::IndexDataType; + use geozero::GeozeroDatasource; + use prost::Message; + use std::collections::HashSet; - fn process(&self, input: Self::Input) -> Self::Output { - (self)(input) + pub struct ParseTileStep {} + + impl Processable for ParseTileStep { + type Input = (TileRequest, TileRequestID, Box<[u8]>); + type Output = (TileRequest, TileRequestID, geozero::mvt::Tile); + + fn process( + &self, + (tile_request, request_id, data): Self::Input, + _context: &mut PipelineContext, + ) -> Self::Output { + let tile = geozero::mvt::Tile::decode(data.as_ref()).expect("failed to load tile"); + (tile_request, request_id, tile) + } + } + + pub struct IndexLayerStep {} + + pub struct TessellateLayerStep {} + + impl Processable for TessellateLayerStep { + type Input = (TileRequest, TileRequestID, geozero::mvt::Tile); + type Output = (); + + fn process( + &self, + (tile_request, request_id, mut tile): Self::Input, + context: &mut PipelineContext, + ) -> Self::Output { + let coords = &tile_request.coords; + + for layer in &mut tile.layers { + let cloned_layer = layer.clone(); + let layer_name: &str = &cloned_layer.name; + if !tile_request.layers.contains(layer_name) { + continue; + } + + tracing::info!("layer {} at {} ready", layer_name, coords); + + let mut tessellator = ZeroTessellator::::default(); + if let Err(e) = layer.process(&mut tessellator) { + context.processor.unavailable_layer(coords, layer_name); + + tracing::error!( + "layer {} at {} tesselation failed {:?}", + layer_name, + &coords, + e + ); + } else { + context.processor.finished_layer_tesselation( + coords, + tessellator.buffer.into(), + tessellator.feature_indices, + cloned_layer, + ) + } + } + + let available_layers: HashSet<_> = tile + .layers + .iter() + .map(|layer| layer.name.clone()) + .collect::>(); + + for missing_layer in tile_request.layers.difference(&available_layers) { + context.processor.unavailable_layer(coords, missing_layer); + + tracing::info!( + "requested layer {} at {} not found in tile", + missing_layer, + &coords + ); + } + + tracing::info!("tile tessellated at {} finished", &tile_request.coords); + + context + .processor + .finished_tile_tesselation(request_id, &tile_request.coords); + } } } #[cfg(test)] mod tests { use crate::io::pipeline::{ - Closure2Processable, ClosureProcessable, EndStep, FnProcessable, PipelineStep, Processable, + Closure2Processable, ClosureProcessable, EndStep, FnProcessable, HeadedPipelineProcessor, + PipelineContext, PipelineStep, Processable, }; + use std::sync::mpsc; - fn add_one(input: u32) -> u8 { + fn add_one(input: u32, context: &mut PipelineContext) -> u8 { input as u8 + 1 } - fn add_two(input: u8) -> u32 { + fn add_two(input: u8, context: &mut PipelineContext) -> u32 { input as u32 + 2 } #[test] fn test() { + let mut context = PipelineContext { + processor: Box::new(HeadedPipelineProcessor { + message_sender: mpsc::channel().0, + }), + }; let output: u32 = PipelineStep { func: FnProcessable { - func: &(add_two as fn(u8) -> u32), + func: &(add_two as fn(u8, &mut PipelineContext) -> u32), }, next: EndStep::default(), } - .process(5u8); + .process(5u8, &mut context); assert_eq!(output, 7); let output = PipelineStep { - func: &(add_one as fn(u32) -> u8), + func: &(add_one as fn(u32, &mut PipelineContext) -> u8), next: PipelineStep { - func: &(add_two as fn(u8) -> u32), + func: &(add_two as fn(u8, &mut PipelineContext) -> u32), next: EndStep::default(), }, } - .process(5); + .process(5u32, &mut context); assert_eq!(output, 8); let output: u32 = PipelineStep { func: ClosureProcessable { - func: |input: u8| -> u32 { + func: |input: u8, context| -> u32 { return input as u32 + 2; }, phantom_i: Default::default(), @@ -166,17 +325,17 @@ mod tests { }, next: EndStep::default(), } - .process(5u8); + .process(5u8, &mut context); assert_eq!(output, 7); let output: u32 = PipelineStep { func: Closure2Processable { - func: |input: u8| -> u32 { input as u32 + 2 }, + func: |input: u8, context| -> u32 { input as u32 + 2 }, }, next: EndStep::default(), } - .process(5u8); + .process(5u8, &mut context); assert_eq!(output, 7); } From 94b8f27bf860fb7bd2cfe46bcf98c61b884a554b Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Mon, 30 May 2022 16:13:52 +0200 Subject: [PATCH 11/31] Use pipeline in headless rendering --- maplibre-demo/Cargo.toml | 2 + maplibre-demo/src/main.rs | 94 ++++++++++++++++--- maplibre/src/context.rs | 1 - maplibre/src/io/mod.rs | 1 - maplibre/src/io/pipeline.rs | 65 +++++++++++-- maplibre/src/io/scheduler.rs | 12 +-- maplibre/src/map_schedule.rs | 1 - .../src/platform/noweb/schedule_method.rs | 8 +- maplibre/src/render/stages/extract_stage.rs | 62 ++++++++++++ maplibre/src/render/stages/mod.rs | 9 +- maplibre/src/render/stages/queue_stage.rs | 26 +++-- maplibre/src/render/stages/upload_stage.rs | 17 +--- maplibre/src/stages/mod.rs | 3 +- .../src/stages/populate_tile_store_stage.rs | 2 +- maplibre/src/stages/request_stage.rs | 30 +++--- .../src/{io => stages}/shared_thread_state.rs | 0 16 files changed, 250 insertions(+), 83 deletions(-) create mode 100644 maplibre/src/render/stages/extract_stage.rs rename maplibre/src/{io => stages}/shared_thread_state.rs (100%) diff --git a/maplibre-demo/Cargo.toml b/maplibre-demo/Cargo.toml index abd35546..ee6d315b 100644 --- a/maplibre-demo/Cargo.toml +++ b/maplibre-demo/Cargo.toml @@ -16,6 +16,8 @@ trace = ["maplibre/trace", "tracing-subscriber", "tracing-tracy", "tracy-client" env_logger = "0.9" maplibre = { path = "../maplibre", version = "0.0.2" } maplibre-winit = { path = "../maplibre-winit", version = "0.0.1" } +geozero = { version = "0.9.4", default-features = false, features = ["with-mvt", "with-geo"]} + tokio = "1.18" wgpu = "0.12" diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index f5a43265..68025903 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -1,15 +1,24 @@ +use geozero::mvt::tile; +use maplibre::benchmarking::tessellation::{IndexDataType, OverAlignedVertexBuffer}; use maplibre::coords::WorldTileCoords; use maplibre::error::Error; +use maplibre::io::pipeline::steps::build_vector_tile_pipeline; +use maplibre::io::pipeline::Processable; +use maplibre::io::pipeline::{PipelineContext, PipelineProcessor}; use maplibre::io::scheduler::ScheduleMethod; use maplibre::io::source_client::{HttpClient, HttpSourceClient}; +use maplibre::io::{LayerTessellateMessage, TileRequest, TileRequestID}; use maplibre::map_schedule::{EventuallyMapContext, MapSchedule}; use maplibre::platform::http_client::ReqwestHttpClient; use maplibre::platform::run_multithreaded; use maplibre::platform::schedule_method::TokioScheduleMethod; use maplibre::render::settings::RendererSettings; +use maplibre::render::ShaderVertex; use maplibre::window::{EventLoop, MapWindow, MapWindowConfig, WindowSize}; use maplibre::MapBuilder; use maplibre_winit::winit::{WinitEventLoop, WinitMapWindow, WinitMapWindowConfig, WinitWindow}; +use std::any::Any; +use std::collections::HashSet; use wgpu::TextureFormat; #[cfg(feature = "trace")] @@ -21,21 +30,25 @@ fn enable_tracing() { tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); } -pub struct HeadlessMapWindowConfig; +pub struct HeadlessMapWindowConfig { + size: WindowSize, +} impl MapWindowConfig for HeadlessMapWindowConfig { type MapWindow = HeadlessMapWindow; fn create(&self) -> Self::MapWindow { - Self::MapWindow {} + Self::MapWindow { size: self.size } } } -pub struct HeadlessMapWindow; +pub struct HeadlessMapWindow { + size: WindowSize, +} impl MapWindow for HeadlessMapWindow { fn size(&self) -> WindowSize { - WindowSize::new(1920, 1080).unwrap() + self.size } } @@ -52,10 +65,38 @@ fn run_in_window() { }) } +#[derive(Default)] +struct HeadlessPipelineProcessor { + layers: Vec, +} + +impl PipelineProcessor for HeadlessPipelineProcessor { + fn finished_tile_tesselation(&mut self, request_id: TileRequestID, coords: &WorldTileCoords) {} + + fn unavailable_layer(&mut self, coords: &WorldTileCoords, layer_name: &str) {} + + fn finished_layer_tesselation( + &mut self, + coords: &WorldTileCoords, + buffer: OverAlignedVertexBuffer, + feature_indices: Vec, + layer_data: tile::Layer, + ) { + self.layers.push(LayerTessellateMessage::TessellatedLayer { + coords: *coords, + buffer, + feature_indices, + layer_data, + }) + } +} + fn run_headless() { run_multithreaded(async { let mut map = MapBuilder::new() - .with_map_window_config(HeadlessMapWindowConfig) + .with_map_window_config(HeadlessMapWindowConfig { + size: WindowSize::new(1000, 1000).unwrap(), + }) .with_http_client(ReqwestHttpClient::new(None)) .with_schedule_method(TokioScheduleMethod::new()) .with_renderer_settings(RendererSettings { @@ -72,20 +113,45 @@ fn run_headless() { let coords = WorldTileCoords::from((0, 0, 0)); let request_id = 0; - /* let x = match http_source_client.fetch(&coords).await { - Ok(data) => state.process_tile(0, data.into_boxed_slice()).unwrap(), - Err(e) => { - log::error!("{:?}", &e); + let data = http_source_client + .fetch(&coords) + .await + .unwrap() + .into_boxed_slice(); - state.tile_unavailable(&coords, request_id).unwrap() - } + let mut processor = HeadlessPipelineProcessor::default(); + let mut pipeline_context = PipelineContext { + processor: Box::new(processor), }; + let pipeline = build_vector_tile_pipeline(); + pipeline.process( + ( + TileRequest { + coords, + layers: HashSet::from(["boundary".to_owned(), "water".to_owned()]), + }, + request_id, + data, + ), + &mut pipeline_context, + ); - match map.map_schedule_mut().map_context { - EventuallyMapContext::Full(a) => a.tile_cache.put_tessellated_layer(), + match &mut map.map_schedule_mut().map_context { + EventuallyMapContext::Full(context) => { + while let Some(v) = pipeline_context + .processor + .as_any_mut() + .downcast_mut::() + .unwrap() + .layers + .pop() + { + context.tile_cache.put_tessellated_layer(v); + } + } EventuallyMapContext::Premature(_) => {} EventuallyMapContext::_Uninitialized => {} - }*/ + } match map.map_schedule_mut().update_and_redraw() { Ok(_) => {} diff --git a/maplibre/src/context.rs b/maplibre/src/context.rs index 06a2d545..e5a3f04e 100644 --- a/maplibre/src/context.rs +++ b/maplibre/src/context.rs @@ -1,5 +1,4 @@ use crate::coords::{Zoom, TILE_SIZE}; -use crate::io::shared_thread_state::SharedThreadState; use crate::io::tile_cache::TileCache; use crate::io::TessellateMessage; use crate::render::camera::{Camera, Perspective, ViewProjection}; diff --git a/maplibre/src/io/mod.rs b/maplibre/src/io/mod.rs index 7310a9a1..001d8611 100644 --- a/maplibre/src/io/mod.rs +++ b/maplibre/src/io/mod.rs @@ -15,7 +15,6 @@ pub mod static_tile_fetcher; pub mod geometry_index; pub mod pipeline; -pub mod shared_thread_state; pub mod tile_cache; pub mod tile_request_state; diff --git a/maplibre/src/io/pipeline.rs b/maplibre/src/io/pipeline.rs index 3bb416c7..b615945a 100644 --- a/maplibre/src/io/pipeline.rs +++ b/maplibre/src/io/pipeline.rs @@ -2,12 +2,14 @@ use crate::coords::WorldTileCoords; use crate::io::{LayerTessellateMessage, TessellateMessage, TileRequestID, TileTessellateMessage}; use crate::render::ShaderVertex; use crate::tessellation::{IndexDataType, OverAlignedVertexBuffer}; +use downcast_rs::{impl_downcast, Downcast}; use geozero::mvt::tile; +use std::any::Any; use std::marker::PhantomData; use std::process::Output; use std::sync::mpsc; -pub trait PipelineProcessor { +pub trait PipelineProcessor: Downcast { fn finished_tile_tesselation(&mut self, request_id: TileRequestID, coords: &WorldTileCoords); fn unavailable_layer(&mut self, coords: &WorldTileCoords, layer_name: &str); fn finished_layer_tesselation( @@ -20,6 +22,8 @@ pub trait PipelineProcessor { ); } +impl_downcast!(PipelineProcessor); + pub struct HeadedPipelineProcessor { pub message_sender: mpsc::Sender, } @@ -80,7 +84,7 @@ where P: Processable, N: Processable, { - func: P, + process: P, next: N, } @@ -93,7 +97,7 @@ where type Output = N::Output; fn process(&self, input: Self::Input, context: &mut PipelineContext) -> Self::Output { - let output = self.func.process(input, context); + let output = self.process.process(input, context); self.next.process(output, context) } } @@ -177,8 +181,8 @@ impl Processable for Closure2Processable { } } -mod steps { - use crate::io::pipeline::{PipelineContext, Processable}; +pub mod steps { + use crate::io::pipeline::{EndStep, PipelineContext, PipelineStep, Processable}; use crate::io::{TileRequest, TileRequestID}; use crate::tessellation::zero_tessellator::ZeroTessellator; use crate::tessellation::IndexDataType; @@ -269,6 +273,47 @@ mod steps { .finished_tile_tesselation(request_id, &tile_request.coords); } } + + pub fn build_vector_tile_pipeline( + ) -> impl Processable::Input> { + PipelineStep { + process: ParseTileStep {}, + next: PipelineStep { + process: TessellateLayerStep {}, + next: EndStep::default(), + }, + } + } + + #[cfg(test)] + mod tests { + use crate::io::pipeline::steps::build_vector_tile_pipeline; + use crate::io::pipeline::{HeadedPipelineProcessor, PipelineContext, Processable}; + use crate::io::TileRequest; + use std::sync::mpsc; + + #[test] + fn test() { + let mut context = PipelineContext { + processor: Box::new(HeadedPipelineProcessor { + message_sender: mpsc::channel().0, + }), + }; + + let pipeline = build_vector_tile_pipeline(); + let output = pipeline.process( + ( + TileRequest { + coords: (0, 0, 0).into(), + layers: Default::default(), + }, + 0, + Box::new([0]), + ), + &mut context, + ); + } + } } #[cfg(test)] @@ -295,7 +340,7 @@ mod tests { }), }; let output: u32 = PipelineStep { - func: FnProcessable { + process: FnProcessable { func: &(add_two as fn(u8, &mut PipelineContext) -> u32), }, next: EndStep::default(), @@ -305,9 +350,9 @@ mod tests { assert_eq!(output, 7); let output = PipelineStep { - func: &(add_one as fn(u32, &mut PipelineContext) -> u8), + process: &(add_one as fn(u32, &mut PipelineContext) -> u8), next: PipelineStep { - func: &(add_two as fn(u8, &mut PipelineContext) -> u32), + process: &(add_two as fn(u8, &mut PipelineContext) -> u32), next: EndStep::default(), }, } @@ -316,7 +361,7 @@ mod tests { assert_eq!(output, 8); let output: u32 = PipelineStep { - func: ClosureProcessable { + process: ClosureProcessable { func: |input: u8, context| -> u32 { return input as u32 + 2; }, @@ -330,7 +375,7 @@ mod tests { assert_eq!(output, 7); let output: u32 = PipelineStep { - func: Closure2Processable { + process: Closure2Processable { func: |input: u8, context| -> u32 { input as u32 + 2 }, }, next: EndStep::default(), diff --git a/maplibre/src/io/scheduler.rs b/maplibre/src/io/scheduler.rs index f086f385..0c3fe58c 100644 --- a/maplibre/src/io/scheduler.rs +++ b/maplibre/src/io/scheduler.rs @@ -5,8 +5,6 @@ use std::pin::Pin; use crate::error::Error; -use crate::io::shared_thread_state::SharedThreadState; - /// Async/await scheduler. pub struct Scheduler where @@ -38,18 +36,12 @@ pub trait ScheduleMethod: 'static { #[cfg(not(feature = "no-thread-safe-futures"))] fn schedule( &self, - shared_thread_state: SharedThreadState, - future_factory: Box< - (dyn (FnOnce(SharedThreadState) -> Pin + Send>>) + Send), - >, + future_factory: Box<(dyn (FnOnce() -> Pin + Send>>) + Send)>, ) -> Result<(), Error>; #[cfg(feature = "no-thread-safe-futures")] fn schedule( &self, - shared_thread_state: SharedThreadState, - future_factory: Box< - (dyn (FnOnce(SharedThreadState) -> Pin>>) + Send), - >, + future_factory: Box<(dyn (FnOnce() -> Pin>>) + Send)>, ) -> Result<(), Error>; } diff --git a/maplibre/src/map_schedule.rs b/maplibre/src/map_schedule.rs index d5c18516..e3c70083 100644 --- a/maplibre/src/map_schedule.rs +++ b/maplibre/src/map_schedule.rs @@ -4,7 +4,6 @@ use crate::context::{MapContext, ViewState}; use crate::error::Error; use crate::io::geometry_index::GeometryIndex; use crate::io::scheduler::Scheduler; -use crate::io::shared_thread_state::SharedThreadState; use crate::io::source_client::{HttpClient, HttpSourceClient, SourceClient}; use crate::io::tile_cache::TileCache; use crate::io::tile_request_state::TileRequestState; diff --git a/maplibre/src/platform/noweb/schedule_method.rs b/maplibre/src/platform/noweb/schedule_method.rs index b05395b6..17a0b12b 100644 --- a/maplibre/src/platform/noweb/schedule_method.rs +++ b/maplibre/src/platform/noweb/schedule_method.rs @@ -1,5 +1,4 @@ use crate::error::Error; -use crate::io::shared_thread_state::SharedThreadState; use crate::ScheduleMethod; use std::future::Future; use std::pin::Pin; @@ -16,14 +15,11 @@ impl TokioScheduleMethod { impl ScheduleMethod for TokioScheduleMethod { fn schedule( &self, - shared_thread_state: SharedThreadState, future_factory: Box< - (dyn (FnOnce(SharedThreadState) -> Pin + Send + 'static>>) - + Send - + 'static), + (dyn (FnOnce() -> Pin + Send + 'static>>) + Send + 'static), >, ) -> Result<(), Error> { - tokio::task::spawn((future_factory)(shared_thread_state)); + tokio::task::spawn((future_factory)()); Ok(()) } } diff --git a/maplibre/src/render/stages/extract_stage.rs b/maplibre/src/render/stages/extract_stage.rs new file mode 100644 index 00000000..0c2bb925 --- /dev/null +++ b/maplibre/src/render/stages/extract_stage.rs @@ -0,0 +1,62 @@ +//! Extracts data from the current state. + +use crate::context::MapContext; +use crate::coords::{ViewRegion, Zoom}; +use crate::io::tile_cache::TileCache; +use crate::io::LayerTessellateMessage; +use crate::render::camera::ViewProjection; +use crate::render::render_phase::RenderPhase; +use crate::render::resource::IndexEntry; +use crate::render::shaders::{ + ShaderCamera, ShaderFeatureStyle, ShaderGlobals, ShaderLayerMetadata, Vec4f32, +}; +use crate::render::tile_view_pattern::TileInView; +use crate::render::util::Eventually::Initialized; +use crate::schedule::Stage; +use crate::{RenderState, Renderer, Style}; +use std::iter; + +#[derive(Default)] +pub struct ExtractStage; + +impl Stage for ExtractStage { + fn run( + &mut self, + MapContext { + view_state, + renderer: + Renderer { + state: + RenderState { + mask_phase, + tile_phase, + tile_view_pattern, + buffer_pool, + .. + }, + .. + }, + .. + }: &mut MapContext, + ) { + if let (Initialized(tile_view_pattern), Initialized(buffer_pool)) = + (tile_view_pattern, &buffer_pool) + { + let visible_level = view_state.visible_level(); + + let view_proj = view_state.view_projection(); + + let view_region = view_state + .camera + .view_region_bounding_box(&view_proj.invert()) + .map(|bounding_box| { + ViewRegion::new(bounding_box, 0, *view_state.zoom, visible_level) + }); + + if let Some(view_region) = &view_region { + let zoom = view_state.zoom(); + tile_view_pattern.update_pattern(view_region, buffer_pool, zoom); + } + } + } +} diff --git a/maplibre/src/render/stages/mod.rs b/maplibre/src/render/stages/mod.rs index 55969992..3b44e2d9 100644 --- a/maplibre/src/render/stages/mod.rs +++ b/maplibre/src/render/stages/mod.rs @@ -6,6 +6,7 @@ use graph_runner_stage::GraphRunnerStage; use resource_stage::ResourceStage; use upload_stage::UploadStage; +mod extract_stage; mod graph_runner_stage; mod phase_sort_stage; mod queue_stage; @@ -14,6 +15,7 @@ mod upload_stage; mod write_surface_buffer_stage; use crate::multi_stage; +use crate::render::stages::extract_stage::ExtractStage; use crate::render::stages::phase_sort_stage::PhaseSortStage; use crate::render::stages::queue_stage::QueueStage; use crate::render::stages::write_surface_buffer_stage::WriteSurfaceBufferStage; @@ -48,7 +50,12 @@ impl StageLabel for RenderStageLabel { } } -multi_stage!(PrepareStage, upload: UploadStage, resource: ResourceStage); +multi_stage!( + PrepareStage, + resource: ResourceStage, + extract: ExtractStage, + upload: UploadStage +); pub fn register_render_stages(schedule: &mut Schedule) { schedule.add_stage(RenderStageLabel::Prepare, PrepareStage::default()); diff --git a/maplibre/src/render/stages/queue_stage.rs b/maplibre/src/render/stages/queue_stage.rs index 61e4dbe0..7b605d87 100644 --- a/maplibre/src/render/stages/queue_stage.rs +++ b/maplibre/src/render/stages/queue_stage.rs @@ -23,15 +23,27 @@ impl Stage for QueueStage { fn run( &mut self, MapContext { - renderer: Renderer { state, .. }, + view_state, + renderer: + Renderer { + state: + RenderState { + mask_phase, + tile_phase, + tile_view_pattern, + buffer_pool, + .. + }, + .. + }, .. }: &mut MapContext, ) { - state.mask_phase.items.clear(); - state.tile_phase.items.clear(); + mask_phase.items.clear(); + tile_phase.items.clear(); if let (Initialized(tile_view_pattern), Initialized(buffer_pool)) = - (&state.tile_view_pattern, &state.buffer_pool) + (tile_view_pattern, &buffer_pool) { let index = buffer_pool.index(); @@ -43,7 +55,7 @@ impl Stage for QueueStage { let shape_to_render = fallback.as_ref().unwrap_or(shape); // Draw mask - state.mask_phase.add(tile_in_view.clone()); + mask_phase.add(tile_in_view.clone()); if let Some(entries) = index.get_layers(&shape_to_render.coords) { let mut layers_to_render: Vec<&IndexEntry> = Vec::from_iter(entries); @@ -51,9 +63,7 @@ impl Stage for QueueStage { for entry in layers_to_render { // Draw tile - state - .tile_phase - .add((entry.clone(), shape_to_render.clone())) + tile_phase.add((entry.clone(), shape_to_render.clone())) } } else { tracing::trace!("No layers found at {}", &shape_to_render.coords); diff --git a/maplibre/src/render/stages/upload_stage.rs b/maplibre/src/render/stages/upload_stage.rs index 72420496..ac3d8b1a 100644 --- a/maplibre/src/render/stages/upload_stage.rs +++ b/maplibre/src/render/stages/upload_stage.rs @@ -59,10 +59,8 @@ impl Stage for UploadStage { .map(|bounding_box| ViewRegion::new(bounding_box, 0, *view_state.zoom, visible_level)); if let Some(view_region) = &view_region { - let zoom = view_state.zoom(); - self.upload_tile_geometry(state, queue, tile_cache, style, view_region); - self.update_tile_view_pattern(state, queue, view_region, &view_proj, zoom); + self.upload_tile_view_pattern(state, queue, &view_proj); self.update_metadata(); } } @@ -139,22 +137,15 @@ impl UploadStage { } #[tracing::instrument(skip_all)] - pub fn update_tile_view_pattern( + pub fn upload_tile_view_pattern( &self, RenderState { - tile_view_pattern, - buffer_pool, - .. + tile_view_pattern, .. }: &mut RenderState, queue: &wgpu::Queue, - view_region: &ViewRegion, view_proj: &ViewProjection, - zoom: Zoom, ) { - if let (Initialized(tile_view_pattern), Initialized(buffer_pool)) = - (tile_view_pattern, buffer_pool) - { - tile_view_pattern.update_pattern(view_region, buffer_pool, zoom); + if let Initialized(tile_view_pattern) = tile_view_pattern { tile_view_pattern.upload_pattern(queue, view_proj); } } diff --git a/maplibre/src/stages/mod.rs b/maplibre/src/stages/mod.rs index 56ad24a7..e4c8a317 100644 --- a/maplibre/src/stages/mod.rs +++ b/maplibre/src/stages/mod.rs @@ -1,18 +1,19 @@ //! [Stages](Stage) for requesting and preparing data use crate::io::geometry_index::GeometryIndex; -use crate::io::shared_thread_state::SharedThreadState; use crate::io::source_client::{HttpSourceClient, SourceClient}; use crate::io::tile_request_state::TileRequestState; use crate::io::TessellateMessage; use crate::schedule::Schedule; use crate::stages::populate_tile_store_stage::PopulateTileStore; +use crate::stages::shared_thread_state::SharedThreadState; use crate::{HttpClient, ScheduleMethod, Scheduler}; use request_stage::RequestStage; use std::sync::{mpsc, Arc, Mutex}; mod populate_tile_store_stage; mod request_stage; +mod shared_thread_state; pub type MessageSender = mpsc::Sender; pub type MessageReceiver = mpsc::Receiver; diff --git a/maplibre/src/stages/populate_tile_store_stage.rs b/maplibre/src/stages/populate_tile_store_stage.rs index 4f6df7f0..51e4d4ef 100644 --- a/maplibre/src/stages/populate_tile_store_stage.rs +++ b/maplibre/src/stages/populate_tile_store_stage.rs @@ -1,9 +1,9 @@ //! Receives data from async threads and populates the [`crate::io::tile_cache::TileCache`]. use crate::context::MapContext; -use crate::io::shared_thread_state::SharedThreadState; use crate::io::{TessellateMessage, TileTessellateMessage}; use crate::schedule::Stage; +use crate::stages::shared_thread_state::SharedThreadState; use crate::stages::MessageReceiver; use std::sync::mpsc; diff --git a/maplibre/src/stages/request_stage.rs b/maplibre/src/stages/request_stage.rs index b64bca8f..d2240e98 100644 --- a/maplibre/src/stages/request_stage.rs +++ b/maplibre/src/stages/request_stage.rs @@ -3,11 +3,11 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, WorldTileCoords}; use crate::error::Error; -use crate::io::shared_thread_state::SharedThreadState; use crate::io::source_client::{HttpSourceClient, SourceClient}; use crate::io::tile_cache::TileCache; use crate::io::TileRequest; use crate::schedule::Stage; +use crate::stages::shared_thread_state::SharedThreadState; use crate::{HttpClient, ScheduleMethod, Style}; use std::collections::HashSet; @@ -134,23 +134,21 @@ where let client = SourceClient::Http(self.http_source_client.clone()); let coords = *coords; + let state = self.shared_thread_state.clone(); self.scheduler - .schedule( - self.shared_thread_state.clone(), - Box::new(move |state: SharedThreadState| { - Box::pin(async move { - match client.fetch(&coords).await { - Ok(data) => state - .process_tile(request_id, data.into_boxed_slice()) - .unwrap(), - Err(e) => { - log::error!("{:?}", &e); - state.tile_unavailable(&coords, request_id).unwrap() - } + .schedule(Box::new(move || { + Box::pin(async move { + match client.fetch(&coords).await { + Ok(data) => state + .process_tile(request_id, data.into_boxed_slice()) + .unwrap(), + Err(e) => { + log::error!("{:?}", &e); + state.tile_unavailable(&coords, request_id).unwrap() } - }) - }), - ) + } + }) + })) .unwrap(); } diff --git a/maplibre/src/io/shared_thread_state.rs b/maplibre/src/stages/shared_thread_state.rs similarity index 100% rename from maplibre/src/io/shared_thread_state.rs rename to maplibre/src/stages/shared_thread_state.rs From 1c297ff10b895cd6f229390c7e30dc5d88a65f6e Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Mon, 30 May 2022 17:06:27 +0200 Subject: [PATCH 12/31] Improve the style significantly. The colors are taken from the default OSM map style --- maplibre/src/style/style.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/maplibre/src/style/style.rs b/maplibre/src/style/style.rs index 0de67453..f9e8a7d5 100644 --- a/maplibre/src/style/style.rs +++ b/maplibre/src/style/style.rs @@ -33,7 +33,7 @@ impl Default for Style { minzoom: None, metadata: None, paint: Some(LayerPaint::Line(LinePaint { - line_color: Some(Color::from_str("lightgreen").unwrap()), + line_color: Some(Color::from_str("#c8facc").unwrap()), })), source: None, source_layer: Some("park".to_string()), @@ -46,7 +46,7 @@ impl Default for Style { minzoom: None, metadata: None, paint: Some(LayerPaint::Line(LinePaint { - line_color: Some(Color::from_str("lightgreen").unwrap()), + line_color: Some(Color::from_str("#e0dfdf").unwrap()), })), source: None, source_layer: Some("landuse".to_string()), @@ -59,20 +59,20 @@ impl Default for Style { minzoom: None, metadata: None, paint: Some(LayerPaint::Line(LinePaint { - line_color: Some(Color::from_str("lightgreen").unwrap()), + line_color: Some(Color::from_str("#aedfa3").unwrap()), })), source: None, source_layer: Some("landcover".to_string()), }, StyleLayer { index: 3, - id: "1transportation".to_string(), + id: "transportation".to_string(), typ: "line".to_string(), maxzoom: None, minzoom: None, metadata: None, paint: Some(LayerPaint::Line(LinePaint { - line_color: Some(Color::from_str("violet").unwrap()), + line_color: Some(Color::from_str("#ffffff").unwrap()), })), source: None, source_layer: Some("transportation".to_string()), @@ -85,7 +85,7 @@ impl Default for Style { minzoom: None, metadata: None, paint: Some(LayerPaint::Line(LinePaint { - line_color: Some(Color::from_str("grey").unwrap()), + line_color: Some(Color::from_str("#d9d0c9").unwrap()), })), source: None, source_layer: Some("building".to_string()), @@ -98,7 +98,7 @@ impl Default for Style { minzoom: None, metadata: None, paint: Some(LayerPaint::Line(LinePaint { - line_color: Some(Color::from_str("blue").unwrap()), + line_color: Some(Color::from_str("#aad3df").unwrap()), })), source: None, source_layer: Some("water".to_string()), @@ -111,7 +111,7 @@ impl Default for Style { minzoom: None, metadata: None, paint: Some(LayerPaint::Line(LinePaint { - line_color: Some(Color::from_str("blue").unwrap()), + line_color: Some(Color::from_str("#aad3df").unwrap()), })), source: None, source_layer: Some("waterway".to_string()), From ad6a0be20d0841667baeea01e3c041ce6d43e290 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Mon, 30 May 2022 17:34:30 +0200 Subject: [PATCH 13/31] Start working on a specific headless API --- maplibre-demo/src/main.rs | 33 +++++++-------- maplibre-winit/src/winit/mod.rs | 8 +++- maplibre/src/io/pipeline.rs | 6 ++- maplibre/src/lib.rs | 40 +++++++++++++----- maplibre/src/map_schedule.rs | 74 +++++++++++++++++++++++++++++++-- maplibre/src/render/settings.rs | 2 - maplibre/src/window.rs | 4 +- 7 files changed, 129 insertions(+), 38 deletions(-) diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index 68025903..fbd7ce23 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -8,7 +8,7 @@ use maplibre::io::pipeline::{PipelineContext, PipelineProcessor}; use maplibre::io::scheduler::ScheduleMethod; use maplibre::io::source_client::{HttpClient, HttpSourceClient}; use maplibre::io::{LayerTessellateMessage, TileRequest, TileRequestID}; -use maplibre::map_schedule::{EventuallyMapContext, MapSchedule}; +use maplibre::map_schedule::{EventuallyMapContext, InteractiveMapSchedule}; use maplibre::platform::http_client::ReqwestHttpClient; use maplibre::platform::run_multithreaded; use maplibre::platform::schedule_method::TokioScheduleMethod; @@ -136,21 +136,19 @@ fn run_headless() { &mut pipeline_context, ); - match &mut map.map_schedule_mut().map_context { - EventuallyMapContext::Full(context) => { - while let Some(v) = pipeline_context - .processor - .as_any_mut() - .downcast_mut::() - .unwrap() - .layers - .pop() - { - context.tile_cache.put_tessellated_layer(v); - } - } - EventuallyMapContext::Premature(_) => {} - EventuallyMapContext::_Uninitialized => {} + let mut processor = pipeline_context.teardown(); + + while let Some(v) = processor + .as_any_mut() + .downcast_mut::() + .unwrap() + .layers + .pop() + { + map.map_schedule_mut() + .map_context + .tile_cache + .put_tessellated_layer(v); } match map.map_schedule_mut().update_and_redraw() { @@ -170,5 +168,6 @@ fn main() { #[cfg(feature = "trace")] enable_tracing(); - run_headless() + run_headless(); + run_in_window(); } diff --git a/maplibre-winit/src/winit/mod.rs b/maplibre-winit/src/winit/mod.rs index 52c089b5..302060e7 100644 --- a/maplibre-winit/src/winit/mod.rs +++ b/maplibre-winit/src/winit/mod.rs @@ -7,7 +7,7 @@ use winit::event::{ElementState, KeyboardInput, VirtualKeyCode, WindowEvent}; use winit::event_loop::ControlFlow; use crate::input::{InputController, UpdateState}; -use maplibre::map_schedule::MapSchedule; +use maplibre::map_schedule::InteractiveMapSchedule; use maplibre::window::{EventLoop, HeadedMapWindow, MapWindow, MapWindowConfig}; use winit::event::Event; @@ -71,7 +71,11 @@ where SM: ScheduleMethod, HC: HttpClient, { - fn run(mut self, mut map_schedule: MapSchedule, max_frames: Option) { + fn run( + mut self, + mut map_schedule: InteractiveMapSchedule, + max_frames: Option, + ) { let mut last_render_time = Instant::now(); let mut current_frame: u64 = 0; diff --git a/maplibre/src/io/pipeline.rs b/maplibre/src/io/pipeline.rs index b615945a..3e451039 100644 --- a/maplibre/src/io/pipeline.rs +++ b/maplibre/src/io/pipeline.rs @@ -70,7 +70,11 @@ pub struct PipelineContext { pub processor: Box, } -impl PipelineContext {} +impl PipelineContext { + pub fn teardown(self) -> Box { + self.processor + } +} pub trait Processable { type Input; diff --git a/maplibre/src/lib.rs b/maplibre/src/lib.rs index e5258bea..44258a11 100644 --- a/maplibre/src/lib.rs +++ b/maplibre/src/lib.rs @@ -18,7 +18,7 @@ use crate::io::scheduler::{ScheduleMethod, Scheduler}; use crate::io::source_client::HttpClient; -use crate::map_schedule::MapSchedule; +use crate::map_schedule::{InteractiveMapSchedule, SimpleMapSchedule}; use crate::render::settings::{RendererSettings, WgpuSettings}; use crate::render::{RenderState, Renderer}; use crate::style::Style; @@ -53,7 +53,7 @@ where SM: ScheduleMethod, HC: HttpClient, { - map_schedule: MapSchedule, + map_schedule: InteractiveMapSchedule, window: MWC::MapWindow, } @@ -95,11 +95,31 @@ where self.window.run(self.map_schedule, max_frames); } - pub fn map_schedule(&self) -> &MapSchedule { + pub fn map_schedule(&self) -> &InteractiveMapSchedule { &self.map_schedule } - pub fn map_schedule_mut(&mut self) -> &mut MapSchedule { + pub fn map_schedule_mut(&mut self) -> &mut InteractiveMapSchedule { + &mut self.map_schedule + } +} + +pub struct HeadlessMap +where + MWC: MapWindowConfig, + SM: ScheduleMethod, + HC: HttpClient, +{ + map_schedule: SimpleMapSchedule, + window: MWC::MapWindow, +} +impl HeadlessMap +where + MWC: MapWindowConfig, + SM: ScheduleMethod, + HC: HttpClient, +{ + pub fn map_schedule_mut(&mut self) -> &mut SimpleMapSchedule { &mut self.map_schedule } } @@ -147,7 +167,7 @@ where .await .ok(); Map { - map_schedule: MapSchedule::new( + map_schedule: InteractiveMapSchedule::new( self.map_window_config, window_size, renderer, @@ -161,7 +181,7 @@ where } } - pub async fn initialize_headless(self) -> Map { + pub async fn initialize_headless(self) -> HeadlessMap { let window = self.map_window_config.create(); let window_size = window.size(); @@ -171,17 +191,15 @@ where self.renderer_settings.clone(), ) .await - .ok(); - Map { - map_schedule: MapSchedule::new( + .expect("Failed to initialize renderer"); + HeadlessMap { + map_schedule: SimpleMapSchedule::new( self.map_window_config, window_size, renderer, self.scheduler, self.http_client, self.style, - self.wgpu_settings, - self.renderer_settings, ), window, } diff --git a/maplibre/src/map_schedule.rs b/maplibre/src/map_schedule.rs index e3c70083..7931078e 100644 --- a/maplibre/src/map_schedule.rs +++ b/maplibre/src/map_schedule.rs @@ -64,7 +64,7 @@ impl EventuallyMapContext { } /// Stores the state of the map, dispatches tile fetching and caching, tessellation and drawing. -pub struct MapSchedule +pub struct InteractiveMapSchedule where MWC: MapWindowConfig, SM: ScheduleMethod, @@ -82,7 +82,7 @@ where suspended: bool, } -impl MapSchedule +impl InteractiveMapSchedule where MWC: MapWindowConfig, SM: ScheduleMethod, @@ -103,7 +103,7 @@ where let mut schedule = Schedule::default(); let http_source_client: HttpSourceClient = HttpSourceClient::new(http_client); - //register_stages(&mut schedule, http_source_client, Box::new(scheduler)); + register_stages(&mut schedule, http_source_client, Box::new(scheduler)); register_render_stages(&mut schedule); @@ -213,3 +213,71 @@ where } } } + +/// Stores the state of the map, dispatches tile fetching and caching, tessellation and drawing. +pub struct SimpleMapSchedule +where + MWC: MapWindowConfig, + SM: ScheduleMethod, + HC: HttpClient, +{ + map_window_config: MWC, + + pub map_context: MapContext, + + schedule: Schedule, + scheduler: Scheduler, + http_client: HC, +} + +impl SimpleMapSchedule +where + MWC: MapWindowConfig, + SM: ScheduleMethod, + HC: HttpClient, +{ + pub fn new( + map_window_config: MWC, + window_size: WindowSize, + renderer: Renderer, + scheduler: Scheduler, + http_client: HC, + style: Style, + ) -> Self { + let view_state = ViewState::new(&window_size); + let tile_cache = TileCache::new(); + let mut schedule = Schedule::default(); + + register_render_stages(&mut schedule); + + Self { + map_window_config, + map_context: MapContext { + view_state, + style, + tile_cache, + renderer, + }, + schedule, + scheduler, + http_client, + } + } + + #[tracing::instrument(name = "update_and_redraw", skip_all)] + pub fn update_and_redraw(&mut self) -> Result<(), Error> { + self.schedule.run(&mut self.map_context); + + Ok(()) + } + + pub fn schedule(&self) -> &Schedule { + &self.schedule + } + pub fn scheduler(&self) -> &Scheduler { + &self.scheduler + } + pub fn http_client(&self) -> &HC { + &self.http_client + } +} diff --git a/maplibre/src/render/settings.rs b/maplibre/src/render/settings.rs index 6e628345..fd10faea 100644 --- a/maplibre/src/render/settings.rs +++ b/maplibre/src/render/settings.rs @@ -102,7 +102,6 @@ impl Default for Msaa { pub struct RendererSettings { pub msaa: Msaa, pub texture_format: wgpu::TextureFormat, - pub surface_type: SurfaceType, } impl Default for RendererSettings { @@ -110,7 +109,6 @@ impl Default for RendererSettings { Self { msaa: Msaa::default(), texture_format: COLOR_TEXTURE_FORMAT, - surface_type: SurfaceType::Headed, } } } diff --git a/maplibre/src/window.rs b/maplibre/src/window.rs index 867b8e45..cac3cf98 100644 --- a/maplibre/src/window.rs +++ b/maplibre/src/window.rs @@ -1,6 +1,6 @@ //! Utilities for the window system. -use crate::{HttpClient, MapSchedule, ScheduleMethod}; +use crate::{HttpClient, InteractiveMapSchedule, ScheduleMethod}; use raw_window_handle::{HasRawWindowHandle, RawWindowHandle}; /// Window with a [carte::window::WindowSize]. @@ -26,7 +26,7 @@ where SM: ScheduleMethod, HC: HttpClient, { - fn run(self, map_schedule: MapSchedule, max_frames: Option); + fn run(self, map_schedule: InteractiveMapSchedule, max_frames: Option); } /// Window size with a width and an height in pixels. From c40f39a72cffc3a80fb05ab8f00f22cb9a957a18 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Tue, 31 May 2022 14:20:52 +0200 Subject: [PATCH 14/31] Refactor thread state and pipeline --- maplibre-demo/src/main.rs | 18 +- maplibre/src/context.rs | 1 - maplibre/src/io/mod.rs | 75 +----- maplibre/src/io/pipeline.rs | 222 +++--------------- maplibre/src/io/pipeline_steps.rs | 153 ++++++++++++ maplibre/src/io/tile_cache.rs | 25 +- maplibre/src/map_schedule.rs | 1 - maplibre/src/render/stages/extract_stage.rs | 1 - .../src/render/stages/phase_sort_stage.rs | 1 - maplibre/src/render/stages/queue_stage.rs | 1 - maplibre/src/render/stages/upload_stage.rs | 2 +- .../stages/write_surface_buffer_stage.rs | 1 - maplibre/src/stages/mod.rs | 205 +++++++++++++++- .../src/stages/populate_tile_store_stage.rs | 4 +- maplibre/src/stages/request_stage.rs | 2 +- maplibre/src/stages/shared_thread_state.rs | 170 -------------- 16 files changed, 426 insertions(+), 456 deletions(-) create mode 100644 maplibre/src/io/pipeline_steps.rs delete mode 100644 maplibre/src/stages/shared_thread_state.rs diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index fbd7ce23..19b81808 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -2,12 +2,12 @@ use geozero::mvt::tile; use maplibre::benchmarking::tessellation::{IndexDataType, OverAlignedVertexBuffer}; use maplibre::coords::WorldTileCoords; use maplibre::error::Error; -use maplibre::io::pipeline::steps::build_vector_tile_pipeline; use maplibre::io::pipeline::Processable; use maplibre::io::pipeline::{PipelineContext, PipelineProcessor}; +use maplibre::io::pipeline_steps::build_vector_tile_pipeline; use maplibre::io::scheduler::ScheduleMethod; use maplibre::io::source_client::{HttpClient, HttpSourceClient}; -use maplibre::io::{LayerTessellateMessage, TileRequest, TileRequestID}; +use maplibre::io::{TileRequest, TileRequestID}; use maplibre::map_schedule::{EventuallyMapContext, InteractiveMapSchedule}; use maplibre::platform::http_client::ReqwestHttpClient; use maplibre::platform::run_multithreaded; @@ -65,9 +65,17 @@ fn run_in_window() { }) } +struct TessellatedLayer { + coords: WorldTileCoords, + buffer: OverAlignedVertexBuffer, + /// Holds for each feature the count of indices. + feature_indices: Vec, + layer_data: tile::Layer, +} + #[derive(Default)] struct HeadlessPipelineProcessor { - layers: Vec, + layers: Vec, } impl PipelineProcessor for HeadlessPipelineProcessor { @@ -82,7 +90,7 @@ impl PipelineProcessor for HeadlessPipelineProcessor { feature_indices: Vec, layer_data: tile::Layer, ) { - self.layers.push(LayerTessellateMessage::TessellatedLayer { + self.layers.push(TessellatedLayer { coords: *coords, buffer, feature_indices, @@ -148,7 +156,7 @@ fn run_headless() { map.map_schedule_mut() .map_context .tile_cache - .put_tessellated_layer(v); + .put_tessellated_layer_(v.coords, v.buffer, v.feature_indices, v.layer_data); } match map.map_schedule_mut().update_and_redraw() { diff --git a/maplibre/src/context.rs b/maplibre/src/context.rs index e5a3f04e..5182ebf8 100644 --- a/maplibre/src/context.rs +++ b/maplibre/src/context.rs @@ -1,6 +1,5 @@ use crate::coords::{Zoom, TILE_SIZE}; use crate::io::tile_cache::TileCache; -use crate::io::TessellateMessage; use crate::render::camera::{Camera, Perspective, ViewProjection}; use crate::util::ChangeObserver; use crate::{Renderer, ScheduleMethod, Style, WindowSize}; diff --git a/maplibre/src/io/mod.rs b/maplibre/src/io/mod.rs index 001d8611..6614e4b5 100644 --- a/maplibre/src/io/mod.rs +++ b/maplibre/src/io/mod.rs @@ -15,83 +15,10 @@ pub mod static_tile_fetcher; pub mod geometry_index; pub mod pipeline; +pub mod pipeline_steps; pub mod tile_cache; pub mod tile_request_state; -/// Contains a `Tile` if the fetch was successful otherwise `Unavailable`. -pub enum TileFetchResult { - Unavailable { - coords: WorldTileCoords, - }, - Tile { - coords: WorldTileCoords, - data: Box<[u8]>, - }, -} - -impl fmt::Debug for TileFetchResult { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "TileFetchResult({})", - match self { - TileFetchResult::Unavailable { coords, .. } => coords, - TileFetchResult::Tile { coords, .. } => coords, - } - ) - } -} - -/// [crate::io::TileTessellateMessage] or [crate::io::LayerTessellateMessage] tessellation message. -pub enum TessellateMessage { - Tile(TileTessellateMessage), - Layer(LayerTessellateMessage), -} - -/// The result of the tessellation of a tile. -pub struct TileTessellateMessage { - pub request_id: TileRequestID, - pub coords: WorldTileCoords, -} - -/// `TessellatedLayer` contains the result of the tessellation for a specific layer, otherwise -/// `UnavailableLayer` if the layer doesn't exist. -pub enum LayerTessellateMessage { - UnavailableLayer { - coords: WorldTileCoords, - layer_name: String, - }, - TessellatedLayer { - coords: WorldTileCoords, - buffer: OverAlignedVertexBuffer, - /// Holds for each feature the count of indices. - feature_indices: Vec, - layer_data: tile::Layer, - }, -} - -impl fmt::Debug for LayerTessellateMessage { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "LayerTessellateMessage{}", self.get_coords()) - } -} - -impl LayerTessellateMessage { - pub fn get_coords(&self) -> WorldTileCoords { - match self { - LayerTessellateMessage::UnavailableLayer { coords, .. } => *coords, - LayerTessellateMessage::TessellatedLayer { coords, .. } => *coords, - } - } - - pub fn layer_name(&self) -> &str { - match self { - LayerTessellateMessage::UnavailableLayer { layer_name, .. } => layer_name.as_str(), - LayerTessellateMessage::TessellatedLayer { layer_data, .. } => &layer_data.name, - } - } -} - /// A request for a tile at the given coordinates and in the given layers. #[derive(Clone)] pub struct TileRequest { diff --git a/maplibre/src/io/pipeline.rs b/maplibre/src/io/pipeline.rs index 3e451039..fc2025a6 100644 --- a/maplibre/src/io/pipeline.rs +++ b/maplibre/src/io/pipeline.rs @@ -1,5 +1,6 @@ use crate::coords::WorldTileCoords; -use crate::io::{LayerTessellateMessage, TessellateMessage, TileRequestID, TileTessellateMessage}; +use crate::io::geometry_index::IndexedGeometry; +use crate::io::TileRequestID; use crate::render::ShaderVertex; use crate::tessellation::{IndexDataType, OverAlignedVertexBuffer}; use downcast_rs::{impl_downcast, Downcast}; @@ -10,8 +11,8 @@ use std::process::Output; use std::sync::mpsc; pub trait PipelineProcessor: Downcast { - fn finished_tile_tesselation(&mut self, request_id: TileRequestID, coords: &WorldTileCoords); - fn unavailable_layer(&mut self, coords: &WorldTileCoords, layer_name: &str); + fn finished_tile_tesselation(&mut self, request_id: TileRequestID, coords: &WorldTileCoords) {} + fn unavailable_layer(&mut self, coords: &WorldTileCoords, layer_name: &str) {} fn finished_layer_tesselation( &mut self, coords: &WorldTileCoords, @@ -19,53 +20,18 @@ pub trait PipelineProcessor: Downcast { // Holds for each feature the count of indices. feature_indices: Vec, layer_data: tile::Layer, - ); + ) { + } + fn finished_layer_indexing( + &mut self, + coords: &WorldTileCoords, + geometries: Vec>, + ) { + } } impl_downcast!(PipelineProcessor); -pub struct HeadedPipelineProcessor { - pub message_sender: mpsc::Sender, -} - -impl PipelineProcessor for HeadedPipelineProcessor { - fn finished_tile_tesselation(&mut self, request_id: TileRequestID, coords: &WorldTileCoords) { - self.message_sender - .send(TessellateMessage::Tile(TileTessellateMessage { - request_id, - coords: *coords, - })) - .unwrap(); - } - - fn unavailable_layer(&mut self, coords: &WorldTileCoords, layer_name: &str) { - self.message_sender.send(TessellateMessage::Layer( - LayerTessellateMessage::UnavailableLayer { - coords: *coords, - layer_name: layer_name.to_owned(), - }, - )); - } - fn finished_layer_tesselation( - &mut self, - coords: &WorldTileCoords, - buffer: OverAlignedVertexBuffer, - feature_indices: Vec, - layer_data: tile::Layer, - ) { - self.message_sender - .send(TessellateMessage::Layer( - LayerTessellateMessage::TessellatedLayer { - coords: *coords, - buffer, - feature_indices, - layer_data, - }, - )) - .unwrap(); - } -} - pub struct PipelineContext { pub processor: Box, } @@ -92,6 +58,16 @@ where next: N, } +impl PipelineStep +where + P: Processable, + N: Processable, +{ + pub fn new(process: P, next: N) -> Self { + Self { process, next } + } +} + impl Processable for PipelineStep where P: Processable, @@ -106,11 +82,18 @@ where } } -#[derive(Default)] pub struct EndStep { phantom: PhantomData, } +impl Default for EndStep { + fn default() -> Self { + Self { + phantom: PhantomData::default(), + } + } +} + impl Processable for EndStep { type Input = I; type Output = I; @@ -185,149 +168,18 @@ impl Processable for Closure2Processable { } } -pub mod steps { - use crate::io::pipeline::{EndStep, PipelineContext, PipelineStep, Processable}; - use crate::io::{TileRequest, TileRequestID}; - use crate::tessellation::zero_tessellator::ZeroTessellator; - use crate::tessellation::IndexDataType; - use geozero::GeozeroDatasource; - use prost::Message; - use std::collections::HashSet; - - pub struct ParseTileStep {} - - impl Processable for ParseTileStep { - type Input = (TileRequest, TileRequestID, Box<[u8]>); - type Output = (TileRequest, TileRequestID, geozero::mvt::Tile); - - fn process( - &self, - (tile_request, request_id, data): Self::Input, - _context: &mut PipelineContext, - ) -> Self::Output { - let tile = geozero::mvt::Tile::decode(data.as_ref()).expect("failed to load tile"); - (tile_request, request_id, tile) - } - } - - pub struct IndexLayerStep {} - - pub struct TessellateLayerStep {} - - impl Processable for TessellateLayerStep { - type Input = (TileRequest, TileRequestID, geozero::mvt::Tile); - type Output = (); - - fn process( - &self, - (tile_request, request_id, mut tile): Self::Input, - context: &mut PipelineContext, - ) -> Self::Output { - let coords = &tile_request.coords; - - for layer in &mut tile.layers { - let cloned_layer = layer.clone(); - let layer_name: &str = &cloned_layer.name; - if !tile_request.layers.contains(layer_name) { - continue; - } - - tracing::info!("layer {} at {} ready", layer_name, coords); - - let mut tessellator = ZeroTessellator::::default(); - if let Err(e) = layer.process(&mut tessellator) { - context.processor.unavailable_layer(coords, layer_name); - - tracing::error!( - "layer {} at {} tesselation failed {:?}", - layer_name, - &coords, - e - ); - } else { - context.processor.finished_layer_tesselation( - coords, - tessellator.buffer.into(), - tessellator.feature_indices, - cloned_layer, - ) - } - } - - let available_layers: HashSet<_> = tile - .layers - .iter() - .map(|layer| layer.name.clone()) - .collect::>(); - - for missing_layer in tile_request.layers.difference(&available_layers) { - context.processor.unavailable_layer(coords, missing_layer); - - tracing::info!( - "requested layer {} at {} not found in tile", - missing_layer, - &coords - ); - } - - tracing::info!("tile tessellated at {} finished", &tile_request.coords); - - context - .processor - .finished_tile_tesselation(request_id, &tile_request.coords); - } - } - - pub fn build_vector_tile_pipeline( - ) -> impl Processable::Input> { - PipelineStep { - process: ParseTileStep {}, - next: PipelineStep { - process: TessellateLayerStep {}, - next: EndStep::default(), - }, - } - } - - #[cfg(test)] - mod tests { - use crate::io::pipeline::steps::build_vector_tile_pipeline; - use crate::io::pipeline::{HeadedPipelineProcessor, PipelineContext, Processable}; - use crate::io::TileRequest; - use std::sync::mpsc; - - #[test] - fn test() { - let mut context = PipelineContext { - processor: Box::new(HeadedPipelineProcessor { - message_sender: mpsc::channel().0, - }), - }; - - let pipeline = build_vector_tile_pipeline(); - let output = pipeline.process( - ( - TileRequest { - coords: (0, 0, 0).into(), - layers: Default::default(), - }, - 0, - Box::new([0]), - ), - &mut context, - ); - } - } -} - #[cfg(test)] mod tests { use crate::io::pipeline::{ Closure2Processable, ClosureProcessable, EndStep, FnProcessable, HeadedPipelineProcessor, - PipelineContext, PipelineStep, Processable, + PipelineContext, PipelineProcessor, PipelineStep, Processable, }; use std::sync::mpsc; + pub struct DummyPipelineProcessor; + + impl PipelineProcessor for DummyPipelineProcessor {} + fn add_one(input: u32, context: &mut PipelineContext) -> u8 { input as u8 + 1 } @@ -339,9 +191,7 @@ mod tests { #[test] fn test() { let mut context = PipelineContext { - processor: Box::new(HeadedPipelineProcessor { - message_sender: mpsc::channel().0, - }), + processor: Box::new(DummyPipelineProcessor), }; let output: u32 = PipelineStep { process: FnProcessable { diff --git a/maplibre/src/io/pipeline_steps.rs b/maplibre/src/io/pipeline_steps.rs new file mode 100644 index 00000000..9749c90a --- /dev/null +++ b/maplibre/src/io/pipeline_steps.rs @@ -0,0 +1,153 @@ +use crate::io::geometry_index::IndexProcessor; +use crate::io::pipeline::{EndStep, PipelineContext, PipelineStep, Processable}; +use crate::io::{TileRequest, TileRequestID}; +use crate::tessellation::zero_tessellator::ZeroTessellator; +use crate::tessellation::IndexDataType; +use geozero::GeozeroDatasource; +use prost::Message; +use std::collections::HashSet; + +pub struct ParseTileStep; + +impl Processable for ParseTileStep { + type Input = (TileRequest, TileRequestID, Box<[u8]>); + type Output = (TileRequest, TileRequestID, geozero::mvt::Tile); + + // TODO (perf): Maybe force inline + fn process( + &self, + (tile_request, request_id, data): Self::Input, + _context: &mut PipelineContext, + ) -> Self::Output { + let tile = geozero::mvt::Tile::decode(data.as_ref()).expect("failed to load tile"); + (tile_request, request_id, tile) + } +} + +pub struct IndexLayerStep; + +impl Processable for IndexLayerStep { + type Input = (TileRequest, TileRequestID, geozero::mvt::Tile); + type Output = (TileRequest, TileRequestID, geozero::mvt::Tile); + + // TODO (perf): Maybe force inline + fn process( + &self, + (tile_request, request_id, tile): Self::Input, + context: &mut PipelineContext, + ) -> Self::Output { + let index = IndexProcessor::new(); + + context + .processor + .finished_layer_indexing(&tile_request.coords, index.get_geometries()); + (tile_request, request_id, tile) + } +} + +pub struct TessellateLayerStep; + +impl Processable for TessellateLayerStep { + type Input = (TileRequest, TileRequestID, geozero::mvt::Tile); + type Output = (TileRequest, TileRequestID, geozero::mvt::Tile); + + // TODO (perf): Maybe force inline + fn process( + &self, + (tile_request, request_id, mut tile): Self::Input, + context: &mut PipelineContext, + ) -> Self::Output { + let coords = &tile_request.coords; + + for layer in &mut tile.layers { + let cloned_layer = layer.clone(); + let layer_name: &str = &cloned_layer.name; + if !tile_request.layers.contains(layer_name) { + continue; + } + + tracing::info!("layer {} at {} ready", layer_name, coords); + + let mut tessellator = ZeroTessellator::::default(); + if let Err(e) = layer.process(&mut tessellator) { + context.processor.unavailable_layer(coords, layer_name); + + tracing::error!( + "layer {} at {} tesselation failed {:?}", + layer_name, + &coords, + e + ); + } else { + context.processor.finished_layer_tesselation( + coords, + tessellator.buffer.into(), + tessellator.feature_indices, + cloned_layer, + ) + } + } + + let available_layers: HashSet<_> = tile + .layers + .iter() + .map(|layer| layer.name.clone()) + .collect::>(); + + for missing_layer in tile_request.layers.difference(&available_layers) { + context.processor.unavailable_layer(coords, missing_layer); + + tracing::info!( + "requested layer {} at {} not found in tile", + missing_layer, + &coords + ); + } + + tracing::info!("tile tessellated at {} finished", &tile_request.coords); + + context + .processor + .finished_tile_tesselation(request_id, &tile_request.coords); + + (tile_request, request_id, tile) + } +} + +pub fn build_vector_tile_pipeline( +) -> impl Processable::Input> { + PipelineStep::new( + ParseTileStep, + PipelineStep::new(TessellateLayerStep, EndStep::default()), + ) +} + +#[cfg(test)] +mod tests { + use super::build_vector_tile_pipeline; + use crate::io::pipeline::{PipelineContext, PipelineProcessor, Processable}; + use crate::io::TileRequest; + pub struct DummyPipelineProcessor; + + impl PipelineProcessor for DummyPipelineProcessor {} + + #[test] + fn test() { + let mut context = PipelineContext { + processor: Box::new(DummyPipelineProcessor), + }; + + let pipeline = build_vector_tile_pipeline(); + let output = pipeline.process( + ( + TileRequest { + coords: (0, 0, 0).into(), + layers: Default::default(), + }, + 0, + Box::new([0]), + ), + &mut context, + ); + } +} diff --git a/maplibre/src/io/tile_cache.rs b/maplibre/src/io/tile_cache.rs index 2166b391..56f6140b 100644 --- a/maplibre/src/io/tile_cache.rs +++ b/maplibre/src/io/tile_cache.rs @@ -1,14 +1,15 @@ //! Tile cache. use crate::coords::{Quadkey, WorldTileCoords}; - -use crate::io::LayerTessellateMessage; - +use crate::render::ShaderVertex; +use crate::stages::LayerTessellateMessage; +use crate::tessellation::{IndexDataType, OverAlignedVertexBuffer}; +use geozero::mvt::tile; use std::collections::{btree_map, BTreeMap, HashSet}; /// Stores the multiple [crate::io::LayerTessellateMessage] of a cached tile. pub struct CachedTile { - layers: Vec, + layers: Vec, // TODO: Changen type here, its no message } impl CachedTile { @@ -22,6 +23,7 @@ impl CachedTile { /// Stores and provides access to a quad tree of cached tiles with world tile coords. #[derive(Default)] pub struct TileCache { + // TODO: Change name to TileStore cache: BTreeMap, } @@ -32,6 +34,21 @@ impl TileCache { } } + pub fn put_tessellated_layer_( + &mut self, + coords: WorldTileCoords, + buffer: OverAlignedVertexBuffer, + feature_indices: Vec, + layer_data: tile::Layer, + ) { + self.put_tessellated_layer(LayerTessellateMessage::TessellatedLayer { + coords, + buffer, + feature_indices, + layer_data, + }) + } + /// Inserts a tessellated layer into the quad tree at its world tile coords. /// If the space is vacant, the tessellated layer is inserted into a new /// [crate::io::tile_cache::CachedTile]. diff --git a/maplibre/src/map_schedule.rs b/maplibre/src/map_schedule.rs index 7931078e..59ae97c1 100644 --- a/maplibre/src/map_schedule.rs +++ b/maplibre/src/map_schedule.rs @@ -7,7 +7,6 @@ use crate::io::scheduler::Scheduler; use crate::io::source_client::{HttpClient, HttpSourceClient, SourceClient}; use crate::io::tile_cache::TileCache; use crate::io::tile_request_state::TileRequestState; -use crate::io::TessellateMessage; use crate::render::register_render_stages; use crate::schedule::{Schedule, Stage}; use crate::stages::register_stages; diff --git a/maplibre/src/render/stages/extract_stage.rs b/maplibre/src/render/stages/extract_stage.rs index 0c2bb925..c6920c63 100644 --- a/maplibre/src/render/stages/extract_stage.rs +++ b/maplibre/src/render/stages/extract_stage.rs @@ -3,7 +3,6 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, Zoom}; use crate::io::tile_cache::TileCache; -use crate::io::LayerTessellateMessage; use crate::render::camera::ViewProjection; use crate::render::render_phase::RenderPhase; use crate::render::resource::IndexEntry; diff --git a/maplibre/src/render/stages/phase_sort_stage.rs b/maplibre/src/render/stages/phase_sort_stage.rs index 51022053..1e989b6d 100644 --- a/maplibre/src/render/stages/phase_sort_stage.rs +++ b/maplibre/src/render/stages/phase_sort_stage.rs @@ -3,7 +3,6 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, Zoom}; use crate::io::tile_cache::TileCache; -use crate::io::LayerTessellateMessage; use crate::render::camera::ViewProjection; use crate::render::render_phase::RenderPhase; use crate::render::resource::IndexEntry; diff --git a/maplibre/src/render/stages/queue_stage.rs b/maplibre/src/render/stages/queue_stage.rs index 7b605d87..c9051491 100644 --- a/maplibre/src/render/stages/queue_stage.rs +++ b/maplibre/src/render/stages/queue_stage.rs @@ -3,7 +3,6 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, Zoom}; use crate::io::tile_cache::TileCache; -use crate::io::LayerTessellateMessage; use crate::render::camera::ViewProjection; use crate::render::resource::IndexEntry; use crate::render::shaders::{ diff --git a/maplibre/src/render/stages/upload_stage.rs b/maplibre/src/render/stages/upload_stage.rs index ac3d8b1a..7122a33b 100644 --- a/maplibre/src/render/stages/upload_stage.rs +++ b/maplibre/src/render/stages/upload_stage.rs @@ -3,7 +3,6 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, Zoom}; use crate::io::tile_cache::TileCache; -use crate::io::LayerTessellateMessage; use crate::render::camera::ViewProjection; use crate::render::resource::IndexEntry; use crate::render::shaders::{ @@ -14,6 +13,7 @@ use crate::render::util::Eventually::Initialized; use crate::schedule::Stage; use crate::{RenderState, Renderer, Style}; +use crate::stages::LayerTessellateMessage; use std::iter; #[derive(Default)] diff --git a/maplibre/src/render/stages/write_surface_buffer_stage.rs b/maplibre/src/render/stages/write_surface_buffer_stage.rs index adcba764..139ecfc4 100644 --- a/maplibre/src/render/stages/write_surface_buffer_stage.rs +++ b/maplibre/src/render/stages/write_surface_buffer_stage.rs @@ -3,7 +3,6 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, Zoom}; use crate::io::tile_cache::TileCache; -use crate::io::LayerTessellateMessage; use crate::render::camera::ViewProjection; use crate::render::render_phase::RenderPhase; use crate::render::resource::{BufferDimensions, BufferedTextureHead, Head, IndexEntry}; diff --git a/maplibre/src/stages/mod.rs b/maplibre/src/stages/mod.rs index e4c8a317..a6b0ab02 100644 --- a/maplibre/src/stages/mod.rs +++ b/maplibre/src/stages/mod.rs @@ -1,22 +1,32 @@ //! [Stages](Stage) for requesting and preparing data +use crate::coords::{WorldCoords, WorldTileCoords, Zoom}; +use crate::error::Error; use crate::io::geometry_index::GeometryIndex; +use crate::io::geometry_index::{IndexProcessor, IndexedGeometry, TileIndex}; +use crate::io::pipeline::{PipelineContext, PipelineProcessor}; +use crate::io::pipeline_steps::build_vector_tile_pipeline; use crate::io::source_client::{HttpSourceClient, SourceClient}; use crate::io::tile_request_state::TileRequestState; -use crate::io::TessellateMessage; +use crate::io::{TileRequest, TileRequestID}; +use crate::render::ShaderVertex; use crate::schedule::Schedule; use crate::stages::populate_tile_store_stage::PopulateTileStore; -use crate::stages::shared_thread_state::SharedThreadState; +use crate::tessellation::zero_tessellator::ZeroTessellator; +use crate::tessellation::{IndexDataType, OverAlignedVertexBuffer}; use crate::{HttpClient, ScheduleMethod, Scheduler}; +use geozero::mvt::tile; +use geozero::GeozeroDatasource; +use prost::Message; use request_stage::RequestStage; +use std::collections::HashSet; +use std::fmt; use std::sync::{mpsc, Arc, Mutex}; +use crate::io::pipeline::Processable; + mod populate_tile_store_stage; mod request_stage; -mod shared_thread_state; - -pub type MessageSender = mpsc::Sender; -pub type MessageReceiver = mpsc::Receiver; pub fn register_stages( schedule: &mut Schedule, @@ -41,3 +51,186 @@ pub fn register_stages( PopulateTileStore::new(shared_thread_state, message_receiver), ); } + +type MessageSender = mpsc::Sender; +type MessageReceiver = mpsc::Receiver; + +/// [crate::io::TileTessellateMessage] or [crate::io::LayerTessellateMessage] tessellation message. +enum TessellateMessage { + Tile(TileTessellateMessage), + Layer(LayerTessellateMessage), +} + +/// The result of the tessellation of a tile. +struct TileTessellateMessage { + pub request_id: TileRequestID, + pub coords: WorldTileCoords, +} + +/// `TessellatedLayer` contains the result of the tessellation for a specific layer, otherwise +/// `UnavailableLayer` if the layer doesn't exist. +enum LayerTessellateMessage { + UnavailableLayer { + coords: WorldTileCoords, + layer_name: String, + }, + TessellatedLayer { + coords: WorldTileCoords, + buffer: OverAlignedVertexBuffer, + /// Holds for each feature the count of indices. + feature_indices: Vec, + layer_data: tile::Layer, + }, +} + +impl fmt::Debug for LayerTessellateMessage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "LayerTessellateMessage{}", self.get_coords()) + } +} + +impl LayerTessellateMessage { + pub fn get_coords(&self) -> WorldTileCoords { + match self { + LayerTessellateMessage::UnavailableLayer { coords, .. } => *coords, + LayerTessellateMessage::TessellatedLayer { coords, .. } => *coords, + } + } + + pub fn layer_name(&self) -> &str { + match self { + LayerTessellateMessage::UnavailableLayer { layer_name, .. } => layer_name.as_str(), + LayerTessellateMessage::TessellatedLayer { layer_data, .. } => &layer_data.name, + } + } +} + +pub struct HeadedPipelineProcessor { + state: SharedThreadState, +} + +impl PipelineProcessor for HeadedPipelineProcessor { + fn finished_tile_tesselation(&mut self, request_id: TileRequestID, coords: &WorldTileCoords) { + self.state + .message_sender + .send(TessellateMessage::Tile(TileTessellateMessage { + request_id, + coords: *coords, + })) + .unwrap(); + } + + fn unavailable_layer(&mut self, coords: &WorldTileCoords, layer_name: &str) { + self.state.message_sender.send(TessellateMessage::Layer( + LayerTessellateMessage::UnavailableLayer { + coords: *coords, + layer_name: layer_name.to_owned(), + }, + )); + } + + fn finished_layer_tesselation( + &mut self, + coords: &WorldTileCoords, + buffer: OverAlignedVertexBuffer, + feature_indices: Vec, + layer_data: tile::Layer, + ) { + self.state + .message_sender + .send(TessellateMessage::Layer( + LayerTessellateMessage::TessellatedLayer { + coords: *coords, + buffer, + feature_indices, + layer_data, + }, + )) + .unwrap(); + } + + fn finished_layer_indexing( + &mut self, + coords: &WorldTileCoords, + geometries: Vec>, + ) { + if let Ok(mut geometry_index) = self.state.geometry_index.lock() { + geometry_index.index_tile(&coords, TileIndex::Linear { list: geometries }) + } + } +} + +/// Stores and provides access to the thread safe data shared between the schedulers. +#[derive(Clone)] +pub struct SharedThreadState { + pub tile_request_state: Arc>, + pub message_sender: mpsc::Sender, + pub geometry_index: Arc>, +} + +impl SharedThreadState { + fn get_tile_request(&self, request_id: TileRequestID) -> Option { + self.tile_request_state + .lock() + .ok() + .and_then(|tile_request_state| tile_request_state.get_tile_request(request_id).cloned()) + } + + #[tracing::instrument(skip_all)] + pub fn process_tile(&self, request_id: TileRequestID, data: Box<[u8]>) -> Result<(), Error> { + if let Some(tile_request) = self.get_tile_request(request_id) { + let mut processor = HeadedPipelineProcessor { + state: self.clone(), + }; + let mut pipeline_context = PipelineContext { + processor: Box::new(processor), + }; + let pipeline = build_vector_tile_pipeline(); + pipeline.process((tile_request, request_id, data), &mut pipeline_context); + } + + Ok(()) + } + + pub fn tile_unavailable( + &self, + coords: &WorldTileCoords, + request_id: TileRequestID, + ) -> Result<(), Error> { + if let Some(tile_request) = self.get_tile_request(request_id) { + for to_load in &tile_request.layers { + tracing::warn!("layer {} at {} unavailable", to_load, coords); + self.message_sender.send(TessellateMessage::Layer( + LayerTessellateMessage::UnavailableLayer { + coords: tile_request.coords, + layer_name: to_load.to_string(), + }, + ))?; + } + } + + Ok(()) + } + + #[tracing::instrument(skip_all)] + pub fn query_point( + &self, + world_coords: &WorldCoords, + z: u8, + zoom: Zoom, + ) -> Option>> { + if let Ok(geometry_index) = self.geometry_index.lock() { + geometry_index + .query_point(world_coords, z, zoom) + .map(|geometries| { + geometries + .iter() + .cloned() + .cloned() + .collect::>>() + }) + } else { + unimplemented!() + } + } +} diff --git a/maplibre/src/stages/populate_tile_store_stage.rs b/maplibre/src/stages/populate_tile_store_stage.rs index 51e4d4ef..9d0bb961 100644 --- a/maplibre/src/stages/populate_tile_store_stage.rs +++ b/maplibre/src/stages/populate_tile_store_stage.rs @@ -1,10 +1,8 @@ //! Receives data from async threads and populates the [`crate::io::tile_cache::TileCache`]. use crate::context::MapContext; -use crate::io::{TessellateMessage, TileTessellateMessage}; use crate::schedule::Stage; -use crate::stages::shared_thread_state::SharedThreadState; -use crate::stages::MessageReceiver; +use crate::stages::{MessageReceiver, SharedThreadState, TessellateMessage, TileTessellateMessage}; use std::sync::mpsc; pub struct PopulateTileStore { diff --git a/maplibre/src/stages/request_stage.rs b/maplibre/src/stages/request_stage.rs index d2240e98..d4756672 100644 --- a/maplibre/src/stages/request_stage.rs +++ b/maplibre/src/stages/request_stage.rs @@ -7,7 +7,7 @@ use crate::io::source_client::{HttpSourceClient, SourceClient}; use crate::io::tile_cache::TileCache; use crate::io::TileRequest; use crate::schedule::Stage; -use crate::stages::shared_thread_state::SharedThreadState; +use crate::stages::SharedThreadState; use crate::{HttpClient, ScheduleMethod, Style}; use std::collections::HashSet; diff --git a/maplibre/src/stages/shared_thread_state.rs b/maplibre/src/stages/shared_thread_state.rs deleted file mode 100644 index 97e9fdeb..00000000 --- a/maplibre/src/stages/shared_thread_state.rs +++ /dev/null @@ -1,170 +0,0 @@ -//! Shared thread state. - -use crate::coords::{WorldCoords, WorldTileCoords, Zoom}; -use crate::error::Error; -use crate::io::geometry_index::{GeometryIndex, IndexProcessor, IndexedGeometry, TileIndex}; -use crate::io::tile_request_state::TileRequestState; -use crate::io::{ - LayerTessellateMessage, TessellateMessage, TileRequest, TileRequestID, TileTessellateMessage, -}; - -use std::collections::HashSet; - -use crate::tessellation::zero_tessellator::ZeroTessellator; - -use geozero::GeozeroDatasource; -use prost::Message; -use std::sync::{mpsc, Arc, Mutex}; - -/// Stores and provides access to the thread safe data shared between the schedulers. -#[derive(Clone)] -pub struct SharedThreadState { - pub tile_request_state: Arc>, - pub message_sender: mpsc::Sender, - pub geometry_index: Arc>, -} - -impl SharedThreadState { - fn get_tile_request(&self, request_id: TileRequestID) -> Option { - self.tile_request_state - .lock() - .ok() - .and_then(|tile_request_state| tile_request_state.get_tile_request(request_id).cloned()) - } - - #[tracing::instrument(skip_all)] - pub fn process_tile(&self, request_id: TileRequestID, data: Box<[u8]>) -> Result<(), Error> { - if let Some(tile_request) = self.get_tile_request(request_id) { - let coords = tile_request.coords; - - tracing::info!("parsing tile {} with {}bytes", &coords, data.len()); - - let _span_ = tracing::span!(tracing::Level::TRACE, "parse_tile_bytes").entered(); - - let mut tile = geozero::mvt::Tile::decode(data.as_ref()).expect("failed to load tile"); - - let index = IndexProcessor::new(); - - for layer in &mut tile.layers { - let cloned_layer = layer.clone(); - let layer_name: &str = &cloned_layer.name; - if !tile_request.layers.contains(layer_name) { - continue; - } - - tracing::info!("layer {} at {} ready", layer_name, &coords); - - let mut tessellator = ZeroTessellator::default(); - if let Err(e) = layer.process(&mut tessellator) { - self.message_sender.send(TessellateMessage::Layer( - LayerTessellateMessage::UnavailableLayer { - coords, - layer_name: layer_name.to_owned(), - }, - ))?; - - tracing::error!( - "layer {} at {} tesselation failed {:?}", - layer_name, - &coords, - e - ); - } else { - self.message_sender.send(TessellateMessage::Layer( - LayerTessellateMessage::TessellatedLayer { - coords, - buffer: tessellator.buffer.into(), - feature_indices: tessellator.feature_indices, - layer_data: cloned_layer, - }, - ))?; - } - - // TODO - // layer.process(&mut index).unwrap(); - } - - let available_layers: HashSet<_> = tile - .layers - .iter() - .map(|layer| layer.name.clone()) - .collect::>(); - - for missing_layer in tile_request.layers.difference(&available_layers) { - self.message_sender.send(TessellateMessage::Layer( - LayerTessellateMessage::UnavailableLayer { - coords, - layer_name: missing_layer.to_owned(), - }, - ))?; - - tracing::info!( - "requested layer {} at {} not found in tile", - missing_layer, - &coords - ); - } - - tracing::info!("tile tessellated at {} finished", &tile_request.coords); - - self.message_sender - .send(TessellateMessage::Tile(TileTessellateMessage { - request_id, - coords: tile_request.coords, - }))?; - - if let Ok(mut geometry_index) = self.geometry_index.lock() { - geometry_index.index_tile( - &coords, - TileIndex::Linear { - list: index.get_geometries(), - }, - ) - } - } - - Ok(()) - } - - pub fn tile_unavailable( - &self, - coords: &WorldTileCoords, - request_id: TileRequestID, - ) -> Result<(), Error> { - if let Some(tile_request) = self.get_tile_request(request_id) { - for to_load in &tile_request.layers { - tracing::warn!("layer {} at {} unavailable", to_load, coords); - self.message_sender.send(TessellateMessage::Layer( - LayerTessellateMessage::UnavailableLayer { - coords: tile_request.coords, - layer_name: to_load.to_string(), - }, - ))?; - } - } - - Ok(()) - } - - #[tracing::instrument(skip_all)] - pub fn query_point( - &self, - world_coords: &WorldCoords, - z: u8, - zoom: Zoom, - ) -> Option>> { - if let Ok(geometry_index) = self.geometry_index.lock() { - geometry_index - .query_point(world_coords, z, zoom) - .map(|geometries| { - geometries - .iter() - .cloned() - .cloned() - .collect::>>() - }) - } else { - unimplemented!() - } - } -} From deb442287e7ed005a4bb49da946aa852a41e2557 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Tue, 31 May 2022 15:28:11 +0200 Subject: [PATCH 15/31] Refactor thread state and pipeline --- maplibre-demo/src/main.rs | 23 +-- maplibre/src/context.rs | 4 +- maplibre/src/io/mod.rs | 2 +- .../io/{tile_cache.rs => tile_repository.rs} | 88 +++++----- maplibre/src/map_schedule.rs | 20 +-- maplibre/src/render/stages/extract_stage.rs | 2 +- .../src/render/stages/phase_sort_stage.rs | 2 +- maplibre/src/render/stages/queue_stage.rs | 2 +- maplibre/src/render/stages/upload_stage.rs | 17 +- .../stages/write_surface_buffer_stage.rs | 2 +- maplibre/src/stages/message.rs | 155 ++++++++++++++++++ maplibre/src/stages/mod.rs | 134 +-------------- .../src/stages/populate_tile_store_stage.rs | 19 ++- maplibre/src/stages/request_stage.rs | 14 +- 14 files changed, 263 insertions(+), 221 deletions(-) rename maplibre/src/io/{tile_cache.rs => tile_repository.rs} (64%) create mode 100644 maplibre/src/stages/message.rs diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index 19b81808..0f45805e 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -7,6 +7,7 @@ use maplibre::io::pipeline::{PipelineContext, PipelineProcessor}; use maplibre::io::pipeline_steps::build_vector_tile_pipeline; use maplibre::io::scheduler::ScheduleMethod; use maplibre::io::source_client::{HttpClient, HttpSourceClient}; +use maplibre::io::tile_repository::StoredLayer; use maplibre::io::{TileRequest, TileRequestID}; use maplibre::map_schedule::{EventuallyMapContext, InteractiveMapSchedule}; use maplibre::platform::http_client::ReqwestHttpClient; @@ -65,24 +66,12 @@ fn run_in_window() { }) } -struct TessellatedLayer { - coords: WorldTileCoords, - buffer: OverAlignedVertexBuffer, - /// Holds for each feature the count of indices. - feature_indices: Vec, - layer_data: tile::Layer, -} - #[derive(Default)] struct HeadlessPipelineProcessor { - layers: Vec, + layers: Vec, } impl PipelineProcessor for HeadlessPipelineProcessor { - fn finished_tile_tesselation(&mut self, request_id: TileRequestID, coords: &WorldTileCoords) {} - - fn unavailable_layer(&mut self, coords: &WorldTileCoords, layer_name: &str) {} - fn finished_layer_tesselation( &mut self, coords: &WorldTileCoords, @@ -90,7 +79,7 @@ impl PipelineProcessor for HeadlessPipelineProcessor { feature_indices: Vec, layer_data: tile::Layer, ) { - self.layers.push(TessellatedLayer { + self.layers.push(StoredLayer::TessellatedLayer { coords: *coords, buffer, feature_indices, @@ -127,7 +116,7 @@ fn run_headless() { .unwrap() .into_boxed_slice(); - let mut processor = HeadlessPipelineProcessor::default(); + let processor = HeadlessPipelineProcessor::default(); let mut pipeline_context = PipelineContext { processor: Box::new(processor), }; @@ -155,8 +144,8 @@ fn run_headless() { { map.map_schedule_mut() .map_context - .tile_cache - .put_tessellated_layer_(v.coords, v.buffer, v.feature_indices, v.layer_data); + .tile_repository + .put_tessellated_layer(v); } match map.map_schedule_mut().update_and_redraw() { diff --git a/maplibre/src/context.rs b/maplibre/src/context.rs index 5182ebf8..68d5e2a9 100644 --- a/maplibre/src/context.rs +++ b/maplibre/src/context.rs @@ -1,5 +1,5 @@ use crate::coords::{Zoom, TILE_SIZE}; -use crate::io::tile_cache::TileCache; +use crate::io::tile_repository::TileRepository; use crate::render::camera::{Camera, Perspective, ViewProjection}; use crate::util::ChangeObserver; use crate::{Renderer, ScheduleMethod, Style, WindowSize}; @@ -59,6 +59,6 @@ pub struct MapContext { pub view_state: ViewState, pub style: Style, - pub tile_cache: TileCache, + pub tile_repository: TileRepository, pub renderer: Renderer, } diff --git a/maplibre/src/io/mod.rs b/maplibre/src/io/mod.rs index 6614e4b5..4e73b91f 100644 --- a/maplibre/src/io/mod.rs +++ b/maplibre/src/io/mod.rs @@ -16,7 +16,7 @@ pub mod static_tile_fetcher; pub mod geometry_index; pub mod pipeline; pub mod pipeline_steps; -pub mod tile_cache; +pub mod tile_repository; pub mod tile_request_state; /// A request for a tile at the given coordinates and in the given layers. diff --git a/maplibre/src/io/tile_cache.rs b/maplibre/src/io/tile_repository.rs similarity index 64% rename from maplibre/src/io/tile_cache.rs rename to maplibre/src/io/tile_repository.rs index 56f6140b..5318df44 100644 --- a/maplibre/src/io/tile_cache.rs +++ b/maplibre/src/io/tile_repository.rs @@ -2,18 +2,48 @@ use crate::coords::{Quadkey, WorldTileCoords}; use crate::render::ShaderVertex; -use crate::stages::LayerTessellateMessage; use crate::tessellation::{IndexDataType, OverAlignedVertexBuffer}; use geozero::mvt::tile; use std::collections::{btree_map, BTreeMap, HashSet}; -/// Stores the multiple [crate::io::LayerTessellateMessage] of a cached tile. -pub struct CachedTile { - layers: Vec, // TODO: Changen type here, its no message +/// A layer which is stored for future use. +pub enum StoredLayer { + UnavailableLayer { + coords: WorldTileCoords, + layer_name: String, + }, + TessellatedLayer { + coords: WorldTileCoords, + buffer: OverAlignedVertexBuffer, + /// Holds for each feature the count of indices. + feature_indices: Vec, + layer_data: tile::Layer, + }, } -impl CachedTile { - pub fn new(first_layer: LayerTessellateMessage) -> Self { +impl StoredLayer { + pub fn get_coords(&self) -> WorldTileCoords { + match self { + StoredLayer::UnavailableLayer { coords, .. } => *coords, + StoredLayer::TessellatedLayer { coords, .. } => *coords, + } + } + + pub fn layer_name(&self) -> &str { + match self { + StoredLayer::UnavailableLayer { layer_name, .. } => layer_name.as_str(), + StoredLayer::TessellatedLayer { layer_data, .. } => &layer_data.name, + } + } +} + +/// Stores multiple [StoredLayers](StoredLayer). +pub struct StoredTile { + layers: Vec, +} + +impl StoredTile { + pub fn new(first_layer: StoredLayer) -> Self { Self { layers: vec![first_layer], } @@ -22,50 +52,34 @@ impl CachedTile { /// Stores and provides access to a quad tree of cached tiles with world tile coords. #[derive(Default)] -pub struct TileCache { - // TODO: Change name to TileStore - cache: BTreeMap, +pub struct TileRepository { + tree: BTreeMap, } -impl TileCache { +impl TileRepository { pub fn new() -> Self { Self { - cache: BTreeMap::new(), + tree: BTreeMap::new(), } } - pub fn put_tessellated_layer_( - &mut self, - coords: WorldTileCoords, - buffer: OverAlignedVertexBuffer, - feature_indices: Vec, - layer_data: tile::Layer, - ) { - self.put_tessellated_layer(LayerTessellateMessage::TessellatedLayer { - coords, - buffer, - feature_indices, - layer_data, - }) - } - /// Inserts a tessellated layer into the quad tree at its world tile coords. /// If the space is vacant, the tessellated layer is inserted into a new - /// [crate::io::tile_cache::CachedTile]. + /// [crate::io::tile_repository::CachedTile]. /// If the space is occupied, the tessellated layer is added to the current - /// [crate::io::tile_cache::CachedTile]. - pub fn put_tessellated_layer(&mut self, message: LayerTessellateMessage) { - if let Some(entry) = message + /// [crate::io::tile_repository::CachedTile]. + pub fn put_tessellated_layer(&mut self, layer: StoredLayer) { + if let Some(entry) = layer .get_coords() .build_quad_key() - .map(|key| self.cache.entry(key)) + .map(|key| self.tree.entry(key)) { match entry { btree_map::Entry::Vacant(entry) => { - entry.insert(CachedTile::new(message)); + entry.insert(StoredTile::new(layer)); } btree_map::Entry::Occupied(mut entry) => { - entry.get_mut().layers.push(message); + entry.get_mut().layers.push(layer); } } } @@ -76,10 +90,10 @@ impl TileCache { pub fn iter_tessellated_layers_at( &self, coords: &WorldTileCoords, - ) -> Option + '_> { + ) -> Option + '_> { coords .build_quad_key() - .and_then(|key| self.cache.get(&key)) + .and_then(|key| self.tree.get(&key)) .map(|results| results.layers.iter()) } @@ -90,7 +104,7 @@ impl TileCache { coords: &WorldTileCoords, layers: &mut HashSet, ) { - if let Some(cached_tile) = coords.build_quad_key().and_then(|key| self.cache.get(&key)) { + if let Some(cached_tile) = coords.build_quad_key().and_then(|key| self.tree.get(&key)) { let tessellated_set: HashSet = cached_tile .layers .iter() @@ -103,7 +117,7 @@ impl TileCache { /// Checks if a layer is missing from the given layers set at the given coords. pub fn is_layers_missing(&self, coords: &WorldTileCoords, layers: &HashSet) -> bool { - if let Some(cached_tile) = coords.build_quad_key().and_then(|key| self.cache.get(&key)) { + if let Some(cached_tile) = coords.build_quad_key().and_then(|key| self.tree.get(&key)) { let tessellated_set: HashSet<&str> = cached_tile .layers .iter() diff --git a/maplibre/src/map_schedule.rs b/maplibre/src/map_schedule.rs index 59ae97c1..88115ec1 100644 --- a/maplibre/src/map_schedule.rs +++ b/maplibre/src/map_schedule.rs @@ -1,11 +1,11 @@ -//! Stores the state of the map such as `[crate::coords::Zoom]`, `[crate::camera::Camera]`, `[crate::style::Style]`, `[crate::io::tile_cache::TileCache]` and more. +//! Stores the state of the map such as `[crate::coords::Zoom]`, `[crate::camera::Camera]`, `[crate::style::Style]`, `[crate::io::tile_repository::TileCache]` and more. use crate::context::{MapContext, ViewState}; use crate::error::Error; use crate::io::geometry_index::GeometryIndex; use crate::io::scheduler::Scheduler; use crate::io::source_client::{HttpClient, HttpSourceClient, SourceClient}; -use crate::io::tile_cache::TileCache; +use crate::io::tile_repository::TileRepository; use crate::io::tile_request_state::TileRequestState; use crate::render::register_render_stages; use crate::schedule::{Schedule, Stage}; @@ -23,7 +23,7 @@ pub struct PrematureMapContext { view_state: ViewState, style: Style, - tile_cache: TileCache, + tile_repository: TileRepository, wgpu_settings: WgpuSettings, renderer_settings: RendererSettings, @@ -44,7 +44,7 @@ impl EventuallyMapContext { EventuallyMapContext::Premature(PrematureMapContext { view_state, style, - tile_cache, + tile_repository, .. }) => { mem::replace( @@ -52,7 +52,7 @@ impl EventuallyMapContext { EventuallyMapContext::Full(MapContext { view_state, style, - tile_cache, + tile_repository, renderer, }), ); @@ -98,7 +98,7 @@ where renderer_settings: RendererSettings, ) -> Self { let view_state = ViewState::new(&window_size); - let tile_cache = TileCache::new(); + let tile_repository = TileRepository::new(); let mut schedule = Schedule::default(); let http_source_client: HttpSourceClient = HttpSourceClient::new(http_client); @@ -112,14 +112,14 @@ where None => EventuallyMapContext::Premature(PrematureMapContext { view_state, style, - tile_cache, + tile_repository, wgpu_settings, renderer_settings, }), Some(renderer) => EventuallyMapContext::Full(MapContext { view_state, style, - tile_cache, + tile_repository, renderer, }), }, @@ -244,7 +244,7 @@ where style: Style, ) -> Self { let view_state = ViewState::new(&window_size); - let tile_cache = TileCache::new(); + let tile_repository = TileRepository::new(); let mut schedule = Schedule::default(); register_render_stages(&mut schedule); @@ -254,7 +254,7 @@ where map_context: MapContext { view_state, style, - tile_cache, + tile_repository, renderer, }, schedule, diff --git a/maplibre/src/render/stages/extract_stage.rs b/maplibre/src/render/stages/extract_stage.rs index c6920c63..73f9cb65 100644 --- a/maplibre/src/render/stages/extract_stage.rs +++ b/maplibre/src/render/stages/extract_stage.rs @@ -2,7 +2,7 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, Zoom}; -use crate::io::tile_cache::TileCache; +use crate::io::tile_repository::TileRepository; use crate::render::camera::ViewProjection; use crate::render::render_phase::RenderPhase; use crate::render::resource::IndexEntry; diff --git a/maplibre/src/render/stages/phase_sort_stage.rs b/maplibre/src/render/stages/phase_sort_stage.rs index 1e989b6d..62d4192c 100644 --- a/maplibre/src/render/stages/phase_sort_stage.rs +++ b/maplibre/src/render/stages/phase_sort_stage.rs @@ -2,7 +2,7 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, Zoom}; -use crate::io::tile_cache::TileCache; +use crate::io::tile_repository::TileRepository; use crate::render::camera::ViewProjection; use crate::render::render_phase::RenderPhase; use crate::render::resource::IndexEntry; diff --git a/maplibre/src/render/stages/queue_stage.rs b/maplibre/src/render/stages/queue_stage.rs index c9051491..ecbbaed6 100644 --- a/maplibre/src/render/stages/queue_stage.rs +++ b/maplibre/src/render/stages/queue_stage.rs @@ -2,7 +2,7 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, Zoom}; -use crate::io::tile_cache::TileCache; +use crate::io::tile_repository::TileRepository; use crate::render::camera::ViewProjection; use crate::render::resource::IndexEntry; use crate::render::shaders::{ diff --git a/maplibre/src/render/stages/upload_stage.rs b/maplibre/src/render/stages/upload_stage.rs index 7122a33b..114fc11d 100644 --- a/maplibre/src/render/stages/upload_stage.rs +++ b/maplibre/src/render/stages/upload_stage.rs @@ -2,7 +2,7 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, Zoom}; -use crate::io::tile_cache::TileCache; +use crate::io::tile_repository::{StoredLayer, TileRepository}; use crate::render::camera::ViewProjection; use crate::render::resource::IndexEntry; use crate::render::shaders::{ @@ -13,7 +13,6 @@ use crate::render::util::Eventually::Initialized; use crate::schedule::Stage; use crate::{RenderState, Renderer, Style}; -use crate::stages::LayerTessellateMessage; use std::iter; #[derive(Default)] @@ -26,7 +25,7 @@ impl Stage for UploadStage { MapContext { view_state, style, - tile_cache, + tile_repository, renderer: Renderer { queue, state, .. }, .. }: &mut MapContext, @@ -59,7 +58,7 @@ impl Stage for UploadStage { .map(|bounding_box| ViewRegion::new(bounding_box, 0, *view_state.zoom, visible_level)); if let Some(view_region) = &view_region { - self.upload_tile_geometry(state, queue, tile_cache, style, view_region); + self.upload_tile_geometry(state, queue, tile_repository, style, view_region); self.upload_tile_view_pattern(state, queue, &view_proj); self.update_metadata(); } @@ -89,7 +88,7 @@ impl UploadStage { /*let source_layer = entry.style_layer.source_layer.as_ref().unwrap(); if let Some(result) = scheduler - .get_tile_cache() + .get_tile_repository() .iter_tessellated_layers_at(&world_coords) .unwrap() .find(|layer| source_layer.as_str() == layer.layer_name()) @@ -155,7 +154,7 @@ impl UploadStage { &self, RenderState { buffer_pool, .. }: &mut RenderState, queue: &wgpu::Queue, - tile_cache: &TileCache, + tile_repository: &TileRepository, style: &Style, view_region: &ViewRegion, ) { @@ -165,7 +164,7 @@ impl UploadStage { let loaded_layers = buffer_pool .get_loaded_layers_at(&world_coords) .unwrap_or_default(); - if let Some(available_layers) = tile_cache + if let Some(available_layers) = tile_repository .iter_tessellated_layers_at(&world_coords) .map(|layers| { layers @@ -187,10 +186,10 @@ impl UploadStage { .map(|color| color.into()); match message { - LayerTessellateMessage::UnavailableLayer { coords: _, .. } => { + StoredLayer::UnavailableLayer { coords: _, .. } => { /*self.buffer_pool.mark_layer_unavailable(*coords);*/ } - LayerTessellateMessage::TessellatedLayer { + StoredLayer::TessellatedLayer { coords, feature_indices, layer_data, diff --git a/maplibre/src/render/stages/write_surface_buffer_stage.rs b/maplibre/src/render/stages/write_surface_buffer_stage.rs index 139ecfc4..8a4ec4a6 100644 --- a/maplibre/src/render/stages/write_surface_buffer_stage.rs +++ b/maplibre/src/render/stages/write_surface_buffer_stage.rs @@ -2,7 +2,7 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, Zoom}; -use crate::io::tile_cache::TileCache; +use crate::io::tile_repository::TileRepository; use crate::render::camera::ViewProjection; use crate::render::render_phase::RenderPhase; use crate::render::resource::{BufferDimensions, BufferedTextureHead, Head, IndexEntry}; diff --git a/maplibre/src/stages/message.rs b/maplibre/src/stages/message.rs new file mode 100644 index 00000000..5c5d9cc1 --- /dev/null +++ b/maplibre/src/stages/message.rs @@ -0,0 +1,155 @@ +use crate::coords::{WorldCoords, WorldTileCoords, Zoom}; +use crate::error::Error; +use crate::io::geometry_index::{GeometryIndex, IndexedGeometry}; +use crate::io::pipeline::PipelineContext; +use crate::io::pipeline::Processable; +use crate::io::pipeline_steps::build_vector_tile_pipeline; +use crate::io::tile_repository::StoredLayer; +use crate::io::tile_request_state::TileRequestState; +use crate::io::{TileRequest, TileRequestID}; +use crate::render::ShaderVertex; +use crate::stages::HeadedPipelineProcessor; +use crate::tessellation::{IndexDataType, OverAlignedVertexBuffer}; +use geozero::mvt::tile; +use std::fmt; +use std::sync::{mpsc, Arc, Mutex}; + +pub type MessageSender = mpsc::Sender; +pub type MessageReceiver = mpsc::Receiver; + +/// [crate::io::TileTessellateMessage] or [crate::io::LayerTessellateMessage] tessellation message. +pub enum TessellateMessage { + Tile(TileTessellateMessage), + Layer(LayerTessellateMessage), +} + +/// The result of the tessellation of a tile. +pub struct TileTessellateMessage { + pub request_id: TileRequestID, + pub coords: WorldTileCoords, +} + +/// `TessellatedLayer` contains the result of the tessellation for a specific layer, otherwise +/// `UnavailableLayer` if the layer doesn't exist. +pub enum LayerTessellateMessage { + UnavailableLayer { + coords: WorldTileCoords, + layer_name: String, + }, + TessellatedLayer { + coords: WorldTileCoords, + buffer: OverAlignedVertexBuffer, + /// Holds for each feature the count of indices. + feature_indices: Vec, + layer_data: tile::Layer, + }, +} + +impl Into for LayerTessellateMessage { + fn into(self) -> StoredLayer { + match self { + LayerTessellateMessage::UnavailableLayer { coords, layer_name } => { + StoredLayer::UnavailableLayer { coords, layer_name } + } + LayerTessellateMessage::TessellatedLayer { + coords, + buffer, + feature_indices, + layer_data, + } => StoredLayer::TessellatedLayer { + coords, + buffer, + feature_indices, + layer_data, + }, + } + } +} + +impl fmt::Debug for LayerTessellateMessage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "LayerTessellateMessage{}", + match self { + LayerTessellateMessage::UnavailableLayer { coords, .. } => coords, + LayerTessellateMessage::TessellatedLayer { coords, .. } => coords, + } + ) + } +} + +/// Stores and provides access to the thread safe data shared between the schedulers. +#[derive(Clone)] +pub struct SharedThreadState { + pub tile_request_state: Arc>, + pub message_sender: mpsc::Sender, + pub geometry_index: Arc>, +} + +impl SharedThreadState { + fn get_tile_request(&self, request_id: TileRequestID) -> Option { + self.tile_request_state + .lock() + .ok() + .and_then(|tile_request_state| tile_request_state.get_tile_request(request_id).cloned()) + } + + #[tracing::instrument(skip_all)] + pub fn process_tile(&self, request_id: TileRequestID, data: Box<[u8]>) -> Result<(), Error> { + if let Some(tile_request) = self.get_tile_request(request_id) { + let mut processor = HeadedPipelineProcessor { + state: self.clone(), + }; + let mut pipeline_context = PipelineContext { + processor: Box::new(processor), + }; + let pipeline = build_vector_tile_pipeline(); + pipeline.process((tile_request, request_id, data), &mut pipeline_context); + } + + Ok(()) + } + + pub fn tile_unavailable( + &self, + coords: &WorldTileCoords, + request_id: TileRequestID, + ) -> Result<(), Error> { + if let Some(tile_request) = self.get_tile_request(request_id) { + for to_load in &tile_request.layers { + tracing::warn!("layer {} at {} unavailable", to_load, coords); + self.message_sender.send(TessellateMessage::Layer( + LayerTessellateMessage::UnavailableLayer { + coords: tile_request.coords, + layer_name: to_load.to_string(), + }, + ))?; + } + } + + Ok(()) + } + + #[tracing::instrument(skip_all)] + pub fn query_point( + &self, + world_coords: &WorldCoords, + z: u8, + zoom: Zoom, + ) -> Option>> { + if let Ok(geometry_index) = self.geometry_index.lock() { + geometry_index + .query_point(world_coords, z, zoom) + .map(|geometries| { + geometries + .iter() + .cloned() + .cloned() + .collect::>>() + }) + } else { + unimplemented!() + } + } +} diff --git a/maplibre/src/stages/mod.rs b/maplibre/src/stages/mod.rs index a6b0ab02..d16bf916 100644 --- a/maplibre/src/stages/mod.rs +++ b/maplibre/src/stages/mod.rs @@ -24,7 +24,13 @@ use std::fmt; use std::sync::{mpsc, Arc, Mutex}; use crate::io::pipeline::Processable; +use crate::io::tile_repository::StoredLayer; +use crate::stages::message::{ + LayerTessellateMessage, MessageReceiver, MessageSender, SharedThreadState, TessellateMessage, + TileTessellateMessage, +}; +mod message; mod populate_tile_store_stage; mod request_stage; @@ -52,59 +58,6 @@ pub fn register_stages( ); } -type MessageSender = mpsc::Sender; -type MessageReceiver = mpsc::Receiver; - -/// [crate::io::TileTessellateMessage] or [crate::io::LayerTessellateMessage] tessellation message. -enum TessellateMessage { - Tile(TileTessellateMessage), - Layer(LayerTessellateMessage), -} - -/// The result of the tessellation of a tile. -struct TileTessellateMessage { - pub request_id: TileRequestID, - pub coords: WorldTileCoords, -} - -/// `TessellatedLayer` contains the result of the tessellation for a specific layer, otherwise -/// `UnavailableLayer` if the layer doesn't exist. -enum LayerTessellateMessage { - UnavailableLayer { - coords: WorldTileCoords, - layer_name: String, - }, - TessellatedLayer { - coords: WorldTileCoords, - buffer: OverAlignedVertexBuffer, - /// Holds for each feature the count of indices. - feature_indices: Vec, - layer_data: tile::Layer, - }, -} - -impl fmt::Debug for LayerTessellateMessage { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "LayerTessellateMessage{}", self.get_coords()) - } -} - -impl LayerTessellateMessage { - pub fn get_coords(&self) -> WorldTileCoords { - match self { - LayerTessellateMessage::UnavailableLayer { coords, .. } => *coords, - LayerTessellateMessage::TessellatedLayer { coords, .. } => *coords, - } - } - - pub fn layer_name(&self) -> &str { - match self { - LayerTessellateMessage::UnavailableLayer { layer_name, .. } => layer_name.as_str(), - LayerTessellateMessage::TessellatedLayer { layer_data, .. } => &layer_data.name, - } - } -} - pub struct HeadedPipelineProcessor { state: SharedThreadState, } @@ -159,78 +112,3 @@ impl PipelineProcessor for HeadedPipelineProcessor { } } } - -/// Stores and provides access to the thread safe data shared between the schedulers. -#[derive(Clone)] -pub struct SharedThreadState { - pub tile_request_state: Arc>, - pub message_sender: mpsc::Sender, - pub geometry_index: Arc>, -} - -impl SharedThreadState { - fn get_tile_request(&self, request_id: TileRequestID) -> Option { - self.tile_request_state - .lock() - .ok() - .and_then(|tile_request_state| tile_request_state.get_tile_request(request_id).cloned()) - } - - #[tracing::instrument(skip_all)] - pub fn process_tile(&self, request_id: TileRequestID, data: Box<[u8]>) -> Result<(), Error> { - if let Some(tile_request) = self.get_tile_request(request_id) { - let mut processor = HeadedPipelineProcessor { - state: self.clone(), - }; - let mut pipeline_context = PipelineContext { - processor: Box::new(processor), - }; - let pipeline = build_vector_tile_pipeline(); - pipeline.process((tile_request, request_id, data), &mut pipeline_context); - } - - Ok(()) - } - - pub fn tile_unavailable( - &self, - coords: &WorldTileCoords, - request_id: TileRequestID, - ) -> Result<(), Error> { - if let Some(tile_request) = self.get_tile_request(request_id) { - for to_load in &tile_request.layers { - tracing::warn!("layer {} at {} unavailable", to_load, coords); - self.message_sender.send(TessellateMessage::Layer( - LayerTessellateMessage::UnavailableLayer { - coords: tile_request.coords, - layer_name: to_load.to_string(), - }, - ))?; - } - } - - Ok(()) - } - - #[tracing::instrument(skip_all)] - pub fn query_point( - &self, - world_coords: &WorldCoords, - z: u8, - zoom: Zoom, - ) -> Option>> { - if let Ok(geometry_index) = self.geometry_index.lock() { - geometry_index - .query_point(world_coords, z, zoom) - .map(|geometries| { - geometries - .iter() - .cloned() - .cloned() - .collect::>>() - }) - } else { - unimplemented!() - } - } -} diff --git a/maplibre/src/stages/populate_tile_store_stage.rs b/maplibre/src/stages/populate_tile_store_stage.rs index 9d0bb961..54064a6d 100644 --- a/maplibre/src/stages/populate_tile_store_stage.rs +++ b/maplibre/src/stages/populate_tile_store_stage.rs @@ -1,8 +1,9 @@ -//! Receives data from async threads and populates the [`crate::io::tile_cache::TileCache`]. +//! Receives data from async threads and populates the [`crate::io::tile_repository::TileCache`]. +use super::{MessageReceiver, SharedThreadState, TessellateMessage, TileTessellateMessage}; use crate::context::MapContext; +use crate::io::tile_repository::StoredLayer; use crate::schedule::Stage; -use crate::stages::{MessageReceiver, SharedThreadState, TessellateMessage, TileTessellateMessage}; use std::sync::mpsc; pub struct PopulateTileStore { @@ -20,16 +21,22 @@ impl PopulateTileStore { } impl Stage for PopulateTileStore { - fn run(&mut self, MapContext { tile_cache, .. }: &mut MapContext) { + fn run( + &mut self, + MapContext { + tile_repository, .. + }: &mut MapContext, + ) { if let Ok(result) = self.message_receiver.try_recv() { match result { TessellateMessage::Layer(layer_result) => { + let layer: StoredLayer = layer_result.into(); tracing::trace!( "Layer {} at {} reached main thread", - layer_result.layer_name(), - layer_result.get_coords() + layer.layer_name(), + layer.get_coords() ); - tile_cache.put_tessellated_layer(layer_result); + tile_repository.put_tessellated_layer(layer); } TessellateMessage::Tile(TileTessellateMessage { request_id, coords }) => loop { if let Ok(mut tile_request_state) = diff --git a/maplibre/src/stages/request_stage.rs b/maplibre/src/stages/request_stage.rs index d4756672..da935dc5 100644 --- a/maplibre/src/stages/request_stage.rs +++ b/maplibre/src/stages/request_stage.rs @@ -4,7 +4,7 @@ use crate::context::MapContext; use crate::coords::{ViewRegion, WorldTileCoords}; use crate::error::Error; use crate::io::source_client::{HttpSourceClient, SourceClient}; -use crate::io::tile_cache::TileCache; +use crate::io::tile_repository::TileRepository; use crate::io::TileRequest; use crate::schedule::Stage; use crate::stages::SharedThreadState; @@ -48,7 +48,7 @@ where MapContext { view_state, style, - tile_cache, + tile_repository, .. }: &mut MapContext, ) { @@ -65,7 +65,7 @@ where { if let Some(view_region) = &view_region { // FIXME: We also need to request tiles from layers above if we are over the maximum zoom level - self.try_failed = self.request_tiles_in_view(tile_cache, style, view_region); + self.try_failed = self.request_tiles_in_view(tile_repository, style, view_region); } } @@ -82,7 +82,7 @@ where #[tracing::instrument(skip_all)] fn request_tiles_in_view( &self, - tile_cache: &TileCache, + tile_repository: &TileRepository, style: &Style, view_region: &ViewRegion, ) -> bool { @@ -97,7 +97,7 @@ where if coords.build_quad_key().is_some() { // TODO: Make tesselation depend on style? try_failed = self - .try_request_tile(tile_cache, &coords, &source_layers) + .try_request_tile(tile_repository, &coords, &source_layers) .unwrap(); } } @@ -106,11 +106,11 @@ where fn try_request_tile( &self, - tile_cache: &TileCache, + tile_repository: &TileRepository, coords: &WorldTileCoords, layers: &HashSet, ) -> Result { - if !tile_cache.is_layers_missing(coords, layers) { + if !tile_repository.is_layers_missing(coords, layers) { return Ok(false); } From 79a4f6d1955d43cc087f8b020f247e730a66e472 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Tue, 31 May 2022 16:31:17 +0200 Subject: [PATCH 16/31] Fix web --- maplibre-winit/src/winit/web.rs | 51 +- maplibre/src/map_schedule.rs | 4 +- .../src/render/copy_surface_to_buffer_node.rs | 1 - maplibre/src/render/main_pass.rs | 13 +- maplibre/src/render/mod.rs | 28 + .../src/render/stages/graph_runner_stage.rs | 43 +- maplibre/src/render/stages/mod.rs | 59 +- web/demo/package-lock.json | 7 +- web/lib/package-lock.json | 627 ------------------ web/lib/src/index.ts | 2 +- web/lib/src/legacy.worker.ts | 2 + web/src/platform/legacy_webworker_fetcher.rs | 1 - web/src/platform/mod.rs | 2 +- web/src/platform/schedule_method.rs | 8 +- 14 files changed, 115 insertions(+), 733 deletions(-) diff --git a/maplibre-winit/src/winit/web.rs b/maplibre-winit/src/winit/web.rs index 73912b4b..4d81d517 100644 --- a/maplibre-winit/src/winit/web.rs +++ b/maplibre-winit/src/winit/web.rs @@ -4,42 +4,12 @@ use super::WinitEventLoop; use super::WinitMapWindow; use super::WinitMapWindowConfig; use super::WinitWindow; +use maplibre::window::HeadedMapWindow; +use maplibre::window::MapWindowConfig; use maplibre::window::{MapWindow, WindowSize}; use winit::platform::web::WindowBuilderExtWebSys; -impl MapWindow for WinitMapWindow { - type EventLoop = WinitEventLoop; - type RawWindow = WinitWindow; - type MapWindowConfig = WinitMapWindowConfig; - - fn size(&self) -> WindowSize { - let size = self.window.inner_size(); - - WindowSize::new(size.width, size.height).expect("failed to get window dimensions.") - } - - fn inner(&self) -> &Self::RawWindow { - &self.window - } -} - -fn create(map_window_config: &Self::MapWindowConfig) -> Self { - let event_loop = WinitEventLoop::new(); - - let window: winit::window::Window = WindowBuilder::new() - .with_canvas(Some(get_canvas(&map_window_config.canvas_id))) - .build(&event_loop) - .unwrap(); - - let size = get_body_size().unwrap(); - window.set_inner_size(size); - Self { - window, - event_loop: Some(event_loop), - } -} - impl MapWindowConfig for WinitMapWindowConfig { type MapWindow = WinitMapWindow; @@ -47,7 +17,7 @@ impl MapWindowConfig for WinitMapWindowConfig { let event_loop = WinitEventLoop::new(); let window: winit::window::Window = WindowBuilder::new() - .with_canvas(Some(get_canvas(&map_window_config.canvas_id))) + .with_canvas(Some(get_canvas(&self.canvas_id))) .build(&event_loop) .unwrap(); @@ -60,6 +30,21 @@ impl MapWindowConfig for WinitMapWindowConfig { } } +impl MapWindow for WinitMapWindow { + fn size(&self) -> WindowSize { + let size = self.window.inner_size(); + + WindowSize::new(size.width, size.height).expect("failed to get window dimensions.") + } +} +impl HeadedMapWindow for WinitMapWindow { + type RawWindow = WinitWindow; + + fn inner(&self) -> &Self::RawWindow { + &self.window + } +} + pub fn get_body_size() -> Option> { let web_window: web_sys::Window = web_sys::window().unwrap(); let document = web_window.document().unwrap(); diff --git a/maplibre/src/map_schedule.rs b/maplibre/src/map_schedule.rs index 88115ec1..a8d56f90 100644 --- a/maplibre/src/map_schedule.rs +++ b/maplibre/src/map_schedule.rs @@ -104,7 +104,7 @@ where let http_source_client: HttpSourceClient = HttpSourceClient::new(http_client); register_stages(&mut schedule, http_source_client, Box::new(scheduler)); - register_render_stages(&mut schedule); + register_render_stages(&mut schedule, false); Self { map_window_config, @@ -247,7 +247,7 @@ where let tile_repository = TileRepository::new(); let mut schedule = Schedule::default(); - register_render_stages(&mut schedule); + register_render_stages(&mut schedule, true); Self { map_window_config, diff --git a/maplibre/src/render/copy_surface_to_buffer_node.rs b/maplibre/src/render/copy_surface_to_buffer_node.rs index c38cc484..da0af63d 100644 --- a/maplibre/src/render/copy_surface_to_buffer_node.rs +++ b/maplibre/src/render/copy_surface_to_buffer_node.rs @@ -2,7 +2,6 @@ use crate::render::graph::{Node, NodeRunError, RenderContext, RenderGraphContext use crate::render::render_commands::{DrawMasks, DrawTiles}; use crate::render::render_phase::{PhaseItem, RenderCommand}; use crate::render::resource::{Head, TrackedRenderPass}; -use crate::render::stages::draw_graph; use crate::render::util::FloatOrd; use crate::render::Eventually::Initialized; use crate::render::RenderState; diff --git a/maplibre/src/render/main_pass.rs b/maplibre/src/render/main_pass.rs index cfa2299d..b5374556 100644 --- a/maplibre/src/render/main_pass.rs +++ b/maplibre/src/render/main_pass.rs @@ -7,12 +7,21 @@ use crate::render::graph::{Node, NodeRunError, RenderContext, RenderGraphContext use crate::render::render_commands::{DrawMasks, DrawTiles}; use crate::render::render_phase::{PhaseItem, RenderCommand}; use crate::render::resource::TrackedRenderPass; -use crate::render::stages::draw_graph; use crate::render::util::FloatOrd; use crate::render::Eventually::Initialized; -use crate::render::RenderState; +use crate::render::{draw_graph, main_graph, RenderState}; use std::ops::{Deref, Range}; +pub mod graph { + // Labels for input nodes + pub mod input {} + // Labels for non-input nodes + pub mod node { + pub const MAIN_PASS_DEPENDENCIES: &str = "main_pass_dependencies"; + pub const MAIN_PASS_DRIVER: &str = "main_pass_driver"; + } +} + pub struct MainPassNode {} impl MainPassNode { diff --git a/maplibre/src/render/mod.rs b/maplibre/src/render/mod.rs index 7628a673..4ac852cc 100644 --- a/maplibre/src/render/mod.rs +++ b/maplibre/src/render/mod.rs @@ -32,6 +32,7 @@ use log::info; use std::sync::Arc; // Rendering internals +#[cfg(not(target_arch = "wasm32"))] mod copy_surface_to_buffer_node; mod graph; mod graph_runner; @@ -410,3 +411,30 @@ mod tests { RenderGraphRunner::run(&graph, &device, &queue, &render_state); } } + +// Plugins that contribute to the RenderGraph should use the following label conventions: +// 1. Graph modules should have a NAME, input module, and node module (where relevant) +// 2. The "top level" graph is the plugin module root. Just add things like `pub mod node` directly under the plugin module +// 3. "sub graph" modules should be nested beneath their parent graph module +pub mod main_graph { + // Labels for input nodes + pub mod input {} + // Labels for non-input nodes + pub mod node { + pub const MAIN_PASS_DEPENDENCIES: &str = "main_pass_dependencies"; + pub const MAIN_PASS_DRIVER: &str = "main_pass_driver"; + } +} + +/// Labels for the "draw" graph +pub mod draw_graph { + pub const NAME: &str = "draw"; + // Labels for input nodes + pub mod input {} + // Labels for non-input nodes + pub mod node { + pub const MAIN_PASS: &str = "main_pass"; + #[cfg(not(target_arch = "wasm32"))] + pub const COPY: &str = "copy"; + } +} diff --git a/maplibre/src/render/stages/graph_runner_stage.rs b/maplibre/src/render/stages/graph_runner_stage.rs index 9a9f7cb6..b0253bd8 100644 --- a/maplibre/src/render/stages/graph_runner_stage.rs +++ b/maplibre/src/render/stages/graph_runner_stage.rs @@ -1,12 +1,6 @@ //! Executes the [`RenderGraph`] current render graph. -// Plugins that contribute to the RenderGraph should use the following label conventions: -// 1. Graph modules should have a NAME, input module, and node module (where relevant) -// 2. The "top level" graph is the plugin module root. Just add things like `pub mod node` directly under the plugin module -// 3. "sub graph" modules should be nested beneath their parent graph module - use crate::context::MapContext; -use crate::render::copy_surface_to_buffer_node::CopySurfaceBufferNode; use crate::render::graph::{EmptyNode, RenderGraph}; use crate::render::graph_runner::RenderGraphRunner; use crate::render::main_pass::{MainPassDriverNode, MainPassNode}; @@ -15,46 +9,13 @@ use crate::schedule::Stage; use crate::Renderer; use log::error; -pub mod node { - pub const MAIN_PASS_DEPENDENCIES: &str = "main_pass_dependencies"; - pub const MAIN_PASS_DRIVER: &str = "main_pass_driver"; -} - -pub mod draw_graph { - pub const NAME: &str = "draw"; - pub mod input {} - pub mod node { - pub const MAIN_PASS: &str = "main_pass"; - pub const COPY: &str = "copy"; - } -} - /// Updates the [`RenderGraph`] with all of its nodes and then runs it to render the entire frame. pub struct GraphRunnerStage { graph: RenderGraph, } -impl Default for GraphRunnerStage { - fn default() -> Self { - let mut graph = RenderGraph::default(); - - let mut draw_graph = RenderGraph::default(); - draw_graph.add_node(draw_graph::node::MAIN_PASS, MainPassNode::new()); - let input_node_id = draw_graph.set_input(vec![]); - draw_graph - .add_node_edge(input_node_id, draw_graph::node::MAIN_PASS) - .unwrap(); - draw_graph.add_node(draw_graph::node::COPY, CopySurfaceBufferNode::default()); - draw_graph - .add_node_edge(draw_graph::node::MAIN_PASS, draw_graph::node::COPY) - .unwrap(); - - graph.add_sub_graph(draw_graph::NAME, draw_graph); - graph.add_node(node::MAIN_PASS_DEPENDENCIES, EmptyNode); - graph.add_node(node::MAIN_PASS_DRIVER, MainPassDriverNode); - graph - .add_node_edge(node::MAIN_PASS_DEPENDENCIES, node::MAIN_PASS_DRIVER) - .unwrap(); +impl GraphRunnerStage { + pub fn new(graph: RenderGraph) -> Self { Self { graph } } } diff --git a/maplibre/src/render/stages/mod.rs b/maplibre/src/render/stages/mod.rs index 3b44e2d9..676644d4 100644 --- a/maplibre/src/render/stages/mod.rs +++ b/maplibre/src/render/stages/mod.rs @@ -1,6 +1,13 @@ //! Rendering specific [Stages](Stage) use crate::context::MapContext; +use crate::multi_stage; +use crate::render::graph::{EmptyNode, RenderGraph, RenderGraphError}; +use crate::render::main_pass::{MainPassDriverNode, MainPassNode}; +use crate::render::stages::extract_stage::ExtractStage; +use crate::render::stages::phase_sort_stage::PhaseSortStage; +use crate::render::stages::queue_stage::QueueStage; +use crate::render::{draw_graph, main_graph}; use crate::schedule::{MultiStage, Schedule, Stage, StageLabel}; use graph_runner_stage::GraphRunnerStage; use resource_stage::ResourceStage; @@ -12,15 +19,9 @@ mod phase_sort_stage; mod queue_stage; mod resource_stage; mod upload_stage; +#[cfg(not(target_arch = "wasm32"))] mod write_surface_buffer_stage; -use crate::multi_stage; -use crate::render::stages::extract_stage::ExtractStage; -use crate::render::stages::phase_sort_stage::PhaseSortStage; -use crate::render::stages::queue_stage::QueueStage; -use crate::render::stages::write_surface_buffer_stage::WriteSurfaceBufferStage; -pub use graph_runner_stage::{draw_graph, node}; - /// The labels of the default App rendering stages. #[derive(Debug, Hash, PartialEq, Eq, Clone)] pub enum RenderStageLabel { @@ -57,13 +58,45 @@ multi_stage!( upload: UploadStage ); -pub fn register_render_stages(schedule: &mut Schedule) { +pub fn register_render_stages( + schedule: &mut Schedule, + headless: bool, +) -> Result<(), RenderGraphError> { + let mut graph = RenderGraph::default(); + + let mut draw_graph = RenderGraph::default(); + draw_graph.add_node(draw_graph::node::MAIN_PASS, MainPassNode::new()); + let input_node_id = draw_graph.set_input(vec![]); + draw_graph.add_node_edge(input_node_id, draw_graph::node::MAIN_PASS)?; + + #[cfg(not(target_arch = "wasm32"))] + if headless { + use crate::render::copy_surface_to_buffer_node::CopySurfaceBufferNode; + draw_graph.add_node(draw_graph::node::COPY, CopySurfaceBufferNode::default()); + draw_graph.add_node_edge(draw_graph::node::MAIN_PASS, draw_graph::node::COPY)?; + } + + graph.add_sub_graph(draw_graph::NAME, draw_graph); + graph.add_node(main_graph::node::MAIN_PASS_DEPENDENCIES, EmptyNode); + graph.add_node(main_graph::node::MAIN_PASS_DRIVER, MainPassDriverNode); + graph.add_node_edge( + main_graph::node::MAIN_PASS_DEPENDENCIES, + main_graph::node::MAIN_PASS_DRIVER, + )?; + schedule.add_stage(RenderStageLabel::Prepare, PrepareStage::default()); schedule.add_stage(RenderStageLabel::Queue, QueueStage::default()); schedule.add_stage(RenderStageLabel::PhaseSort, PhaseSortStage::default()); - schedule.add_stage(RenderStageLabel::Render, GraphRunnerStage::default()); - schedule.add_stage( - RenderStageLabel::Cleanup, - WriteSurfaceBufferStage::default(), - ); + schedule.add_stage(RenderStageLabel::Render, GraphRunnerStage::new(graph)); + + #[cfg(not(target_arch = "wasm32"))] + if headless { + use crate::render::stages::write_surface_buffer_stage::WriteSurfaceBufferStage; + schedule.add_stage( + RenderStageLabel::Cleanup, + WriteSurfaceBufferStage::default(), + ); + } + + Ok(()) } diff --git a/web/demo/package-lock.json b/web/demo/package-lock.json index 47cd8f38..8478c320 100644 --- a/web/demo/package-lock.json +++ b/web/demo/package-lock.json @@ -28,7 +28,6 @@ "../lib": { "name": "maplibre-rs", "version": "0.0.1", - "hasInstallScript": true, "license": "MIT", "dependencies": { "spectorjs": "^0.9.27", @@ -36,10 +35,9 @@ }, "devDependencies": { "@chialab/esbuild-plugin-env": "^0.15.3", - "@chialab/esbuild-plugin-meta-url": "^0.15.15", + "@chialab/esbuild-plugin-meta-url": "^0.15.28", "esbuild": "^0.14.38", "esbuild-plugin-inline-worker": "^0.1.1", - "patch-package": "^6.4.7", "ts-loader": "^9.2.8", "typescript": "^4.5.4", "wasm-pack": "^0.10.2" @@ -5892,10 +5890,9 @@ "version": "file:../lib", "requires": { "@chialab/esbuild-plugin-env": "^0.15.3", - "@chialab/esbuild-plugin-meta-url": "^0.15.15", + "@chialab/esbuild-plugin-meta-url": "^0.15.28", "esbuild": "^0.14.38", "esbuild-plugin-inline-worker": "^0.1.1", - "patch-package": "^6.4.7", "spectorjs": "^0.9.27", "ts-loader": "^9.2.8", "typescript": "^4.5.4", diff --git a/web/lib/package-lock.json b/web/lib/package-lock.json index 823db7d9..50a9f44d 100644 --- a/web/lib/package-lock.json +++ b/web/lib/package-lock.json @@ -7,7 +7,6 @@ "": { "name": "maplibre-rs", "version": "0.0.1", - "hasInstallScript": true, "license": "MIT", "dependencies": { "spectorjs": "^0.9.27", @@ -18,7 +17,6 @@ "@chialab/esbuild-plugin-meta-url": "^0.15.28", "esbuild": "^0.14.38", "esbuild-plugin-inline-worker": "^0.1.1", - "patch-package": "^6.4.7", "ts-loader": "^9.2.8", "typescript": "^4.5.4", "wasm-pack": "^0.10.2" @@ -288,12 +286,6 @@ "license": "Apache-2.0", "peer": true }, - "node_modules/@yarnpkg/lockfile": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz", - "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==", - "dev": true - }, "node_modules/acorn": { "version": "8.7.0", "dev": true, @@ -315,18 +307,6 @@ "acorn": "^8" } }, - "node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/axios": { "version": "0.21.4", "dev": true, @@ -424,20 +404,6 @@ "license": "CC-BY-4.0", "peer": true }, - "node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/chownr": { "version": "2.0.0", "dev": true, @@ -455,27 +421,6 @@ "node": ">=6.0" } }, - "node_modules/ci-info": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", - "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", - "dev": true - }, - "node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", - "dev": true - }, "node_modules/commondir": { "version": "1.0.1", "dev": true, @@ -486,31 +431,6 @@ "dev": true, "license": "MIT" }, - "node_modules/cross-spawn": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", - "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", - "dev": true, - "dependencies": { - "nice-try": "^1.0.4", - "path-key": "^2.0.1", - "semver": "^5.5.0", - "shebang-command": "^1.2.0", - "which": "^1.2.9" - }, - "engines": { - "node": ">=4.8" - } - }, - "node_modules/cross-spawn/node_modules/semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true, - "bin": { - "semver": "bin/semver" - } - }, "node_modules/detect-libc": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", @@ -614,15 +534,6 @@ "node": ">=6" } }, - "node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/eslint-scope": { "version": "5.1.1", "dev": true, @@ -726,15 +637,6 @@ "node": ">=8" } }, - "node_modules/find-yarn-workspace-root": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/find-yarn-workspace-root/-/find-yarn-workspace-root-2.0.0.tgz", - "integrity": "sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ==", - "dev": true, - "dependencies": { - "micromatch": "^4.0.2" - } - }, "node_modules/follow-redirects": { "version": "1.14.9", "dev": true, @@ -754,20 +656,6 @@ } } }, - "node_modules/fs-extra": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", - "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", - "dev": true, - "dependencies": { - "graceful-fs": "^4.1.2", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" - }, - "engines": { - "node": ">=6 <7 || >=8" - } - }, "node_modules/fs-minipass": { "version": "2.1.0", "dev": true, @@ -814,15 +702,6 @@ "dev": true, "license": "ISC" }, - "node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", - "dev": true, - "engines": { - "node": ">=4" - } - }, "node_modules/inflight": { "version": "1.0.6", "dev": true, @@ -837,33 +716,6 @@ "dev": true, "license": "ISC" }, - "node_modules/is-ci": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", - "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", - "dev": true, - "dependencies": { - "ci-info": "^2.0.0" - }, - "bin": { - "is-ci": "bin.js" - } - }, - "node_modules/is-docker": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", - "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", - "dev": true, - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-number": { "version": "7.0.0", "dev": true, @@ -872,24 +724,6 @@ "node": ">=0.12.0" } }, - "node_modules/is-wsl": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", - "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", - "dev": true, - "dependencies": { - "is-docker": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", - "dev": true - }, "node_modules/jest-worker": { "version": "27.5.1", "dev": true, @@ -934,24 +768,6 @@ "license": "MIT", "peer": true }, - "node_modules/jsonfile": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", - "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", - "dev": true, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, - "node_modules/klaw-sync": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/klaw-sync/-/klaw-sync-6.0.0.tgz", - "integrity": "sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ==", - "dev": true, - "dependencies": { - "graceful-fs": "^4.1.11" - } - }, "node_modules/loader-runner": { "version": "4.3.0", "dev": true, @@ -1055,12 +871,6 @@ "node": "*" } }, - "node_modules/minimist": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", - "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", - "dev": true - }, "node_modules/minipass": { "version": "3.1.6", "dev": true, @@ -1101,12 +911,6 @@ "license": "MIT", "peer": true }, - "node_modules/nice-try": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", - "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", - "dev": true - }, "node_modules/node-releases": { "version": "2.0.3", "dev": true, @@ -1121,31 +925,6 @@ "wrappy": "1" } }, - "node_modules/open": { - "version": "7.4.2", - "resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz", - "integrity": "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==", - "dev": true, - "dependencies": { - "is-docker": "^2.0.0", - "is-wsl": "^2.1.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/os-tmpdir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", - "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/p-limit": { "version": "2.3.0", "dev": true, @@ -1179,54 +958,6 @@ "node": ">=6" } }, - "node_modules/patch-package": { - "version": "6.4.7", - "resolved": "https://registry.npmjs.org/patch-package/-/patch-package-6.4.7.tgz", - "integrity": "sha512-S0vh/ZEafZ17hbhgqdnpunKDfzHQibQizx9g8yEf5dcVk3KOflOfdufRXQX8CSEkyOQwuM/bNz1GwKvFj54kaQ==", - "dev": true, - "dependencies": { - "@yarnpkg/lockfile": "^1.1.0", - "chalk": "^2.4.2", - "cross-spawn": "^6.0.5", - "find-yarn-workspace-root": "^2.0.0", - "fs-extra": "^7.0.1", - "is-ci": "^2.0.0", - "klaw-sync": "^6.0.0", - "minimist": "^1.2.0", - "open": "^7.4.2", - "rimraf": "^2.6.3", - "semver": "^5.6.0", - "slash": "^2.0.0", - "tmp": "^0.0.33" - }, - "bin": { - "patch-package": "index.js" - }, - "engines": { - "npm": ">5" - } - }, - "node_modules/patch-package/node_modules/rimraf": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", - "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", - "dev": true, - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - } - }, - "node_modules/patch-package/node_modules/semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true, - "bin": { - "semver": "bin/semver" - } - }, "node_modules/path-exists": { "version": "4.0.0", "dev": true, @@ -1243,15 +974,6 @@ "node": ">=0.10.0" } }, - "node_modules/path-key": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", - "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", - "dev": true, - "engines": { - "node": ">=4" - } - }, "node_modules/picocolors": { "version": "1.0.0", "dev": true, @@ -1355,36 +1077,6 @@ "randombytes": "^2.1.0" } }, - "node_modules/shebang-command": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", - "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", - "dev": true, - "dependencies": { - "shebang-regex": "^1.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/shebang-regex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", - "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/slash": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz", - "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==", - "dev": true, - "engines": { - "node": ">=6" - } - }, "node_modules/source-map": { "version": "0.6.1", "dev": true, @@ -1408,18 +1100,6 @@ "version": "0.9.27", "license": "MIT" }, - "node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/tapable": { "version": "2.2.1", "dev": true, @@ -1560,18 +1240,6 @@ "node": ">= 8" } }, - "node_modules/tmp": { - "version": "0.0.33", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", - "dev": true, - "dependencies": { - "os-tmpdir": "~1.0.2" - }, - "engines": { - "node": ">=0.6.0" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "dev": true, @@ -1677,15 +1345,6 @@ "node": ">=4.2.0" } }, - "node_modules/universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", - "dev": true, - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/uri-js": { "version": "4.4.1", "dev": true, @@ -1829,18 +1488,6 @@ "url": "https://opencollective.com/webpack" } }, - "node_modules/which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "dev": true, - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "which": "bin/which" - } - }, "node_modules/wrappy": { "version": "1.0.2", "dev": true, @@ -2076,12 +1723,6 @@ "dev": true, "peer": true }, - "@yarnpkg/lockfile": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz", - "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==", - "dev": true - }, "acorn": { "version": "8.7.0", "dev": true, @@ -2093,15 +1734,6 @@ "peer": true, "requires": {} }, - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } - }, "axios": { "version": "0.21.4", "dev": true, @@ -2159,17 +1791,6 @@ "dev": true, "peer": true }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, "chownr": { "version": "2.0.0", "dev": true @@ -2179,27 +1800,6 @@ "dev": true, "peer": true }, - "ci-info": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", - "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", - "dev": true - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", - "dev": true - }, "commondir": { "version": "1.0.1", "dev": true @@ -2208,27 +1808,6 @@ "version": "0.0.1", "dev": true }, - "cross-spawn": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", - "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", - "dev": true, - "requires": { - "nice-try": "^1.0.4", - "path-key": "^2.0.1", - "semver": "^5.5.0", - "shebang-command": "^1.2.0", - "which": "^1.2.9" - }, - "dependencies": { - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true - } - } - }, "detect-libc": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", @@ -2297,12 +1876,6 @@ "dev": true, "peer": true }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", - "dev": true - }, "eslint-scope": { "version": "5.1.1", "dev": true, @@ -2371,30 +1944,10 @@ "path-exists": "^4.0.0" } }, - "find-yarn-workspace-root": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/find-yarn-workspace-root/-/find-yarn-workspace-root-2.0.0.tgz", - "integrity": "sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ==", - "dev": true, - "requires": { - "micromatch": "^4.0.2" - } - }, "follow-redirects": { "version": "1.14.9", "dev": true }, - "fs-extra": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", - "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", - "dev": true, - "requires": { - "graceful-fs": "^4.1.2", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" - } - }, "fs-minipass": { "version": "2.1.0", "dev": true, @@ -2427,12 +1980,6 @@ "version": "4.2.10", "dev": true }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", - "dev": true - }, "inflight": { "version": "1.0.6", "dev": true, @@ -2445,40 +1992,10 @@ "version": "2.0.4", "dev": true }, - "is-ci": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", - "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", - "dev": true, - "requires": { - "ci-info": "^2.0.0" - } - }, - "is-docker": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", - "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", - "dev": true - }, "is-number": { "version": "7.0.0", "dev": true }, - "is-wsl": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", - "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", - "dev": true, - "requires": { - "is-docker": "^2.0.0" - } - }, - "isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", - "dev": true - }, "jest-worker": { "version": "27.5.1", "dev": true, @@ -2509,24 +2026,6 @@ "dev": true, "peer": true }, - "jsonfile": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", - "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", - "dev": true, - "requires": { - "graceful-fs": "^4.1.6" - } - }, - "klaw-sync": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/klaw-sync/-/klaw-sync-6.0.0.tgz", - "integrity": "sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ==", - "dev": true, - "requires": { - "graceful-fs": "^4.1.11" - } - }, "loader-runner": { "version": "4.3.0", "dev": true, @@ -2592,12 +2091,6 @@ "brace-expansion": "^1.1.7" } }, - "minimist": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", - "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", - "dev": true - }, "minipass": { "version": "3.1.6", "dev": true, @@ -2622,12 +2115,6 @@ "dev": true, "peer": true }, - "nice-try": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", - "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", - "dev": true - }, "node-releases": { "version": "2.0.3", "dev": true, @@ -2640,22 +2127,6 @@ "wrappy": "1" } }, - "open": { - "version": "7.4.2", - "resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz", - "integrity": "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==", - "dev": true, - "requires": { - "is-docker": "^2.0.0", - "is-wsl": "^2.1.1" - } - }, - "os-tmpdir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", - "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", - "dev": true - }, "p-limit": { "version": "2.3.0", "dev": true, @@ -2674,44 +2145,6 @@ "version": "2.2.0", "dev": true }, - "patch-package": { - "version": "6.4.7", - "resolved": "https://registry.npmjs.org/patch-package/-/patch-package-6.4.7.tgz", - "integrity": "sha512-S0vh/ZEafZ17hbhgqdnpunKDfzHQibQizx9g8yEf5dcVk3KOflOfdufRXQX8CSEkyOQwuM/bNz1GwKvFj54kaQ==", - "dev": true, - "requires": { - "@yarnpkg/lockfile": "^1.1.0", - "chalk": "^2.4.2", - "cross-spawn": "^6.0.5", - "find-yarn-workspace-root": "^2.0.0", - "fs-extra": "^7.0.1", - "is-ci": "^2.0.0", - "klaw-sync": "^6.0.0", - "minimist": "^1.2.0", - "open": "^7.4.2", - "rimraf": "^2.6.3", - "semver": "^5.6.0", - "slash": "^2.0.0", - "tmp": "^0.0.33" - }, - "dependencies": { - "rimraf": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", - "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", - "dev": true, - "requires": { - "glob": "^7.1.3" - } - }, - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true - } - } - }, "path-exists": { "version": "4.0.0", "dev": true @@ -2720,12 +2153,6 @@ "version": "1.0.1", "dev": true }, - "path-key": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", - "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", - "dev": true - }, "picocolors": { "version": "1.0.0", "dev": true, @@ -2782,27 +2209,6 @@ "randombytes": "^2.1.0" } }, - "shebang-command": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", - "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", - "dev": true, - "requires": { - "shebang-regex": "^1.0.0" - } - }, - "shebang-regex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", - "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", - "dev": true - }, - "slash": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz", - "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==", - "dev": true - }, "source-map": { "version": "0.6.1", "dev": true, @@ -2820,15 +2226,6 @@ "spectorjs": { "version": "0.9.27" }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - }, "tapable": { "version": "2.2.1", "dev": true @@ -2914,15 +2311,6 @@ } } }, - "tmp": { - "version": "0.0.33", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", - "dev": true, - "requires": { - "os-tmpdir": "~1.0.2" - } - }, "to-regex-range": { "version": "5.0.1", "dev": true, @@ -2983,12 +2371,6 @@ "version": "4.6.3", "dev": true }, - "universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", - "dev": true - }, "uri-js": { "version": "4.4.1", "dev": true, @@ -3086,15 +2468,6 @@ "dev": true, "peer": true }, - "which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "dev": true, - "requires": { - "isexe": "^2.0.0" - } - }, "wrappy": { "version": "1.0.2", "dev": true diff --git a/web/lib/src/index.ts b/web/lib/src/index.ts index ffc0e661..56c00921 100644 --- a/web/lib/src/index.ts +++ b/web/lib/src/index.ts @@ -1,4 +1,4 @@ -import init, {create_pool_scheduler, new_thread_local_state, run} from "./wasm-pack" +import init, {create_pool_scheduler, run} from "./wasm-pack" import {Spector} from "spectorjs" import {WebWorkerMessageType} from "./types" import { diff --git a/web/lib/src/legacy.worker.ts b/web/lib/src/legacy.worker.ts index 0f7b1c25..c9a6010a 100644 --- a/web/lib/src/legacy.worker.ts +++ b/web/lib/src/legacy.worker.ts @@ -1,3 +1,4 @@ +/* import init, {InitOutput, tessellate_layers} from "./wasm-pack" import {WebWorkerMessageType} from "./types" @@ -30,3 +31,4 @@ onmessage = async message => { break } } +*/ diff --git a/web/src/platform/legacy_webworker_fetcher.rs b/web/src/platform/legacy_webworker_fetcher.rs index c099ba28..4129fb88 100644 --- a/web/src/platform/legacy_webworker_fetcher.rs +++ b/web/src/platform/legacy_webworker_fetcher.rs @@ -5,7 +5,6 @@ use wasm_bindgen::prelude::*; use maplibre::coords::TileCoords; use maplibre::io::scheduler::Scheduler; -use maplibre::io::shared_thread_state::SharedThreadState; use maplibre::io::TileRequestID; diff --git a/web/src/platform/mod.rs b/web/src/platform/mod.rs index 4b07af78..8522af98 100644 --- a/web/src/platform/mod.rs +++ b/web/src/platform/mod.rs @@ -1,4 +1,4 @@ pub mod http_client; -pub mod legacy_webworker_fetcher; +//pub mod legacy_webworker_fetcher; pub mod pool; pub mod schedule_method; diff --git a/web/src/platform/schedule_method.rs b/web/src/platform/schedule_method.rs index 6596d74d..8949ccca 100644 --- a/web/src/platform/schedule_method.rs +++ b/web/src/platform/schedule_method.rs @@ -8,7 +8,6 @@ use web_sys::Worker; use maplibre::error::Error; use maplibre::io::scheduler::ScheduleMethod; -use maplibre::io::shared_thread_state::SharedThreadState; use super::pool::WorkerPool; @@ -37,17 +36,14 @@ impl WebWorkerPoolScheduleMethod { impl ScheduleMethod for WebWorkerPoolScheduleMethod { fn schedule( &self, - shared_thread_state: SharedThreadState, future_factory: Box< - (dyn (FnOnce(SharedThreadState) -> Pin + 'static>>) - + Send - + 'static), + (dyn (FnOnce() -> Pin + 'static>>) + Send + 'static), >, ) -> Result<(), Error> { self.pool .run(move || { wasm_bindgen_futures::future_to_promise(async move { - future_factory(shared_thread_state).await; + future_factory().await; Ok(JsValue::undefined()) }) }) From e1706b02283609c7301179515f2fbb8d3ccd915d Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Tue, 31 May 2022 20:49:03 +0200 Subject: [PATCH 17/31] Remove mem::replace usage --- maplibre/src/map_schedule.rs | 15 ++++++--------- maplibre/src/render/util/mod.rs | 4 ++-- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/maplibre/src/map_schedule.rs b/maplibre/src/map_schedule.rs index a8d56f90..4308e583 100644 --- a/maplibre/src/map_schedule.rs +++ b/maplibre/src/map_schedule.rs @@ -47,15 +47,12 @@ impl EventuallyMapContext { tile_repository, .. }) => { - mem::replace( - self, - EventuallyMapContext::Full(MapContext { - view_state, - style, - tile_repository, - renderer, - }), - ); + *self = EventuallyMapContext::Full(MapContext { + view_state, + style, + tile_repository, + renderer, + }); } EventuallyMapContext::_Uninitialized => {} } diff --git a/maplibre/src/render/util/mod.rs b/maplibre/src/render/util/mod.rs index befa9d42..30805a5f 100644 --- a/maplibre/src/render/util/mod.rs +++ b/maplibre/src/render/util/mod.rs @@ -79,7 +79,7 @@ where }; if should_replace { - mem::replace(self, Eventually::Initialized(f())); + *self = Eventually::Initialized(f()); } } } @@ -87,7 +87,7 @@ impl Eventually { #[tracing::instrument(name = "initialize", skip_all)] pub fn initialize(&mut self, f: impl FnOnce() -> T) { if let Eventually::Uninitialized = self { - mem::replace(self, Eventually::Initialized(f())); + *self = Eventually::Initialized(f()); } } From e1aceabd0b5eb494fe7d8d390809fb90e6af257e Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Tue, 31 May 2022 20:49:16 +0200 Subject: [PATCH 18/31] Improve pipeline types --- maplibre/src/io/pipeline.rs | 82 ++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/maplibre/src/io/pipeline.rs b/maplibre/src/io/pipeline.rs index fc2025a6..144dcbbb 100644 --- a/maplibre/src/io/pipeline.rs +++ b/maplibre/src/io/pipeline.rs @@ -121,26 +121,24 @@ impl Processable for fn(input: I, context: &mut PipelineContext) -> O { } } -pub struct FnProcessable { - func: &'static fn(I, context: &mut PipelineContext) -> O, -} - -impl Processable for FnProcessable { - type Input = I; - type Output = O; - - fn process(&self, input: Self::Input, context: &mut PipelineContext) -> Self::Output { - (self.func)(input, context) - } -} - pub struct ClosureProcessable where F: Fn(I, &mut PipelineContext) -> O, { func: F, phantom_i: PhantomData, - phantom_o: PhantomData, +} + +impl From for ClosureProcessable +where + F: Fn(I, &mut PipelineContext) -> O, +{ + fn from(func: F) -> Self { + ClosureProcessable { + func, + phantom_i: PhantomData::default(), + } + } } impl Processable for ClosureProcessable @@ -155,24 +153,10 @@ where } } -pub struct Closure2Processable { - func: fn(I, context: &mut PipelineContext) -> O, -} - -impl Processable for Closure2Processable { - type Input = I; - type Output = O; - - fn process(&self, input: Self::Input, context: &mut PipelineContext) -> Self::Output { - (self.func)(input, context) - } -} - #[cfg(test)] mod tests { use crate::io::pipeline::{ - Closure2Processable, ClosureProcessable, EndStep, FnProcessable, HeadedPipelineProcessor, - PipelineContext, PipelineProcessor, PipelineStep, Processable, + ClosureProcessable, EndStep, PipelineContext, PipelineProcessor, PipelineStep, Processable, }; use std::sync::mpsc; @@ -194,9 +178,7 @@ mod tests { processor: Box::new(DummyPipelineProcessor), }; let output: u32 = PipelineStep { - process: FnProcessable { - func: &(add_two as fn(u8, &mut PipelineContext) -> u32), - }, + process: add_two as fn(u8, &mut PipelineContext) -> u32, next: EndStep::default(), } .process(5u8, &mut context); @@ -204,9 +186,9 @@ mod tests { assert_eq!(output, 7); let output = PipelineStep { - process: &(add_one as fn(u32, &mut PipelineContext) -> u8), + process: add_one as fn(u32, &mut PipelineContext) -> u8, next: PipelineStep { - process: &(add_two as fn(u8, &mut PipelineContext) -> u32), + process: add_two as fn(u8, &mut PipelineContext) -> u32, next: EndStep::default(), }, } @@ -214,28 +196,42 @@ mod tests { assert_eq!(output, 8); + let mut a = 3; + let closure = |input: u8, context: &mut PipelineContext| -> u32 { + return input as u32 + 2 + a; + }; let output: u32 = PipelineStep { process: ClosureProcessable { - func: |input: u8, context| -> u32 { - return input as u32 + 2; - }, + func: closure, phantom_i: Default::default(), - phantom_o: Default::default(), }, next: EndStep::default(), } .process(5u8, &mut context); - assert_eq!(output, 7); + assert_eq!(output, 10); + let processable = + ClosureProcessable::from(|input: u8, context: &mut PipelineContext| -> u32 { + return input as u32 + 2 + a; + }); let output: u32 = PipelineStep { - process: Closure2Processable { - func: |input: u8, context| -> u32 { input as u32 + 2 }, - }, + process: processable, next: EndStep::default(), } .process(5u8, &mut context); - assert_eq!(output, 7); + assert_eq!(output, 10); + + let output: u32 = PipelineStep::, _>::new( + (|input: u8, context: &mut PipelineContext| -> u32 { + return input as u32 + 2 + a; + }) + .into(), + EndStep::::default(), + ) + .process(5u8, &mut context); + + assert_eq!(output, 10); } } From b80aa42fdb58b85af85c84c658b973e8fed0d507 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Tue, 31 May 2022 21:03:20 +0200 Subject: [PATCH 19/31] Simplify pipeline API --- maplibre-demo/src/main.rs | 2 +- maplibre/src/io/pipeline.rs | 21 ++++++++++++++++++--- maplibre/src/io/pipeline_steps.rs | 18 ++++++++++-------- maplibre/src/map_schedule.rs | 6 +++--- maplibre/src/stages/message.rs | 7 ++----- maplibre/src/stages/mod.rs | 15 +++++++++------ 6 files changed, 43 insertions(+), 26 deletions(-) diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index 0f45805e..9532afcb 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -133,7 +133,7 @@ fn run_headless() { &mut pipeline_context, ); - let mut processor = pipeline_context.teardown(); + let mut processor = pipeline_context.take_processor(); while let Some(v) = processor .as_any_mut() diff --git a/maplibre/src/io/pipeline.rs b/maplibre/src/io/pipeline.rs index 144dcbbb..0ead12ad 100644 --- a/maplibre/src/io/pipeline.rs +++ b/maplibre/src/io/pipeline.rs @@ -33,12 +33,27 @@ pub trait PipelineProcessor: Downcast { impl_downcast!(PipelineProcessor); pub struct PipelineContext { - pub processor: Box, + processor: Box, } impl PipelineContext { - pub fn teardown(self) -> Box { - self.processor + pub fn new

(processor: P) -> Self + where + P: PipelineProcessor, + { + Self { + processor: Box::new(processor), + } + } + + pub fn take_processor

(self) -> Option> + where + P: PipelineProcessor, + { + self.processor.into_any().downcast::

().ok() + } + pub fn processor_mut(&mut self) -> &mut dyn PipelineProcessor { + self.processor.as_mut() } } diff --git a/maplibre/src/io/pipeline_steps.rs b/maplibre/src/io/pipeline_steps.rs index 9749c90a..894d4385 100644 --- a/maplibre/src/io/pipeline_steps.rs +++ b/maplibre/src/io/pipeline_steps.rs @@ -39,7 +39,7 @@ impl Processable for IndexLayerStep { let index = IndexProcessor::new(); context - .processor + .processor_mut() .finished_layer_indexing(&tile_request.coords, index.get_geometries()); (tile_request, request_id, tile) } @@ -70,7 +70,9 @@ impl Processable for TessellateLayerStep { let mut tessellator = ZeroTessellator::::default(); if let Err(e) = layer.process(&mut tessellator) { - context.processor.unavailable_layer(coords, layer_name); + context + .processor_mut() + .unavailable_layer(coords, layer_name); tracing::error!( "layer {} at {} tesselation failed {:?}", @@ -79,7 +81,7 @@ impl Processable for TessellateLayerStep { e ); } else { - context.processor.finished_layer_tesselation( + context.processor_mut().finished_layer_tesselation( coords, tessellator.buffer.into(), tessellator.feature_indices, @@ -95,7 +97,9 @@ impl Processable for TessellateLayerStep { .collect::>(); for missing_layer in tile_request.layers.difference(&available_layers) { - context.processor.unavailable_layer(coords, missing_layer); + context + .processor_mut() + .unavailable_layer(coords, missing_layer); tracing::info!( "requested layer {} at {} not found in tile", @@ -107,7 +111,7 @@ impl Processable for TessellateLayerStep { tracing::info!("tile tessellated at {} finished", &tile_request.coords); context - .processor + .processor_mut() .finished_tile_tesselation(request_id, &tile_request.coords); (tile_request, request_id, tile) @@ -133,9 +137,7 @@ mod tests { #[test] fn test() { - let mut context = PipelineContext { - processor: Box::new(DummyPipelineProcessor), - }; + let mut context = PipelineContext::new(DummyPipelineProcessor); let pipeline = build_vector_tile_pipeline(); let output = pipeline.process( diff --git a/maplibre/src/map_schedule.rs b/maplibre/src/map_schedule.rs index 4308e583..d38eb3b0 100644 --- a/maplibre/src/map_schedule.rs +++ b/maplibre/src/map_schedule.rs @@ -101,7 +101,7 @@ where let http_source_client: HttpSourceClient = HttpSourceClient::new(http_client); register_stages(&mut schedule, http_source_client, Box::new(scheduler)); - register_render_stages(&mut schedule, false); + register_render_stages(&mut schedule, false).unwrap(); Self { map_window_config, @@ -194,7 +194,7 @@ where ) .await .unwrap(); - &self.map_context.make_full(renderer); + self.map_context.make_full(renderer); true } EventuallyMapContext::_Uninitialized => false, @@ -244,7 +244,7 @@ where let tile_repository = TileRepository::new(); let mut schedule = Schedule::default(); - register_render_stages(&mut schedule, true); + register_render_stages(&mut schedule, true).unwrap(); Self { map_window_config, diff --git a/maplibre/src/stages/message.rs b/maplibre/src/stages/message.rs index 5c5d9cc1..7c441db7 100644 --- a/maplibre/src/stages/message.rs +++ b/maplibre/src/stages/message.rs @@ -98,12 +98,9 @@ impl SharedThreadState { #[tracing::instrument(skip_all)] pub fn process_tile(&self, request_id: TileRequestID, data: Box<[u8]>) -> Result<(), Error> { if let Some(tile_request) = self.get_tile_request(request_id) { - let mut processor = HeadedPipelineProcessor { + let mut pipeline_context = PipelineContext::new(HeadedPipelineProcessor { state: self.clone(), - }; - let mut pipeline_context = PipelineContext { - processor: Box::new(processor), - }; + }); let pipeline = build_vector_tile_pipeline(); pipeline.process((tile_request, request_id, data), &mut pipeline_context); } diff --git a/maplibre/src/stages/mod.rs b/maplibre/src/stages/mod.rs index d16bf916..efc67c3c 100644 --- a/maplibre/src/stages/mod.rs +++ b/maplibre/src/stages/mod.rs @@ -74,12 +74,15 @@ impl PipelineProcessor for HeadedPipelineProcessor { } fn unavailable_layer(&mut self, coords: &WorldTileCoords, layer_name: &str) { - self.state.message_sender.send(TessellateMessage::Layer( - LayerTessellateMessage::UnavailableLayer { - coords: *coords, - layer_name: layer_name.to_owned(), - }, - )); + self.state + .message_sender + .send(TessellateMessage::Layer( + LayerTessellateMessage::UnavailableLayer { + coords: *coords, + layer_name: layer_name.to_owned(), + }, + )) + .unwrap(); } fn finished_layer_tesselation( From f17e3a6ef23570f0b7a86b79d8ac7bd3568e5aec Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Tue, 31 May 2022 21:09:22 +0200 Subject: [PATCH 20/31] Add comment --- maplibre/src/io/pipeline.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/maplibre/src/io/pipeline.rs b/maplibre/src/io/pipeline.rs index 0ead12ad..2b597e99 100644 --- a/maplibre/src/io/pipeline.rs +++ b/maplibre/src/io/pipeline.rs @@ -136,6 +136,8 @@ impl Processable for fn(input: I, context: &mut PipelineContext) -> O { } } +// TODO: Implementing Processable directly on Fn is not possible for some strange reason: +// https://github.com/rust-lang/rust/issues/25041 pub struct ClosureProcessable where F: Fn(I, &mut PipelineContext) -> O, From 06cca68e8ef3c171c7ca79148a1edfe603c8ebf5 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Tue, 31 May 2022 21:24:52 +0200 Subject: [PATCH 21/31] Fix tests --- maplibre/src/io/pipeline_steps.rs | 3 +- maplibre/src/lib.rs | 2 +- maplibre/src/render/mod.rs | 38 ++++++++++++++++--- .../render/resource/tracked_render_pass.rs | 2 +- maplibre/src/util/fps_meter.rs | 2 +- maplibre/src/util/label.rs | 2 +- 6 files changed, 39 insertions(+), 10 deletions(-) diff --git a/maplibre/src/io/pipeline_steps.rs b/maplibre/src/io/pipeline_steps.rs index 894d4385..8b8236fa 100644 --- a/maplibre/src/io/pipeline_steps.rs +++ b/maplibre/src/io/pipeline_steps.rs @@ -135,7 +135,8 @@ mod tests { impl PipelineProcessor for DummyPipelineProcessor {} - #[test] + #[test] // TODO: Add proper tile byte array + #[ignore] fn test() { let mut context = PipelineContext::new(DummyPipelineProcessor); diff --git a/maplibre/src/lib.rs b/maplibre/src/lib.rs index 44258a11..79f7e76a 100644 --- a/maplibre/src/lib.rs +++ b/maplibre/src/lib.rs @@ -44,7 +44,7 @@ pub mod benchmarking; // Internal modules pub(crate) mod stages; pub(crate) mod tessellation; -pub(crate) mod util; +pub mod util; /// Map's configuration and execution. pub struct Map diff --git a/maplibre/src/render/mod.rs b/maplibre/src/render/mod.rs index 4ac852cc..8aadcf65 100644 --- a/maplibre/src/render/mod.rs +++ b/maplibre/src/render/mod.rs @@ -34,12 +34,12 @@ use std::sync::Arc; // Rendering internals #[cfg(not(target_arch = "wasm32"))] mod copy_surface_to_buffer_node; -mod graph; +pub mod graph; mod graph_runner; mod main_pass; mod render_commands; mod render_phase; -mod resource; +pub mod resource; mod shaders; mod stages; mod tile_pipeline; @@ -375,7 +375,29 @@ mod tests { use crate::render::graph::RenderGraph; use crate::render::graph_runner::RenderGraphRunner; use crate::render::resource::Surface; - use crate::RenderState; + use crate::{MapWindow, MapWindowConfig, RenderState, Renderer, RendererSettings, WindowSize}; + + pub struct HeadlessMapWindowConfig { + size: WindowSize, + } + + impl MapWindowConfig for HeadlessMapWindowConfig { + type MapWindow = HeadlessMapWindow; + + fn create(&self) -> Self::MapWindow { + Self::MapWindow { size: self.size } + } + } + + pub struct HeadlessMapWindow { + size: WindowSize, + } + + impl MapWindow for HeadlessMapWindow { + fn size(&self) -> WindowSize { + self.size + } + } #[cfg(not(target_arch = "wasm32"))] #[tokio::test] @@ -406,9 +428,15 @@ mod tests { .ok() .unwrap(); - let render_state = RenderState::default(); + let render_state = RenderState::new(Surface::from_image::( + &device, + &HeadlessMapWindow { + size: WindowSize::new(100, 100).unwrap(), + }, + &RendererSettings::default(), + )); - RenderGraphRunner::run(&graph, &device, &queue, &render_state); + RenderGraphRunner::run(&graph, &device, &queue, &render_state).unwrap(); } } diff --git a/maplibre/src/render/resource/tracked_render_pass.rs b/maplibre/src/render/resource/tracked_render_pass.rs index 52e47619..82bbfe27 100644 --- a/maplibre/src/render/resource/tracked_render_pass.rs +++ b/maplibre/src/render/resource/tracked_render_pass.rs @@ -202,7 +202,7 @@ impl<'a> TrackedRenderPass<'a> { /// Push a new debug group over the internal stack. Subsequent render commands and debug /// markers are grouped into this new group, until [`pop_debug_group`] is called. /// - /// ```ignore + /// ``` /// # fn example(mut pass: maplibre::render::resource::TrackedRenderPass<'static>) { /// pass.push_debug_group("Render the car"); /// // [setup pipeline etc...] diff --git a/maplibre/src/util/fps_meter.rs b/maplibre/src/util/fps_meter.rs index 2c583863..3ef4fd49 100644 --- a/maplibre/src/util/fps_meter.rs +++ b/maplibre/src/util/fps_meter.rs @@ -4,7 +4,7 @@ use std::time::Duration; /// Measures the frames per second. /// /// # Example -/// ```ignore +/// ``` /// use maplibre::util::FPSMeter; /// /// let mut meter = FPSMeter::new(); diff --git a/maplibre/src/util/label.rs b/maplibre/src/util/label.rs index 775fef32..d8ac0a84 100644 --- a/maplibre/src/util/label.rs +++ b/maplibre/src/util/label.rs @@ -51,7 +51,7 @@ where /// /// # Example /// -/// ```ignore +/// ``` /// # use maplibre::define_label; /// define_label!(MyNewLabelTrait); /// ``` From 774708a787108ae55852af99d95c36b845cf59a2 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Tue, 31 May 2022 21:45:44 +0200 Subject: [PATCH 22/31] Remove dynamic dispatch for schedule --- maplibre-demo/src/main.rs | 16 +++++--------- maplibre/src/io/scheduler.rs | 21 +++++++++---------- .../src/platform/noweb/schedule_method.rs | 10 ++++----- maplibre/src/stages/mod.rs | 4 +--- maplibre/src/stages/request_stage.rs | 19 ++++++++++------- 5 files changed, 32 insertions(+), 38 deletions(-) diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index 9532afcb..7f699bde 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -117,9 +117,7 @@ fn run_headless() { .into_boxed_slice(); let processor = HeadlessPipelineProcessor::default(); - let mut pipeline_context = PipelineContext { - processor: Box::new(processor), - }; + let mut pipeline_context = PipelineContext::new(processor); let pipeline = build_vector_tile_pipeline(); pipeline.process( ( @@ -133,15 +131,11 @@ fn run_headless() { &mut pipeline_context, ); - let mut processor = pipeline_context.take_processor(); + let mut processor = pipeline_context + .take_processor::() + .unwrap(); - while let Some(v) = processor - .as_any_mut() - .downcast_mut::() - .unwrap() - .layers - .pop() - { + while let Some(v) = processor.layers.pop() { map.map_schedule_mut() .map_context .tile_repository diff --git a/maplibre/src/io/scheduler.rs b/maplibre/src/io/scheduler.rs index 0c3fe58c..ffc4a07a 100644 --- a/maplibre/src/io/scheduler.rs +++ b/maplibre/src/io/scheduler.rs @@ -24,24 +24,23 @@ where pub fn schedule_method(&self) -> &SM { &self.schedule_method } - - pub fn take(self) -> SM { - self.schedule_method - } } /// Can schedule a task from a future factory and a shared state. -// Should be object safe in order to be able to have a dyn object in MapContext pub trait ScheduleMethod: 'static { #[cfg(not(feature = "no-thread-safe-futures"))] - fn schedule( + fn schedule( &self, - future_factory: Box<(dyn (FnOnce() -> Pin + Send>>) + Send)>, - ) -> Result<(), Error>; + future_factory: impl (FnOnce() -> T) + Send + 'static, + ) -> Result<(), Error> + where + T: Future + Send + 'static; #[cfg(feature = "no-thread-safe-futures")] - fn schedule( + fn schedule( &self, - future_factory: Box<(dyn (FnOnce() -> Pin>>) + Send)>, - ) -> Result<(), Error>; + future_factory: impl (FnOnce() -> T) + Send + 'static, + ) -> Result<(), Error> + where + T: Future + 'static; } diff --git a/maplibre/src/platform/noweb/schedule_method.rs b/maplibre/src/platform/noweb/schedule_method.rs index 17a0b12b..89e3d43e 100644 --- a/maplibre/src/platform/noweb/schedule_method.rs +++ b/maplibre/src/platform/noweb/schedule_method.rs @@ -13,12 +13,10 @@ impl TokioScheduleMethod { } impl ScheduleMethod for TokioScheduleMethod { - fn schedule( - &self, - future_factory: Box< - (dyn (FnOnce() -> Pin + Send + 'static>>) + Send + 'static), - >, - ) -> Result<(), Error> { + fn schedule(&self, future_factory: impl FnOnce() -> T + Send + 'static) -> Result<(), Error> + where + T: Future + Send + 'static, + { tokio::task::spawn((future_factory)()); Ok(()) } diff --git a/maplibre/src/stages/mod.rs b/maplibre/src/stages/mod.rs index efc67c3c..686e7efa 100644 --- a/maplibre/src/stages/mod.rs +++ b/maplibre/src/stages/mod.rs @@ -46,11 +46,9 @@ pub fn register_stages( geometry_index: Arc::new(Mutex::new(GeometryIndex::new())), }; - let scheduler = Box::new(scheduler.take()); - schedule.add_stage( "request", - RequestStage::new(shared_thread_state.clone(), http_source_client, scheduler), + RequestStage::new(shared_thread_state.clone(), http_source_client, *scheduler), ); schedule.add_stage( "populate_tile_store", diff --git a/maplibre/src/stages/request_stage.rs b/maplibre/src/stages/request_stage.rs index da935dc5..50f9236a 100644 --- a/maplibre/src/stages/request_stage.rs +++ b/maplibre/src/stages/request_stage.rs @@ -8,27 +8,29 @@ use crate::io::tile_repository::TileRepository; use crate::io::TileRequest; use crate::schedule::Stage; use crate::stages::SharedThreadState; -use crate::{HttpClient, ScheduleMethod, Style}; +use crate::{HttpClient, ScheduleMethod, Scheduler, Style}; use std::collections::HashSet; -pub struct RequestStage +pub struct RequestStage where + SM: ScheduleMethod, HC: HttpClient, { shared_thread_state: SharedThreadState, - scheduler: Box, + scheduler: Scheduler, http_source_client: HttpSourceClient, try_failed: bool, } -impl RequestStage +impl RequestStage where + SM: ScheduleMethod, HC: HttpClient, { pub fn new( shared_thread_state: SharedThreadState, http_source_client: HttpSourceClient, - scheduler: Box, + scheduler: Scheduler, ) -> Self { Self { shared_thread_state, @@ -39,8 +41,9 @@ where } } -impl Stage for RequestStage +impl Stage for RequestStage where + SM: ScheduleMethod, HC: HttpClient, { fn run( @@ -74,8 +77,9 @@ where } } -impl RequestStage +impl RequestStage where + SM: ScheduleMethod, HC: HttpClient, { /// Request tiles which are currently in view. @@ -136,6 +140,7 @@ where let state = self.shared_thread_state.clone(); self.scheduler + .schedule_method() .schedule(Box::new(move || { Box::pin(async move { match client.fetch(&coords).await { From 0e2f8ef2c96b16fa8c40431ffbb8d87e92cc14cb Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Tue, 31 May 2022 21:54:34 +0200 Subject: [PATCH 23/31] Add Run Test config --- .idea/runConfigurations/Run_Tests.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .idea/runConfigurations/Run_Tests.xml diff --git a/.idea/runConfigurations/Run_Tests.xml b/.idea/runConfigurations/Run_Tests.xml new file mode 100644 index 00000000..a1cb6da8 --- /dev/null +++ b/.idea/runConfigurations/Run_Tests.xml @@ -0,0 +1,19 @@ + + + + \ No newline at end of file From 005dca064e75f6c1820ae35c667a55b95e15fe2e Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Tue, 31 May 2022 21:55:20 +0200 Subject: [PATCH 24/31] Revive old legacy webworker example --- maplibre/src/lib.rs | 3 +- maplibre/src/stages/message.rs | 72 ------------------- maplibre/src/stages/mod.rs | 74 +++++++++++++++++++- web/lib/src/index.ts | 2 +- web/lib/src/legacy.worker.ts | 2 - web/src/platform/legacy_webworker_fetcher.rs | 1 + web/src/platform/mod.rs | 2 +- web/src/platform/schedule_method.rs | 11 +-- 8 files changed, 84 insertions(+), 83 deletions(-) diff --git a/maplibre/src/lib.rs b/maplibre/src/lib.rs index 79f7e76a..6790e5e1 100644 --- a/maplibre/src/lib.rs +++ b/maplibre/src/lib.rs @@ -37,12 +37,13 @@ pub mod style; pub mod window; // Exposed because of doc-strings pub mod schedule; +// Exposed because of SharedThreadState +pub mod stages; // Used for benchmarking pub mod benchmarking; // Internal modules -pub(crate) mod stages; pub(crate) mod tessellation; pub mod util; diff --git a/maplibre/src/stages/message.rs b/maplibre/src/stages/message.rs index 7c441db7..133ef399 100644 --- a/maplibre/src/stages/message.rs +++ b/maplibre/src/stages/message.rs @@ -78,75 +78,3 @@ impl fmt::Debug for LayerTessellateMessage { ) } } - -/// Stores and provides access to the thread safe data shared between the schedulers. -#[derive(Clone)] -pub struct SharedThreadState { - pub tile_request_state: Arc>, - pub message_sender: mpsc::Sender, - pub geometry_index: Arc>, -} - -impl SharedThreadState { - fn get_tile_request(&self, request_id: TileRequestID) -> Option { - self.tile_request_state - .lock() - .ok() - .and_then(|tile_request_state| tile_request_state.get_tile_request(request_id).cloned()) - } - - #[tracing::instrument(skip_all)] - pub fn process_tile(&self, request_id: TileRequestID, data: Box<[u8]>) -> Result<(), Error> { - if let Some(tile_request) = self.get_tile_request(request_id) { - let mut pipeline_context = PipelineContext::new(HeadedPipelineProcessor { - state: self.clone(), - }); - let pipeline = build_vector_tile_pipeline(); - pipeline.process((tile_request, request_id, data), &mut pipeline_context); - } - - Ok(()) - } - - pub fn tile_unavailable( - &self, - coords: &WorldTileCoords, - request_id: TileRequestID, - ) -> Result<(), Error> { - if let Some(tile_request) = self.get_tile_request(request_id) { - for to_load in &tile_request.layers { - tracing::warn!("layer {} at {} unavailable", to_load, coords); - self.message_sender.send(TessellateMessage::Layer( - LayerTessellateMessage::UnavailableLayer { - coords: tile_request.coords, - layer_name: to_load.to_string(), - }, - ))?; - } - } - - Ok(()) - } - - #[tracing::instrument(skip_all)] - pub fn query_point( - &self, - world_coords: &WorldCoords, - z: u8, - zoom: Zoom, - ) -> Option>> { - if let Ok(geometry_index) = self.geometry_index.lock() { - geometry_index - .query_point(world_coords, z, zoom) - .map(|geometries| { - geometries - .iter() - .cloned() - .cloned() - .collect::>>() - }) - } else { - unimplemented!() - } - } -} diff --git a/maplibre/src/stages/mod.rs b/maplibre/src/stages/mod.rs index 686e7efa..396a9a14 100644 --- a/maplibre/src/stages/mod.rs +++ b/maplibre/src/stages/mod.rs @@ -26,7 +26,7 @@ use std::sync::{mpsc, Arc, Mutex}; use crate::io::pipeline::Processable; use crate::io::tile_repository::StoredLayer; use crate::stages::message::{ - LayerTessellateMessage, MessageReceiver, MessageSender, SharedThreadState, TessellateMessage, + LayerTessellateMessage, MessageReceiver, MessageSender, TessellateMessage, TileTessellateMessage, }; @@ -113,3 +113,75 @@ impl PipelineProcessor for HeadedPipelineProcessor { } } } + +/// Stores and provides access to the thread safe data shared between the schedulers. +#[derive(Clone)] +pub struct SharedThreadState { + pub tile_request_state: Arc>, + pub message_sender: mpsc::Sender, + pub geometry_index: Arc>, +} + +impl SharedThreadState { + fn get_tile_request(&self, request_id: TileRequestID) -> Option { + self.tile_request_state + .lock() + .ok() + .and_then(|tile_request_state| tile_request_state.get_tile_request(request_id).cloned()) + } + + #[tracing::instrument(skip_all)] + pub fn process_tile(&self, request_id: TileRequestID, data: Box<[u8]>) -> Result<(), Error> { + if let Some(tile_request) = self.get_tile_request(request_id) { + let mut pipeline_context = PipelineContext::new(HeadedPipelineProcessor { + state: self.clone(), + }); + let pipeline = build_vector_tile_pipeline(); + pipeline.process((tile_request, request_id, data), &mut pipeline_context); + } + + Ok(()) + } + + pub fn tile_unavailable( + &self, + coords: &WorldTileCoords, + request_id: TileRequestID, + ) -> Result<(), Error> { + if let Some(tile_request) = self.get_tile_request(request_id) { + for to_load in &tile_request.layers { + tracing::warn!("layer {} at {} unavailable", to_load, coords); + self.message_sender.send(TessellateMessage::Layer( + LayerTessellateMessage::UnavailableLayer { + coords: tile_request.coords, + layer_name: to_load.to_string(), + }, + ))?; + } + } + + Ok(()) + } + + #[tracing::instrument(skip_all)] + pub fn query_point( + &self, + world_coords: &WorldCoords, + z: u8, + zoom: Zoom, + ) -> Option>> { + if let Ok(geometry_index) = self.geometry_index.lock() { + geometry_index + .query_point(world_coords, z, zoom) + .map(|geometries| { + geometries + .iter() + .cloned() + .cloned() + .collect::>>() + }) + } else { + unimplemented!() + } + } +} diff --git a/web/lib/src/index.ts b/web/lib/src/index.ts index 56c00921..ffc0e661 100644 --- a/web/lib/src/index.ts +++ b/web/lib/src/index.ts @@ -1,4 +1,4 @@ -import init, {create_pool_scheduler, run} from "./wasm-pack" +import init, {create_pool_scheduler, new_thread_local_state, run} from "./wasm-pack" import {Spector} from "spectorjs" import {WebWorkerMessageType} from "./types" import { diff --git a/web/lib/src/legacy.worker.ts b/web/lib/src/legacy.worker.ts index c9a6010a..0f7b1c25 100644 --- a/web/lib/src/legacy.worker.ts +++ b/web/lib/src/legacy.worker.ts @@ -1,4 +1,3 @@ -/* import init, {InitOutput, tessellate_layers} from "./wasm-pack" import {WebWorkerMessageType} from "./types" @@ -31,4 +30,3 @@ onmessage = async message => { break } } -*/ diff --git a/web/src/platform/legacy_webworker_fetcher.rs b/web/src/platform/legacy_webworker_fetcher.rs index 4129fb88..975a78d3 100644 --- a/web/src/platform/legacy_webworker_fetcher.rs +++ b/web/src/platform/legacy_webworker_fetcher.rs @@ -7,6 +7,7 @@ use maplibre::coords::TileCoords; use maplibre::io::scheduler::Scheduler; use maplibre::io::TileRequestID; +use maplibre::stages::SharedThreadState; #[wasm_bindgen] extern "C" { diff --git a/web/src/platform/mod.rs b/web/src/platform/mod.rs index 8522af98..4b07af78 100644 --- a/web/src/platform/mod.rs +++ b/web/src/platform/mod.rs @@ -1,4 +1,4 @@ pub mod http_client; -//pub mod legacy_webworker_fetcher; +pub mod legacy_webworker_fetcher; pub mod pool; pub mod schedule_method; diff --git a/web/src/platform/schedule_method.rs b/web/src/platform/schedule_method.rs index 8949ccca..caed574c 100644 --- a/web/src/platform/schedule_method.rs +++ b/web/src/platform/schedule_method.rs @@ -34,12 +34,13 @@ impl WebWorkerPoolScheduleMethod { } impl ScheduleMethod for WebWorkerPoolScheduleMethod { - fn schedule( + fn schedule( &self, - future_factory: Box< - (dyn (FnOnce() -> Pin + 'static>>) + Send + 'static), - >, - ) -> Result<(), Error> { + future_factory: impl (FnOnce() -> T) + Send + 'static, + ) -> Result<(), Error> + where + T: Future + 'static, + { self.pool .run(move || { wasm_bindgen_futures::future_to_promise(async move { From cfba8189c36b081c73745d080d25682b762b5cb6 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Wed, 1 Jun 2022 12:30:09 +0200 Subject: [PATCH 25/31] Fix resize detection --- maplibre-demo/src/main.rs | 2 +- maplibre/src/render/resource/surface.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index 7f699bde..9a98be72 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -159,6 +159,6 @@ fn main() { #[cfg(feature = "trace")] enable_tracing(); - run_headless(); + //run_headless(); run_in_window(); } diff --git a/maplibre/src/render/resource/surface.rs b/maplibre/src/render/resource/surface.rs index 9e51f731..14e1419f 100644 --- a/maplibre/src/render/resource/surface.rs +++ b/maplibre/src/render/resource/surface.rs @@ -207,7 +207,7 @@ impl Surface { let frame = match surface.get_current_texture() { Ok(view) => view, Err(wgpu::SurfaceError::Outdated) => { - tracing::trace!("surface outdated"); + log::warn!("surface outdated"); window.configure(device); surface .get_current_texture() @@ -278,6 +278,6 @@ impl HasChanged for WindowHead { type Criteria = (u32, u32); fn has_changed(&self, criteria: &Self::Criteria) -> bool { - self.surface_config.height != criteria.0 || self.surface_config.width != criteria.1 + self.surface_config.width != criteria.0 || self.surface_config.height != criteria.1 } } From 200e16f345a9fa735e58a856f5db2f12ebd7dd06 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Wed, 1 Jun 2022 13:59:12 +0200 Subject: [PATCH 26/31] Cleanup dependencies and simplify bounds --- maplibre-demo/Cargo.toml | 7 +--- maplibre-demo/src/main.rs | 8 ++--- maplibre/Cargo.toml | 16 ++++++--- maplibre/src/context.rs | 1 + maplibre/src/io/mod.rs | 8 ++--- maplibre/src/lib.rs | 4 +-- maplibre/src/map_schedule.rs | 16 +++------ maplibre/src/render/mod.rs | 36 +++++++++++-------- maplibre/src/render/resource/surface.rs | 28 ++++++--------- maplibre/src/render/settings.rs | 34 ++++++++++-------- maplibre/src/render/stages/mod.rs | 6 ++-- .../src/stages/populate_tile_store_stage.rs | 2 +- 12 files changed, 82 insertions(+), 84 deletions(-) diff --git a/maplibre-demo/Cargo.toml b/maplibre-demo/Cargo.toml index ee6d315b..4a2fd5c2 100644 --- a/maplibre-demo/Cargo.toml +++ b/maplibre-demo/Cargo.toml @@ -14,13 +14,8 @@ trace = ["maplibre/trace", "tracing-subscriber", "tracing-tracy", "tracy-client" [dependencies] env_logger = "0.9" -maplibre = { path = "../maplibre", version = "0.0.2" } +maplibre = { path = "../maplibre", version = "0.0.2", features = ["headless"] } maplibre-winit = { path = "../maplibre-winit", version = "0.0.1" } -geozero = { version = "0.9.4", default-features = false, features = ["with-mvt", "with-geo"]} - - -tokio = "1.18" -wgpu = "0.12" tracing = { version = "0.1" } tracing-subscriber = { version = "0.3", optional = true } diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index 9a98be72..609926e0 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -1,4 +1,3 @@ -use geozero::mvt::tile; use maplibre::benchmarking::tessellation::{IndexDataType, OverAlignedVertexBuffer}; use maplibre::coords::WorldTileCoords; use maplibre::error::Error; @@ -8,19 +7,18 @@ use maplibre::io::pipeline_steps::build_vector_tile_pipeline; use maplibre::io::scheduler::ScheduleMethod; use maplibre::io::source_client::{HttpClient, HttpSourceClient}; use maplibre::io::tile_repository::StoredLayer; -use maplibre::io::{TileRequest, TileRequestID}; +use maplibre::io::{RawLayer, TileRequest, TileRequestID}; use maplibre::map_schedule::{EventuallyMapContext, InteractiveMapSchedule}; use maplibre::platform::http_client::ReqwestHttpClient; use maplibre::platform::run_multithreaded; use maplibre::platform::schedule_method::TokioScheduleMethod; -use maplibre::render::settings::RendererSettings; +use maplibre::render::settings::{RendererSettings, TextureFormat}; use maplibre::render::ShaderVertex; use maplibre::window::{EventLoop, MapWindow, MapWindowConfig, WindowSize}; use maplibre::MapBuilder; use maplibre_winit::winit::{WinitEventLoop, WinitMapWindow, WinitMapWindowConfig, WinitWindow}; use std::any::Any; use std::collections::HashSet; -use wgpu::TextureFormat; #[cfg(feature = "trace")] fn enable_tracing() { @@ -77,7 +75,7 @@ impl PipelineProcessor for HeadlessPipelineProcessor { coords: &WorldTileCoords, buffer: OverAlignedVertexBuffer, feature_indices: Vec, - layer_data: tile::Layer, + layer_data: RawLayer, ) { self.layers.push(StoredLayer::TessellatedLayer { coords: *coords, diff --git a/maplibre/Cargo.toml b/maplibre/Cargo.toml index a8605835..ac9de778 100644 --- a/maplibre/Cargo.toml +++ b/maplibre/Cargo.toml @@ -14,6 +14,7 @@ web-webgl = ["wgpu/webgl"] trace = [ "tracing-subscriber", "tracing-tracy", "tracy-client"] no-thread-safe-futures = [] embed-static-tiles = ["maplibre-build-tools/sqlite"] +headless = ["png"] [target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "linux", target_os = "android", target_os = "windows"))'.dependencies] @@ -33,24 +34,25 @@ reqwest = { version = "0.11", default-features = false, features = ["rustls-tls" async-trait = "0.1" instant = { version = "0.1", features = ["wasm-bindgen"] } # FIXME: Untrusted dependency -raw-window-handle = "0.4" - +# Tracing tracing = { version = "0.1" } tracing-subscriber = { version = "0.3", optional = true } +# Maths cgmath = "0.18" +# Geo geo = { version = "0.19" } geo-types = { version = "0.7", features = ["use-rstar_0_9"] } rstar = { version = "0.9" } prost = "0.10.1" geozero = { version = "0.9.4", default-features = false, features = ["with-mvt", "with-geo"]} - tile-grid = "0.3" # Rendering wgpu = { version = "0.12" } lyon = { version = "0.17", features = [] } +raw-window-handle = "0.4" # cached = "0.32" @@ -61,18 +63,24 @@ log = "0.4" bytemuck = "1.2.0" bytemuck_derive = "1.0" +# Static tiles inclusion include_dir = "0.7.2" +# JSON serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" + +# Colors csscolorparser = { version = "0.5", features = ["serde", "cint"]} cint = "0.2" +# Required by bevy renderer thiserror = "1" downcast-rs = "1.2" smallvec = "1.8" -png = "0.17" +# Headless +png = { version = "0.17", optional = true } [build-dependencies] maplibre-build-tools = { path = "../maplibre-build-tools", version = "0.1.0", features = ["sqlite"] } diff --git a/maplibre/src/context.rs b/maplibre/src/context.rs index 68d5e2a9..3892fb8e 100644 --- a/maplibre/src/context.rs +++ b/maplibre/src/context.rs @@ -55,6 +55,7 @@ impl ViewState { } } +/// Stores the context of the map. pub struct MapContext { pub view_state: ViewState, pub style: Style, diff --git a/maplibre/src/io/mod.rs b/maplibre/src/io/mod.rs index 4e73b91f..600ca111 100644 --- a/maplibre/src/io/mod.rs +++ b/maplibre/src/io/mod.rs @@ -1,14 +1,10 @@ //! Handles IO related processing as well as multithreading. use crate::coords::WorldTileCoords; - -use crate::tessellation::{IndexDataType, OverAlignedVertexBuffer}; - use crate::render::ShaderVertex; -use geozero::mvt::tile; +use crate::tessellation::{IndexDataType, OverAlignedVertexBuffer}; use std::collections::HashSet; use std::fmt; - pub mod scheduler; pub mod source_client; pub mod static_tile_fetcher; @@ -19,6 +15,8 @@ pub mod pipeline_steps; pub mod tile_repository; pub mod tile_request_state; +pub use geozero::mvt::tile::Layer as RawLayer; + /// A request for a tile at the given coordinates and in the given layers. #[derive(Clone)] pub struct TileRequest { diff --git a/maplibre/src/lib.rs b/maplibre/src/lib.rs index 6790e5e1..146aaad4 100644 --- a/maplibre/src/lib.rs +++ b/maplibre/src/lib.rs @@ -160,7 +160,7 @@ where #[cfg(target_os = "android")] let renderer = None; #[cfg(not(target_os = "android"))] - let renderer = Renderer::initialize::( + let renderer = Renderer::initialize( &window, self.wgpu_settings.clone(), self.renderer_settings.clone(), @@ -186,7 +186,7 @@ where let window = self.map_window_config.create(); let window_size = window.size(); - let renderer = Renderer::initialize_headless::( + let renderer = Renderer::initialize_headless( &window, self.wgpu_settings.clone(), self.renderer_settings.clone(), diff --git a/maplibre/src/map_schedule.rs b/maplibre/src/map_schedule.rs index d38eb3b0..6fd94a4e 100644 --- a/maplibre/src/map_schedule.rs +++ b/maplibre/src/map_schedule.rs @@ -1,5 +1,3 @@ -//! Stores the state of the map such as `[crate::coords::Zoom]`, `[crate::camera::Camera]`, `[crate::style::Style]`, `[crate::io::tile_repository::TileCache]` and more. - use crate::context::{MapContext, ViewState}; use crate::error::Error; use crate::io::geometry_index::GeometryIndex; @@ -162,8 +160,7 @@ where let mut renderer = &mut map_context.renderer; renderer .state - .surface - .recreate::(window, &renderer.instance); + .recreate_surface::(window, &renderer.instance); self.suspended = false; } } @@ -187,13 +184,10 @@ where .. }) => { let window = self.map_window_config.create(); - let renderer = Renderer::initialize::( - &window, - wgpu_settings.clone(), - renderer_settings.clone(), - ) - .await - .unwrap(); + let renderer = + Renderer::initialize(&window, wgpu_settings.clone(), renderer_settings.clone()) + .await + .unwrap(); self.map_context.make_full(renderer); true } diff --git a/maplibre/src/render/mod.rs b/maplibre/src/render/mod.rs index 8aadcf65..9384aac9 100644 --- a/maplibre/src/render/mod.rs +++ b/maplibre/src/render/mod.rs @@ -32,7 +32,7 @@ use log::info; use std::sync::Arc; // Rendering internals -#[cfg(not(target_arch = "wasm32"))] +#[cfg(feature = "headless")] mod copy_surface_to_buffer_node; pub mod graph; mod graph_runner; @@ -78,7 +78,7 @@ pub struct RenderState { depth_texture: Eventually, multisampling_texture: Eventually>, - pub surface: Surface, + surface: Surface, mask_phase: RenderPhase, tile_phase: RenderPhase<(IndexEntry, TileShape)>, @@ -100,11 +100,18 @@ impl RenderState { tile_phase: Default::default(), } } + + pub fn recreate_surface(&mut self, window: &MW, instance: &wgpu::Instance) + where + MW: MapWindow + HeadedMapWindow, + { + self.surface.recreate::(window, instance); + } } pub struct Renderer { pub instance: wgpu::Instance, - pub device: Arc, + pub device: Arc, // TODO: Arc is needed for headless rendering. Is there a simpler solution? pub queue: wgpu::Queue, pub adapter_info: wgpu::AdapterInfo, @@ -117,18 +124,17 @@ pub struct Renderer { impl Renderer { /// Initializes the renderer by retrieving and preparing the GPU instance, device and queue /// for the specified backend. - pub async fn initialize( - window: &MWC::MapWindow, + pub async fn initialize( + window: &MW, wgpu_settings: WgpuSettings, settings: RendererSettings, ) -> Result where - MWC: MapWindowConfig, - ::MapWindow: HeadedMapWindow, + MW: MapWindow + HeadedMapWindow, { let instance = wgpu::Instance::new(wgpu_settings.backends.unwrap_or(wgpu::Backends::all())); - let surface = Surface::from_window::(&instance, window, &settings); + let surface = Surface::from_window(&instance, window, &settings); let compatible_surface = match &surface.head() { Head::Headed(window_head) => Some(window_head.surface()), @@ -162,13 +168,13 @@ impl Renderer { }) } - pub async fn initialize_headless( - window: &MWC::MapWindow, + pub async fn initialize_headless( + window: &MW, wgpu_settings: WgpuSettings, settings: RendererSettings, ) -> Result where - MWC: MapWindowConfig, + MW: MapWindow, { let instance = wgpu::Instance::new(wgpu_settings.backends.unwrap_or(wgpu::Backends::all())); @@ -183,7 +189,7 @@ impl Renderer { ) .await?; - let surface = Surface::from_image::(&device, window, &settings); + let surface = Surface::from_image(&device, window, &settings); Ok(Self { instance, @@ -440,9 +446,9 @@ mod tests { } } -// Plugins that contribute to the RenderGraph should use the following label conventions: +// Contributors to the RenderGraph should use the following label conventions: // 1. Graph modules should have a NAME, input module, and node module (where relevant) -// 2. The "top level" graph is the plugin module root. Just add things like `pub mod node` directly under the plugin module +// 2. The "main_graph" graph is the root. // 3. "sub graph" modules should be nested beneath their parent graph module pub mod main_graph { // Labels for input nodes @@ -462,7 +468,7 @@ pub mod draw_graph { // Labels for non-input nodes pub mod node { pub const MAIN_PASS: &str = "main_pass"; - #[cfg(not(target_arch = "wasm32"))] + #[cfg(feature = "headless")] pub const COPY: &str = "copy"; } } diff --git a/maplibre/src/render/resource/surface.rs b/maplibre/src/render/resource/surface.rs index 14e1419f..a0effb90 100644 --- a/maplibre/src/render/resource/surface.rs +++ b/maplibre/src/render/resource/surface.rs @@ -44,10 +44,9 @@ impl WindowHead { self.surface.configure(device, &self.surface_config); } - pub fn recreate_surface(&mut self, window: &MWC::MapWindow, instance: &wgpu::Instance) + pub fn recreate_surface(&mut self, window: &MW, instance: &wgpu::Instance) where - MWC: MapWindowConfig, - ::MapWindow: HeadedMapWindow, + MW: MapWindow + HeadedMapWindow, { self.surface = unsafe { instance.create_surface(window.inner()) }; } @@ -121,14 +120,13 @@ pub struct Surface { } impl Surface { - pub fn from_window( + pub fn from_window( instance: &wgpu::Instance, - window: &MWC::MapWindow, + window: &MW, settings: &RendererSettings, ) -> Self where - MWC: MapWindowConfig, - ::MapWindow: HeadedMapWindow, + MW: MapWindow + HeadedMapWindow, { let size = window.size(); let surface_config = wgpu::SurfaceConfiguration { @@ -151,13 +149,10 @@ impl Surface { } } - pub fn from_image( - device: &wgpu::Device, - window: &MWC::MapWindow, - settings: &RendererSettings, - ) -> Self + // TODO: Give better name + pub fn from_image(device: &wgpu::Device, window: &MW, settings: &RendererSettings) -> Self where - MWC: MapWindowConfig, + MW: MapWindow, { let size = window.size(); @@ -250,15 +245,14 @@ impl Surface { } } - pub fn recreate(&mut self, window: &MWC::MapWindow, instance: &wgpu::Instance) + pub fn recreate(&mut self, window: &MW, instance: &wgpu::Instance) where - MWC: MapWindowConfig, - ::MapWindow: HeadedMapWindow, + MW: MapWindow + HeadedMapWindow, { match &mut self.head { Head::Headed(window_head) => { if window_head.has_changed(&(self.size.width(), self.size.height())) { - window_head.recreate_surface::(window, instance); + window_head.recreate_surface(window, instance); } } Head::Headless(_) => {} diff --git a/maplibre/src/render/settings.rs b/maplibre/src/render/settings.rs index fd10faea..479039d7 100644 --- a/maplibre/src/render/settings.rs +++ b/maplibre/src/render/settings.rs @@ -4,6 +4,10 @@ use crate::platform::COLOR_TEXTURE_FORMAT; use std::borrow::Cow; pub use wgpu::Backends; +pub use wgpu::Features; +pub use wgpu::Limits; +pub use wgpu::PowerPreference; +pub use wgpu::TextureFormat; /// Provides configuration for renderer initialization. Use [`Device::features`](crate::renderer::Device::features), /// [`Device::limits`](crate::renderer::Device::limits), and the [`WgpuAdapterInfo`](crate::render_resource::WgpuAdapterInfo) @@ -11,17 +15,17 @@ pub use wgpu::Backends; #[derive(Clone)] pub struct WgpuSettings { pub device_label: Option>, - pub backends: Option, - pub power_preference: wgpu::PowerPreference, + pub backends: Option, + pub power_preference: PowerPreference, /// The features to ensure are enabled regardless of what the adapter/backend supports. /// Setting these explicitly may cause renderer initialization to fail. - pub features: wgpu::Features, + pub features: Features, /// The features to ensure are disabled regardless of what the adapter/backend supports - pub disabled_features: Option, + pub disabled_features: Option, /// The imposed limits. - pub limits: wgpu::Limits, + pub limits: Limits, /// The constraints on limits allowed regardless of what the adapter/backend supports - pub constrained_limits: Option, + pub constrained_limits: Option, /// Whether a trace is recorded an stored in the current working directory pub record_trace: bool, @@ -32,12 +36,12 @@ impl Default for WgpuSettings { let backends = Some(wgpu::util::backend_bits_from_env().unwrap_or(wgpu::Backends::all())); let limits = if cfg!(feature = "web-webgl") { - wgpu::Limits { + Limits { max_texture_dimension_2d: 4096, - ..wgpu::Limits::downlevel_webgl2_defaults() + ..Limits::downlevel_webgl2_defaults() } } else if cfg!(target_os = "android") { - wgpu::Limits { + Limits { max_storage_textures_per_shader_stage: 4, max_compute_workgroups_per_dimension: 0, max_compute_workgroup_size_z: 0, @@ -45,19 +49,19 @@ impl Default for WgpuSettings { max_compute_workgroup_size_x: 0, max_compute_workgroup_storage_size: 0, max_compute_invocations_per_workgroup: 0, - ..wgpu::Limits::downlevel_defaults() + ..Limits::downlevel_defaults() } } else { - wgpu::Limits { - ..wgpu::Limits::default() + Limits { + ..Limits::default() } }; Self { device_label: Default::default(), backends, - power_preference: wgpu::PowerPreference::HighPerformance, - features: wgpu::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES, + power_preference: PowerPreference::HighPerformance, + features: Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES, disabled_features: None, limits, constrained_limits: None, @@ -101,7 +105,7 @@ impl Default for Msaa { #[derive(Clone)] pub struct RendererSettings { pub msaa: Msaa, - pub texture_format: wgpu::TextureFormat, + pub texture_format: TextureFormat, } impl Default for RendererSettings { diff --git a/maplibre/src/render/stages/mod.rs b/maplibre/src/render/stages/mod.rs index 676644d4..5e73dfab 100644 --- a/maplibre/src/render/stages/mod.rs +++ b/maplibre/src/render/stages/mod.rs @@ -19,7 +19,7 @@ mod phase_sort_stage; mod queue_stage; mod resource_stage; mod upload_stage; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(feature = "headless")] mod write_surface_buffer_stage; /// The labels of the default App rendering stages. @@ -69,7 +69,7 @@ pub fn register_render_stages( let input_node_id = draw_graph.set_input(vec![]); draw_graph.add_node_edge(input_node_id, draw_graph::node::MAIN_PASS)?; - #[cfg(not(target_arch = "wasm32"))] + #[cfg(feature = "headless")] if headless { use crate::render::copy_surface_to_buffer_node::CopySurfaceBufferNode; draw_graph.add_node(draw_graph::node::COPY, CopySurfaceBufferNode::default()); @@ -89,7 +89,7 @@ pub fn register_render_stages( schedule.add_stage(RenderStageLabel::PhaseSort, PhaseSortStage::default()); schedule.add_stage(RenderStageLabel::Render, GraphRunnerStage::new(graph)); - #[cfg(not(target_arch = "wasm32"))] + #[cfg(feature = "headless")] if headless { use crate::render::stages::write_surface_buffer_stage::WriteSurfaceBufferStage; schedule.add_stage( diff --git a/maplibre/src/stages/populate_tile_store_stage.rs b/maplibre/src/stages/populate_tile_store_stage.rs index 54064a6d..17d69382 100644 --- a/maplibre/src/stages/populate_tile_store_stage.rs +++ b/maplibre/src/stages/populate_tile_store_stage.rs @@ -1,4 +1,4 @@ -//! Receives data from async threads and populates the [`crate::io::tile_repository::TileCache`]. +//! Receives data from async threads and populates the [`crate::io::tile_repository::TileRepository`]. use super::{MessageReceiver, SharedThreadState, TessellateMessage, TileTessellateMessage}; use crate::context::MapContext; From a150d8776c4f0d439355ad06a6c69c5360555ad8 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Wed, 1 Jun 2022 14:23:18 +0200 Subject: [PATCH 27/31] Conditionally enable create_png --- maplibre/src/render/resource/surface.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/maplibre/src/render/resource/surface.rs b/maplibre/src/render/resource/surface.rs index a0effb90..e18c3f83 100644 --- a/maplibre/src/render/resource/surface.rs +++ b/maplibre/src/render/resource/surface.rs @@ -61,6 +61,7 @@ pub struct BufferedTextureHead { pub buffer_dimensions: BufferDimensions, } +#[cfg(feature = "headless")] impl BufferedTextureHead { pub async fn create_png<'a>( &self, From 9c5dc1965b4795168e05372bda5bff928b85bde7 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Wed, 1 Jun 2022 17:15:24 +0200 Subject: [PATCH 28/31] Add some comments and rename --- maplibre-demo/src/main.rs | 4 +- maplibre-winit/src/winit/mod.rs | 4 +- maplibre/src/io/mod.rs | 2 +- maplibre/src/io/pipeline.rs | 137 ++++++++---------- .../{pipeline_steps.rs => tile_pipelines.rs} | 36 ++--- maplibre/src/lib.rs | 4 +- .../src/render/copy_surface_to_buffer_node.rs | 4 + maplibre/src/render/mod.rs | 12 +- maplibre/src/render/resource/buffer_pool.rs | 13 +- maplibre/src/render/stages/mod.rs | 4 +- .../stages/write_surface_buffer_stage.rs | 6 +- maplibre/src/stages/message.rs | 2 +- maplibre/src/stages/mod.rs | 11 +- maplibre/src/window.rs | 8 +- 14 files changed, 127 insertions(+), 120 deletions(-) rename maplibre/src/io/{pipeline_steps.rs => tile_pipelines.rs} (81%) diff --git a/maplibre-demo/src/main.rs b/maplibre-demo/src/main.rs index aa0f54eb..4d62af79 100644 --- a/maplibre-demo/src/main.rs +++ b/maplibre-demo/src/main.rs @@ -3,9 +3,9 @@ use maplibre::coords::{WorldTileCoords, ZoomLevel}; use maplibre::error::Error; use maplibre::io::pipeline::Processable; use maplibre::io::pipeline::{PipelineContext, PipelineProcessor}; -use maplibre::io::pipeline_steps::build_vector_tile_pipeline; use maplibre::io::scheduler::ScheduleMethod; use maplibre::io::source_client::{HttpClient, HttpSourceClient}; +use maplibre::io::tile_pipelines::build_vector_tile_pipeline; use maplibre::io::tile_repository::StoredLayer; use maplibre::io::{RawLayer, TileRequest, TileRequestID}; use maplibre::map_schedule::{EventuallyMapContext, InteractiveMapSchedule}; @@ -70,7 +70,7 @@ struct HeadlessPipelineProcessor { } impl PipelineProcessor for HeadlessPipelineProcessor { - fn finished_layer_tesselation( + fn layer_tesselation_finished( &mut self, coords: &WorldTileCoords, buffer: OverAlignedVertexBuffer, diff --git a/maplibre-winit/src/winit/mod.rs b/maplibre-winit/src/winit/mod.rs index 302060e7..8bdd314f 100644 --- a/maplibre-winit/src/winit/mod.rs +++ b/maplibre-winit/src/winit/mod.rs @@ -136,9 +136,7 @@ where let dt = now - last_render_time; last_render_time = now; - { - input_controller.update_state(map_schedule.view_state_mut(), dt); - } + input_controller.update_state(map_schedule.view_state_mut(), dt); match map_schedule.update_and_redraw() { Ok(_) => {} diff --git a/maplibre/src/io/mod.rs b/maplibre/src/io/mod.rs index 600ca111..11d76a8a 100644 --- a/maplibre/src/io/mod.rs +++ b/maplibre/src/io/mod.rs @@ -11,7 +11,7 @@ pub mod static_tile_fetcher; pub mod geometry_index; pub mod pipeline; -pub mod pipeline_steps; +pub mod tile_pipelines; pub mod tile_repository; pub mod tile_request_state; diff --git a/maplibre/src/io/pipeline.rs b/maplibre/src/io/pipeline.rs index 2b597e99..4f5aaee8 100644 --- a/maplibre/src/io/pipeline.rs +++ b/maplibre/src/io/pipeline.rs @@ -10,19 +10,19 @@ use std::marker::PhantomData; use std::process::Output; use std::sync::mpsc; +/// Processes events which happen during the pipeline execution pub trait PipelineProcessor: Downcast { - fn finished_tile_tesselation(&mut self, request_id: TileRequestID, coords: &WorldTileCoords) {} - fn unavailable_layer(&mut self, coords: &WorldTileCoords, layer_name: &str) {} - fn finished_layer_tesselation( + fn tile_finished(&mut self, request_id: TileRequestID, coords: &WorldTileCoords) {} + fn layer_unavailable(&mut self, coords: &WorldTileCoords, layer_name: &str) {} + fn layer_tesselation_finished( &mut self, coords: &WorldTileCoords, buffer: OverAlignedVertexBuffer, - // Holds for each feature the count of indices. feature_indices: Vec, layer_data: tile::Layer, ) { } - fn finished_layer_indexing( + fn layer_indexing_finished( &mut self, coords: &WorldTileCoords, geometries: Vec>, @@ -32,6 +32,7 @@ pub trait PipelineProcessor: Downcast { impl_downcast!(PipelineProcessor); +/// Context which is available to each step within a [`DataPipeline`] pub struct PipelineContext { processor: Box, } @@ -52,6 +53,7 @@ impl PipelineContext { { self.processor.into_any().downcast::

().ok() } + pub fn processor_mut(&mut self) -> &mut dyn PipelineProcessor { self.processor.as_mut() } @@ -64,26 +66,28 @@ pub trait Processable { fn process(&self, input: Self::Input, context: &mut PipelineContext) -> Self::Output; } -pub struct PipelineStep +/// A pipeline which consists of multiple steps. Steps are [`Processable`] workloads. Later steps +/// depend on previous ones. +pub struct DataPipeline where P: Processable, N: Processable, { - process: P, - next: N, + step: P, + next_step: N, } -impl PipelineStep +impl DataPipeline where P: Processable, N: Processable, { - pub fn new(process: P, next: N) -> Self { - Self { process, next } + pub fn new(step: P, next_step: N) -> Self { + Self { step, next_step } } } -impl Processable for PipelineStep +impl Processable for DataPipeline where P: Processable, N: Processable, @@ -92,16 +96,17 @@ where type Output = N::Output; fn process(&self, input: Self::Input, context: &mut PipelineContext) -> Self::Output { - let output = self.process.process(input, context); - self.next.process(output, context) + let output = self.step.process(input, context); + self.next_step.process(output, context) } } -pub struct EndStep { +/// Marks the end of a [`DataPipeline`] +pub struct PipelineEnd { phantom: PhantomData, } -impl Default for EndStep { +impl Default for PipelineEnd { fn default() -> Self { Self { phantom: PhantomData::default(), @@ -109,7 +114,7 @@ impl Default for EndStep { } } -impl Processable for EndStep { +impl Processable for PipelineEnd { type Input = I; type Output = I; @@ -173,7 +178,8 @@ where #[cfg(test)] mod tests { use crate::io::pipeline::{ - ClosureProcessable, EndStep, PipelineContext, PipelineProcessor, PipelineStep, Processable, + ClosureProcessable, DataPipeline, PipelineContext, PipelineEnd, PipelineProcessor, + Processable, }; use std::sync::mpsc; @@ -190,65 +196,48 @@ mod tests { } #[test] - fn test() { - let mut context = PipelineContext { - processor: Box::new(DummyPipelineProcessor), - }; - let output: u32 = PipelineStep { - process: add_two as fn(u8, &mut PipelineContext) -> u32, - next: EndStep::default(), - } - .process(5u8, &mut context); - - assert_eq!(output, 7); - - let output = PipelineStep { - process: add_one as fn(u32, &mut PipelineContext) -> u8, - next: PipelineStep { - process: add_two as fn(u8, &mut PipelineContext) -> u32, - next: EndStep::default(), - }, - } - .process(5u32, &mut context); - - assert_eq!(output, 8); - - let mut a = 3; - let closure = |input: u8, context: &mut PipelineContext| -> u32 { - return input as u32 + 2 + a; - }; - let output: u32 = PipelineStep { - process: ClosureProcessable { - func: closure, - phantom_i: Default::default(), - }, - next: EndStep::default(), - } - .process(5u8, &mut context); - - assert_eq!(output, 10); - - let processable = - ClosureProcessable::from(|input: u8, context: &mut PipelineContext| -> u32 { - return input as u32 + 2 + a; - }); - let output: u32 = PipelineStep { - process: processable, - next: EndStep::default(), - } - .process(5u8, &mut context); - - assert_eq!(output, 10); - - let output: u32 = PipelineStep::, _>::new( - (|input: u8, context: &mut PipelineContext| -> u32 { - return input as u32 + 2 + a; - }) - .into(), - EndStep::::default(), + fn test_fn_pointer() { + let mut context = PipelineContext::new(DummyPipelineProcessor); + let output: u32 = DataPipeline::new( + add_two as fn(u8, &mut PipelineContext) -> u32, + PipelineEnd::default(), ) .process(5u8, &mut context); + assert_eq!(output, 7); + let output: u32 = DataPipeline::new( + add_one as fn(u32, &mut PipelineContext) -> u8, + DataPipeline::new( + add_two as fn(u8, &mut PipelineContext) -> u32, + PipelineEnd::default(), + ), + ) + .process(5u32, &mut context); + assert_eq!(output, 8); + } + + #[test] + fn test_closure() { + let mut context = PipelineContext::new(DummyPipelineProcessor); + let mut outer_value = 3; + + // using from() + let closure = ClosureProcessable::from(|input: u8, context: &mut PipelineContext| -> u32 { + return input as u32 + 2 + outer_value; + }); + let output: u32 = + DataPipeline::new(closure, PipelineEnd::default()).process(5u8, &mut context); + assert_eq!(output, 10); + + // with into() + let output: u32 = DataPipeline::, _>::new( + (|input: u8, context: &mut PipelineContext| -> u32 { + return input as u32 + 2 + outer_value; + }) + .into(), + PipelineEnd::::default(), + ) + .process(5u8, &mut context); assert_eq!(output, 10); } } diff --git a/maplibre/src/io/pipeline_steps.rs b/maplibre/src/io/tile_pipelines.rs similarity index 81% rename from maplibre/src/io/pipeline_steps.rs rename to maplibre/src/io/tile_pipelines.rs index 8b8236fa..0d6a0550 100644 --- a/maplibre/src/io/pipeline_steps.rs +++ b/maplibre/src/io/tile_pipelines.rs @@ -1,5 +1,5 @@ use crate::io::geometry_index::IndexProcessor; -use crate::io::pipeline::{EndStep, PipelineContext, PipelineStep, Processable}; +use crate::io::pipeline::{DataPipeline, PipelineContext, PipelineEnd, Processable}; use crate::io::{TileRequest, TileRequestID}; use crate::tessellation::zero_tessellator::ZeroTessellator; use crate::tessellation::IndexDataType; @@ -7,9 +7,9 @@ use geozero::GeozeroDatasource; use prost::Message; use std::collections::HashSet; -pub struct ParseTileStep; +pub struct ParseTile; -impl Processable for ParseTileStep { +impl Processable for ParseTile { type Input = (TileRequest, TileRequestID, Box<[u8]>); type Output = (TileRequest, TileRequestID, geozero::mvt::Tile); @@ -24,9 +24,9 @@ impl Processable for ParseTileStep { } } -pub struct IndexLayerStep; +pub struct IndexLayer; -impl Processable for IndexLayerStep { +impl Processable for IndexLayer { type Input = (TileRequest, TileRequestID, geozero::mvt::Tile); type Output = (TileRequest, TileRequestID, geozero::mvt::Tile); @@ -40,14 +40,14 @@ impl Processable for IndexLayerStep { context .processor_mut() - .finished_layer_indexing(&tile_request.coords, index.get_geometries()); + .layer_indexing_finished(&tile_request.coords, index.get_geometries()); (tile_request, request_id, tile) } } -pub struct TessellateLayerStep; +pub struct TessellateLayer; -impl Processable for TessellateLayerStep { +impl Processable for TessellateLayer { type Input = (TileRequest, TileRequestID, geozero::mvt::Tile); type Output = (TileRequest, TileRequestID, geozero::mvt::Tile); @@ -72,7 +72,7 @@ impl Processable for TessellateLayerStep { if let Err(e) = layer.process(&mut tessellator) { context .processor_mut() - .unavailable_layer(coords, layer_name); + .layer_unavailable(coords, layer_name); tracing::error!( "layer {} at {} tesselation failed {:?}", @@ -81,7 +81,7 @@ impl Processable for TessellateLayerStep { e ); } else { - context.processor_mut().finished_layer_tesselation( + context.processor_mut().layer_tesselation_finished( coords, tessellator.buffer.into(), tessellator.feature_indices, @@ -99,7 +99,7 @@ impl Processable for TessellateLayerStep { for missing_layer in tile_request.layers.difference(&available_layers) { context .processor_mut() - .unavailable_layer(coords, missing_layer); + .layer_unavailable(coords, missing_layer); tracing::info!( "requested layer {} at {} not found in tile", @@ -112,23 +112,23 @@ impl Processable for TessellateLayerStep { context .processor_mut() - .finished_tile_tesselation(request_id, &tile_request.coords); + .tile_finished(request_id, &tile_request.coords); (tile_request, request_id, tile) } } -pub fn build_vector_tile_pipeline( -) -> impl Processable::Input> { - PipelineStep::new( - ParseTileStep, - PipelineStep::new(TessellateLayerStep, EndStep::default()), +pub fn build_vector_tile_pipeline() -> impl Processable::Input> { + DataPipeline::new( + ParseTile, + DataPipeline::new(TessellateLayer, PipelineEnd::default()), ) } #[cfg(test)] mod tests { use super::build_vector_tile_pipeline; + use crate::coords::ZoomLevel; use crate::io::pipeline::{PipelineContext, PipelineProcessor, Processable}; use crate::io::TileRequest; pub struct DummyPipelineProcessor; @@ -144,7 +144,7 @@ mod tests { let output = pipeline.process( ( TileRequest { - coords: (0, 0, 0).into(), + coords: (0, 0, ZoomLevel::default()).into(), layers: Default::default(), }, 0, diff --git a/maplibre/src/lib.rs b/maplibre/src/lib.rs index 146aaad4..6fe7472e 100644 --- a/maplibre/src/lib.rs +++ b/maplibre/src/lib.rs @@ -47,7 +47,8 @@ pub mod benchmarking; pub(crate) mod tessellation; pub mod util; -/// Map's configuration and execution. +/// The [`Map`] defines the public interface of the map renderer. +// DO NOT IMPLEMENT INTERNALS ON THIS STRUCT. pub struct Map where MWC: MapWindowConfig, @@ -114,6 +115,7 @@ where map_schedule: SimpleMapSchedule, window: MWC::MapWindow, } + impl HeadlessMap where MWC: MapWindowConfig, diff --git a/maplibre/src/render/copy_surface_to_buffer_node.rs b/maplibre/src/render/copy_surface_to_buffer_node.rs index da0af63d..1daf4920 100644 --- a/maplibre/src/render/copy_surface_to_buffer_node.rs +++ b/maplibre/src/render/copy_surface_to_buffer_node.rs @@ -1,3 +1,7 @@ +//! Node which copies the contents of the GPU-side texture in [`BufferedTextureHead`] to an +//! unmapped GPU-side buffer. This buffer will be mapped in +//! [`crate::render::stages::write_surface_buffer_stage::WriteSurfaceBufferStage`]. + use crate::render::graph::{Node, NodeRunError, RenderContext, RenderGraphContext, SlotInfo}; use crate::render::render_commands::{DrawMasks, DrawTiles}; use crate::render::render_phase::{PhaseItem, RenderCommand}; diff --git a/maplibre/src/render/mod.rs b/maplibre/src/render/mod.rs index 9384aac9..a4bf5103 100644 --- a/maplibre/src/render/mod.rs +++ b/maplibre/src/render/mod.rs @@ -31,17 +31,19 @@ use crate::{HeadedMapWindow, MapWindow, MapWindowConfig}; use log::info; use std::sync::Arc; -// Rendering internals #[cfg(feature = "headless")] -mod copy_surface_to_buffer_node; +// Exposed because it should be addable conditionally +pub mod copy_surface_to_buffer_node; pub mod graph; +pub mod resource; +pub mod stages; + +// Rendering internals mod graph_runner; mod main_pass; mod render_commands; mod render_phase; -pub mod resource; mod shaders; -mod stages; mod tile_pipeline; mod tile_view_pattern; mod util; @@ -434,7 +436,7 @@ mod tests { .ok() .unwrap(); - let render_state = RenderState::new(Surface::from_image::( + let render_state = RenderState::new(Surface::from_image( &device, &HeadlessMapWindow { size: WindowSize::new(100, 100).unwrap(), diff --git a/maplibre/src/render/resource/buffer_pool.rs b/maplibre/src/render/resource/buffer_pool.rs index da4082fc..f3794929 100644 --- a/maplibre/src/render/resource/buffer_pool.rs +++ b/maplibre/src/render/resource/buffer_pool.rs @@ -578,6 +578,7 @@ impl RingIndex { #[cfg(test)] mod tests { + use crate::coords::ZoomLevel; use crate::style::layer::StyleLayer; use lyon::tessellation::VertexBuffers; @@ -638,7 +639,7 @@ mod tests { for _ in 0..2 { pool.allocate_layer_geometry( &queue, - (0, 0, 0).into(), + (0, 0, ZoomLevel::default()).into(), style_layer.clone(), &data48bytes_aligned, 2, @@ -652,7 +653,7 @@ mod tests { pool.allocate_layer_geometry( &queue, - (0, 0, 0).into(), + (0, 0, ZoomLevel::default()).into(), style_layer.clone(), &data24bytes_aligned, 2, @@ -666,7 +667,7 @@ mod tests { pool.allocate_layer_geometry( &queue, - (0, 0, 0).into(), + (0, 0, ZoomLevel::default()).into(), style_layer.clone(), &data24bytes_aligned, 2, @@ -678,7 +679,7 @@ mod tests { pool.allocate_layer_geometry( &queue, - (0, 0, 0).into(), + (0, 0, ZoomLevel::default()).into(), style_layer.clone(), &data24bytes_aligned, 2, @@ -689,7 +690,7 @@ mod tests { pool.allocate_layer_geometry( &queue, - (0, 0, 0).into(), + (0, 0, ZoomLevel::default()).into(), style_layer.clone(), &data24bytes_aligned, 2, @@ -700,7 +701,7 @@ mod tests { pool.allocate_layer_geometry( &queue, - (0, 0, 0).into(), + (0, 0, ZoomLevel::default()).into(), style_layer, &data24bytes_aligned, 2, diff --git a/maplibre/src/render/stages/mod.rs b/maplibre/src/render/stages/mod.rs index 5e73dfab..782215a9 100644 --- a/maplibre/src/render/stages/mod.rs +++ b/maplibre/src/render/stages/mod.rs @@ -19,8 +19,10 @@ mod phase_sort_stage; mod queue_stage; mod resource_stage; mod upload_stage; + #[cfg(feature = "headless")] -mod write_surface_buffer_stage; +// Exposed because it should be addable conditionally +pub mod write_surface_buffer_stage; /// The labels of the default App rendering stages. #[derive(Debug, Hash, PartialEq, Eq, Clone)] diff --git a/maplibre/src/render/stages/write_surface_buffer_stage.rs b/maplibre/src/render/stages/write_surface_buffer_stage.rs index 8a4ec4a6..a320af73 100644 --- a/maplibre/src/render/stages/write_surface_buffer_stage.rs +++ b/maplibre/src/render/stages/write_surface_buffer_stage.rs @@ -1,4 +1,5 @@ -//! Sorts items of the [RenderPhases](RenderPhase). +//! Stage which writes the current contents of the GPU/CPU buffer in [`BufferedTextureHead`] +//! to disk as PNG. use crate::context::MapContext; use crate::coords::{ViewRegion, Zoom}; @@ -18,6 +19,7 @@ use std::future::Future; use std::io::Write; use std::iter; use std::ops::Deref; +use std::sync::Arc; use tokio::runtime::Handle; use tokio::task; use wgpu::{BufferAsyncError, BufferSlice}; @@ -38,7 +40,7 @@ impl Stage for WriteSurfaceBufferStage { match state.surface.head() { Head::Headed(_) => {} Head::Headless(buffered_texture) => { - let buffered_texture = buffered_texture.clone(); + let buffered_texture: Arc = buffered_texture.clone(); let device = device.clone(); let current_frame = self.frame; diff --git a/maplibre/src/stages/message.rs b/maplibre/src/stages/message.rs index 133ef399..23cbcb95 100644 --- a/maplibre/src/stages/message.rs +++ b/maplibre/src/stages/message.rs @@ -3,7 +3,7 @@ use crate::error::Error; use crate::io::geometry_index::{GeometryIndex, IndexedGeometry}; use crate::io::pipeline::PipelineContext; use crate::io::pipeline::Processable; -use crate::io::pipeline_steps::build_vector_tile_pipeline; +use crate::io::tile_pipelines::build_vector_tile_pipeline; use crate::io::tile_repository::StoredLayer; use crate::io::tile_request_state::TileRequestState; use crate::io::{TileRequest, TileRequestID}; diff --git a/maplibre/src/stages/mod.rs b/maplibre/src/stages/mod.rs index 05edde01..0c17c72a 100644 --- a/maplibre/src/stages/mod.rs +++ b/maplibre/src/stages/mod.rs @@ -7,8 +7,8 @@ use crate::io::geometry_index::GeometryIndex; use crate::io::geometry_index::{IndexProcessor, IndexedGeometry, TileIndex}; use crate::io::pipeline::Processable; use crate::io::pipeline::{PipelineContext, PipelineProcessor}; -use crate::io::pipeline_steps::build_vector_tile_pipeline; use crate::io::source_client::{HttpSourceClient, SourceClient}; +use crate::io::tile_pipelines::build_vector_tile_pipeline; use crate::io::tile_repository::StoredLayer; use crate::io::tile_request_state::TileRequestState; use crate::io::{TileRequest, TileRequestID}; @@ -34,6 +34,7 @@ mod message; mod populate_tile_store_stage; mod request_stage; +/// Register stages required for requesting and preparing new tiles. pub fn register_stages( schedule: &mut Schedule, http_source_client: HttpSourceClient, @@ -61,7 +62,7 @@ pub struct HeadedPipelineProcessor { } impl PipelineProcessor for HeadedPipelineProcessor { - fn finished_tile_tesselation(&mut self, request_id: TileRequestID, coords: &WorldTileCoords) { + fn tile_finished(&mut self, request_id: TileRequestID, coords: &WorldTileCoords) { self.state .message_sender .send(TessellateMessage::Tile(TileTessellateMessage { @@ -71,7 +72,7 @@ impl PipelineProcessor for HeadedPipelineProcessor { .unwrap(); } - fn unavailable_layer(&mut self, coords: &WorldTileCoords, layer_name: &str) { + fn layer_unavailable(&mut self, coords: &WorldTileCoords, layer_name: &str) { self.state .message_sender .send(TessellateMessage::Layer( @@ -83,7 +84,7 @@ impl PipelineProcessor for HeadedPipelineProcessor { .unwrap(); } - fn finished_layer_tesselation( + fn layer_tesselation_finished( &mut self, coords: &WorldTileCoords, buffer: OverAlignedVertexBuffer, @@ -103,7 +104,7 @@ impl PipelineProcessor for HeadedPipelineProcessor { .unwrap(); } - fn finished_layer_indexing( + fn layer_indexing_finished( &mut self, coords: &WorldTileCoords, geometries: Vec>, diff --git a/maplibre/src/window.rs b/maplibre/src/window.rs index cac3cf98..d59059c8 100644 --- a/maplibre/src/window.rs +++ b/maplibre/src/window.rs @@ -3,23 +3,29 @@ use crate::{HttpClient, InteractiveMapSchedule, ScheduleMethod}; use raw_window_handle::{HasRawWindowHandle, RawWindowHandle}; -/// Window with a [carte::window::WindowSize]. +/// Window of a certain [`WindowSize`]. This can either be a proper window or a headless one. pub trait MapWindow { fn size(&self) -> WindowSize; } +/// Window which references a physical `RawWindow`. This is only implemented by headed windows and +/// not by headless windows. pub trait HeadedMapWindow: MapWindow { type RawWindow: HasRawWindowHandle; fn inner(&self) -> &Self::RawWindow; } +/// A configuration for a window which determines the corresponding implementation of a +/// [`MapWindow`] and is able to create it. pub trait MapWindowConfig: 'static { type MapWindow: MapWindow; fn create(&self) -> Self::MapWindow; } +/// The event loop is responsible for processing events and propagating them to the map renderer. +/// Only non-headless windows use an [`EventLoop`]. pub trait EventLoop where MWC: MapWindowConfig, From 3f821e7f66d0be9e8f584539c8b7bc1ce1e87307 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Wed, 1 Jun 2022 17:21:24 +0200 Subject: [PATCH 29/31] Run more tests in CI --- .github/actions/android/action.yml | 4 ++++ .github/actions/apple/action.yml | 6 +++++- .github/actions/demo/macos/action.yml | 4 ++++ .github/actions/tests/action.yml | 18 ++++++++++++++++++ .github/actions/webgpu/action.yml | 5 ++++- .github/workflows/on_main_push.yml | 5 +++++ .github/workflows/on_pull_request.yml | 5 +++++ 7 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 .github/actions/tests/action.yml diff --git a/.github/actions/android/action.yml b/.github/actions/android/action.yml index fe69f6c5..a0bdc7b0 100644 --- a/.github/actions/android/action.yml +++ b/.github/actions/android/action.yml @@ -24,3 +24,7 @@ runs: env "AR_aarch64-linux-android=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar" \ env "CC_aarch64-linux-android=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android30-clang" \ just check maplibre-android aarch64-linux-android + - name: Test + shell: bash + # TODO: Additional test runs for different targets + run: just test maplibre-android aarch64-linux-android \ No newline at end of file diff --git a/.github/actions/apple/action.yml b/.github/actions/apple/action.yml index 8ffe1ea7..07ec9498 100644 --- a/.github/actions/apple/action.yml +++ b/.github/actions/apple/action.yml @@ -24,8 +24,12 @@ runs: run: just check apple x86_64-apple-darwin - name: Check aarch64 darwin shell: bash + # TODO: Additional clippy checks for different targets (iOS) run: just check apple aarch64-apple-darwin - # TODO: Additional clippy checks for iOS + - name: Test + shell: bash + # TODO: Additional test runs for different targets + run: just test apple aarch64-apple-darwin - name: Build Example shell: bash run: cd apple/xcode && xcodebuild -scheme "example (iOS)" -arch arm64 -sdk iphoneos build CODE_SIGNING_ALLOWED=NO diff --git a/.github/actions/demo/macos/action.yml b/.github/actions/demo/macos/action.yml index 36869ec4..3078507c 100644 --- a/.github/actions/demo/macos/action.yml +++ b/.github/actions/demo/macos/action.yml @@ -19,6 +19,10 @@ runs: - name: Build shell: bash run: cd apple/xcode && xcodebuild -scheme "example (macOS)" build CODE_SIGNING_ALLOWED=NO MACOSX_DEPLOYMENT_TARGET=10.9 -derivedDataPath build + - name: Test + shell: bash + # TODO: Additional test runs for different targets + run: just test apple aarch64-apple-darwin - uses: actions/upload-artifact@v3 with: name: maplibre-x86_64-apple-darwin-demo diff --git a/.github/actions/tests/action.yml b/.github/actions/tests/action.yml new file mode 100644 index 00000000..61327ab9 --- /dev/null +++ b/.github/actions/tests/action.yml @@ -0,0 +1,18 @@ +name: tests +description: Run tests + +runs: + using: "composite" + steps: + - uses: extractions/setup-just@v1 + - name: Install toolchain + shell: bash + run: just default-toolchain + - uses: Swatinem/rust-cache@v1 + - name: Install Dependencies + shell: bash + run: sudo apt-get install -y libwayland-dev libxkbcommon-dev # Required for winit + - name: Test + shell: bash + # TODO: Additional test runs for different targets + run: just test maplibre x86_64-unknown-linux-gnu diff --git a/.github/actions/webgpu/action.yml b/.github/actions/webgpu/action.yml index 86a28ca3..a947c84b 100644 --- a/.github/actions/webgpu/action.yml +++ b/.github/actions/webgpu/action.yml @@ -17,4 +17,7 @@ runs: run: just web-demo build - name: Check shell: bash - run: just check web wasm32-unknown-unknown \ No newline at end of file + run: just check web wasm32-unknown-unknown + - name: Check + shell: bash + run: just test web wasm32-unknown-unknown diff --git a/.github/workflows/on_main_push.yml b/.github/workflows/on_main_push.yml index adeaffaa..527824c5 100644 --- a/.github/workflows/on_main_push.yml +++ b/.github/workflows/on_main_push.yml @@ -17,6 +17,11 @@ jobs: steps: - uses: actions/checkout@v2 - uses: ./.github/actions/benchmarks + run-tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/tests build-android: runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml index 41de25d9..947aa23f 100644 --- a/.github/workflows/on_pull_request.yml +++ b/.github/workflows/on_pull_request.yml @@ -17,6 +17,11 @@ jobs: steps: - uses: actions/checkout@v2 - uses: ./.github/actions/benchmarks + run-tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/tests build-android: runs-on: ubuntu-20.04 steps: From 873e351890dc720d0876296f3f230419004b901c Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Wed, 1 Jun 2022 17:50:07 +0200 Subject: [PATCH 30/31] Fix tests for various platforms --- .cargo/config.toml | 3 ++- .github/actions/android/action.yml | 10 ++++++---- .github/actions/apple/action.yml | 10 +++++----- .github/actions/demo/linux/action.yml | 6 +++--- .github/actions/demo/macos/action.yml | 8 +++++--- .github/actions/demo/windows/action.yml | 5 ++++- .github/actions/webgl/action.yml | 3 +++ .github/actions/webgpu/action.yml | 4 ++-- .github/workflows/on_main_push.yml | 2 +- .github/workflows/on_pull_request.yml | 2 +- justfile | 8 +++++++- maplibre/src/render/mod.rs | 1 + web/Cargo.toml | 3 +++ web/src/lib.rs | 12 ++++++++++++ 14 files changed, 55 insertions(+), 22 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 8c8d3ba5..54e293df 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -7,4 +7,5 @@ rustflags = [ # Enables the possibility to import memory into wasm. # Without --shared-memory it is not possible to use shared WebAssembly.Memory. "-C", "link-args=--shared-memory --import-memory", -] \ No newline at end of file +] +runner = 'wasm-bindgen-test-runner' diff --git a/.github/actions/android/action.yml b/.github/actions/android/action.yml index a0bdc7b0..13da6c7a 100644 --- a/.github/actions/android/action.yml +++ b/.github/actions/android/action.yml @@ -12,6 +12,7 @@ runs: - name: Build shell: bash run: just build-android + # TODO: Additional clippy checks for different targets - name: Check x86_64 shell: bash run: | @@ -24,7 +25,8 @@ runs: env "AR_aarch64-linux-android=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar" \ env "CC_aarch64-linux-android=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android30-clang" \ just check maplibre-android aarch64-linux-android - - name: Test - shell: bash - # TODO: Additional test runs for different targets - run: just test maplibre-android aarch64-linux-android \ No newline at end of file + # FIXME: Requires cross-compilation + #- name: Test + # shell: bash + # # TODO: Additional test runs for different targets + # run: just test maplibre-android aarch64-linux-android \ No newline at end of file diff --git a/.github/actions/apple/action.yml b/.github/actions/apple/action.yml index 07ec9498..d0913ff1 100644 --- a/.github/actions/apple/action.yml +++ b/.github/actions/apple/action.yml @@ -22,14 +22,14 @@ runs: - name: Check x86_64 darwin shell: bash run: just check apple x86_64-apple-darwin - - name: Check aarch64 darwin + - name: Check x86_64 darwin shell: bash # TODO: Additional clippy checks for different targets (iOS) - run: just check apple aarch64-apple-darwin - - name: Test + run: just check apple x86_64-apple-darwin + - name: Test x86_64 darwin shell: bash - # TODO: Additional test runs for different targets - run: just test apple aarch64-apple-darwin + # TODO: Additional test runs for different targets (Different targets might require emulation) + run: just test apple x86_64-apple-darwin - name: Build Example shell: bash run: cd apple/xcode && xcodebuild -scheme "example (iOS)" -arch arm64 -sdk iphoneos build CODE_SIGNING_ALLOWED=NO diff --git a/.github/actions/demo/linux/action.yml b/.github/actions/demo/linux/action.yml index f04ebf10..825dc60a 100644 --- a/.github/actions/demo/linux/action.yml +++ b/.github/actions/demo/linux/action.yml @@ -15,12 +15,12 @@ runs: - name: Build shell: bash run: cargo build -p maplibre-demo - - name: Test - shell: bash - run: just test maplibre-demo x86_64-unknown-linux-gnu - name: Check shell: bash run: just check maplibre-demo x86_64-unknown-linux-gnu + - name: Test x86_64 linux + shell: bash + run: just test maplibre-demo x86_64-unknown-linux-gnu - uses: actions/upload-artifact@v2 with: name: maplibre-rs diff --git a/.github/actions/demo/macos/action.yml b/.github/actions/demo/macos/action.yml index 3078507c..769b8b6d 100644 --- a/.github/actions/demo/macos/action.yml +++ b/.github/actions/demo/macos/action.yml @@ -19,10 +19,12 @@ runs: - name: Build shell: bash run: cd apple/xcode && xcodebuild -scheme "example (macOS)" build CODE_SIGNING_ALLOWED=NO MACOSX_DEPLOYMENT_TARGET=10.9 -derivedDataPath build - - name: Test + - name: Check x86_64 darwin shell: bash - # TODO: Additional test runs for different targets - run: just test apple aarch64-apple-darwin + run: just check maplibre-demo x86_64-apple-darwin + - name: Test x86_64 darwin + shell: bash + run: just test maplibre-demo x86_64-apple-darwin - uses: actions/upload-artifact@v3 with: name: maplibre-x86_64-apple-darwin-demo diff --git a/.github/actions/demo/windows/action.yml b/.github/actions/demo/windows/action.yml index d0225c55..05eaa866 100644 --- a/.github/actions/demo/windows/action.yml +++ b/.github/actions/demo/windows/action.yml @@ -22,7 +22,10 @@ runs: - name: Build shell: bash run: cargo build -p maplibre-demo --release --target x86_64-pc-windows-msvc - - name: Test + - name: Check x86_64 windows + shell: bash + run: just check maplibre-demo x86_64-pc-windows-msvc + - name: Test x86_64 windows shell: bash run: just test maplibre-demo x86_64-pc-windows-msvc - uses: actions/upload-artifact@v3 diff --git a/.github/actions/webgl/action.yml b/.github/actions/webgl/action.yml index 2a6e96e9..c0fedbe9 100644 --- a/.github/actions/webgl/action.yml +++ b/.github/actions/webgl/action.yml @@ -18,3 +18,6 @@ runs: - name: Check shell: bash run: just check web wasm32-unknown-unknown + - name: Test + shell: bash + run: just web-test "web-webgl" diff --git a/.github/actions/webgpu/action.yml b/.github/actions/webgpu/action.yml index a947c84b..5f851891 100644 --- a/.github/actions/webgpu/action.yml +++ b/.github/actions/webgpu/action.yml @@ -18,6 +18,6 @@ runs: - name: Check shell: bash run: just check web wasm32-unknown-unknown - - name: Check + - name: Test shell: bash - run: just test web wasm32-unknown-unknown + run: just web-test "" diff --git a/.github/workflows/on_main_push.yml b/.github/workflows/on_main_push.yml index 527824c5..a1633e7c 100644 --- a/.github/workflows/on_main_push.yml +++ b/.github/workflows/on_main_push.yml @@ -62,7 +62,7 @@ jobs: source: docs/book/. destination: docs key: ${{ secrets.SSH_KEY_MAXAMMANN_ORG }} - build-ios: + build-apple: runs-on: macos-11 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml index 947aa23f..65b86dfd 100644 --- a/.github/workflows/on_pull_request.yml +++ b/.github/workflows/on_pull_request.yml @@ -42,7 +42,7 @@ jobs: steps: - uses: actions/checkout@v2 - uses: ./.github/actions/docs - build-ios: + build-apple: runs-on: macos-11 steps: - uses: actions/checkout@v2 diff --git a/justfile b/justfile index b8fe2e51..de95a609 100644 --- a/justfile +++ b/justfile @@ -45,12 +45,18 @@ web-lib TARGET: nightly-toolchain (web-install "lib") web-demo TARGET: (web-install "demo") cd web/demo && npm run {{TARGET}} +web-test FEATURES: nightly-toolchain + export RUSTUP_TOOLCHAIN=$NIGHTLY_TOOLCHAIN && cargo test -p web --features "{{FEATURES}}" --target wasm32-unknown-unknown -Z build-std=std,panic_abort + #profile-bench: # cargo flamegraph --bench render -- --bench -build-android: print-android-env +build-android: nightly-toolchain print-android-env export RUSTUP_TOOLCHAIN=$NIGHTLY_TOOLCHAIN && cd android/gradle && ./gradlew assembleDebug +test-android TARGET: nightly-toolchain print-android-env + export RUSTUP_TOOLCHAIN=$NIGHTLY_TOOLCHAIN && cargo test -p maplibre-android --target {{TARGET}} -Z build-std=std,panic_abort + # language=bash print-android-env: #!/usr/bin/env bash diff --git a/maplibre/src/render/mod.rs b/maplibre/src/render/mod.rs index a4bf5103..7f38200e 100644 --- a/maplibre/src/render/mod.rs +++ b/maplibre/src/render/mod.rs @@ -409,6 +409,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[tokio::test] + #[ignore] // FIXME: We do not have a GPU in CI async fn test_render() { let graph = RenderGraph::default(); diff --git a/web/Cargo.toml b/web/Cargo.toml index 7bc068bc..97721658 100644 --- a/web/Cargo.toml +++ b/web/Cargo.toml @@ -35,3 +35,6 @@ wasm-bindgen = "0.2" wasm-bindgen-futures = "0.4" console_log = { version = "0.2", features = ["color"] } tracing-wasm = { version = "0.2", optional = true } # FIXME: Low quality dependency + +[dev-dependencies] +wasm-bindgen-test = "0.3" \ No newline at end of file diff --git a/web/src/lib.rs b/web/src/lib.rs index 7ade3b26..b2478607 100644 --- a/web/src/lib.rs +++ b/web/src/lib.rs @@ -65,3 +65,15 @@ pub async fn run(scheduler_ptr: *mut Scheduler) { // std::mem::forget(scheduler); } + +#[cfg(test)] +/// See https://rustwasm.github.io/wasm-bindgen/wasm-bindgen-test/browsers.html +mod tests { + use wasm_bindgen_test::*; + wasm_bindgen_test_configure!(run_in_browser); + + #[wasm_bindgen_test] + fn pass() { + assert_eq!(1, 1); + } +} From cef804ef186478d04b9ca061ad0da94095e8bfd3 Mon Sep 17 00:00:00 2001 From: Maximilian Ammann Date: Wed, 1 Jun 2022 18:10:49 +0200 Subject: [PATCH 31/31] Install missing dependency --- .github/actions/webgl/action.yml | 4 +++- .github/actions/webgpu/action.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/webgl/action.yml b/.github/actions/webgl/action.yml index c0fedbe9..f2cfec1a 100644 --- a/.github/actions/webgl/action.yml +++ b/.github/actions/webgl/action.yml @@ -20,4 +20,6 @@ runs: run: just check web wasm32-unknown-unknown - name: Test shell: bash - run: just web-test "web-webgl" + run: | + cargo install wasm-bindgen-cli --version "0.2.80" + just web-test "web-webgl" diff --git a/.github/actions/webgpu/action.yml b/.github/actions/webgpu/action.yml index 5f851891..43618cb0 100644 --- a/.github/actions/webgpu/action.yml +++ b/.github/actions/webgpu/action.yml @@ -20,4 +20,6 @@ runs: run: just check web wasm32-unknown-unknown - name: Test shell: bash - run: just web-test "" + run: | + cargo install wasm-bindgen-cli --version "0.2.80" + just web-test ""