Enable bechmarks (#129)

* Fix benchmarks

* Add test data

* Update drivers

* No vulkan-sdk

.github/actions/download-test-data/action.yml

@@ -0,0 +1,9 @@
+name: download-test-data
+description: Download test data
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download
+      shell: bash
+      run: wget -O test-data/munich-15.mbtiles https://maplibre-rs-test-data.pages.dev/munich-15.mbtiles

.github/actions/install-driver/action.yml

@@ -0,0 +1,20 @@
+name: install-driver
+description: Install drivers
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Mesa Dependencies
+      shell: bash
+      run: |
+        sudo apt-get update -y -qq
+        
+        # Get latest drivers. The lavapipe in Ubuntu 20.04 SEGVs.
+        sudo add-apt-repository ppa:oibaf/graphics-drivers -y
+
+        sudo apt install -y libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
+    - name: wgpu info
+      shell: bash
+      run: |
+        cargo install --git "https://github.com/gfx-rs/wgpu" wgpu-info
+        wgpu-info

.github/workflows/run-benchmarks.yml

@@ -14,9 +14,10 @@ jobs:
         shell: bash
         run: just default-toolchain
       - uses: Swatinem/rust-cache@v1
-      - name: Install Dependencies
+      - name: Install GPU Drivers
+        uses: ./.github/actions/install-driver
+      - name: Download test data
+        uses: ./.github/actions/download-test-data
+      - name: Benchmark
         shell: bash
-        run: sudo apt-get install -y libwayland-dev libxkbcommon-dev # Required for winit
-      - name: Check
-        shell: bash
-        run: just check benchmarks x86_64-unknown-linux-gnu
+        run: WGPU_BACKEND=vulkan just benchmark

.github/workflows/run-tests.yml

@@ -14,14 +14,8 @@ jobs:
         shell: bash
         run: just default-toolchain
       - uses: Swatinem/rust-cache@v1
-      - name: Install Mesa Dependencies
-        shell: bash
-        run: sudo apt install -y libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
-      - name: wgpu info
-        shell: bash
-        run: |
-          cargo install --git "https://github.com/gfx-rs/wgpu" wgpu-info
-          wgpu-info
+      - name: Install GPU Drivers
+        uses: ./.github/actions/install-driver
       - name: Test Vulkan
         shell: bash
         # TODO: Additional test runs for different targets

.gitignore

@@ -26,4 +26,6 @@ logs/
 perf.data*
 *.folded
 *flamegraph.svg
-*flamechart.svg
+*flamechart.svg
+
+frame_*.png

benchmarks/Cargo.toml

@@ -6,7 +6,16 @@ categories = []
 edition = "2021"
 
 [dependencies]
-maplibre = { path = "../maplibre" }
+maplibre = { path = "../maplibre", features = ["headless", "embed-static-tiles"] }
 
 [dev-dependencies]
-criterion = "0.3"
+criterion = { version = "0.3", features = ["async_tokio"] }
+tokio = "1.18.2"
+
+[[bench]]
+name = "render"
+harness = false
+
+[[bench]]
+name = "data"
+harness = false

benchmarks/benches/data.rs

@@ -0,0 +1,73 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use maplibre::benchmarking::io::static_tile_fetcher::StaticTileFetcher;
+use maplibre::benchmarking::tessellation::Tessellated;
+use maplibre::coords::{TileCoords, WorldTileCoords, ZoomLevel};
+use maplibre::io::pipeline::{PipelineContext, PipelineProcessor, Processable};
+use maplibre::io::tile_pipelines::{ParseTile, TessellateLayer};
+use maplibre::io::TileRequest;
+use maplibre::style::source::TileAddressingScheme;
+use std::collections::HashSet;
+use std::io::Cursor;
+
+const MUNICH_COORDS: TileCoords = TileCoords {
+    x: 17425,
+    y: 11365,
+    z: ZoomLevel::new(15u8),
+};
+
+pub struct DummyPipelineProcessor;
+
+impl PipelineProcessor for DummyPipelineProcessor {}
+
+fn parse_tile(c: &mut Criterion) {
+    let fetcher = StaticTileFetcher::new();
+
+    c.bench_function("parse", |b| {
+        b.iter(|| {
+            let request = TileRequest {
+                coords: MUNICH_COORDS
+                    .into_world_tile(TileAddressingScheme::XYZ)
+                    .unwrap(),
+                layers: HashSet::from(["boundary".to_owned(), "water".to_owned()]),
+            };
+            let data = fetcher
+                .sync_fetch_tile(&MUNICH_COORDS)
+                .unwrap()
+                .into_boxed_slice();
+            ParseTile::default().process(
+                (request, 0, data),
+                &mut PipelineContext::new(DummyPipelineProcessor),
+            );
+        })
+    });
+}
+
+fn tessellate_tile(c: &mut Criterion) {
+    let fetcher = StaticTileFetcher::new();
+    let request = TileRequest {
+        coords: MUNICH_COORDS
+            .into_world_tile(TileAddressingScheme::XYZ)
+            .unwrap(),
+        layers: HashSet::from(["boundary".to_owned(), "water".to_owned()]),
+    };
+    let data = fetcher
+        .sync_fetch_tile(&MUNICH_COORDS)
+        .unwrap()
+        .into_boxed_slice();
+    let parsed = ParseTile::default().process(
+        (request, 0, data),
+        &mut PipelineContext::new(DummyPipelineProcessor),
+    );
+
+    c.bench_function("tessselate", |b| {
+        b.iter(|| {
+            TessellateLayer::default().process(
+                parsed.clone(),
+                &mut PipelineContext::new(DummyPipelineProcessor),
+            );
+        })
+    });
+}
+
+criterion_group!(benches, parse_tile, tessellate_tile);
+criterion_main!(benches);

benchmarks/benches/render.rs

@@ -1,17 +1,100 @@
 use criterion::{criterion_group, criterion_main, Criterion};
-use maplibre::window::FromWindow;
-use maplibre::{MapBuilder, ScheduleMethod, TokioScheduleMethod};
+use maplibre::coords::{WorldTileCoords, ZoomLevel};
+use maplibre::error::Error;
+use maplibre::headless::utils::HeadlessPipelineProcessor;
+use maplibre::headless::HeadlessMapWindowConfig;
+use maplibre::io::pipeline::PipelineContext;
+use maplibre::io::pipeline::Processable;
+use maplibre::io::source_client::HttpSourceClient;
+use maplibre::io::tile_pipelines::build_vector_tile_pipeline;
+use maplibre::io::TileRequest;
+use maplibre::platform::http_client::ReqwestHttpClient;
+use maplibre::platform::run_multithreaded;
+use maplibre::platform::schedule_method::TokioScheduleMethod;
+use maplibre::render::settings::{RendererSettings, TextureFormat};
+use maplibre::window::WindowSize;
+use maplibre::MapBuilder;
+use std::collections::HashSet;
 
-fn render(c: &mut Criterion) {
-    c.bench_function("render", |b| {
-        b.iter(|| {
-            MapBuilder::from_window("A fantastic window!")
-                .with_schedule_method(ScheduleMethod::Tokio(TokioScheduleMethod::new()))
+fn headless_render(c: &mut Criterion) {
+    c.bench_function("headless_render", |b| {
+        let mut map = run_multithreaded(async {
+            let mut map = MapBuilder::new()
+                .with_map_window_config(HeadlessMapWindowConfig {
+                    size: WindowSize::new(1000, 1000).unwrap(),
+                })
+                .with_http_client(ReqwestHttpClient::new(None))
+                .with_schedule_method(TokioScheduleMethod::new())
+                .with_renderer_settings(RendererSettings {
+                    texture_format: TextureFormat::Rgba8UnormSrgb,
+                    ..RendererSettings::default()
+                })
                 .build()
-                .run_sync_with_max_frames(1000);
-        })
+                .initialize_headless()
+                .await;
+
+            let http_source_client: HttpSourceClient<ReqwestHttpClient> =
+                HttpSourceClient::new(ReqwestHttpClient::new(None));
+
+            let coords = WorldTileCoords::from((0, 0, ZoomLevel::default()));
+            let request_id = 0;
+
+            let data = http_source_client
+                .fetch(&coords)
+                .await
+                .unwrap()
+                .into_boxed_slice();
+
+            let processor = HeadlessPipelineProcessor::default();
+            let mut pipeline_context = PipelineContext::new(processor);
+            let pipeline = build_vector_tile_pipeline();
+            pipeline.process(
+                (
+                    TileRequest {
+                        coords,
+                        layers: HashSet::from(["boundary".to_owned(), "water".to_owned()]),
+                    },
+                    request_id,
+                    data,
+                ),
+                &mut pipeline_context,
+            );
+
+            let mut processor = pipeline_context
+                .take_processor::<HeadlessPipelineProcessor>()
+                .unwrap();
+
+            while let Some(v) = processor.layers.pop() {
+                map.map_schedule_mut()
+                    .map_context
+                    .tile_repository
+                    .put_tessellated_layer(v);
+            }
+
+            map
+        });
+
+        b.to_async(
+            tokio::runtime::Builder::new_multi_thread()
+                .enable_all()
+                .build()
+                .unwrap(),
+        )
+        .iter(|| {
+            match map.map_schedule_mut().update_and_redraw() {
+                Ok(_) => {}
+                Err(Error::Render(e)) => {
+                    eprintln!("{}", e);
+                    if e.should_exit() {}
+                }
+                e => eprintln!("{:?}", e),
+            };
+            async {}
+        });
     });
 }
 
-criterion_group!(benches, render);
+criterion_group!(name = benches;
+    config = Criterion::default().significance_level(0.1).sample_size(20);
+    targets = headless_render);
 criterion_main!(benches);

benchmarks/benches/tessellation.rs

@@ -1,44 +0,0 @@
-use criterion::{criterion_group, criterion_main, Criterion};
-// use lyon::tessellation::VertexBuffers;
-use maplibre::benchmarking::io::static_tile_fetcher::StaticTileFetcher;
-use maplibre::benchmarking::tessellation::Tessellated;
-use std::io::Cursor;
-
-const MUNICH_X: u32 = 17425;
-const MUNICH_Y: u32 = 11365;
-const MUNICH_Z: u8 = 15;
-
-fn parse_tile(c: &mut Criterion) {
-    let fetcher = StaticTileFetcher::new();
-
-    /*    c.bench_function("parse", |b| {
-        b.iter(|| {
-            parse_tile_reader(&mut Cursor::new(
-                fetcher
-                    .sync_fetch_tile(&(MUNICH_X, MUNICH_Y, MUNICH_Z).into())
-                    .unwrap(),
-            ))
-            .expect("failed to load tile")
-        })
-    });*/
-}
-
-fn tessellate_tile(c: &mut Criterion) {
-    /*    let fetcher = StaticTileFetcher::new();
-    let tile = parse_tile_reader(&mut Cursor::new(
-        fetcher
-            .sync_fetch_tile(&(MUNICH_X, MUNICH_Y, MUNICH_Z).into())
-            .unwrap(),
-    ))
-    .expect("failed to load tile");
-    let layer = tile.layers().first().unwrap();
-
-    fn tessselate(layer: &Layer) {
-        let _: (VertexBuffers<_, u32>, _) = layer.tessellate().unwrap();
-    }
-
-    c.bench_function("tessselate", |b| b.iter(|| tessselate(layer)));*/
-}
-
-criterion_group!(benches, parse_tile, tessellate_tile);
-criterion_main!(benches);

justfile

@@ -18,6 +18,7 @@ fixup:
   cargo clippy --no-deps -p maplibre --fix
   cargo clippy --allow-dirty --no-deps -p maplibre-winit --fix
   cargo clippy --allow-dirty --no-deps -p maplibre-demo --fix
+  cargo clippy --allow-dirty --no-deps -p benchmarks --fix
   # Web
   cargo clippy --allow-dirty --no-deps -p web --target wasm32-unknown-unknown --fix
   cargo clippy --allow-dirty --no-deps -p maplibre --target wasm32-unknown-unknown --fix
@@ -32,6 +33,9 @@ check PROJECT ARCH: install-clippy
 test PROJECT ARCH:
   cargo test -p {{PROJECT}} --target {{ARCH}}
 
+benchmark:
+  cargo bench -p benchmarks
+
 install-rustfmt:
   rustup component add rustfmt
 

maplibre-demo/src/main.rs

@@ -19,6 +19,7 @@ use maplibre::window::{EventLoop, WindowSize};
 use maplibre::MapBuilder;
 use maplibre_winit::winit::WinitMapWindowConfig;
 
+use maplibre::headless::utils::HeadlessPipelineProcessor;
 use std::collections::HashSet;
 
 #[cfg(feature = "trace")]
@@ -44,28 +45,6 @@ fn run_in_window() {
     })
 }
 
-#[derive(Default)]
-struct HeadlessPipelineProcessor {
-    layers: Vec<StoredLayer>,
-}
-
-impl PipelineProcessor for HeadlessPipelineProcessor {
-    fn layer_tesselation_finished(
-        &mut self,
-        coords: &WorldTileCoords,
-        buffer: OverAlignedVertexBuffer<ShaderVertex, IndexDataType>,
-        feature_indices: Vec<u32>,
-        layer_data: RawLayer,
-    ) {
-        self.layers.push(StoredLayer::TessellatedLayer {
-            coords: *coords,
-            buffer,
-            feature_indices,
-            layer_data,
-        })
-    }
-}
-
 fn run_headless() {
     run_multithreaded(async {
         let mut map = MapBuilder::new()

maplibre/build.rs

@@ -3,28 +3,12 @@
 //! This script is built and executed just before building the package.
 //! It will validate the WGSL (WebGPU Shading Language) shaders and embed static files.
 
-#[cfg(feature = "embed-static-tiles")]
-use maplibre_build_tools::mbtiles::extract;
 use maplibre_build_tools::wgsl::validate_project_wgsl;
 
 const MUNICH_X: u32 = 17425;
 const MUNICH_Y: u32 = 11365;
 const MUNICH_Z: u8 = 15;
 
-/// Tiles which can be used by StaticTileFetcher.
-#[cfg(feature = "embed-static-tiles")]
-fn clean_static_tiles() -> std::path::PathBuf {
-    let out_dir = std::env::var("OUT_DIR").unwrap();
-
-    let out = std::path::Path::new(&out_dir).join("extracted-tiles");
-
-    if out.exists() && out.is_dir() {
-        std::fs::remove_dir_all(&out).unwrap()
-    }
-
-    out
-}
-
 /*use std::fs::File;
 use std::io::BufReader;
 use serde_json::Value;*/
@@ -47,18 +31,32 @@ fn generate_type_def() -> Option<u32> {
     Some(5)
 }
 
-#[cfg(feature = "embed-static-tiles")]
 fn embed_tiles_statically() {
+    use maplibre_build_tools::mbtiles::extract;
     use std::env;
     use std::path::Path;
+
+    /// Tiles which can be used by StaticTileFetcher.
+    fn clean_static_tiles() -> std::path::PathBuf {
+        let out_dir = std::env::var("OUT_DIR").unwrap();
+
+        let out = std::path::Path::new(&out_dir).join("extracted-tiles");
+
+        if out.exists() && out.is_dir() {
+            std::fs::remove_dir_all(&out).unwrap()
+        }
+
+        out
+    }
+
     let out = clean_static_tiles();
 
     let root_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
 
-    let source = Path::new(&root_dir).join(format!("test-data/munich-{}.mbtiles", MUNICH_Z));
+    let source = Path::new(&root_dir).join(format!("../test-data/munich-{}.mbtiles", MUNICH_Z));
 
     if source.exists() {
-        println!("cargo:rustc-cfg=static_tiles");
+        println!("cargo:rustc-cfg=static_tiles_found");
         // Pack tiles around Munich HBF (100 tiles in each direction)
         extract(
             source,
@@ -76,6 +74,5 @@ fn embed_tiles_statically() {
 fn main() {
     validate_project_wgsl();
 
-    #[cfg(feature = "embed-static-tiles")]
     embed_tiles_statically();
 }

maplibre/src/coords.rs

@@ -62,6 +62,9 @@ impl fmt::Debug for Quadkey {
 pub struct ZoomLevel(u8);
 
 impl ZoomLevel {
+    pub const fn new(z: u8) -> Self {
+        ZoomLevel(z)
+    }
     pub fn is_root(self) -> bool {
         self.0 == 0
     }
@@ -236,12 +239,12 @@ impl TileCoords {
     }
 }
 
-impl From<(u32, u32, u8)> for TileCoords {
-    fn from(tuple: (u32, u32, u8)) -> Self {
+impl From<(u32, u32, ZoomLevel)> for TileCoords {
+    fn from(tuple: (u32, u32, ZoomLevel)) -> Self {
         TileCoords {
             x: tuple.0,
             y: tuple.1,
-            z: ZoomLevel::from(tuple.2),
+            z: tuple.2,
         }
     }
 }

maplibre/src/headless.rs

@@ -242,3 +242,34 @@ impl Stage for WriteSurfaceBufferStage {
         }
     }
 }
+
+pub mod utils {
+    use crate::coords::WorldTileCoords;
+    use crate::io::pipeline::PipelineProcessor;
+    use crate::io::tile_repository::StoredLayer;
+    use crate::io::RawLayer;
+    use crate::render::ShaderVertex;
+    use crate::tessellation::{IndexDataType, OverAlignedVertexBuffer};
+
+    #[derive(Default)]
+    pub struct HeadlessPipelineProcessor {
+        pub layers: Vec<StoredLayer>,
+    }
+
+    impl PipelineProcessor for HeadlessPipelineProcessor {
+        fn layer_tesselation_finished(
+            &mut self,
+            coords: &WorldTileCoords,
+            buffer: OverAlignedVertexBuffer<ShaderVertex, IndexDataType>,
+            feature_indices: Vec<u32>,
+            layer_data: RawLayer,
+        ) {
+            self.layers.push(StoredLayer::TessellatedLayer {
+                coords: *coords,
+                buffer,
+                feature_indices,
+                layer_data,
+            })
+        }
+    }
+}

maplibre/src/io/mod.rs

@@ -1,15 +1,15 @@
 //! Handles IO related processing as well as multithreading.
 
 use crate::coords::WorldTileCoords;
-
 use std::collections::HashSet;
 use std::fmt;
-pub mod scheduler;
-pub mod source_client;
-pub mod static_tile_fetcher;
 
 pub mod geometry_index;
 pub mod pipeline;
+pub mod scheduler;
+pub mod source_client;
+#[cfg(feature = "embed-static-tiles")]
+pub mod static_tile_fetcher;
 pub mod tile_pipelines;
 pub mod tile_repository;
 pub mod tile_request_state;

maplibre/src/io/static_tile_fetcher.rs

@@ -8,11 +8,11 @@ use include_dir::Dir;
 use crate::coords::TileCoords;
 use crate::error::Error;
 
-#[cfg(static_tiles)]
+#[cfg(static_tiles_found)]
 use include_dir::include_dir;
-#[cfg(static_tiles)]
+#[cfg(static_tiles_found)]
 static TILES: Dir = include_dir!("$OUT_DIR/extracted-tiles");
-#[cfg(not(static_tiles))]
+#[cfg(not(static_tiles_found))]
 static TILES: Dir = Dir::new("/path", &[]);
 
 /// Load PBF files which were statically embedded in the `build.rs`
@@ -38,7 +38,7 @@ impl StaticTileFetcher {
     /// could not be fetched.
     pub fn sync_fetch_tile(&self, coords: &TileCoords) -> Result<Vec<u8>, Error> {
         if TILES.entries().is_empty() {
-            log::error!(
+            panic!(
                 "There are not tiles statically embedded in this binary! StaticTileFetcher will \
                 not return any tiles!"
             )
@@ -55,8 +55,11 @@ impl StaticTileFetcher {
 
 #[cfg(test)]
 mod tests {
+    use super::StaticTileFetcher;
+    use crate::coords::WorldTileCoords;
+    use crate::style::source::TileAddressingScheme;
 
-    #[cfg(all(static_tiles, not(target_arch = "wasm32")))]
+    #[cfg(static_tiles_found)]
     #[tokio::test]
     async fn test_tiles_available() {
         const MUNICH_X: i32 = 17425;

maplibre/src/io/tile_pipelines.rs

@@ -7,6 +7,7 @@ use geozero::GeozeroDatasource;
 use prost::Message;
 use std::collections::HashSet;
 
+#[derive(Default)]
 pub struct ParseTile;
 
 impl Processable for ParseTile {
@@ -24,6 +25,7 @@ impl Processable for ParseTile {
     }
 }
 
+#[derive(Default)]
 pub struct IndexLayer;
 
 impl Processable for IndexLayer {
@@ -45,6 +47,7 @@ impl Processable for IndexLayer {
     }
 }
 
+#[derive(Default)]
 pub struct TessellateLayer;
 
 impl Processable for TessellateLayer {