From 63da707210f32c36880c1a0c683251198a725bb7 Mon Sep 17 00:00:00 2001 From: Max Ammann Date: Sat, 18 Feb 2023 12:56:56 +0100 Subject: [PATCH] Introduce Plugin API along with Raster Tiles (#253) * Implement basic raster rendering * Fix web builds for raster rendering (e.g. APC) * Implement plugin system using a "Tile Component System" pattern heavily inspired by bevy * Remove tile_repository and integrate into world structures * Migrate from specific stages to systems * Introduce a RasterPlugin and VectorPlugin * Add feature gate for headless rendering to demo * Remove pipelines for simplicity * Add message tags using Labels * Add run config * Add show bash path for Windows CI * Do not use symlinks for cargo binstall * Introduce DebugPlugin * Introduce HeadlessPlugin * Add more layers to vector processing benchmark --------- Co-authored-by: Quillasp Co-authored-by: Antoine Drabble --- .github/actions/cargo-install/action.yml | 12 +- .github/workflows/demo-windows.yml | 3 + .gitignore | 3 +- .idea/runConfigurations/Build_maplibre.xml | 19 + ..._headless_demo__debug_enable_tracing__.xml | 2 +- benchmarks/benches/data.rs | 90 ++--- benchmarks/benches/render.rs | 23 +- .../ecs-design.drawio.svg | 4 + maplibre-build-tools/src/wgsl.rs | 2 +- maplibre-demo/Cargo.toml | 3 +- maplibre-demo/src/headless.rs | 22 +- maplibre-demo/src/main.rs | 17 +- maplibre-winit/src/input/pan_handler.rs | 11 +- maplibre-winit/src/input/query_handler.rs | 16 +- maplibre-winit/src/input/shift_handler.rs | 11 +- maplibre-winit/src/input/tilt_handler.rs | 11 +- maplibre-winit/src/input/zoom_handler.rs | 11 +- maplibre-winit/src/lib.rs | 21 +- maplibre-winit/src/noweb.rs | 33 +- maplibre/Cargo.toml | 5 +- maplibre/src/context.rs | 10 +- maplibre/src/coords.rs | 2 +- maplibre/src/debug/cleanup_system.rs | 11 + maplibre/src/{render => debug}/debug_pass.rs | 36 +- maplibre/src/debug/mod.rs | 93 +++++ maplibre/src/debug/queue_system.rs | 34 ++ maplibre/src/debug/render_commands.rs | 58 +++ maplibre/src/debug/resource_system.rs | 56 +++ maplibre/src/environment.rs | 17 +- maplibre/src/headless/environment.rs | 9 +- maplibre/src/headless/graph_node.rs | 24 +- maplibre/src/headless/map.rs | 182 +++++----- maplibre/src/headless/mod.rs | 70 +++- maplibre/src/headless/{stage.rs => system.rs} | 21 +- maplibre/src/io/apc.rs | 201 +++++++---- maplibre/src/io/geometry_index.rs | 20 +- maplibre/src/io/mod.rs | 36 +- maplibre/src/io/pipeline.rs | 288 --------------- maplibre/src/io/source_client.rs | 29 +- maplibre/src/io/source_type.rs | 92 +++++ maplibre/src/io/static_tile_fetcher.rs | 1 + maplibre/src/io/tile_pipelines.rs | 177 ---------- maplibre/src/io/tile_repository.rs | 228 ------------ maplibre/src/io/transferables.rs | 172 --------- maplibre/src/lib.rs | 17 +- maplibre/src/map.rs | 75 ++-- maplibre/src/platform/mod.rs | 8 +- maplibre/src/platform/noweb/http_client.rs | 1 + maplibre/src/platform/noweb/mod.rs | 20 ++ maplibre/src/plugin/mod.rs | 16 + maplibre/src/raster/mod.rs | 96 +++++ maplibre/src/raster/populate_world_system.rs | 61 ++++ maplibre/src/raster/process_raster.rs | 86 +++++ maplibre/src/raster/queue_system.rs | 58 +++ maplibre/src/raster/render_commands.rs | 93 +++++ maplibre/src/raster/request_system.rs | 158 +++++++++ maplibre/src/raster/resource/mod.rs | 3 + maplibre/src/raster/resource/raster.rs | 91 +++++ maplibre/src/raster/resource_system.rs | 56 +++ maplibre/src/raster/transferables.rs | 135 +++++++ maplibre/src/raster/upload_system.rs | 105 ++++++ maplibre/src/render/eventually.rs | 21 ++ maplibre/src/render/graph/graph.rs | 36 +- maplibre/src/render/graph/mod.rs | 12 +- maplibre/src/render/graph/node.rs | 12 +- maplibre/src/render/graph_runner.rs | 26 +- maplibre/src/render/main_pass.rs | 42 ++- maplibre/src/render/mod.rs | 271 +++++++++------ maplibre/src/render/render_commands.rs | 172 ++------- maplibre/src/render/render_phase/draw.rs | 39 ++- maplibre/src/render/render_phase/mod.rs | 65 +++- maplibre/src/render/resource/buffer.rs | 12 + maplibre/src/render/resource/globals.rs | 37 -- maplibre/src/render/resource/mod.rs | 20 +- maplibre/src/render/resource/surface.rs | 2 +- maplibre/src/render/resource/texture.rs | 22 +- .../render/{ => resource}/tile_pipeline.rs | 58 +-- maplibre/src/render/shaders/mod.rs | 103 +++++- .../src/render/shaders/tile_debug.vertex.wgsl | 4 +- .../src/render/shaders/tile_mask.vertex.wgsl | 4 +- .../render/shaders/tile_raster.fragment.wgsl | 14 + .../render/shaders/tile_raster.vertex.wgsl | 48 +++ maplibre/src/render/stages/extract_stage.rs | 43 --- maplibre/src/render/stages/mod.rs | 67 ---- .../src/render/stages/phase_sort_stage.rs | 25 -- maplibre/src/render/stages/queue_stage.rs | 64 ---- maplibre/src/render/stages/upload_stage.rs | 222 ------------ maplibre/src/render/systems/cleanup_system.rs | 16 + .../graph_runner_system.rs} | 30 +- maplibre/src/render/systems/mod.rs | 8 + .../resource_system.rs} | 98 ++---- .../src/render/systems/sort_phase_system.rs | 14 + .../systems/tile_view_pattern_system.rs | 38 ++ maplibre/src/render/systems/upload_system.rs | 28 ++ maplibre/src/render/tile_view_pattern.rs | 238 ------------- maplibre/src/render/tile_view_pattern/mod.rs | 230 ++++++++++++ .../src/render/tile_view_pattern/pattern.rs | 154 ++++++++ maplibre/src/schedule.rs | 47 ++- maplibre/src/stages/mod.rs | 95 ----- .../src/stages/populate_tile_store_stage.rs | 97 ------ maplibre/src/stages/request_stage.rs | 165 --------- maplibre/src/style/layer.rs | 8 +- maplibre/src/style/mod.rs | 9 +- maplibre/src/style/raster.rs | 54 +++ maplibre/src/style/style.rs | 45 +-- maplibre/src/tcs/mod.rs | 30 ++ maplibre/src/tcs/resources.rs | 237 +++++++++++++ maplibre/src/tcs/system/function.rs | 41 +++ maplibre/src/tcs/system/mod.rs | 50 +++ maplibre/src/tcs/system/stage.rs | 34 ++ maplibre/src/tcs/tiles.rs | 329 ++++++++++++++++++ maplibre/src/tcs/world.rs | 11 + maplibre/src/util/mod.rs | 16 +- maplibre/src/vector/mod.rs | 139 ++++++++ maplibre/src/vector/populate_world_system.rs | 78 +++++ maplibre/src/vector/process_vector.rs | 207 +++++++++++ maplibre/src/vector/queue_system.rs | 57 +++ maplibre/src/vector/render_commands.rs | 102 ++++++ maplibre/src/vector/request_system.rs | 168 +++++++++ .../resource/buffer_pool.rs | 309 ++++++++-------- maplibre/src/vector/resource/mod.rs | 3 + maplibre/src/vector/resource_system.rs | 59 ++++ maplibre/src/vector/transferables.rs | 265 ++++++++++++++ maplibre/src/vector/upload_system.rs | 186 ++++++++++ maplibre/src/{world.rs => view_state.rs} | 58 +-- web/Cargo.toml | 2 + web/demo/package-lock.json | 1 + web/flatbuffer/basic.fbs | 1 - web/flatbuffer/layer_missing.fbs | 8 + web/flatbuffer/layer_raster.fbs | 12 + web/flatbuffer/layer_unavailable.fbs | 10 - web/lib/build.mjs | 1 + web/lib/src/index.ts | 6 +- web/src/error.rs | 12 +- web/src/lib.rs | 44 ++- web/src/platform/http_client.rs | 7 +- web/src/platform/mod.rs | 16 + web/src/platform/singlethreaded/apc.rs | 169 ++++++--- .../platform/singlethreaded/transferables.rs | 227 +++++++++--- .../platform/singlethreaded/wasm_entries.rs | 53 ++- 140 files changed, 5818 insertions(+), 3210 deletions(-) create mode 100644 .idea/runConfigurations/Build_maplibre.xml create mode 100644 docs/src/development-documents/ecs-design.drawio.svg create mode 100644 maplibre/src/debug/cleanup_system.rs rename maplibre/src/{render => debug}/debug_pass.rs (57%) create mode 100644 maplibre/src/debug/mod.rs create mode 100644 maplibre/src/debug/queue_system.rs create mode 100644 maplibre/src/debug/render_commands.rs create mode 100644 maplibre/src/debug/resource_system.rs rename maplibre/src/headless/{stage.rs => system.rs} (78%) delete mode 100644 maplibre/src/io/pipeline.rs create mode 100644 maplibre/src/io/source_type.rs delete mode 100644 maplibre/src/io/tile_pipelines.rs delete mode 100644 maplibre/src/io/tile_repository.rs delete mode 100644 maplibre/src/io/transferables.rs create mode 100644 maplibre/src/plugin/mod.rs create mode 100644 maplibre/src/raster/mod.rs create mode 100644 maplibre/src/raster/populate_world_system.rs create mode 100644 maplibre/src/raster/process_raster.rs create mode 100644 maplibre/src/raster/queue_system.rs create mode 100644 maplibre/src/raster/render_commands.rs create mode 100644 maplibre/src/raster/request_system.rs create mode 100644 maplibre/src/raster/resource/mod.rs create mode 100644 maplibre/src/raster/resource/raster.rs create mode 100644 maplibre/src/raster/resource_system.rs create mode 100644 maplibre/src/raster/transferables.rs create mode 100644 maplibre/src/raster/upload_system.rs create mode 100644 maplibre/src/render/resource/buffer.rs delete mode 100644 maplibre/src/render/resource/globals.rs rename maplibre/src/render/{ => resource}/tile_pipeline.rs (76%) create mode 100644 maplibre/src/render/shaders/tile_raster.fragment.wgsl create mode 100644 maplibre/src/render/shaders/tile_raster.vertex.wgsl delete mode 100644 maplibre/src/render/stages/extract_stage.rs delete mode 100644 maplibre/src/render/stages/mod.rs delete mode 100644 maplibre/src/render/stages/phase_sort_stage.rs delete mode 100644 maplibre/src/render/stages/queue_stage.rs delete mode 100644 maplibre/src/render/stages/upload_stage.rs create mode 100644 maplibre/src/render/systems/cleanup_system.rs rename maplibre/src/render/{stages/graph_runner_stage.rs => systems/graph_runner_system.rs} (75%) create mode 100644 maplibre/src/render/systems/mod.rs rename maplibre/src/render/{stages/resource_stage.rs => systems/resource_system.rs} (57%) create mode 100644 maplibre/src/render/systems/sort_phase_system.rs create mode 100644 maplibre/src/render/systems/tile_view_pattern_system.rs create mode 100644 maplibre/src/render/systems/upload_system.rs delete mode 100644 maplibre/src/render/tile_view_pattern.rs create mode 100644 maplibre/src/render/tile_view_pattern/mod.rs create mode 100644 maplibre/src/render/tile_view_pattern/pattern.rs delete mode 100644 maplibre/src/stages/mod.rs delete mode 100644 maplibre/src/stages/populate_tile_store_stage.rs delete mode 100644 maplibre/src/stages/request_stage.rs create mode 100644 maplibre/src/style/raster.rs create mode 100644 maplibre/src/tcs/mod.rs create mode 100644 maplibre/src/tcs/resources.rs create mode 100644 maplibre/src/tcs/system/function.rs create mode 100644 maplibre/src/tcs/system/mod.rs create mode 100644 maplibre/src/tcs/system/stage.rs create mode 100644 maplibre/src/tcs/tiles.rs create mode 100644 maplibre/src/tcs/world.rs create mode 100644 maplibre/src/vector/mod.rs create mode 100644 maplibre/src/vector/populate_world_system.rs create mode 100644 maplibre/src/vector/process_vector.rs create mode 100644 maplibre/src/vector/queue_system.rs create mode 100644 maplibre/src/vector/render_commands.rs create mode 100644 maplibre/src/vector/request_system.rs rename maplibre/src/{render => vector}/resource/buffer_pool.rs (86%) create mode 100644 maplibre/src/vector/resource/mod.rs create mode 100644 maplibre/src/vector/resource_system.rs create mode 100644 maplibre/src/vector/transferables.rs create mode 100644 maplibre/src/vector/upload_system.rs rename maplibre/src/{world.rs => view_state.rs} (68%) create mode 100644 web/flatbuffer/layer_missing.fbs create mode 100644 web/flatbuffer/layer_raster.fbs delete mode 100644 web/flatbuffer/layer_unavailable.fbs diff --git a/.github/actions/cargo-install/action.yml b/.github/actions/cargo-install/action.yml index f1bb9d05..ea269ac6 100644 --- a/.github/actions/cargo-install/action.yml +++ b/.github/actions/cargo-install/action.yml @@ -12,7 +12,7 @@ runs: - name: Install binstall run: | if command -v cargo-binstall &> /dev/null; then - echo "binstall already found" + echo "binstall already found at $(command -v cargo-binstall)" exit 0 fi @@ -61,9 +61,13 @@ runs: echo "Skipping binary install" fi - # Upgrade - cargo binstall --no-confirm --force cargo-binstall + # Upgrade binstall to latest version + # Do not use symlinks because they behave weird on Windows(+Bash) + cargo binstall --no-confirm --force --no-symlinks cargo-binstall + # Use latest version to reinstall binstall + cargo binstall --no-confirm --force --no-symlinks cargo-binstall shell: bash - name: Install ${{ inputs.name }} shell: bash - run: cargo binstall --no-confirm --force ${{ inputs.name }} + # Do not use symlinks because they behave weird on Windows(+Bash) + run: cargo binstall --no-confirm --force --no-symlinks ${{ inputs.name }} diff --git a/.github/workflows/demo-windows.yml b/.github/workflows/demo-windows.yml index ecddb5d1..826ea9fa 100644 --- a/.github/workflows/demo-windows.yml +++ b/.github/workflows/demo-windows.yml @@ -19,6 +19,9 @@ jobs: - name: Show PATH shell: powershell run: Write-Host $env:path + - name: Show bash PATH + shell: bash + run: echo $PATH - uses: actions/checkout@v3 - name: Setup uses: ./.github/actions/setup diff --git a/.gitignore b/.gitignore index cfac52ae..3e092010 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ # Generated by Cargo # will have compiled files and executables -debug/ target/ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries @@ -28,4 +27,4 @@ perf.data* *flamegraph.svg *flamechart.svg -frame_*.png \ No newline at end of file +frame_*.png diff --git a/.idea/runConfigurations/Build_maplibre.xml b/.idea/runConfigurations/Build_maplibre.xml new file mode 100644 index 00000000..ff04b3d1 --- /dev/null +++ b/.idea/runConfigurations/Build_maplibre.xml @@ -0,0 +1,19 @@ + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/Run_headless_demo__debug_enable_tracing__.xml b/.idea/runConfigurations/Run_headless_demo__debug_enable_tracing__.xml index a5645315..482a9f1b 100644 --- a/.idea/runConfigurations/Run_headless_demo__debug_enable_tracing__.xml +++ b/.idea/runConfigurations/Run_headless_demo__debug_enable_tracing__.xml @@ -1,6 +1,6 @@ -