From 0a416189b6ec0dec6a088085dca35c7bb7c0f91d Mon Sep 17 00:00:00 2001 From: teoxoy <28601907+teoxoy@users.noreply.github.com> Date: Fri, 11 Apr 2025 17:15:45 +0200 Subject: [PATCH] [test] add OOM tests --- .config/nextest.toml | 5 + tests/tests/wgpu-gpu/main.rs | 1 + tests/tests/wgpu-gpu/oom.rs | 230 +++++++++++++++++++++++++++++++++++ 3 files changed, 236 insertions(+) create mode 100644 tests/tests/wgpu-gpu/oom.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 5295fbb5d..64b3b84cb 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,6 +1,7 @@ # None of our tests should take longer than 45s, and if they've gone 2x that, # terminate them to prevent infinite run-on. [profile.default] +default-filter = "!test(~oom_test)" slow-timeout = { period = "45s", terminate-after = 2 } fail-fast = false retries = 0 @@ -33,3 +34,7 @@ retries = 1 [[profile.default.overrides]] filter = 'test(compile_fail)' slow-timeout = { period = "3m", terminate-after = 2 } + +[[profile.default.overrides]] +filter = 'test(~oom_test)' +threads-required = "num-test-threads" diff --git a/tests/tests/wgpu-gpu/main.rs b/tests/tests/wgpu-gpu/main.rs index d7014fa63..31246697c 100644 --- a/tests/tests/wgpu-gpu/main.rs +++ b/tests/tests/wgpu-gpu/main.rs @@ -36,6 +36,7 @@ mod mem_leaks; mod nv12_texture; mod occlusion_query; mod oob_indexing; +mod oom; mod pipeline; mod pipeline_cache; mod poll; diff --git a/tests/tests/wgpu-gpu/oom.rs b/tests/tests/wgpu-gpu/oom.rs new file mode 100644 index 000000000..7ed6c7459 --- /dev/null +++ b/tests/tests/wgpu-gpu/oom.rs @@ -0,0 +1,230 @@ +use wgpu::{ + AccelerationStructureFlags, AccelerationStructureGeometryFlags, + AccelerationStructureUpdateMode, Backends, BlasGeometrySizeDescriptors, + BlasTriangleGeometrySizeDescriptor, BufferDescriptor, BufferUsages, CreateBlasDescriptor, + CreateTlasDescriptor, Error, ErrorFilter, Extent3d, Features, QuerySetDescriptor, QueryType, + TextureDescriptor, TextureDimension, TextureFormat, TextureUsages, VertexFormat, +}; +use wgpu_test::{gpu_test, FailureCase, GpuTestConfiguration, TestParameters}; + +// Tests in this file must all end with "OOM_TEST" so that nextest doesn't run any other tests while it runs one of the OOM tests. +// This is done so that other tests that create resources will not fail with OOM errors due to the OOM tests running in parallel. + +/// Backends for which OOM detection is implemented +const OOM_DETECTION_IMPL: Backends = Backends::DX12.union(Backends::VULKAN); + +// All tests skip llvmpipe. +// Even though llvmpipe supports VK_EXT_memory_budget it's happy to continue creating resources until +// the process crashes with SIGABRT "memory allocation of X bytes failed" or the test times out. + +/// Nr of resources tests will try to create before failing. +const LOOP_BOUND: u32 = 1_000_000; + +#[gpu_test] +static TEXTURE_OOM_TEST: GpuTestConfiguration = GpuTestConfiguration::new() + .parameters( + TestParameters::default() + .skip(FailureCase::backend(!OOM_DETECTION_IMPL)) + // see comment at the top of the file + .skip(FailureCase::backend_adapter(Backends::VULKAN, "llvmpipe")), + ) + .run_async(|ctx| async move { + let mut textures = Vec::new(); + for _ in 0..LOOP_BOUND { + ctx.device.push_error_scope(ErrorFilter::OutOfMemory); + let texture = ctx.device.create_texture(&TextureDescriptor { + label: None, + size: Extent3d { + width: 2048, + height: 2048, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: TextureDimension::D2, + format: TextureFormat::Rgba16Float, + usage: TextureUsages::RENDER_ATTACHMENT, + view_formats: &[], + }); + if let Some(err) = ctx.device.pop_error_scope().await { + match err { + Error::OutOfMemory { .. } => { + return; + } + _ => unreachable!(), + } + } + textures.push(texture); + } + panic!("Failed to OOM after {LOOP_BOUND} iterations."); + }); + +#[gpu_test] +static BUFFER_OOM_TEST: GpuTestConfiguration = GpuTestConfiguration::new() + .parameters( + TestParameters::default() + .skip(FailureCase::backend(!OOM_DETECTION_IMPL)) + // see comment at the top of the file + .skip(FailureCase::backend_adapter(Backends::VULKAN, "llvmpipe")), + ) + .run_async(|ctx| async move { + let mut buffers = Vec::new(); + for _ in 0..LOOP_BOUND { + ctx.device.push_error_scope(ErrorFilter::OutOfMemory); + let buffer = ctx.device.create_buffer(&BufferDescriptor { + label: None, + size: 256 * 1024 * 1024, + usage: BufferUsages::STORAGE, + mapped_at_creation: false, + }); + if let Some(err) = ctx.device.pop_error_scope().await { + match err { + Error::OutOfMemory { .. } => { + return; + } + _ => unreachable!(), + } + } + buffers.push(buffer); + } + panic!("Failed to OOM after {LOOP_BOUND} iterations."); + }); + +#[gpu_test] +static MAPPING_BUFFER_OOM_TEST: GpuTestConfiguration = GpuTestConfiguration::new() + .parameters( + TestParameters::default() + .skip(FailureCase::backend(!OOM_DETECTION_IMPL)) + // see comment at the top of the file + .skip(FailureCase::backend_adapter(Backends::VULKAN, "llvmpipe")), + ) + .run_async(|ctx| async move { + let mut buffers = Vec::new(); + for _ in 0..LOOP_BOUND { + ctx.device.push_error_scope(ErrorFilter::OutOfMemory); + let buffer = ctx.device.create_buffer(&BufferDescriptor { + label: None, + size: 256 * 1024 * 1024, + usage: BufferUsages::COPY_SRC | BufferUsages::MAP_WRITE, + mapped_at_creation: false, + }); + if let Some(err) = ctx.device.pop_error_scope().await { + match err { + Error::OutOfMemory { .. } => { + return; + } + _ => unreachable!(), + } + } + buffers.push(buffer); + } + panic!("Failed to OOM after {LOOP_BOUND} iterations."); + }); + +#[gpu_test] +static QUERY_SET_OOM_TEST: GpuTestConfiguration = GpuTestConfiguration::new() + .parameters( + TestParameters::default() + .skip(FailureCase::backend(!OOM_DETECTION_IMPL)) + // see comment at the top of the file + .skip(FailureCase::backend_adapter(Backends::VULKAN, "llvmpipe")), + ) + .run_async(|ctx| async move { + let mut query_sets = Vec::new(); + for _ in 0..LOOP_BOUND { + ctx.device.push_error_scope(ErrorFilter::OutOfMemory); + let query_set = ctx.device.create_query_set(&QuerySetDescriptor { + label: None, + ty: QueryType::Occlusion, + count: 4096, + }); + if let Some(err) = ctx.device.pop_error_scope().await { + match err { + Error::OutOfMemory { .. } => { + return; + } + _ => unreachable!(), + } + } + query_sets.push(query_set); + } + panic!("Failed to OOM after {LOOP_BOUND} iterations."); + }); + +#[gpu_test] +static BLAS_OOM_TEST: GpuTestConfiguration = GpuTestConfiguration::new() + .parameters( + TestParameters::default() + .features(Features::EXPERIMENTAL_RAY_TRACING_ACCELERATION_STRUCTURE) + .skip(FailureCase::backend(!OOM_DETECTION_IMPL)) + // https://github.com/gfx-rs/wgpu/issues/6727 + .skip(FailureCase::backend_adapter(Backends::VULKAN, "AMD")) + // see comment at the top of the file + .skip(FailureCase::backend_adapter(Backends::VULKAN, "llvmpipe")), + ) + .run_async(|ctx| async move { + let mut blases = Vec::new(); + for _ in 0..LOOP_BOUND { + ctx.device.push_error_scope(ErrorFilter::OutOfMemory); + let blas = ctx.device.create_blas( + &CreateBlasDescriptor { + label: None, + flags: AccelerationStructureFlags::PREFER_FAST_TRACE, + update_mode: AccelerationStructureUpdateMode::Build, + }, + BlasGeometrySizeDescriptors::Triangles { + descriptors: vec![BlasTriangleGeometrySizeDescriptor { + vertex_format: VertexFormat::Float32x3, + vertex_count: 1024 * 1024, + index_format: None, + index_count: None, + flags: AccelerationStructureGeometryFlags::OPAQUE, + }], + }, + ); + if let Some(err) = ctx.device.pop_error_scope().await { + match err { + Error::OutOfMemory { .. } => { + return; + } + _ => unreachable!(), + } + } + blases.push(blas); + } + panic!("Failed to OOM after {LOOP_BOUND} iterations."); + }); + +#[gpu_test] +static TLAS_OOM_TEST: GpuTestConfiguration = GpuTestConfiguration::new() + .parameters( + TestParameters::default() + .features(Features::EXPERIMENTAL_RAY_TRACING_ACCELERATION_STRUCTURE) + .skip(FailureCase::backend(!OOM_DETECTION_IMPL)) + // https://github.com/gfx-rs/wgpu/issues/6727 + .skip(FailureCase::backend_adapter(Backends::VULKAN, "AMD")) + // see comment at the top of the file + .skip(FailureCase::backend_adapter(Backends::VULKAN, "llvmpipe")), + ) + .run_async(|ctx| async move { + let mut tlases = Vec::new(); + for _ in 0..LOOP_BOUND { + ctx.device.push_error_scope(ErrorFilter::OutOfMemory); + let tlas = ctx.device.create_tlas(&CreateTlasDescriptor { + label: None, + max_instances: 1024 * 1024, + flags: AccelerationStructureFlags::PREFER_FAST_TRACE, + update_mode: AccelerationStructureUpdateMode::Build, + }); + if let Some(err) = ctx.device.pop_error_scope().await { + match err { + Error::OutOfMemory { .. } => { + return; + } + _ => unreachable!(), + } + } + tlases.push(tlas); + } + panic!("Failed to OOM after {LOOP_BOUND} iterations."); + });