command buffer buffer memory init tracking is only filled if necessary now

This commit is contained in:
Andreas Reich 2021-01-25 22:16:33 +01:00
parent 22fb9ab1eb
commit 492027fe6e
10 changed files with 198 additions and 122 deletions

View File

@ -9,7 +9,7 @@ use crate::{
},
hub::Resource,
id::{BindGroupLayoutId, BufferId, DeviceId, SamplerId, TextureViewId, Valid},
memory_init_tracker::ResourceMemoryInitTrackerAction,
memory_init_tracker::MemoryInitTrackerAction,
track::{TrackerSet, DUMMY_SELECTOR},
validation::{MissingBufferUsageError, MissingTextureUsageError},
FastHashMap, Label, LifeGuard, MultiRefCount, Stored, MAX_BIND_GROUPS,
@ -589,7 +589,7 @@ pub struct BindGroup<B: hal::Backend> {
pub(crate) layout_id: Valid<BindGroupLayoutId>,
pub(crate) life_guard: LifeGuard,
pub(crate) used: TrackerSet,
pub(crate) used_buffer_ranges: Vec<ResourceMemoryInitTrackerAction<BufferId>>,
pub(crate) used_buffer_ranges: Vec<MemoryInitTrackerAction<BufferId>>,
pub(crate) dynamic_binding_info: Vec<BindGroupDynamicBindingData>,
}

View File

@ -123,7 +123,7 @@ impl<B: GfxBackend> CommandAllocator<B> {
device_id,
trackers: TrackerSet::new(B::VARIANT),
used_swap_chains: Default::default(),
used_buffer_ranges: Default::default(),
buffer_memory_init_actions: Default::default(),
limits,
private_features,
has_labels: label.is_some(),

View File

@ -49,7 +49,7 @@ use crate::{
},
hub::{GfxBackend, GlobalIdentityHandlerFactory, Hub, Resource, Storage, Token},
id,
memory_init_tracker::{MemoryInitKind, ResourceMemoryInitTrackerAction},
memory_init_tracker::{MemoryInitKind, MemoryInitTrackerAction},
resource::BufferUse,
span,
track::{TrackerSet, UsageConflict},
@ -160,7 +160,7 @@ impl RenderBundleEncoder {
let mut commands = Vec::new();
let mut base = self.base.as_ref();
let mut pipeline_layout_id = None::<id::Valid<id::PipelineLayoutId>>;
let mut used_buffer_ranges = Vec::new();
let mut buffer_memory_init_actions = Vec::new();
for &command in base.commands {
match command {
@ -206,7 +206,7 @@ impl RenderBundleEncoder {
.map_pass_err(scope);
}
used_buffer_ranges
buffer_memory_init_actions
.extend(bind_group.used_buffer_ranges.iter().map(|x| x.clone()));
state.set_bind_group(index, bind_group_id, bind_group.layout_id, offsets);
@ -267,7 +267,7 @@ impl RenderBundleEncoder {
Some(s) => offset + s.get(),
None => buffer.size,
};
used_buffer_ranges.push(ResourceMemoryInitTrackerAction {
buffer_memory_init_actions.push(MemoryInitTrackerAction {
id: buffer_id,
range: offset..end,
kind: MemoryInitKind::NeedsInitializedMemory,
@ -294,7 +294,7 @@ impl RenderBundleEncoder {
Some(s) => offset + s.get(),
None => buffer.size,
};
used_buffer_ranges.push(ResourceMemoryInitTrackerAction {
buffer_memory_init_actions.push(MemoryInitTrackerAction {
id: buffer_id,
range: offset..end,
kind: MemoryInitKind::NeedsInitializedMemory,
@ -411,7 +411,7 @@ impl RenderBundleEncoder {
check_buffer_usage(buffer.usage, wgt::BufferUsage::INDIRECT)
.map_pass_err(scope)?;
used_buffer_ranges.push(ResourceMemoryInitTrackerAction {
buffer_memory_init_actions.push(MemoryInitTrackerAction {
id: buffer_id,
range: 0..buffer.size,
kind: MemoryInitKind::NeedsInitializedMemory,
@ -442,7 +442,7 @@ impl RenderBundleEncoder {
.map_pass_err(scope)?;
let stride = 4 * 4; // 4 integers, vertexCount, instanceCount, firstVertex, firstInstance
used_buffer_ranges.push(ResourceMemoryInitTrackerAction {
buffer_memory_init_actions.push(MemoryInitTrackerAction {
id: buffer_id,
range: 0..stride,
kind: MemoryInitKind::NeedsInitializedMemory,
@ -482,7 +482,7 @@ impl RenderBundleEncoder {
ref_count: device.life_guard.add_ref(),
},
used: state.trackers,
used_buffer_ranges,
buffer_memory_init_actions,
context: self.context,
life_guard: LifeGuard::new(desc.label.borrow_or_default()),
})
@ -531,7 +531,7 @@ pub struct RenderBundle {
base: BasePass<RenderCommand>,
pub(crate) device_id: Stored<id::DeviceId>,
pub(crate) used: TrackerSet,
pub(crate) used_buffer_ranges: Vec<ResourceMemoryInitTrackerAction<id::BufferId>>,
pub(crate) buffer_memory_init_actions: Vec<MemoryInitTrackerAction<id::BufferId>>,
pub(crate) context: RenderPassContext,
pub(crate) life_guard: LifeGuard,
}

View File

@ -11,7 +11,7 @@ use crate::{
},
hub::{GfxBackend, Global, GlobalIdentityHandlerFactory, Storage, Token},
id,
memory_init_tracker::{MemoryInitKind, ResourceMemoryInitTrackerAction},
memory_init_tracker::{MemoryInitKind, MemoryInitTrackerAction},
resource::{Buffer, BufferUse, Texture},
span,
track::{TrackerSet, UsageConflict},
@ -147,6 +147,8 @@ pub enum ComputePassErrorInner {
end_offset: u64,
buffer_size: u64,
},
#[error("buffer {0:?} is invalid or destroyed")]
InvalidBuffer(id::BufferId),
#[error(transparent)]
ResourceUsageConflict(#[from] UsageConflict),
#[error(transparent)]
@ -330,9 +332,18 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
.validate_dynamic_bindings(&temp_offsets)
.map_pass_err(scope)?;
cmd_buf
.used_buffer_ranges
.extend(bind_group.used_buffer_ranges.iter().map(|x| x.clone()));
cmd_buf.buffer_memory_init_actions.extend(
bind_group
.used_buffer_ranges
.iter()
.filter(|action| match buffer_guard.get(action.id) {
Ok(buffer) => {
!buffer.initialization_status.is_initialized(&action.range)
}
Err(_) => false,
})
.map(|action| action.clone()),
);
if let Some((pipeline_layout_id, follow_ups)) = state.binder.provide_entry(
index as usize,
@ -519,13 +530,20 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
.map_pass_err(scope)?;
let stride = 3 * 4; // 3 integers, x/y/z group size
cmd_buf
.used_buffer_ranges
.push(ResourceMemoryInitTrackerAction {
id: buffer_id,
range: offset..(offset + stride),
kind: MemoryInitKind::NeedsInitializedMemory,
});
let used_buffer_range = offset..(offset + stride);
if !indirect_buffer
.initialization_status
.is_initialized(&used_buffer_range)
{
cmd_buf
.buffer_memory_init_actions
.push(MemoryInitTrackerAction {
id: buffer_id,
range: used_buffer_range,
kind: MemoryInitKind::NeedsInitializedMemory,
});
}
state
.flush_states(

View File

@ -24,7 +24,7 @@ use crate::{
device::{all_buffer_stages, all_image_stages},
hub::{GfxBackend, Global, GlobalIdentityHandlerFactory, Storage, Token},
id,
memory_init_tracker::ResourceMemoryInitTrackerAction,
memory_init_tracker::MemoryInitTrackerAction,
resource::{Buffer, Texture},
span,
track::TrackerSet,
@ -47,7 +47,7 @@ pub struct CommandBuffer<B: hal::Backend> {
pub(crate) device_id: Stored<id::DeviceId>,
pub(crate) trackers: TrackerSet,
pub(crate) used_swap_chains: SmallVec<[Stored<id::SwapChainId>; 1]>,
pub(crate) used_buffer_ranges: Vec<ResourceMemoryInitTrackerAction<id::BufferId>>,
pub(crate) buffer_memory_init_actions: Vec<MemoryInitTrackerAction<id::BufferId>>,
limits: wgt::Limits,
private_features: PrivateFeatures,
has_labels: bool,
@ -58,17 +58,6 @@ pub struct CommandBuffer<B: hal::Backend> {
}
impl<B: GfxBackend> CommandBuffer<B> {
fn get_encoder(
storage: &Storage<Self, id::CommandEncoderId>,
id: id::CommandEncoderId,
) -> Result<&Self, CommandEncoderError> {
match storage.get(id) {
Ok(cmd_buf) if cmd_buf.is_recording => Ok(cmd_buf),
Ok(_) => Err(CommandEncoderError::NotRecording),
Err(_) => Err(CommandEncoderError::Invalid),
}
}
fn get_encoder_mut(
storage: &mut Storage<Self, id::CommandEncoderId>,
id: id::CommandEncoderId,

View File

@ -17,7 +17,7 @@ use crate::{
},
hub::{GfxBackend, Global, GlobalIdentityHandlerFactory, Storage, Token},
id,
memory_init_tracker::{MemoryInitKind, ResourceMemoryInitTrackerAction},
memory_init_tracker::{MemoryInitKind, MemoryInitTrackerAction},
pipeline::PipelineFlags,
resource::{BufferUse, Texture, TextureUse, TextureView, TextureViewInner},
span,
@ -1025,12 +1025,12 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
let (device_guard, mut token) = hub.devices.read(&mut token);
let (cmd_buf_raw, trackers, used_swapchain, used_buffer_ranges, query_reset_state) = {
let (cmd_buf_raw, trackers, used_swapchain, query_reset_state) = {
// read-only lock guard
let (cmb_guard, mut token) = hub.command_buffers.read(&mut token);
let (mut cmb_guard, mut token) = hub.command_buffers.write(&mut token);
let cmd_buf =
CommandBuffer::get_encoder(&*cmb_guard, encoder_id).map_pass_err(scope)?;
CommandBuffer::get_encoder_mut(&mut *cmb_guard, encoder_id).map_pass_err(scope)?;
let device = &device_guard[cmd_buf.device_id.value];
let mut raw = device.cmd_allocator.extend(cmd_buf);
unsafe {
@ -1080,7 +1080,6 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
let mut string_offset = 0;
let mut active_query = None;
let mut query_reset_state = QueryResetMap::new();
let mut used_buffer_ranges = Vec::new();
for command in base.commands {
match *command {
@ -1119,8 +1118,18 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
.merge_extend(&bind_group.used)
.map_pass_err(scope)?;
used_buffer_ranges
.extend(bind_group.used_buffer_ranges.iter().map(|x| x.clone()));
cmd_buf.buffer_memory_init_actions.extend(
bind_group
.used_buffer_ranges
.iter()
.filter(|action| match buffer_guard.get(action.id) {
Ok(buffer) => {
!buffer.initialization_status.is_initialized(&action.range)
}
Err(_) => false,
})
.map(|action| action.clone()),
);
if let Some((pipeline_layout_id, follow_ups)) = state.binder.provide_entry(
index as usize,
@ -1298,11 +1307,15 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
state.index.format = Some(index_format);
state.index.update_limit();
used_buffer_ranges.push(ResourceMemoryInitTrackerAction {
id: buffer_id,
range: offset..end,
kind: MemoryInitKind::NeedsInitializedMemory,
});
if !buffer.initialization_status.is_initialized(&(offset..end)) {
cmd_buf
.buffer_memory_init_actions
.push(MemoryInitTrackerAction {
id: buffer_id,
range: offset..end,
kind: MemoryInitKind::NeedsInitializedMemory,
});
}
let range = hal::buffer::SubRange {
offset,
@ -1347,11 +1360,16 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
};
vertex_state.bound = true;
used_buffer_ranges.push(ResourceMemoryInitTrackerAction {
id: buffer_id,
range: offset..(offset + vertex_state.total_size),
kind: MemoryInitKind::NeedsInitializedMemory,
});
let used_range = offset..(offset + vertex_state.total_size);
if !buffer.initialization_status.is_initialized(&used_range) {
cmd_buf
.buffer_memory_init_actions
.push(MemoryInitTrackerAction {
id: buffer_id,
range: used_range,
kind: MemoryInitKind::NeedsInitializedMemory,
});
}
let range = hal::buffer::SubRange {
offset,
@ -1605,11 +1623,18 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
.map_pass_err(scope);
}
used_buffer_ranges.push(ResourceMemoryInitTrackerAction {
id: buffer_id,
range: offset..end_offset,
kind: MemoryInitKind::NeedsInitializedMemory,
});
if !indirect_buffer
.initialization_status
.is_initialized(&(offset..end_offset))
{
cmd_buf
.buffer_memory_init_actions
.push(MemoryInitTrackerAction {
id: buffer_id,
range: offset..end_offset,
kind: MemoryInitKind::NeedsInitializedMemory,
});
}
match indexed {
false => unsafe {
@ -1694,6 +1719,18 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
})
.map_pass_err(scope);
}
if !indirect_buffer
.initialization_status
.is_initialized(&(offset..end_offset))
{
cmd_buf
.buffer_memory_init_actions
.push(MemoryInitTrackerAction {
id: buffer_id,
range: offset..end_offset,
kind: MemoryInitKind::NeedsInitializedMemory,
});
}
let begin_count_offset = count_buffer_offset;
let end_count_offset = count_buffer_offset + 4;
@ -1705,17 +1742,18 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
})
.map_pass_err(scope);
}
used_buffer_ranges.push(ResourceMemoryInitTrackerAction {
id: buffer_id,
range: offset..end_offset,
kind: MemoryInitKind::NeedsInitializedMemory,
});
used_buffer_ranges.push(ResourceMemoryInitTrackerAction {
id: count_buffer_id,
range: count_buffer_offset..end_count_offset,
kind: MemoryInitKind::NeedsInitializedMemory,
});
if !count_buffer
.initialization_status
.is_initialized(&(count_buffer_offset..end_count_offset))
{
cmd_buf
.buffer_memory_init_actions
.push(MemoryInitTrackerAction {
id: count_buffer_id,
range: count_buffer_offset..end_count_offset,
kind: MemoryInitKind::NeedsInitializedMemory,
});
}
match indexed {
false => unsafe {
@ -1849,8 +1887,18 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
.map_err(RenderPassErrorInner::IncompatibleRenderBundle)
.map_pass_err(scope)?;
used_buffer_ranges
.extend(bundle.used_buffer_ranges.iter().map(|x| x.clone()));
cmd_buf.buffer_memory_init_actions.extend(
bundle
.buffer_memory_init_actions
.iter()
.filter(|action| match buffer_guard.get(action.id) {
Ok(buffer) => {
!buffer.initialization_status.is_initialized(&action.range)
}
Err(_) => false,
})
.map(|action| action.clone()),
);
unsafe {
bundle.execute(
@ -1882,13 +1930,7 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
}
let (trackers, used_swapchain) = info.finish(&*texture_guard).map_pass_err(scope)?;
(
raw,
trackers,
used_swapchain,
used_buffer_ranges,
query_reset_state,
)
(raw, trackers, used_swapchain, query_reset_state)
};
let (mut cmb_guard, mut token) = hub.command_buffers.write(&mut token);
@ -1899,7 +1941,6 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
CommandBuffer::get_encoder_mut(&mut *cmb_guard, encoder_id).map_pass_err(scope)?;
cmd_buf.has_labels |= base.label.is_some();
cmd_buf.used_swap_chains.extend(used_swapchain);
cmd_buf.used_buffer_ranges.extend(used_buffer_ranges);
#[cfg(feature = "trace")]
if let Some(ref mut list) = cmd_buf.commands {

View File

@ -10,7 +10,7 @@ use crate::{
device::{all_buffer_stages, all_image_stages},
hub::{GfxBackend, Global, GlobalIdentityHandlerFactory, Storage, Token},
id::{BufferId, CommandEncoderId, TextureId},
memory_init_tracker::{MemoryInitKind, ResourceMemoryInitTrackerAction},
memory_init_tracker::{MemoryInitKind, MemoryInitTrackerAction},
resource::{BufferUse, Texture, TextureErrorDimension, TextureUse},
span,
track::TextureSelector,
@ -404,20 +404,32 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
}
// Make sure source is initialized memory and mark dest as initialized.
cmd_buf
.used_buffer_ranges
.push(ResourceMemoryInitTrackerAction {
id: destination,
range: destination_offset..(destination_offset + size),
kind: MemoryInitKind::ImplicitlyInitialized,
});
cmd_buf
.used_buffer_ranges
.push(ResourceMemoryInitTrackerAction {
id: source,
range: source_offset..(source_offset + size),
kind: MemoryInitKind::NeedsInitializedMemory,
});
let used_dst_buffer_range = destination_offset..(destination_offset + size);
if !dst_buffer
.initialization_status
.is_initialized(&used_dst_buffer_range)
{
cmd_buf
.buffer_memory_init_actions
.push(MemoryInitTrackerAction {
id: destination,
range: used_dst_buffer_range,
kind: MemoryInitKind::ImplicitlyInitialized,
});
}
let used_src_buffer_range = source_offset..(source_offset + size);
if !src_buffer
.initialization_status
.is_initialized(&used_src_buffer_range)
{
cmd_buf
.buffer_memory_init_actions
.push(MemoryInitTrackerAction {
id: source,
range: used_src_buffer_range,
kind: MemoryInitKind::NeedsInitializedMemory,
});
}
let region = hal::command::BufferCopy {
src: source_offset,
@ -532,14 +544,20 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
copy_size,
)?;
cmd_buf
.used_buffer_ranges
.push(ResourceMemoryInitTrackerAction {
id: source.buffer,
range: source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy),
kind: MemoryInitKind::NeedsInitializedMemory,
});
// TODO: Mark dest texture memory as implicitly initialized here.
let used_src_buffer_range =
source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy);
if !src_buffer
.initialization_status
.is_initialized(&used_src_buffer_range)
{
cmd_buf
.buffer_memory_init_actions
.push(MemoryInitTrackerAction {
id: source.buffer,
range: used_src_buffer_range,
kind: MemoryInitKind::NeedsInitializedMemory,
});
}
let (block_width, _) = dst_texture.format.describe().block_dimensions;
if !conv::is_valid_copy_dst_texture_format(dst_texture.format) {
@ -688,16 +706,20 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
))?
}
cmd_buf
.used_buffer_ranges
.push(ResourceMemoryInitTrackerAction {
id: destination.buffer,
range: destination.layout.offset
..(destination.layout.offset + required_buffer_bytes_in_copy),
kind: MemoryInitKind::ImplicitlyInitialized,
});
// TODO: Mark dest texture memory as required to be initialized here.
let used_dst_buffer_range =
destination.layout.offset..(destination.layout.offset + required_buffer_bytes_in_copy);
if !dst_buffer
.initialization_status
.is_initialized(&used_dst_buffer_range)
{
cmd_buf
.buffer_memory_init_actions
.push(MemoryInitTrackerAction {
id: destination.buffer,
range: used_dst_buffer_range,
kind: MemoryInitKind::ImplicitlyInitialized,
});
}
// WebGPU uses the physical size of the texture for copies whereas vulkan uses
// the virtual size. We have passed validation, so it's safe to use the
// image extent data directly. We want the provided copy size to be no larger than

View File

@ -211,7 +211,7 @@ fn map_buffer<B: hal::Backend>(
if let Some(uninitialized_ranges) = buffer
.initialization_status
.drain_uninitialized_ranges(offset..(size + offset))
.drain_uninitialized_ranges(&(offset..(size + offset)))
{
for uninitialized_range in uninitialized_ranges {
let num_bytes = uninitialized_range.end - uninitialized_range.start;
@ -1366,7 +1366,7 @@ impl<B: GfxBackend> Device<B> {
return Err(Error::BindingZeroSize(bb.buffer_id));
}
used_buffer_ranges.push(ResourceMemoryInitTrackerAction {
used_buffer_ranges.push(MemoryInitTrackerAction {
id: bb.buffer_id,
range: bb.offset..(bb.offset + bind_size),
kind: MemoryInitKind::NeedsInitializedMemory,
@ -2611,12 +2611,12 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
unsafe { ptr::write_bytes(ptr.as_ptr(), 0, buffer.size as usize) };
buffer
.initialization_status
.drain_uninitialized_ranges(0..buffer.size)
.drain_uninitialized_ranges(&(0..buffer.size))
.unwrap()
.for_each(drop);
stage
.initialization_status
.drain_uninitialized_ranges(0..buffer.size)
.drain_uninitialized_ranges(&(0..buffer.size))
.unwrap()
.for_each(drop);

View File

@ -277,7 +277,7 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
let dst = buffer_guard.get_mut(buffer_id).unwrap();
if let Some(uninitialized_ranges) = dst
.initialization_status
.drain_uninitialized_ranges(buffer_offset..(buffer_offset + data_size))
.drain_uninitialized_ranges(&(buffer_offset..(buffer_offset + data_size)))
{
uninitialized_ranges.for_each(drop);
}
@ -498,14 +498,14 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
.get(cmb_id)
.map_err(|_| QueueSubmitError::InvalidCommandBuffer(cmb_id))?;
for buffer_use in cmdbuf.used_buffer_ranges.iter() {
for buffer_use in cmdbuf.buffer_memory_init_actions.iter() {
let buffer = buffer_guard
.get_mut(buffer_use.id)
.map_err(|_| QueueSubmitError::DestroyedBuffer(buffer_use.id))?;
if let Some(uninitialized_ranges) = buffer
.initialization_status
.drain_uninitialized_ranges(buffer_use.range.clone())
.drain_uninitialized_ranges(&buffer_use.range)
{
match buffer_use.kind {
MemoryInitKind::ImplicitlyInitialized => {

View File

@ -2,14 +2,14 @@ use std::ops::Range;
#[derive(Debug, Clone)]
pub(crate) enum MemoryInitKind {
// The memory range is going to be written by an already initialized source, thus doesn't need extra attention.
// The memory range is going to be written by an already initialized source, thus doesn't need extra attention other than marking as initialized.
ImplicitlyInitialized,
// The memory range is going to be read, therefore needs to ensure prior initialization.
NeedsInitializedMemory,
}
#[derive(Debug, Clone)]
pub(crate) struct ResourceMemoryInitTrackerAction<ResourceId> {
pub(crate) struct MemoryInitTrackerAction<ResourceId> {
pub(crate) id: ResourceId,
pub(crate) range: Range<wgt::BufferAddress>,
pub(crate) kind: MemoryInitKind,
@ -34,10 +34,16 @@ impl MemoryInitTracker {
}
}
pub(crate) fn is_initialized(&self, range: &Range<wgt::BufferAddress>) -> bool {
self.uninitialized_ranges
.allocated_ranges()
.all(|r: Range<wgt::BufferAddress>| r.start >= range.end || r.end <= range.start)
}
#[must_use]
pub(crate) fn drain_uninitialized_ranges<'a>(
&'a mut self,
range: Range<wgt::BufferAddress>,
range: &Range<wgt::BufferAddress>,
) -> Option<impl Iterator<Item = Range<wgt::BufferAddress>> + 'a> {
let mut uninitialized_ranges: Vec<Range<wgt::BufferAddress>> = self
.uninitialized_ranges