From 09abb910e2bf3bee6eea69dfd92b98843454d7fd Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Wed, 2 Oct 2019 16:12:02 +0200 Subject: [PATCH] unlimited dimensions --- src/dimension.rs | 29 ++++++++++++++++++-- src/error.rs | 3 ++ src/file.rs | 64 +++++++++++++++++++++++++++++++++++++------ src/group.rs | 4 +++ src/variable.rs | 71 ++++++++++++++++++++++++++++++++++++------------ tests/lib.rs | 70 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 211 insertions(+), 30 deletions(-) diff --git a/src/dimension.rs b/src/dimension.rs index edc53e6..bfa1b72 100644 --- a/src/dimension.rs +++ b/src/dimension.rs @@ -5,15 +5,37 @@ use netcdf_sys::*; #[derive(Debug, Clone)] pub struct Dimension { pub(crate) name: String, - pub(crate) len: usize, + /// None when unlimited (size = 0) + pub(crate) len: Option, pub(crate) id: nc_type, + pub(crate) ncid: nc_type, } #[allow(clippy::len_without_is_empty)] impl Dimension { pub fn len(&self) -> usize { - self.len + match self.len { + Some(x) => x.get(), + None => { + let err; + let mut len = 0; + unsafe { + let _l = LOCK.lock().unwrap(); + err = nc_inq_dimlen(self.ncid, self.id, &mut len); + } + if err != NC_NOERR { + // Should log this... + return 0; + } + len + } + } } + + pub fn is_unlimited(&self) -> bool { + self.len.is_none() + } + pub fn name(&self) -> &str { &self.name } @@ -35,8 +57,9 @@ impl Dimension { Ok(Dimension { name: name.into(), - len, + len: core::num::NonZeroUsize::new(len), id: dimid, + ncid: grpid, }) } } diff --git a/src/error.rs b/src/error.rs index b54f0e0..6e12d1f 100644 --- a/src/error.rs +++ b/src/error.rs @@ -28,6 +28,8 @@ pub enum Error { AlreadyExists(String), /// Could not find variable/attribute/etc NotFound(String), + /// Slice lenghts are ambigious + Ambiguous, } impl std::error::Error for Error { @@ -87,6 +89,7 @@ impl fmt::Display for Error { write!(f, "netcdf error({}): {}", x, msg.to_string_lossy()) } + Error::Ambiguous => write!(f, "Could not find an appropriate length of the slices"), } } } diff --git a/src/file.rs b/src/file.rs index 3771632..c83fd03 100644 --- a/src/file.rs +++ b/src/file.rs @@ -195,7 +195,10 @@ impl Drop for File { use super::dimension::Dimension; -fn get_group_dimensions(ncid: nc_type) -> error::Result> { +fn get_group_dimensions( + ncid: nc_type, + unlimited_dims: &[nc_type], +) -> error::Result> { let mut ndims: nc_type = 0; let err; unsafe { @@ -233,9 +236,16 @@ fn get_group_dimensions(ncid: nc_type) -> error::Result error::Result error::Result> { +fn get_dimensions_of_var( + ncid: nc_type, + varid: nc_type, + unlimited_dims: &[nc_type], +) -> error::Result> { let mut ndims = 0; let err; unsafe { @@ -342,9 +356,16 @@ fn get_dimensions_of_var(ncid: nc_type, varid: nc_type) -> error::Result error::Result error::Result> { +fn get_variables( + ncid: nc_type, + unlimited_dims: &[nc_type], +) -> error::Result> { let err; let mut nvars = 0; unsafe { @@ -394,7 +418,7 @@ fn get_variables(ncid: nc_type) -> error::Result> { return Err(err.into()); } let attributes = get_attributes(ncid, varid)?; - let dimensions = get_dimensions_of_var(ncid, varid)?; + let dimensions = get_dimensions_of_var(ncid, varid, unlimited_dims)?; let cstr = std::ffi::CString::new( name.into_iter() @@ -444,8 +468,9 @@ fn get_groups( } let mut groups = HashMap::with_capacity(ngroups as usize); for grpid in grpids.into_iter() { - let dimensions = get_group_dimensions(grpid)?; - let variables = get_variables(grpid)?; + let unlim_dims = get_unlimited_dimensions(grpid)?; + let dimensions = get_group_dimensions(grpid, &unlim_dims)?; + let variables = get_variables(grpid, &unlim_dims)?; let mut parent_dimensions = parent_dim.to_vec(); parent_dimensions.push(dimensions.clone()); let subgroups = get_groups(grpid, &parent_dimensions)?; @@ -479,14 +504,35 @@ fn get_groups( Ok(groups) } +fn get_unlimited_dimensions(ncid: nc_type) -> error::Result> { + let err; + let mut nunlim = 0; + unsafe { + err = nc_inq_unlimdims(ncid, &mut nunlim, std::ptr::null_mut()); + } + if err != NC_NOERR { + return Err(err.into()); + } + let mut uldim = vec![0; nunlim as usize]; + let err; + unsafe { + err = nc_inq_unlimdims(ncid, std::ptr::null_mut(), uldim.as_mut_ptr()); + } + if err != NC_NOERR { + return Err(err.into()); + } + Ok(uldim) +} + fn parse_file(ncid: nc_type) -> error::Result { let _l = LOCK.lock().unwrap(); - let dimensions = get_group_dimensions(ncid)?; + let unlimited_dimensions = get_unlimited_dimensions(ncid)?; + let dimensions = get_group_dimensions(ncid, &unlimited_dimensions)?; let attributes = get_attributes(ncid, NC_GLOBAL)?; - let variables = get_variables(ncid)?; + let variables = get_variables(ncid, &unlimited_dimensions)?; // Empty parent group let groups = get_groups(ncid, &[])?; diff --git a/src/group.rs b/src/group.rs index f0223b9..b1561cd 100644 --- a/src/group.rs +++ b/src/group.rs @@ -78,6 +78,10 @@ impl Group { Ok(self.dimensions.get_mut(name).unwrap()) } + pub fn add_unlimited_dimension(&mut self, name: &str) -> error::Result<&mut Dimension> { + self.add_dimension(name, 0) + } + /// Create a Variable into the dataset, without writting any data into it. pub fn add_variable(&mut self, name: &str, dims: &[&str]) -> error::Result<&mut Variable> where diff --git a/src/variable.rs b/src/variable.rs index 4f22def..918a07e 100644 --- a/src/variable.rs +++ b/src/variable.rs @@ -134,7 +134,7 @@ macro_rules! impl_numeric { return Err(error::Error::IndexLen); } for i in 0..x.len() { - if x[i] >= variable.dimensions[i].len { + if x[i] >= variable.dimensions[i].len() { return Err(error::Error::IndexMismatch); } } @@ -170,7 +170,7 @@ macro_rules! impl_numeric { let slice_len = match slice_len { Some(x) => x, None => { - _slice_len = variable.dimensions.iter().map(|x| x.len).collect(); + _slice_len = variable.dimensions.iter().map(|x| x.len()).collect(); &_slice_len } }; @@ -219,7 +219,7 @@ macro_rules! impl_numeric { .dimensions .iter() .zip(indices.iter()) - .map(|(x, i)| (x.len).wrapping_sub(*i)) + .map(|(x, i)| (x.len()).saturating_sub(*i)) .collect(); &_slice_len } @@ -227,10 +227,10 @@ macro_rules! impl_numeric { let mut values_len = None; for i in 0..indices.len() { - if indices[i] >= variable.dimensions[i].len { + if indices[i] >= variable.dimensions[i].len() { return Err(error::Error::IndexMismatch); } - if (indices[i] + slice_len[i]) > variable.dimensions[i].len { + if (indices[i] + slice_len[i]) > variable.dimensions[i].len() { return Err(error::Error::SliceMismatch); } values_len = Some(values_len.unwrap_or(1) * slice_len[i]); @@ -277,8 +277,9 @@ macro_rules! impl_numeric { if x.len() != variable.dimensions.len() { return Err(error::Error::IndexLen); } - for i in 0..x.len() { - if x[i] >= variable.dimensions[i].len { + // Checking indices matching the variable dims + for (x, v) in x.iter().zip(&variable.dimensions) { + if !v.is_unlimited() && *x >= v.len() { return Err(error::Error::IndexMismatch); } } @@ -327,29 +328,63 @@ macro_rules! impl_numeric { let slice_len = match slice_len { Some(x) => x, None => { - _slice_len = variable + let num_unlimited = variable .dimensions .iter() - .zip(indices.iter()) - .map(|(x, i)| (x.len).wrapping_sub(*i)) - .collect(); - &_slice_len + .fold(0, |acc, v| acc + v.is_unlimited() as usize); + if num_unlimited == 0 { + _slice_len = variable + .dimensions + .iter() + .zip(indices.iter()) + .map(|(x, i)| x.len().saturating_sub(*i)) + .collect(); + &_slice_len + } else if num_unlimited == 1 { + let cube_length = variable.dimensions.iter().fold(1, |acc, x| { + if x.is_unlimited() { + acc + } else { + acc * x.len() + } + }); + let len_unlim = values.len() / cube_length; + _slice_len = variable + .dimensions + .iter() + .zip(indices.iter()) + .map(|(x, i)| { + if x.is_unlimited() { + len_unlim + } else { + x.len().saturating_sub(*i) + } + }) + .collect(); + &_slice_len + } else { + return Err(error::Error::Ambiguous); + } } }; let mut values_len = None; - for i in 0..indices.len() { - if indices[i] >= variable.dimensions[i].len { + for ((i, v), s) in indices + .iter() + .zip(variable.dimensions.iter()) + .zip(slice_len.iter()) + { + if !v.is_unlimited() && *i > v.len() { return Err(error::Error::IndexMismatch); } - if (indices[i] + slice_len[i]) > variable.dimensions[i].len { + if !v.is_unlimited() && (*i + s) > v.len() { return Err(error::Error::SliceMismatch); } // Check for empty slice - if slice_len[i] == 0 { + if *s == 0 { return Err(error::Error::ZeroSlice); } - values_len = Some(values_len.unwrap_or(1) * slice_len[i]); + values_len = Some(values_len.unwrap_or(1usize).saturating_mul(*s)); } if values_len.is_none() || values_len.unwrap() != values.len() { return Err(error::Error::BufferLen( @@ -360,7 +395,7 @@ macro_rules! impl_numeric { let err: nc_type; unsafe { - let _g = LOCK.lock().unwrap(); + let _l = LOCK.lock().unwrap(); err = $nc_put_vara_type( variable.ncid, variable.varid, diff --git a/tests/lib.rs b/tests/lib.rs index a7c0565..1de8ea8 100644 --- a/tests/lib.rs +++ b/tests/lib.rs @@ -856,3 +856,73 @@ fn add_conflicting_variables() { ); assert_eq!(10, file.variables()["x"].dimensions()[0].len()); } + +#[test] +fn unlimited_dimension_single_putting() { + let d = tempfile::tempdir().unwrap(); + let mut file = netcdf::create(d.path().join("unlim_single.nc")).unwrap(); + + file.add_unlimited_dimension("x").unwrap(); + file.add_unlimited_dimension("y").unwrap(); + + let var = &mut file.add_variable::("var", &["x", "y"]).unwrap(); + var.set_fill_value(0u8).unwrap(); + + var.put_value(1, None).unwrap(); + assert_eq!(var.dimensions()[0].len(), 1); + assert_eq!(var.dimensions()[1].len(), 1); + var.put_value(2, Some(&[0, 1])).unwrap(); + assert_eq!(var.dimensions()[0].len(), 1); + assert_eq!(var.dimensions()[1].len(), 2); + var.put_value(3, Some(&[2, 0])).unwrap(); + assert_eq!(var.dimensions()[0].len(), 3); + assert_eq!(var.dimensions()[1].len(), 2); + + check_equal(var, &[1, 2, 0, 0, 3, 0]); +} + +fn check_equal(var: &netcdf::Variable, check: &[T]) +where + T: netcdf::variable::Numeric + std::clone::Clone + std::default::Default + std::fmt::Debug + std::cmp::PartialEq, +{ + let mut v: Vec = vec![Default::default(); check.len()]; + var.values_to(&mut v, None, None).unwrap(); + assert_eq!(v.as_slice(), check); +} + +#[test] +fn unlimited_dimension_multi_putting() { + let d = tempfile::tempdir().unwrap(); + let mut file = netcdf::create(d.path().join("unlim_multi.nc")).unwrap(); + + file.add_unlimited_dimension("x").unwrap(); + file.add_unlimited_dimension("y").unwrap(); + file.add_dimension("z", 2).unwrap(); + file.add_unlimited_dimension("x2").unwrap(); + file.add_unlimited_dimension("x3").unwrap(); + file.add_unlimited_dimension("x4").unwrap(); + + let var = &mut file.add_variable::("one_unlim", &["x", "z"]).unwrap(); + var.put_values(&[0u8, 1, 2, 3], None, None).unwrap(); + check_equal(var, &[0u8, 1, 2, 3]); + var.put_values(&[0u8, 1, 2, 3, 4, 5, 6, 7], None, None).unwrap(); + check_equal(var, &[0u8, 1, 2, 3, 4, 5, 6, 7]); + + let var = &mut file + .add_variable::("unlim_first", &["z", "x2"]) + .unwrap(); + var.put_values(&[0u8, 1, 2, 3], None, None).unwrap(); + check_equal(var, &[0u8, 1, 2, 3]); + var.put_values(&[0u8, 1, 2, 3, 4, 5, 6, 7], None, None).unwrap(); + check_equal(var, &[0u8, 1, 2, 3, 4, 5, 6, 7]); + + let var = &mut file.add_variable::("two_unlim", &["x3", "x4"]).unwrap(); + var.set_fill_value(0u8).unwrap(); + let e = var.put_values(&[0u8, 1, 2, 3], None, None); + assert_eq!(e.unwrap_err(), netcdf::error::Error::Ambiguous); + var.put_values(&[0u8, 1, 2, 3], None, Some(&[1, 4])).unwrap(); + check_equal(var, &[0u8, 1, 2, 3]); + var.put_values(&[4u8, 5, 6], None, Some(&[3, 1])).unwrap(); + + check_equal(var, &[4, 1, 2, 3, 5, 0, 0, 0, 6, 0, 0, 0]); +}