archive-gh-me/netcdf
1
0
Fork 0
mirror of https://github.com/georust/netcdf.git synced 2026-01-25 15:02:13 +00:00
Code Issues Projects Releases Wiki Activity

unlimited dimensions

Browse Source
This commit is contained in:
Magnus Ulimoen 2019-10-02 16:12:02 +02:00
parent d35fe217c9
commit 09abb910e2
6 changed files with 211 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/dimension.rs
View File

@ -5,15 +5,37 @@ use netcdf_sys::*;
#[derive(Debug, Clone)]
pub struct Dimension {
pub(crate) name: String,
pub(crate) len: usize,
/// None when unlimited (size = 0)
pub(crate) len: Option<core::num::NonZeroUsize>,
pub(crate) id: nc_type,
pub(crate) ncid: nc_type,
}
#[allow(clippy::len_without_is_empty)]
impl Dimension {
pub fn len(&self) -> usize {
self.len
match self.len {
Some(x) => x.get(),
None => {
let err;
let mut len = 0;
unsafe {
let _l = LOCK.lock().unwrap();
err = nc_inq_dimlen(self.ncid, self.id, &mut len);
}
if err != NC_NOERR {
// Should log this...
return 0;
}
len
}
}
}
pub fn is_unlimited(&self) -> bool {
self.len.is_none()
}
pub fn name(&self) -> &str {
&self.name
}
@ -35,8 +57,9 @@ impl Dimension {
Ok(Dimension {
name: name.into(),
len,
len: core::num::NonZeroUsize::new(len),
id: dimid,
ncid: grpid,
})
}
}

3
src/error.rs
View File

@ -28,6 +28,8 @@ pub enum Error {
AlreadyExists(String),
/// Could not find variable/attribute/etc
NotFound(String),
/// Slice lenghts are ambigious
Ambiguous,
}
impl std::error::Error for Error {
@ -87,6 +89,7 @@ impl fmt::Display for Error {
write!(f, "netcdf error({}): {}", x, msg.to_string_lossy())
}
Error::Ambiguous => write!(f, "Could not find an appropriate length of the slices"),
}
}
}

64
src/file.rs
View File

@ -195,7 +195,10 @@ impl Drop for File {
use super::dimension::Dimension;
fn get_group_dimensions(ncid: nc_type) -> error::Result<HashMap<String, Dimension>> {
fn get_group_dimensions(
ncid: nc_type,
unlimited_dims: &[nc_type],
) -> error::Result<HashMap<String, Dimension>> {
let mut ndims: nc_type = 0;
let err;
unsafe {
@ -233,9 +236,16 @@ fn get_group_dimensions(ncid: nc_type) -> error::Result<HashMap<String, Dimensio
.map(|x| *x as u8)
.collect();
let name = String::from_utf8(name).unwrap();
let len = if unlimited_dims.contains(&dimid) {
None
} else {
Some(unsafe { core::num::NonZeroUsize::new_unchecked(len) })
};
dimensions.insert(
name.clone(),
Dimension {
ncid,
name,
len,
id: dimid,
@ -284,7 +294,11 @@ fn get_attributes(ncid: nc_type, varid: nc_type) -> error::Result<HashMap<String
Ok(attributes)
}
fn get_dimensions_of_var(ncid: nc_type, varid: nc_type) -> error::Result<Vec<Dimension>> {
fn get_dimensions_of_var(
ncid: nc_type,
varid: nc_type,
unlimited_dims: &[nc_type],
) -> error::Result<Vec<Dimension>> {
let mut ndims = 0;
let err;
unsafe {
@ -342,9 +356,16 @@ fn get_dimensions_of_var(ncid: nc_type, varid: nc_type) -> error::Result<Vec<Dim
.unwrap();
let name = cstr.to_string_lossy().into_owned();
let unlimited = unlimited_dims.contains(&dimid);
let len = if unlimited {
None
} else {
Some(unsafe { core::num::NonZeroUsize::new_unchecked(dimlen) })
};
let d = Dimension {
ncid,
name,
len: dimlen,
len: len,
id: dimid,
};
dimensions.push(d);
@ -354,7 +375,10 @@ fn get_dimensions_of_var(ncid: nc_type, varid: nc_type) -> error::Result<Vec<Dim
}
use super::Variable;
fn get_variables(ncid: nc_type) -> error::Result<HashMap<String, Variable>> {
fn get_variables(
ncid: nc_type,
unlimited_dims: &[nc_type],
) -> error::Result<HashMap<String, Variable>> {
let err;
let mut nvars = 0;
unsafe {
@ -394,7 +418,7 @@ fn get_variables(ncid: nc_type) -> error::Result<HashMap<String, Variable>> {
return Err(err.into());
}
let attributes = get_attributes(ncid, varid)?;
let dimensions = get_dimensions_of_var(ncid, varid)?;
let dimensions = get_dimensions_of_var(ncid, varid, unlimited_dims)?;
let cstr = std::ffi::CString::new(
name.into_iter()
@ -444,8 +468,9 @@ fn get_groups(
}
let mut groups = HashMap::with_capacity(ngroups as usize);
for grpid in grpids.into_iter() {
let dimensions = get_group_dimensions(grpid)?;
let variables = get_variables(grpid)?;
let unlim_dims = get_unlimited_dimensions(grpid)?;
let dimensions = get_group_dimensions(grpid, &unlim_dims)?;
let variables = get_variables(grpid, &unlim_dims)?;
let mut parent_dimensions = parent_dim.to_vec();
parent_dimensions.push(dimensions.clone());
let subgroups = get_groups(grpid, &parent_dimensions)?;
@ -479,14 +504,35 @@ fn get_groups(
Ok(groups)
}
fn get_unlimited_dimensions(ncid: nc_type) -> error::Result<Vec<nc_type>> {
let err;
let mut nunlim = 0;
unsafe {
err = nc_inq_unlimdims(ncid, &mut nunlim, std::ptr::null_mut());
}
if err != NC_NOERR {
return Err(err.into());
}
let mut uldim = vec![0; nunlim as usize];
let err;
unsafe {
err = nc_inq_unlimdims(ncid, std::ptr::null_mut(), uldim.as_mut_ptr());
}
if err != NC_NOERR {
return Err(err.into());
}
Ok(uldim)
}
fn parse_file(ncid: nc_type) -> error::Result<Group> {
let _l = LOCK.lock().unwrap();
let dimensions = get_group_dimensions(ncid)?;
let unlimited_dimensions = get_unlimited_dimensions(ncid)?;
let dimensions = get_group_dimensions(ncid, &unlimited_dimensions)?;
let attributes = get_attributes(ncid, NC_GLOBAL)?;
let variables = get_variables(ncid)?;
let variables = get_variables(ncid, &unlimited_dimensions)?;
// Empty parent group
let groups = get_groups(ncid, &[])?;

4
src/group.rs
View File

@ -78,6 +78,10 @@ impl Group {
Ok(self.dimensions.get_mut(name).unwrap())
}
pub fn add_unlimited_dimension(&mut self, name: &str) -> error::Result<&mut Dimension> {
self.add_dimension(name, 0)
}
/// Create a Variable into the dataset, without writting any data into it.
pub fn add_variable<T>(&mut self, name: &str, dims: &[&str]) -> error::Result<&mut Variable>
where

71
src/variable.rs
View File

@ -134,7 +134,7 @@ macro_rules! impl_numeric {
return Err(error::Error::IndexLen);
}
for i in 0..x.len() {
if x[i] >= variable.dimensions[i].len {
if x[i] >= variable.dimensions[i].len() {
return Err(error::Error::IndexMismatch);
}
}
@ -170,7 +170,7 @@ macro_rules! impl_numeric {
let slice_len = match slice_len {
Some(x) => x,
None => {
_slice_len = variable.dimensions.iter().map(|x| x.len).collect();
_slice_len = variable.dimensions.iter().map(|x| x.len()).collect();
&_slice_len
}
};
@ -219,7 +219,7 @@ macro_rules! impl_numeric {
.dimensions
.iter()
.zip(indices.iter())
.map(|(x, i)| (x.len).wrapping_sub(*i))
.map(|(x, i)| (x.len()).saturating_sub(*i))
.collect();
&_slice_len
}
@ -227,10 +227,10 @@ macro_rules! impl_numeric {
let mut values_len = None;
for i in 0..indices.len() {
if indices[i] >= variable.dimensions[i].len {
if indices[i] >= variable.dimensions[i].len() {
return Err(error::Error::IndexMismatch);
}
if (indices[i] + slice_len[i]) > variable.dimensions[i].len {
if (indices[i] + slice_len[i]) > variable.dimensions[i].len() {
return Err(error::Error::SliceMismatch);
}
values_len = Some(values_len.unwrap_or(1) * slice_len[i]);
@ -277,8 +277,9 @@ macro_rules! impl_numeric {
if x.len() != variable.dimensions.len() {
return Err(error::Error::IndexLen);
}
for i in 0..x.len() {
if x[i] >= variable.dimensions[i].len {
// Checking indices matching the variable dims
for (x, v) in x.iter().zip(&variable.dimensions) {
if !v.is_unlimited() && *x >= v.len() {
return Err(error::Error::IndexMismatch);
}
}
@ -327,29 +328,63 @@ macro_rules! impl_numeric {
let slice_len = match slice_len {
Some(x) => x,
None => {
_slice_len = variable
let num_unlimited = variable
.dimensions
.iter()
.zip(indices.iter())
.map(|(x, i)| (x.len).wrapping_sub(*i))
.collect();
&_slice_len
.fold(0, |acc, v| acc + v.is_unlimited() as usize);
if num_unlimited == 0 {
_slice_len = variable
.dimensions
.iter()
.zip(indices.iter())
.map(|(x, i)| x.len().saturating_sub(*i))
.collect();
&_slice_len
} else if num_unlimited == 1 {
let cube_length = variable.dimensions.iter().fold(1, |acc, x| {
if x.is_unlimited() {
acc
} else {
acc * x.len()
}
});
let len_unlim = values.len() / cube_length;
_slice_len = variable
.dimensions
.iter()
.zip(indices.iter())
.map(|(x, i)| {
if x.is_unlimited() {
len_unlim
} else {
x.len().saturating_sub(*i)
}
})
.collect();
&_slice_len
} else {
return Err(error::Error::Ambiguous);
}
}
};
let mut values_len = None;
for i in 0..indices.len() {
if indices[i] >= variable.dimensions[i].len {
for ((i, v), s) in indices
.iter()
.zip(variable.dimensions.iter())
.zip(slice_len.iter())
{
if !v.is_unlimited() && *i > v.len() {
return Err(error::Error::IndexMismatch);
}
if (indices[i] + slice_len[i]) > variable.dimensions[i].len {
if !v.is_unlimited() && (*i + s) > v.len() {
return Err(error::Error::SliceMismatch);
}
// Check for empty slice
if slice_len[i] == 0 {
if *s == 0 {
return Err(error::Error::ZeroSlice);
}
values_len = Some(values_len.unwrap_or(1) * slice_len[i]);
values_len = Some(values_len.unwrap_or(1usize).saturating_mul(*s));
}
if values_len.is_none() || values_len.unwrap() != values.len() {
return Err(error::Error::BufferLen(
@ -360,7 +395,7 @@ macro_rules! impl_numeric {
let err: nc_type;
unsafe {
let _g = LOCK.lock().unwrap();
let _l = LOCK.lock().unwrap();
err = $nc_put_vara_type(
variable.ncid,
variable.varid,

70
tests/lib.rs
View File

@ -856,3 +856,73 @@ fn add_conflicting_variables() {
);
assert_eq!(10, file.variables()["x"].dimensions()[0].len());
}
#[test]
fn unlimited_dimension_single_putting() {
let d = tempfile::tempdir().unwrap();
let mut file = netcdf::create(d.path().join("unlim_single.nc")).unwrap();
file.add_unlimited_dimension("x").unwrap();
file.add_unlimited_dimension("y").unwrap();
let var = &mut file.add_variable::<u8>("var", &["x", "y"]).unwrap();
var.set_fill_value(0u8).unwrap();
var.put_value(1, None).unwrap();
assert_eq!(var.dimensions()[0].len(), 1);
assert_eq!(var.dimensions()[1].len(), 1);
var.put_value(2, Some(&[0, 1])).unwrap();
assert_eq!(var.dimensions()[0].len(), 1);
assert_eq!(var.dimensions()[1].len(), 2);
var.put_value(3, Some(&[2, 0])).unwrap();
assert_eq!(var.dimensions()[0].len(), 3);
assert_eq!(var.dimensions()[1].len(), 2);
check_equal(var, &[1, 2, 0, 0, 3, 0]);
}
fn check_equal<T>(var: &netcdf::Variable, check: &[T])
where
T: netcdf::variable::Numeric + std::clone::Clone + std::default::Default + std::fmt::Debug + std::cmp::PartialEq,
{
let mut v: Vec<T> = vec![Default::default(); check.len()];
var.values_to(&mut v, None, None).unwrap();
assert_eq!(v.as_slice(), check);
}
#[test]
fn unlimited_dimension_multi_putting() {
let d = tempfile::tempdir().unwrap();
let mut file = netcdf::create(d.path().join("unlim_multi.nc")).unwrap();
file.add_unlimited_dimension("x").unwrap();
file.add_unlimited_dimension("y").unwrap();
file.add_dimension("z", 2).unwrap();
file.add_unlimited_dimension("x2").unwrap();
file.add_unlimited_dimension("x3").unwrap();
file.add_unlimited_dimension("x4").unwrap();
let var = &mut file.add_variable::<u8>("one_unlim", &["x", "z"]).unwrap();
var.put_values(&[0u8, 1, 2, 3], None, None).unwrap();
check_equal(var, &[0u8, 1, 2, 3]);
var.put_values(&[0u8, 1, 2, 3, 4, 5, 6, 7], None, None).unwrap();
check_equal(var, &[0u8, 1, 2, 3, 4, 5, 6, 7]);
let var = &mut file
.add_variable::<u8>("unlim_first", &["z", "x2"])
.unwrap();
var.put_values(&[0u8, 1, 2, 3], None, None).unwrap();
check_equal(var, &[0u8, 1, 2, 3]);
var.put_values(&[0u8, 1, 2, 3, 4, 5, 6, 7], None, None).unwrap();
check_equal(var, &[0u8, 1, 2, 3, 4, 5, 6, 7]);
let var = &mut file.add_variable::<u8>("two_unlim", &["x3", "x4"]).unwrap();
var.set_fill_value(0u8).unwrap();
let e = var.put_values(&[0u8, 1, 2, 3], None, None);
assert_eq!(e.unwrap_err(), netcdf::error::Error::Ambiguous);
var.put_values(&[0u8, 1, 2, 3], None, Some(&[1, 4])).unwrap();
check_equal(var, &[0u8, 1, 2, 3]);
var.put_values(&[4u8, 5, 6], None, Some(&[3, 1])).unwrap();
check_equal(var, &[4, 1, 2, 3, 5, 0, 0, 0, 6, 0, 0, 0]);
}