Class: HDF5::Dataset

Inherits:
Object
  • Object
show all
Defined in:
lib/hdf5/dataset.rb

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(parent_id, name) ⇒ Dataset

Returns a new instance of Dataset.



213
214
215
 
# File 'lib/hdf5/dataset.rb', line 213

def initialize(parent_id, name)
  initialize_from_id(HDF5::FFI.H5Dopen2(parent_id, name, HDF5::DEFAULT_PROPERTY_LIST), name, nil)
end

Class Method Details

.create(parent_id, name, data = nil, shape: nil, dtype: nil, maxshape: nil, chunks: nil, compression: nil, compression_opts: nil, shuffle: false, fletcher32: false, fillvalue: nil, context: nil) ⇒ Object 



4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/hdf5/dataset.rb', line 4

def create(parent_id, name, data = nil, shape: nil, dtype: nil, maxshape: nil, chunks: nil, compression: nil,
           compression_opts: nil, shuffle: false, fletcher32: false, fillvalue: nil, context: nil)
  empty_data = data.is_a?(HDF5::Empty)
  string_data = HDF5::StringCodec.string_data?(data)
  _string_values, string_shape = HDF5::StringCodec.normalize_data(data) if string_data
  unless data.nil? || string_data || empty_data
    narray = HDF5::DataHelpers.normalize_data(data,
                                              label: 'Dataset data')
  end
  unless string_data
    dtype_object = if empty_data
                     data.dtype
                   else
                     (dtype ? DType.for_symbol(dtype) : DType.for_numo(narray))
                   end
  end
  type_id = string_data ? HDF5::StringCodec.datatype_id : dtype_object.storage_type_id
  shape = string_data ? string_shape : narray.shape if shape.nil? && !data.nil? && !empty_data
  raise HDF5::Error, 'shape: and dtype: are required when data: is omitted' if data.nil? && (!shape || !dtype)
  raise HDF5::Error, 'Dataset shape must match data shape' if narray && shape != narray.shape
  raise HDF5::ShapeError, 'Dataset shape must match string data shape' if string_data && shape != string_shape

  raise HDF5::Error, 'Null datasets cannot have maxshape or chunks' if empty_data && (maxshape || chunks)

  validate_maxshape(maxshape, shape) if maxshape
  chunks = :auto if maxshape && chunks.nil?
  dataspace_id = create_dataspace(shape, maxshape)
  raise HDF5::Error, "Failed to create dataspace for dataset: #{name}" if dataspace_id < 0

  dcpl_id = create_property_list(shape, dtype_object, chunks:, compression:, compression_opts:, shuffle:, fletcher32:,
                                                      fillvalue:)

  dataset = from_id(
    HDF5::FFI.H5Dcreate2(parent_id, name, type_id, dataspace_id, HDF5::DEFAULT_PROPERTY_LIST,
                         dcpl_id || HDF5::DEFAULT_PROPERTY_LIST, HDF5::DEFAULT_PROPERTY_LIST), name, context
  )
  dataset.write(data) if string_data
  dataset.write(narray) if narray
  return dataset unless block_given?

  begin
    yield dataset
  ensure
    dataset.close
  end
rescue StandardError
  if dataset
    dataset.close unless dataset.closed?
    HDF5::FFI.H5Ldelete(parent_id, name, HDF5::DEFAULT_PROPERTY_LIST)
  end
  raise
ensure
  HDF5::FFI.H5Tclose(type_id) if string_data && type_id && type_id >= 0
  HDF5::FFI.H5Pclose(dcpl_id) if dcpl_id && dcpl_id >= 0
  HDF5::FFI.H5Sclose(dataspace_id) if dataspace_id && dataspace_id >= 0
end

.open(parent_id, name, context: nil) ⇒ Object 



61
62
63
64
65
66
67
68
69
70
 
# File 'lib/hdf5/dataset.rb', line 61

def open(parent_id, name, context: nil)
  dataset = from_id(HDF5::FFI.H5Dopen2(parent_id, name, HDF5::DEFAULT_PROPERTY_LIST), name, context)
  return dataset unless block_given?

  begin
    yield dataset
  ensure
    dataset.close
  end
end

Instance Method Details

#[](*selection) ⇒ Object 



476
477
478
 
# File 'lib/hdf5/dataset.rb', line 476

def [](*selection)
  read(selection: selection)
end

#[]=(*selection, value) ⇒ Object 



480
481
482
 
# File 'lib/hdf5/dataset.rb', line 480

def []=(*selection, value)
  write(value, selection: selection)
end

#append(data, axis: 0) ⇒ Object 



382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
 
# File 'lib/hdf5/dataset.rb', line 382

def append(data, axis: 0)
  ensure_open!
  values = HDF5::DataHelpers.normalize_data(data, label: 'Dataset data')
  current_shape = shape
  raise HDF5::Error, 'Cannot append to a Null dataset' if current_shape.nil?
  raise HDF5::Error, 'Cannot append to a scalar dataset' if current_shape.empty?
  raise IndexError, "Invalid append axis: #{axis}" unless axis.is_a?(Integer) && axis.between?(0,
                                                                                               current_shape.length - 1)
  raise HDF5::Error, 'Appended data rank must match dataset rank' unless values.shape.length == current_shape.length
  raise HDF5::Error, 'Appended data shape must match all non-appended dimensions' unless
    values.shape.each_with_index.all? { |dimension, index| index == axis || dimension == current_shape[index] }

  source_dtype = DType.for_numo(values)
  target_dtype = dtype
  raise ConversionError, "Cannot safely cast #{source_dtype.to_sym} to #{target_dtype.to_sym}" unless
    source_dtype.castable_to?(target_dtype)
  return self if values.shape[axis].zero?

  new_shape = current_shape.dup
  new_shape[axis] += values.shape[axis]
  resize(new_shape)
  selection = current_shape.each_with_index.map do |dimension, index|
    index == axis ? dimension...new_shape[index] : 0...dimension
  end
  write(values, selection: selection)
  self
rescue StandardError => e
  raise unless current_shape && new_shape && shape == new_shape

  begin
    resize(current_shape)
  rescue StandardError => rollback_error
    raise HDF5::Error,
          "Append failed (#{e.message}) and extent rollback failed (#{rollback_error.message}); current shape: #{shape.inspect}"
  end
  raise e
end

#attrs ⇒ Object 



217
218
219
220
 
# File 'lib/hdf5/dataset.rb', line 217

def attrs
  ensure_open!
  @attrs ||= AttributeManager.new(@dataset_id, @context)
end

#chunks ⇒ Object 



311
312
313
314
315
316
317
318
319
320
321
322
323
324
 
# File 'lib/hdf5/dataset.rb', line 311

def chunks
  ensure_open!
  property_list_id = HDF5::FFI.H5Dget_create_plist(@dataset_id)
  raise HDF5::Error, 'Failed to get dataset creation properties' if property_list_id < 0
  return nil unless HDF5::FFI.H5Pget_layout(property_list_id) == :H5D_CHUNKED

  dimensions = ::FFI::MemoryPointer.new(:ulong_long, shape.length)
  rank = HDF5::FFI.H5Pget_chunk(property_list_id, shape.length, dimensions)
  raise HDF5::Error, 'Failed to get chunk dimensions' if rank < 0

  dimensions.read_array_of_uint64(rank)
ensure
  HDF5::FFI.H5Pclose(property_list_id) if property_list_id && property_list_id >= 0
end

#close ⇒ Object 



265
266
267
268
269
270
 
# File 'lib/hdf5/dataset.rb', line 265

def close
  return if @dataset_id.nil?

  @context ? @context.close(@dataset_id) : HDF5::FFI.H5Dclose(@dataset_id)
  @dataset_id = nil
end

#closed? ⇒ Boolean

Returns:

  • (Boolean) 


272
273
274
 
# File 'lib/hdf5/dataset.rb', line 272

def closed?
  @dataset_id.nil? || (@context && @context.closed?)
end

#dtype ⇒ Object 



276
277
278
279
280
281
282
283
284
 
# File 'lib/hdf5/dataset.rb', line 276

def dtype
  ensure_open!
  datatype_id = HDF5::FFI.H5Dget_type(@dataset_id)
  raise HDF5::Error, 'Failed to get datatype' if datatype_id < 0

  DType.for_hdf5(datatype_id)
ensure
  HDF5::FFI.H5Tclose(datatype_id) if datatype_id && datatype_id >= 0
end

#each_block(max_bytes:) ⇒ Object

Raises:

  • (ArgumentError) 


497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
 
# File 'lib/hdf5/dataset.rb', line 497

def each_block(max_bytes:)
  return enum_for(__method__, max_bytes:) unless block_given?

  ensure_open!
  raise ArgumentError, 'max_bytes must be a positive integer' unless max_bytes.is_a?(Integer) && max_bytes.positive?

  current_dtype = dtype
  raise ArgumentError, 'max_bytes is smaller than one dataset element' if max_bytes < current_dtype.itemsize

  current_shape = shape
  if current_shape.empty?
    yield [], read
    return
  end
  return if current_shape.any?(&:zero?)

  block_shape = block_shape_for(current_shape, max_bytes / current_dtype.itemsize)
  each_block_selection(current_shape, block_shape) do |selection|
    yield selection, read(selection: selection)
  end
end

#each_chunk ⇒ Object

Raises:



519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
 
# File 'lib/hdf5/dataset.rb', line 519

def each_chunk
  return enum_for(__method__) unless block_given?

  ensure_open!

  chunk_shape = chunks
  raise HDF5::Error, 'each_chunk requires a chunked dataset' unless chunk_shape

  current_shape = shape
  return if current_shape.any?(&:zero?)

  each_block_selection(current_shape, chunk_shape) do |selection|
    yield selection, read(selection: selection)
  end
end

#fillvalue ⇒ Object 



344
345
346
347
348
349
350
351
352
353
354
355
356
357
 
# File 'lib/hdf5/dataset.rb', line 344

def fillvalue
  ensure_open!
  dtype_object = dtype
  property_list_id = HDF5::FFI.H5Dget_create_plist(@dataset_id)
  raise HDF5::Error, 'Failed to get dataset creation properties' if property_list_id < 0

  buffer = ::FFI::MemoryPointer.new(:char, dtype_object.itemsize)
  status = HDF5::FFI.H5Pget_fill_value(property_list_id, dtype_object.memory_type_id, buffer)
  raise HDF5::Error, 'Failed to get dataset fill value' if status < 0

  dtype_object.numo_class.from_binary(buffer.read_bytes(dtype_object.itemsize), []).extract
ensure
  HDF5::FFI.H5Pclose(property_list_id) if property_list_id && property_list_id >= 0
end

#maxshape ⇒ Object 



326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
 
# File 'lib/hdf5/dataset.rb', line 326

def maxshape
  ensure_open!
  dataspace_id = HDF5::FFI.H5Dget_space(@dataset_id)
  raise HDF5::Error, 'Failed to get dataset dataspace' if dataspace_id < 0

  rank = HDF5::FFI.H5Sget_simple_extent_ndims(dataspace_id)
  return [] if rank.zero?

  maximums = ::FFI::MemoryPointer.new(:ulong_long, rank)
  status = HDF5::FFI.H5Sget_simple_extent_dims(dataspace_id, nil, maximums)
  raise HDF5::Error, 'Failed to get dataset maximum shape' if status < 0

  unlimited = (1 << (::FFI.type_size(:ulong_long) * 8)) - 1
  maximums.read_array_of_uint64(rank).map { |dimension| dimension == unlimited ? nil : dimension }
ensure
  HDF5::FFI.H5Sclose(dataspace_id) if dataspace_id && dataspace_id >= 0
end

#ndim ⇒ Object 



303
304
305
 
# File 'lib/hdf5/dataset.rb', line 303

def ndim
  shape&.length
end

#read(selection: nil, dtype: nil, casting: :safe) ⇒ Object 



420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
 
# File 'lib/hdf5/dataset.rb', line 420

def read(selection: nil, dtype: nil, casting: :safe)
  ensure_open!
  type_id = HDF5::FFI.H5Dget_type(@dataset_id)
  raise HDF5::Error, 'Failed to get dataset datatype' if type_id < 0
  if dtype && HDF5::FFI.H5Tget_class(type_id) == :H5T_STRING
    raise ConversionError,
          'dtype is not supported for string datasets'
  end
  return read_string(type_id, selection:) if HDF5::FFI.H5Tget_class(type_id) == :H5T_STRING

  source_dtype = DType.for_hdf5(type_id)
  current_dtype = dtype ? DType.for_symbol(dtype) : source_dtype
  raise ConversionError, "Cannot safely cast #{source_dtype.to_sym} to #{current_dtype.to_sym}" unless
    source_dtype.castable_to?(current_dtype, casting:)

  current_shape = shape
  return HDF5::Empty.new(current_dtype) if current_shape.nil?

  normalized_selection = Selection.normalize(selection, current_shape)
  return current_dtype.numo_class.zeros(*normalized_selection.result_shape) if normalized_selection.size.zero?

  file_space_id = HDF5::FFI.H5Dget_space(@dataset_id)
  raise HDF5::Error, 'Failed to get dataset dataspace' if file_space_id < 0

  select_hyperslab(file_space_id, normalized_selection)
  memory_space_id = create_memory_dataspace(normalized_selection.result_shape)
  raise HDF5::Error, 'Failed to create memory dataspace' if memory_space_id < 0
  raise HDF5::Error, 'File and memory selections have different sizes' unless
    HDF5::FFI.H5Sget_select_npoints(file_space_id) == HDF5::FFI.H5Sget_select_npoints(memory_space_id)

  bytesize = normalized_selection.size * current_dtype.itemsize
  buffer = ::FFI::MemoryPointer.new(:char, bytesize)
  status = HDF5::FFI.H5Dread(@dataset_id, current_dtype.memory_type_id, memory_space_id, file_space_id,
                             HDF5::DEFAULT_PROPERTY_LIST, buffer)
  raise HDF5::Error, 'Failed to read dataset' if status < 0

  result = HDF5::DataHelpers.from_binary(current_dtype, buffer.read_bytes(bytesize),
                                         normalized_selection.result_shape)
  return result unless normalized_selection.scalar?

  scalar = result.extract
  current_dtype.kind == :bool ? !scalar.zero? : scalar
ensure
  HDF5::FFI.H5Tclose(type_id) if type_id && type_id >= 0
  HDF5::FFI.H5Sclose(memory_space_id) if memory_space_id && memory_space_id >= 0
  HDF5::FFI.H5Sclose(file_space_id) if file_space_id && file_space_id >= 0
end

#read_array(selection: nil, flatten: false, dtype: nil, casting: :safe) ⇒ Object 



468
469
470
471
472
473
474
 
# File 'lib/hdf5/dataset.rb', line 468

def read_array(selection: nil, flatten: false, dtype: nil, casting: :safe)
  value = read(selection:, dtype:, casting:)
  return value unless value.is_a?(Numo::NArray)

  array = value.to_a
  flatten ? array.flatten : array
end

#read_into(destination, selection: nil, casting: :safe) ⇒ Object

Raises:



484
485
486
487
488
489
490
491
492
493
494
495
 
# File 'lib/hdf5/dataset.rb', line 484

def read_into(destination, selection: nil, casting: :safe)
  ensure_open!
  raise HDF5::Error, 'read_into destination must be a Numo::NArray' unless destination.is_a?(Numo::NArray)

  values = read(selection:, dtype: DType.for_numo(destination).to_sym, casting:)
  unless destination.shape == values.shape
    raise HDF5::Error,
          'read_into destination shape must match selection shape'
  end

  destination.store(values)
end

#resize(new_shape) ⇒ Object

Raises:



359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
 
# File 'lib/hdf5/dataset.rb', line 359

def resize(new_shape)
  ensure_open!
  raise HDF5::Error, 'Cannot resize a Null dataset' if shape.nil?
  unless new_shape.is_a?(Array) && new_shape.length == shape.length
    raise HDF5::Error,
          'Dataset shape must be an Array matching dataset rank'
  end
  unless new_shape.all? { |dimension| dimension.is_a?(Integer) && dimension >= 0 }
    raise HDF5::Error, 'Dataset dimensions must be non-negative integers'
  end

  maxshape.zip(new_shape).each do |maximum, dimension|
    raise HDF5::Error, 'Dataset shape exceeds maxshape' if maximum && dimension > maximum
  end

  dimensions = ::FFI::MemoryPointer.new(:ulong_long, new_shape.length)
  dimensions.write_array_of_ulong_long(new_shape)
  status = HDF5::FFI.H5Dset_extent(@dataset_id, dimensions)
  raise HDF5::Error, 'Failed to resize dataset' if status < 0

  self
end

#shape ⇒ Object 



286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
 
# File 'lib/hdf5/dataset.rb', line 286

def shape
  ensure_open!
  dataspace_id = HDF5::FFI.H5Dget_space(@dataset_id)
  raise HDF5::Error, 'Failed to get dataspace' if dataspace_id < 0
  return nil if HDF5::FFI.H5Sget_simple_extent_type(dataspace_id) == :H5S_NULL

  ndims = HDF5::FFI.H5Sget_simple_extent_ndims(dataspace_id)
  raise HDF5::Error, 'Failed to get number of dimensions' if ndims < 0

  dims = ::FFI::MemoryPointer.new(:ulong_long, ndims)
  HDF5::FFI.H5Sget_simple_extent_dims(dataspace_id, dims, nil)

  dims.read_array_of_uint64(ndims)
ensure
  HDF5::FFI.H5Sclose(dataspace_id) if dataspace_id && dataspace_id >= 0
end

#size ⇒ Object 



307
308
309
 
# File 'lib/hdf5/dataset.rb', line 307

def size
  shape&.inject(1, :*) || 0
end

#write(data, selection: nil, casting: :safe) ⇒ Object 



222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
 
# File 'lib/hdf5/dataset.rb', line 222

def write(data, selection: nil, casting: :safe)
  ensure_open!
  return write_string(data, selection:) if HDF5::StringCodec.string_data?(data)

  normalized_selection = Selection.normalize(selection, shape)
  values = if data.is_a?(Numeric)
             target_dtype = dtype
             if normalized_selection.scalar?
               target_dtype.numo_class.cast(data)
             else
               target_dtype.numo_class.ones(*normalized_selection.result_shape) * data
             end
           else
             HDF5::DataHelpers.normalize_data(data, label: 'Dataset data')
           end
  raise HDF5::Error, 'Dataset shape must match data shape' unless values.shape == normalized_selection.result_shape

  dtype_object = DType.for_numo(values)
  target_dtype = dtype
  raise ConversionError, "Cannot safely cast #{dtype_object.to_sym} to #{target_dtype.to_sym}" unless
    dtype_object.castable_to?(target_dtype, casting:)
  return data if normalized_selection.size.zero?

  buffer = HDF5::DataHelpers.buffer_for(values)
  file_space_id = HDF5::FFI.H5Dget_space(@dataset_id)
  raise HDF5::Error, 'Failed to get dataset dataspace' if file_space_id < 0

  select_hyperslab(file_space_id, normalized_selection)
  memory_space_id = create_memory_dataspace(normalized_selection.result_shape)
  raise HDF5::Error, 'Failed to create memory dataspace' if memory_space_id < 0
  raise HDF5::Error, 'File and memory selections have different sizes' unless
    HDF5::FFI.H5Sget_select_npoints(file_space_id) == HDF5::FFI.H5Sget_select_npoints(memory_space_id)

  status = HDF5::FFI.H5Dwrite(@dataset_id, dtype_object.memory_type_id, memory_space_id, file_space_id,
                              HDF5::DEFAULT_PROPERTY_LIST, buffer)
  raise HDF5::Error, 'Failed to write dataset' if status < 0

  data
ensure
  HDF5::FFI.H5Sclose(memory_space_id) if memory_space_id && memory_space_id >= 0
  HDF5::FFI.H5Sclose(file_space_id) if file_space_id && file_space_id >= 0
end