Module: NumRu::GPhys::NetCDF_IO

Defined in:
lib/numru/gphys/gphys_netcdf_io.rb

Class Method Summary collapse

Class Method Details

.__files2varray(files, varname, dim = nil) ⇒ Object



523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 523

def __files2varray( files, varname, dim=nil )
	if files.is_a?(NetCDF)
	  # Single file. Returns a VArrayNetCDF. dim is ignored.
	  file = files
	  var = file.var( varname )
	  raise "variable '#{varname}' not found in #{file.inspect}" if !var
	  VArrayNetCDF.new( var )
	elsif files.is_a?(NArray)
	  # Suppose that files is a NArray of NetCDF. Returns a VArrayCompsite.
	  if dim.is_a?(Integer) && dim>=0 && dim<files.rank
	    files = files[ *([0]*dim+[true]+[0]*(files.rank-dim-1)) ]
	  end
	  varys = NArray.object( *files.shape )
	  for i in 0...files.length
	    var = files[i].var( varname )
	    raise "variable '#{varname}' not found in #{files[i].path}" if !var
	    varys[i] = VArrayNetCDF.new( var )
	  end
	  if files.length != 1
	    VArrayComposite.new( varys )
	  else
	    varys[0]
	  end
	else
	  raise TypeError, "not a NetCDF or NArray"
	end
end

.__files_dim_matching(files, varname) ⇒ Object

re-arrange the array of files and check some inconsistencies

e.g.)
   files = NArray[ file0.nc, file1.nc, file2.nc, file3.nc ]
   The files have a variable, u(lat,lev,time),
   which is tiled to four parts as the followings:
              |    lat  |     lev    | time |
     file0.nc | -90...0 | 1000...500 | 0..1 |
     file1.nc |   0..90 | 1000...500 | 0..1 |
     file2.nc | -90...0 |  500..100  | 0..1 |
     file3.nc |   0..90 |  500..100  | 0..1 |

   Then the output is
      NArray[ [ [ file0.nc, file1.nc ],
                [ file2.nc, file3.nc ] ] ].

The order of the elements of the input array is arbitrary.


588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 588

def __files_dim_matching( files, varname )
	# files: NArray of NetCDF

	return files if files.length == 1

	# < read the first file and check its rank >

	file0 = files[0]
	ncvar0 = file0.var(varname)
	if files.rank > ncvar0.rank
	  raise ArgumentError, "rank of files > rank of data"
	end
	convention = NetCDF_Conventions.find( ncvar0.file )
	axposnames = convention::coord_var_names(ncvar0)

	#< find correspoding dimensions >

  rank = axposnames.length
  cvals = Array.new(rank){ Array.new }
  sign = Array.new(rank)
  units = Array.new(rank)
  files.each do |file|
    axposnames.each_with_index do |axposname, i|
      nvax = file.var(axposname)
      raise "No coordinate variable: #{axposname}' not found" if !nvax
      axpos = VArrayNetCDF.new( nvax )
      axv0 = axpos.val[0]
      if units[i]
        un = axpos.units
        axv0 = un.convert2(axv0,units[i]) unless un == units[i]
      else # the first file
        units[i] = axpos.units
        sign[i] = (axpos.val[-1] >= axv0 ? 1 : -1)
      end
      cvals[i].push axv0
    end
  end
  # For the above example,
  # cvals = [ [  -90,    0, -90,   0 ],
  #           [ 1000, 1000, 500, 500 ],
  #           [    0,    0,   0,   0 ] ]
  # sign =  [ 1, -1, 1 ]

  axs = Array.new(rank)
  cvals.each_with_index do |cval, i|
    axs[i] = cval.uniq
    axs[i].sort!{|a,b| (a <=> b)*sign[i] }
  end
  # For the above example,
  # axs   = [ [ -90, 0 ], [ 1000, 500 ], [ 0 ] ]
  files_new = NArray.object( *axs.map{|ax| ax.length} )
  # For the above example,
  # shape of the files_new is [2, 2, 1]
  files.length.times do |n|
    file = files[n]
    idx = Array.new(rank)
    rank.times do |i|
      axv0 = cvals[i][n]
      idx[i] = axs[i].index(axv0)
    end
    # For the above example,
    # idx =
    #       [ 0, 0, 0 ] for n==0
    #       [ 1, 0, 0 ] for n==1
    #       [ 0, 1, 0 ] for n==2
    #       [ 1, 1, 0 ] for n==3
    files_new[*idx] = file
  end
  if files_new.eq(nil).count_true > 0
    raise "No dimension correspondence found"
  end
  files = files_new

	files
end

.__interpret_files(files, varname) ⇒ Object



551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 551

def __interpret_files( files, varname )
	case files
	when NetCDF
	  ncvar0 = files.var( varname )
	when String
	  files = NetCDF.open(files)
	  ncvar0 = files.var( varname )
	when NArray, Array, Regexp
	  files = NArray[ *files ] if files.is_a?(Array)
	  files = GPhys::NetCDF_IO.__to_na_of_netcdf( files )
	  files = GPhys::NetCDF_IO.__files_dim_matching( files, varname )
	  ncvar0 = files[0].var( varname )
	else
	  raise TypeError, "argument files: not a NetCDF, String, NArray, Array, or Regexp"
	end
	[files, ncvar0]
end

.__to_na_of_netcdf(files) ⇒ Object



664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 664

def __to_na_of_netcdf( files )
	case files
	when NArray
	  case files[0]
	  when NetCDF
	    files.each{|f|
 raise TypeError, "non-NetCDF obj included" if !f.is_a?(NetCDF)
	    }
	  when String
	    files.each{|f|
 raise TypeError, "non-String obj included" if !f.is_a?(String)
	    }
	    for i in 0...files.length
 files[i] = NetCDF.open(files[i])
	    end
	  end
	when Regexp
	  pat = files
	  if  /^(.*)\\?\/(.*)$/ =~ (pat.source)
	    d=$1
	    f=$2
	    dir = d.gsub(/\\/,'') + '/'
	    pat = Regexp.new(f)
	  else
	    dir = './'
	  end
	  match_data = Array.new
	  fnames = Array.new
	  first_time = true
	  Dir.open(dir).each{|fn| 
	    if pat =~ fn
 fnames.push(fn)
 ary = Regexp.last_match.to_a
 ary.shift  # ==> ary == [$1,$2,...]
        ary.each_with_index{|v,i|
		match_data[i] = Array.new if !match_data[i]
		match_data[i].push(v) if !match_data[i].include?(v)
 }
 if first_time
		body = fn

		first_time = false
 end
	    end
	  }
	  if match_data.length == 0
	    raise ArgumentError, "found no files that matches #{files.source}"
	  end
	  match_data.each{|ary| ary.sort!}
	  shape = match_data.collect{|ary| ary.length}
	  files = NArray.object(*shape)
	  fnames.each{|fn|
	    pat =~ fn
	    ary = Regexp.last_match.to_a
	    ary.shift  # ==> ary == [$1,$2,...]
      index = Array.new        
      ary.each_with_index{|v,i|
 index[i] = match_data[i].index(v)
	    }
	    files[*index] = NetCDF.open(dir+fn)
	  }
	else
	  raise TypeError, "unsupported type #{pat.class}"
	end
	files
end

.def_var(file, name, ntype, dimensions, vary = nil) ⇒ Object



439
440
441
# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 439

def def_var(file, name, ntype, dimensions, vary=nil)
	VArrayNetCDF.def_var(file, name, ntype, dimensions, vary)
end

.each_along_dims_write(gphyses, files, *loopdims, &block) ⇒ Object



457
458
459
460
461
462
463
464
# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 457

def each_along_dims_write(gphyses, files, *loopdims, &block)
	files = [files] if !files.is_a?(Array)
	files.each do |fl|
	  NetCDF_Conventions.add_history(fl, "#{File.basename($0)}")
	end
	IO_Common::each_along_dims_write(gphyses, files, loopdims, NetCDF_IO,
&block)
end

.is_a_NetCDF?(filename) ⇒ Boolean

Returns:

  • (Boolean)


304
305
306
307
308
309
310
311
312
313
# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 304

def is_a_NetCDF?(filename)
  file = nil
  begin
    file = File.open(filename,"rb")
    str = file.read(3)
  ensure
    file.close if file
  end
  return str=="CDF"
end

.open(files, varname) ⇒ Object



315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 315

def open(files, varname)
	files, ncvar0 = __interpret_files( files, varname )
	data = __files2varray( files, varname )
	convention = NetCDF_Conventions.find( ncvar0.file )
	axposnames = convention::coord_var_names( ncvar0 )
	rank = data.rank
	bare_index = [ false ] * rank # will be true if coord var is not found

  axes = Array.new
	var_names = ncvar0.file.var_names
	for i in 0...rank
	  if var_names.include?(axposnames[i])
	    axpos = __files2varray( files, axposnames[i], i )
	  else
	    bare_index[i]=true
	    na = NArray.float(ncvar0.shape_current[i]).indgen!
	    axpos = VArray.new( na )
      axpos.name = axposnames[i]         # added by S. OTSUKA
	  end
	  cell_center, cell_bounds_name = convention::cell_center?( axpos )
	  cell_bounds, cell_center_name = convention::cell_bounds?( axpos )
	  cell = cell_center || cell_bounds
	  axis = Axis.new(cell,bare_index[i])
	  if !cell
	    axis.set_pos( axpos )
	  else
	    if cell_center
 if cell_bounds_name
		varray_cell_bounds = __files2varray(files, cell_bounds_name, i)
		axis.set_cell(axpos, varray_cell_bounds).set_pos_to_center
 else
		p "cell bounds are guessed"
		axis.set_cell_guess_bounds(axpos).set_pos_to_center
 end
	    else  # then it is cell_bounds
 if cell_center_name
		varray_cell_center = __files2varray(files, cell_center_name, i)
		axis.set_cell(varray_cell_center, axpos).set_pos_to_bounds
 else
		p "cell center is guessed"
		axis.set_cell_guess_center(axpos).set_pos_to_bounds
 end
	    end
	  end
	  
	  aux = convention::aux_var_names( axpos )
	  if aux
	    aux.each{|k,varname| 
 axis.set_aux( k, __files2varray(files,varname,i) )
	    }
	  end
	  
	  axes[i] = axis
	end

	grid = Grid.new( *axes )

  if (nms = convention::assoc_coord_names(data)) && !(nms.include?(varname))
    if files.is_a?(NArray)
      assoc_crds = nms.collect{|nm| 
        index_files_assoc = [0]*(files.rank) 
        tmp_assoc = GPhys::NetCDF_IO.open(files[0],nm)
        tmp_assoc.rank.times{|i_assoc|
          axes.length.times{|i_axes|
            if tmp_assoc.axis(i_assoc).name == axes[i_axes].name
              index_files_assoc[i_axes] = true
            end
          }
        }
        GPhys::NetCDF_IO.open(files[*(index_files_assoc)],nm)
      }
    else
      assoc_crds = nms.collect{|nm| GPhys::NetCDF_IO.open(files,nm)}
    end
    grid.set_assoc_coords(assoc_crds)
  end

	GPhys.new(grid,data)
end

.var_names(files) ⇒ Object



466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 466

def var_names(files)
	case files
	when NetCDF
    file = files
	  opened = true
	when String
	  file = NetCDF.open(files)
	  opened = false
	when NArray, Array, Regexp
	  files = NArray[ *files ] if files.is_a?(Array)
	  files = GPhys::NetCDF_IO.__to_na_of_netcdf( files )
	  files = GPhys::NetCDF_IO.__files_dim_matching( files, varname )
	  file = files[0]
	  files.length.times{|i|
	    files[i].close unless i==0
    }
	  opened = false
	else
	  raise TypeError, "argument files: not a NetCDF, String, NArray, Array, or Regexp"
	end
	var_names = file.var_names
	file.close unless opened
	return var_names
end

.var_names_except_coordinates(files) ⇒ Object



490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 490

def var_names_except_coordinates(files)
	case files
	when NetCDF
    file = files
	  opened = true
	when String
	  file = NetCDF.open(files)
	  opened = false
	when NArray, Array, Regexp
	  files = NArray[ *files ] if files.is_a?(Array)
	  files = GPhys::NetCDF_IO.__to_na_of_netcdf( files )
	  files = GPhys::NetCDF_IO.__files_dim_matching( files, varname )
	  file = files[0]
	  files.length.times{|i|
	    files[i].close unless i==0
    }
	  opened = false
	else
	  raise TypeError, "argument files: not a NetCDF, String, NArray, Array, or Regexp"
	end
  var_names = Array.new
  file.var_names.each{|name|
    f,var = __interpret_files( files, name )
    if var.rank>1 || var.name!=var.dim_names[0]
      var_names.push(name)
    end
  }
	file.close unless opened
  return var_names
end

.write(file, gphys, name = nil) ⇒ Object



443
444
445
446
447
448
449
450
451
452
453
454
455
# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 443

def write(file, gphys, name=nil)
  write_grid(file, gphys)
	NetCDF_Conventions.add_history(file, "#{File.basename($0)} wrote "+gphys.name)
  data = gphys.data
  if gphys.has_assoccoord?
    if data.class != VArray
      data = data.copy
    end
    data.put_att("coordinates",gphys.assoccoordnames.join(' '))
  end
	VArrayNetCDF.write(file, data, name, gphys.axnames)
	nil
end

.write_grid(file, grid_or_gphys) ⇒ Object



395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 395

def write_grid(file, grid_or_gphys)
	newaxes = Array.new
	(0...(grid_or_gphys.rank)).each{|i|
	  ax = grid_or_gphys.axis(i)
	  dimname = ax.pos.name
	  length = ax.pos.length
	  isfx = 0
	  altdimnames = Hash.new
	  newax = ax.collect{ |va|
	    if va.length == length
 dimnames = [dimname]
	    else
 if (nm=altdimnames[va.length])
		dimnames = [nm]
 else
		dimnames = [ (altdimnames[va.length] = dimname+isfx.to_s) ]
		isfx += 1
 end
	    end
	    if !(already=file.var(va.name))
 newva = VArrayNetCDF.write(file, va, nil, dimnames )
	    else
        #< var with the same name exists --> check it >
        if va.shape_current != already.shape_current
		raise "#{va.name} exisits but the shape is different" 
 #elsif va[0].val != already[0].val #(not allowed in the def mode)
 #	raise "#{va.name} exisits but the first element doesn't agree" 
        else
		newva = VArrayNetCDF.new(already)
 end
	    end
	    newva
	  }
	  newaxes.push( newax )
	}
  if acnms = grid_or_gphys.assoccoordnames
    acnms.each{|nm| 
      acgp = grid_or_gphys.assoc_coord_gphys(nm)
      VArrayNetCDF.write(file, acgp.data, nil, acgp.axnames)
    }
  end
	Grid.new( *newaxes )
end