Module: NumRu::GPhys::NetCDF_IO

Defined in:

lib/numru/gphys/gphys_netcdf_io.rb

Class Method Summary

• .__files2varray(files, varname, dim = nil) ⇒ Object
• .__files_dim_matching(files, varname) ⇒ Object

  – re-arrange the array of files and check some inconsistencies.

• .__interpret_files(files, varname) ⇒ Object
• .__to_na_of_netcdf(files) ⇒ Object
• .def_var(file, name, ntype, dimensions, vary = nil) ⇒ Object
• .each_along_dims_write(gphyses, files, *loopdims, &block) ⇒ Object
• .is_a_NetCDF?(filename) ⇒ Boolean
• .open(files, varname) ⇒ Object
• .var_names(files) ⇒ Object
• .var_names_except_coordinates(files) ⇒ Object
• .write(file, gphys, name = nil) ⇒ Object
• .write_grid(file, grid_or_gphys) ⇒ Object

Class Method Details

.__files2varray(files, varname, dim = nil) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 524

def __files2varray( files, varname, dim=nil )
  if files.is_a?(NetCDF)
    # Single file. Returns a VArrayNetCDF. dim is ignored.
    file = files
    var = file.var( varname )
    raise "variable '#{varname}' not found in #{file.inspect}" if !var
    VArrayNetCDF.new( var )
  elsif files.is_a?(NArray)
    # Suppose that files is a NArray of NetCDF. Returns a VArrayCompsite.
    if dim.is_a?(Integer) && dim>=0 && dim<files.rank
      files = files[ *([0]*dim+[true]+[0]*(files.rank-dim-1)) ]
    end
    varys = NArray.object( *files.shape )
    for i in 0...files.length
      var = files[i].var( varname )
      raise "variable '#{varname}' not found in #{files[i].path}" if !var
      varys[i] = VArrayNetCDF.new( var )
    end
    if files.length != 1
      VArrayComposite.new( varys )
    else
      varys[0]
    end
  else
    raise TypeError, "not a NetCDF or NArray"
  end
end

.__files_dim_matching(files, varname) ⇒ Object

–

re-arrange the array of files and check some inconsistencies

e.g.)
   files = NArray[ file0.nc, file1.nc, file2.nc, file3.nc ]
   The files have a variable, u(lat,lev,time),
   which is tiled to four parts as the followings:
              |    lat  |     lev    | time |
     file0.nc | -90...0 | 1000...500 | 0..1 |
     file1.nc |   0..90 | 1000...500 | 0..1 |
     file2.nc | -90...0 |  500..100  | 0..1 |
     file3.nc |   0..90 |  500..100  | 0..1 |

   Then the output is
      NArray[ [ [ file0.nc, file1.nc ],
                [ file2.nc, file3.nc ] ] ].

The order of the elements of the input array is arbitrary.

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 589

def __files_dim_matching( files, varname )
  # files: NArray of NetCDF

  return files if files.length == 1

  # < read the first file and check its rank >

  file0 = files[0]
  ncvar0 = file0.var(varname)
  if files.rank > ncvar0.rank
    raise ArgumentError, "rank of files > rank of data"
  end
  convention = NetCDF_Conventions.find( ncvar0.file )
  axposnames = convention::coord_var_names(ncvar0)

  #< find correspoding dimensions >

  rank = axposnames.length
  cvals = Array.new(rank){ Array.new }
  sign = Array.new(rank)
  units = Array.new(rank)
  files.each do |file|
    axposnames.each_with_index do |axposname, i|
      nvax = file.var(axposname)
      raise "No coordinate variable: #{axposname}' not found" if !nvax
      axpos = VArrayNetCDF.new( nvax )
      axv0 = axpos.val[0]
      if units[i]
        un = axpos.units
        axv0 = un.convert2(axv0,units[i]) unless un == units[i]
      else # the first file
        units[i] = axpos.units
        sign[i] = (axpos.val[-1] >= axv0 ? 1 : -1)
      end
      cvals[i].push axv0
    end
  end
  # For the above example,
  # cvals = [ [  -90,    0, -90,   0 ],
  #           [ 1000, 1000, 500, 500 ],
  #           [    0,    0,   0,   0 ] ]
  # sign =  [ 1, -1, 1 ]

  axs = Array.new(rank)
  cvals.each_with_index do |cval, i|
    axs[i] = cval.uniq
    axs[i].sort!{|a,b| (a <=> b)*sign[i] }
  end
  # For the above example,
  # axs   = [ [ -90, 0 ], [ 1000, 500 ], [ 0 ] ]
  files_new = NArray.object( *axs.map{|ax| ax.length} )
  # For the above example,
  # shape of the files_new is [2, 2, 1]
  files.length.times do |n|
    file = files[n]
    idx = Array.new(rank)
    rank.times do |i|
      axv0 = cvals[i][n]
      idx[i] = axs[i].index(axv0)
    end
    # For the above example,
    # idx =
    #       [ 0, 0, 0 ] for n==0
    #       [ 1, 0, 0 ] for n==1
    #       [ 0, 1, 0 ] for n==2
    #       [ 1, 1, 0 ] for n==3
    files_new[*idx] = file
  end
  if files_new.eq(nil).count_true > 0
    raise "No dimension correspondence found"
  end
  files = files_new

  files
end

.__interpret_files(files, varname) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 552

def __interpret_files( files, varname )
  case files
  when NetCDF
    ncvar0 = files.var( varname )
  when String
    files = NetCDF.open(files)
    ncvar0 = files.var( varname )
  when NArray, Array, Regexp
    files = NArray[ *files ] if files.is_a?(Array)
    files = GPhys::NetCDF_IO.__to_na_of_netcdf( files )
    files = GPhys::NetCDF_IO.__files_dim_matching( files, varname )
    ncvar0 = files[0].var( varname )
  else
    raise TypeError, "argument files: not a NetCDF, String, NArray, Array, or Regexp"
  end
  [files, ncvar0]
end

.__to_na_of_netcdf(files) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 665

def __to_na_of_netcdf( files )
  case files
  when NArray
    case files[0]
    when NetCDF
      files.each{|f|
 raise TypeError, "non-NetCDF obj included" if !f.is_a?(NetCDF)
      }
    when String
      files.each{|f|
 raise TypeError, "non-String obj included" if !f.is_a?(String)
      }
      for i in 0...files.length
 files[i] = NetCDF.open(files[i])
      end
    end
  when Regexp
    pat = files
    if  /^(.*)\\?\/(.*)$/ =~ (pat.source)
      d=$1
      f=$2
      dir = d.gsub(/\\/,'') + '/'
      pat = Regexp.new(f)
    else
      dir = './'
    end
    match_data = Array.new
    fnames = Array.new
    first_time = true
    Dir.open(dir).each{|fn| 
      if pat =~ fn
 fnames.push(fn)
 ary = Regexp.last_match.to_a
 ary.shift  # ==> ary == [$1,$2,...]
        ary.each_with_index{|v,i|
    match_data[i] = Array.new if !match_data[i]
    match_data[i].push(v) if !match_data[i].include?(v)
 }
 if first_time
    body = fn

    first_time = false
 end
      end
    }
    if match_data.length == 0
      raise ArgumentError, "found no files that matches #{files.source}"
    end
    match_data.each{|ary| ary.sort!}
    shape = match_data.collect{|ary| ary.length}
    files = NArray.object(*shape)
    fnames.each{|fn|
      pat =~ fn
      ary = Regexp.last_match.to_a
      ary.shift  # ==> ary == [$1,$2,...]
      index = Array.new        
      ary.each_with_index{|v,i|
 index[i] = match_data[i].index(v)
      }
      files[*index] = NetCDF.open(dir+fn)
    }
  else
    raise TypeError, "unsupported type #{pat.class}"
  end
  files
end

.def_var(file, name, ntype, dimensions, vary = nil) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 440

def def_var(file, name, ntype, dimensions, vary=nil)
  VArrayNetCDF.def_var(file, name, ntype, dimensions, vary)
end

.each_along_dims_write(gphyses, files, *loopdims, &block) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 458

def each_along_dims_write(gphyses, files, *loopdims, &block)
  files = [files] if !files.is_a?(Array)
  files.each do |fl|
    NetCDF_Conventions.add_history(fl, "#{File.basename($0)}")
  end
  IO_Common::each_along_dims_write(gphyses, files, loopdims, NetCDF_IO,
&block)
end

.is_a_NetCDF?(filename) ⇒ Boolean

Returns:

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 304

def is_a_NetCDF?(filename)
  file = nil
  begin
    file = File.open(filename,"rb")
    str = file.read(3)
  ensure
    file.close if file
  end
  return str=="CDF"
end

.open(files, varname) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 315

def open(files, varname)
  files, ncvar0 = __interpret_files( files, varname )
  data = __files2varray( files, varname )
  convention = NetCDF_Conventions.find( ncvar0.file )
  axposnames = convention::coord_var_names( ncvar0 )
  rank = data.rank
  bare_index = [ false ] * rank # will be true if coord var is not found

  axes = Array.new
  var_names = ncvar0.file.var_names
  for i in 0...rank
    if var_names.include?(axposnames[i])
      axpos = __files2varray( files, axposnames[i], i )
    else
      bare_index[i]=true
      na = NArray.float(ncvar0.shape_current[i]).indgen!
      axpos = VArray.new( na )
      axpos.name = axposnames[i]         # added by S. OTSUKA
    end
    cell_center, cell_bounds_name = convention::cell_center?( axpos )
    cell_bounds, cell_center_name = convention::cell_bounds?( axpos )
    cell = cell_center || cell_bounds
    axis = Axis.new(cell,bare_index[i])
    if !cell
      axis.set_pos( axpos )
    else
      if cell_center
 if cell_bounds_name
    varray_cell_bounds = __files2varray(files, cell_bounds_name, i)
    axis.set_cell(axpos, varray_cell_bounds).set_pos_to_center
 else
    p "cell bounds are guessed"
    axis.set_cell_guess_bounds(axpos).set_pos_to_center
 end
      else  # then it is cell_bounds
 if cell_center_name
    varray_cell_center = __files2varray(files, cell_center_name, i)
    axis.set_cell(varray_cell_center, axpos).set_pos_to_bounds
 else
    p "cell center is guessed"
    axis.set_cell_guess_center(axpos).set_pos_to_bounds
 end
      end
    end
    
    aux = convention::aux_var_names( axpos )
    if aux
      aux.each{|k,varname| 
 axis.set_aux( k, __files2varray(files,varname,i) )
      }
    end
    
    axes[i] = axis
  end

  grid = Grid.new( *axes )

  if (nms = convention::assoc_coord_names(data)) && !(nms.include?(varname))
    if files.is_a?(NArray)
      assoc_crds = nms.collect{|nm| 
        index_files_assoc = [0]*(files.rank) 
        tmp_assoc = GPhys::NetCDF_IO.open(files[0],nm)
        tmp_assoc.rank.times{|i_assoc|
          axes.length.times{|i_axes|
            if tmp_assoc.axis(i_assoc).name == axes[i_axes].name
              index_files_assoc[i_axes] = true
            end
          }
        }
        GPhys::NetCDF_IO.open(files[*(index_files_assoc)],nm)
      }
    else
      assoc_crds = nms.collect{|nm| GPhys::NetCDF_IO.open(files,nm)}
    end
    grid.set_assoc_coords(assoc_crds)
  end

  GPhys.new(grid,data)
end

.var_names(files) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 467

def var_names(files)
  case files
  when NetCDF
    file = files
    opened = true
  when String
    file = NetCDF.open(files)
    opened = false
  when NArray, Array, Regexp
    files = NArray[ *files ] if files.is_a?(Array)
    files = GPhys::NetCDF_IO.__to_na_of_netcdf( files )
    files = GPhys::NetCDF_IO.__files_dim_matching( files, varname )
    file = files[0]
    files.length.times{|i|
      files[i].close unless i==0
    }
    opened = false
  else
    raise TypeError, "argument files: not a NetCDF, String, NArray, Array, or Regexp"
  end
  var_names = file.var_names
  file.close unless opened
  return var_names
end

.var_names_except_coordinates(files) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 491

def var_names_except_coordinates(files)
  case files
  when NetCDF
    file = files
    opened = true
  when String
    file = NetCDF.open(files)
    opened = false
  when NArray, Array, Regexp
    files = NArray[ *files ] if files.is_a?(Array)
    files = GPhys::NetCDF_IO.__to_na_of_netcdf( files )
    files = GPhys::NetCDF_IO.__files_dim_matching( files, varname )
    file = files[0]
    files.length.times{|i|
      files[i].close unless i==0
    }
    opened = false
  else
    raise TypeError, "argument files: not a NetCDF, String, NArray, Array, or Regexp"
  end
  var_names = Array.new
  file.var_names.each{|name|
    f,var = __interpret_files( files, name )
    if var.rank>1 || var.name!=var.dim_names[0]
      var_names.push(name)
    end
  }
  file.close unless opened
  return var_names
end

.write(file, gphys, name = nil) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 444

def write(file, gphys, name=nil)
  write_grid(file, gphys)
  NetCDF_Conventions.add_history(file, "#{File.basename($0)} wrote "+gphys.name)
  data = gphys.data
  if gphys.has_assoccoord?
    if data.class != VArray
      data = data.copy
    end
    data.put_att("coordinates",gphys.assoccoordnames.join(' '))
  end
  VArrayNetCDF.write(file, data, name, gphys.axnames)
  nil
end

.write_grid(file, grid_or_gphys) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 395

def write_grid(file, grid_or_gphys)
  newaxes = Array.new
  (0...(grid_or_gphys.rank)).each{|i|
    ax = grid_or_gphys.axis(i)
    dimname = ax.pos.name
    length = ax.pos.length
    isfx = 0
    altdimnames = Hash.new
    newax = ax.collect{ |va|
      if va.length == length
 dimnames = [dimname]
      else
 if (nm=altdimnames[va.length])
    dimnames = [nm]
 else
    dimnames = [ (altdimnames[va.length] = dimname+isfx.to_s) ]
    isfx += 1
 end
      end
      if !(already=file.var(va.name))
 newva = VArrayNetCDF.write(file, va, nil, dimnames )
      else
        #< var with the same name exists --> check it >
        if va.shape_current != already.shape_current
    raise "#{va.name} exisits but the shape is different" 
 #elsif va[0].val != already[0].val #(not allowed in the def mode)
 #  raise "#{va.name} exisits but the first element doesn't agree" 
        else
    newva = VArrayNetCDF.new(already)
 end
      end
      newva
    }
    newaxes.push( newax )
  }
  if acnms = grid_or_gphys.assoccoordnames
    (nms = file.var_names) and acnms -= nms
    acnms.each{|nm| 
      acgp = grid_or_gphys.assoc_coord_gphys(nm)
      VArrayNetCDF.write(file, acgp.data, nil, acgp.axnames)
    }
  end
  Grid.new( *newaxes )
end