Module: Hdfs

Includes:
Java
Defined in:
lib/hdfs_jruby.rb,
lib/hdfs_jruby/file.rb,
lib/hdfs_jruby/version.rb

Defined Under Namespace

Classes: Configuration, File, FileSystem, FsPermission, Path

Constant Summary collapse

JAR_PATTERN_0_20 = 
"hadoop-core-*.jar"
HADOOP_HOME = 
ENV["HADOOP_HOME"]
VERSION = 
"0.0.3"

Class Method Summary collapse

Class Method Details

._conv(stat) ⇒ Object 



169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/hdfs_jruby.rb', line 169

def _conv(stat)
  file_info = {}
  file_info['path'] = stat.getPath.to_s
  file_info['length'] = stat.getLen.to_i
  file_info['modificationTime'] = stat.getModificationTime.to_i
  file_info['owner'] = stat.getOwner.to_s
  file_info['group'] = stat.getGroup.to_s
  file_info['permission'] = stat.getPermission.toShort.to_i
  file_info['type'] = !stat.isDir ? 'FILE': 'DIRECTORY'
  return file_info
end

._path(path) ⇒ Object 



162
163
164
165
166
167
 
# File 'lib/hdfs_jruby.rb', line 162

def _path(path)
  if path.nil?
    raise "path is nil"
  end
  Path.new(path)
end

.delete(path, r = false) ⇒ Object 



96
97
98
 
# File 'lib/hdfs_jruby.rb', line 96

def delete(path, r=false)
  @fs.delete(_path(path), r)
end

.directory?(path) ⇒ Boolean

Returns:

  • (Boolean) 


104
105
106
 
# File 'lib/hdfs_jruby.rb', line 104

def directory?(path)
  @fs.isDirectory(_path(path))
end

.exists?(path) ⇒ Boolean

Returns:

  • (Boolean) 


88
89
90
 
# File 'lib/hdfs_jruby.rb', line 88

def exists?(path)
  @fs.exists(_path(path))
end

.file?(path) ⇒ Boolean

Returns:

  • (Boolean) 


100
101
102
 
# File 'lib/hdfs_jruby.rb', line 100

def file?(path)
  @fs.isFile(_path(path))
end

.get(remote, local) ⇒ Object 



120
121
122
 
# File 'lib/hdfs_jruby.rb', line 120

def get(remote, local)
  @fs.copyToLocalFile(Path.new(remote), Path.new(local))
end

.get_home_directory ⇒ Object 



124
125
126
 
# File 'lib/hdfs_jruby.rb', line 124

def get_home_directory()
  @fs.getHomeDirectory()
end

.get_working_directory ⇒ Object 



128
129
130
 
# File 'lib/hdfs_jruby.rb', line 128

def get_working_directory()
  @fs.getWorkingDirectory()
end

.list(path, opts = {}) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/hdfs_jruby.rb', line 64

def list(path, opts={})
  use_glob = opts[:glob] ? true : false
  p = _path(path)

  list = nil
  if use_glob
    list = @fs.globStatus(p)
  else
    list = @fs.listStatus(p)
  end
    
  if ! block_given?
    ret_list = []
    list.each do | stat |
      ret_list << _conv(stat)
    end
    return ret_list
  else
    list.each do | stat |
      yield _conv(stat)
    end
  end
end

.ls(path) ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/hdfs_jruby.rb', line 39

def ls(path)
  p = _path(path)
  list = @fs.globStatus(p)
  ret_list = []
  list.each do |stat|
    if stat.isDir
      sub_list = @fs.listStatus(stat.getPath)
      sub_list.each do | s |
        if block_given?
          yield _conv(s)
        else
          ret_list << _conv(s)
        end
      end
    else
      if block_given?
        yield _conv(stat)
      else
        ret_list << _conv(stat)
      end
    end
  end
  ret_list if ! block_given?
end

.mkdir(path) ⇒ Object 



112
113
114
 
# File 'lib/hdfs_jruby.rb', line 112

def mkdir(path)
  @fs.mkdirs(_path(path))
end

.move(src, dst) ⇒ Object 



92
93
94
 
# File 'lib/hdfs_jruby.rb', line 92

def move(src, dst)
  @fs.rename(Path.new(src), Path.new(dst))
end

.put(local, remote) ⇒ Object 



116
117
118
 
# File 'lib/hdfs_jruby.rb', line 116

def put(local, remote)
  @fs.copyFromLocalFile(Path.new(local), Path.new(remote))
end

.set_owner(path, owner, group) ⇒ Object 



140
141
142
 
# File 'lib/hdfs_jruby.rb', line 140

def set_owner(path, owner, group)
  @fs.setOwner(_path(path), owner, group)
end

.set_permission(path, perm) ⇒ Object 



136
137
138
 
# File 'lib/hdfs_jruby.rb', line 136

def set_permission(path, perm)
  @fs.setPermission(_path(path), org.apache.hadoop.fs.permission.FsPermission.new(perm))
end

.set_working_directory(path) ⇒ Object 



132
133
134
 
# File 'lib/hdfs_jruby.rb', line 132

def set_working_directory(path)
  @fs.setWorkingDirectory(_path())
end

.size(path) ⇒ Object 



108
109
110
 
# File 'lib/hdfs_jruby.rb', line 108

def size(path)
  @fs.getFileStatus(_path(path)).getLen()
end