Module: WebHDFS::FileUtils
- Defined in:
- lib/webhdfs/fileutils.rb
Class Method Summary collapse
-
.append(path, body, options = {}) ⇒ Object
Public: Append to HDFS file.
-
.chmod(mode, list, options = {}) ⇒ Object
Public: Change permission of one or more directories/files.
-
.chown(user, group, list, options = {}) ⇒ Object
Public: Change an ownership of one or more directories/files.
-
.copy_from_local(file, path, options = {}) ⇒ Object
Public: Copy local file into HDFS.
-
.copy_from_local_via_stream(file, path, options = {}) ⇒ Object
Public: Copy local file into HDFS with IOStream.
-
.copy_to_local(path, file, options = {}) ⇒ Object
Public: Copy remote HDFS file into local.
-
.mkdir(list, options = {}) ⇒ Object
Public: Create one or more directories.
-
.mkdir_p ⇒ Object
Public: Create one or more directories.
-
.private_module_function(name) ⇒ Object
Internal: make functin private.
-
.rename(src, dst, options = {}) ⇒ Object
Public: Rename a file or directory.
-
.rm(list, options = {}) ⇒ Object
Public: Remove one or more directories or files.
-
.rmr(list, options = {}) ⇒ Object
Public: Remove one or more directories/files recursively.
-
.set_atime(list, time, options = {}) ⇒ Object
Public: Set an access time of files.
-
.set_httpfs_mode(mode = true) ⇒ Object
Public: Set httpfs mode enable/disable.
-
.set_kerberos(mode = true) ⇒ Object
Public: Set kerberos authentication enable/disable.
-
.set_mtime(list, time, options = {}) ⇒ Object
Public: Set a modification time of files.
-
.set_repl_factor(list, num, options = {}) ⇒ Object
Public: Set a replication factor of files.
-
.set_server(host, port, user = nil, doas = nil, proxy_address = nil, proxy_port = nil) ⇒ Object
Public: Set hostname and port number of WebHDFS.
-
.set_ssl(mode = true) ⇒ Object
Public: Set ssl enable/disable.
-
.set_ssl_ca_file(ca_file) ⇒ Object
Public: Set ssl ca_file.
-
.set_ssl_verify_mode(mode) ⇒ Object
Public: Set ssl verify mode.
Instance Method Summary collapse
-
#client ⇒ Object
Internal.
-
#fu_log(msg) ⇒ Object
Internal: Logging.
Class Method Details
.append(path, body, options = {}) ⇒ Object
Public: Append to HDFS file
path - HDFS file path body - contents options - :buffersize, :verbose
Examples
FileUtils.append 'remote_path', 'contents'
183 184 185 186 187 |
# File 'lib/webhdfs/fileutils.rb', line 183 def append(path, body, ={}) opts = .dup fu_log "append #{body.bytesize} bytes to #{path}" if opts.delete(:verbose) client.append(path, body, opts) end |
.chmod(mode, list, options = {}) ⇒ Object
297 298 299 300 301 302 303 304 305 306 |
# File 'lib/webhdfs/fileutils.rb', line 297 def chmod(mode, list, ={}) opts = .dup list = [list].flatten fu_log sprintf('chmod %o %s', mode, list.join(' ')) if opts.delete(:verbose) mode = ('%03o' % mode) if mode.is_a? Integer c = client list.each { |entry| c.chmod(entry, mode, opts) } end |
.chown(user, group, list, options = {}) ⇒ Object
321 322 323 324 325 326 327 328 329 330 331 |
# File 'lib/webhdfs/fileutils.rb', line 321 def chown(user, group, list, ={}) opts = .dup list = [list].flatten fu_log sprintf('chown %s%s', [user,group].compact.join(':') + ' ', list.join(' ')) if opts.delete(:verbose) c = client list.each { |entry| c.chown(entry, {:owner => user, :group => group}) } end |
.copy_from_local(file, path, options = {}) ⇒ Object
Public: Copy local file into HDFS
file - local file path path - HDFS file path options - :overwrite, :blocksize, :replication, :mode, :buffersize, :verbose
Examples
FileUtils.copy_from_local 'local_file', 'remote_file'
114 115 116 117 118 119 120 121 122 123 124 125 126 |
# File 'lib/webhdfs/fileutils.rb', line 114 def copy_from_local(file, path, ={}) opts = .dup fu_log "copy_from_local local=#{file} hdfs=#{path}" if opts.delete(:verbose) if mode = opts.delete(:mode) mode = ('%03o' % mode) if mode.is_a? Integer else mode = '644' end opts[:permission] = mode opts[:overwrite] ||= true client.create(path, File.new(file, 'rb').read(File.size(file)), opts) end |
.copy_from_local_via_stream(file, path, options = {}) ⇒ Object
Public: Copy local file into HDFS with IOStream
file - local file IO handle path - HDFS file path options - :overwrite, :blocksize, :replication, :mode, :buffersize, :verbose
Examples
FileUtils.copy_from_local_via_stream 'local_file_IO_handle', 'remote_file'
139 140 141 142 143 144 145 146 147 148 149 150 151 |
# File 'lib/webhdfs/fileutils.rb', line 139 def copy_from_local_via_stream(file, path, ={}) opts = .dup fu_log "copy_from_local_via_stream local=#{file} hdfs=#{path}" if opts.delete(:verbose) if mode = opts.delete(:mode) mode = ('%03o' % mode) if mode.is_a? Integer else mode = '644' end opts[:permission] = mode opts[:overwrite] ||= true client.create(path, File.new(file, 'rb'), opts) end |
.copy_to_local(path, file, options = {}) ⇒ Object
Public: Copy remote HDFS file into local
path - HDFS file path file - local file path options - :offset, :length, :buffersize, :verbose
Examples
FileUtils.copy_to_local 'remote_file', 'local_file'
164 165 166 167 168 169 170 |
# File 'lib/webhdfs/fileutils.rb', line 164 def copy_to_local(path, file, ={}) opts = .dup fu_log "copy_to_local hdfs=#{path} local=#{file}" if opts.delete(:verbose) File.open(file, "wb") do |f| f.write client.read(path, opts) end end |
.mkdir(list, options = {}) ⇒ Object
201 202 203 204 205 206 207 208 209 210 211 212 213 214 |
# File 'lib/webhdfs/fileutils.rb', line 201 def mkdir(list, ={}) opts = .dup list = [list].flatten fu_log "mkdir #{[:mode] ? ('-m %03o ' % [:mode]) : ''}#{list.join ' '}" if opts.delete(:verbose) if mode = opts[:mode] mode = ('0%03o' % mode) if mode.is_a? Integer else mode = '0755' end c = client list.each { |dir| c.mkdir(dir, {:permission => mode}) } end |
.mkdir_p ⇒ Object
Public: Create one or more directories.
list - directory name, or list of them options - :mode, :verbose
Examples
FileUtils.mkdir 'test'
FileUtils.mkdir %w( tmp data )
FileUtils.mkdir 'tmp', :mode => 0700
Public: Create one or more directories recursively.
list - directory name, or list of them options - :mode, :verbose
Examples
FileUtils.mkdir_p 'dir/subdir'
FileUtils.mkdir_p %w( tmp data )
FileUtils.mkdir_p 'dir/subdir', :mode => 0700
228 229 230 231 232 233 234 235 236 237 238 239 240 241 |
# File 'lib/webhdfs/fileutils.rb', line 228 def mkdir(list, ={}) opts = .dup list = [list].flatten fu_log "mkdir #{[:mode] ? ('-m %03o ' % [:mode]) : ''}#{list.join ' '}" if opts.delete(:verbose) if mode = opts[:mode] mode = ('0%03o' % mode) if mode.is_a? Integer else mode = '0755' end c = client list.each { |dir| c.mkdir(dir, {:permission => mode}) } end |
.private_module_function(name) ⇒ Object
Internal: make functin private
401 402 403 404 |
# File 'lib/webhdfs/fileutils.rb', line 401 def self.private_module_function(name) module_function name private_class_method name end |
.rename(src, dst, options = {}) ⇒ Object
Public: Rename a file or directory.
src - from dst - to options - :verbose
Examples
FileUtils.rename 'from', 'to'
279 280 281 282 283 |
# File 'lib/webhdfs/fileutils.rb', line 279 def rename(src, dst, ={}) opts = .dup fu_log "rename #{src} #{dst}" if opts.delete(:verbose) client.rename(src, dst, opts) end |
.rm(list, options = {}) ⇒ Object
242 243 244 245 246 247 248 249 250 |
# File 'lib/webhdfs/fileutils.rb', line 242 def rm(list, ={}) opts = .dup list = [list].flatten fu_log "rm #{list.join ' '}" if opts.delete(:verbose) c = client list.each { |dir| c.delete(dir, {:recursive => opts[:recursive] || false}) } end |
.rmr(list, options = {}) ⇒ Object
264 265 266 |
# File 'lib/webhdfs/fileutils.rb', line 264 def rmr(list, ={}) self.rm(list, .merge({:recursive => true})) end |
.set_atime(list, time, options = {}) ⇒ Object
Public: Set an access time of files
list - file/directory name or list of them time - new access time options - :verbose
Examples
FileUtils.set_atime 'file', Time.now
366 367 368 369 370 371 372 373 374 375 |
# File 'lib/webhdfs/fileutils.rb', line 366 def set_atime(list, time, ={}) opts = .dup list = [list].flatten time = time.to_i fu_log sprintf('set_atime %s %d', list.join(' '), time) if opts.delete(:verbose) c = client list.each { |entry| c.touch(entry, {:accesstime => time}) } end |
.set_httpfs_mode(mode = true) ⇒ Object
Public: Set httpfs mode enable/disable
mode - boolean (default true)
Examples
FileUtils.set_httpfs_mode
47 48 49 |
# File 'lib/webhdfs/fileutils.rb', line 47 def set_httpfs_mode(mode=true) @fu_httpfs_mode = mode end |
.set_kerberos(mode = true) ⇒ Object
Public: Set kerberos authentication enable/disable
mode - boolean (default true)
Examples
FileUtils.set_kerberos
99 100 101 |
# File 'lib/webhdfs/fileutils.rb', line 99 def set_kerberos(mode=true) @fu_kerberos = mode end |
.set_mtime(list, time, options = {}) ⇒ Object
Public: Set a modification time of files
list - file/directory name or list of them time - new modification time options - :verbose
Examples
FileUtils.set_mtime 'file', Time.now
388 389 390 391 392 393 394 395 396 397 |
# File 'lib/webhdfs/fileutils.rb', line 388 def set_mtime(list, time, ={}) opts = .dup list = [list].flatten time = time.to_i fu_log sprintf('set_mtime %s %d', list.join(' '), time) if opts.delete(:verbose) c = client list.each { |entry| c.touch(entry, {:modificationtime => time}) } end |
.set_repl_factor(list, num, options = {}) ⇒ Object
Public: Set a replication factor of files
list - file/directory name or list of them num - replication factor options - :verbose
Examples
FileUtils.set_repl_factor 'file', 3
344 345 346 347 348 349 350 351 352 353 |
# File 'lib/webhdfs/fileutils.rb', line 344 def set_repl_factor(list, num, ={}) opts = .dup list = [list].flatten fu_log sprintf('set_repl_factor %s %d', list.join(' '), num) if opts.delete(:verbose) c = client list.each { |entry| c.replication(entry, num, opts) } end |
.set_server(host, port, user = nil, doas = nil, proxy_address = nil, proxy_port = nil) ⇒ Object
Public: Set hostname and port number of WebHDFS
host - hostname port - port user - username doas - proxy user name proxy_address - address of the net http proxy to use proxy_port - port of the net http proxy to use
Examples
FileUtils.set_server 'localhost', 50070
29 30 31 32 33 34 35 36 |
# File 'lib/webhdfs/fileutils.rb', line 29 def set_server(host, port, user=nil, doas=nil, proxy_address=nil, proxy_port=nil) @fu_host = host @fu_port = port @fu_user = user @fu_doas = doas @fu_paddr = proxy_address @fu_pport = proxy_port end |
.set_ssl(mode = true) ⇒ Object
60 61 62 |
# File 'lib/webhdfs/fileutils.rb', line 60 def set_ssl(mode=true) @fu_ssl = mode end |
.set_ssl_ca_file(ca_file) ⇒ Object
73 74 75 |
# File 'lib/webhdfs/fileutils.rb', line 73 def set_ssl_ca_file(ca_file) @fu_ssl_ca_file = ca_file end |
.set_ssl_verify_mode(mode) ⇒ Object
86 87 88 |
# File 'lib/webhdfs/fileutils.rb', line 86 def set_ssl_verify_mode(mode) @fu_ssl_verify_mode = mode end |
Instance Method Details
#client ⇒ Object
Internal
417 418 419 420 421 422 423 424 425 426 427 |
# File 'lib/webhdfs/fileutils.rb', line 417 def client client = WebHDFS::Client.new(@fu_host, @fu_port, @fu_user, @fu_doas, @fu_paddr, @fu_pport) if @fu_httpfs_mode client.httpfs_mode = true end client.ssl = true if @fu_ssl client.ssl_ca_file = @fu_ssl_ca_file if @fu_ssl_ca_file client.ssl_verify_mode = @fu_ssl_verify_mode if @fu_ssl_verify_mode client.kerberos = true if @fu_kerberos client end |
#fu_log(msg) ⇒ Object
Internal: Logging
409 410 411 412 413 |
# File 'lib/webhdfs/fileutils.rb', line 409 def fu_log(msg) @fileutils_output ||= $stderr @fileutils_label ||= '' @fileutils_output.puts @fileutils_label + msg end |