Module: Open

Defined in:
lib/rbbt/util/open.rb

Defined Under Namespace

Classes: OpenGzipError, OpenURLError

Constant Summary collapse

REMOTE_CACHEDIR = 
"/tmp/open_cache"
LAST_TIME =

Remote WGET

{}

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.repository_dirs ⇒ Object

Returns the value of attribute repository_dirs.



15
16
17
 
# File 'lib/rbbt/util/open.rb', line 15

def repository_dirs
  @repository_dirs
end

Class Method Details

.add_cache(url, data, options = {}) ⇒ Object 



120
121
122
123
 
# File 'lib/rbbt/util/open.rb', line 120

def self.add_cache(url, data, options = {})
  file = File.join(REMOTE_CACHEDIR, digest_url(url, options))
  Misc.sensiblewrite(file, data)
end

.cachedir ⇒ Object 



34
35
36
 
# File 'lib/rbbt/util/open.rb', line 34

def self.cachedir
  REMOTE_CACHEDIR
end

.cachedir=(cachedir) ⇒ Object 



29
30
31
32
 
# File 'lib/rbbt/util/open.rb', line 29

def self.cachedir=(cachedir)
  REMOTE_CACHEDIR.replace cachedir
  FileUtils.mkdir_p REMOTE_CACHEDIR unless File.exist? REMOTE_CACHEDIR
end

.can_open?(file) ⇒ Boolean

Returns:

  • (Boolean) 


399
400
401
 
# File 'lib/rbbt/util/open.rb', line 399

def self.can_open?(file)
  String === file and (File.exists?(file) or remote?(file))
end

.clean_cache(url, options = {}) ⇒ Object

Open Read Write



316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
 
# File 'lib/rbbt/util/open.rb', line 316

def self.clean_cache(url, options = {})
  options = Misc.add_defaults options, :noz => false, :mode => 'r'

  wget_options = options[:wget_options] || {}
  wget_options[:nice] = options.delete(:nice)
  wget_options[:nice_key] = options.delete(:nice_key)
  wget_options[:quiet] = options.delete(:quiet)
  wget_options["--post-data="] = options.delete(:post) if options.include? :post
  wget_options["--post-file"] = options.delete("--post-file") if options.include? "--post-file"
  wget_options["--post-file="] = options.delete("--post-file=") if options.include? "--post-file="
  wget_options[:cookies] = options.delete(:cookies)

  cache_file = in_cache(url, wget_options)
  Misc.lock(cache_file) do
    FileUtils.rm(cache_file)
  end if cache_file
end

.digest_url(url, options = {}) ⇒ Object 



94
95
96
97
 
# File 'lib/rbbt/util/open.rb', line 94

def self.digest_url(url, options = {})
  params = [url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]).split("\n").sort * "\n" : "")]
  digest = Misc.digest(params.inspect)
end

.exists?(file) ⇒ Boolean

Returns:

  • (Boolean) 


263
264
265
266
267
268
269
270
 
# File 'lib/rbbt/util/open.rb', line 263

def self.exists?(file)
  if (dir_sub_path = find_repo_dir(file))
    dir_sub_path.push file
    exists_in_repo(*dir_sub_path)
  else
    File.exists? file
  end
end

.exists_in_repo(dir, sub_path, content) ⇒ Object 



171
172
173
174
175
176
 
# File 'lib/rbbt/util/open.rb', line 171

def self.exists_in_repo(dir, sub_path, content)
  repo = get_repo_from_dir(dir)
  repo.read_and_close do
    repo.include? sub_path
  end
end

.file_open(file, grep, mode = 'r', invert_grep = false) ⇒ Object 



204
205
206
207
208
209
210
211
212
213
214
215
216
 
# File 'lib/rbbt/util/open.rb', line 204

def self.file_open(file, grep, mode = 'r', invert_grep = false)
  if (dir_sub_path = find_repo_dir(file))
    stream = get_stream_from_repo(*dir_sub_path)
  else
    stream =  File.open(file, mode)
  end

  if grep
    grep(stream, grep, invert_grep)
  else
    stream
  end
end

.file_write(file, content, mode = 'w') ⇒ Object 



218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
 
# File 'lib/rbbt/util/open.rb', line 218

def self.file_write(file, content, mode = 'w')
  if (dir_sub_path = find_repo_dir(file))
    dir_sub_path.push content
    save_content_in_repo(*dir_sub_path)
  else
    File.open(file, mode) do |f|
      begin
        f.flock(File::LOCK_EX)
        f.write content 
        f.flock(File::LOCK_UN)
      ensure
        f.close unless f.closed?
      end
    end
  end
end

.find_repo_dir(file) ⇒ Object 



178
179
180
181
182
183
184
185
186
 
# File 'lib/rbbt/util/open.rb', line 178

def self.find_repo_dir(file)
  self.repository_dirs.each do |dir|
    if file.start_with? dir
      sub_path = file.to_s[dir.length..-1]
      return [dir, sub_path]
    end
  end
  nil
end

.get_repo_from_dir(dir) ⇒ Object 



138
139
140
141
142
143
144
 
# File 'lib/rbbt/util/open.rb', line 138

def self.get_repo_from_dir(dir)
  @repos ||= {}
  @repos[dir] ||= begin
                    repo_path = File.join(dir, '.file_repo')
                    Persist.open_tokyocabinet(repo_path, false, :clean,TokyoCabinet::BDB )
                  end
end

.get_stream_from_repo(dir, sub_path) ⇒ Object 



146
147
148
149
150
151
 
# File 'lib/rbbt/util/open.rb', line 146

def self.get_stream_from_repo(dir, sub_path)
  repo = get_repo_from_dir(dir)
  repo.read_and_close do
    StringIO.new repo[sub_path]
  end
end

.grep(stream, grep, invert = false) ⇒ Object

Grep



127
128
129
130
131
132
133
134
135
136
 
# File 'lib/rbbt/util/open.rb', line 127

def self.grep(stream, grep, invert = false)
  case 
  when Array === grep
    TmpFile.with_file(grep * "\n", false) do |f|
      CMD.cmd("grep #{invert ? '-v' : ''}", "-w" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
    end
  else
    CMD.cmd("grep #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{stream.force_close if stream.respond_to? :force_close})
  end
end

.gunzip(stream) ⇒ Object

Decompression



285
286
287
288
289
290
291
 
# File 'lib/rbbt/util/open.rb', line 285

def self.gunzip(stream)
  if String === stream
    Zlib::Inflate.inflate(stream)
  else
    CMD.cmd("gunzip", :pipe => true, :in => stream, :post => proc{stream.force_close if stream.respond_to? :force_close})
  end
end

.gzip?(file) ⇒ Boolean

Returns:

  • (Boolean) 


305
306
307
 
# File 'lib/rbbt/util/open.rb', line 305

def self.gzip?(file)
  !! (file =~ /\.gz$/)
end

.in_cache(url, options = {}) ⇒ Object

Cache



100
101
102
103
104
105
106
107
 
# File 'lib/rbbt/util/open.rb', line 100

def self.in_cache(url, options = {})
  filename = File.join(REMOTE_CACHEDIR, digest_url(url, options))
  if File.exists? filename
    return filename 
  else
    nil
  end
end

.lock(file, options = {}, &block) ⇒ Object 



272
273
274
275
276
277
278
279
280
 
# File 'lib/rbbt/util/open.rb', line 272

def self.lock(file, options = {}, &block)
  if (dir_sub_path = find_repo_dir(file))
    dir, sub_path = dir_sub_path
    repo = get_repo_from_dir(dir)
    Misc.lock_in_repo(repo, sub_path, &block)
  else
    Misc.lock(file, true, options, &block)
  end
end

.mv(source, target) ⇒ Object 



235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
 
# File 'lib/rbbt/util/open.rb', line 235

def self.mv(source, target)
  dir_sub_path_source = find_repo_dir(source)
  dir_sub_path_target = find_repo_dir(target)

  return FileUtils.mv source, target if dir_sub_path_source.nil? and dir_sub_path_target.nil?

  if dir_sub_path_source.nil?
    save_content_in_repo(dir_sub_path_target[0], dir_sub_path_target[1], Open.read(source))
    return nil
  end

  if dir_sub_path_target.nil?
    Open.write(target, get_stream_from_repo(dir_sub_path_source))
    return nil
  end

  repo_source = get_repo_from_dir(dir_sub_path_source[0])
  repo_target = get_repo_from_dir(dir_sub_path_target[0])

  repo_source.write do
    repo_target.write do
      repo_source[dir_sub_path_source[1]] = repo_target[dir_sub_path_target[1]]
    end
  end

  return nil
end

.open(url, options = {}) ⇒ Object 



334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
 
# File 'lib/rbbt/util/open.rb', line 334

def self.open(url, options = {})
  if IO === url
    if block_given?
      res = yield url 
      url.close
      return res
    else
      return url 
    end
  end
  options = Misc.add_defaults options, :noz => false, :mode => 'r'

  mode = Misc.process_options options, :mode

  wget_options = options[:wget_options] || {}
  wget_options[:nice] = options.delete(:nice)
  wget_options[:nice_key] = options.delete(:nice_key)
  wget_options[:quiet] = options.delete(:quiet)
  wget_options["--post-data="] = options.delete(:post) if options.include? :post
  wget_options["--post-file"] = options.delete("--post-file") if options.include? "--post-file"
  wget_options["--post-file="] = options.delete("--post-file=") if options.include? "--post-file="
  wget_options[:cookies] = options.delete(:cookies)

  io = case
       when (IO === url or StringIO === url)
         url
       when (not remote?(url))
         file_open(url, options[:grep], mode, options[:invert_grep])
       when (options[:nocache] and options[:nocache] != :update)
         # What about grep?
         wget(url, wget_options)
       when (options[:nocache] != :update and in_cache(url, wget_options))
         file_open(in_cache(url, wget_options), options[:grep], mode, options[:invert_grep])
       else
         io = wget(url, wget_options)
         add_cache(url, io, wget_options)
         io.close
         file_open(in_cache(url, wget_options), options[:grep], mode, options[:invert_grep])
       end
  io = unzip(io)  if ((String === url and zip?(url))  and not options[:noz]) or options[:zip]
  io = gunzip(io) if ((String === url and gzip?(url)) and not options[:noz]) or options[:gzip]

  if block_given?
    begin
      return yield(io)
    rescue
      io.abort if io.respond_to? :abort
      io.join if io.respond_to? :join
      raise $!
    ensure
      io.join if io.respond_to? :join
    end
  else
    io
  end

  class << io;
    attr_accessor :filename
  end

  io.filename = url.to_s

  io
end

.read(file, options = {}, &block) ⇒ Object 



403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
 
# File 'lib/rbbt/util/open.rb', line 403

def self.read(file, options = {}, &block)
  open(file, options) do |f|
    if block_given?
      res = []
      while not f.eof?
        l = f.gets
        l = Misc.fixutf8(l) 
        res << yield(l)
      end
      res
    else
      Misc.fixutf8(f.read)
    end
  end
end

.remote?(file) ⇒ Boolean

Questions

Returns:

  • (Boolean) 


301
302
303
 
# File 'lib/rbbt/util/open.rb', line 301

def self.remote?(file)
  !! (file =~ /^(?:https?|ftp):\/\//)
end

.remove_from_cache(url, options = {}) ⇒ Object 



109
110
111
112
113
114
115
116
117
118
 
# File 'lib/rbbt/util/open.rb', line 109

def self.remove_from_cache(url, options = {})
  digest = Misc.digest([url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)

  filename = File.join(REMOTE_CACHEDIR, digest)
  if File.exists? filename
    FileUtils.rm filename 
  else
    nil
  end
end

.remove_from_repo(dir, sub_path, recursive = false) ⇒ Object 



160
161
162
163
164
165
166
167
168
169
 
# File 'lib/rbbt/util/open.rb', line 160

def self.remove_from_repo(dir, sub_path, recursive = false)
  repo = get_repo_from_dir(dir)
  repo.write_and_close do
    if recursive
      repo.outlist repo.range sub_path, true, sub_path.sub(/.$/,('\1'.ord + 1).chr), false
    else
      repo.outlist sub_path
    end
  end
end

.rm(file) ⇒ Object 



188
189
190
191
192
193
194
 
# File 'lib/rbbt/util/open.rb', line 188

def self.rm(file)
  if (dir_sub_path = find_repo_dir(file))
    remove_from_repo(*dir_sub_path)
  else
    FileUtils.rm(file)
  end
end

.rm_rf(file) ⇒ Object 



196
197
198
199
200
201
202
 
# File 'lib/rbbt/util/open.rb', line 196

def self.rm_rf(file)
  if (dir_sub_path = find_repo_dir(file))
    remove_from_repo(dir_sub_path[0], dir_sub_path[1], true)
  else
    FileUtils.rm_rf(file)
  end
end

.save_content_in_repo(dir, sub_path, content) ⇒ Object 



153
154
155
156
157
158
 
# File 'lib/rbbt/util/open.rb', line 153

def self.save_content_in_repo(dir, sub_path, content)
  repo = get_repo_from_dir(dir)
  repo.write_and_close do
    repo[sub_path] = content
  end
end

.unzip(stream) ⇒ Object 



293
294
295
296
297
 
# File 'lib/rbbt/util/open.rb', line 293

def self.unzip(stream)
  TmpFile.with_file(stream.read) do |filename|
    StringIO.new(CMD.cmd("unzip '{opt}' #{filename}", "-p" => true, :pipe => true).read)
  end
end

.wait(lag, key = nil) ⇒ Object 



40
41
42
43
44
45
46
47
48
 
# File 'lib/rbbt/util/open.rb', line 40

def self.wait(lag, key = nil)
  time = Time.now   

  if LAST_TIME[key] != nil && (time < LAST_TIME[key] + lag)
    sleep (LAST_TIME[key] + lag) - time
  end

  LAST_TIME[key] = Time.now   
end

.wget(url, options = {}) ⇒ Object 



50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/rbbt/util/open.rb', line 50

def self.wget(url, options = {})
  Log.low "WGET:\n -URL: #{ url }\n -OPTIONS: #{options.inspect}"
  options = Misc.add_defaults options, "--user-agent=" => 'firefox', :pipe => true

  wait(options[:nice], options[:nice_key]) if options[:nice]
  options.delete(:nice)
  options.delete(:nice_key)

  pipe  = options.delete(:pipe)
  quiet = options.delete(:quiet)
  post  = options.delete(:post)
  cookies = options.delete(:cookies)

  options["--quiet"]     = quiet if options["--quiet"].nil?
  options["--post-data="] ||= post if post

  if cookies
    options["--save-cookies"] = cookies
    options["--load-cookies"] = cookies
    options["--keep-session-cookies"] = true
  end


  stderr = case
           when options['stderr']
             options['stderr'] 
           when options['--quiet']
             false
           else
             nil
           end

  begin
    wget_options = options.merge( '-O' => '-')
    wget_options[:pipe] = pipe unless pipe.nil?
    wget_options[:stderr] = stderr unless stderr.nil?

    CMD.cmd("wget '#{ url }'", wget_options)
  rescue
   STDERR.puts $!.backtrace.inspect
   raise OpenURLError, "Error reading remote url: #{ url }.\n#{$!.message}"
  end
end

.write(file, content = nil, options = {}) ⇒ Object 



419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
 
# File 'lib/rbbt/util/open.rb', line 419

def self.write(file, content = nil, options = {})
  options = Misc.add_defaults options, :mode => 'w'

  mode = Misc.process_options options, :mode

  FileUtils.mkdir_p File.dirname(file)
  case
  when block_given?
    begin
      File.open(file, mode) do |f| 
        yield f
      end
    rescue Exception
      FileUtils.rm file if File.exists? file
      raise $!
    end
  when content.nil?
    File.open(file, mode){|f| f.write "" }
  when String === content
    file_write(file, content, mode)
  else
    begin
      File.open(file, mode) do |f| 
        f.flock(File::LOCK_EX)
        while block = content.read(2014)
          f.write block
        end
        f.flock(File::LOCK_UN)
      end
    rescue Exception
      FileUtils.rm_rf file if File.exists? file
      raise $!
    end
    content.close
  end
end

.zip?(file) ⇒ Boolean

Returns:

  • (Boolean) 


309
310
311
 
# File 'lib/rbbt/util/open.rb', line 309

def self.zip?(file)
  !! (file =~ /\.zip/)
end