Class: RHACK::Frame

Inherits:
Object
  • Object
show all
Defined in:
lib/rhack/dl.rb,
lib/rhack/frame.rb

Constant Summary collapse

@@cache = 
{}

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(*args) ⇒ Frame

Returns a new instance of Frame.



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/rhack/frame.rb', line 27

def initialize *args
  args << 10 unless args[-1].is Fixnum
  args.insert -2, {} unless args[-2].is Hash
  opts = args[-2]
  if scouts = (opts[:scouts] || opts[:threads])
    args[-1] = scouts
  end
  @opts = {:eval => Johnson::Enabled, :redir => true, :cp => true, :result => Page}.merge!(opts)
  args[-2] = @opts
  if args[0].is String
    url = args[0]
    'http://' >> url if url !~ /^\w+:\/\//
    update_loc url
  else
    @loc = {}
    @static = false
  end
  @ss  = ScoutSquad *args
end

Instance Attribute Details

#loc ⇒ Object (readonly)

Returns the value of attribute loc.



24
25
26
 
# File 'lib/rhack/frame.rb', line 24

def loc
  @loc
end

#opts ⇒ Object (readonly)

Returns the value of attribute opts.



24
25
26
 
# File 'lib/rhack/frame.rb', line 24

def opts
  @opts
end

#ss ⇒ Object (readonly)

Returns the value of attribute ss.



24
25
26
 
# File 'lib/rhack/frame.rb', line 24

def ss
  @ss
end

#static ⇒ Object (readonly)

Returns the value of attribute static.



24
25
26
 
# File 'lib/rhack/frame.rb', line 24

def static
  @static
end

#use_cache ⇒ Object (readonly)

Returns the value of attribute use_cache.



24
25
26
 
# File 'lib/rhack/frame.rb', line 24

def use_cache
  @use_cache
end

#write_to ⇒ Object (readonly)

Returns the value of attribute write_to.



24
25
26
 
# File 'lib/rhack/frame.rb', line 24

def write_to
  @write_to
end

Instance Method Details

#[](i) ⇒ Object 



67
 
# File 'lib/rhack/frame.rb', line 67

def [](i) @ss[i] end

#anchor ⇒ Object 



60
61
62
 
# File 'lib/rhack/frame.rb', line 60

def anchor
  retarget @loc.href
end

#check_mapfile(df, opts = {}) ⇒ Object 



182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/rhack/dl.rb', line 182

def check_mapfile(df, opts={})
  opts.reverse_merge! :psize => :auto, :threads => 1
  map = read_mapfile df
  if map
    L << map
    if map.rest.empty?
      puts "#{df} is loaded"
      $log << 'deleting mapfile'
      File.delete df+'.map'
      []
    else
      if opts[:len] and map.len != opts[:len]
        raise "Incorrect file size for #{df}"
      end
      psize = configure_psize *opts.values_at(:len, :psize, :threads)
      [psize, map.rest.div(psize)]
    end
  else
    write_mapfile df, opts[:len]
    psize = configure_psize *opts.values_at(:len, :psize, :threads)
    $log << (0...opts[:len]).div(psize)
    [psize, (0...opts[:len]).div(psize)]
  end
end

#clear_speedometer(scout) ⇒ Object 



296
297
298
299
 
# File 'lib/rhack/dl.rb', line 296

def clear_speedometer(scout)
  return  unless  @print_progress
  scout.http.on_progress
end

#configure_psize(len, psize, threads) ⇒ Object 



251
252
253
254
255
256
257
258
 
# File 'lib/rhack/dl.rb', line 251

def configure_psize(len, psize, threads)
  case psize
    when Numeric; psize.to_i
    when :auto; len > 100000 ? len/threads+1 : len
    when :mb; 1.mb
    else raise ArgumentError, "Incorrect value for part size #{psize}:#{psize.class}"
  end
end

#copy_cookies!(i = 0) ⇒ Object 



69
70
71
 
# File 'lib/rhack/frame.rb', line 69

def copy_cookies! i=0
  @ss.each {|s| s.cookies.replace @ss[i].cookies}
end

#dl(uri, df = File.basename(uri.parse(:uri).path), psize = :auto, opts = {}) ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/rhack/dl.rb', line 93

def dl(uri, df=File.basename(uri.parse(:uri).path), psize=:auto, opts={})
  dled = 0
  lock = ''
  callback = lambda {|len, pos, body|
    if body != :careful_dl
      begin
        write(df, body, pos)
      rescue => e
        binding.start_interaction
        raise
      end
      if (dled += body.size) == len
        File.delete lock if File.file? lock
        yield df if block_given?
      end
    else
      lock = lock_file df, len, pos # filename, filesize, partsize
    end
  }
  opts[:threads] ||= @ss.size-1
  get_distr(uri, psize, opts[:threads], opts[:start].to_i, &callback)
  Curl.wait unless block_given?
  df
end

#drop_cache!(use = nil) ⇒ Object 



83
84
85
86
87
 
# File 'lib/rhack/frame.rb', line 83

def drop_cache! use=nil
  @@cache.clear
  GC.start
  @use_cache = use if use.in [true, false]      
end

#each(&block) ⇒ Object 



66
 
# File 'lib/rhack/frame.rb', line 66

def each(&block) @ss.each &block end

#exec(*args, &callback) ⇒ Object Also known as: get

opts are :eval, :json, :hash, :wait, :proc_result, :save_result, :load_scripts, :zip, :thread_safe, :result, :stream, :raw, :xhr + any opts for Scouts in one hash



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/rhack/frame.rb', line 96

def exec *args, &callback
  many, order, orders, with_opts = interpret_request *args
  L.log({:many => many, :order => order, :orders => orders, :with_opts => with_opts})
  
  if !Johnson::Enabled and with_opts[:eval]
    L < "failed to use option :eval because Johnson is disabled"
    with_opts.delete :eval
  end
  # JS Runtime is not thread-safe and must be created in curl thread
  # if we aren't said explicitly about the opposite
  Johnson::Runtime.set_browser_for_curl with_opts
  
  if many then  exec_many orders, with_opts, &callback
  else            exec_one order, with_opts, &callback    end
end

#get_cached(*links) ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/rhack/dl.rb', line 7

def get_cached(*links)
  res = []
  expire = links[-1] == :expire ? links.pop : false
  links.parses(:uri).each_with_index {|url, i|
    next if url.path[/ads|count|stats/]
    file = Cache.load url, !expire
    if file
      if expire
        @ss.next.loadGet(url.href, :headers=>{'If-Modified-Since'=>file.date}) {|c|
          if c.res.code == 200
            res << [i, (data = c.res.body)]
            Cache.save url, data, false
          else
            res << [i, file.is(String) ? file : read(file.path)]
          end
        }
      else
        res << [i, file.is(String) ? file : read(file.path)]
      end
    else
      @ss.next.loadGet(url.href) {|c|
        if c.res.code == 200
          res << [i, (data = c.res.body)]
          Cache.save url, data, !expire
        end
      }
    end
  }
  Curl.wait
  links.size == 1 ? res[0][1] : res.sort!.lasts
end

#get_distr(uri, psize, threads, start = 0, print_progress = $verbose) ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/rhack/dl.rb', line 39

def get_distr(uri, psize, threads, start=0, print_progress=$verbose)
  raise ConfigError, "Insufficient Scouts in the Frame for distributed downloading" if @ss.size < 2
  @print_progress, code, stop_download, @ss_reserve = print_progress, nil, false, []
  (s = @ss.next).http.on_header {|h|
    next h.size unless h[/Content-Length: (\d+)|HTTP\/1\.[01] (\d+)[^\r]+|^\s*$/]
    if code = $2
      if code != '200'
        L << "#$& getting #{uri}; interrupting request."
        s.http.on_header() # set default process
        next 0
      end
      next h.size
    end
    
    s.http.on_header() # set default process
    if !$1 # конец хедера, content-length отсутствует
      L << "No Content-Length header; trying to load a whole #{uri} at once!"
      s.loadGet {|c| yield c.res.body.size, 0, c.res.body}
      next 0
    end
    
    len = $1.to_i - start
    psize = configure_psize(len, psize, threads)
    parts = (len/psize.to_f).ceil
    setup_speedometer(uri, parts, len)
    yield len, psize, :careful_dl if len > (@opts[:careful_dl] || 10.mb)
    
    @ss_reserve = @ss[threads+1..-1]
    @ss = @ss[0..threads]
    (0...parts).each {|n|
      break if stop_download
      
      s = @ss.next
      run_speedometer(s, len, n)
      s.loadGet(uri, :headers => {
        'Range' => "bytes=#{start + n*psize}-#{start + (n+1)*psize - 1}"
      }) {|c|
        clear_speedometer(s)
        if c.res.code/10 == 20
          yield len, n*psize, c.res.body
        else
          L << "#{c.res} during get #{uri.inspect}; interrupting request."
          stop_download = true
        end
      }
    }
    0
  }
  s.raise_err = false
  s.loadGet validate uri
ensure
  @ss.concat @ss_reserve || []
end

#inspect ⇒ Object 



89
90
91
92
 
# File 'lib/rhack/frame.rb', line 89

def inspect
  sssize = @ss.size
  "<#Frame @ #{@ss.untargeted ? 'no target' : @loc.root}: #{sssize} #{sssize == 1 ? 'scout' : 'scouts'}#{', static'+(' => '[email protected] if @static.is(Hash)) if @static}, cookies #{@ss[0].cookieProc ? 'on' : 'off'}>"
end

#interpret_request(*args) ⇒ Object 



114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
 
# File 'lib/rhack/frame.rb', line 114

def interpret_request(*args)
  body, mp, url, opts = args.dup.get_opts [nil, false, nil], @opts
  L.log [body, mp, url, opts]
  zip = opts.delete :zip
  verb = opts.delete :verb
  many = order = orders = post = false
  # Default options set is for POST
  if mp.is String or mp.kinda Array and !(url.is String or url.kinda Array)
  # if second arg is String, then that's url
    url, mp, post = mp.dup, false, true
  #  L.debug "url #{url.inspect} has been passed as second argument instead of third"
  # But if we have only one argument actually passed 
  # except for options hash, then believe it's GET
  elsif body.is String or body.kinda [String]
    L.debug "first parameter (#{body.inspect}) was implicitly taken as url#{' '+body.class if body.kinda Array}, but last paramter is of type #{url.class}, too" if url
    url = body.dup
  elsif !body
    url = nil
  else
    url = url.dup if url
    mp, post = !!mp, true
  end
  
  if post
    put = verb == :put
    validate_zip url, body if zip
    if zip or url.kinda Array or body.kinda Array
      many    = true
      unless put or body.kinda [Hash]
        raise TypeError, "body of post request must be a hash array, params was
   (#{args.inspect[1..-2]})"
      end
 
      if zip or url.kinda Array
        validate_some url
        orders = zip ? body.zip(url) : url.xprod(body, :inverse)
      else
        url = validate url
        orders = body.xprod url
      end
      if put
        orders.each {|o| o.unshift :loadPut}
      else
        orders.each {|o| o.unshift :loadPost and o.insert 2, mp}
      end
    else
      unless put or body.is Hash
        raise TypeError, "body of post request must be a hash, params was
   (#{args.inspect[1..-2]})"
      end
 
      url = validate url
      order = put ? [:loadPut, body, url] : [:loadPost, body, mp, url]
    end
  else
    del = verb == :delete
    if url.kinda Array
      many  = true
      validate_some url
      orders = [del ? :loadDelete : :loadGet].xprod url
    else
      url = validate url
      order = [del ? :loadDelete : :loadGet, url]
    end
  end
  if !order.b and !orders.b
    raise ArgumentError, "failed to run blank request#{'s' if many}, params was
 (#{args.inspect[1..-2]})"
  end
   
  opts[:wait] = opts[:sync] if :sync.in opts
  opts[:wait] = true if !:wait.in(opts) and 
                :proc_result.in(opts) ? !opts[:proc_result] : opts[:save_result]
  opts[:eval] = false if opts[:json] or opts[:hash] or opts[:raw]
  opts[:load_scripts] = self if opts[:load_scripts]
  opts[:stream] = true if opts[:raw]
  (opts[:headers] ||= {})['X-Requested-With'] = 'XMLHttpRequest' if opts[:xhr]
  [many, order, orders, opts]
end

#next ⇒ Object 



64
 
# File 'lib/rhack/frame.rb', line 64

def next() @ss.next end

#rand ⇒ Object 



65
 
# File 'lib/rhack/frame.rb', line 65

def rand() @ss.rand end

#read_mapfile(df) ⇒ Object 



207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
 
# File 'lib/rhack/dl.rb', line 207

def read_mapfile(df)
  df += '.map'
  text = read df
  $log << "mapfile read: #{text}"
  if text.b
    text[/^(\d+)\0+(\d+)\0*\n/]
    map = {}
    $log << [$1,$2]
    if $1 and $1 == $2
      map.rest = []
    else
      map.len, *map.parts = text.chop/"\n"
      map.len = map.len.to_i
      map.parts.map! {|part| part /= '-'; part[0].to_i..part[1].to_i}
      $log << map.parts
      map.rest = (0...map.len) - XRange(*map.parts)
    end
    map
  end
end

#retarget(to, forced = nil) ⇒ Object Also known as: target= 



53
54
55
56
57
 
# File 'lib/rhack/frame.rb', line 53

def retarget to, forced=nil
  to = 'http://' + to if to !~ /^\w+:/
  @ss.update to, forced
  update_loc to
end

#run_speedometer(scout, len, n) ⇒ Object 



280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
 
# File 'lib/rhack/dl.rb', line 280

def run_speedometer(scout, len, n)
  return  unless  @print_progress
  scout.http.on_progress {|dl_need, dl_now, *ul|
    if !@stop_print
      @progress[n] = dl_now
      percents = (@sum = @progress.sum)*100/len
      print @str%[@progress.select_b.size, @speed]+"\n%%[#{'@'*percents}#{' '*(100-percents)}]\r\b\r"+@bs
      if percents == 100
        puts "\v"*@newlines
        @stop_print = true
      end
    end
    true
  }
end

#setup_speedometer(uri, parts, len) ⇒ Object 



262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
 
# File 'lib/rhack/dl.rb', line 262

def setup_speedometer(uri, parts, len)
  return  unless  @print_progress
  @progress = Array.new(parts, 0)
  @stop_print, @speed, @sum, *@speedometer = false, '', 0, Time.now, 0
  @str = "Downloading #{uri.gsub '%', '%%'} (#{len.bytes}) in %03s streams, %07s/s:"
  @bs = "\b\r"*(@newlines = (uri.unpack('U*').size+len.bytes.size+42)/(ENV['COLUMNS'] || 80).to_i)
  Thread.new {
    until @stop_print
      sleep 0.2
      now = Time.now
      if now > @speedometer[0] and @sum > @speedometer[1]
        @speed.replace(((@sum - @speedometer[1])/(now - @speedometer[0])).to_i.bytes)
        @speedometer.replace [now, @sum]
      end
    end
  }
end

#simple_dl(uri, df = File.basename(uri.parse(:uri).path), opts = {}) ⇒ Object 



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/rhack/dl.rb', line 118

def simple_dl(uri, df=File.basename(uri.parse(:uri).path), opts={})
  opts.reverse_merge! :psize => :auto, :threads => 1, :print_progress => $verbose
  L << opts
  
  @print_progress = opts[:print_progress]
  unless len = opts[:len] || (map = read_mapfile(df) and map.len)
    return @ss.next.loadHead(uri) {|c| $log << c
      if len = c.res['Content-Length']
        simple_dl(uri, df, opts.merge(:len => len.to_i))
      else L.warn "Can't get file size, so it has no sence to download this way. Or maybe it's just an error. Check ObjectSpace.find(#{c.res.object_id}) out."
      end
    }
  end
  
  psize, parts = check_mapfile(df, opts)
  return unless psize
  L << [psize, parts]
  setup_speedometer(uri, parts.size, len)
  
  obtained uri do |uri|
    if opts[:threads] == 1
      start = opts[:start].to_i || (parts[0] && parts[0].begin) || 0
      scout = opts[:scout] || @ss.next
      $log << [uri, scout]
      (loadget = lambda {|n|
        run_speedometer(scout, len, n)
        from = start + n*psize
        to = start + (n+1)*psize - 1
        scout.loadGet(uri, :headers => {'Range' => "bytes=#{from}-#{to}"}) {|c|
          begin
            $log << "writing #{df} from #{from}: #{c.res.body.inspect}"
            write(df, c.res.body, from)
          rescue => e
            binding.start_interaction
            raise
          end
          if write_mapfile(df, from, to)
            clear_speedometer(scout)
            L.warn "file completely dl'ed, but (n+1)*psize <= len: (#{n}+1)*#{psize} <= #{len}" if (n+1)*psize <= len 
            yield df if block_given?
          elsif (n+1)*psize <= len 
            loadget[n+1] 
          end
        }
      })[0]
    else
      exec(uri, opts.merge(:raw => true, :ranges => parts)) {|c|
        L << c.res
        range = c.req.range
        begin
          write(df, c.res.body, range.begin)
        rescue => e
          binding.start_interaction
          raise
        end
        if write_mapfile(df, range.begin, range.end)
          @ss.each {|s| s.http.on_progress} if @print_progress
          yield df if block_given?
        end
      }
    end
  end
end

#update_loc(url) ⇒ Object 



47
48
49
50
51
 
# File 'lib/rhack/frame.rb', line 47

def update_loc url
  @loc = url.parse :uri
  # be careful, if you set :static => false, frame will be unable to use "path" url
  @static = @opts.fetch(:static, @loc)
end

#use_cache!(opts = {}) ⇒ Object 



73
74
75
76
77
78
79
80
81
 
# File 'lib/rhack/frame.rb', line 73

def use_cache! opts={}
  if opts == false
    @use_cache = false
  else
    @@cache = opts[:pages].kinda(Hash) ? opts[:pages] : opts[:pages].map_hash {|p| [p.href, p]} if opts[:pages]
    #@write_to = opts[:write_to] if :write_to.in opts
    @use_cache = true
  end
end

#write_mapfile(df, *args) ⇒ Object 



228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
 
# File 'lib/rhack/dl.rb', line 228

def write_mapfile(df, *args)
  df += '.map'
  map = ''
  if args.size != 2
    len = args.shift
    map << len.to_s.ljust(22, "\0") << "\n" if File.file? df
  end
  if args.any?
    read(df)[/^(\d+)\0+(\d+)\0*\n/]
    $log << "mapfile read"
    $log << [$1,$2]
    dled = $2.to_i + args[1] - args[0] + 1
    return true if dled == $1.to_i
    map << "#{args[0]}..#{args[1]}\n"
    $log << 'writing mapfile'
    write(df, dled.to_s.ljust(11, "\0"), 11)
  end
  $log << [df, map]
  $log << 'writing mapfile'
  write df, map
  nil
end