Module: ListSpider

Defined in:
lib/list_spider.rb,
lib/list_spider/version.rb

Constant Summary collapse

RANDOM_TIME = 
-1
NO_LIMIT_CONCURRENT = 
-1
DEFAULT_CONCURRNET_MAX = 
50
DEFAULT_INTERVAL = 
0
VERSION = 
'2.5.0'.freeze

Class Method Summary collapse

Class Method Details

.add_task(task) ⇒ Object 



143
144
145
146
147
148
149
150
151
152
153
 
# File 'lib/list_spider.rb', line 143

def add_task(task)
  if task.is_a? Array
    need_down_list = filter_list(task)
    @down_list += need_down_list
  elsif task.is_a?TaskStruct
    need_down_list = filter_list([task])
    @down_list += need_down_list
  else
    puts "error task type:#{task.class}"
  end
end

.get_list(down_list, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX) ⇒ Object 



122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'lib/list_spider.rb', line 122

def get_list(down_list, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
  if interval.is_a? Range
    @random_time_range = interval
    interval = RANDOM_TIME
  end

  @down_list = filter_list(down_list)
  @interval = interval
  @max = max
  @max = @down_list.size if @max == NO_LIMIT_CONCURRENT
  @succeed_size = 0
  @failed_size = 0

  puts "total size:#{@down_list.size}"
  event_machine_start_list(next_task, method(:complete))
end

.get_one(task, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX) ⇒ Object 



139
140
141
 
# File 'lib/list_spider.rb', line 139

def get_one(task, interval: DEFAULT_INTERVAL, max: DEFAULT_CONCURRNET_MAX)
  get_list([task], interval: interval, max: max)
end