9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
|
# File 'lib/web_crawler/application.rb', line 9
def test
urls = FactoryUrl.new('http://www.superjob.ru/rabota/554/veb-programmist/?from=$1', [[140]]).factory
logger.info "start requests with #{urls.join(' ')} in 4 processes"
targets = BatchRequest.new(urls).process
logger.info "#{targets.size} targets collected"
urls = Follower.new(targets, same_host: false).collect { |url| url =~ /vacancy\/\?id=\d+/ }
logger.info "#{urls.size} urls collected"
logger.info "start requests with in 4 processes"
puts BatchRequest.new(urls).process.inspect
""
end
|