Class: IfanrSpider

Inherits:
Flute::SpiderBase show all
Includes:
RssMiddleware
Defined in:
lib/template/spiders/ifanr_spider.rb

Instance Method Summary collapse

Methods included from RssMiddleware

#items, #rss_items

Methods inherited from Flute::SpiderBase

#init_queue, #items, #manager, #manager_network, middleware, #network, opts, #pipeline, #reset_manager, #run_queue, set_name, set_network, start_urls

Instance Method Details

#after_parse(requests) ⇒ Object 



23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/template/spiders/ifanr_spider.rb', line 23

def after_parse(requests)
  requests.map {|request|
    html = request.response_html
    item = request.meta
    item.description = html.og_description
    unless html.og_image.blank?
      item.image = html.og_image
    end
    item
  }
end

#before_parse(items) ⇒ Object 



7
8
9
10
11
12
 
# File 'lib/template/spiders/ifanr_spider.rb', line 7

def before_parse(items)
  items.map {|item|
    WebItem.new url: item.url,
                title: item.title
  }
end

#parse(items) ⇒ Object 



14
15
16
17
18
19
20
21
 
# File 'lib/template/spiders/ifanr_spider.rb', line 14

def parse(items)
  items.map do |item|
    request = network.request(item.url)
    request.meta = item
    manager.queue request
    request
  end
end