Class: Statement::Feed

Inherits:
Object
  • Object
show all
Defined in:
lib/statement/feed.rb

Class Method Summary collapse

Class Method Details

.batch(urls) ⇒ Object 



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/statement/feed.rb', line 11

def self.batch(urls)
  results = []
  failures = []
  hydra = Typhoeus::Hydra.new
  urls.each do |url|
    req = Typhoeus::Request.new(url)
    req.on_complete do |response|
      if response.success?
        doc = Nokogiri::XML(response.body)
        results << parse_rss(doc, url)
      else
        failures << url
      end
    end
    hydra.queue(req)
  end
  hydra.run
  [results.flatten, failures]
end

.date_from_rss_item(link) ⇒ Object 



39
40
41
42
43
44
45
46
47
 
# File 'lib/statement/feed.rb', line 39

def self.date_from_rss_item(link)
  if !link.xpath('pubDate').text.empty?
    Date.parse(link.xpath('pubDate').text)
  elsif !link.xpath('pubdate').empty?
    Date.parse(link.xpath('pubdate').text)
  else
    nil
  end
end

.from_rss(url) ⇒ Object 



49
50
51
52
53
 
# File 'lib/statement/feed.rb', line 49

def self.from_rss(url)
  doc = open_rss(url)
  return unless doc
  parse_rss(doc, url)
end

.open_rss(url) ⇒ Object 



31
32
33
34
35
36
37
 
# File 'lib/statement/feed.rb', line 31

def self.open_rss(url)
  begin
    Nokogiri::XML(open(url))
  rescue
    nil
  end
end

.parse_rss(doc, url) ⇒ Object 



55
56
57
58
59
60
61
62
63
64
 
# File 'lib/statement/feed.rb', line 55

def self.parse_rss(doc, url)
  links = doc.xpath('//item')
  results = links.map do |link|
    abs_link = Utils.absolute_link(url, link.xpath('link').text)
    abs_link = "http://www.burr.senate.gov/public/"+ link.xpath('link').text if url == 'http://www.burr.senate.gov/public/index.cfm?FuseAction=RSS.Feed'
    abs_link = link.xpath('link').text[37..-1] if url == "http://www.johanns.senate.gov/public/?a=RSS.Feed"
    { :source => url, :url => abs_link, :title => link.xpath('title').text, :date => date_from_rss_item(link), :domain => URI.parse(url).host }
  end
  Utils.remove_generic_urls!(results)
end