Class: Feedzirra::HttpMulti

Inherits:
Object
  • Object
show all
Defined in:
lib/feedzirra/http_multi.rb

Overview

Handles HTTP requests for Feedzirra, including registration of on success and on failure callbacks.

Constant Summary collapse

DEFAULTS = 
{
  :backend => {
    :class => Feedzirra::Backend::Memory 
  }  
}

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(*args) ⇒ HttpMulti

Returns a new instance of HttpMulti.



15
16
17
18
19
20
21
 
# File 'lib/feedzirra/http_multi.rb', line 15

def initialize(*args)
  @options          = DEFAULTS.merge(args.extract_options!)
  @retrievables     = args.flatten
  @multi            = Curl::Multi.new
  @responses        = { }
  @backend          = @options[:backend][:class].new
end

Instance Attribute Details

#multi ⇒ Object (readonly)

Returns the value of attribute multi.



7
8
9
 
# File 'lib/feedzirra/http_multi.rb', line 7

def multi
  @multi
end

#options ⇒ Object (readonly)

Returns the value of attribute options.



7
8
9
 
# File 'lib/feedzirra/http_multi.rb', line 7

def options
  @options
end

#responses ⇒ Object (readonly)

Returns the value of attribute responses.



7
8
9
 
# File 'lib/feedzirra/http_multi.rb', line 7

def responses
  @responses
end

#retrievables ⇒ Object (readonly)

Returns the value of attribute retrievables.



7
8
9
 
# File 'lib/feedzirra/http_multi.rb', line 7

def retrievables
  @retrievables
end

Instance Method Details

#add_to_multi(retrievable, retrievable_queue) ⇒ Object

Generic method for building Curl::Multi object. Retrievable may be a Feed or a String URL.



41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/feedzirra/http_multi.rb', line 41

def add_to_multi(retrievable, retrievable_queue)
  if retrievable.respond_to?(:feed_url)
    url = retrievable.feed_url
  else
    url = retrievable
    retrievable = @backend.get(url)  # Try to fetch the last retrieval from backend
  end

  easy = build_curl_easy(url, retrievable, retrievable_queue)
  @multi.add(easy)
end

#build_curl_easy(url, retrievable, retrievable_queue) ⇒ Object

builds a Curl::Easy object that can be added to Curl::Multi.



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/feedzirra/http_multi.rb', line 54

def build_curl_easy(url, retrievable, retrievable_queue)

  easy = Curl::Easy.new(url) do |curl|
    curl = set_curl_configuration(curl, retrievable)

    curl.on_success do |c|
      on_success_handler(c, url, retrievable, retrievable_queue)
    end

    curl.on_failure do |c|
      if c.response_code == 304
        on_success_handler(c, url, retrievable, retrievable_queue)
      else
        on_failure_handler(c, url, retrievable, retrievable_queue)                          
      end
    end
  end 
       
  easy
end

#decode_content(curl) ⇒ Object

Decodes the XML document if it was compressed.

Parameters

[curl_requestCurl::Easy] The Curl::Easy response object from the request.

Returns

A decoded string of XML.



165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
 
# File 'lib/feedzirra/http_multi.rb', line 165

def decode_content(curl)
  if curl.header_str.match(/Content-Encoding: gzip/)
    begin
      gz =  Zlib::GzipReader.new(StringIO.new(curl.body_str))
      xml = gz.read
      gz.close
    rescue Zlib::GzipFile::Error 
      # Maybe this is not gzipped?
      xml = c.body_str
    end
  elsif curl.header_str.match(/Content-Encoding: deflate/)
    xml = Zlib::Inflate.inflate(curl.body_str)
  else
    xml = curl.body_str
  end

  xml
end

#etag_from_header(header) ⇒ Object

Determines the etag from the request headers.

Parameters

[header] Raw request header returned from the request

Returns

A string of the etag or nil if it cannot be found in the headers.



124
125
126
127
 
# File 'lib/feedzirra/http_multi.rb', line 124

def etag_from_header(header)
  header =~ /.*ETag:\s(.*)\r/
  $1
end

#last_modified_from_header(header) ⇒ Object

Determines the last modified date from the request headers.

Parameters

[header] Raw request header returned from the request

Returns

A Time object of the last modified date or nil if it cannot be found in the headers.



135
136
137
138
 
# File 'lib/feedzirra/http_multi.rb', line 135

def last_modified_from_header(header)
  header =~ /.*Last-Modified:\s(.*)\r/
  Time.parse($1) if $1
end

#on_failure_handler(curl, url, retrievable, retrievable_queue) ⇒ Object

Handles failed Curl responses.



112
113
114
115
116
 
# File 'lib/feedzirra/http_multi.rb', line 112

def on_failure_handler(curl, url, retrievable, retrievable_queue)      
  add_to_multi(multi, retrievable_queue.shift, retrievable_queue, responses, options) unless retrievable_queue.empty?
  responses[url] = curl.response_code
  @options[:on_failure].call(retrievable, curl.response_code, curl.header_str, curl.body_str) if options.has_key?(:on_failure)
end

#on_success_handler(curl, url, retrievable, retrievable_queue) ⇒ Object

Handles successful Curl responses.



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/feedzirra/http_multi.rb', line 85

def on_success_handler(curl, url, retrievable, retrievable_queue)
  add_to_multi(retrievable_queue.shift, retrievable_queue) unless retrievable_queue.empty?
  
  begin      
    if curl.response_code == 304
      updated_feed = retrievable
      updated_feed.mark_all_entries_old!
    else
      updated_feed = parser_for_xml(curl.body_str).run
      updated_feed.feed_url = curl.last_effective_url
      updated_feed.etag = etag_from_header(curl.header_str)
      updated_feed.last_modified = last_modified_from_header(curl.header_str)          
      set_updated_feed_entries!(retrievable, updated_feed)
    end

    @backend.set(url, updated_feed)          

    responses[url] = updated_feed
    @options[:on_success].call(retrievable) if @options.has_key?(:on_success)

  rescue Exception => e
    puts "Caught exception, but we're throwing it away: #{e}"
    @options[:on_failure].call(retrievable, curl.response_code, curl.header_str, curl.body_str) if @options.has_key?(:on_failure)
  end      
end

#parser_for_xml(xml) ⇒ Object 



140
141
142
 
# File 'lib/feedzirra/http_multi.rb', line 140

def parser_for_xml(xml)
  Feedzirra::FeedParser.new(xml)
end

#prepare ⇒ Object

Breaks the urls into chunks of 30 because of weird errors encountered on entering more items. As one finishes it pops another off the queue.



31
32
33
34
35
36
37
 
# File 'lib/feedzirra/http_multi.rb', line 31

def prepare
  retrievable_queue = @retrievables.dup
  
  retrievable_queue.slice!(0, 30).each do |retrievable|
    add_to_multi(retrievable, retrievable_queue)
  end
end

#run ⇒ Object

Prepares the curl object and calls #perform



24
25
26
27
 
# File 'lib/feedzirra/http_multi.rb', line 24

def run
  prepare
  @multi.perform
end

#set_curl_configuration(curl, retrievable = nil) ⇒ Object

Accepts a Curl::Easy object with an optional set of options and returns a Curl::Easy object with options merged into the defaults.



147
148
149
150
151
152
153
154
155
156
157
 
# File 'lib/feedzirra/http_multi.rb', line 147

def set_curl_configuration(curl, retrievable = nil)
  curl.headers["User-Agent"]        = @options[:user_agent] || Feedzirra::USER_AGENT
  curl.headers["If-Modified-Since"] = @options[:if_modified_since].httpdate if @options.has_key?(:if_modified_since)
  curl.headers["If-None-Match"]     = retrievable.etag if (retrievable.respond_to?(:etag) && retrievable.etag)
  curl.headers["Accept-Encoding"]   = 'gzip, deflate' if @options.has_key?(:compress)

  curl.follow_location = true
  curl.userpwd = @options[:http_authentication].join(':') if @options.has_key?(:http_authentication)

  curl
end

#set_updated_feed_entries!(retrievable, updated_feed) ⇒ Object 



75
76
77
78
79
80
81
82
 
# File 'lib/feedzirra/http_multi.rb', line 75

def set_updated_feed_entries!(retrievable, updated_feed)
  if retrievable.respond_to?(:update_from_feed) 
    retrievable.update_from_feed(updated_feed) 
  else
    # all elements are "new", since we weren't dealing with a Feed element.
    updated_feed.mark_all_entries_new!
  end    
end