Class: HttpUtilities::Http::Mechanize::Client

Inherits:
Object
  • Object
show all
Includes:
Logger, ProxySupport, UserAgent
Defined in:
lib/http_utilities/http/mechanize/client.rb

Constant Summary

Constants included from UserAgent

UserAgent::USER_AGENTS

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from UserAgent

#set_user_agent

Methods included from ProxySupport

#generate_proxy_options, #proxy_model_defined?, #set_credentials, #set_from_array, #set_from_hash, #set_from_object, #set_from_string, #set_proxy_options

Methods included from Logger

#log

Constructor Details

#initialize(options = {}) ⇒ Client

Returns a new instance of Client.



14
15
16
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 14

def initialize(options = {})
  init_agent(options)
end

Instance Attribute Details

#agent ⇒ Object

Returns the value of attribute agent.



8
9
10
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 8

def agent
  @agent
end

#proxy ⇒ Object

Returns the value of attribute proxy.



8
9
10
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 8

def proxy
  @proxy
end

#user_agent ⇒ Object

Returns the value of attribute user_agent.



8
9
10
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 8

def user_agent
  @user_agent
end

Instance Method Details

#get_form(url_or_page, form_identifier = {}, options = {}) ⇒ Object 



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 87

def get_form(url_or_page, form_identifier = {}, options = {})
  form        =   nil
  index       =   form_identifier.delete(:index) { |el| 0 }
  page        =   (url_or_page.is_a?(String)) ? get_page(url_or_page, options) : url_or_page
  
  if page
    if form_identifier.empty?
      form    =   page.forms[index]
    else
      forms   =   page.forms_with(form_identifier)
      form    =   (forms && forms.any?) ? forms[index] : nil
    end
  end
  
  return form
end

#get_page(url_or_page, options = {}) ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 73

def get_page(url_or_page, options = {})
  page        =   nil
  
  if (url_or_page.is_a?(String))
    page      =   open_url(url_or_page, options)
  else
    page      =   url_or_page
  end
  
  page        =   (page && page.is_a?(::Mechanize::Page)) ? page : nil #Occasionally proxies will yield Mechanize::File instead of a proper page
  
  return page
end

#get_parser(page) ⇒ Object 



179
180
181
182
183
184
185
186
187
188
189
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 179

def get_parser(page)
  parser = nil

  if (page.is_a?(::Mechanize::Page))
    parser = page.parser
  elsif (page.is_a?(::Mechanize::File))
    parser = Nokogiri::HTML(page.body, nil, "utf-8")
  end

  return parser
end

#init_agent(options = {}) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 18

def init_agent(options = {})
  verbose                   =   options.fetch(:verbose, false)
  logger                    =   options.fetch(:logger, STDOUT)
  
  self.agent                =   ::Mechanize.new
  self.agent.log            =   ::Logger.new(logger) if (verbose)
  
  self.proxy                =   {}
  self.set_proxy_options(options)
  
  if self.proxy[:host] && self.proxy[:port]
    log(:info, "[HttpUtilities::Http::Mechanize::Client] - Will use proxy #{self.proxy[:host]}:#{self.proxy[:port]} for Mechanize.")
    self.agent.set_proxy(self.proxy[:host], self.proxy[:port], self.proxy[:username], self.proxy[:password])
  end
  
  self.set_user_agent
  (self.user_agent) ? self.agent.user_agent = self.user_agent : self.agent.user_agent_alias = 'Mac Safari'
  
  timeout                   =   options.fetch(:timeout, 300)
  self.agent.open_timeout   =   self.agent.read_timeout = timeout if timeout
end

#open_url(url, options = {}, retries = 3) ⇒ Object 



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 45

def open_url(url, options = {}, retries = 3)
  page = nil

  begin
    page = self.agent.get(url)

  rescue Net::HTTPNotFound, ::Mechanize::ResponseCodeError => error
    log(:error, "[HttpUtilities::Http::Mechanize::Client] - Response Code Error occurred for url #{url}. Error class: #{error.class.name}. Error message: #{error.message}")
    
    if retries > 0
      reset_agent(options)
      retries -= 1
      retry
    end
    
  rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT, Errno::ECONNRESET, Timeout::Error, Net::HTTPUnauthorized, Net::HTTPForbidden => connection_error
    log(:error, "[HttpUtilities::Http::Mechanize::Client] - Error occurred. Error class: #{connection_error.class.name}. Message: #{connection_error.message}")

    if retries > 0
      reset_agent(options)
      retries -= 1
      retry
    end
  end

  return page
end

#reset_agent(options = {}) ⇒ Object 



40
41
42
43
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 40

def reset_agent(options = {})
  self.agent, self.proxy, self.user_agent = nil
  init_agent(options)
end

#reset_radio_buttons(form) ⇒ Object 



131
132
133
134
135
136
137
138
139
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 131

def reset_radio_buttons(form)
  radio_buttons = form.radiobuttons
  
  radio_buttons.each do |radio_button|
    radio_button.checked = false
  end if (form && radio_buttons && radio_buttons.any?)

  return form
end

#set_form_and_submit(url_or_page, form_identifier = {}, submit_identifier = :first, fields = {}, options = {}, retries = 3) ⇒ Object 



104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 104

def set_form_and_submit(url_or_page, form_identifier = {}, submit_identifier = :first, fields = {}, options = {}, retries = 3)
  should_reset_radio_buttons  =   options.fetch(:should_reset_radio_buttons, false)
  page                        =   get_page(url_or_page, options)
  form                        =   page ? get_form(page, form_identifier) : nil
  response_page               =   nil

  if (form)
    form.action               =     "#{url_or_page}#{form.action}"  if (url_or_page.is_a?(String) && form.action.starts_with?("#"))
    form                      =     reset_radio_buttons(form)       if (should_reset_radio_buttons)
    form                      =     set_form_fields(form, fields)
    button                    =     (submit_identifier.nil? || submit_identifier.eql?(:first)) ? form.buttons.first : form.button_with(submit_identifier)

    begin
      response_page           =     form.submit(button)
    rescue Exception => e
      log(:error, "[HttpUtilities::Http::Mechanize::Client] - Failed to submit form. Error: #{e.class.name} - #{e.message}.")
    end

  elsif (!form && retries > 0)
    log(:info, "[HttpUtilities::Http::Mechanize::Client] - Couldn't find page or form with identifier #{form_identifier.inspect}")
    retries -= 1
    set_form_and_submit(url_or_page, form_identifier, submit_identifier, fields, options, retries)
  end

  return response_page
end

#set_form_field(form, key, value) ⇒ Object 



151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 151

def set_form_field(form, key, value)
  type          =   value.fetch(:type, :input)
  identifier    =   value.fetch(:identifier, :name)
  
  if (type.eql?(:input))
    log(:info, "[HttpUtilities::Http::Mechanize::Client] - Setting text field #{key} to value #{value[:value]}.")
    form.has_field?(key.to_s) ? form.field_with(identifier => key.to_s).value = value[:value].to_s : set_form_fields(form, value[:fallbacks])
  
  elsif (type.eql?(:checkbox))
    log(:info, "[HttpUtilities::Http::Mechanize::Client] - Setting checkbox #{key} to checked: #{value[:checked]}.")
    form.checkbox_with(identifier => key.to_s).checked        =   value[:checked]
  
  elsif (type.eql?(:radiobutton))
    log(:info, "[HttpUtilities::Http::Mechanize::Client] - Setting radio button #{key} to checked: #{value[:checked]}.")
    form.radiobutton_with(identifier => key.to_s).checked     =   value[:checked]
  
  elsif (type.eql?(:select))
    log(:info, "[HttpUtilities::Http::Mechanize::Client] - Setting select/dropdown #{key} to value: #{value[:value]}.")
    form.field_with(identifier => key.to_s).value             =   value[:value].to_s
  
  elsif (type.eql?(:file_upload))
    log(:info, "[HttpUtilities::Http::Mechanize::Client] - Setting file upload #{key} to value #{value[:value]}.")
    form.file_upload_with(identifier => key.to_s).file_name   =   value[:value].to_s
  end

  return form
end

#set_form_fields(form, fields) ⇒ Object 



141
142
143
144
145
146
147
148
149
 
# File 'lib/http_utilities/http/mechanize/client.rb', line 141

def set_form_fields(form, fields)
  if (form && fields && !fields.empty?)
    fields.each do |key, value|
      form = set_form_field(form, key, value)
    end
  end

  return form
end