Module: HttpCrawler::Client

Included in:
Proxy::TestProxyApi::Client, Web::Baidu::Client
Defined in:
lib/http_crawler/client.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#http ⇒ Object (readonly)

Returns the value of attribute http.



24
25
26
 
# File 'lib/http_crawler/client.rb', line 24

def http
  @http
end

#uri ⇒ Object (readonly)

Returns the value of attribute uri.



24
25
26
 
# File 'lib/http_crawler/client.rb', line 24

def uri
  @uri
end

Class Method Details

.for(web_name, *args) ⇒ Object

接收格式web_name = “biquge_duquanben” 返回 HttpCrawler::Web::BiqugeDuquanben::Client 实例



10
11
12
 
# File 'lib/http_crawler/client.rb', line 10

def for(web_name, *args)
  "HttpCrawler::Web::#{web_name.camelize}::Client".constantize.new(*args)
end

.for_module(module_name, *args) ⇒ Object

接收格式module_name = “HttpCrawler::Web::BiqugeDuquanben” 返回 HttpCrawler::Web::BiqugeDuquanben::Client 实例



19
20
21
 
# File 'lib/http_crawler/client.rb', line 19

def for_module(module_name, *args)
  "#{module_name}::Client".constantize.new(*args)
end

Instance Method Details

#add_error_url(url_string) ⇒ Object

添加错误的url地址，表示这里面的url都是异常地址，存的是正则



50
51
52
 
# File 'lib/http_crawler/client.rb', line 50

def add_error_url(url_string)
  @http.error_urls << url_string
end

#auto_proxy=(value) ⇒ Object 



77
78
79
80
81
 
# File 'lib/http_crawler/client.rb', line 77

def auto_proxy=(value)
  Rails.logger.debug "自动更新代理"
  @http.auto_proxy = value
  @http.update_proxy if (value == true && @http.proxy? == false)
end

#header ⇒ Object 



61
62
63
 
# File 'lib/http_crawler/client.rb', line 61

def header
  @header ||= init_header
end

#init_header ⇒ Object 



65
66
67
 
# File 'lib/http_crawler/client.rb', line 65

def init_header
  nil
end

#init_http ⇒ Object

初始化http参数



45
46
47
 
# File 'lib/http_crawler/client.rb', line 45

def init_http

end

#init_uri ⇒ Object

init_uri 如果未初始化@uri,则会报错

继承类需要实现 @uri = URI("http://host")



57
58
59
 
# File 'lib/http_crawler/client.rb', line 57

def init_uri
  @uri = nil
end

#initialize ⇒ Object

init_uri 如果未初始化@uri,则会报错

继承类需要重定义 init_uri



30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/http_crawler/client.rb', line 30

def initialize
  raise "Client uri为空" unless init_uri
  @http = HttpCrawler::HTTP.new(uri.host, uri.port)

  @http.use_ssl = (uri.scheme == "https")

  @http.open_timeout = 5
  @http.read_timeout = 5
  @http.proxy_key = "#{self.class}"
  init_http

  Rails.logger.debug "proxy_key => #{@http.proxy_key}"
end

#update_header(parameter = {}) ⇒ Object 



69
70
71
 
# File 'lib/http_crawler/client.rb', line 69

def update_header(parameter = {})
  nil
end

#update_proxy(proxy = {}) ⇒ Object 



73
74
75
 
# File 'lib/http_crawler/client.rb', line 73

def update_proxy(proxy = {})
  @http.update_proxy(proxy)
end

#validation_page?(*arg) ⇒ Boolean

是否验证码界面

Returns:

  • (Boolean) 


84
85
86
 
# File 'lib/http_crawler/client.rb', line 84

def validation_page?(*arg)
  false
end