55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
# File 'lib/cacho.rb', line 55
def request(verb, url, options = {})
uri = self.uri(url, options)
loop do
request = Net::HTTP.const_get(verb.capitalize).new(uri.request_uri)
@callbacks[:before_request].(request) if @callbacks[:before_request]
request["User-Agent"] = "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0"
if options.include?(:headers)
options[:headers].each do |key, value|
request[key] = value
end
end
request["Accept-Encoding"] = "gzip"
if verb == :post
post_data = options.fetch(:data, {})
case options[:content_type]
when :json
request["Content-Type"] = "application/json; charset=utf-8"
request.body = post_data.to_json
else
request.body = URI.encode_www_form(post_data)
end
end
if options[:content_encoding] == :deflate
request["Content-Encoding"] = "deflate"
request.body = Zlib::Deflate.deflate(request.body)
end
$stderr.puts("-> #{verb.upcase} #{uri}")
if verb_idempotent?(verb)
res = protect { @http.request(uri, request) }
else
res = @http.request(uri, request)
end
body = res.body
if res["Content-Encoding"] == "gzip"
body = Zlib::GzipReader.new(StringIO.new(body)).read
end
if res["Content-Type"]
if charset = res["Content-Type"][/\bcharset=(.+)\b/, 1]
body.force_encoding(charset)
body.encode!(Encoding.default_external)
end
if res["Content-Type"].start_with?("application/json")
parsed = JSON.parse(body)
else
parsed = body
end
end
if @callbacks[:rate_limit_detector]
if seconds = @callbacks[:rate_limit_detector].(res, body, parsed)
$stderr.puts("Rate limited for #{seconds} seconds.")
sleep(seconds)
next
end
end
case res.code
when "200"
return parsed
when "303" , "302" , "301"
if res["Location"].to_s.index(/^https?:\/\//)
uri = URI(res["Location"])
else
uri = URI.join(uri, res["Location"])
end
$stderr.puts("Redirected to #{res["Location"]} (#{uri})")
when "404"
return nil
else
raise "Got #{res.code}: #{body.inspect}"
end
end
end
|