Class: Down::NetHttp

Inherits:
Backend show all
Defined in:
lib/down/net_http.rb

Defined Under Namespace

Modules: DownloadedFile

Instance Method Summary collapse

Methods inherited from Backend

download, open

Constructor Details

#initialize(options = {}) ⇒ NetHttp

Returns a new instance of NetHttp.



14
15
16
 
# File 'lib/down/net_http.rb', line 14

def initialize(options = {})
  @options = { "User-Agent" => "Down/#{Down::VERSION}" }.merge(options)
end

Instance Method Details

#download(uri, options = {}) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/down/net_http.rb', line 18

def download(uri, options = {})
  options = @options.merge(options)

  max_size            = options.delete(:max_size)
  max_redirects       = options.delete(:max_redirects) || 2
  progress_proc       = options.delete(:progress_proc)
  content_length_proc = options.delete(:content_length_proc)

  open_uri_options = {
    content_length_proc: proc { |size|
      if size && max_size && size > max_size
        raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
      end
      content_length_proc.call(size) if content_length_proc
    },
    progress_proc: proc { |current_size|
      if max_size && current_size > max_size
        raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
      end
      progress_proc.call(current_size) if progress_proc
    },
    redirect: false,
  }

  if options[:proxy]
    proxy    = URI(options.delete(:proxy))
    user     = proxy.user
    password = proxy.password

    if user || password
      proxy.user     = nil
      proxy.password = nil

      open_uri_options[:proxy_http_basic_authentication] = [proxy.to_s, user, password]
    else
      open_uri_options[:proxy] = proxy.to_s
    end
  end

  open_uri_options.merge!(options)

  tries = max_redirects + 1

  begin
    uri = URI(uri)
    raise Down::InvalidUrl, "URL scheme needs to be http or https" unless uri.is_a?(URI::HTTP)
  rescue URI::InvalidURIError => exception
    raise Down::InvalidUrl, exception.message
  end

  if uri.user || uri.password
    open_uri_options[:http_basic_authentication] ||= [uri.user, uri.password]
    uri.user = nil
    uri.password = nil
  end

  begin
    downloaded_file = uri.open(open_uri_options)
  rescue OpenURI::HTTPRedirect => exception
    if (tries -= 1) > 0
      uri = exception.uri

      if !exception.io.meta["set-cookie"].to_s.empty?
        open_uri_options["Cookie"] = exception.io.meta["set-cookie"]
      end

      retry
    else
      raise Down::TooManyRedirects, "too many redirects"
    end
  rescue OpenURI::HTTPError => exception
    code, message = exception.io.status
    response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
    response = response_class.new(nil, code, message)
    exception.io.metas.each do |name, values|
      values.each { |value| response.add_field(name, value) }
    end

    response_error!(response)
  rescue => exception
    request_error!(exception)
  end

  # open-uri will return a StringIO instead of a Tempfile if the filesize is
  # less than 10 KB, so if it happens we convert it back to Tempfile. We want
  # to do this with a Tempfile as well, because open-uri doesn't preserve the
  # file extension, so we want to run it against #copy_to_tempfile which
  # does.
  open_uri_file = downloaded_file
  downloaded_file = copy_to_tempfile(uri.path, open_uri_file)
  OpenURI::Meta.init downloaded_file, open_uri_file

  downloaded_file.extend Down::NetHttp::DownloadedFile
  downloaded_file
end

#open(uri, options = {}) ⇒ Object 



114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/down/net_http.rb', line 114

def open(uri, options = {})
  options = @options.merge(options)

  begin
    uri = URI(uri)
    raise Down::InvalidUrl, "URL scheme needs to be http or https" unless uri.is_a?(URI::HTTP)
  rescue URI::InvalidURIError => exception
    raise Down::InvalidUrl, exception.message
  end

  http_class = Net::HTTP

  if options[:proxy]
    proxy = URI(options[:proxy])
    http_class = Net::HTTP::Proxy(proxy.hostname, proxy.port, proxy.user, proxy.password)
  end

  http = http_class.new(uri.host, uri.port)

  # taken from open-uri implementation
  if uri.is_a?(URI::HTTPS)
    http.use_ssl = true
    http.verify_mode = options[:ssl_verify_mode] || OpenSSL::SSL::VERIFY_PEER
    store = OpenSSL::X509::Store.new
    if options[:ssl_ca_cert]
      Array(options[:ssl_ca_cert]).each do |cert|
        File.directory?(cert) ? store.add_path(cert) : store.add_file(cert)
      end
    else
      store.set_default_paths
    end
    http.cert_store = store
  end

  http.read_timeout = options[:read_timeout] if options.key?(:read_timeout)
  http.open_timeout = options[:open_timeout] if options.key?(:open_timeout)

  request_headers = options.select { |key, value| key.is_a?(String) }
  request_headers["Accept-Encoding"] = "" # otherwise FiberError can be raised

  get = Net::HTTP::Get.new(uri.request_uri, request_headers)
  get.basic_auth(uri.user, uri.password) if uri.user || uri.password

  request = Fiber.new do
    http.start do
      http.request(get) do |response|
        Fiber.yield response
        response.instance_variable_set("@read", true)
      end
    end
  end

  begin
    response = request.resume
  rescue => exception
    request_error!(exception)
  end

  response_error!(response) unless (200..299).cover?(response.code.to_i)

  body_chunks = Enumerator.new do |yielder|
    begin
      response.read_body { |chunk| yielder << chunk }
    rescue => exception
      request_error!(exception)
    end
  end

  Down::ChunkedIO.new(
    chunks:     body_chunks,
    size:       response["Content-Length"] && response["Content-Length"].to_i,
    encoding:   response.type_params["charset"],
    rewindable: options.fetch(:rewindable, true),
    on_close:   -> { request.resume }, # close HTTP connnection
    data: {
      status:   response.code.to_i,
      headers:  response.each_header.inject({}) { |headers, (downcased_name, value)|
                  name = downcased_name.split("-").map(&:capitalize).join("-")
                  headers.merge!(name => value)
                },
      response: response,
    },
  )
end