Class: Curl::Multi

Inherits:
Object
  • Object
show all
Defined in:
lib/curl/multi.rb,
ext/curb_multi.c

Defined Under Namespace

Classes: DownloadError

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.Curl::Multi.autoclose ⇒ Object

Get the global default autoclose setting for all Curl::Multi Handles.



182
183
184
 
# File 'ext/curb_multi.c', line 182

VALUE ruby_curl_multi_get_autoclose(VALUE klass) {
  return cCurlMutiAutoClose == 1 ? Qtrue : Qfalse;
}

.Curl::Multi.autoclose( = true) ⇒ true

Automatically close open connections after each request. Otherwise, the connection will remain open for reuse until the next GC

Returns:

  • (true) 


170
171
172
173
 
# File 'ext/curb_multi.c', line 170

VALUE ruby_curl_multi_set_autoclose(VALUE klass, VALUE onoff) {
  cCurlMutiAutoClose = ((onoff == Qtrue) ? 1 : 0);
  return onoff;
}

.Curl::Multi.default_timeout( = 4) ⇒ 4

Get the global default time out for all Curl::Multi Handles.

Returns:

  • (4) 


158
159
160
 
# File 'ext/curb_multi.c', line 158

VALUE ruby_curl_multi_get_default_timeout(VALUE klass) {
  return LONG2NUM(cCurlMutiDefaulttimeout);
}

.Curl::Multi.default_timeout( = 4) ⇒ 4

Set the global default time out for all Curl::Multi Handles. This value is used when libcurl cannot determine a timeout value when calling curl_multi_timeout.

Returns:

  • (4) 


146
147
148
149
 
# File 'ext/curb_multi.c', line 146

VALUE ruby_curl_multi_set_default_timeout(VALUE klass, VALUE timeout) {
  cCurlMutiDefaulttimeout = NUM2LONG(timeout);
  return timeout;
}

.download(urls, easy_options = {}, multi_options = {}, download_paths = nil, &blk) ⇒ Object

call-seq:

Curl::Multi.download(){|c|}

will create 2 new files file1.txt and file2.txt

2 files will be opened, and remain open until the call completes

when using the :post or :put method, urls should be a hash, including the individual post fields per post



187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/curl/multi.rb', line 187

def download(urls,easy_options={},multi_options={},download_paths=nil,&blk)
  errors = []
  procs = []
  files = []
  urls_with_config = []
  url_to_download_paths = {}

  urls.each_with_index do|urlcfg,i|
    if urlcfg.is_a?(Hash)
      url = url[:url]
    else
      url = urlcfg
    end

    if download_paths and download_paths[i]
      download_path = download_paths[i]
    else
      download_path = File.basename(url)
    end

    file = lambda do|dp|
      file = File.open(dp,"wb")
      procs << (lambda {|data| file.write data; data.size })
      files << file
      file
    end.call(download_path)

    if urlcfg.is_a?(Hash)
      urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options))
    else
      urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options)
    end
    url_to_download_paths[url] = {:path => download_path, :file => file} # store for later
  end

  if blk
    # when injecting the block, ensure file is closed before yielding
    Curl::Multi.http(urls_with_config, multi_options) do |c,code,method|
      info = url_to_download_paths[c.url]
      begin
        file = info[:file]
        files.reject!{|f| f == file }
        file.close
      rescue => e
        errors << e
      end
      blk.call(c,info[:path])
    end
  else
    Curl::Multi.http(urls_with_config, multi_options)
  end

ensure
  files.each {|f|
    begin
      f.close
    rescue => e
      errors << e
    end
  }
  if errors.any?
    de = Curl::Multi::DownloadError.new
    de.errors = errors
    raise de
  end
end

.get(urls, easy_options = {}, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.get(['url1','url2','url3','url4','url5'], :follow_location => true) do|easy|
  easy
end

Blocking call to fetch multiple url’s in parallel.



14
15
16
17
18
19
20
 
# File 'lib/curl/multi.rb', line 14

def get(urls, easy_options={}, multi_options={}, &blk)
  url_confs = []
  urls.each do|url|
    url_confs << {:url => url, :method => :get}.merge(easy_options)
  end
  self.http(url_confs, multi_options) {|c,code,method| blk.call(c) if blk }
end

.http(urls_with_config, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.http( [

{ :url => 'url1', :method => :post,
  :post_fields => {'field1' => 'value1', 'field2' => 'value2'} },
{ :url => 'url2', :method => :get,
  :follow_location => true, :max_redirects => 3 },
{ :url => 'url3', :method => :put, :put_data => File.open('file.txt','rb') },
{ :url => 'url4', :method => :head }

], => Curl::CURLPIPE_HTTP1)

Blocking call to issue multiple HTTP requests with varying verb’s.

urls_with_config: is a hash of url’s pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used multi_options: options for the multi handle blk: a callback, that yeilds when a handle is completed



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/curl/multi.rb', line 88

def http(urls_with_config, multi_options={}, &blk)
  m = Curl::Multi.new

  # maintain a sane number of easy handles
  multi_options[:max_connects] = max_connects = multi_options.key?(:max_connects) ? multi_options[:max_connects] : 10

  free_handles = [] # keep a list of free easy handles

  # configure the multi handle
  multi_options.each { |k,v| m.send("#{k}=", v) }
  callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_redirect,:on_missing,:on_body,:on_header]

  add_free_handle = proc do|conf, easy|
    c       = conf.dup # avoid being destructive to input
    url     = c.delete(:url)
    method  = c.delete(:method)
    headers = c.delete(:headers)

    easy    = Curl::Easy.new if easy.nil?

    easy.url = url

    # assign callbacks
    callbacks.each do |cb|
      cbproc = c.delete(cb)
      easy.send(cb,&cbproc) if cbproc
    end

    case method
    when :post
      fields = c.delete(:post_fields)
      # set the post post using the url fields
      easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&')
    when :put
      easy.put_data = c.delete(:put_data)
    when :head
      easy.head = true
    when :delete
      easy.delete = true
    when :get
    else
      # XXX: nil is treated like a GET
    end

    # headers is a special key
    headers.each {|k,v| easy.headers[k] = v } if headers

    #
    # use the remaining options as specific configuration to the easy handle
    # bad options should raise an undefined method error
    #
    c.each { |k,v| easy.send("#{k}=",v) }

    easy.on_complete {|curl|
      free_handles << curl
      blk.call(curl,curl.response_code,method) if blk
    }
    m.add(easy)
  end

  max_connects.times do
    conf = urls_with_config.pop
    add_free_handle.call(conf, nil) if conf
    break if urls_with_config.empty?
  end

  consume_free_handles = proc do
    # as we idle consume free handles
    if urls_with_config.size > 0 && free_handles.size > 0
      easy = free_handles.pop
      conf = urls_with_config.pop
      add_free_handle.call(conf, easy) if conf
    end
  end

  if urls_with_config.empty?
    m.perform
  else
    until urls_with_config.empty?
      m.perform do
        consume_free_handles.call
      end
      consume_free_handles.call
    end
    free_handles = nil
  end

end

.Curl::Multi.new ⇒ #<Curl::Easy...

Create a new Curl::Multi instance

Returns ].

Returns:



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
# File 'ext/curb_multi.c', line 121

VALUE ruby_curl_multi_new(VALUE klass) {
  ruby_curl_multi *rbcm = ALLOC(ruby_curl_multi);
  if (!rbcm) {
    rb_raise(rb_eNoMemError, "Failed to allocate memory for Curl::Multi");
  }

  ruby_curl_multi_init(rbcm);

  /*
   * The mark routine will be called by the garbage collector during its ``mark'' phase.
   * If your structure references other Ruby objects, then your mark function needs to
   * identify these objects using rb_gc_mark(value). If the structure doesn't reference
   * other Ruby objects, you can simply pass 0 as a function pointer.
  */
  return Data_Wrap_Struct(klass, 0, curl_multi_free, rbcm);
}

.post(urls_with_config, easy_options = {}, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
                  {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
                  {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}],
                 { :follow_location => true, :multipart_form_post => true },
                 {:pipeline => Curl::CURLPIPE_HTTP1}) do|easy|
  easy_handle_on_request_complete
end

Blocking call to POST multiple form’s in parallel.

urls_with_config: is a hash of url’s pointing to the postfields to send easy_options: are a set of common options to set on all easy handles multi_options: options to set on the Curl::Multi handle



38
39
40
41
42
43
44
 
# File 'lib/curl/multi.rb', line 38

def post(urls_with_config, easy_options={}, multi_options={}, &blk)
  url_confs = []
  urls_with_config.each do|uconf|
    url_confs << uconf.merge(:method => :post).merge(easy_options)
  end
  self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
end

.put(urls_with_config, easy_options = {}, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.put([{:url => 'url1', :put_data => "some message"},
                 {:url => 'url2', :put_data => IO.read('filepath')},
                 {:url => 'url3', :put_data => "maybe another string or socket?"],
                 {:follow_location => true},
                 {:pipeline => Curl::CURLPIPE_HTTP1}) do|easy|
  easy_handle_on_request_complete
end

Blocking call to POST multiple form’s in parallel.

urls_with_config: is a hash of url’s pointing to the postfields to send easy_options: are a set of common options to set on all easy handles multi_options: options to set on the Curl::Multi handle



62
63
64
65
66
67
68
 
# File 'lib/curl/multi.rb', line 62

def put(urls_with_config, easy_options={}, multi_options={}, &blk)
  url_confs = []
  urls_with_config.each do|uconf|
    url_confs << uconf.merge(:method => :put).merge(easy_options)
  end
  self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
end

Instance Method Details

#_add(easy) ⇒ Object

multi = Curl::Multi.new easy = Curl::Easy.new(‘url’)

multi.add(easy)

Add an easy handle to the multi stack



271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
 
# File 'ext/curb_multi.c', line 271

VALUE ruby_curl_multi_add(VALUE self, VALUE easy) {
  CURLMcode mcode;
  ruby_curl_easy *rbce;
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  Data_Get_Struct(easy, ruby_curl_easy, rbce);

  /* setup the easy handle */
  ruby_curl_easy_setup( rbce );

  mcode = curl_multi_add_handle(rbcm->handle, rbce->curl);
  if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
    ruby_curl_easy_cleanup(easy, rbce);

    raise_curl_multi_error_exception(mcode);
  }

  rbcm->active++;

  /* Increase the running count, so that the perform loop keeps running.
   * If this number is not correct, the next call to curl_multi_perform will correct it. */
  rbcm->running++;

  /* track a reference to associated multi handle */
  rbce->multi = self;

  return self;
}

#_close ⇒ Object

multi.close after closing the multi handle all connections will be closed and the handle will no longer be usable



1158
1159
1160
1161
1162
1163
1164
 
# File 'ext/curb_multi.c', line 1158

VALUE ruby_curl_multi_close(VALUE self) {
  ruby_curl_multi *rbcm;
  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  curl_multi_cleanup(rbcm->handle);
  ruby_curl_multi_init(rbcm);
  return self;
}

#_remove(rb_easy_handle) ⇒ Object

multi = Curl::Multi.new easy = Curl::Easy.new(‘url’)

multi.add(easy)

# sometime later multi.remove(easy)

Remove an easy handle from a multi stack.

Will raise an exception if the easy handle is not found



315
316
317
318
319
320
321
322
323
 
# File 'ext/curb_multi.c', line 315

VALUE ruby_curl_multi_remove(VALUE self, VALUE rb_easy_handle) {
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);

  rb_curl_multi_remove(rbcm, rb_easy_handle);

  return self;
}

#add(easy) ⇒ Object 



269
270
271
272
273
274
 
# File 'lib/curl/multi.rb', line 269

def add(easy)
  return self if requests[easy.object_id]
  requests[easy.object_id] = easy
  _add(easy)
  self
end

#cancel! ⇒ Object 



255
256
257
258
259
 
# File 'lib/curl/multi.rb', line 255

def cancel!
  requests.each do |_,easy|
    remove(easy)
  end
end

#close ⇒ Object 



283
284
285
286
287
288
289
290
 
# File 'lib/curl/multi.rb', line 283

def close
  requests.values.each {|easy|
    _remove(easy)
  }
  @requests = {}
  _close
  self
end

#idle? ⇒ Boolean

Returns:

  • (Boolean) 


261
262
263
 
# File 'lib/curl/multi.rb', line 261

def idle?
  requests.empty?
end

#max_connects=(count) ⇒ Object

multi = Curl::Multi.new multi.max_connects = 800

Set the max connections in the cache for a multi handle



199
200
201
202
203
204
205
206
207
208
209
 
# File 'ext/curb_multi.c', line 199

static VALUE ruby_curl_multi_max_connects(VALUE self, VALUE count) {
#ifdef HAVE_CURLMOPT_MAXCONNECTS
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);

  curl_multi_setopt(rbcm->handle, CURLMOPT_MAXCONNECTS, NUM2LONG(count));
#endif

  return count;
}

#max_host_connections=(count) ⇒ Object

multi = Curl::Multi.new multi.max_host_connections = 1

Set the max number of connections per host



218
219
220
221
222
223
224
225
226
227
228
 
# File 'ext/curb_multi.c', line 218

static VALUE ruby_curl_multi_max_host_connections(VALUE self, VALUE count) {
#ifdef HAVE_CURLMOPT_MAX_HOST_CONNECTIONS
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);

  curl_multi_setopt(rbcm->handle, CURLMOPT_MAX_HOST_CONNECTIONS, NUM2LONG(count));
#endif

  return count;
}

#perform(*args) ⇒ Object

multi = Curl::Multi.new easy1 = Curl::Easy.new(‘url’) easy2 = Curl::Easy.new(‘url’)

multi.add(easy1) multi.add(easy2)

multi.perform do

# while idle other code my execute here

end

Run multi handles, looping selecting when data can be transfered



849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
 
# File 'ext/curb_multi.c', line 849

VALUE ruby_curl_multi_socket_perform(int argc, VALUE *argv, VALUE self) {
  ruby_curl_multi *rbcm;
  VALUE block = Qnil;
  rb_scan_args(argc, argv, "0&", &block);

  Data_Get_Struct(self, ruby_curl_multi, rbcm);

  multi_socket_ctx ctx;
  ctx.sock_map = st_init_numtable();
  ctx.timeout_ms = -1;

  /* install socket/timer callbacks */
  curl_multi_setopt(rbcm->handle, CURLMOPT_SOCKETFUNCTION, multi_socket_cb);
  curl_multi_setopt(rbcm->handle, CURLMOPT_SOCKETDATA, &ctx);
  curl_multi_setopt(rbcm->handle, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
  curl_multi_setopt(rbcm->handle, CURLMOPT_TIMERDATA, &ctx);

  /* run using socket action loop with ensure-cleanup */
  struct socket_drive_args body_args = { self, rbcm, &ctx, block };
  struct socket_cleanup_args ensure_args = { rbcm, &ctx };
  rb_ensure(ruby_curl_multi_socket_drive_body, (VALUE)&body_args, ruby_curl_multi_socket_drive_ensure, (VALUE)&ensure_args);

  /* finalize */
  rb_curl_multi_read_info(self, rbcm->handle);
  if (block != Qnil) rb_funcall(block, rb_intern("call"), 1, self);
  if (cCurlMutiAutoClose == 1) rb_funcall(self, rb_intern("close"), 0);

  return Qtrue;
}

#pipeline=(method) ⇒ Object

multi = Curl::Multi.new multi.pipeline = true

Pass a long set to 1 for HTTP/1.1 pipelining, 2 for HTTP/2 multiplexing, or 0 to disable.

Enabling pipelining on a multi handle will make it attempt to perform HTTP Pipelining as

far as possible for transfers using this handle. This means that if you add a second request that can use an already existing connection, the second request will be “piped” on the same connection rather than being executed in parallel. (Added in 7.16.0, multiplex added in 7.43.0)



242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
 
# File 'ext/curb_multi.c', line 242

static VALUE ruby_curl_multi_pipeline(VALUE self, VALUE method) {
#ifdef HAVE_CURLMOPT_PIPELINING
  ruby_curl_multi *rbcm;

  long value;

  if (method == Qtrue) {
    value = 1;
  } else if (method == Qfalse) {
    value  = 0;
  } else {
    value = NUM2LONG(method);
  } 

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  curl_multi_setopt(rbcm->handle, CURLMOPT_PIPELINING, value);
#endif
  return method == Qtrue ? 1 : 0;
}

#remove(easy) ⇒ Object 



276
277
278
279
280
281
 
# File 'lib/curl/multi.rb', line 276

def remove(easy)
  return self if !requests[easy.object_id]
  requests.delete(easy.object_id)
  _remove(easy)
  self
end

#requests ⇒ Object 



265
266
267
 
# File 'lib/curl/multi.rb', line 265

def requests
  @requests ||= {}
end