Class: Curl::Multi

Inherits:
Object
  • Object
show all
Defined in:
lib/curb.rb,
ext/curb_multi.c

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.get(urls, easy_options = {}, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.get('url1','url2','url3','url4','url5', :follow_location => true) do|easy|
  easy
end

Blocking call to fetch multiple url’s in parallel.



65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/curb.rb', line 65

def get(urls, easy_options={}, multi_options={}, &blk)
  m = Curl::Multi.new
  # configure the multi handle
  multi_options.each do|k,v|
    m.send("#{k}=", v)
  end

  # create and configure each easy handle
  urls.each do|url|
    c = Curl::Easy.new(url)
    easy_options.each do|k,v|
      c.send("#{k}=",v)
    end
    c.on_complete {|curl| blk.call curl } if blk
    m.add(c)
  end
  m.perform
end

.Curl::Multi.new ⇒ #<Curl::Easy...>

Create a new Curl::Multi instance

Returns:



80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'ext/curb_multi.c', line 80

VALUE ruby_curl_multi_new(VALUE klass) {
  VALUE new_curlm;

  ruby_curl_multi *rbcm = ALLOC(ruby_curl_multi);

  rbcm->handle = curl_multi_init();

  rbcm->requests = rb_hash_new();

  rbcm->active = 0;
  rbcm->running = 0;

  new_curlm = Data_Wrap_Struct(klass, curl_multi_mark, curl_multi_free, rbcm);

  return new_curlm;
}

.post(urls_with_config, easy_options, multi_options, &blk) ⇒ Object

call-seq:

Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
                  {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
                  {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}],
                 { :follow_location => true, :multipart_form_post => true },
                 {:pipeline => true }) do|easy|
  easy_handle_on_request_complete
end

Blocking call to POST multiple form’s in parallel.

urls_with_config: is a hash of url’s pointing to the postfields to send easy_options: are a set of common options to set on all easy handles multi_options: options to set on the Curl::Multi handle



100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/curb.rb', line 100

def post(urls_with_config, easy_options, multi_options, &blk)
  m = Curl::Multi.new
  # configure the multi handle
  multi_options.each do|k,v|
    m.send("#{k}=", v)
  end

  urls_with_config.each do|conf|
    c = conf.dup # avoid being destructive to input
    url    = c.delete(:url)
    fields = c.delete(:post_fields)
    easy   = Curl::Easy.new(url)
    # set the post post using the url fields
    easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&')
    # configure the easy handle
    easy_options.each do|k,v|
      easy.send("#{k}=",v)
    end
    easy.on_complete {|curl| blk.call curl } if blk
    m.add(easy)
  end
  m.perform
end

Instance Method Details

#add(easy) ⇒ Object

multi = Curl::Multi.new easy = Curl::Easy.new(‘url’)

multi.add(easy)

Add an easy handle to the multi stack



190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
 
# File 'ext/curb_multi.c', line 190

VALUE ruby_curl_multi_add(VALUE self, VALUE easy) {
  CURLMcode mcode;
  ruby_curl_easy *rbce;
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  Data_Get_Struct(easy, ruby_curl_easy, rbce);

  mcode = curl_multi_add_handle(rbcm->handle, rbce->curl);
  if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
    raise_curl_multi_error_exception(mcode);
  }

  /* save a pointer to self */
  rbce->self = easy;

  /* setup the easy handle */
  ruby_curl_easy_setup( rbce, &(rbce->bodybuf), &(rbce->headerbuf), &(rbce->curl_headers) );

  rbcm->active++;
  if (mcode == CURLM_CALL_MULTI_PERFORM) {
    curl_multi_perform(rbcm->handle, &(rbcm->running));
  }

  rb_hash_aset( rbcm->requests, easy, easy );
  // active should equal INT2FIX(RHASH(rbcm->requests)->tbl->num_entries)

  if (rbcm->active > rbcm->running) {
    rb_curl_multi_read_info(self, rbcm->handle);
  }

  return self;
}

#cancel! ⇒ Object

Cancels all requests currently being made on this Curl::Multi handle.



288
289
290
291
292
293
294
295
296
297
 
# File 'ext/curb_multi.c', line 288

static VALUE ruby_curl_multi_cancel(VALUE self) {
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  
  rb_hash_foreach( rbcm->requests, ruby_curl_multi_cancel_callback, (VALUE)rbcm );
  
  // for chaining
  return self;
}

#idle? ⇒ Boolean

Returns whether or not this Curl::Multi handle is processing any requests. E.g. this returns true when multi.requests.length == 0.

Returns:

  • (Boolean) 


130
131
132
133
134
135
136
137
138
139
140
 
# File 'ext/curb_multi.c', line 130

static VALUE ruby_curl_multi_idle(VALUE self) {
  ruby_curl_multi *rbcm;
  
  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  
  if ( FIX2INT( rb_funcall(rbcm->requests, rb_intern("length"), 0) ) == 0 ) {
    return Qtrue;
  } else {
    return Qfalse;
  }
}

#max_connects=(count) ⇒ Object

multi = Curl::Multi.new multi.max_connects = 800

Set the max connections in the cache for a multi handle



149
150
151
152
153
154
155
156
157
158
 
# File 'ext/curb_multi.c', line 149

static VALUE ruby_curl_multi_max_connects(VALUE self, VALUE count) {
#ifdef HAVE_CURLMOPT_MAXCONNECTS
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  curl_multi_setopt(rbcm->handle, CURLMOPT_MAXCONNECTS, NUM2INT(count));
#endif

  return count;
}

#perform ⇒ Object

multi = Curl::Multi.new easy1 = Curl::Easy.new(‘url’) easy2 = Curl::Easy.new(‘url’)

multi.add(easy1) multi.add(easy2)

multi.perform do

# while idle other code my execute here

end

Run multi handles, looping selecting when data can be transfered



384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
 
# File 'ext/curb_multi.c', line 384

VALUE ruby_curl_multi_perform(VALUE self) {
  CURLMcode mcode;
  ruby_curl_multi *rbcm;
  int maxfd, rc;
  fd_set fdread, fdwrite, fdexcep;

  long timeout_milliseconds;
  struct timeval tv = {0, 0};

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  //rb_gc_mark(self);

  rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );

  while(rbcm->running) {
    FD_ZERO(&fdread);
    FD_ZERO(&fdwrite);
    FD_ZERO(&fdexcep);

    /* load the fd sets from the multi handle */
    mcode = curl_multi_fdset(rbcm->handle, &fdread, &fdwrite, &fdexcep, &maxfd);
    if (mcode != CURLM_OK) {
      raise_curl_multi_error_exception(mcode);
    }

#ifdef HAVE_CURL_MULTI_TIMEOUT
    /* get the curl suggested time out */
    mcode = curl_multi_timeout(rbcm->handle, &timeout_milliseconds);
    if (mcode != CURLM_OK) {
      raise_curl_multi_error_exception(mcode);
    }
#else
    /* libcurl doesn't have a timeout method defined... make a wild guess */
    timeout_milliseconds = -1;
#endif
    //printf("libcurl says wait: %ld ms or %ld s\n", timeout_milliseconds, timeout_milliseconds/1000);

    if (timeout_milliseconds == 0) { /* no delay */
      rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
      continue;
    }
    else if(timeout_milliseconds < 0) {
      timeout_milliseconds = 500; /* wait half a second, libcurl doesn't know how long to wait */
    }
#ifdef __APPLE_CC__
    if(timeout_milliseconds > 1000) {
      timeout_milliseconds = 1000; /* apple libcurl sometimes reports huge timeouts... let's cap it */
    }
#endif

    tv.tv_sec = timeout_milliseconds / 1000; // convert milliseconds to seconds
    tv.tv_usec = (timeout_milliseconds % 1000) * 1000; // get the remainder of milliseconds and convert to micro seconds

    rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
    switch(rc) {
    case -1:
      rb_raise(rb_eRuntimeError, "select(): %s", strerror(errno));
      break;
    case 0:
      if (rb_block_given_p()) {
        rb_yield(self);
      }
    default: 
      rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
      break;
    }

  }

  return Qtrue;
}

#pipeline=(onoff) ⇒ Object

multi = Curl::Multi.new multi.pipeline = true

Pass a long set to 1 to enable or 0 to disable. Enabling pipelining on a multi handle will make it attempt to perform HTTP Pipelining as far as possible for transfers using this handle. This means that if you add a second request that can use an already existing connection, the second request will be “piped” on the same connection rather than being executed in parallel. (Added in 7.16.0)



171
172
173
174
175
176
177
178
179
 
# File 'ext/curb_multi.c', line 171

static VALUE ruby_curl_multi_pipeline(VALUE self, VALUE onoff) {
#ifdef HAVE_CURLMOPT_PIPELINING
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  curl_multi_setopt(rbcm->handle, CURLMOPT_PIPELINING, onoff == Qtrue ? 1 : 0);
#endif
  return onoff;
}

#remove(easy) ⇒ Object

multi = Curl::Multi.new easy = Curl::Easy.new(‘url’)

multi.add(easy)

# sometime later multi.remove(easy)

Remove an easy handle from a multi stack.

Will raise an exception if the easy handle is not found



238
239
240
241
242
243
244
245
246
247
248
249
 
# File 'ext/curb_multi.c', line 238

VALUE ruby_curl_multi_remove(VALUE self, VALUE easy) {
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);

  ruby_curl_easy *rbce;
  Data_Get_Struct(easy, ruby_curl_easy, rbce);

  rb_curl_multi_remove(rbcm,easy);

  return self;
}

#requests ⇒ Array

Returns an array containing all the active requests on this Curl::Multi object.

Returns:

  • (Array) 


110
111
112
113
114
115
116
117
118
119
120
121
 
# File 'ext/curb_multi.c', line 110

static VALUE ruby_curl_multi_requests(VALUE self) {
  ruby_curl_multi *rbcm;
  
  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  
  VALUE result_array = rb_ary_new();
  
  // iterate over the requests hash, and stuff references into the array.
  rb_hash_foreach( rbcm->requests, ruby_curl_multi_requests_callback, result_array );
  
  return result_array;
}