Class: Hyperll::HyperLogLogPlus

Inherits:
Object
  • Object
show all
Defined in:
lib/hyperll/hyper_log_log_plus.rb,
ext/hyperll/hyper_log_log_plus.c

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.new(*args) ⇒ Object 



405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 405

static VALUE rb_hyperllp_new(int argc, VALUE *argv, VALUE klass) {
  VALUE p, sp;
  rb_scan_args(argc, argv, "11", &p, &sp);

  if (NIL_P(sp)) sp = INT2NUM(0);

  hyperllp *hllp = ALLOC(hyperllp);
  hyperllp_init(hllp, NUM2INT(p), NUM2INT(sp));
  VALUE hllpv = Data_Wrap_Struct(klass, 0, hyperllp_free, hllp);

  if (hllp->p < 4) rb_raise(rb_eArgError, "p must be >= 4");
  if (hllp->sp >= 32) rb_raise(rb_eArgError, "sp must be < 32");
  if (hllp->sp != 0 && hllp->p > hllp->sp) rb_raise(rb_eArgError, "p must be <= sp");

  return hllpv;
}

.unserialize(rserialized) ⇒ Object 



305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 305

static VALUE rb_hyperllp_unserialize(VALUE klass, VALUE rserialized) {
  Check_Type(rserialized, T_STRING);

  int size = RSTRING_LEN(rserialized);
  uint8_t *serialized = (uint8_t*)RSTRING_PTR(rserialized);

  // The first four bytes are a version number which we do not use
  int offset = 4;
  int len = 0;

  int p = varint_read_unsigned(serialized + offset, size - offset, &len);
  if (len < 0) {
    rb_raise(rb_eArgError, "invalid serialized p value");
    goto error;
  }
  offset += len;

  int sp = varint_read_unsigned(serialized + offset, size - offset, &len);
  if (len < 0) {
    rb_raise(rb_eArgError, "invalid serialized sp value");
    goto error;
  }
  offset += len;

  int format = varint_read_unsigned(serialized + offset, size - offset, &len);
  if (len < 0) {
    rb_raise(rb_eArgError, "invalid serialized format value");
    goto error;
  }
  offset += len;

  hyperllp *hllp = ALLOC(hyperllp);
  hyperllp_init(hllp, p, sp);
  VALUE hllpv = Data_Wrap_Struct(klass, 0, hyperllp_free, hllp);

  if (format == FORMAT_NORMAL) {
    register_set *rset = hllp->register_set;

    int rsetlen = varint_read_unsigned(serialized + offset, size - offset, &len);
    if (len < 0) {
      rb_raise(rb_eArgError, "invalid register size value");
      goto error;
    }
    offset += len;

    int rsetsize = rsetlen / 4;
    if ((size - offset) < rsetlen) {
      rb_raise(rb_eArgError, "not enough register set bytes (got %d, needed %d)", size - offset, rsetlen);
      goto error;
    } else if (rsetsize != rset->size) {
      rb_raise(rb_eArgError, "register set size does not correspond to p value (got %d, expected %d)", rsetsize, rset->size);
      goto error;
    }

    uint8_t *rsetbytes = (serialized + offset);
    for (int i = 0; i < rsetsize; i++) {
      uint32_t value = (((uint32_t)rsetbytes[i*4 + 0]) << 24) |
                       (((uint32_t)rsetbytes[i*4 + 1]) << 16) |
                       (((uint32_t)rsetbytes[i*4 + 2]) <<  8) |
                       (((uint32_t)rsetbytes[i*4 + 3]));

      rset->values[i] = value;
    }

    hllp->format = FORMAT_NORMAL;
  } else if (format == FORMAT_SPARSE) {
    sparse_set *sset = hllp->sparse_set;

    int ssetsize = varint_read_unsigned(serialized + offset, size - offset, &len);
    if (len < 0) {
      rb_raise(rb_eArgError, "invalid sparse size value");
      goto error;
    } else if (ssetsize > sset->capacity) {
      rb_raise(rb_eArgError, "sparse set size does not correspond to capacity value (got %d, expected %d)", ssetsize, sset->capacity);
      goto error;
    }
    offset += len;

    len = delta_bytes_uncompress(serialized + offset, size - offset, sset->values);
    if (len < 0) {
      rb_raise(rb_eArgError, "invalid sparse set representation");
      goto error;
    } else if (len != ssetsize) {
      rb_raise(rb_eArgError, "did not find as many sparse set values as expected (got %d, expected %d)", len, ssetsize);
      goto error;
    } else {
      sset->size = len;
    }

    hllp->format = FORMAT_SPARSE;
  } else {
    rb_raise(rb_eArgError, "invalid format value (got %d)", format);
    goto error;
  }

  return hllpv;
error:
  return Qnil;
}

Instance Method Details

#cardinality ⇒ Object 



465
466
467
468
469
470
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 465

static VALUE rb_hyperllp_cardinality(VALUE self) {
  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  return INT2NUM(hyperllp_cardinality(hllp));
}

#convert_to_normal ⇒ Object 



523
524
525
526
527
528
529
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 523

static VALUE rb_hyperllp_convert_to_normal(VALUE self) {
  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  hyperllp_convert_to_normal(hllp);
  return self;
}

#format ⇒ Object 



422
423
424
425
426
427
428
429
430
431
432
433
434
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 422

static VALUE rb_hyperllp_format(VALUE self) {
  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  switch(hllp->format) {
    case FORMAT_NORMAL:
      return ID2SYM(rb_intern("normal"));
    case FORMAT_SPARSE:
      return ID2SYM(rb_intern("sparse"));
  }

  return Qnil;
}

#format=(format) ⇒ Object 



436
437
438
439
440
441
442
443
444
445
446
447
448
449
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 436

static VALUE rb_hyperllp_format_set(VALUE self, VALUE format) {
  Check_Type(format, T_SYMBOL);

  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  if (format == ID2SYM(rb_intern("normal"))) {
    hllp->format = FORMAT_NORMAL;
  } else if (format == ID2SYM(rb_intern("sparse"))) {
    hllp->format = FORMAT_SPARSE;
  }

  return format;
}

#merge(*args) ⇒ Object 



472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 472

static VALUE rb_hyperllp_merge(int argc, VALUE *argv, VALUE self) {
  VALUE others;
  rb_scan_args(argc, argv, "*", &others);

  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  int len = RARRAY_LEN(others);
  for (int i = 0; i < len; i++) {
    hyperllp *ohllp;
    Data_Get_Struct(rb_ary_entry(others, i), hyperllp, ohllp);
    if (!ohllp || hllp->p != ohllp->p) {
      rb_raise(rb_eArgError, "cannot merge estimators of different sizes");
      return Qnil;
    } else {
      hyperllp_merge(hllp, ohllp);
    }
  }

  return self;
}

#p ⇒ Object 



451
452
453
454
455
456
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 451

static VALUE rb_hyperllp_p(VALUE self) {
  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  return INT2NUM(hllp->p);
}

#serialize ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/hyperll/hyper_log_log_plus.rb', line 5

def serialize
  str = ""
  str << [-2].pack("N") # -VERSION
  str << Varint.write_unsigned_var_int(p).pack("C*")
  str << Varint.write_unsigned_var_int(sp).pack("C*")

  case format
  when :normal
    str << Varint.write_unsigned_var_int(0).pack("C*")

    rs_bytes = raw_register_set
    str << Varint.write_unsigned_var_int(rs_bytes.length * 4).pack("C*")
    str << rs_bytes.pack("N*")
  when :sparse
    str << Varint.write_unsigned_var_int(1).pack("C*")

    ss_bytes = raw_sparse_set
    str << Varint.write_unsigned_var_int(ss_bytes.length).pack("C*")
    str << DeltaBytes.compress(ss_bytes).pack("C*")
  end

  str
end

#sp ⇒ Object 



458
459
460
461
462
463
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 458

static VALUE rb_hyperllp_sp(VALUE self) {
  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  return INT2NUM(hllp->sp);
}