Class: Hyperll::HyperLogLogPlus

Inherits:
Object
  • Object
show all
Defined in:
lib/hyperll/hyper_log_log_plus.rb,
ext/hyperll/hyper_log_log_plus.c

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.new(*args) ⇒ Object 



406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 406

static VALUE rb_hyperllp_new(int argc, VALUE *argv, VALUE klass) {
  VALUE p, sp;
  rb_scan_args(argc, argv, "11", &p, &sp);

  if (NIL_P(sp)) sp = INT2NUM(0);

  hyperllp *hllp = ALLOC(hyperllp);
  hyperllp_init(hllp, NUM2INT(p), NUM2INT(sp));
  VALUE hllpv = Data_Wrap_Struct(klass, 0, hyperllp_free, hllp);

  if (hllp->p < 4) rb_raise(rb_eArgError, "p must be >= 4");
  if (hllp->sp >= 32) rb_raise(rb_eArgError, "sp must be < 32");
  if (hllp->sp != 0 && hllp->p > hllp->sp) rb_raise(rb_eArgError, "p must be <= sp");

  return hllpv;
}

.unserialize(rserialized) ⇒ Object 



305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 305

static VALUE rb_hyperllp_unserialize(VALUE klass, VALUE rserialized) {
  Check_Type(rserialized, T_STRING);

  int size = RSTRING_LEN(rserialized);
  uint8_t *serialized = (uint8_t*)RSTRING_PTR(rserialized);

  // The first four bytes are a version number which we do not use
  int offset = 4;
  int len = 0;

  int p = varint_read_unsigned(serialized + offset, size - offset, &len);
  if (len < 0) {
    rb_raise(rb_eArgError, "invalid serialized p value");
    goto error;
  }
  offset += len;

  int sp = varint_read_unsigned(serialized + offset, size - offset, &len);
  if (len < 0) {
    rb_raise(rb_eArgError, "invalid serialized sp value");
    goto error;
  }
  offset += len;

  int format = varint_read_unsigned(serialized + offset, size - offset, &len);
  if (len < 0) {
    rb_raise(rb_eArgError, "invalid serialized format value");
    goto error;
  }
  offset += len;

  hyperllp *hllp = ALLOC(hyperllp);
  hyperllp_init(hllp, p, sp);
  VALUE hllpv = Data_Wrap_Struct(klass, 0, hyperllp_free, hllp);

  if (format == FORMAT_NORMAL) {
    register_set *rset = hllp->register_set;

    int rsetlen = varint_read_unsigned(serialized + offset, size - offset, &len);
    if (len < 0) {
      rb_raise(rb_eArgError, "invalid register size value");
      goto error;
    }
    offset += len;

    int rsetsize = rsetlen / 4;
    if ((size - offset) < rsetlen) {
      rb_raise(rb_eArgError, "not enough register set bytes (got %d, needed %d)", size - offset, rsetlen);
      goto error;
    } else if (rsetsize != rset->size) {
      rb_raise(rb_eArgError, "register set size does not correspond to p value (got %d, expected %d)", rsetsize, rset->size);
      goto error;
    }

    uint8_t *rsetbytes = (serialized + offset);
    for (int i = 0; i < rsetsize; i++) {
      uint32_t value = (((uint32_t)rsetbytes[i*4 + 0]) << 24) |
                       (((uint32_t)rsetbytes[i*4 + 1]) << 16) |
                       (((uint32_t)rsetbytes[i*4 + 2]) <<  8) |
                       (((uint32_t)rsetbytes[i*4 + 3]));

      rset->values[i] = value;
    }

    hllp->format = FORMAT_NORMAL;
  } else if (format == FORMAT_SPARSE) {
    sparse_set *sset = hllp->sparse_set;

    int ssetsize = varint_read_unsigned(serialized + offset, size - offset, &len);
    if (len < 0) {
      rb_raise(rb_eArgError, "invalid sparse size value");
      goto error;
    } else if (ssetsize > sset->capacity) {
      rb_raise(rb_eArgError, "sparse set size does not correspond to capacity value (got %d, expected %d)", ssetsize, sset->capacity);
      goto error;
    }
    offset += len;

    len = delta_bytes_uncompress(serialized + offset, size - offset, sset->values);
    if (len < 0) {
      rb_raise(rb_eArgError, "invalid sparse set representation");
      goto error;
    } else if (len != ssetsize) {
      rb_raise(rb_eArgError, "did not find as many sparse set values as expected (got %d, expected %d)", len, ssetsize);
      goto error;
    } else {
      sset->size = len;
    }

    hllp->format = FORMAT_SPARSE;
  } else {
    rb_raise(rb_eArgError, "invalid format value (got %d)", format);
    goto error;
  }

  RB_GC_GUARD(rserialized);
  return hllpv;
error:
  return Qnil;
}

Instance Method Details

#cardinality ⇒ Object 



466
467
468
469
470
471
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 466

static VALUE rb_hyperllp_cardinality(VALUE self) {
  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  return INT2NUM(hyperllp_cardinality(hllp));
}

#convert_to_normal ⇒ Object 



524
525
526
527
528
529
530
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 524

static VALUE rb_hyperllp_convert_to_normal(VALUE self) {
  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  hyperllp_convert_to_normal(hllp);
  return self;
}

#format ⇒ Object 



423
424
425
426
427
428
429
430
431
432
433
434
435
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 423

static VALUE rb_hyperllp_format(VALUE self) {
  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  switch(hllp->format) {
    case FORMAT_NORMAL:
      return ID2SYM(rb_intern("normal"));
    case FORMAT_SPARSE:
      return ID2SYM(rb_intern("sparse"));
  }

  return Qnil;
}

#format=(format) ⇒ Object 



437
438
439
440
441
442
443
444
445
446
447
448
449
450
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 437

static VALUE rb_hyperllp_format_set(VALUE self, VALUE format) {
  Check_Type(format, T_SYMBOL);

  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  if (format == ID2SYM(rb_intern("normal"))) {
    hllp->format = FORMAT_NORMAL;
  } else if (format == ID2SYM(rb_intern("sparse"))) {
    hllp->format = FORMAT_SPARSE;
  }

  return format;
}

#merge(*args) ⇒ Object 



473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 473

static VALUE rb_hyperllp_merge(int argc, VALUE *argv, VALUE self) {
  VALUE others;
  rb_scan_args(argc, argv, "*", &others);

  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  int len = RARRAY_LEN(others);
  for (int i = 0; i < len; i++) {
    hyperllp *ohllp;
    Data_Get_Struct(rb_ary_entry(others, i), hyperllp, ohllp);
    if (!ohllp || hllp->p != ohllp->p) {
      rb_raise(rb_eArgError, "cannot merge estimators of different sizes");
      return Qnil;
    } else {
      hyperllp_merge(hllp, ohllp);
    }
  }

  return self;
}

#p ⇒ Object 



452
453
454
455
456
457
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 452

static VALUE rb_hyperllp_p(VALUE self) {
  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  return INT2NUM(hllp->p);
}

#serialize ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/hyperll/hyper_log_log_plus.rb', line 5

def serialize
  str = ""
  str << [-2].pack("N") # -VERSION
  str << Varint.write_unsigned_var_int(p).pack("C*")
  str << Varint.write_unsigned_var_int(sp).pack("C*")

  case format
  when :normal
    str << Varint.write_unsigned_var_int(0).pack("C*")

    rs_bytes = raw_register_set
    str << Varint.write_unsigned_var_int(rs_bytes.length * 4).pack("C*")
    str << rs_bytes.pack("N*")
  when :sparse
    str << Varint.write_unsigned_var_int(1).pack("C*")

    ss_bytes = raw_sparse_set
    str << Varint.write_unsigned_var_int(ss_bytes.length).pack("C*")
    str << DeltaBytes.compress(ss_bytes).pack("C*")
  end

  str
end

#sp ⇒ Object 



459
460
461
462
463
464
 
# File 'ext/hyperll/hyper_log_log_plus.c', line 459

static VALUE rb_hyperllp_sp(VALUE self) {
  hyperllp *hllp;
  Data_Get_Struct(self, hyperllp, hllp);

  return INT2NUM(hllp->sp);
}