Class: CACategorical

Inherits:
CAObject show all
Defined in:
lib/carray/categorical.rb

Overview

Categorical column: dense integer codes plus a label vocabulary, so each element is an index into a small set of categories. Structurally the same idea as a pandas Categorical or an Arrow dictionary array.

Implemented as a read-only non-numeric Face over the codes array — the storage is the integer codes, while the surface is CA_FIXLEN so numeric kernels are gated off (cat + 1 raises; arithmetic on category codes is not meaningful).

Constant Summary collapse

UNPACK_FORMAT =

raw-byte unpack format per storage (codes) data_type, native endian. The FIXLEN surface delivers a per-cell fetch as an N-byte String; this decodes it back into the integer code.

{
  CA_INT8  => "c", CA_UINT8  => "C",
  CA_INT16 => "s", CA_UINT16 => "S",
  CA_INT32 => "l", CA_UINT32 => "L",
  CA_INT64 => "q", CA_UINT64 => "Q",
}.freeze
SENTINEL =

The exclusion sentinel per codes data_type: the all-ones bit pattern, read as type-max for an unsigned data type and as -1 for a signed one. Either way it is out of every valid [0, k) range and byte-identical to a pandas / Arrow missing code.

{
  CA_UINT8  => 0xFF,               CA_INT8  => -1,
  CA_UINT16 => 0xFFFF,             CA_INT16 => -1,
  CA_UINT32 => 0xFFFFFFFF,         CA_INT32 => -1,
  CA_UINT64 => 0xFFFFFFFFFFFFFFFF, CA_INT64 => -1,
}.freeze

Constants inherited from CArray

CArray::HAVE_COMPLEX, CArray::LAZY_BINCMP_OP_IDS, CArray::LAZY_BINCMP_TOL_OP_IDS, CArray::LAZY_BINOP_OP_IDS, CArray::LAZY_MONOP_OP_IDS, CArray::LAZY_TRIOP_OP_IDS, CArray::UNSPECIFIED, CArray::VERSION

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from CArray

#%, #/, #<=>, #==, #T, #[], #[]=, _scan_float, _scan_int, #addr2index, addr2index, #address, align_addr, align_nearest_addr, #all, #all_masked?, #ancestors, #any, #any_masked?, #as_boolean, #as_byte, #as_cmplx128, #as_cmplx256, #as_cmplx64, #as_complex, #as_dcomplex, #as_double, #as_fixlen, #as_float, #as_float128, #as_float32, #as_float64, #as_int, #as_int16, #as_int32, #as_int64, #as_int8, #as_object, #as_short, #as_strided, #as_type, #as_uint16, #as_uint32, #as_uint64, #as_uint8, attach, #attach, #attach!, attach!, #attached?, #attr, #attrs, #axis2addr, #axis_group, big_endian?, #bin, #bin_to, #bincount_nd, #bitarray, #bitfield, #bits, #block_view, #blocks, boolean, #boolean, #boolean?, #broadcast_to, #bsearch, #bsearch_addr, #byte, byte, #bytes, cast, #cast_with, #categorize, #choose, #clip, #clip_int16, #clip_int32, #clip_int64, #clip_int8, #clip_uint16, #clip_uint32, #clip_uint64, #clip_uint8, cmplx128, #cmplx128, cmplx64, #cmplx64, #coerce, #collect!, #collect_addr!, #collect_index!, #collect_with_addr!, #collect_with_index!, complex, #complex, #complex?, concatenate, #concatenate, #conditional, const_string, #convert, #copy, #count_masked, #count_not_masked, #crop, #data_class, #data_class=, #data_type, data_type_code, data_type_name, #data_type_name, dcomplex, #dcomplex, #delete_block, #diagonal, #dim, #dim0, #dim1, #dim2, #dim3, #dim_view, #div, #div!, #divmod, double, #double, #drop_axis, dump, #dump_binary, #each, #each_addr, each_index, #each_index, #each_slab, #each_with_addr, #each_with_index, #elem_copy, #elem_decr, #elem_fetch, #elem_incr, #elem_mask, #elem_masked?, #elem_max, #elem_min, #elem_store, #elem_swap, #elem_unmask, #elements, #empty?, endian, #endian, #entity?, #eql?, #face?, #fake, #false, #farray, #field, #fields, #fields_at, #fill, #fill_copy, #first, fixlen, #fixlen, #fixlen?, fixlen_string, #flags, #flatten, #flip, float, #float, float32, #float32, float64, #float64, #float?, #fmod, #fmod!, #format, format, #frac, #freeze, #from_bit_string, from_bit_string, from_memory_view, fuse, #gather_nd, #grid, #group_by_category, #group_by_run, guard_undef, guess_type_and_bytes, #has_attr?, #has_data_class?, #has_mask?, #hash, #histogram, #histogram1d, #histogram2d, #imag, #imag=, #index, index2addr, #index2addr, #indices, #inherit_mask, #inherit_mask_replace, #initialize_copy, #insert_axis, #insert_block, #int, int, #int16, int16, #int32, int32, #int64, int64, #int8, int8, #integer?, #invert_mask, #is_masked, #is_mode, #is_not_masked, #is_real, jit_each, jit_for, jit_map, #join, #last, #lazy, #length, #linear_fetch, #linear_section, little_endian?, load, load_arrow_tensor, #load_binary, #locate_nearest_addr, #lookup, #map!, #map_addr!, #map_index!, #map_slab, #map_with_addr!, #map_with_index!, #marshal_dump, #marshal_load, #mask, #mask=, #mask_array?, #mask_duplicates, #mask_eq, #mask_invalid, #mask_where, #median, meld, #meld, #members, memory_view_available?, memory_view_reject_reason, meshgrid, #mod, #mod!, montage, mosaic, #ndim, #none, #normalize_axes, #normalize_axis, normalize_axis, #normalize_index, #numeric?, #nunique, #obj_type, #object, object, #object?, #order, #pack_bits, #parent, #partition, #partition_copy, #paste, #percentile, #project, promote_list, #put_nd, #quantile, #random, #random!, #randomn, #randomn!, #rank, #read_only?, #real, #real=, #real?, #reduce_slab, #refer, #replace_where, #reshape, #resize, result_type, #roll, #root_array, #round, #same_shape?, save, #save_arrow_tensor, #scalar?, #scale, #scale!, #scatter_add!, #scatter_max!, #scatter_min!, #scatter_mul!, #scatter_replace!, #scatter_sub!, #search, #search_addr, #search_nearest, #search_nearest_addr, select, #seq, #seq!, #set_attr, #set_read_only_flag, #shape, #shift, short, #short, #shuffle, #shuffle!, #size, sizeof, #sliding_windows, #snap, #snap_to, #sort, sort_addr, #sort_copy, #source_code, #span, #span!, #split, #st, stack, #stack, string, #strip_mask, struct, #swap_bytes, tabulate, #template, #then_else, #tile, time, #time, time_range, time_series, #timedelta, #to_a, #to_bit_string, #to_ca, #to_s, #to_type, #transpose, #true, uint16, #uint16, uint32, #uint32, uint64, #uint64, uint8, #uint8, #unfold, union, #unmask, #valid_addr?, #valid_index?, #validity_bits, #value, #value_array?, #virtual?, #where, #window, #windows, wrap, wrap_memory_view, wrap_readonly, wrap_writable

Methods included from CArray::DataTypeExtension

#arange, #empty, #eye, #full, #identity, #linspace, #ones, #zeros

Constructor Details

#initialize(codes, labels) ⇒ CACategorical

codes : integer CArray, the storage parent. Excluded cells are both masked AND store the type-max sentinel value (= the all-ones bit pattern, which is signed -1 byte-for-byte — the pandas / Arrow missing code). Because the Face is READONLY the two never desync, so consumers may rely on either: the mask (CArray-native) or the sentinel (axis-group's out-of-range skip, zero-copy export). labels: Array | CArray, the vocabulary; labels = category.

Allocates a READONLY CACategorical Face whose storage is codes and whose vocabulary is labels. The label list is copied and frozen so codes always index a stable vocabulary.

Parameters:

  • codes (CArray) —

    integer code storage.

  • labels (Array, CArray) —

    category vocabulary.



126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/carray/categorical.rb', line 126

def initialize(codes, labels)
  # Own a frozen copy of the vocabulary: the categorical is READONLY and its
  # codes index into labels, so the label list must not change under it. We
  # copy first so a caller's array is never frozen as a side effect; the
  # label objects themselves are left untouched (container-level freeze).
  @labels = (labels.respond_to?(:to_a) ? labels.to_a : Array(labels)).dup.freeze
  super(CA_FIXLEN, codes.dim,
        bytes:     codes.bytes,
        storage:   codes.data_type,
        parent:    codes,
        read_only: true,
        face:      true)
  # Mark the codes storage read-only so the READONLY guarantee holds at the
  # root, not just on the Face. Without this the Face is read_only but its
  # parent is writable, so `cat.codes[i] = x` silently mutates the categorical
  # (and any grouping cache derived from it). We set the CA_FLAG_READ_ONLY flag
  # rather than #freeze: freeze also freezes the Ruby object, which propagates
  # through views/Faces (a reshape of frozen codes is frozen) and would block
  # the grouping cache from memoising. The flag gives the same write protection
  # (mutations raise) while keeping the object non-frozen. One-way: it takes
  # ownership of `codes` (categorize / from_codes build or receive it, mask
  # already derived above); a caller keeping a mutable array must pass `.copy`.
  codes.set_read_only_flag
end

Instance Attribute Details

#labels ⇒ Object (readonly)

Returns the value of attribute labels.



151
152
153
 
# File 'lib/carray/categorical.rb', line 151

def labels
  @labels
end

Class Method Details

.from_codes(codes, labels) ⇒ CACategorical

Wrap already-dense codes + labels with no discovery — the import receiver for a pandas Categorical or an Arrow dictionary. codes becomes the Face's storage parent verbatim (zero-copy when it is a wrapped memory view), and from_codes takes ownership of it.

Excluded cells are identified by the all-ones sentinel value (type-max for unsigned codes, -1 for signed — both the pandas / Arrow missing code) and masked here, so the categorical is well-formed regardless of whether the caller pre-masked. Only the mask buffer is touched; the code bytes are left intact (so a pandas byte-reinterpret round-trips).

Returns a CACategorical wrapping already-dense integer codes with the given labels, without discovery. codes becomes the Face's storage parent; excluded cells (identified by the type-max sentinel value) are masked automatically.

Parameters:

  • codes (CArray) —

    integer code storage.

  • labels (Array, CArray) —

    category vocabulary indexed by code.

Returns:

Raises:

  • (ArgumentError) —

    when codes is not an integer CArray.



100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/carray/categorical.rb', line 100

def from_codes(codes, labels)
  unless codes.is_a?(CArray) && SENTINEL.key?(codes.data_type)
    got = codes.is_a?(CArray) ? codes.data_type : codes.class
    raise ArgumentError, "from_codes: codes must be an integer CArray (got #{got})"
  end
  excluded = codes.eq(SENTINEL[codes.data_type])
  if excluded.count(true) > 0
    codes.mask = codes.has_mask? ? (codes.mask | excluded) : excluded
  end
  new(codes, labels)
end

Instance Method Details

#build_grouping ⇒ self

Force-build (and cache) the whole grouping plan up front — sort_addr, reduceat_index, category_sizes — for prepare-ahead use before a batch of groupings. Lazy building already covers correctness; this is the explicit "pay the counting sort now" handle (e.g. right after df.group_by(col)). Returns self so it chains.

Eagerly builds and caches the grouping plan (sort_addr / reduceat_index / category_sizes). Optional — the plan is built lazily on first grouping access — but useful to pay the counting sort once ahead of a batch.

Returns:

  • (self) 


403
404
405
406
407
 
# File 'lib/carray/categorical.rb', line 403

def build_grouping
  sort_addr        # pulls category_sizes; reduceat_index shares category_sizes
  reduceat_index
  self
end

#category_sizes ⇒ CArray Also known as: bincount

Per-category counts as a length-k array aligned to #labels. Trailing empty categories are kept as 0 (unlike codes.bincount, which truncates them), so labels.zip(category_sizes.to_a) always pairs up.

Memoised: the codes are read-only (immutable storage), so the counts are a pure function of the categorical and stay valid for its lifetime. The same cached array backs #reduceat_index and #sort_addr, so a wide aggregate over the same categorical pays the count once. (Do not mutate the returned array — it is shared; take .copy for a scratch buffer.) A derived-view / composite categorical caches too: read-only rides from the codes as a flag, not a Ruby freeze, so the object stays non-frozen and the memo ivar sticks.

Returns per-category counts as a CArray with one entry per #labels aligned to #labels. Trailing empty categories are kept as 0 so labels.zip(category_sizes.to_a) always pairs up.

Returns:



252
253
254
255
256
257
258
259
 
# File 'lib/carray/categorical.rb', line 252

def category_sizes
  return @_category_sizes if @_category_sizes
  bc  = codes.bincount
  out = CArray.new(bc.data_type, [@labels.size])   # new zero-fills
  out[0...bc.elements] = bc if bc.elements > 0
  @_category_sizes = out
  out
end

#codes ⇒ CArray

The raw integer codes (= the storage parent). On a derived view this is the correspondingly sliced/reshaped codes, since codes ride the chain. Excluded cells are masked and store the type-max sentinel; the same array serves the axis-group kernel (out-of-range skip) and the pandas / Arrow bridge (byte-reinterpret to signed -1) with no conversion.

Returns the raw integer code CArray backing self. Excluded cells are masked and store the type-max sentinel. The array is read-only — the categorical owns immutable codes, so codes[i] = x raises; use codes.copy for a mutable copy.

Returns:



164
165
166
 
# File 'lib/carray/categorical.rb', line 164

def codes
  parent
end

#copy_state(src) ⇒ Object

Face hook: carry the vocabulary across lifted views (slice / reshape / …). The codes ride along automatically as the Face's parent.



170
171
172
 
# File 'lib/carray/categorical.rb', line 170

def copy_state(src)
  @labels = src.labels
end

#count(label) ⇒ Integer

Returns the number of cells whose category equals label (0 for an unknown label).

Parameters:

  • label (Object) —

    category to count.

Returns:

  • (Integer) 


231
232
233
234
 
# File 'lib/carray/categorical.rb', line 231

def count(label)
  code = @labels.index(label)
  code ? codes.count(code) : 0
end

#difference(other, sort: false) ⇒ CArray

Returns object CArray of the labels only self has.

Returns:

  • (CArray) —

    object CArray of the labels only self has.



334
335
336
 
# File 'lib/carray/categorical.rb', line 334

def difference (other, sort: false)
  unique.difference(label_array(other), sort: sort)
end

#eq(label) ⇒ CArray

Boolean mask of cells whose category == label. Excluded cells stay UNDEF (their category is unknown); an unknown label yields an all-false mask.

Returns a boolean CArray marking cells whose category equals label. Excluded cells stay masked; an unknown label yields an all-false result.

Parameters:

  • label (Object) —

    category to match.

Returns:



214
215
216
 
# File 'lib/carray/categorical.rb', line 214

def eq(label)
  codes.eq(@labels.index(label) || @labels.size)
end

#face_state_compatible?(other) ⇒ Boolean

Face hook: the homogeneity gate for multi-parent constructions (CArray.promote_list / CArray.stack / anything that Face-lifts a list). A code only means anything against the vocabulary it was assigned from, so two categoricals may share one lifted Face only when they index the same labels in the same code order.

Same labels in a different code order is refused rather than re-coded, for the same reason CATime refuses a unit mismatch it knows how to convert: this is a predicate consulted after the parents are assembled, with no channel to rewrite storage — and the codes are read-only by construction, so agreeing would mean silently materialising fresh codes inside what the caller asked for as a view. Build the shared vocabulary up front instead: keys.categorize(labels: shared).

Returns:

  • (Boolean) 


187
188
189
 
# File 'lib/carray/categorical.rb', line 187

def face_state_compatible?(other)
  other.is_a?(CACategorical) && @labels == other.labels
end

#inspect ⇒ String

Returns a short summary showing element count, category count, and vocabulary.

Returns:

  • (String) 


484
485
486
 
# File 'lib/carray/categorical.rb', line 484

def inspect
  "#<CACategorical n=#{elements} categories=#{@labels.size} labels=#{@labels.inspect}>"
end

#intersection(other, sort: false) ⇒ CArray

Returns object CArray of the labels present in both.

Returns:

  • (CArray) —

    object CArray of the labels present in both.



328
329
330
 
# File 'lib/carray/categorical.rb', line 328

def intersection (other, sort: false)
  unique.intersection(label_array(other), sort: sort)
end

#is_in(values) ⇒ CArray

Returns a boolean CArray, true where the cell's label is in values. An unknown label matches nothing. Masked cells stay masked.

Parameters:

Returns:



321
322
323
324
 
# File 'lib/carray/categorical.rb', line 321

def is_in (values)
  wanted = label_list(values).filter_map { |l| @labels.index(l) }
  codes.is_in(CArray.int64(wanted.size) { |i| wanted[i] })
end

#locate_addr(ref) ⇒ CArray

Returns for each cell, its label's address in ref (UNDEF when the label does not occur there).

Returns:

  • (CArray) —

    for each cell, its label's address in ref (UNDEF when the label does not occur there).



347
348
349
 
# File 'lib/carray/categorical.rb', line 347

def locate_addr (ref)
  label_values.locate_addr(label_array(ref))
end

#mode(axis: nil) ⇒ CArray+

Returns the most frequent label(s). Runs in label space because the modal values come back sorted ascending, which for labels is not the code order.

Returns:



312
313
314
 
# File 'lib/carray/categorical.rb', line 312

def mode (axis: nil)
  label_values.mode(axis: axis)
end

#ne(label) ⇒ CArray

Returns the complement of #eq.

Parameters:

  • label (Object) —

    category to compare against.

Returns:



222
223
224
 
# File 'lib/carray/categorical.rb', line 222

def ne(label)
  codes.ne(@labels.index(label) || @labels.size)
end

#reduceat_index ⇒ CArray

Segment start offsets into the category-contiguous layout produced by #sort_addr: reduceat_index is where category c's block begins, so block c spans reduceat_index[c] ... reduceat_index[c] + category_sizes[c]. Empty categories repeat the following start (a zero-width block). Length k, aligned to #labels; pair with #category_sizes for the block lengths. On data already laid out in category order the offsets index it directly.

Returns an int64 CArray, one entry per #labels, of segment start offsets aligned to #labels. Pair with #category_sizes for lengths.

Returns:



466
467
468
469
470
471
472
473
474
475
476
477
478
 
# File 'lib/carray/categorical.rb', line 466

def reduceat_index
  return @_reduceat_index if @_reduceat_index
  k = @labels.size
  out = CArray.int64(k > 0 ? k : 0)
  if k > 1
    out[0] = 0
    out[1..-1] = category_sizes.cumsum.int64[0..-2]
  elsif k == 1
    out[0] = 0
  end
  @_reduceat_index = out
  out
end

#sort_addr ⇒ CArray

Flat addresses that gather self into category-contiguous order: every cell of category 0 first, then 1, ..., then k-1, with excluded cells last. Built by a counting sort over the codes (O(n + k), stable): the segment starts (an exclusive prefix scan of #category_sizes) drive a scatter that places each source index into its category's block in source order; excluded cells (masked, or code out of range 0...k) are appended at the tail in source order. The first #category_sizes.sum addresses are exactly the classified cells in category order — CACategoricalIterator's permutation. For an N-D categorical the addresses are into the raveled storage, so value.reshape(elements)[cat.sort_addr] produces the contiguous blocks. Memoised (see the grouping-plan note above): the counting sort is the dominant grouping cost, so it is computed once and shared by every payload column and by CACategoricalIterator. Do not mutate the returned array.

Returns a length-CArray#elements integer CArray of flat storage addresses that order the cells by category (excluded cells last).

Returns:



426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
 
# File 'lib/carray/categorical.rb', line 426

def sort_addr
  return @_sort_addr if @_sort_addr
  n      = elements
  k      = @labels.size
  cs     = category_sizes                       # per-category counts
  nvalid = cs.sum
  cur    = CArray.int64(k > 0 ? k : 0)          # segment starts, consumed as cursor
  cur[1..-1] = cs.cumsum.int64[0..-2] if k > 1
  flat   = codes.reshape(n)
  seq    = CArray.int64(n).seq!                 # source indices, scattered as payload
  out    = CArray.int64(n)
  if nvalid == n
    flat.send(:__categorical_scatter__, seq, cur, out, k) # all valid: scatter straight in
  else
    if nvalid > 0
      valid = CArray.int64(nvalid)              # scatter target must be an entity, not a view
      flat.send(:__categorical_scatter__, seq, cur, valid, k)
      out[0...nvalid] = valid
    end
    # Excluded = every cell the scatter skips: code out of [0, k), OR masked
    # (a from_codes pre-masked cell keeps a valid code but is excluded). Both
    # must be caught or the tail slot count would not add up to n - nvalid.
    excluded = flat.value.ge(k).or(flat.value.lt(0))
    excluded = excluded.or(flat.is_masked) if flat.has_mask?
    out[nvalid..-1] = seq[excluded]             # excluded cells, source order
  end
  @_sort_addr = out
  out
end

#storage_to_scalar(raw) ⇒ Object

Face hook: decode a per-cell code into its category label. An out-of-range code (e.g. an unmasked external sentinel) decodes to nil rather than a wrong category via Ruby negative indexing.



194
195
196
197
 
# File 'lib/carray/categorical.rb', line 194

def storage_to_scalar(raw)
  code = raw.is_a?(String) ? raw.unpack1(UNPACK_FORMAT.fetch(parent.data_type)) : raw
  (code < 0 || code >= @labels.size) ? nil : @labels[code]
end

#union(other, sort: false) ⇒ CArray

Returns object CArray of the labels of either side.

Returns:

  • (CArray) —

    object CArray of the labels of either side.



340
341
342
 
# File 'lib/carray/categorical.rb', line 340

def union (other, sort: false)
  unique.union(label_array(other), sort: sort)
end

#unique(sort: false) ⇒ CArray

Returns the labels that occur, in first-appearance order. A category with no cells is not included (use #labels for the vocabulary).

Parameters:

  • sort (Boolean) (defaults to: false) —

    when true, sort the labels ascending.

Returns:

  • (CArray) —

    object CArray of labels.



283
284
285
286
 
# File 'lib/carray/categorical.rb', line 283

def unique (sort: false)
  u = labels_for(codes.unique)
  sort ? u.sort : u
end

#value_counts(sort: false) ⇒ Array(CArray, CArray)

Returns [labels, counts] for the categories that occur. #category_sizes is the aligned-to-#labels counterpart, which keeps the empty ones.

Parameters:

  • sort (false, :count, :value) (defaults to: false) —

    pair ordering.

Returns:



293
294
295
296
297
298
299
300
301
302
303
304
305
 
# File 'lib/carray/categorical.rb', line 293

def value_counts (sort: false)
  unless [false, :count, :value].include?(sort)
    raise ArgumentError, "value_counts: sort must be false, :count, or :value"
  end
  code_values, counts = codes.value_counts(sort: sort == :count ? :count : false)
  values = labels_for(code_values)
  return [values, counts] unless sort == :value
  # Ascending *label*: code order is the vocabulary's, so reorder here.
  la    = values.to_a
  order = (0...la.size).sort_by { |i| [la[i], i] }
  idx   = CArray.int64(order.size) { |i| order[i] }
  [values[idx], counts[idx]]
end