Class: IdPack::IdPacker
- Inherits:
-
Object
- Object
- IdPack::IdPacker
- Defined in:
- lib/id_pack/id_packer.rb
Overview
This is a module to encode an integer array into our compressed format. Basically there are only 2 methods in this module, encode and decode.
Usage:
encode:
a usual use case of encode is to provide the server with object ids
that have already been fetched and hence we don't need their data to
be returned
Example:
IdPack::IdPacker.new.encode([5, 6, 21, 23, 25]) # => "_F~C_P.V"
decode:
mainly used by the server to convert the compressed string back into
the integer array
Example:
IdPack::IdPacker.new.decode("_F~C_P.V") # => [5, 6, 21, 23, 25]
Defined Under Namespace
Classes: InvalidEncodedCharException
Constant Summary collapse
- SPACES_PREFIX =
'_'
- BINARY_PREFIX =
'.'
- RANGE_PREFIX =
'~'
- WINDOW_SIZE =
10
- EXCLUDE_NIL =
true
- ENCODED_NUMBER_CHARS =
(('A'..'Z').to_a + ('a'..'z').to_a + ('0'..'9').to_a).join + '-'
Instance Method Summary collapse
-
#decode(encoded_caches) ⇒ Object
“_F~C_P.V” => [5, 6, 21, 23, 25].
- #decode_sync_str(sync_str, base_timestamp = 0) ⇒ Object
-
#encode(array, window_size = WINDOW_SIZE, exclude_nil = EXCLUDE_NIL, output_charset = ENCODED_NUMBER_CHARS) ⇒ Object
- 5, 6, 21, 23, 25
-
> “_F~C_P.V”.
-
#encode_sync_str(id_synced_at) ⇒ Object
Input: id_synced_at: { 1 => synced_at_1_timestamp, 2 => synced_at_2_timestamp, 10 => synced_at_10_timestamp, … }.
Instance Method Details
#decode(encoded_caches) ⇒ Object
“_F~C_P.V”
> [5, 6, 21, 23, 25]
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 |
# File 'lib/id_pack/id_packer.rb', line 123 def decode(encoded_caches) curr_encoded_string_prefix = nil ids = [] start_id = 0 encoded_number = '' encoded_caches.each_char do |c| if [SPACES_PREFIX, BINARY_PREFIX, RANGE_PREFIX].include?(c) unless curr_encoded_string_prefix == nil ids_to_include, end_id = convert_encoded_number_to_ids(curr_encoded_string_prefix, encoded_number, start_id) ids = ids.concat(ids_to_include) start_id = end_id + (c == SPACES_PREFIX ? 0 : 1) end curr_encoded_string_prefix = c encoded_number = '' else encoded_number = encoded_number + c end end unless curr_encoded_string_prefix == nil ids_to_include, end_id = convert_encoded_number_to_ids(curr_encoded_string_prefix, encoded_number, start_id) ids = ids.concat(ids_to_include) start_id = end_id + 1 end ids rescue InvalidEncodedCharException # corrupted encoded_caches, assume nothing cached [] end |
#decode_sync_str(sync_str, base_timestamp = 0) ⇒ Object
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 |
# File 'lib/id_pack/id_packer.rb', line 196 def decode_sync_str sync_str, = 0 # format of sync_str: # min_last_synced_at, # "encoded_0", diff_last_requested_at_0, # "encoded_1", diff_last_requested_at_1, # "encoded_2", diff_last_requested_at_2, ... sync_str = sync_str.encode('UTF-8', 'UTF-8', :invalid => :replace) encoded_min_last_synced_at, *encoded_ranges = sync_str.split(',') min_last_synced_at = LZString.decompress_from_encoded_uri_component(encoded_min_last_synced_at).to_i grouped_encoded_ranges = encoded_ranges.inject([]) do |grouped, encoded_range| grouped << [] if grouped.last.nil? || grouped.last.length >= 2 grouped.last << encoded_range grouped end grouped_encoded_ranges.inject({}) do |synced_at_map, (encoded_caches, encoded_diff_last_synced_at)| primary_keys_str = LZString.decompress_from_encoded_uri_component(encoded_caches) primary_keys = primary_keys_str.split(",") if primary_keys.first.to_i.to_s == primary_keys.first primary_keys.map!(&:to_i) else primary_keys = primary_keys_str.scan(/.{32}/).map do |uuid_str| [uuid_str[0,8], uuid_str[8,4], uuid_str[12,4], uuid_str[16,4], uuid_str[20,16]].join("-") end end diff_last_synced_at = LZString.decompress_from_encoded_uri_component(encoded_diff_last_synced_at).to_i last_synced_at = min_last_synced_at + diff_last_synced_at + primary_keys.each do |key| synced_at_map[key] = [synced_at_map[key], last_synced_at].compact.max end synced_at_map end rescue # invalid sync_str, return empty map {} end |
#encode(array, window_size = WINDOW_SIZE, exclude_nil = EXCLUDE_NIL, output_charset = ENCODED_NUMBER_CHARS) ⇒ Object
- 5, 6, 21, 23, 25
-
> “_F~C_P.V”
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 |
# File 'lib/id_pack/id_packer.rb', line 38 def encode array, window_size = WINDOW_SIZE, exclude_nil = EXCLUDE_NIL, output_charset = ENCODED_NUMBER_CHARS encoded_array = '' ranges = convert_numbers_to_ranges array.uniq.sort prev_end = 0 curr_start = 1 spaces = 0 group_with_prev = false ranges_to_group = [] binary_number = '' decimal_number = 0 encoded_string = '' ranges.each_with_index do |range, i| spaces = range.begin - prev_end if group_with_prev if range.end - curr_start + 1 == window_size ranges_to_group << range binary_number = convert_ranges_to_binary_number ranges_to_group decimal_number = convert_binary_number_to_decimal_number binary_number encoded_string = BINARY_PREFIX + encode_decimal_number(decimal_number, output_charset) encoded_array += encoded_string ranges_to_group = [] group_with_prev = false elsif range.end - curr_start + 1 >= window_size if ranges_to_group.length == 1 encoded_string = RANGE_PREFIX + encode_decimal_number(ranges_to_group.first.size, output_charset) encoded_array += encoded_string else binary_number = convert_ranges_to_binary_number ranges_to_group decimal_number = convert_binary_number_to_decimal_number binary_number encoded_string = BINARY_PREFIX + encode_decimal_number(decimal_number, output_charset) encoded_array += encoded_string end ranges_to_group = [] encoded_string = SPACES_PREFIX + encode_decimal_number(spaces, output_charset) encoded_array += encoded_string if range.size >= window_size encoded_string = RANGE_PREFIX + encode_decimal_number(range.size, output_charset) encoded_array += encoded_string group_with_prev = false else ranges_to_group.push range curr_start = range.begin group_with_prev = true end else ranges_to_group.push range end else if spaces >= 0 encoded_string = SPACES_PREFIX + encode_decimal_number(spaces, output_charset) encoded_array += encoded_string end if range.size >= window_size encoded_string = RANGE_PREFIX + encode_decimal_number(range.size, output_charset) encoded_array += encoded_string else ranges_to_group.push range curr_start = range.begin group_with_prev = true end end prev_end = range.end end if ranges_to_group.length == 1 encoded_string = RANGE_PREFIX + encode_decimal_number(ranges_to_group.first.size, output_charset) encoded_array += encoded_string elsif ranges_to_group.length > 0 binary_number = convert_ranges_to_binary_number ranges_to_group decimal_number = convert_binary_number_to_decimal_number binary_number encoded_string = BINARY_PREFIX + encode_decimal_number(decimal_number, output_charset) encoded_array += encoded_string end encoded_array end |
#encode_sync_str(id_synced_at) ⇒ Object
Input: id_synced_at:
1 => synced_at_1_timestamp,
2 => synced_at_2_timestamp,
10 => synced_at_10_timestamp, ...
Expected output of sync_str: min_last_synced_at,\ “encoded_0”,diff_last_synced_at_0,\ “encoded_1”,diff_last_synced_at_1,\ “encoded_2”,diff_last_synced_at_2, …
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 |
# File 'lib/id_pack/id_packer.rb', line 169 def encode_sync_str id_synced_at min_synced_at = id_synced_at.values.min encoded_min_synced_at = LZString.compress_to_encoded_uri_component(min_synced_at.to_s) grouped_synced_at = id_synced_at.group_by do |id, synced_at| synced_at end grouped_synced_at.inject([encoded_min_synced_at]) do |sync_str_arr, (synced_at, ids_group)| ids = ids_group.map do |id_group| int_id = id_group[0].to_s.to_i int_id && int_id.to_s == id_group[0].to_s ? int_id : id_group[0].to_s end joined_ids = ids.first.is_a?(String) ? ids.join("").gsub(/-/, "") : ids.join(",") encoded_indices = LZString.compress_to_encoded_uri_component(joined_ids) diff_synced_at = synced_at - min_synced_at encoded_diff_synced_at = LZString.compress_to_encoded_uri_component(diff_synced_at.to_s) sync_str_arr << "#{encoded_indices},#{encoded_diff_synced_at}" end.join(",") end |