Module: ActiveSupport::Multibyte::Unicode

Extended by:
Unicode
Included in:
Unicode
Defined in:
lib/active_support/multibyte/unicode.rb

Defined Under Namespace

Classes: Codepoint, UnicodeDatabase

Constant Summary collapse

NORMALIZATION_FORMS =

A list of all available normalization forms. See www.unicode.org/reports/tr15/tr15-29.html for more information about normalization.

[:c, :kc, :d, :kd]
UNICODE_VERSION =

The Unicode version that is supported by the implementation

"9.0.0"
HANGUL_SBASE =

Hangul character boundaries and properties

0xAC00
HANGUL_LBASE = 
0x1100
HANGUL_VBASE = 
0x1161
HANGUL_TBASE = 
0x11A7
HANGUL_LCOUNT = 
19
HANGUL_VCOUNT = 
21
HANGUL_TCOUNT = 
28
HANGUL_NCOUNT = 
HANGUL_VCOUNT * HANGUL_TCOUNT
HANGUL_SCOUNT = 
11172
HANGUL_SLAST = 
HANGUL_SBASE + HANGUL_SCOUNT

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#default_normalization_form ⇒ Object

The default normalization used for operations that require normalization. It can be set to any of the normalizations in NORMALIZATION_FORMS.

ActiveSupport::Multibyte::Unicode.default_normalization_form = :c



19
20
21
 
# File 'lib/active_support/multibyte/unicode.rb', line 19

def default_normalization_form
  @default_normalization_form
end

Instance Method Details

#compose(codepoints) ⇒ Object

Compose decomposed characters to the composed form.



159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
 
# File 'lib/active_support/multibyte/unicode.rb', line 159

def compose(codepoints)
  pos = 0
  eoa = codepoints.length - 1
  starter_pos = 0
  starter_char = codepoints[0]
  previous_combining_class = -1
  while pos < eoa
    pos += 1
    lindex = starter_char - HANGUL_LBASE
    # -- Hangul
    if 0 <= lindex && lindex < HANGUL_LCOUNT
      vindex = codepoints[starter_pos + 1] - HANGUL_VBASE rescue vindex = -1
      if 0 <= vindex && vindex < HANGUL_VCOUNT
        tindex = codepoints[starter_pos + 2] - HANGUL_TBASE rescue tindex = -1
        if 0 <= tindex && tindex < HANGUL_TCOUNT
          j = starter_pos + 2
          eoa -= 2
        else
          tindex = 0
          j = starter_pos + 1
          eoa -= 1
        end
        codepoints[starter_pos..j] = (lindex * HANGUL_VCOUNT + vindex) * HANGUL_TCOUNT + tindex + HANGUL_SBASE
      end
      starter_pos += 1
      starter_char = codepoints[starter_pos]
    # -- Other characters
    else
      current_char = codepoints[pos]
      current = database.codepoints[current_char]
      if current.combining_class > previous_combining_class
        if ref = database.composition_map[starter_char]
          composition = ref[current_char]
        else
          composition = nil
        end
        unless composition.nil?
          codepoints[starter_pos] = composition
          starter_char = composition
          codepoints.delete_at pos
          eoa -= 1
          pos -= 1
          previous_combining_class = -1
        else
          previous_combining_class = current.combining_class
        end
      else
        previous_combining_class = current.combining_class
      end
      if current.combining_class == 0
        starter_pos = pos
        starter_char = codepoints[pos]
      end
    end
  end
  codepoints
end

#decompose(type, codepoints) ⇒ Object

Decompose composed characters to the decomposed form.



138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/active_support/multibyte/unicode.rb', line 138

def decompose(type, codepoints)
  codepoints.inject([]) do |decomposed, cp|
    # if it's a hangul syllable starter character
    if HANGUL_SBASE <= cp && cp < HANGUL_SLAST
      sindex = cp - HANGUL_SBASE
      ncp = [] # new codepoints
      ncp << HANGUL_LBASE + sindex / HANGUL_NCOUNT
      ncp << HANGUL_VBASE + (sindex % HANGUL_NCOUNT) / HANGUL_TCOUNT
      tindex = sindex % HANGUL_TCOUNT
      ncp << (HANGUL_TBASE + tindex) unless tindex == 0
      decomposed.concat ncp
    # if the codepoint is decomposable in with the current decomposition type
    elsif (ncp = database.codepoints[cp].decomp_mapping) && (!database.codepoints[cp].decomp_type || type == :compatibility)
      decomposed.concat decompose(type, ncp.dup)
    else
      decomposed << cp
    end
  end
end

#downcase(string) ⇒ Object 



282
283
284
 
# File 'lib/active_support/multibyte/unicode.rb', line 282

def downcase(string)
  apply_mapping string, :lowercase_mapping
end

#in_char_class?(codepoint, classes) ⇒ Boolean

Detect whether the codepoint is in a certain character class. Returns true when it’s in the specified character class and false otherwise. Valid character classes are: :cr, :lf, :l, :v, :lv, :lvt and :t.

Primarily used by the grapheme cluster support.

Returns:

  • (Boolean) 


40
41
42
 
# File 'lib/active_support/multibyte/unicode.rb', line 40

def in_char_class?(codepoint, classes)
  classes.detect { |c| database.boundary[c] === codepoint } ? true : false
end

#normalize(string, form = nil) ⇒ Object

Returns the KC normalization of the string by default. NFKC is considered the best normalization form for passing strings to databases and validations.

  • string - The string to perform normalization on.

  • form - The form you want to normalize in. Should be one of the following: :c, :kc, :d, or :kd. Default is ActiveSupport::Multibyte::Unicode.default_normalization_form.



264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
 
# File 'lib/active_support/multibyte/unicode.rb', line 264

def normalize(string, form = nil)
  form ||= @default_normalization_form
  # See http://www.unicode.org/reports/tr15, Table 1
  codepoints = string.codepoints.to_a
  case form
  when :d
    reorder_characters(decompose(:canonical, codepoints))
  when :c
    compose(reorder_characters(decompose(:canonical, codepoints)))
  when :kd
    reorder_characters(decompose(:compatibility, codepoints))
  when :kc
    compose(reorder_characters(decompose(:compatibility, codepoints)))
    else
    raise ArgumentError, "#{form} is not a valid normalization variant", caller
  end.pack("U*".freeze)
end

#pack_graphemes(unpacked) ⇒ Object

Reverse operation of unpack_graphemes.

Unicode.pack_graphemes(Unicode.unpack_graphemes('क्षि')) # => 'क्षि'



117
118
119
 
# File 'lib/active_support/multibyte/unicode.rb', line 117

def pack_graphemes(unpacked)
  unpacked.flatten.pack("U*")
end

#reorder_characters(codepoints) ⇒ Object

Re-order codepoints so the string becomes canonical.



122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/active_support/multibyte/unicode.rb', line 122

def reorder_characters(codepoints)
  length = codepoints.length - 1
  pos = 0
  while pos < length do
    cp1, cp2 = database.codepoints[codepoints[pos]], database.codepoints[codepoints[pos + 1]]
    if (cp1.combining_class > cp2.combining_class) && (cp2.combining_class > 0)
      codepoints[pos..pos + 1] = cp2.code, cp1.code
      pos += (pos > 0 ? -1 : 1)
    else
      pos += 1
    end
  end
  codepoints
end

#swapcase(string) ⇒ Object 



290
291
292
 
# File 'lib/active_support/multibyte/unicode.rb', line 290

def swapcase(string)
  apply_mapping string, :swapcase_mapping
end

#tidy_bytes(string, force = false) ⇒ Object

Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string.

Passing true will forcibly tidy all bytes, assuming that the string’s encoding is entirely CP1252 or ISO-8859-1.



224
225
226
227
228
 
# File 'lib/active_support/multibyte/unicode.rb', line 224

def tidy_bytes(string, force = false)
  return string if string.empty?
  return recode_windows1252_chars(string) if force
  string.scrub { |bad| recode_windows1252_chars(bad) }
end

#unpack_graphemes(string) ⇒ Object

Unpack the string at grapheme boundaries. Returns a list of character lists.

Unicode.unpack_graphemes('क्षि') # => [[2325, 2381], [2359], [2367]]
Unicode.unpack_graphemes('Café') # => [[67], [97], [102], [233]]



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/active_support/multibyte/unicode.rb', line 49

def unpack_graphemes(string)
  codepoints = string.codepoints.to_a
  unpacked = []
  pos = 0
  marker = 0
  eoc = codepoints.length
  while (pos < eoc)
    pos += 1
    previous = codepoints[pos - 1]
    current = codepoints[pos]

    # See http://unicode.org/reports/tr29/#Grapheme_Cluster_Boundary_Rules
    should_break =
      if pos == eoc
        true
      # GB3. CR X LF
      elsif previous == database.boundary[:cr] && current == database.boundary[:lf]
        false
      # GB4. (Control|CR|LF) ÷
      elsif previous && in_char_class?(previous, [:control, :cr, :lf])
        true
      # GB5. ÷ (Control|CR|LF)
      elsif in_char_class?(current, [:control, :cr, :lf])
        true
      # GB6. L X (L|V|LV|LVT)
      elsif database.boundary[:l] === previous && in_char_class?(current, [:l, :v, :lv, :lvt])
        false
      # GB7. (LV|V) X (V|T)
      elsif in_char_class?(previous, [:lv, :v]) && in_char_class?(current, [:v, :t])
        false
      # GB8. (LVT|T) X (T)
      elsif in_char_class?(previous, [:lvt, :t]) && database.boundary[:t] === current
        false
      # GB9. X (Extend | ZWJ)
      elsif in_char_class?(current, [:extend, :zwj])
        false
      # GB9a. X SpacingMark
      elsif database.boundary[:spacingmark] === current
        false
      # GB9b. Prepend X
      elsif database.boundary[:prepend] === previous
        false
      # GB10. (E_Base | EBG) Extend* X E_Modifier
      elsif (marker...pos).any? { |i| in_char_class?(codepoints[i], [:e_base, :e_base_gaz]) && codepoints[i + 1...pos].all? { |c| database.boundary[:extend] === c } } && database.boundary[:e_modifier] === current
        false
      # GB11. ZWJ X (Glue_After_Zwj | EBG)
      elsif database.boundary[:zwj] === previous && in_char_class?(current, [:glue_after_zwj, :e_base_gaz])
        false
      # GB12. ^ (RI RI)* RI X RI
      # GB13. [^RI] (RI RI)* RI X RI
      elsif codepoints[marker..pos].all? { |c| database.boundary[:regional_indicator] === c } && codepoints[marker..pos].count { |c| database.boundary[:regional_indicator] === c }.even?
        false
      # GB999. Any ÷ Any
      else
        true
      end

    if should_break
      unpacked << codepoints[marker..pos - 1]
      marker = pos
    end
  end
  unpacked
end

#upcase(string) ⇒ Object 



286
287
288
 
# File 'lib/active_support/multibyte/unicode.rb', line 286

def upcase(string)
  apply_mapping string, :uppercase_mapping
end