Module: ActiveSupport::Multibyte::Unicode

Extended by:
Unicode
Included in:
Unicode
Defined in:
lib/active_support/multibyte/unicode.rb

Defined Under Namespace

Classes: Codepoint, UnicodeDatabase

Constant Summary collapse

NORMALIZATION_FORMS =

A list of all available normalization forms. See www.unicode.org/reports/tr15/tr15-29.html for more information about normalization.

[:c, :kc, :d, :kd]
UNICODE_VERSION =

The Unicode version that is supported by the implementation

'8.0.0'
HANGUL_SBASE =

Hangul character boundaries and properties

0xAC00
HANGUL_LBASE = 
0x1100
HANGUL_VBASE = 
0x1161
HANGUL_TBASE = 
0x11A7
HANGUL_LCOUNT = 
19
HANGUL_VCOUNT = 
21
HANGUL_TCOUNT = 
28
HANGUL_NCOUNT = 
HANGUL_VCOUNT * HANGUL_TCOUNT
HANGUL_SCOUNT = 
11172
HANGUL_SLAST = 
HANGUL_SBASE + HANGUL_SCOUNT
HANGUL_JAMO_FIRST = 
0x1100
HANGUL_JAMO_LAST = 
0x11FF
WHITESPACE =

All the unicode whitespace

[
  (0x0009..0x000D).to_a, # White_Space # Cc   [5] <control-0009>..<control-000D>
  0x0020,                # White_Space # Zs       SPACE
  0x0085,                # White_Space # Cc       <control-0085>
  0x00A0,                # White_Space # Zs       NO-BREAK SPACE
  0x1680,                # White_Space # Zs       OGHAM SPACE MARK
  (0x2000..0x200A).to_a, # White_Space # Zs  [11] EN QUAD..HAIR SPACE
  0x2028,                # White_Space # Zl       LINE SEPARATOR
  0x2029,                # White_Space # Zp       PARAGRAPH SEPARATOR
  0x202F,                # White_Space # Zs       NARROW NO-BREAK SPACE
  0x205F,                # White_Space # Zs       MEDIUM MATHEMATICAL SPACE
  0x3000,                # White_Space # Zs       IDEOGRAPHIC SPACE
].flatten.freeze
LEADERS_AND_TRAILERS =

BOM (byte order mark) can also be seen as whitespace, it’s a non-rendering character used to distinguish between little and big endian. This is not an issue in utf-8, so it must be ignored.

WHITESPACE + [65279]
TRAILERS_PAT = 
/(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+\Z/u
LEADERS_PAT = 
/\A(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+/u

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#default_normalization_form ⇒ Object

The default normalization used for operations that require normalization. It can be set to any of the normalizations in NORMALIZATION_FORMS.

ActiveSupport::Multibyte::Unicode.default_normalization_form = :c



20
21
22
 
# File 'lib/active_support/multibyte/unicode.rb', line 20

def default_normalization_form
  @default_normalization_form
end

Class Method Details

.codepoints_to_pattern(array_of_codepoints) ⇒ Object

Returns a regular expression pattern that matches the passed Unicode codepoints.



59
60
61
 
# File 'lib/active_support/multibyte/unicode.rb', line 59

def self.codepoints_to_pattern(array_of_codepoints) #:nodoc:
  array_of_codepoints.collect{ |e| [e].pack 'U*'.freeze }.join('|'.freeze)
end

Instance Method Details

#compose(codepoints) ⇒ Object

Compose decomposed characters to the composed form.



180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
 
# File 'lib/active_support/multibyte/unicode.rb', line 180

def compose(codepoints)
  pos = 0
  eoa = codepoints.length - 1
  starter_pos = 0
  starter_char = codepoints[0]
  previous_combining_class = -1
  while pos < eoa
    pos += 1
    lindex = starter_char - HANGUL_LBASE
    # -- Hangul
    if 0 <= lindex and lindex < HANGUL_LCOUNT
      vindex = codepoints[starter_pos+1] - HANGUL_VBASE rescue vindex = -1
      if 0 <= vindex and vindex < HANGUL_VCOUNT
        tindex = codepoints[starter_pos+2] - HANGUL_TBASE rescue tindex = -1
        if 0 <= tindex and tindex < HANGUL_TCOUNT
          j = starter_pos + 2
          eoa -= 2
        else
          tindex = 0
          j = starter_pos + 1
          eoa -= 1
        end
        codepoints[starter_pos..j] = (lindex * HANGUL_VCOUNT + vindex) * HANGUL_TCOUNT + tindex + HANGUL_SBASE
      end
      starter_pos += 1
      starter_char = codepoints[starter_pos]
    # -- Other characters
    else
      current_char = codepoints[pos]
      current = database.codepoints[current_char]
      if current.combining_class > previous_combining_class
        if ref = database.composition_map[starter_char]
          composition = ref[current_char]
        else
          composition = nil
        end
        unless composition.nil?
          codepoints[starter_pos] = composition
          starter_char = composition
          codepoints.delete_at pos
          eoa -= 1
          pos -= 1
          previous_combining_class = -1
        else
          previous_combining_class = current.combining_class
        end
      else
        previous_combining_class = current.combining_class
      end
      if current.combining_class == 0
        starter_pos = pos
        starter_char = codepoints[pos]
      end
    end
  end
  codepoints
end

#decompose(type, codepoints) ⇒ Object

Decompose composed characters to the decomposed form.



159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/active_support/multibyte/unicode.rb', line 159

def decompose(type, codepoints)
  codepoints.inject([]) do |decomposed, cp|
    # if it's a hangul syllable starter character
    if HANGUL_SBASE <= cp and cp < HANGUL_SLAST
      sindex = cp - HANGUL_SBASE
      ncp = [] # new codepoints
      ncp << HANGUL_LBASE + sindex / HANGUL_NCOUNT
      ncp << HANGUL_VBASE + (sindex % HANGUL_NCOUNT) / HANGUL_TCOUNT
      tindex = sindex % HANGUL_TCOUNT
      ncp << (HANGUL_TBASE + tindex) unless tindex == 0
      decomposed.concat ncp
    # if the codepoint is decomposable in with the current decomposition type
    elsif (ncp = database.codepoints[cp].decomp_mapping) and (!database.codepoints[cp].decomp_type || type == :compatibility)
      decomposed.concat decompose(type, ncp.dup)
    else
      decomposed << cp
    end
  end
end

#downcase(string) ⇒ Object 



303
304
305
 
# File 'lib/active_support/multibyte/unicode.rb', line 303

def downcase(string)
  apply_mapping string, :lowercase_mapping
end

#in_char_class?(codepoint, classes) ⇒ Boolean

Detect whether the codepoint is in a certain character class. Returns true when it’s in the specified character class and false otherwise. Valid character classes are: :cr, :lf, :l, :v, :lv, :lvt and :t.

Primarily used by the grapheme cluster support.

Returns:

  • (Boolean) 


71
72
73
 
# File 'lib/active_support/multibyte/unicode.rb', line 71

def in_char_class?(codepoint, classes)
  classes.detect { |c| database.boundary[c] === codepoint } ? true : false
end

#normalize(string, form = nil) ⇒ Object

Returns the KC normalization of the string by default. NFKC is considered the best normalization form for passing strings to databases and validations.

  • string - The string to perform normalization on.

  • form - The form you want to normalize in. Should be one of the following: :c, :kc, :d, or :kd. Default is ActiveSupport::Multibyte::Unicode.default_normalization_form.



285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
 
# File 'lib/active_support/multibyte/unicode.rb', line 285

def normalize(string, form=nil)
  form ||= @default_normalization_form
  # See http://www.unicode.org/reports/tr15, Table 1
  codepoints = string.codepoints.to_a
  case form
    when :d
      reorder_characters(decompose(:canonical, codepoints))
    when :c
      compose(reorder_characters(decompose(:canonical, codepoints)))
    when :kd
      reorder_characters(decompose(:compatibility, codepoints))
    when :kc
      compose(reorder_characters(decompose(:compatibility, codepoints)))
    else
      raise ArgumentError, "#{form} is not a valid normalization variant", caller
  end.pack('U*'.freeze)
end

#pack_graphemes(unpacked) ⇒ Object

Reverse operation of unpack_graphemes.

Unicode.pack_graphemes(Unicode.unpack_graphemes('क्षि')) # => 'क्षि'



138
139
140
 
# File 'lib/active_support/multibyte/unicode.rb', line 138

def pack_graphemes(unpacked)
  unpacked.flatten.pack('U*')
end

#reorder_characters(codepoints) ⇒ Object

Re-order codepoints so the string becomes canonical.



143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/active_support/multibyte/unicode.rb', line 143

def reorder_characters(codepoints)
  length = codepoints.length- 1
  pos = 0
  while pos < length do
    cp1, cp2 = database.codepoints[codepoints[pos]], database.codepoints[codepoints[pos+1]]
    if (cp1.combining_class > cp2.combining_class) && (cp2.combining_class > 0)
      codepoints[pos..pos+1] = cp2.code, cp1.code
      pos += (pos > 0 ? -1 : 1)
    else
      pos += 1
    end
  end
  codepoints
end

#swapcase(string) ⇒ Object 



311
312
313
 
# File 'lib/active_support/multibyte/unicode.rb', line 311

def swapcase(string)
  apply_mapping string, :swapcase_mapping
end

#tidy_bytes(string, force = false) ⇒ Object

Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string.

Passing true will forcibly tidy all bytes, assuming that the string’s encoding is entirely CP1252 or ISO-8859-1.



245
246
247
248
249
 
# File 'lib/active_support/multibyte/unicode.rb', line 245

def tidy_bytes(string, force = false)
  return string if string.empty?
  return recode_windows1252_chars(string) if force
  string.scrub { |bad| recode_windows1252_chars(bad) }
end

#unpack_graphemes(string) ⇒ Object

Unpack the string at grapheme boundaries. Returns a list of character lists.

Unicode.unpack_graphemes('क्षि') # => [[2325, 2381], [2359], [2367]]
Unicode.unpack_graphemes('Café') # => [[67], [97], [102], [233]]



80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/active_support/multibyte/unicode.rb', line 80

def unpack_graphemes(string)
  codepoints = string.codepoints.to_a
  unpacked = []
  pos = 0
  marker = 0
  eoc = codepoints.length
  while(pos < eoc)
    pos += 1
    previous = codepoints[pos-1]
    current = codepoints[pos]

    should_break =
      # GB3. CR X LF
      if previous == database.boundary[:cr] and current == database.boundary[:lf]
        false
      # GB4. (Control|CR|LF) ÷
      elsif previous and in_char_class?(previous, [:control,:cr,:lf])
        true
      # GB5. ÷ (Control|CR|LF)
      elsif in_char_class?(current, [:control,:cr,:lf])
        true
      # GB6. L X (L|V|LV|LVT)
      elsif database.boundary[:l] === previous and in_char_class?(current, [:l,:v,:lv,:lvt])
        false
      # GB7. (LV|V) X (V|T)
      elsif in_char_class?(previous, [:lv,:v]) and in_char_class?(current, [:v,:t])
        false
      # GB8. (LVT|T) X (T)
      elsif in_char_class?(previous, [:lvt,:t]) and database.boundary[:t] === current
        false
      # GB8a. Regional_Indicator X Regional_Indicator
      elsif database.boundary[:regional_indicator] === previous and database.boundary[:regional_indicator] === current
        false
      # GB9. X Extend
      elsif database.boundary[:extend] === current
        false
      # GB9a. X SpacingMark
      elsif database.boundary[:spacingmark] === current
        false
      # GB9b. Prepend X
      elsif database.boundary[:prepend] === previous
        false
      # GB10. Any ÷ Any
      else
        true
      end

    if should_break
      unpacked << codepoints[marker..pos-1]
      marker = pos
    end
  end
  unpacked
end

#upcase(string) ⇒ Object 



307
308
309
 
# File 'lib/active_support/multibyte/unicode.rb', line 307

def upcase(string)
  apply_mapping string, :uppercase_mapping
end