Class: NormalizeCountry::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/normalize_country/scanner.rb

Constant Summary collapse

PATTERN =

Need numbers to match ISO codes Need “-” to match emoji names Need regex to match emojis

/[[:word:]]+/

Instance Method Summary collapse

Constructor Details

#initialize(s) ⇒ Tokenizer

Returns a new instance of Tokenizer.



8
9
10
 
# File 'lib/normalize_country/scanner.rb', line 8

def initialize(s)
  @scanner = StringScanner.new(s)
end

Instance Method Details

#end? ⇒ Boolean

Returns:

  • (Boolean) 


23
24
25
 
# File 'lib/normalize_country/scanner.rb', line 23

def end?
  peek.nil?
end

#peek ⇒ Object 



17
18
19
20
21
 
# File 'lib/normalize_country/scanner.rb', line 17

def peek
  match = scan
  @scanner.unscan if match
  match
end

#scan ⇒ Object 



12
13
14
15
 
# File 'lib/normalize_country/scanner.rb', line 12

def scan
  @scanner.scan_until(PATTERN)
  @scanner.matched
end