Class: MiniCSS::CSS::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/minicss/css/tokenizer.rb

Instance Method Summary collapse

Constructor Details

#initialize(input, allow_unicode_ranges: false) ⇒ Tokenizer

Returns a new instance of Tokenizer.



8
9
10
11
12
13
14
15
16
 
# File 'lib/minicss/css/tokenizer.rb', line 8

def initialize(input, allow_unicode_ranges: false)
  str = (input.is_a?(IO) ? input.read : input.to_s)
        .encode("UTF-8", invalid: :replace, undef: :replace, replace: "\uFFFD")
        .gsub("\r\n", "\n")
        .tr("\u000C", "\n")
        .tr("\u0000", "\uFFFD")
  @source_str = str
  @scanner = MiniCSS::CSS::Scanner.new(str, allow_unicode_ranges)
end

Instance Method Details

#hydrate_tokens ⇒ Object 



20
21
22
23
24
 
# File 'lib/minicss/css/tokenizer.rb', line 20

def hydrate_tokens
  @scanner.tokens.each do |tok|
    tok.literal ||= @source_str[tok.pos_start.offset...tok.pos_end.offset]
  end
end

#tokenize ⇒ Object 



26
27
28
29
30
 
# File 'lib/minicss/css/tokenizer.rb', line 26

def tokenize
  @scanner.consume_token until @scanner.eof?
  hydrate_tokens
  nil
end

#tokens ⇒ Object 



18
 
# File 'lib/minicss/css/tokenizer.rb', line 18

def tokens = @scanner.tokens