Class: Hocon::Impl::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/hocon/impl/tokenizer.rb

Defined Under Namespace

Classes: TokenIterator, TokenizerProblemError

Constant Summary collapse

Tokens = 
Hocon::Impl::Tokens
ConfigBugOrBrokenError = 
Hocon::ConfigError::ConfigBugOrBrokenError

Class Method Summary collapse

Class Method Details

.as_string(codepoint) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/hocon/impl/tokenizer.rb', line 24

def self.as_string(codepoint)
  if codepoint == "\n"
    "newline"
  elsif codepoint == "\t"
    "tab"
  elsif codepoint == -1
    "end of file"
  elsif codepoint =~ /[[:cntrl:]]/
    "control character 0x%x" % codepoint
  else
    "%c" % codepoint
  end
end

.render(tokens) ⇒ Object 



44
45
46
47
48
49
50
 
# File 'lib/hocon/impl/tokenizer.rb', line 44

def self.render(tokens)
  rendered_text = ""
  while (t = tokens.next)
    rendered_text << t.token_text
  end
  rendered_text
end

.tokenize(origin, input, syntax) ⇒ Object

Tokenizes a Reader. Does not close the reader; you have to arrange to do that after you’re done with the returned iterator.



40
41
42
 
# File 'lib/hocon/impl/tokenizer.rb', line 40

def self.tokenize(origin, input, syntax)
  TokenIterator.new(origin, input, syntax != Hocon::ConfigSyntax::JSON)
end