Class: SLF0::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/slf0/tokenizer.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(slf0) ⇒ Tokenizer

Returns a new instance of Tokenizer.



6
7
8
 
# File 'lib/slf0/tokenizer.rb', line 6

def initialize(slf0)
  @scanner = StringScanner.new(slf0)
end

Instance Attribute Details

#scanner ⇒ Object (readonly)

Returns the value of attribute scanner.



5
6
7
 
# File 'lib/slf0/tokenizer.rb', line 5

def scanner
  @scanner
end

Class Method Details

.tokenize(slf0) ⇒ Object 



11
12
13
 
# File 'lib/slf0/tokenizer.rb', line 11

def self.tokenize(slf0)
  new(slf0).tokenize
end

Instance Method Details

#scan_length(length) ⇒ Object 



55
56
57
58
59
 
# File 'lib/slf0/tokenizer.rb', line 55

def scan_length(length)
  string = scanner.string[scanner.pos, length]
  scanner.pos += length
  string
end

#skip_header! ⇒ Object 



20
21
22
 
# File 'lib/slf0/tokenizer.rb', line 20

def skip_header!
  raise 'missing header' unless scanner.skip(/SLF0/)
end

#tokenize ⇒ Object 



15
16
17
18
 
# File 'lib/slf0/tokenizer.rb', line 15

def tokenize
  skip_header!
  tokenize_body!
end

#tokenize_body! ⇒ Object 



24
25
26
27
28
29
30
31
32
33
 
# File 'lib/slf0/tokenizer.rb', line 24

def tokenize_body!
  body = []
  until scanner.eos?
    object = tokenize_double_field || tokenize_field || tokenize_object_list_nil
    raise "malformed no object: #{scanner.rest.inspect}\n\nafter: #{body.inspect}" unless object

    body << object
  end
  body
end

#tokenize_double_field ⇒ Object 



61
62
63
64
65
66
 
# File 'lib/slf0/tokenizer.rb', line 61

def tokenize_double_field
  return unless (hex = scanner.scan(/\h*\^/)&.chop)

  double = [hex.to_i(16)].pack('Q<').unpack1('G')
  SLF0::Token::Double.new double
end

#tokenize_field ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/slf0/tokenizer.rb', line 35

def tokenize_field
  raise 'no int found' unless (int = scanner.scan(/\d+/).to_i)

  case scanner.get_byte
  when '#'
    SLF0::Token::Int.new int
  when '%'
    SLF0::Token::ClassName.new scan_length(int).freeze
  when '@'
    SLF0::Token::ClassNameRef.new int
  when '"'
    SLF0::Token::String.new scan_length(int).tr("\r", "\n").freeze
  when '('
    SLF0::Token::ObjectList.new int
  else
    scanner.unscan
    nil
  end
end

#tokenize_object_list_nil ⇒ Object 



68
69
70
71
72
 
# File 'lib/slf0/tokenizer.rb', line 68

def tokenize_object_list_nil
  return unless scanner.skip(/-/)

  SLF0::Token::ObjectListNil.new nil
end