Class: BaseTokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/support/base_tokenizer.rb

Defined Under Namespace

Classes: ScanError

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source) ⇒ BaseTokenizer

Constructor. Initialize a tokenizer for Skeem.

Parameters:

  • source (String) —

    Skeem text to tokenize.



13
14
15
16
 
# File 'lib/support/base_tokenizer.rb', line 13

def initialize(source)
  @scanner = StringScanner.new('')
  restart(source)
end

Instance Attribute Details

#line_start ⇒ Object (readonly)

Returns the value of attribute line_start.



7
8
9
 
# File 'lib/support/base_tokenizer.rb', line 7

def line_start
  @line_start
end

#lineno ⇒ Object (readonly)

Returns the value of attribute lineno.



6
7
8
 
# File 'lib/support/base_tokenizer.rb', line 6

def lineno
  @lineno
end

#scanner ⇒ Object (readonly)

Returns the value of attribute scanner.



5
6
7
 
# File 'lib/support/base_tokenizer.rb', line 5

def scanner
  @scanner
end

Instance Method Details

#restart(source) ⇒ Object

Parameters:

  • source (String) —

    Skeem text to tokenize.



19
20
21
22
23
 
# File 'lib/support/base_tokenizer.rb', line 19

def restart(source)
  @scanner.string = source
  @lineno = 1
  @line_start = 0
end

#tokens ⇒ Array<SkmToken>

Returns | Returns a sequence of tokens.

Returns:

  • (Array<SkmToken>) —

    | Returns a sequence of tokens



26
27
28
29
30
31
32
33
34
 
# File 'lib/support/base_tokenizer.rb', line 26

def tokens
  tok_sequence = []
  until @scanner.eos?
    token = _next_token
    tok_sequence << token unless token.nil?
  end

  return tok_sequence
end