Class: Rox::Core::StringTokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/rox/core/roxx/string_tokenizer.rb

Instance Method Summary collapse

Constructor Details

#initialize(text, delim, return_delims) ⇒ StringTokenizer

Returns a new instance of StringTokenizer.



4
5
6
7
8
9
10
11
 
# File 'lib/rox/core/roxx/string_tokenizer.rb', line 4

def initialize(text, delim, return_delims)
  @current_position = 0
  @new_position = -1
  @text = text
  @max_position = text.length
  @delimiters = delim
  @ret_delims = return_delims
end

Instance Method Details

#delimiter?(char) ⇒ Boolean

Returns:

  • (Boolean) 


43
44
45
 
# File 'lib/rox/core/roxx/string_tokenizer.rb', line 43

def delimiter?(char)
  @delimiters.include?(char)
end

#more_tokens? ⇒ Boolean

Returns:

  • (Boolean) 


47
48
49
50
 
# File 'lib/rox/core/roxx/string_tokenizer.rb', line 47

def more_tokens?
  @new_position = skip_delimiters(@current_position)
  @new_position < @max_position
end

#next_token(delim = nil) ⇒ Object

Raises:

  • (ArgumentError) 


52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/rox/core/roxx/string_tokenizer.rb', line 52

def next_token(delim = nil)
  delims_changed = false
  unless delim.nil?
    @delimiters = delim
    delims_changed = true
  end

  @current_position = @new_position >= 0 && !delims_changed ? @new_position : skip_delimiters(@current_position)
  @new_position = -1

  raise ArgumentError, 'invalid operation' if @current_position >= @max_position

  start = @current_position
  @current_position = scan_token(@current_position)
  @text[start...@current_position]
end

#scan_token(start_pos) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/rox/core/roxx/string_tokenizer.rb', line 26

def scan_token(start_pos)
  position = start_pos
  while position < @max_position
    c = @text[position]
    break if delimiter?(c)

    position += 1
  end

  if @ret_delims && start_pos == position
    c = @text[position]
    position += 1 if delimiter?(c)
  end

  position
end

#skip_delimiters(start_pos) ⇒ Object

Raises:

  • (ArgumentError) 


13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/rox/core/roxx/string_tokenizer.rb', line 13

def skip_delimiters(start_pos)
  raise ArgumentError, 'delimiters is nil' if @delimiters.nil?

  position = start_pos
  while !@ret_delims && position < @max_position
    c = @text[position]
    break unless delimiter?(c)

    position += 1
  end
  position
end