Class: WirisPlugin::Tokenizer

Inherits:
Object
  • Object
show all
Includes:
Wiris
Defined in:
lib/com/wiris/util/type/Tokenizer.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(delimiters, takeDelimiters) ⇒ Tokenizer

Returns a new instance of Tokenizer.



8
9
10
11
12
 
# File 'lib/com/wiris/util/type/Tokenizer.rb', line 8

def initialize(delimiters, takeDelimiters)
    super()
    self.delimiters = delimiters
    self.takeDelimiters = takeDelimiters
end

Instance Attribute Details

#delimiters ⇒ Object

Returns the value of attribute delimiters.



6
7
8
 
# File 'lib/com/wiris/util/type/Tokenizer.rb', line 6

def delimiters
  @delimiters
end

#takeDelimiters ⇒ Object

Returns the value of attribute takeDelimiters.



7
8
9
 
# File 'lib/com/wiris/util/type/Tokenizer.rb', line 7

def takeDelimiters
  @takeDelimiters
end

Instance Method Details

#getDelimiters ⇒ Object 



13
14
15
 
# File 'lib/com/wiris/util/type/Tokenizer.rb', line 13

def getDelimiters()
    return self.delimiters
end

#splitTokens(text) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/com/wiris/util/type/Tokenizer.rb', line 16

def splitTokens(text)
    tokens = Array.new()
    it = Utf8::getIterator(text)
    block = StringBuf.new()
    while it::hasNext()
        c = Utf8::uchr(it::next())
        if self.delimiters::indexOf(c) >= 0
            tokens::push(block::toString())
            if self.takeDelimiters
                tokens::push(c)
            end
            block = StringBuf.new()
        else 
            block::add(c)
        end
    end
    last = block::toString()
    if !(last == "")
        tokens::push(last)
    end
    return tokens
end