Method: String#tokenize

Defined in:
lib/lexeme/core_extensions.rb

#tokenize ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
 
# File 'lib/lexeme/core_extensions.rb', line 2

def tokenize 
  @lexer ||= Lexeme.define do 
    use_language :natural
  end
  
  string_content = self 

  @lexer.analyze do
    from_string string_content
  end

  @lexer.tokens
end