Class: String

Inherits:
Object
  • Object
show all
Defined in:
lib/lexeme/core_extensions.rb

Instance Method Summary collapse

Instance Method Details

#to_tokens ⇒ Object 



16
17
18
19
 
# File 'lib/lexeme/core_extensions.rb', line 16

def to_tokens
  warn "String#to_tokens is obsolete, use String#tokenize instead"
  tokenize
end

#tokenize ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
 
# File 'lib/lexeme/core_extensions.rb', line 2

def tokenize 
  @lexer ||= Lexeme.define do 
    use_language :natural
  end
  
  string_content = self 

  @lexer.analyze do
    from_string string_content
  end

  @lexer.tokens
end