Class: SocialTokenizer::Tokenizer

Inherits:
Tokenizer::Tokenizer
  • Object
show all
Defined in:
lib/social_tokenizer/tokenizer.rb

Instance Method Summary collapse

Instance Method Details

#social_tokenize(string) ⇒ Object 



3
4
5
6
7
8
9
 
# File 'lib/social_tokenizer/tokenizer.rb', line 3

def social_tokenize(string)
  for token_string in self.tokenize(string)
    SocialTokenizer::TokenType.check_all(token_string)
  end

  SocialTokenizer::TokenType.all
end