Method: TextRank::KeywordExtractor#tokenize

Defined in:
lib/text_rank/keyword_extractor.rb

#tokenize(text) ⇒ Array<String>

Filters and tokenizes text

Parameters:

  • text (String) —

    unfiltered text to be tokenized

Returns:

  • (Array<String>) —

    tokens



98
99
100
101
102
 
# File 'lib/text_rank/keyword_extractor.rb', line 98

def tokenize(text)
  filtered_text = apply_char_filters(text)
  tokens = Tokenizer.tokenize(filtered_text, *tokenizer_regular_expressions)
  apply_token_filters(tokens)
end