Class: Qiita::Elasticsearch::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/qiita/elasticsearch/tokenizer.rb

Constant Summary collapse

DEFAULT_DOWNCASED_FIELDS = 
[]
DEFAULT_FILTERABLE_FIELDS = 
[]
DEFAULT_HIERARCHAL_FIELDS = 
[]
DEFAULT_RANGE_FIELDS = 
[]
TOKEN_PATTERN = 
/
  (?<token_string>
    (?<minus>-)?
    (?:(?<field_name>\w+):)?
    (?:
      (?:"(?<quoted_term>.*?)(?<!\\)")
      |
      (?<term>\S+)
    )
  )
/x

Instance Method Summary collapse

Constructor Details

#initialize(downcased_fields: nil, filterable_fields: nil, hierarchal_fields: nil, matchable_fields: nil, range_fields: nil) ⇒ Tokenizer

Returns a new instance of Tokenizer.

Parameters:

  • downcased_fields (Array<String>, nil) (defaults to: nil)
  • filterable_fields (Array<String>, nil) (defaults to: nil)
  • hierarchal_fields (Array<String>, nil) (defaults to: nil)
  • matchable_fields (Array<String>, nil) (defaults to: nil)
  • range_fields (Array<String>, nil) (defaults to: nil) 


31
32
33
34
35
36
37
 
# File 'lib/qiita/elasticsearch/tokenizer.rb', line 31

def initialize(downcased_fields: nil, filterable_fields: nil, hierarchal_fields: nil, matchable_fields: nil, range_fields: nil)
  @downcased_fields = downcased_fields
  @filterable_fields = filterable_fields
  @hierarchal_fields = hierarchal_fields
  @matchable_fields = matchable_fields
  @range_fields = range_fields
end

Instance Method Details

#tokenize(query_string) ⇒ Array<Qiita::Elasticsearch::Token>

Parameters:

  • query_string (String) —

    Raw query string

Returns:



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/qiita/elasticsearch/tokenizer.rb', line 41

def tokenize(query_string)
  query_string.scan(TOKEN_PATTERN).map do |token_string, minus, field_name, quoted_term, term|
    term ||= quoted_term
    if !field_name.nil? && !filterable_fields.include?(field_name)
      term = "#{field_name}:#{term}"
      field_name = nil
    end
    token = token_class(field_name).new(
      downcased: downcased_fields.include?(field_name),
      field_name: field_name,
      minus: minus,
      quoted: !quoted_term.nil?,
      term: term,
      token_string: token_string,
    )
    token.matchable_fields = @matchable_fields if token.is_a?(MatchableToken)
    token
  end
end