Class: Metacrunch::Elasticsearch::Source

Inherits:
Object
  • Object
show all
Defined in:
lib/metacrunch/elasticsearch/source.rb

Constant Summary collapse

DEFAULT_OPTIONS = 
{
  total_hits_callback: nil,
  search_options: {
    size: 100,
    scroll: "1m",
    sort: ["_doc"]
  }
}

Instance Method Summary collapse

Constructor Details

#initialize(elasticsearch_client, options = {}) ⇒ Source

Returns a new instance of Source.



15
16
17
18
 
# File 'lib/metacrunch/elasticsearch/source.rb', line 15

def initialize(elasticsearch_client, options = {})
  @client = elasticsearch_client
  @options = DEFAULT_OPTIONS.deep_merge(options)
end

Instance Method Details

#each(&block) ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/metacrunch/elasticsearch/source.rb', line 20

def each(&block)
  return enum_for(__method__) unless block_given?

  # Perform search request and yield the first results if any
  search_options = @options[:search_options]
  result = @client.search(search_options)
  call_total_hits_callback(result)
  yield_hits(result, &block)

  # Scroll over the rest of result set and yield the results until the set is empty.
  while (
    # Note: semantic of 'and' is important here. Do not use '&&'.
    result = @client.scroll(scroll_id: result["_scroll_id"], scroll: search_options[:scroll]) and result["hits"]["hits"].present?
  ) do
    yield_hits(result, &block)
  end
ensure
  # Clear scroll to free up resources.
  @client.clear_scroll(scroll_id: result["_scroll_id"]) if result
end