Class: Jetel::Loaders::Elasticsearch

Inherits:
Loader
  • Object
show all
Defined in:
lib/jetel/loaders/elasticsearch/elasticsearch.rb

Instance Attribute Summary collapse

Attributes inherited from Loader

#uri

Instance Method Summary collapse

Constructor Details

#initialize(uri) ⇒ Elasticsearch

Returns a new instance of Elasticsearch.



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/jetel/loaders/elasticsearch/elasticsearch.rb', line 12

def initialize(uri)
  super

  tmp = uri.split('://')
  tmp = tmp[1].split('@')

  parts = tmp[0].split(':')
  user = parts[0]
  password = parts[1]

  parts = tmp[1].split('/')
  host, port = parts[0].split(':')
  @index, @document_type = parts[1], parts[2]

  opts = {
    :host => host,
    :port => (port && port.to_i) || 9200,
    # :options => '',
    # :tty => '',
    # :bucket => bucket,
    # :username => user,
    # :password => password,
    # :connection_timeout => 360e6,
    # :timeout => 360e6
  }

  @client = ::Elasticsearch::Client.new(opts)

  puts client.cluster.health

  # client.index index: index, type: document_type, body: {title: 'Test'}
end

Instance Attribute Details

#client ⇒ Object (readonly)

Returns the value of attribute client.



10
11
12
 
# File 'lib/jetel/loaders/elasticsearch/elasticsearch.rb', line 10

def client
  @client
end

#document_type ⇒ Object (readonly)

Returns the value of attribute document_type.



10
11
12
 
# File 'lib/jetel/loaders/elasticsearch/elasticsearch.rb', line 10

def document_type
  @document_type
end

#index ⇒ Object (readonly)

Returns the value of attribute index.



10
11
12
 
# File 'lib/jetel/loaders/elasticsearch/elasticsearch.rb', line 10

def index
  @index
end

Instance Method Details

#load(modul, source, file, opts) ⇒ Object 



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/jetel/loaders/elasticsearch/elasticsearch.rb', line 45

def load(modul, source, file, opts)
  super

  cache = []
  CSV.open(file, 'rt', :headers => true, :converters => :all) do |csv|
    csv.each do |row|
      cache << {
        create: {
          _index: @index,
          _type: @document_type,
          # _id: 1,
          data: row.to_hash
        }
      }
      if cache.length === 5_000
        client.bulk(body: cache)
        cache = []
        print '.'
      end
    end

    if cache.length > 0
      client.bulk(body: cache)
      cache = []
    end
  end
end