Class: Jetel::Loaders::Elasticsearch

Inherits:
Loader
  • Object
show all
Defined in:
lib/jetel/loaders/elasticsearch/elasticsearch.rb

Instance Attribute Summary collapse

Attributes inherited from Loader

#uri

Instance Method Summary collapse

Constructor Details

#initialize(uri) ⇒ Elasticsearch

Returns a new instance of Elasticsearch.



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'lib/jetel/loaders/elasticsearch/elasticsearch.rb', line 34

def initialize(uri)
  super

  tmp = uri.split('://')
  tmp = tmp[1].split('@')

  parts = tmp[0].split(':')
  user = parts[0]
  password = parts[1]

  parts = tmp[1].split('/')
  host, port = parts[0].split(':')
  @index, @document_type = parts[1], parts[2]

  opts = {
    :host => host,
    :port => (port && port.to_i) || 9200,
    # :options => '',
    # :tty => '',
    # :bucket => bucket,
    # :username => user,
    # :password => password,
    # :connection_timeout => 360e6,
    # :timeout => 360e6
  }

  @client = ::Elasticsearch::Client.new(opts)

  puts client.cluster.health

  # client.index index: index, type: document_type, body: {title: 'Test'}
end

Instance Attribute Details

#client ⇒ Object (readonly)

Returns the value of attribute client.



32
33
34
 
# File 'lib/jetel/loaders/elasticsearch/elasticsearch.rb', line 32

def client
  @client
end

#document_type ⇒ Object (readonly)

Returns the value of attribute document_type.



32
33
34
 
# File 'lib/jetel/loaders/elasticsearch/elasticsearch.rb', line 32

def document_type
  @document_type
end

#index ⇒ Object (readonly)

Returns the value of attribute index.



32
33
34
 
# File 'lib/jetel/loaders/elasticsearch/elasticsearch.rb', line 32

def index
  @index
end

Instance Method Details

#load(modul, source, file, opts) ⇒ Object 



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/jetel/loaders/elasticsearch/elasticsearch.rb', line 67

def load(modul, source, file, opts)
  super

  cache = []
  CSV.open(file, 'rt', :headers => true, :converters => :all) do |csv|
    csv.each do |row|
      cache << {
        create: {
          _index: @index,
          _type: @document_type,
          # _id: 1,
          data: row.to_hash
        }
      }
      if cache.length === 5_000
        client.bulk(body: cache)
        cache = []
        print '.'
      end
    end

    if cache.length > 0
      client.bulk(body: cache)
      cache = []
    end
  end
end