Module: Elasticshelf::Client

Extended by:
Client
Included in:
Client
Defined in:
lib/elasticshelf.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#client ⇒ Object (readonly)

Returns the value of attribute client.



8
9
10
 
# File 'lib/elasticshelf.rb', line 8

def client
  @client
end

#cutoff_days ⇒ Object

Returns the value of attribute cutoff_days.



14
15
16
 
# File 'lib/elasticshelf.rb', line 14

def cutoff_days
  @cutoff_days
end

#date_separator ⇒ Object

Returns the value of attribute date_separator.



14
15
16
 
# File 'lib/elasticshelf.rb', line 14

def date_separator
  @date_separator
end

#errors ⇒ Object (readonly)

Returns the value of attribute errors.



8
9
10
 
# File 'lib/elasticshelf.rb', line 8

def errors
  @errors
end

#get_info ⇒ Object (readonly)

Returns the value of attribute get_info.



10
11
12
 
# File 'lib/elasticshelf.rb', line 10

def get_info
  @get_info
end

#ignore_unavailable ⇒ Object

Returns the value of attribute ignore_unavailable.



20
21
22
 
# File 'lib/elasticshelf.rb', line 20

def ignore_unavailable
  @ignore_unavailable
end

#include_global_state ⇒ Object

Returns the value of attribute include_global_state.



20
21
22
 
# File 'lib/elasticshelf.rb', line 20

def include_global_state
  @include_global_state
end

#indices ⇒ Object

Returns the value of attribute indices.



18
19
20
 
# File 'lib/elasticshelf.rb', line 18

def indices
  @indices
end

#indices_affected ⇒ Object (readonly)

Returns the value of attribute indices_affected.



12
13
14
 
# File 'lib/elasticshelf.rb', line 12

def indices_affected
  @indices_affected
end

#indices_closed ⇒ Object (readonly)

Returns the value of attribute indices_closed.



12
13
14
 
# File 'lib/elasticshelf.rb', line 12

def indices_closed
  @indices_closed
end

#indices_deleted ⇒ Object (readonly)

Returns the value of attribute indices_deleted.



12
13
14
 
# File 'lib/elasticshelf.rb', line 12

def indices_deleted
  @indices_deleted
end

#indices_expired ⇒ Object (readonly)

Returns the value of attribute indices_expired.



12
13
14
 
# File 'lib/elasticshelf.rb', line 12

def indices_expired
  @indices_expired
end

#indices_prefix ⇒ Object

Returns the value of attribute indices_prefix.



14
15
16
 
# File 'lib/elasticshelf.rb', line 14

def indices_prefix
  @indices_prefix
end

#indices_rejected ⇒ Object (readonly)

Returns the value of attribute indices_rejected.



12
13
14
 
# File 'lib/elasticshelf.rb', line 12

def indices_rejected
  @indices_rejected
end

#indices_withered ⇒ Object (readonly)

Returns the value of attribute indices_withered.



12
13
14
 
# File 'lib/elasticshelf.rb', line 12

def indices_withered
  @indices_withered
end

#repo ⇒ Object

Returns the value of attribute repo.



16
17
18
 
# File 'lib/elasticshelf.rb', line 16

def repo
  @repo
end

#repo_compressed ⇒ Object

Returns the value of attribute repo_compressed.



16
17
18
 
# File 'lib/elasticshelf.rb', line 16

def repo_compressed
  @repo_compressed
end

#repo_location ⇒ Object

Returns the value of attribute repo_location.



16
17
18
 
# File 'lib/elasticshelf.rb', line 16

def repo_location
  @repo_location
end

#repo_type ⇒ Object

Returns the value of attribute repo_type.



16
17
18
 
# File 'lib/elasticshelf.rb', line 16

def repo_type
  @repo_type
end

#results ⇒ Object (readonly)

Returns the value of attribute results.



8
9
10
 
# File 'lib/elasticshelf.rb', line 8

def results
  @results
end

#snapshot ⇒ Object

Returns the value of attribute snapshot.



18
19
20
 
# File 'lib/elasticshelf.rb', line 18

def snapshot
  @snapshot
end

#snapshot_indices_expired_name ⇒ Object

Returns the value of attribute snapshot_indices_expired_name.



22
23
24
 
# File 'lib/elasticshelf.rb', line 22

def snapshot_indices_expired_name
  @snapshot_indices_expired_name
end

#version ⇒ Object (readonly)

Returns the value of attribute version.



10
11
12
 
# File 'lib/elasticshelf.rb', line 10

def version
  @version
end

#wait_for_completion ⇒ Object

Returns the value of attribute wait_for_completion.



20
21
22
 
# File 'lib/elasticshelf.rb', line 20

def wait_for_completion
  @wait_for_completion
end

Instance Method Details

#close_expired_indices ⇒ Object 



100
101
102
103
104
105
106
 
# File 'lib/elasticshelf.rb', line 100

def close_expired_indices
  find_expired_indices if blank? @indices_expired
  @indices_expired.each do |k,v|
    close_index(k)
    puts "closed index '#{k}' expired #{v} days ago"
  end
end

#close_index(index_name) ⇒ Object 



94
95
96
97
98
 
# File 'lib/elasticshelf.rb', line 94

def close_index(index_name)
  try do
    @indices_closed[index_name] = @client.indices.close(index: index_name)
  end
end

#cutoff_days_as_date ⇒ Object 



345
346
347
348
349
 
# File 'lib/elasticshelf.rb', line 345

def cutoff_days_as_date
  return {"error" => "cutoff_days must be zero or greater"} if @cutoff_days.to_i < 0
  cutoff_date = DateTime.now.new_offset(0) - @cutoff_days
  set_time_to_beginning_of_day(cutoff_date.strftime("%Y.%m.%d"))
end

#cutoff_days_as_date_to_s ⇒ Object 



351
352
353
354
355
 
# File 'lib/elasticshelf.rb', line 351

def cutoff_days_as_date_to_s
  cdad = cutoff_days_as_date
  cdad = cdad.strftime("%Y.%m.%d") if cdad.is_a? DateTime
  cdad
end

#delete_expired_indices ⇒ Object 



118
119
120
121
122
123
124
125
 
# File 'lib/elasticshelf.rb', line 118

def delete_expired_indices
  reset_errors_results
  find_expired_indices if blank? @indices_expired
  @indices_expired.each do |k,v|
    delete_index(k)
    puts "deleted '#{k}' expired #{v} days ago"
  end
end

#delete_index(index_name) ⇒ Object 



108
109
110
111
112
113
114
115
116
 
# File 'lib/elasticshelf.rb', line 108

def delete_index(index_name)
  if ['_all','*'].include? index_name.downcase
    @errors = {"error" => "deleting all indices is not allowed"}
    return
  end
  try do
    @indices_deleted[index_name] = @client.indices.delete(index: index_name)
  end
end

#find_expired_indices ⇒ Object 



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/elasticshelf.rb', line 52

def find_expired_indices
  try do
    return {} if blank? @indices_prefix
    return {} if blank? @date_separator
    return {} if @cutoff_days.nil?
    return {} if @cutoff_days.to_i < 0
    @indices_expired = {}
    # cutoff_date = cutoff_days_as_date
    # cutoff_date_beginning_of_day = set_time_to_beginning_of_day(cutoff_date.strftime("%Y.%m.%d"))
    cutoff_date_beginning_of_day = cutoff_days_as_date
    required_date_parts = 3 # for now, we only do days not hours
    # the following should only return 'closed' indices but it returns all indices ???
    # sorted_indices = @client.indices.get_settings(:index => @indices_prefix, :expand_wildcards => 'closed').keys
    # sorted_indices = @client.indices.get_settings(:index => @indices_prefix, :expand_wildcards => 'open').keys
    sorted_indices = @client.indices.get_settings(:index => @indices_prefix, :expand_wildcards => 'open,closed').keys
    sorted_indices.each do |index_name|
      unprefixed_index_name = index_name.slice(@indices_prefix.size-1..-1)
      date_parts = unprefixed_index_name.split(@date_separator)
      next unless date_parts.size == required_date_parts
      index_time = set_time_to_beginning_of_day(unprefixed_index_name, separator=@date_separator)
      expiry = (cutoff_date_beginning_of_day - index_time).to_i
      if expiry > 0
        @indices_expired[index_name] = expiry
      end
    end
  end
end

#get_cluster_state(index_name, metric = 'metadata') ⇒ Object 



162
163
164
165
166
167
 
# File 'lib/elasticshelf.rb', line 162

def get_cluster_state(index_name, metric='metadata')
  try do
    index_metadata = @client.cluster.state(:index => index_name, :metric => metric)
    index_metadata
  end
end

#get_elasticsearch_version ⇒ Object 



321
322
323
324
325
326
327
328
329
 
# File 'lib/elasticshelf.rb', line 321

def get_elasticsearch_version
  try do
    info_hash = @client.info
    return nil if blank? info_hash
    return nil unless info_hash['status'] == 200
    return nil if blank? info_hash['version']
    info_hash['version']['number']
  end
end

#get_index_state(index_name, metric = 'metadata') ⇒ Object 



155
156
157
158
159
160
 
# File 'lib/elasticshelf.rb', line 155

def get_index_state(index_name, metric='metadata')
  try do
    index_metadata = @client.cluster.state(:index => index_name, :metric => metric)
    index_metadata['metadata']['indices'][index_name]['state']
  end
end

#get_lucene_version ⇒ Object 



331
332
333
334
335
336
337
338
339
 
# File 'lib/elasticshelf.rb', line 331

def get_lucene_version
  try do
    info_hash = @client.info
    return nil if blank? info_hash
    return nil unless info_hash['status'] == 200
    return nil if blank? info_hash['version']
    info_hash['version']['lucene_version']
  end
end

#get_settings_index(index_name) ⇒ Object 



127
128
129
130
131
132
 
# File 'lib/elasticshelf.rb', line 127

def get_settings_index(index_name)
  reset_errors_results
  try do
    @results[index_name] = @client.indices.get_settings(:index => index_name)
  end
end

#index_closed?(index_name) ⇒ Boolean

Returns:

  • (Boolean) 


80
81
82
83
84
85
86
 
# File 'lib/elasticshelf.rb', line 80

def index_closed?(index_name)
  # catch exceptions when index doesn't exist
  try do
    index_metadata = @client.cluster.state(:index => index_name, :metric => 'metadata')
    index_metadata['metadata']['indices'][index_name]['state'] == 'close'
  end
end

#new(arguments = {}) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/elasticshelf.rb', line 24

def new(arguments={})
  @client = Elasticsearch::Transport::Client.new(arguments)
  @wait_for_completion = true
  @ignore_unavailable = true
  @include_global_state = false
  @snapshot_indices_expired_name = nil
  @date_separator = '.'
  @indices_prefix = 'logstash-'
  @cutoff_days = -1
  @repo_type = 'fs'
  @repo_compressed = true
  @indices = []
  @indices_affected = {}
  @indices_rejected = {}
  @indices_expired = {}
  @indices_closed = {}
  @indices_deleted = {}
  @indices_withered = {}
  @results = {}
  @errors = {}
  self
end

#open_index(index_name) ⇒ Object 



88
89
90
91
92
 
# File 'lib/elasticshelf.rb', line 88

def open_index(index_name)
  try do
    @results[index_name] = @client.indices.open(index: index_name)
  end
end

#snapshot_create ⇒ Object 



240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
 
# File 'lib/elasticshelf.rb', line 240

def snapshot_create
  # naming errors as seen in elasticsearch log:
  #   Invalid snapshot name [2014-04-01 20:12:07 UTC], must not contain whitespace
  #   Invalid snapshot name [2014-04-01_20:12:38_UTC], must be lowercase
  try do
    if blank? @indices
      # the whole cluster:
      @results = @client.snapshot.create(repository: @repo,
        snapshot: @snapshot,
        wait_for_completion: @wait_for_completion,
        body: {
          ignore_unavailable: @ignore_unavailable,
          include_global_state: @include_global_state
        }
      )
    else
      @results = @client.snapshot.create(repository: @repo,
        snapshot: @snapshot,
        wait_for_completion: @wait_for_completion,
        body: {
          indices: @indices,
          ignore_unavailable: @ignore_unavailable,
          include_global_state: @include_global_state
        }
      )
    end
  end
end

#snapshot_create_repository ⇒ Object 



169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
 
# File 'lib/elasticshelf.rb', line 169

def snapshot_create_repository
  @results = @errors = {}
  # ensure the location is accessible by Elasticsearch
  @repo_type = 'fs' if blank? @repo_type
  @repo_location = '/var/lib/elasticsearch_snapshots' if blank? @repo_location
  @repo_compressed = true if blank? @repo_compressed
  try do
    @results = @client.snapshot.create_repository(
      repository: @repo,
      body: {
        type: @repo_type,
        settings: {
          location: @repo_location,
          compress: @repo_compressed
        }
      }
    )
  end
end

#snapshot_delete ⇒ Object 



280
281
282
283
284
 
# File 'lib/elasticshelf.rb', line 280

def snapshot_delete
  try do
    @results = @client.snapshot.delete(repository: @repo, snapshot: @snapshot)
  end
end

#snapshot_delete_repository ⇒ Object 



200
201
202
203
204
205
 
# File 'lib/elasticshelf.rb', line 200

def snapshot_delete_repository
  @results = @errors = {}
  try do
    @results = @client.snapshot.delete_repository(repository: @repo)
  end
end

#snapshot_expired_indices ⇒ Object 



207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
 
# File 'lib/elasticshelf.rb', line 207

def snapshot_expired_indices
  @results = @errors = {}
  find_expired_indices
  if blank? @indices_expired
    @results = {"result" => "no expired indices found"}
    return
  end
  open_indices = []
  @indices_expired.keys.each do |index_name|
    # only open indices can be used in a snapshot:
    if index_closed?(index_name)
      @indices_rejected[index_name] = "closed index can't snapshot"
    else
      @indices_affected[index_name] = "open can snapshot"
      open_indices << index_name
    end
  end
  if blank? open_indices
    @results = {"result" => "none of the expired indices are open so can't snapshot"}
    return
  end
  @indices = open_indices.join(',')
  # allow user to override this name:
  if blank? @snapshot_indices_expired_name
    @snapshot = (Time.now.utc.to_s.gsub(' ', '_') +
                 "_#{@indices_prefix.chomp('*')}" +
                 "_cutoff_#{@cutoff_days}_days").downcase
  else
    @snapshot = @snapshot_indices_expired_name
  end
  @results = snapshot_create
end

#snapshot_get ⇒ Object 



269
270
271
272
273
274
275
276
277
278
 
# File 'lib/elasticshelf.rb', line 269

def snapshot_get
  reset_errors_results
  try do
    if blank? @snapshot
      @results = @client.snapshot.get(repository: @repo, snapshot: '_all')
    else
      @results = @client.snapshot.get(repository: @repo, snapshot: @snapshot)
    end
  end
end

#snapshot_get_repository ⇒ Object 



189
190
191
192
193
194
195
196
197
198
 
# File 'lib/elasticshelf.rb', line 189

def snapshot_get_repository
  @results = @errors = {}
  try do
    if @repo
      @results = @client.snapshot.get_repository(repository: @repo)
    else
      @results = @client.snapshot.get_repository
    end
  end
end

#snapshot_restore ⇒ Object 



286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
 
# File 'lib/elasticshelf.rb', line 286

def snapshot_restore
  try do
    if blank? @indices
      # the whole snapshot:
      @results = @client.snapshot.restore(
        repository: @repo,
        snapshot: @snapshot,
        wait_for_completion: @wait_for_completion,
        body: {
          ignore_unavailable: @ignore_unavailable,
          include_global_state: @include_global_state
        }
      )
    else
      # note: @indices must be a string of comma separated index names
      @results = @client.snapshot.restore(
        repository: @repo,
        snapshot: @snapshot,
        wait_for_completion: @wait_for_completion,
        body: {
          indices: @indices,
          ignore_unavailable: @ignore_unavailable,
          include_global_state: @include_global_state
        }
      )
    end
  end
end

#wither_expired_indices ⇒ Object 



146
147
148
149
150
151
152
153
 
# File 'lib/elasticshelf.rb', line 146

def wither_expired_indices
  reset_errors_results
  find_expired_indices if blank? @indices_expired
  @indices_expired.each do |k,v|
    wither_index(k)
    puts "withered '#{k}' expired #{v} days ago"
  end
end

#wither_index(index_name) ⇒ Object 



134
135
136
137
138
139
140
141
142
143
144
 
# File 'lib/elasticshelf.rb', line 134

def wither_index(index_name)
  try do
    # don't try to disable bloom filter on a closed index, it will re-open it:
    if index_closed?(index_name)
      @errors = {"failed" => "Unable to wither (disable bloom filter) index #{index_name}: It's already closed."}
      return
    else
      @indices_withered[index_name] = @client.indices.put_settings(:index => index_name, :body => 'index.codec.bloom.load=false')
    end
  end
end