Class: Backup::Database::Elasticsearch

Inherits:
Base
  • Object
show all
Defined in:
lib/elasticsearch/extensions/backup.rb

Overview

Integration with the Backup gem [backup.github.io/backup/v4/]

This extension allows to backup Elasticsearch indices as flat JSON files on the disk.

Perform the backup with the Backup gem’s command line utility:

$ backup perform -t elasticsearch_backup

The Backup gem can store your backup files on S3, Dropbox and other cloud providers, send notifications about the operation, and so on; read more in the gem documentation.

Examples:

Use the Backup gem’s DSL to configure the backup


require 'elasticsearch/extensions/backup'

Model.new(:elasticsearch_backup, 'Elasticsearch') do

  database Elasticsearch do |db|
    db.url     = 'http://localhost:9200'
    db.indices = 'articles,people'
    db.size    = 500
    db.scroll  = '10m'
  end

  store_with Local do |local|
    local.path = '/tmp/backups'
    local.keep = 3
  end

  compress_with Gzip
end

Use the integration as a standalone script (eg. in a Rake task)


require 'backup'
require 'elasticsearch/extensions/backup'

Backup::Logger.configure do
  logfile.enabled   = true
  logfile.log_path  = '/tmp/backups/log'
end; Backup::Logger.start!

backup  = Backup::Model.new(:elasticsearch, 'Backup Elasticsearch') do
  database Backup::Database::Elasticsearch do |db|
    db.indices = 'test'
  end

  store_with Backup::Storage::Local do |local|
    local.path = '/tmp/backups'
  end
end

backup.perform!

A simple recover script for the backup created in the previous examples


PATH = '/path/to/backup/'

require 'elasticsearch'
client  = Elasticsearch::Client.new log: true
payload = []

Dir[ File.join( PATH, '**', '*.json' ) ].each do |file|
  document = MultiJson.load(File.read(file))
  item = document.merge(data: document['_source'])
  document.delete('_source')
  document.delete('_score')

  payload << { index: item }

  if payload.size == 100
    client.bulk body: payload
    payload = []
  end

  client.bulk body: payload
end

See Also:

Defined Under Namespace

Classes: Error

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(model, database_id = nil, &block) ⇒ Elasticsearch

Returns a new instance of Elasticsearch.



134
135
136
137
138
139
140
141
142
143
144
 
# File 'lib/elasticsearch/extensions/backup.rb', line 134

def initialize(model, database_id = nil, &block)
  super

  @url     ||= 'http://localhost:9200'
  @indices ||= '_all'
  @size    ||= 100
  @scroll  ||= '10m'
  @mode    ||= 'single'

  instance_eval(&block) if block_given?
end

Instance Attribute Details

#indices ⇒ Object

Returns the value of attribute indices.



127
128
129
 
# File 'lib/elasticsearch/extensions/backup.rb', line 127

def indices
  @indices
end

#mode ⇒ Object

Returns the value of attribute mode.



132
133
134
 
# File 'lib/elasticsearch/extensions/backup.rb', line 132

def mode
  @mode
end

#scroll ⇒ Object

Returns the value of attribute scroll.



127
128
129
 
# File 'lib/elasticsearch/extensions/backup.rb', line 127

def scroll
  @scroll
end

#size ⇒ Object

Returns the value of attribute size.



127
128
129
 
# File 'lib/elasticsearch/extensions/backup.rb', line 127

def size
  @size
end

#url ⇒ Object

Returns the value of attribute url.



127
128
129
 
# File 'lib/elasticsearch/extensions/backup.rb', line 127

def url
  @url
end

Instance Method Details

#__perform_single ⇒ Object

Raises:



178
179
180
181
182
183
184
185
186
187
188
189
190
 
# File 'lib/elasticsearch/extensions/backup.rb', line 178

def __perform_single
  r = client.search index: indices, search_type: 'scan', scroll: scroll, size: size
  raise Error, "No scroll_id returned in response:\n#{r.inspect}" unless r['_scroll_id']

  while r = client.scroll(scroll_id: r['_scroll_id'], scroll: scroll) and not r['hits']['hits'].empty? do
    r['hits']['hits'].each do |hit|
      FileUtils.mkdir_p "#{path.join hit['_index'], hit['_type']}"
      File.open("#{path.join hit['_index'], hit['_type'], __sanitize_filename(hit['_id'])}.json", 'w') do |file|
        file.write MultiJson.dump(hit)
      end
    end
  end
end

#__sanitize_filename(name) ⇒ Object 



192
193
194
195
196
197
 
# File 'lib/elasticsearch/extensions/backup.rb', line 192

def __sanitize_filename name
  name
    .encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�")
    .strip
    .tr("\u{202E}%$|:;/\t\r\n\\", "-")
end

#client ⇒ Object 



159
160
161
 
# File 'lib/elasticsearch/extensions/backup.rb', line 159

def client
  @client ||= ::Elasticsearch::Client.new url: url, logger: logger
end

#logger ⇒ Object 



167
168
169
170
171
172
173
174
175
 
# File 'lib/elasticsearch/extensions/backup.rb', line 167

def logger
  logger = Backup::Logger.__send__(:logger)
  logger.instance_eval do
    def debug(*args);end
    # alias :debug :info
    alias :fatal :warn
  end
  logger
end

#path ⇒ Object 



163
164
165
 
# File 'lib/elasticsearch/extensions/backup.rb', line 163

def path
  Pathname.new File.join(dump_path , dump_filename.downcase)
end

#perform! ⇒ Object 



146
147
148
149
150
151
152
153
154
155
156
157
 
# File 'lib/elasticsearch/extensions/backup.rb', line 146

def perform!
  super

  case mode
    when 'single'
      __perform_single
    else
      raise Error, "Unsupported mode [#{mode}]"
  end

  log!(:finished)
end