Class: FC::Storage

Inherits:
DbBase show all
Defined in:
lib/fc/storage.rb

Class Attribute Summary collapse

Attributes inherited from DbBase

#additional_fields, #database_fields, #id

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from DbBase

apply!, create_from_fiels, #delete, find, #reload, #save, set_table, where

Constructor Details

#initialize(params = {}) ⇒ Storage

Returns a new instance of Storage.



33
34
35
36
37
38
39
40
41
 
# File 'lib/fc/storage.rb', line 33

def initialize(params = {})
  path = (params['path'] || params[:path])
  if path && !path.to_s.empty?
    path += '/' unless path[-1] == '/'
    raise "Storage path must be like '/bla/bla../'" unless path.match(/^\/.*\/$/)
    params['path'] = params[:path] = path
  end
  super params
end

Class Attribute Details

.check_time_limit ⇒ Object

Returns the value of attribute check_time_limit.



10
11
12
 
# File 'lib/fc/storage.rb', line 10

def check_time_limit
  @check_time_limit
end

.get_copy_storages_mutex ⇒ Object

Returns the value of attribute get_copy_storages_mutex.



10
11
12
 
# File 'lib/fc/storage.rb', line 10

def get_copy_storages_mutex
  @get_copy_storages_mutex
end

.storages_cache_time ⇒ Object

Returns the value of attribute storages_cache_time.



10
11
12
 
# File 'lib/fc/storage.rb', line 10

def storages_cache_time
  @storages_cache_time
end

Class Method Details

.curr_host ⇒ Object 



16
17
18
 
# File 'lib/fc/storage.rb', line 16

def self.curr_host
  @uname || @uname = `uname -n`.chomp
end

.select_proper_storage_for_create(storages, size, exclude = []) ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/fc/storage.rb', line 20

def self.select_proper_storage_for_create(storages, size, exclude = [])
  list = storages.select do |storage|
    !exclude.include?(storage.name) && storage.up? && storage.size + size < storage.size_limit && storage.write_weight.to_i >= 0
  end
  list = yield(list) if block_given?
  # sort by random(free_rate * write_weight)
  list.map{ |storage| 
    [storage, Kernel.rand(storage.free_rate * (storage.write_weight.to_i == 0 ? 0.01 : storage.write_weight.to_i) * 1000000000)] 
  }.sort{ |a, b|
    a[1] <=> b[1]
  }.map{|el| el[0]}.last
end

.speed_limit_to_rsync_opt(speed_limit) ⇒ Object 



123
124
125
126
 
# File 'lib/fc/storage.rb', line 123

def self.speed_limit_to_rsync_opt(speed_limit)
  return "--bwlimit=#{(speed_limit.to_f * 125.0).ceil} " if speed_limit.to_f > 0
  ''
end

Instance Method Details

#auto_size? ⇒ Boolean

Returns:

  • (Boolean) 


43
44
45
 
# File 'lib/fc/storage.rb', line 43

def auto_size?
  (auto_size || 0) > 0
end

#check_time_delay ⇒ Object 



103
104
105
 
# File 'lib/fc/storage.rb', line 103

def check_time_delay
  Time.new.to_i - check_time.to_i
end

#copy_path(local_path, file_name, try_move = false, speed_limit = nil) ⇒ Object

copy local_path to storage



129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/fc/storage.rb', line 129

def copy_path(local_path, file_name, try_move = false, speed_limit = nil)
  dst_path = "#{self.path}#{file_name}"

  recreate_dirs_cmd = "rm -rf #{dst_path.shellescape}; mkdir -p #{File.dirname(dst_path).shellescape}"

  # recreate dirs anyway if local op
  if try_move && self.class.curr_host == host
    r = `#{recreate_dirs_cmd} 2>&1`
    raise r if $?.exitstatus != 0
  end
  # if we can make mv command
  if try_move && self.class.curr_host == host && File.stat(local_path).dev == File.stat(File.dirname(dst_path)).dev
    r = `mv #{local_path.shellescape} #{dst_path.shellescape} 2>&1`
    raise r if $?.exitstatus != 0
  else
    local_path += '/' if File.stat(local_path).directory?
    cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"#{recreate_dirs_cmd} && ionice -c 2 -n 7 rsync\" #{local_path.shellescape} #{self.host}:\"#{dst_path.shellescape}\""
    r = `#{cmd} 2>&1`
    raise r if $?.exitstatus != 0
  end
end

#copy_to_local(file_name, local_path, speed_limit = nil) ⇒ Object

copy object to local_path



152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
 
# File 'lib/fc/storage.rb', line 152

def copy_to_local(file_name, local_path, speed_limit = nil)
  src_path = "#{self.path}#{file_name}"
  
  r = `rm -rf #{local_path.shellescape}; mkdir -p #{File.dirname(local_path).shellescape} 2>&1`
  raise r if $?.exitstatus != 0

  # if remote file is directory?
  cmd = "ssh -oStrictHostKeyChecking=no -q #{self.host} \"if [ -d #{src_path.shellescape} ]; then /bin/true; else /bin/false; fi\""
  r = `#{cmd} 2>&1`
  src_path += '/' if $?.exitstatus == 0

  cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"ionice -c 2 -n 7 rsync\" #{self.host}:\"#{src_path.shellescape}\" #{local_path.shellescape}"
  r = `#{cmd} 2>&1`
  raise r if $?.exitstatus != 0
end

#delete_file(file_name) ⇒ Object

delete object from storage



169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
 
# File 'lib/fc/storage.rb', line 169

def delete_file(file_name)
  dst_path = "#{self.path}#{file_name}"
  if self.class.curr_host == host
    begin
      File.delete(dst_path)
    rescue Errno::EISDIR
      FileUtils.rm_r(dst_path)
    rescue Errno::ENOENT
    end
  else
    cmd = "ssh -q -oBatchMode=yes -oStrictHostKeyChecking=no #{self.host} \"rm -rf #{dst_path.shellescape}\""
    r = `#{cmd} 2>&1`
    raise r if $?.exitstatus != 0
    
    cmd = "ssh -q -oBatchMode=yes -oStrictHostKeyChecking=no #{self.host} \"ls -la #{dst_path.shellescape}\""
    r = `#{cmd} 2>/dev/null`
    raise "Path #{dst_path} not deleted" unless r.empty?
  end
end

#dump ⇒ Object 



217
218
219
 
# File 'lib/fc/storage.rb', line 217

def dump
  super(%i[check_time autosync_at http_check_time])
end

#file_size(file_name, ignore_errors = false) ⇒ Object

return object size on storage



190
191
192
193
194
195
196
197
198
199
 
# File 'lib/fc/storage.rb', line 190

def file_size(file_name, ignore_errors = false)
  dst_path = "#{self.path}#{file_name}"
  
  cmd = self.class.curr_host == host ? 
    "du -sb #{dst_path.shellescape}" : 
    "ssh -q -oBatchMode=yes -oStrictHostKeyChecking=no #{self.host} \"du -sb #{dst_path.shellescape}\""
  r = ignore_errors ? `#{cmd} 2>/dev/null` : `#{cmd} 2>&1`
  raise r if $?.exitstatus != 0
  r.to_i
end

#free ⇒ Object 



47
48
49
 
# File 'lib/fc/storage.rb', line 47

def free
  size_limit - size
end

#free_rate ⇒ Object 



55
56
57
58
 
# File 'lib/fc/storage.rb', line 55

def free_rate
  rate = free.to_f / size_limit
  rate < 0 ? 0.0 : rate
end

#get_copy_storages ⇒ Object 



81
82
83
84
85
86
87
88
89
90
 
# File 'lib/fc/storage.rb', line 81

def get_copy_storages
  self.class.get_copy_storages_mutex.synchronize do
    unless @copy_storages_cache && Time.new.to_i - @get_copy_storages_time.to_i < self.class.storages_cache_time
      @get_copy_storages_time = Time.new.to_i
      names = copy_storages.to_s.split(',').map{|s| "'#{s}'"}.join(',')
      @copy_storages_cache = names.empty? ? [] : FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
    end
  end
  @copy_storages_cache
end

#get_disk_free_space ⇒ Object 



73
74
75
76
77
78
79
 
# File 'lib/fc/storage.rb', line 73

def get_disk_free_space
  cmd = "df #{self.path.shellescape}"
  cmd = "ssh -q -oBatchMode=yes -oStrictHostKeyChecking=no #{self.host} \"df #{self.path.shellescape}\"" unless self.class.curr_host == host 
  r = `#{cmd} 2>&1`
  raise r if $?.exitstatus != 0
  r.split("\n").last.split(/\s+/)[3].to_i * 1024
end

#get_proper_storage_for_copy(size, exclude = []) ⇒ Object

get available storage for copy by size



213
214
215
 
# File 'lib/fc/storage.rb', line 213

def get_proper_storage_for_copy(size, exclude = [])
  FC::Storage.select_proper_storage_for_create(get_copy_storages, size, exclude)
end

#get_real_size ⇒ Object 



68
69
70
71
 
# File 'lib/fc/storage.rb', line 68

def get_real_size
  size_in_copy = size_in_status 'copy'
  self.size_limit = get_disk_free_space - size_in_copy - auto_size + size
end

#http_check_enabled? ⇒ Boolean

Returns:

  • (Boolean) 


111
112
113
 
# File 'lib/fc/storage.rb', line 111

def http_check_enabled?
  http_check_time.to_i >= 0
end

#http_check_time_delay ⇒ Object 



107
108
109
 
# File 'lib/fc/storage.rb', line 107

def http_check_time_delay
  http_check_enabled? ? Time.new.to_i - http_check_time.to_i : 0
end

#http_up? ⇒ Boolean

Returns:

  • (Boolean) 


119
120
121
 
# File 'lib/fc/storage.rb', line 119

def http_up?
  http_check_time_delay < self.class.check_time_limit
end

#load(data: {}) ⇒ Object 



221
222
223
224
225
226
227
 
# File 'lib/fc/storage.rb', line 221

def load(data: {})
  if name
    data = data.clone
    data.delete(:host)
  end
  super(data: data)
end

#md5_sum(file_name) ⇒ Object

return object md5_sum on storage



202
203
204
205
206
207
208
209
210
 
# File 'lib/fc/storage.rb', line 202

def md5_sum(file_name)
  dst_path = "#{self.path}#{file_name}"
  cmd = self.class.curr_host == host ?
      "find #{dst_path.shellescape} -type f -exec md5sum {} \\; | awk '{print $1}' | sort | md5sum" :
      "ssh -q -oBatchMode=yes -oStrictHostKeyChecking=no #{self.host} \"find #{dst_path.shellescape} -type f -exec md5sum {} \\; | awk '{print \\$1}' | sort | md5sum\""
  r = `#{cmd} 2>&1`
  raise r if $?.exitstatus != 0
  r.to_s[0..31]
end

#size_in_status(status) ⇒ Object 



60
61
62
63
64
65
66
 
# File 'lib/fc/storage.rb', line 60

def size_in_status(status)
  FC::DB.query(%{SELECT sum(i.size) as isize 
    FROM #{FC::ItemStorage.table_name} its 
    join #{FC::Item.table_name} i on i.id = its.item_id 
    where its.storage_name = '#{self.name}' 
    and its.status = '#{status}'}).first['isize'].to_i
end

#size_rate ⇒ Object 



51
52
53
 
# File 'lib/fc/storage.rb', line 51

def size_rate
  size.to_f / size_limit
end

#up? ⇒ Boolean

Returns:

  • (Boolean) 


115
116
117
 
# File 'lib/fc/storage.rb', line 115

def up?
  check_time_delay < self.class.check_time_limit
end

#update_check_time ⇒ Object 



92
93
94
95
 
# File 'lib/fc/storage.rb', line 92

def update_check_time
  self.check_time = Time.new.to_i
  save
end

#update_http_check_time ⇒ Object 



97
98
99
100
101
 
# File 'lib/fc/storage.rb', line 97

def update_http_check_time
  return unless http_check_enabled?
  self.http_check_time = Time.new.to_i
  save
end