Class: Autosync

Inherits:
Object
  • Object
show all
Defined in:
lib/autosync.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(storage, dry_run = false) ⇒ Autosync

Returns a new instance of Autosync.



5
6
7
8
9
10
11
12
13
 
# File 'lib/autosync.rb', line 5

def initialize(storage, dry_run = false)
  @dry_run = dry_run
  @start_time = Time.now.to_i - 3600
  @storage = storage
  @files_to_delete = []
  @items_to_delete = []
  @removed_files_size = 0
  @io_stat = Iostat.new(@storage.path)
end

Instance Attribute Details

#files_to_delete ⇒ Object

Returns the value of attribute files_to_delete.



4
5
6
 
# File 'lib/autosync.rb', line 4

def files_to_delete
  @files_to_delete
end

#items_to_delete ⇒ Object

Returns the value of attribute items_to_delete.



4
5
6
 
# File 'lib/autosync.rb', line 4

def items_to_delete
  @items_to_delete
end

Class Method Details

.error(msg, options = {}) ⇒ Object 



15
16
17
18
 
# File 'lib/autosync.rb', line 15

def self.error(msg, options = {})
  $log.error(msg) if $log
  FC::Error.new(options.merge(:host => FC::Storage.curr_host, :message => msg)).save
end

Instance Method Details

#delete_diffs ⇒ Object 



130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/autosync.rb', line 130

def delete_diffs
  $log.debug("Removing #{@files_to_delete.size} disk entries") if $log
  @files_to_delete.each do |f|
    break if $exit_signal
    delete_disk_entry(f)
  end
  return if $exit_signal
  self.class.error("Autosync removed #{@files_to_delete.size} files/dirs from #{@storage.name}. Size: #{@removed_files_size} bytes") if @removed_files_size > 0
  $log.debug("Removing items #{@items_to_delete.size} from DB for #{@storage.name}") if $log
  counter = 0
  @items_to_delete.each do |item_storage_id|
    its = FC::ItemStorage.where('id = ?', item_storage_id).first
    next unless its
    its.status = 'error'
    its.save
    self.class.error("item does not exist on storage #{@storage.name}", item_storage_id: item_storage_id.to_i) rescue nil
    sleep 10 if (counter += 1) % 1000 == 0
  end
end

#delete_disk_entry(entry) ⇒ Object 



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/autosync.rb', line 108

def delete_disk_entry(entry)
  return false if $exit_signal
  return true unless File.exist?(entry)
  remove = true
  stat = File.stat(entry) rescue nil
  if File.directory?(entry)
    Dir.glob("#{entry}/*").each do |sub_entry|
      relax_drive
      remove = false unless delete_disk_entry(sub_entry)
    end
  else
    mtime = stat ? stat.mtime.to_i : Time.now.to_i
    remove = @start_time > mtime
  end
  if remove
    @removed_files_size += stat.size if stat
    $log.debug("deleting disk entry #{entry}") if $log
    FileUtils.rm_rf(entry)
  end
  remove
end

#fill_db ⇒ Object 



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/autosync.rb', line 37

def fill_db
  $log.debug("Autosync: Reading DB items for #{@storage.name} ...") if $log

  db_struct = {}
  last_item_storage_id = 0
  items_count = 0
  loop do
    items = FC::DB.connect.query(%(
      SELECT its.id, itm.name
      FROM #{FC::Item.table_name} itm
      JOIN #{FC::ItemStorage.table_name} its ON its.item_id = itm.id
      WHERE its.storage_name = '#{@storage.name}'
      AND its.status = 'ready'
      AND its.id > #{last_item_storage_id}
      ORDER BY its.id
      LIMIT 10000
    ), cache_rows: false, symbolize_keys: true).to_a
    break if $exit_signal

    # make tree structure with array of values (items) on leafs
    items.each do |i|
      items_count += 1
      last_item_storage_id = i[:id]
      ref = db_struct
      path = i[:name].split('/')
      last_idx = path.size - 1
      path.each_with_index do |part, idx|
        if idx == last_idx
          ref[part] = [false, i[:id]]
        else
          ref[part] ||= {}
          ref = ref[part]
        end
      end
    end
    break unless items.size == 10_000
  end
  $log.debug("Autosync: Reading DB items for #{@storage.name} done. Items: #{items_count}") if $log
  db_struct
end

#relax_drive ⇒ Object 



33
34
35
 
# File 'lib/autosync.rb', line 33

def relax_drive
  sleep 1 while @io_stat.util > 50
end

#run ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/autosync.rb', line 20

def run
  @db_struct = fill_db
  $log.debug("Autosync: Scanning disk #{@storage.name} (#{@storage.path})") if $log
  scan_disk(@db_struct, '')
  return if $exit_signal
  $log.debug("Autosync: Scanning DB items #{@storage.name}") if $log
  scan_db(@db_struct, '')
  return if $exit_signal
  delete_diffs unless @dry_run
ensure
  @io_stat.stop
end

#scan_db(db_item, node_path) ⇒ Object 



97
98
99
100
101
102
103
104
105
106
 
# File 'lib/autosync.rb', line 97

def scan_db(db_item, node_path)
  return if $exit_signal
  db_item.each do |item_name, item_data|
    if item_data.is_a?(Array) # tree leaf
      @items_to_delete << item_data[1] unless item_data[0]
    else # tree node
      scan_db(item_data, "#{node_path}#{item_name}/")
    end
  end
end

#scan_disk(db_path, relative_path) ⇒ Object 



78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/autosync.rb', line 78

def scan_disk(db_path, relative_path)
  return if $exit_signal
  sleep 0.001
  relax_drive
  Dir.glob("#{@storage.path}#{relative_path}*").each do |disk_entry|
    next if disk_entry == "#{@storage.path}healthcheck"
    db_item = db_path[disk_entry.split('/').last]
    case
    when db_item.is_a?(Array) # tree leaf
      db_item[0] = true # mark db_item as exists on disk
    when db_item.is_a?(Hash) && File.directory?(disk_entry) # tree node
      scan_disk(db_item, "#{disk_entry[@storage.path.size..-1]}/")
    else # not found in db
      mtime = File.stat(disk_entry).mtime.to_i rescue Time.now.to_i
      @files_to_delete << disk_entry if @start_time > mtime # older than 1 hour
    end
  end
end