Class: FsCache

Inherits:
Object
  • Object
show all
Defined in:
lib/fs_cache.rb,
lib/fs_cache/attribute.rb,
lib/fs_cache/attributes/crc.rb,
lib/fs_cache/attributes/size.rb

Overview

Implement a cache of the file system: directories and files presence. Plugins can be used to also cache attributes of the files, like crc, size…

Defined Under Namespace

Modules: Attributes Classes: Attribute

Constant Summary collapse

ATTRIBUTE_PLUGINS_MODULE = 
FsCache::Attributes

Instance Method Summary collapse

Constructor Details

#initialize(attribute_plugins_dirs: []) ⇒ FsCache

Constructor

Parameters

  • attribute_plugins_dirs (Array<String>): List of directories containing possible attribute plugins [default = []]



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/fs_cache.rb', line 14

def initialize(attribute_plugins_dirs: [])
  # List of possible attribute plugins, per attribute name
  # Hash<Symbol, Attribute>
  @attribute_plugins = {}
  # Tree of dependent attributes: for each attribute in this tree, the list of attributes to be invalidated if this attribute changes.
  # Hash<Symbol, Array<Symbol> >
  @dependent_attributes = {}
  # Big database of files information
  # Hash<String, Hash<Symbol,Object> >: For each file name, the file info (can be incomplete if it was never fetched):
  # * *exist* (Boolean): Does the file exist?
  # * *size* (Integer): File size.
  # * *crc* (String): File CRC.
  # * *corruption* (false or Object): Info about this file's corruption, or false if sane.
  @files = Hash.new { |h, k| h[k] = {} }
  # Directories information
  # Hash<String, Hash<Symbol,Object> >: For each directory name, the dir info (can be incomplete if it was never fetched):
  # * *files* (Hash<String,nil>): Set of files (base names)
  # * *dirs* (Hash<String,nil>): Set of directories (base names)
  # * *recursive_dirs* (Hash<String,nil>): Set of recursive sub-directories (full paths)
  # * *recursive_files* (Hash<String,nil>): Set of recursive files (full paths)
  @dirs = Hash.new { |h, k| h[k] = {} }

  # Automatically register attributes from the plugins dirs
  (["#{__dir__}/fs_cache/attributes"] + attribute_plugins_dirs).each do |attribute_plugins_dir|
    Dir.glob("#{attribute_plugins_dir}/*.rb") do |attribute_plugin_file|
      attribute = File.basename(attribute_plugin_file)[0..-4].to_sym
      require attribute_plugin_file
      class_name = attribute.to_s.split('_').collect(&:capitalize).join.to_sym
      if ATTRIBUTE_PLUGINS_MODULE.const_defined?(class_name)
        plugin_class = ATTRIBUTE_PLUGINS_MODULE.const_get(class_name)
        register_attribute_plugin(attribute, plugin_class.new)
      else
        raise "Attributes plugin #{attribute_plugin_file} does not define the class #{class_name} inside #{ATTRIBUTE_PLUGINS_MODULE}" if plugin_class.nil?
      end
    end
  end
end

Instance Method Details

#check(include_attributes: nil, exclude_attributes: []) ⇒ Object

Check our info against file system changes. This detects

  • files that have been deleted,

  • any change in the directories structure,

  • any change in the attributes that are already part of the cache and that are not ignored explicitely.

Parameters

  • include_attributes (Array<Symbol> or nil): List of attributes to scan, or nil for all [default = nil]

  • exclude_attributes (Array<Symbol>): List of attributes to ignore while scanning [default = []]



256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
 
# File 'lib/fs_cache.rb', line 256

def check(include_attributes: nil, exclude_attributes: [])
  progressbar = ProgressBar.create(title: 'Refreshing files info')
  attributes_to_scan = (include_attributes.nil? ? @attribute_plugins.keys : include_attributes) - exclude_attributes
  progressbar.total = @files.size
  @files.each do |file, file_info|
    if File.exist?(file)
      if file_info.key?(:exist) && !file_info[:exist]
        # This file has been added when we thought it was missing
        file_info.replace(exist: true)
      else
        # Check attributes that are already present
        (file_info.keys & attributes_to_scan).each do |attribute|
          current_attribute = file_info[attribute]
          new_attribute = @attribute_plugins[attribute].attribute_for(file)
          if current_attribute != new_attribute
            # Attribute has changed
            file_info[attribute] = new_attribute
            # If some other attributes were depending on this one, invalidate them
            if @dependent_attributes.key?(attribute)
              @dependent_attributes[attribute].each do |dependent_attribute|
                file_info.delete(dependent_attribute)
              end
            end
          end
        end
      end
    elsif !file_info.key?(:exist) || file_info[:exist]
      # This file has been removed when we thought it was there
      file_info.replace(exist: false)
    end
    progressbar.increment
  end
  # Rebuilding @dirs structure needs to make the Dir.glob commands once again. Therefore there is no need to check it. Removing it will rebuild it anyway.
  @dirs.clear
end

#dirs_from(dir) ⇒ Object

Get recursive list of directories from a directory

Parameters

  • dir (String): The directory to get other directories from

Result

  • Array<String>: List of directories



132
133
134
135
136
137
138
139
140
141
142
143
144
 
# File 'lib/fs_cache.rb', line 132

def dirs_from(dir)
  unless @dirs[dir].key?(:recursive_dirs)
    ensure_dir_data(dir)
    recursive_dirs = {}
    @dirs[dir][:dirs].keys.each do |subdir|
      full_subdir = "#{dir}/#{subdir}"
      recursive_dirs[full_subdir] = nil
      recursive_dirs.merge!(Hash[dirs_from(full_subdir).map { |subsubdir| [subsubdir, nil] }])
    end
    @dirs[dir][:recursive_dirs] = recursive_dirs
  end
  @dirs[dir][:recursive_dirs].keys
end

#exist?(file) ⇒ Boolean

Is a file existing?

Parameters

  • file (String): File name

Result

  • String: Is the file existing?

Returns:

  • (Boolean) 


95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/fs_cache.rb', line 95

def exist?(file)
  # If there is at least 1 attribute for this file it means that it exists
  unless @files[file].key?(:exist)
    @files[file][:exist] =
      # If we have an attribute for this file, it means it exist
      if @files[file].size > 0
        true
      else
        dir = File.dirname(file)
        if @dirs.key?(dir)
          # We know about its directory, so we should know if it is there
          @dirs[dir][:files].key?(File.basename(file))
        else
          File.exist?(file)
        end
      end
  end
  @files[file][:exist]
end

#files_from(dir) ⇒ Object

Get recursive list of files from a directory

Parameters

  • dir (String): The directory to get other directories from

Result

  • Array<String>: List of files



152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/fs_cache.rb', line 152

def files_from(dir)
  unless @dirs[dir].key?(:recursive_files)
    ensure_dir_data(dir)
    recursive_files = Hash[@dirs[dir][:files].keys.map { |file| ["#{dir}/#{file}", nil] }]
    @dirs[dir][:dirs].keys.each do |subdir|
      recursive_files.merge!(Hash[files_from("#{dir}/#{subdir}").map { |file| [file, nil] }])
    end
    @dirs[dir][:recursive_files] = recursive_files
  end
  @dirs[dir][:recursive_files].keys
end

#files_in(dir) ⇒ Object

Get list of files from a directory (base names)

Parameters

  • dir (String): The directory to get files from

Result

  • Array<String>: List of file base names



121
122
123
124
 
# File 'lib/fs_cache.rb', line 121

def files_in(dir)
  ensure_dir_data(dir)
  @dirs[dir][:files].keys
end

#from_json(json) ⇒ Object

Get data from JSON.

Parameters

  • json (Object): JSON object



205
206
207
208
209
210
211
 
# File 'lib/fs_cache.rb', line 205

def from_json(json)
  json = json.transform_keys(&:to_sym)
  @files = Hash[json[:files].map { |file, file_info| [file, file_info.transform_keys(&:to_sym)] }]
  @files.default_proc = proc { |h, k| h[k] = {} }
  @dirs = Hash[json[:dirs].map { |dir, dir_info| [dir, dir_info.transform_keys(&:to_sym)] }]
  @dirs.default_proc = proc { |h, k| h[k] = {} }
end

#invalidate(files, include_attributes: nil, exclude_attributes: []) ⇒ Object

Remove attributes for a list of files

Parameters

  • files (Array<String>): The list of files to invalidate attributes for

  • include_attributes (Array<Symbol> or nil): List of attributes to scan, or nil for all [default = nil]

  • exclude_attributes (Array<Symbol>): List of attributes to ignore while scanning [default = []]



298
299
300
301
302
303
304
305
306
307
 
# File 'lib/fs_cache.rb', line 298

def invalidate(files, include_attributes: nil, exclude_attributes: [])
  attributes_to_invalidate = ((include_attributes.nil? ? @attribute_plugins.keys : include_attributes) - exclude_attributes)
  files.each do |file|
    if @files.key?(file)
      attributes_to_invalidate.each do |attribute|
        @files[file].delete(attribute)
      end
    end
  end
end

#notify_file_cp(src, dst) ⇒ Object

Notify the file system of a file copy

Parameters

  • src (String): Origin file

  • dst (String): Destination file



227
228
229
230
231
232
233
234
235
 
# File 'lib/fs_cache.rb', line 227

def notify_file_cp(src, dst)
  if @files.key?(src)
    @files[dst] = @files[src].clone
  else
    @files[src] = { exist: true }
    @files[dst] = { exist: true }
  end
  register_file_in_dirs(dst)
end

#notify_file_mv(src, dst) ⇒ Object

Notify the file system of a file move

Parameters

  • src (String): Origin file

  • dst (String): Destination file



242
243
244
245
 
# File 'lib/fs_cache.rb', line 242

def notify_file_mv(src, dst)
  notify_file_cp(src, dst)
  notify_file_rm(src)
end

#notify_file_rm(file) ⇒ Object

Notify the file system that a given file has been deleted

Parameters

  • file (String): File being deleted



217
218
219
220
 
# File 'lib/fs_cache.rb', line 217

def notify_file_rm(file)
  @files[file] = { exist: false }
  unregister_file_from_dirs(file)
end

#register_attribute_plugin(attribute, plugin) ⇒ Object

Register a new attributes’ plugin. The constructor already registers all plugins found in the plugins directories. This method exists in order to register plugins that could be dynamically instantiated.

Parameters

  • attribute (Symbol): The attribute

  • plugin (Attribute): The attribute plugin



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/fs_cache.rb', line 59

def register_attribute_plugin(attribute, plugin)
  puts "Registering attribute plugin #{attribute}..."
  raise "Attributes plugin #{attribute} is already defined (by class #{@attribute_plugins[attribute].class.name})." if @attribute_plugins.key?(attribute)
  @attribute_plugins[attribute] = plugin
  # Define the getter methods for this attribute, directly in the base class for performance purposes

  # Get the attribute for a given file.
  # Use the cache if possible.
  #
  # Parameters::
  # * *file* (String): File path for which we look for the attribute
  # Result::
  # * Object: Corresponding attribute value, or nil if the file does not exist
  define_singleton_method("#{attribute}_for".to_sym) do |file|
    @files[file][attribute] = plugin.attribute_for(file) if !@files[file].key?(attribute) && exist?(file)
    @files[file][attribute]
  end

  # If there are some helpers, register them too
  if plugin.class.const_defined?(:Helpers)
    helpers_module = plugin.class.const_get(:Helpers)
    self.class.include helpers_module unless helpers_module.nil?
  end
  # If this attribute is dependent on others, remember it too
  plugin.invalidated_on_change_of.each do |parent_attribute|
    @dependent_attributes[parent_attribute] = [] unless @dependent_attributes.key?(parent_attribute)
    @dependent_attributes[parent_attribute] << attribute
  end
end

#scan(dirs, include_attributes: nil, exclude_attributes: []) ⇒ Object

Scan files and directories from a list of directories. Use a progress bar.

Parameters

  • dirs (Array<String>): List of directories to preload

  • include_attributes (Array<Symbol> or nil): List of attributes to scan, or nil for all [default = nil]

  • exclude_attributes (Array<Symbol>): List of attributes to ignore while scanning [default = []]



171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
 
# File 'lib/fs_cache.rb', line 171

def scan(dirs, include_attributes: nil, exclude_attributes: [])
  progressbar = ProgressBar.create(title: 'Indexing files')
  attributes_to_scan = (include_attributes.nil? ? @attribute_plugins.keys : include_attributes) - exclude_attributes
  files = dirs.
    map do |dir|
      dirs_from(dir)
      files_from(dir)
    end.
    flatten
  progressbar.total = files.size
  files.each do |file|
    exist?(file)
    attributes_to_scan.each do |attribute|
      self.send "#{attribute}_for", file
    end
    progressbar.increment
  end
end

#to_json ⇒ Object

Serialize into JSON.

Result

  • Object: JSON object



194
195
196
197
198
199
 
# File 'lib/fs_cache.rb', line 194

def to_json
  {
    files: @files,
    dirs: @dirs
  }
end