Module: FilePool

Defined in:
lib/file_pool.rb,
lib/file_pool/version.rb

Defined Under Namespace

Classes: InvalidFileId

Constant Summary collapse

VERSION = 
"0.6.0"

Class Method Summary collapse

Class Method Details

.add(path, options = {}) ⇒ Object

Add a file to the file pool.

Same as FilePool.add!, but doesn’t throw exceptions.

Parameters:

source (String)

path of the file to add.

options (Hash)

:background (true,false) adding large files can take long (esp. with encryption), true won’t block, default is false

Return Value:

String containing a new unique ID for the file added.

false when the file could not be stored.



136
137
138
139
140
141
 
# File 'lib/file_pool.rb', line 136

def self.add path, options = {}
  self.add!(path, options)

rescue Exception
  return false
end

.add!(orig_path, options = {}) ⇒ Object

Add a file to the file pool.

Creates hard-links (ln) when file at path is on same file system as pool, otherwise copies it. When dealing with large files the latter should be avoided, because it takes more time and space.

Throws standard file exceptions when unable to store the file. See also FilePool.add to avoid it.

Parameters:

path (String)

path of the file to add.

options (Hash)

:background (true,false) adding large files can take long (esp. with encryption), true won’t block, default is false

Return Value:

String containing a new unique ID for the file added.

Raises:



79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/file_pool.rb', line 79

def self.add! orig_path, options = {}
  newid = uuid

  raise Errno::ENOENT unless File.exist?(orig_path)

  child = fork do
    target = path newid

    if @@crypted_mode
      FileUtils.mkpath(id2dir_secured newid)
      path = encrypt_to_tempfile(orig_path)
    else
      path = orig_path
      FileUtils.mkpath(id2dir newid)
    end

    if !@@copy_source and (File.stat(path).dev == File.stat(File.dirname(target)).dev)
      FileUtils.link(path, target)
    else
      FileUtils.copy(path, target)
    end

    # don't chmod if orginal file is same as target (hard-linked)
    if File.stat(orig_path).ino != File.stat(File.dirname(target)).ino
      FileUtils.chmod(@@mode, target) if @@mode
      FileUtils.chown(@@owner, @@group, target)
    end
  end


  if options[:background]
    # don't wait, avoid zombies
    Process.detach(child)
  else
    # block until done
    Process.waitpid(child)
  end

  newid
end

.add_stream(in_stream) ⇒ Object

Add a new file from a stream to the file pool



146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
 
# File 'lib/file_pool.rb', line 146

def self.add_stream in_stream
  newid = uuid

  # ensure target path exists
  if @@crypted_mode
    FileUtils.mkpath(id2dir_secured newid)
  else
    FileUtils.mkpath(id2dir newid)
  end

  child = fork do
    # create the target file to write
    open(path(newid),'w') do |out_stream|
      encrypt in_stream, out_stream
    end
  end

  Process.detach(child)

  newid
end

.path(fid, options = {}) ⇒ Object

Return the file’s path corresponding to the passed file ID, no matter if it exists or not. In encrypting mode the file is first decrypted and the returned path will point to a temporary location of the decrypted file.

To get the path of the encrypted file pass :decrypt => false, as an option.

Parameters:

fid (String)

File ID which was generated by a previous #add operation.

options (Hash)

:decrypt (true,false) In encryption mode don’t decrypt, but return the encrypted file’s path. Defaults to true.

Return Value:

String, absolute path of the file in the pool or to temporary location if it was decrypted.

Raises:



186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
 
# File 'lib/file_pool.rb', line 186

def self.path fid, options={}
  options[:decrypt] = options.fetch(:decrypt, true)

  raise InvalidFileId unless valid?(fid)

  # file present in pool?
  if File.file?(id2dir_secured(fid) + "/#{fid}")
    # present in secured tree
    if @@crypted_mode
      if options[:decrypt]
        # return path of decrypted file (tmp path)
        decrypt_to_tempfile id2dir_secured(fid) + "/#{fid}"
      else
        id2dir_secured(fid) + "/#{fid}"
      end
    else
      id2dir_secured(fid) + "/#{fid}"
    end
  elsif File.file?(id2dir(fid) + "/#{fid}")
    # present in plain tree
    id2dir(fid) + "/#{fid}"
  else
    # not present
    if @@crypted_mode
      id2dir_secured(fid) + "/#{fid}"
    else
      id2dir(fid) + "/#{fid}"
    end
  end
end

.remove(fid) ⇒ Object

Remove a previously added file by its ID. Same as FilePool.remove!, but doesn’t throw exceptions.

Parameters:

fid (String)

File ID which was generated by a previous #add operation.

Return Value:

Boolean, true if file was removed successfully, false else



268
269
270
271
272
 
# File 'lib/file_pool.rb', line 268

def self.remove fid
  self.remove! fid
rescue Exception => ex
  return false
end

.remove!(fid) ⇒ Object

Remove a previously added file by its ID. Same as FilePool.remove, but throws exceptions on failure.

Parameters:

fid (String)

File ID which was generated by a previous #add operation.



252
253
254
 
# File 'lib/file_pool.rb', line 252

def self.remove! fid
  FileUtils.rm path(fid, :decrypt => false)
end

.setup(root, options = {}) ⇒ Object

Setup the root directory of the file pool and configure encryption

Parameters:

root (String)

absolute path of the file pool’s root directory under which all files will be stored.

config_file_path (String)

path to the config file of the filepool.

options (Hash)

  • :secrets_file (String) path to file containing key and IV for encryption (if omitted FilePool does not encrypt/decrypt). If file is not present, the file is initialized with a new random key and IV.

  • :encryption_block_size (Integer) sets the block size for encryption/decryption in bytes. Larger blocks need more memory and less time (less IO). Defaults to 1’048’576 (1 MiB).

  • :copy_source (true,false) if false files added to the pool are hard-linked with the source if source and file pool are on the same file system (default). If set to true files are always copied into the pool.

  • :mode (Integer) File mode to set on all files added to the pool. E.g. mode: 0640 for rw-r----- or symbolic “u=wrx,go=rx” (see Ruby stdlib FileUtils#chmod). Note that the desired mode is not set if the file is hard-linked with the source. Use copy_source:true when to ensure.

  • :owner Owner of the files added to the pool. Note that the desired owner is not set if the file is hard-linked with the source. Use copy_source:true when to ensure.

  • :group Group of the files added to the pool. Note that the desired group is not set if the file is hard-linked with the source. Use copy_source:true when to ensure.



45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/file_pool.rb', line 45

def self.setup root, options={}
  unless(unknown = options.keys - [:encryption_block_size, :secrets_file, :copy_source, :mode, :owner, :group]).empty?
    puts "FilePool Warning: unknown option(s) passed to #setup: #{unknown.inspect}"
  end
  @@root = root
  @@crypted_mode = false
  @@block_size = options[:encryption_block_size] || (1024*1024)
  @@copy_source = options[:copy_source] || false
  @@mode = options[:mode]
  @@group = options[:group]
  @@owner = options[:owner]
  configure options[:secrets_file]
end

.stat ⇒ Object

Returns some statistics about the current pool. (It may be slow if the pool contains very many files as it computes them from scratch.)

Return Value

Hash with keys

:total_number (Integer)

Number of files in pool

:total_size (Integer)

Total number of bytes of all files

:median_size (Float)

Median of file sizes (most frequent size)

:last_add (Time)

Time and Date of last add operation



290
291
292
293
294
295
296
297
298
299
300
301
 
# File 'lib/file_pool.rb', line 290

def self.stat
  all_files = Dir.glob("#{root}_secured/*/*/*/*")
  all_files << Dir.glob("#{root}/*/*/*/*")
  all_stats = all_files.map{|f| File.stat(f) }

  {
    :total_size => all_stats.inject(0){|sum,stat| sum+=stat.size},
    :median_size => median(all_stats.map{|stat| stat.size}),
    :file_number => all_files.length,
    :last_add => all_stats.map{|stat| stat.ctime}.max
  }
end

.stream(fid, options = {}) ⇒ Object

Returns an IO object providing an (unencrypted) stream of data of the given file ID To get the stream of the encrypted data pass :decrypt => false, as an option.

Parameters:

fid (String)

File ID which was generated by a previous #add operation.

options (Hash)

:decrypt (true,false) In encryption mode don’t decrypt, but prioved the encrypted data. Defaults to true.

Return Value:

IO, IO stream open for reading



232
233
234
235
236
237
238
239
240
241
242
 
# File 'lib/file_pool.rb', line 232

def self.stream fid, options={}
  options[:decrypt] = options.fetch(:decrypt, true)

  if path = path(fid, :decrypt => false)
    if @@crypted_mode and options[:decrypt]
      decrypt_to_stream path
    else
      open(path)
    end
  end
end