Class: Dbox::Syncer::Pull

Inherits:
Operation show all
Defined in:
lib/dbox/syncer.rb

Instance Attribute Summary

Attributes inherited from Operation

#database

Instance Method Summary collapse

Methods inherited from Operation

#api, #current_dir_entries_as_hash, #gather_remote_info, #generate_tmpfilename, #local_path, #lookup_id_by_path, #metadata, #process_basic_remote_props, #remote_path, #remove_dotfiles, #remove_tmpfiles, #saving_parent_timestamp, #saving_timestamp, #sort_changelist, #update_file_timestamp

Methods included from Utils

#calculate_hash, #case_insensitive_difference, #case_insensitive_equal, #case_insensitive_join, #case_insensitive_resolve, #find_nonconflicting_path, #local_to_relative_path, #parse_time, #relative_to_local_path, #relative_to_remote_path, #remote_to_relative_path, #time_to_s, #times_equal?

Methods included from Loggable

included, #log

Constructor Details

#initialize(database, api) ⇒ Pull

Returns a new instance of Pull.



168
169
170
 
# File 'lib/dbox/syncer.rb', line 168

def initialize(database, api)
  super(database, api)
end

Instance Method Details

#calculate_changes(dir, operation = :update) ⇒ Object

Raises:

  • (ArgumentError) 


258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
 
# File 'lib/dbox/syncer.rb', line 258

def calculate_changes(dir, operation = :update)
  raise(ArgumentError, "Not a directory: #{dir.inspect}") unless dir[:is_dir]

  out = []
  recur_dirs = []

  # grab the metadata for the current dir (either off the filesystem or from Dropbox)
  res = gather_remote_info(dir)
  if res == :not_modified
    # directory itself was not modified, but we still need to
    # recur on subdirectories
    recur_dirs += database.subdirs(dir[:id]).map {|d| [:update, d] }
  else
    raise(ArgumentError, "Not a directory: #{res.inspect}") unless res[:is_dir]

    # dir may have changed -- calculate changes on contents
    contents = res.delete(:contents)
    if operation == :create || modified?(dir, res)
      res[:parent_id] = dir[:parent_id] if dir[:parent_id]
      res[:parent_path] = dir[:parent_path] if dir[:parent_path]
      out << [operation, res]
    end
    found_paths = []
    existing_entries = current_dir_entries_as_hash(dir)

    # process each entry that came back from dropbox/filesystem
    contents.each do |c|
      found_paths << c[:path]
      if entry = existing_entries[c[:path]]
        c[:id] = entry[:id]
        c[:modified] = parse_time(c[:modified])
        if c[:is_dir]
          # queue dir for later
          c[:remote_hash] = entry[:remote_hash]
          recur_dirs << [:update, c]
        else
          # update iff modified
          out << [:update, c] if modified?(entry, c)
        end
      else
        # create
        c[:modified] = parse_time(c[:modified])
        if c[:is_dir]
          # queue dir for later
          recur_dirs << [:create, c]
        else
          out << [:create, c]
        end
      end
    end

    # add any deletions
    out += case_insensitive_difference(existing_entries.keys, found_paths).map do |p|
      [:delete, existing_entries[p]]
    end
  end

  # recursively process new & existing subdirectories in parallel
  recur_dirs.each do |operation, dir|
    begin
      out += calculate_changes(dir, operation)
    rescue => e
      log.error "Error while caclulating changes for #{operation} on #{dir[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
      out += [[:failed, dir.merge({ :operation => operation, :error => e })]]
    end
  end

  out
end

#create_dir(dir) ⇒ Object 



336
337
338
339
340
341
342
343
 
# File 'lib/dbox/syncer.rb', line 336

def create_dir(dir)
  local_path = dir[:local_path]
  log.info "Creating #{local_path}"
  saving_parent_timestamp(dir) do
    FileUtils.mkdir_p(local_path)
    update_file_timestamp(dir)
  end
end

#create_file(file) ⇒ Object 



357
358
359
360
361
 
# File 'lib/dbox/syncer.rb', line 357

def create_file(file)
  saving_parent_timestamp(file) do
    download_file(file)
  end
end

#delete_dir(dir) ⇒ Object 



349
350
351
352
353
354
355
 
# File 'lib/dbox/syncer.rb', line 349

def delete_dir(dir)
  local_path = dir[:local_path]
  log.info "Deleting #{local_path}"
  saving_parent_timestamp(dir) do
    FileUtils.rm_r(local_path)
  end
end

#delete_file(file) ⇒ Object 



367
368
369
370
371
372
373
 
# File 'lib/dbox/syncer.rb', line 367

def delete_file(file)
  local_path = file[:local_path]
  log.info "Deleting file: #{local_path}"
  saving_parent_timestamp(file) do
    FileUtils.rm_rf(local_path)
  end
end

#download_file(file) ⇒ Object 



375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
 
# File 'lib/dbox/syncer.rb', line 375

def download_file(file)
  local_path = file[:local_path]
  remote_path = file[:remote_path]

  # check to ensure we aren't overwriting an untracked file or a
  # file with local modifications
  clobbering = false
  if entry = database.find_by_path(file[:path])
    clobbering = calculate_hash(local_path) != entry[:local_hash]
  else
    clobbering = File.exists?(local_path)
  end

  # stream files larger than the minimum
  stream = file[:size] && file[:size] > MIN_BYTES_TO_STREAM_DOWNLOAD

  # download to temp file
  tmp = generate_tmpfilename(file[:path])
  File.open(tmp, "wb") do |f|
    api.get_file(remote_path, f, stream)
  end

  # rename old file if clobbering
  if clobbering && File.exists?(local_path)
    backup_path = find_nonconflicting_path(local_path)
    FileUtils.mv(local_path, backup_path)
    backup_relpath = local_to_relative_path(backup_path)
    log.warn "#{file[:path]} had a conflict and the existing copy was renamed to #{backup_relpath} locally"
  end

  # atomic move over to the real file, and update the timestamp
  FileUtils.mv(tmp, local_path)
  update_file_timestamp(file)

  if backup_relpath
    [:conflict, { :original => file[:path], :renamed => backup_relpath }]
  else
    true
  end
end

#execute ⇒ Object 



178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
 
# File 'lib/dbox/syncer.rb', line 178

def execute
  remove_tmpfiles
  dir = database.root_dir
  changes = calculate_changes(dir)
  log.debug "Executing changes:\n" + changes.map {|c| c.inspect }.join("\n")
  parent_ids_of_failed_entries = []
  changelist = { :created => [], :deleted => [], :updated => [], :failed => [] }

  changes.each do |op, c|
    case op
    when :create
      c[:parent_id] ||= lookup_id_by_path(c[:parent_path])
      if c[:is_dir]
        # create the local directory
        create_dir(c)
        database.add_entry(c[:path], true, c[:parent_id], c[:modified], c[:revision], c[:remote_hash], nil)
        changelist[:created] << c[:path]
      else
        # download the new file
        begin
          res = create_file(c)
          local_hash = calculate_hash(c[:local_path])
          database.add_entry(c[:path], false, c[:parent_id], c[:modified], c[:revision], c[:remote_hash], local_hash)
          changelist[:created] << c[:path]
          if res.kind_of?(Array) && res[0] == :conflict
            changelist[:conflicts] ||= []
            changelist[:conflicts] << res[1]
          end
        rescue => e
          log.error "Error while downloading #{c[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
          parent_ids_of_failed_entries << c[:parent_id]
          changelist[:failed] << { :operation => :create, :path => c[:path], :error => e }
        end
      end
    when :update
      if c[:is_dir]
        # update the local directory
        update_dir(c)
        database.update_entry_by_path(c[:path], :modified => c[:modified], :revision => c[:revision], :remote_hash => c[:remote_hash])
        changelist[:updated] << c[:path]
      else
        # download updates to the file
        begin
          res = update_file(c)
          local_hash = calculate_hash(c[:local_path])
          database.update_entry_by_path(c[:path], :modified => c[:modified], :revision => c[:revision], :remote_hash => c[:remote_hash], :local_hash => local_hash)
          changelist[:updated] << c[:path]
          if res.kind_of?(Array) && res[0] == :conflict
            changelist[:conflicts] ||= []
            changelist[:conflicts] << res[1]
          end
        rescue => e
          log.error "Error while downloading #{c[:path]}: #{e.inspect}\n#{e.backtrace.join("\n")}"
          parent_ids_of_failed_entries << c[:parent_id]
          changelist[:failed] << { :operation => :create, :path => c[:path], :error => e }
        end
      end
    when :delete
      # delete the local directory/file
      c[:is_dir] ? delete_dir(c) : delete_file(c)
      database.delete_entry_by_path(c[:path])
      changelist[:deleted] << c[:path]
    when :failed
      parent_ids_of_failed_entries << c[:parent_id]
      changelist[:failed] << { :operation => c[:operation], :path => c[:path], :error => c[:error] }
    else
      raise(RuntimeError, "Unknown operation type: #{op}")
    end
  end

  # clear hashes on any dirs with children that failed so that
  # they are processed again on next pull
  parent_ids_of_failed_entries.uniq.each do |id|
    database.update_entry_by_id(id, :remote_hash => nil)
  end

  # sort & return output
  sort_changelist(changelist)
end

#modified?(entry, res) ⇒ Boolean

Returns:

  • (Boolean) 


328
329
330
331
332
333
334
 
# File 'lib/dbox/syncer.rb', line 328

def modified?(entry, res)
  out = (entry[:revision] != res[:revision]) ||
        !times_equal?(entry[:modified], res[:modified])
  out ||= (entry[:remote_hash] != res[:remote_hash]) if res.has_key?(:remote_hash)
  log.debug "#{entry[:path]} modified? r#{entry[:revision]} vs. r#{res[:revision]}, h#{entry[:remote_hash]} vs. h#{res[:remote_hash]}, t#{time_to_s(entry[:modified])} vs. t#{time_to_s(res[:modified])} => #{out}"
  out
end

#practice ⇒ Object 



172
173
174
175
176
 
# File 'lib/dbox/syncer.rb', line 172

def practice
  dir = database.root_dir
  changes = calculate_changes(dir)
  log.debug "Changes that would be executed:\n" + changes.map {|c| c.inspect }.join("\n")
end

#update_dir(dir) ⇒ Object 



345
346
347
 
# File 'lib/dbox/syncer.rb', line 345

def update_dir(dir)
  update_file_timestamp(dir)
end

#update_file(file) ⇒ Object 



363
364
365
 
# File 'lib/dbox/syncer.rb', line 363

def update_file(file)
  download_file(file)
end