Module: MrMurano::SyncUpDown

Includes:
SyncAllowed, SyncCore, Verbose
Included in:
Gateway::Resources, SolutionBase, Webservice::WebserviceBase
Defined in:
lib/MrMurano/SyncUpDown.rb,
lib/MrMurano/SyncUpDown-Item.rb

Overview

The functionality of a Syncable thing.

This provides the logic for computing what things have changed, and pushing and pulling those things.

Defined Under Namespace

Classes: Item

Constant Summary

Constants included from Verbose

Verbose::TABULARIZE_DATA_FORMAT_ERROR

Instance Method Summary collapse

Methods included from Verbose

ask_yes_no, #ask_yes_no, #assert, assert, cmd_confirm_delete!, #cmd_confirm_delete!, debug, #debug, dump_file_json, dump_file_plain, dump_file_yaml, #dump_output_file, #error, error, #error_file_format!, fancy_ticks, #fancy_ticks, #load_file_json, #load_file_plain, #load_file_yaml, #load_input_file, outf, #outf, #outformat_engine, #pluralize?, pluralize?, #prepare_hash_csv, #read_hashf!, #tabularize, tabularize, verbose, #verbose, warning, #warning, #whirly_interject, whirly_interject, #whirly_linger, whirly_linger, #whirly_msg, whirly_msg, #whirly_pause, whirly_pause, #whirly_start, whirly_start, #whirly_stop, whirly_stop, #whirly_unpause, whirly_unpause

Methods included from SyncCore

#debug_selected, #dodiff, #dodiff_build_cmd, #dodiff_cull_tempfile_paths, #dodiff_do_diff, #dodiff_download_remote, #dodiff_flexible, #dodiff_header_aware, #dodiff_local_to_tempfile, #dodiff_prepare_local_and_diff, #dodiff_resolve_localname, #dodiff_tempfile_paths, #filter_solution, #init_mods_and_chgs_arrs, #item_dirty_set_status, #item_local_there_merged, #item_merged_diff_status, #item_merged_set_status, #item_select_selected!, #items_classify_and_find_duplicates, #items_cull_clashes!, #items_lists, #items_log_duplicates, #items_log_duplicates_there_local, #items_mods_and_chgs!, #items_new_and_old!, #select_selected!, #sort_by_name, #status, #sync_update_progress, #syncable_validate_api_id, #syncdown, #syncdown_item, #syncup, #syncup_item

Methods included from SyncAllowed

#download_item_allowed, #remove_item_allowed, #removelocal_item_allowed, #sync_item_allowed, #upload_item_allowed

Instance Method Details

#config_vars_decode(script, _event_event = nil) ⇒ Object 



483
484
485
 
# File 'lib/MrMurano/SyncUpDown.rb', line 483

def config_vars_decode(script, _event_event=nil)
  script
end

#config_vars_encode(script, _event_event = nil) ⇒ Object 



487
488
489
 
# File 'lib/MrMurano/SyncUpDown.rb', line 487

def config_vars_encode(script, _event_event=nil)
  script
end

#debug_print_localitems(items) ⇒ Object 



451
452
453
454
455
456
 
# File 'lib/MrMurano/SyncUpDown.rb', line 451

def debug_print_localitems(items)
  return unless $cfg['tool.debug']
  loci = items.map { |it| it.location_friendly(full_path: true) }
  item_list = loci.sort.join("\n  ")
  debug "#{self.class}: localitems' matches:\n  #{item_list}"
end

#diff_download(tmp_path, merged, options) ⇒ Object 



203
204
205
 
# File 'lib/MrMurano/SyncUpDown.rb', line 203

def diff_download(tmp_path, merged, options)
  download(tmp_path, merged, options: options, is_tmp: true)
end

#diff_item_write(io, merged, _local, _remote) ⇒ Object 



254
255
256
257
258
 
# File 'lib/MrMurano/SyncUpDown.rb', line 254

def diff_item_write(io, merged, _local, _remote)
  contents = merged[:local_path].read
  contents = config_vars_decode(contents)
  io << contents
end

#docmp(_item_a, _item_b) ⇒ Object

True if itemA and itemB are different

Children objects must override this



73
74
75
 
# File 'lib/MrMurano/SyncUpDown.rb', line 73

def docmp(_item_a, _item_b)
  true
end

#download(local, item, options: {}, is_tmp: false) ⇒ Object

Download an item into local

Children objects should override this or implement #fetch()

Parameters:

  • local (Pathname) —

    Full path of where to download to

  • item (Item) —

    The item to download



163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
 
# File 'lib/MrMurano/SyncUpDown.rb', line 163

def download(local, item, options: {}, is_tmp: false)
  #if item[:bundled]
  #  warning "Not downloading into bundled item #{synckey(item)}"
  #  return
  #end
  id = item[@itemkey.to_sym]
  if id.to_s.empty?
    if @itemkey.to_sym != :id
      debug "Missing '#{@itemkey}', trying :id instead"
      id = item[:id]
    end
    if id.to_s.empty?
      debug %(Missing id: remote: #{item[:name]} / local: #{local} / item: #{item})
      return if options[:ignore_errors]
      error %(Remote item missing :id => #{local})
      say %(You can ignore this error using --ignore-errors)
      exit 1
    end
    debug ":id => #{id}"
  end
  unless is_tmp
    relpath = local.relative_path_from(Pathname.pwd).to_s
    return unless download_item_allowed(relpath)
  end
  # MAYBE: If is_tmp and doing syncdown, just use this file rather
  # than downloading again.
  local.dirname.mkpath
  local.open('wb') do |io|
    # Do not modify remote content when diffing, e.g., do not add #ENDPOINT header.
    untainted = options[:diff] || false
    fetch(id, untainted) do |chunk|
      # First chunk may be header, if not part of script.
      # Second chunk (only only chunk), is script (which may include header).
      encoded = is_tmp && chunk || config_vars_encode(chunk)
      io.write(encoded)
    end
  end
  update_mtime(local, item)
end

#ignore?(path, pattern) ⇒ Boolean

Returns:

  • (Boolean) 


458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
 
# File 'lib/MrMurano/SyncUpDown.rb', line 458

def ignore?(path, pattern)
  # 2017-08-18: [lb] not sure this block should be disabled for no-nesting.
  # The block *was* added for Nested Lua support. But I think it was
  # more necessary because modules.include is now '**/*.lua', not '*/*.lua'.
  # Or maybe this block was because we now use expand_path, not realpath.
  if !$cfg['modules.no-nesting'] && pattern.start_with?('**/')
    # E.g., '**/.*' or '**/*'
    dirname = File.dirname(path)
    return true if ['.', ::File::ALT_SEPARATOR, ::File::SEPARATOR].include?(dirname)
    # There's at least one ancestor directory.
    # Remove the '**', which ::File.fnmatch doesn't recognize, and the path delimiter.
    # 2017-08-08: Why does Rubocop not follow Style/RegexpLiteral here?
    #pattern = pattern.gsub(/^\*\*\//, '')
    pattern = pattern.gsub(%r{^\*\*\/}, '')
  end

  ignore = ::File.fnmatch(pattern, path)
  debug "Excluded #{path}" if ignore
  ignore
end

#ignoring ⇒ Array<String>

Returns array of globs of files to ignore

Returns:

  • (Array<String>) —

    of Strings that are globs



398
399
400
401
 
# File 'lib/MrMurano/SyncUpDown.rb', line 398

def ignoring
  raise 'Missing @project_section' if @project_section.nil?
  $project["#{@project_section}.exclude"]
end

#list ⇒ Array<Item>

Get a list of remote items.

Children objects Must override this

Returns:

  • (Array<Item>) —

    of item details



36
37
38
 
# File 'lib/MrMurano/SyncUpDown.rb', line 36

def list
  []
end

#localitems(from) ⇒ Array<Item>

Get a list of local items rooted at #from

Children rarely need to override this. Only when the locallist is not a set of files in a directory will they need to override it.

Parameters:

  • from (Pathname) —

    Directory of items to scan

Returns:

  • (Array<Item>) —

    Items found



411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
 
# File 'lib/MrMurano/SyncUpDown.rb', line 411

def localitems(from)
  debug "#{self.class}: Getting local items from:\n  #{from}"
  search_in = from.to_s
  sf = searchFor.map { |i| ::File.join(search_in, i) }
  debug "#{self.class}: #{@project_section}.include:\n  #{sf.sort.join("\n  ")}"
  debug "#{self.class}: #{@project_section}.exclude:\n  #{ignoring.sort.join("\n  ")}"
  # 2017-07-27: Add uniq to cull duplicate entries that globbing
  # all the ways might produce, otherwise status/sync/diff complain
  # about duplicate resources. I [lb] think this problem has existed
  # but was exacerbated by the change to support sub-directory scripts
  # (Nested Lua support).
  files = Dir[*sf].collect { |path| File.absolute_path(path) }
  files = files.uniq.flatten.compact.reject do |path|
    if ::File.directory?(path)
      true
    else
      ignoring.any? { |pattern| ignore?(path, pattern) }
    end
  end
  items = files.map do |path|
    # Do not resolve symlinks, just relative paths (. and ..),
    # otherwise it makes nested Lua support tricky, because
    # symlinks might be outside the root item path, and then
    # the nested Lua path looks like ".......some_dir/some_item".
    if $cfg['modules.no-nesting']
      rpath = Pathname.new(path).realpath
    else
      rpath = Pathname.new(path).expand_path
    end
    files_items = to_remote_items(from, rpath)
    files_items.compact.map do |item|
      item[:local_path] = rpath
      item
    end
  end
  items = items.flatten.compact.sort_by { |item| item[:local_path] }
  debug_print_localitems(items)
  sort_by_name(items)
end

#locallist(skip_warn: false) ⇒ Array<Item>

Get a list of local items.

Children should never need to override this. Instead they should override #localitems.

This collects items in the project and all bundles. 2017-07-02: [lb] removed this commented-out code from locallist body.

See "Bundles" comments in TODO.taskpaper.
This code builds the list of local items from all bundle
subdirectories. Would that be how a bundles implementation
works? Or would we rather just iterate over each bundle and
process them separately, rather than all together at once?

 def locallist
   # so. if @locationbase/bundles exists
   #  gather and merge: @locationbase/bundles/*/@location
   # then merge @locationbase/@location
   #
   bundleDir = $cfg['location.bundles'] or 'bundles'
   bundleDir = 'bundles' if bundleDir.nil?
   items = {}
   if (@locationbase + bundleDir).directory?
     (@locationbase + bundleDir).children.sort.each do |bndl|
       if (bndl + @location).exist?
         verbose("Loading from bundle #{bndl.basename}")
         bitems = localitems(bndl + @location)
         bitems.map!{|b| b[:bundled] = true; b} # mark items from bundles.
         # use synckey for quicker merging.
         bitems.each { |b| items[synckey(b)] = b }
       end
     end
   end
 end

Returns:

  • (Array<Item>) —

    items found



306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
 
# File 'lib/MrMurano/SyncUpDown.rb', line 306

def locallist(skip_warn: false)
  items = {}
  if location.exist?
    # Get a list of SyncUpDown::Item's, or a class derived thereof.
    bitems = localitems(location)
    # Check for duplicates first -- two files with the same identity.
    seen = locallist_mark_seen(bitems)
    counts = {}
    bitems.each do |item|
      locallist_add_item(item, items, seen, counts)
    end
  elsif !skip_warn
    locallist_complain_missing
  end
  items.values
end

#locallist_add_item(item, items, seen, counts) ⇒ Object 



332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
 
# File 'lib/MrMurano/SyncUpDown.rb', line 332

def locallist_add_item(item, items, seen, counts)
  skey = synckey(item)
  if seen[skey] > 1
    if items[skey].nil?
      items[skey] = item.clone
      items[skey][:dup_count] = 0
    end
    counts[skey] = counts.key?(skey) && (counts[skey] + 1) || 1
    # Use a unique synckey so all duplicates make it in the list.
    uniq_synckey = "#{skey}-#{counts[skey]}"
    item[:dup_count] = counts[skey]
    # This sets the alias for the output, so duplicates look unique.
    item[@itemkey.to_sym] = uniq_synckey
    items[uniq_synckey] = item
  else
    items[skey] = item
  end
end

#locallist_complain_missing ⇒ Object 



351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
 
# File 'lib/MrMurano/SyncUpDown.rb', line 351

def locallist_complain_missing
  @missing_complaints = [] unless defined?(@missing_complaints)
  return if @missing_complaints.include?(location)
  # MEH/2017-07-31: This message is a little misleading on syncdown,
  #   e.g., in rspec ./spec/cmd_syncdown_spec.rb, one test blows away
  #   local directories and does a syncdown, and on stderr you'll see
  #     Skipping missing location
  #      ‘/tmp/d20170731-3150-1f50uj4/project/specs/resources.yaml’ (Resources)
  #   but then later in the syncdown, that directory and file gets created.
  msg = "Skipping missing location #{fancy_ticks(location)}"
  unless self.class.description.to_s.empty?
    msg += " (#{Inflecto.pluralize(self.class.description)})"
  end
  warning(msg)
  @missing_complaints << location
end

#locallist_mark_seen(bitems) ⇒ Object 



323
324
325
326
327
328
329
330
 
# File 'lib/MrMurano/SyncUpDown.rb', line 323

def locallist_mark_seen(bitems)
  seen = {}
  bitems.each do |item|
    skey = synckey(item)
    seen[skey] = seen.key?(skey) && seen[skey] + 1 || 1
  end
  seen
end

#location ⇒ Pathname

Get the full path for the local versions

Returns:

  • (Pathname) —

    Location for local items



381
382
383
384
 
# File 'lib/MrMurano/SyncUpDown.rb', line 381

def location
  raise 'Missing @project_section' if @project_section.nil?
  Pathname.new($cfg['location.base']) + $project["#{@project_section}.location"]
end

#match(_item, _pattern) ⇒ Bool

Does item match pattern?

Children objects should override this if synckey is not @itemkey

Check child specific patterns against item

Parameters:

  • item (Item) —

    Item to be checked

  • pattern (String) —

    pattern to check with

Returns:

  • (Bool) —

    true or false



142
143
144
 
# File 'lib/MrMurano/SyncUpDown.rb', line 142

def match(_item, _pattern)
  false
end

#remove(_itemkey) ⇒ Object

Remove remote item

Children objects Must override this

Parameters:

  • itemkey (String) —

    The identifying key for this item



45
46
47
48
49
 
# File 'lib/MrMurano/SyncUpDown.rb', line 45

def remove(_itemkey)
  # :nocov:
  raise 'Forgotten implementation'
  # :nocov:
end

#remove_or_clear(itemkey, _thereitem, _modify = false) ⇒ Object 



51
52
53
 
# File 'lib/MrMurano/SyncUpDown.rb', line 51

def remove_or_clear(itemkey, _thereitem, _modify=false)
  remove(itemkey)
end

#removelocal(dest, _item) ⇒ Object

Remove local reference of item

Children objects should override this if move than just unlinking the local item.

Parameters:

  • dest (Pathname) —

    Full path of item to be removed

  • item (Item) —

    Full details of item to be removed



233
234
235
236
 
# File 'lib/MrMurano/SyncUpDown.rb', line 233

def removelocal(dest, _item)
  return unless removelocal_item_allowed(dest)
  dest.unlink if dest.exist?
end

#resolve_config_var_usage!(there, local) ⇒ Object 



479
480
481
 
# File 'lib/MrMurano/SyncUpDown.rb', line 479

def resolve_config_var_usage!(there, local)
  # pass; derived classes should implement.
end

#resurrect_undeletables(localbox, _therebox) ⇒ Object

Some items are considered “undeletable”, meaning if a corresponding file does not exist locally, or if the user deletes such a file, we do not delete it on the server, but instead set it to the empty string. The reverse is also true: if a service script on the platform is empty, we do not need to create a file for it locally.



373
374
375
376
 
# File 'lib/MrMurano/SyncUpDown.rb', line 373

def resurrect_undeletables(localbox, _therebox)
  # It's up to the Syncables to implement this, if they care.
  localbox
end

#searchFor ⇒ Array<String>

Returns array of globs to search for files rubocop:disable Style/MethodName: Use snake_case for method names.

MAYBE/2017-07-18: Rename this. Beware the config has a related keyname.

Returns:

  • (Array<String>) —

    of Strings that are globs



391
392
393
394
 
# File 'lib/MrMurano/SyncUpDown.rb', line 391

def searchFor
  raise 'Missing @project_section' if @project_section.nil?
  $project["#{@project_section}.include"]
end

#syncdown_after(_local) ⇒ Object 



250
251
252
 
# File 'lib/MrMurano/SyncUpDown.rb', line 250

def syncdown_after(_local)
  0
end

#syncdown_before ⇒ Object 



246
247
248
 
# File 'lib/MrMurano/SyncUpDown.rb', line 246

def syncdown_before
  syncable_validate_api_id
end

#synckey(item) ⇒ Object

Get the key used to quickly compare two items

Children objects should override this if synckey is not @itemkey

Parameters:

  • item (Item) —

    The item to get a key from

Returns:

  • (Object) —

    The object to use a comparison key



152
153
154
155
 
# File 'lib/MrMurano/SyncUpDown.rb', line 152

def synckey(item)
  key = @itemkey.to_sym
  item[key]
end

#syncup_after ⇒ Object 



242
243
244
 
# File 'lib/MrMurano/SyncUpDown.rb', line 242

def syncup_after
  0
end

#syncup_before ⇒ Object 



238
239
240
 
# File 'lib/MrMurano/SyncUpDown.rb', line 238

def syncup_before
  syncable_validate_api_id
end

#to_remote_items(root, path) ⇒ Item

Compute a remote item hash from the local path

Children objects should override this.

Parameters:

  • root (Pathname, String) —

    Root path for this resource type from config files

  • path (Pathname, String) —

    Path to local item

Returns:

  • (Item) —

    hash of the details for the remote item for this path



91
92
93
94
95
96
97
98
99
 
# File 'lib/MrMurano/SyncUpDown.rb', line 91

def to_remote_items(root, path)
  # This mess brought to you by Windows short path names.
  path = Dir.glob(path.to_s).first
  root = Dir.glob(root.to_s).first
  path = Pathname.new(path)
  root = Pathname.new(root)
  item = Item.new(name: path.realpath.relative_path_from(root.realpath).to_s)
  [item]
end

#tolocalname(item, itemkey) ⇒ Object

Compute the local name from remote item details

Children objects should override this or #tolocalpath

Parameters:

  • item (Item) —

    listing details for the item.

  • itemkey (Symbol) —

    Key for look up.



108
109
110
 
# File 'lib/MrMurano/SyncUpDown.rb', line 108

def tolocalname(item, itemkey)
  item[itemkey].to_s
end

#tolocalpath(into, item) ⇒ Pathname

Compute the local path from the listing details

If there is already a matching local item, some of its details are also in the item hash.

Children objects should override this or #tolocalname

Parameters:

  • into (Pathname) —

    Root path for this resource type from config files

  • item (Item) —

    listing details for the item.

Returns:

  • (Pathname) —

    path to save (or merge) remote item into



123
124
125
126
127
128
129
130
131
 
# File 'lib/MrMurano/SyncUpDown.rb', line 123

def tolocalpath(into, item)
  return item[:local_path] unless item[:local_path].nil?
  itemkey = @itemkey.to_sym
  name = tolocalname(item, itemkey)
  raise "Bad key(#{itemkey}) for #{item}" if name.nil?
  name = Pathname.new(name) unless name.is_a? Pathname
  name = name.relative_path_from(Pathname.new('/')) if name.absolute?
  into + name
end

#update_mtime(local, item) ⇒ Object

Give the local file the same timestamp as the remote, because diff.

Parameters:

  • local (Pathname) —

    Full path of where to download to

  • item (Item) —

    The item to download



211
212
213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/MrMurano/SyncUpDown.rb', line 211

def update_mtime(local, item)
  return unless item[:updated_at]
  mod_time = item[:updated_at]
  mod_time = Time.parse(mod_time) unless mod_time.is_a?(Time)
  begin
    FileUtils.touch([local.to_path], mtime: mod_time)
  rescue Errno::EACCES => err
    # (lb): This is okay on Windows. (We really need a better solution.)
    unless OS.windows?
      msg = 'Unexpected: touch failed on non-Windows machine'
      warning "#{msg} / host_os: #{RbConfig::CONFIG['host_os']} / err: #{err}"
    end
  end
end

#upload(_src, _item, _modify) ⇒ Object

Upload local item to remote

Children objects Must override this

Parameters:

  • src (Pathname) —

    Full path of where to upload from

  • item (Hash) —

    The item details to upload

  • modify (Bool) —

    True if item exists already and this is changing it



62
63
64
65
66
 
# File 'lib/MrMurano/SyncUpDown.rb', line 62

def upload(_src, _item, _modify)
  # :nocov:
  raise 'Forgotten implementation'
  # :nocov:
end