649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
|
# File 'lib/cnvrg/dataset.rb', line 649
def generate_idx(show_progress = false)
if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
else
old_idx = nil
end
tree_idx = Hash.new(0)
list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject {|x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~ /^#{self.local_path}\/\.cnvrgignore/)}
list_ignore = self.get_ignore_list()
if show_progress
parallel_options = {
:progress => {
:title => "Checking Dataset",
:progress_mark => '=',
:format => "%b>>%i| %p%% %t",
:starting_at => 0,
:total => (list).size,
:autofinish => true
},
in_threads: IDXParallelThreads,
isolation: true
}
else
parallel_options = {
in_threads: IDXParallelThreads,
isolation: true
}
end
Parallel.map(list, parallel_options) do |e|
label = e.gsub(self.local_path + "/", "")
if File.directory? e
if list_ignore.include? label
next
end
tree_idx[label + "/"] = nil
else
if list_ignore.include? label
next
end
sha1 = OpenSSL::Digest::SHA1.file(e).hexdigest
if old_idx.nil? or old_idx.to_h["tree"].nil?
tree_idx[label] = {sha1: sha1, commit_time: nil}
elsif old_idx["tree"][label].nil? or old_idx["tree"][label]["sha1"] != sha1
tree_idx[label] = {sha1: sha1, commit_time: nil}
else
tree_idx[label] = old_idx["tree"][label]
end
end
end
if !old_idx.nil? and !old_idx[:next_commit].nil? and !old_idx[:next_commit].empty?
idx = {commit: old_idx.to_h[:commit], tree: tree_idx, next_commit: old_idx[:next_commit]}
else
idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
end
idx_yaml = idx.to_yaml
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_yaml}
return idx
end
|