Class: Cnvrg::Dataset
- Inherits:
-
Object
- Object
- Cnvrg::Dataset
- Defined in:
- lib/cnvrg/dataset.rb
Instance Attribute Summary collapse
-
#local_path ⇒ Object
readonly
Returns the value of attribute local_path.
-
#owner ⇒ Object
readonly
Returns the value of attribute owner.
-
#slug ⇒ Object
readonly
Returns the value of attribute slug.
-
#title ⇒ Object
readonly
Returns the value of attribute title.
-
#working_dir ⇒ Object
readonly
Returns the value of attribute working_dir.
Class Method Summary collapse
- .blank_clone(owner, dataset_name, dataset_slug) ⇒ Object
- .clone(owner, dataset_name, dataset_slug, remote = false) ⇒ Object
- .clone_tree(commit: 'latest', dataset_home: nil) ⇒ Object
- .delete(dataset_slug, owner) ⇒ Object
- .init(owner, dataset_name, is_public = false, bucket: nil) ⇒ Object
- .init_container(owner, dataset_slug, dataset_name) ⇒ Object
- .verify_cnvrgignore_exist(dataset_name, remote) ⇒ Object
- .verify_datasets(dataset_titles, timeout = 100) ⇒ Object
Instance Method Summary collapse
- #backup_idx ⇒ Object
- #change_url(owner: '', slug: '', title: '') ⇒ Object
- #clone(commit) ⇒ Object
- #compare_commit(commit) ⇒ Object
- #compare_commits(commit) ⇒ Object
- #compare_idx(new_branch, commit = last_local_commit, local_idx = nil, force = false, next_commit = nil) ⇒ Object
- #compare_idx_download(all_files: false, desired_commit: nil) ⇒ Object
- #create_volume ⇒ Object
- #current_status(new_branch) ⇒ Object
- #download_tags_yaml ⇒ Object
- #download_updated_data(current_commit) ⇒ Object
- #generate_chunked_idx(list_files = [], threads: IDXParallelThreads, prefix: '') ⇒ Object
- #generate_cnvrg_ignore ⇒ Object
- #generate_idx(show_progress = false) ⇒ Object
- #get_clone_chunk(latest_id: nil, chunk_size: 1000, offset: 0, commit: 'latest') ⇒ Object
- #get_current_commit ⇒ Object
- #get_dataset(commit: nil, query: nil) ⇒ Object
- #get_idx ⇒ Object
- #get_ignore_list ⇒ Object
- #get_next_commit ⇒ Object
- #get_partial_commit ⇒ Object
- #get_query_file(query_slug) ⇒ Object
- #get_stats(commit: nil, query: nil) ⇒ Object
- #get_storage_client ⇒ Object
- #get_storage_client_fallback ⇒ Object
- #init_home(remote: false) ⇒ Object
-
#initialize(project_home = '', dataset_url: '') ⇒ Dataset
constructor
A new instance of Dataset.
- #last_local_commit ⇒ Object
- #list(owner) ⇒ Object
- #list_all_files(with_ignore = false) ⇒ Object
- #list_commits ⇒ Object
- #list_files(commit_sha1: "latest", limit: 1000, offset: 0, expires: 3600) ⇒ Object
- #remove_next_commit ⇒ Object
- #restore_idx ⇒ Object
- #revert(working_dir) ⇒ Object
- #revert_next_commit ⇒ Object
- #revert_to_last_commit(commit: nil) ⇒ Object
- #search_queries ⇒ Object
- #send_ignore_list ⇒ Object
- #set_idx(idx) ⇒ Object
- #set_next_commit(commit_sha1) ⇒ Object
- #set_partial_commit(commit_sha1) ⇒ Object
- #snapshot ⇒ Object
- #soft_linked? ⇒ Boolean
- #softlinked? ⇒ Boolean
- #update_idx(idx) ⇒ Object
- #update_idx_with_commit!(commit) ⇒ Object
- #update_idx_with_files_commits!(files, commit_time) ⇒ Object
- #update_ignore_list(new_ignore) ⇒ Object
- #upload_tags_via_yml(tag_file = nil) ⇒ Object
- #url ⇒ Object
- #write_idx(tree = nil, commit = nil) ⇒ Object
- #write_success(in_folder = false) ⇒ Object
- #write_tree(tree) ⇒ Object
Constructor Details
#initialize(project_home = '', dataset_url: '') ⇒ Dataset
Returns a new instance of Dataset.
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 |
# File 'lib/cnvrg/dataset.rb', line 10 def initialize(project_home = '', dataset_url: '') begin @info = {} if project_home.present? @local_path = project_home @working_dir = project_home config = YAML.load_file(project_home + "/.cnvrg/config.yml") @title = config[:dataset_name] @slug = config[:dataset_slug] @owner = config[:owner] else owner, slug = Cnvrg::Helpers.extract_owner_slug_from_url(dataset_url, 'datasets') @title = slug @slug = slug @owner = owner @local_path = Dir.pwd end rescue => e end end |
Instance Attribute Details
#local_path ⇒ Object (readonly)
Returns the value of attribute local_path.
4 5 6 |
# File 'lib/cnvrg/dataset.rb', line 4 def local_path @local_path end |
#owner ⇒ Object (readonly)
Returns the value of attribute owner.
4 5 6 |
# File 'lib/cnvrg/dataset.rb', line 4 def owner @owner end |
#slug ⇒ Object (readonly)
Returns the value of attribute slug.
4 5 6 |
# File 'lib/cnvrg/dataset.rb', line 4 def slug @slug end |
#title ⇒ Object (readonly)
Returns the value of attribute title.
4 5 6 |
# File 'lib/cnvrg/dataset.rb', line 4 def title @title end |
#working_dir ⇒ Object (readonly)
Returns the value of attribute working_dir.
4 5 6 |
# File 'lib/cnvrg/dataset.rb', line 4 def working_dir @working_dir end |
Class Method Details
.blank_clone(owner, dataset_name, dataset_slug) ⇒ Object
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 |
# File 'lib/cnvrg/dataset.rb', line 315 def self.blank_clone(owner, dataset_name, dataset_slug) list_dirs = [".cnvrg" ] list_files = [ ".cnvrg/config.yml" ] create_ignore = false if !File.exist? ".cnvrgignore" list_files << ".cnvrgignore" create_ignore = true end cnvrgignore = Helpers.cnvrgignore_content begin config = {dataset_name: dataset_name, dataset_slug: dataset_slug, owner: owner} FileUtils.mkdir_p list_dirs FileUtils.touch list_files File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml} File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore} unless !create_ignore rescue => e return false end return true end |
.clone(owner, dataset_name, dataset_slug, remote = false) ⇒ Object
377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 |
# File 'lib/cnvrg/dataset.rb', line 377 def self.clone(owner, dataset_name, dataset_slug, remote = false) begin list_dirs = [] if !remote list_dirs << dataset_name end list_dirs << "#{dataset_name}/.cnvrg" list_files = [ "#{dataset_name}/.cnvrg/config.yml", ] config = {dataset_name: dataset_name, dataset_slug: dataset_slug, owner: owner} FileUtils.mkdir_p list_dirs FileUtils.touch list_files File.open("#{dataset_name}/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml} rescue => e puts "Exception in clone request:#{e.}" return false end return true end |
.clone_tree(commit: 'latest', dataset_home: nil) ⇒ Object
409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 |
# File 'lib/cnvrg/dataset.rb', line 409 def self.clone_tree(commit: 'latest', dataset_home: nil) @dataset = Cnvrg::Dataset.new(dataset_home) @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset) trees = @files.get_trees(commit: commit) return false if trees.nil? pb = ProgressBar.create(:title => "Download Progress", :progress_mark => '=', :format => "%b>>%i| %p%% %t", :starting_at => 0, :total => trees.size, :autofinish => true) trees.each do |tree| pb.progress += 1 @files.download_dir(dataset_home, tree) end pb.finish @dataset.write_success true end |
.delete(dataset_slug, owner) ⇒ Object
123 124 125 126 |
# File 'lib/cnvrg/dataset.rb', line 123 def self.delete(dataset_slug, owner) response = Cnvrg::API.request("users/#{owner}/datasets/#{dataset_slug}/delete", 'DELETE') return response end |
.init(owner, dataset_name, is_public = false, bucket: nil) ⇒ Object
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 |
# File 'lib/cnvrg/dataset.rb', line 280 def self.init(owner, dataset_name, is_public = false, bucket: nil) list_dirs = [".cnvrg" ] list_files = [ ".cnvrg/config.yml" ] create_ignore = false if !File.exist? ".cnvrgignore" list_files << ".cnvrgignore" create_ignore = true end cnvrgignore = Helpers.cnvrgignore_content begin response = Cnvrg::API.request("cli/create_dataset", 'POST', {title: dataset_name, owner: owner, is_public: is_public, bucket: bucket}) Cnvrg::CLI.is_response_success(response) response = JSON.parse response["result"] dataset_slug = response["slug"] config = {dataset_name: dataset_name, dataset_slug: dataset_slug, owner: owner} FileUtils.mkdir_p list_dirs FileUtils.touch list_files File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml} File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore} unless !create_ignore rescue => e return false end return true end |
.init_container(owner, dataset_slug, dataset_name) ⇒ Object
438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 |
# File 'lib/cnvrg/dataset.rb', line 438 def self.init_container(owner, dataset_slug, dataset_name) cnvrgignore = Helpers.cnvrgignore_content begin list_dirs = [".cnvrg" ] list_files = [ ".cnvrgignore", ".cnvrg/config.yml" ] FileUtils.mkdir_p list_dirs FileUtils.touch list_files config = {dataset_name: dataset_name, dataset_slug: dataset_slug, owner: owner} File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml} File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore} unless File.exist? ".cnvrgignore" rescue => e return false end return true end |
.verify_cnvrgignore_exist(dataset_name, remote) ⇒ Object
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 |
# File 'lib/cnvrg/dataset.rb', line 208 def self.verify_cnvrgignore_exist(dataset_name, remote) path = ".cnvrgignore" if !File.exist? path path = "#{dataset_name}/.cnvrgignore" end ignore_exits = File.exist? path if !ignore_exits begin list_files = [ path ] FileUtils.touch list_files cnvrgignore = Helpers.cnvrgignore_content File.open(path, "w+") {|f| f.write cnvrgignore} rescue => e return false end end end |
.verify_datasets(dataset_titles, timeout = 100) ⇒ Object
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 |
# File 'lib/cnvrg/dataset.rb', line 350 def self.verify_datasets(dataset_titles, timeout = 100) start_time = Time.now.to_i Cnvrg::Logger.log_info("Verifying datasets #{dataset_titles}") Cnvrg::Logger.log_info("Timeout is #{timeout}") while true begin current_time = Time.now.to_i return false if timeout < current_time - start_time all_are_ready = dataset_titles.all? do |dataset_title| config = YAML.load_file("/data/#{dataset_title}/.cnvrg/config.yml") config[:success] == true end return true if all_are_ready Cnvrg::Logger.log_info("Sleeping..") sleep 10 rescue => e Cnvrg::Logger.log_info("Got error") Cnvrg::Logger.log_error(e) sleep 10 end end end |
Instance Method Details
#backup_idx ⇒ Object
104 105 106 107 108 |
# File 'lib/cnvrg/dataset.rb', line 104 def backup_idx Cnvrg::Logger.log_info("Backup idx") idx = self.get_idx File.open("#{self.local_path}/.cnvrg/idx.yml.backup", 'w') {|f| f.write idx.to_yaml} end |
#change_url(owner: '', slug: '', title: '') ⇒ Object
117 118 119 120 |
# File 'lib/cnvrg/dataset.rb', line 117 def change_url(owner: '', slug: '', title: '') config = {dataset_home: title, dataset_slug: slug, owner: owner} File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml} end |
#clone(commit) ⇒ Object
373 374 375 |
# File 'lib/cnvrg/dataset.rb', line 373 def clone(commit) return end |
#compare_commit(commit) ⇒ Object
754 755 756 757 758 759 760 761 762 |
# File 'lib/cnvrg/dataset.rb', line 754 def compare_commit(commit) if commit.nil? or commit.empty? commit = last_local_commit end response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit}) CLI.is_response_success(response, false) update_is_new_branch(response["result"]["new_branch"]) return response["result"]["new_branch"] end |
#compare_commits(commit) ⇒ Object
697 698 699 700 701 |
# File 'lib/cnvrg/dataset.rb', line 697 def compare_commits(commit) response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/compare_commits", 'POST', {compare_commit: commit, current_commit: last_local_commit}) CLI.is_response_success(response, false) return response end |
#compare_idx(new_branch, commit = last_local_commit, local_idx = nil, force = false, next_commit = nil) ⇒ Object
620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 |
# File 'lib/cnvrg/dataset.rb', line 620 def compare_idx(new_branch, commit = last_local_commit, local_idx = nil, force = false, next_commit = nil) if local_idx.nil? local_idx = self.generate_idx end ignore_list = self.get_ignore_list() if force added = [] if local_idx[:tree] added << local_idx[:tree].keys added.flatten! end response = {"result" => {"commit" => next_commit, "tree" => {"added" => added, "updated_on_server" => [], "updated_on_local" => [], "deleted" => [], "conflicts" => []}}} return response end response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit, ignore: ignore_list, next_commit: next_commit}) CLI.is_response_success(response, false) return response end |
#compare_idx_download(all_files: false, desired_commit: nil) ⇒ Object
645 646 647 648 649 650 |
# File 'lib/cnvrg/dataset.rb', line 645 def compare_idx_download(all_files: false, desired_commit: nil) current_commit = self.last_local_commit next_commit = self.get_next_commit ignore_list = self.send_ignore_list() return Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_status", 'POST', {current_commit: current_commit, next_commit: next_commit, ignore: ignore_list, all_files: all_files, desired_commit: desired_commit.presence}) end |
#create_volume ⇒ Object
608 609 610 611 612 |
# File 'lib/cnvrg/dataset.rb', line 608 def create_volume response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST') CLI.is_response_success(response) return response end |
#current_status(new_branch) ⇒ Object
664 665 666 667 668 669 |
# File 'lib/cnvrg/dataset.rb', line 664 def current_status(new_branch) commit = last_local_commit response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status_current", 'POST', {current_commit: commit, new_branch: new_branch}) CLI.is_response_success(response, true) return response end |
#download_tags_yaml ⇒ Object
173 174 175 176 177 178 179 180 181 182 183 |
# File 'lib/cnvrg/dataset.rb', line 173 def response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/tags_yml", 'GET') CLI.is_response_success(response) begin path = self.working_dir File.open("#{path}/#{response["results"]["filename"]}", "w+") {|f| f.write response["results"]["file_content"]} return true rescue return false end end |
#download_updated_data(current_commit) ⇒ Object
614 615 616 617 618 |
# File 'lib/cnvrg/dataset.rb', line 614 def download_updated_data(current_commit) response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_updated_data", 'POST', {current_commit: current_commit}) CLI.is_response_success(response, false) return response end |
#generate_chunked_idx(list_files = [], threads: IDXParallelThreads, prefix: '') ⇒ Object
481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 |
# File 'lib/cnvrg/dataset.rb', line 481 def generate_chunked_idx(list_files = [], threads: IDXParallelThreads, prefix: '') tree = {} Parallel.map(list_files, in_threads: threads) do |file| #check if prefix exists do prefix/path otherwise path label = file.gsub(self.local_path + "/", "") label = "#{prefix}/#{label}" if prefix.present? if File.directory? file tree[label + "/"] = nil else sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest file_name = File.basename file file_size = File.size(file).to_f mime_type = MimeMagic.by_path(file) content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain" relative_path = file.gsub(/^#{@local_path + "/"}/, "") relative_path = "#{prefix}/#{relative_path}" if prefix.present? tree[label] = {sha1: sha1, file_name: file_name, file_size: file_size, content_type: content_type, absolute_path: file, relative_path: relative_path} end end if prefix.present? #add the prefix as dirs to the files #lets say the prefix is a/b/c so we want that a/, a/b/, a/b/c/ will be in our files_list dirs = prefix.split('/') curr_path = [] dirs.each do |dir| curr_path << dir list_files << curr_path.join('/') end end return tree end |
#generate_cnvrg_ignore ⇒ Object
345 346 347 348 |
# File 'lib/cnvrg/dataset.rb', line 345 def generate_cnvrg_ignore cnvrgignore = Helpers.cnvrgignore_content File.open(self.local_path + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore} end |
#generate_idx(show_progress = false) ⇒ Object
548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 |
# File 'lib/cnvrg/dataset.rb', line 548 def generate_idx(show_progress = false) if File.exists? "#{self.local_path}/.cnvrg/idx.yml" old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") else old_idx = nil end tree_idx = Hash.new(0) list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject {|x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~ /^#{self.local_path}\/\.cnvrgignore/)} list_ignore = self.get_ignore_list() if show_progress = { :progress => { :title => "Checking Dataset", :progress_mark => '=', :format => "%b>>%i| %p%% %t", :starting_at => 0, :total => (list).size, :autofinish => true }, in_threads: IDXParallelThreads, isolation: true } else = { in_threads: IDXParallelThreads, isolation: true } end Parallel.map(list, ) do |e| label = e.gsub(self.local_path + "/", "") if File.directory? e if list_ignore.include? label next end tree_idx[label + "/"] = nil else if list_ignore.include? label next end sha1 = OpenSSL::Digest::SHA1.file(e).hexdigest if old_idx.nil? or old_idx.to_h["tree"].nil? tree_idx[label] = {sha1: sha1, commit_time: nil} elsif old_idx["tree"][label].nil? or old_idx["tree"][label]["sha1"] != sha1 tree_idx[label] = {sha1: sha1, commit_time: nil} else tree_idx[label] = old_idx["tree"][label] end end end if !old_idx.nil? and !old_idx[:next_commit].nil? and !old_idx[:next_commit].empty? idx = {commit: old_idx.to_h[:commit], tree: tree_idx, next_commit: old_idx[:next_commit]} else idx = {commit: old_idx.to_h[:commit], tree: tree_idx} end idx_yaml = idx.to_yaml File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_yaml} return idx end |
#get_clone_chunk(latest_id: nil, chunk_size: 1000, offset: 0, commit: 'latest') ⇒ Object
97 98 99 100 101 |
# File 'lib/cnvrg/dataset.rb', line 97 def get_clone_chunk(latest_id: nil, chunk_size: 1000, offset: 0, commit: 'latest') response = Cnvrg::API.request("users/#{@owner}/datasets/#{@slug}/clone_chunk", 'POST', {commit: commit, chunk_size: chunk_size, latest_id: latest_id, offset: offset}) return nil unless Cnvrg::CLI.is_response_success(response, false) response['result']['files']['keys'] end |
#get_current_commit ⇒ Object
746 747 748 749 750 751 752 |
# File 'lib/cnvrg/dataset.rb', line 746 def get_current_commit() if !File.exist? "#{self.local_path}/.cnvrg/idx.yml" return nil end idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") return idx_hash[:commit] end |
#get_dataset(commit: nil, query: nil) ⇒ Object
56 57 58 59 60 61 62 63 64 |
# File 'lib/cnvrg/dataset.rb', line 56 def get_dataset(commit: nil, query: nil) if @dataset_call return @dataset_call end response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'POST',{ commit: commit, query:query}) Cnvrg::CLI.is_response_success(response,true) @dataset_call = response["result"] @dataset_call end |
#get_idx ⇒ Object
464 465 466 467 468 469 470 |
# File 'lib/cnvrg/dataset.rb', line 464 def get_idx if File.exists? "#{self.local_path}/.cnvrg/idx.yml" return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") end {commit: nil, tree: {}} end |
#get_ignore_list ⇒ Object
248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 |
# File 'lib/cnvrg/dataset.rb', line 248 def get_ignore_list ### handle case when after clone .cnvrgignore doesnt exists if not File.exists?(self.local_path + "/.cnvrgignore") self.generate_cnvrg_ignore end ignore_list = [] if not File.exists? "#{self.local_path}/.cnvrgignore" return ignore_list end File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line| line = line.strip if line.start_with? "#" or ignore_list.include? line or line.empty? next end if line.end_with? "/" or File.directory?(line) ignore_list << line all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten ignore_list << all_sub.flatten elsif line.include? "*" regex_list = Dir.glob("**/*#{line}", File::FNM_DOTMATCH).flatten ignore_list << regex_list else ignore_list << line end end return ignore_list.flatten end |
#get_next_commit ⇒ Object
718 719 720 721 722 723 724 |
# File 'lib/cnvrg/dataset.rb', line 718 def get_next_commit() if !File.exist? "#{self.local_path}/.cnvrg/idx.yml" return nil end idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") return idx_hash[:next_commit] end |
#get_partial_commit ⇒ Object
658 659 660 661 |
# File 'lib/cnvrg/dataset.rb', line 658 def get_partial_commit idx = self.get_idx idx.try(:fetch, :partial_commit) end |
#get_query_file(query_slug) ⇒ Object
162 163 164 165 166 167 168 169 170 171 |
# File 'lib/cnvrg/dataset.rb', line 162 def get_query_file(query_slug) response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/search/#{query_slug}", 'GET') CLI.is_response_success(response) row = [["Name", "Full path", "URL"]] response["results"]["query_files"].each do |file| row << [file["name"], file["fullpath"], file["s3_url"]] end return row end |
#get_stats(commit: nil, query: nil) ⇒ Object
91 92 93 94 95 |
# File 'lib/cnvrg/dataset.rb', line 91 def get_stats(commit: nil, query: nil) response = Cnvrg::API.request("users/#{@owner}/datasets/#{@slug}/clone", 'POST', {commit: commit, query: query}) Cnvrg::CLI.is_response_success(response, true) response['result'] end |
#get_storage_client ⇒ Object
71 72 73 74 75 76 77 78 79 |
# File 'lib/cnvrg/dataset.rb', line 71 def get_storage_client response = Cnvrg::API.request("users/#{@owner}/datasets/#{@slug}/client", 'GET') if Cnvrg::CLI.is_response_success(response, false) client_params = response['client'] else client_params = get_storage_client_fallback end Cnvrg::Downloader::Client.factory(client_params) end |
#get_storage_client_fallback ⇒ Object
81 82 83 84 85 86 87 88 |
# File 'lib/cnvrg/dataset.rb', line 81 def get_storage_client_fallback response = Cnvrg::API.request("users/#{@owner}/datasets/#{@slug}/download_multi", "POST", {files: []}) raise StandardError.new("Can't find dataset credentials") unless Cnvrg::CLI.is_response_success(response, false) files = response['files'] storage = files['is_s3'] ? 's3' : 'minio' files['storage'] = storage files end |
#init_home(remote: false) ⇒ Object
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 |
# File 'lib/cnvrg/dataset.rb', line 36 def init_home(remote: false) dataset_home = File.join(Dir.pwd, @slug) if Dir.exists? dataset_home if !remote Cnvrg::CLI.("Error: Conflict with dir #{@slug}", Thor::Shell::Color::RED) if Thor::Shell::Basic.new.no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW Cnvrg::CLI.("Remove dir in order to clone #{@slug}", Thor::Shell::Color::RED) exit(1) end end FileUtils.rm_rf(dataset_home) end # if Dataset.clone(owner, dataset_name, slug, remote) Dataset.clone(@owner, @slug, @slug, remote) @local_path = dataset_home Cnvrg::CLI.('') true end |
#last_local_commit ⇒ Object
128 129 130 131 132 133 134 |
# File 'lib/cnvrg/dataset.rb', line 128 def last_local_commit if !File.exist? "#{self.local_path}/.cnvrg/idx.yml" return nil end idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml") return idx[:commit] end |
#list(owner) ⇒ Object
144 145 146 147 148 149 |
# File 'lib/cnvrg/dataset.rb', line 144 def list(owner) response = Cnvrg::API.request("users/#{owner}/datasets/list", 'GET') CLI.is_response_success(response) return response end |
#list_all_files(with_ignore = false) ⇒ Object
523 524 525 526 527 528 529 530 531 |
# File 'lib/cnvrg/dataset.rb', line 523 def list_all_files(with_ignore = false) list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject {|x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~ /^#{self.local_path}\/\.cnvrgignore/)} #we want that big files will list = list.sort_by {|fn| File.size(fn)} return list if with_ignore list_ignore = self.get_ignore_list.map {|ignore_file| "#{self.local_path}/#{ignore_file}"} (list - list_ignore) end |
#list_commits ⇒ Object
185 186 187 188 189 190 |
# File 'lib/cnvrg/dataset.rb', line 185 def list_commits response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list_commits", 'GET') CLI.is_response_success(response) return response end |
#list_files(commit_sha1: "latest", limit: 1000, offset: 0, expires: 3600) ⇒ Object
403 404 405 406 407 |
# File 'lib/cnvrg/dataset.rb', line 403 def list_files(commit_sha1: "latest", limit: 1000, offset: 0, expires: 3600) response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list", 'GET', {commit_sha1: commit_sha1, limit: limit, offset: offset, expires: expires}) return nil if response.blank? response.to_json end |
#remove_next_commit ⇒ Object
726 727 728 729 730 731 732 733 734 735 |
# File 'lib/cnvrg/dataset.rb', line 726 def remove_next_commit() if !File.exist? "#{self.local_path}/.cnvrg/idx.yml" return nil end idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") idx = Hash.new() idx[:commit] = idx_hash[:next_commit] idx[:tree] = idx_hash[:tree] File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx.to_yaml} end |
#restore_idx ⇒ Object
110 111 112 113 114 115 |
# File 'lib/cnvrg/dataset.rb', line 110 def restore_idx Cnvrg::Logger.log_info("Restore idx because an error.") Cnvrg::Logger.log_method(bind: binding) idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml.backup") self.set_idx(idx) end |
#revert(working_dir) ⇒ Object
793 794 795 796 797 |
# File 'lib/cnvrg/dataset.rb', line 793 def revert(working_dir) FileUtils.rm_rf working_dir # response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET') # CLI.is_response_success(response) end |
#revert_next_commit ⇒ Object
737 738 739 740 741 742 743 744 |
# File 'lib/cnvrg/dataset.rb', line 737 def revert_next_commit() if !File.exist? "#{self.local_path}/.cnvrg/idx.yml" return nil end idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") idx_hash = idx_hash.except(:next_commit) File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml} end |
#revert_to_last_commit(commit: nil) ⇒ Object
512 513 514 515 516 517 518 519 520 521 |
# File 'lib/cnvrg/dataset.rb', line 512 def revert_to_last_commit(commit: nil) if commit.blank? resp = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/last_valid_commit", 'GET') if CLI.is_response_success(resp, false) commit = resp['result']['commit_sha1'] end end self.update_idx_with_commit(commit) if commit.present? self.revert_next_commit end |
#search_queries ⇒ Object
151 152 153 154 155 156 157 158 159 160 |
# File 'lib/cnvrg/dataset.rb', line 151 def search_queries response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/queries/list", 'GET') CLI.is_response_success(response) row = [["name", "id", "created_at", "username"]] response["results"]["queries"].each do |query| row << [query["name"], query["slug"], query["created_at"].in_time_zone.to_s, query["username"]] end return row end |
#send_ignore_list ⇒ Object
671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 |
# File 'lib/cnvrg/dataset.rb', line 671 def send_ignore_list() begin ignore_list = [] File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line| line = line.strip if line.start_with? "#" or ignore_list.include? line or line.empty? next end if line.end_with? "/" ignore_list << line.gsub("/", "") ignore_list << line + "." elsif line.include? "*" line = line.gsub("*", ".*") ignore_list << line else ignore_list << line end end return ignore_list.flatten rescue return [] end end |
#set_idx(idx) ⇒ Object
472 473 474 |
# File 'lib/cnvrg/dataset.rb', line 472 def set_idx(idx) File.open("#{self.local_path}/.cnvrg/idx.yml", 'w+') {|f| f.write idx.to_yaml} end |
#set_next_commit(commit_sha1) ⇒ Object
703 704 705 706 707 708 709 710 711 712 713 714 715 716 |
# File 'lib/cnvrg/dataset.rb', line 703 def set_next_commit(commit_sha1) if !File.exist? "#{self.local_path}/.cnvrg/idx.yml" idx_hash = Hash.new() idx_hash[:commit] = "" idx_hash[:tree] = "" else idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") end idx_hash[:next_commit] = commit_sha1 File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml} return true end |
#set_partial_commit(commit_sha1) ⇒ Object
652 653 654 655 656 |
# File 'lib/cnvrg/dataset.rb', line 652 def set_partial_commit(commit_sha1) idx = self.get_idx idx[:partial_commit] = commit_sha1 self.set_idx(idx) end |
#snapshot ⇒ Object
136 137 138 139 140 141 142 |
# File 'lib/cnvrg/dataset.rb', line 136 def snapshot commit = last_local_commit response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST', {data_commit: commit}) CLI.is_response_success(response) return response end |
#soft_linked? ⇒ Boolean
32 33 34 |
# File 'lib/cnvrg/dataset.rb', line 32 def soft_linked? @dataset_call["dataset_type"] == "soft_link_dataset" end |
#softlinked? ⇒ Boolean
66 67 68 |
# File 'lib/cnvrg/dataset.rb', line 66 def softlinked? get_dataset["dataset_type"] == "soft_link_dataset" end |
#update_idx(idx) ⇒ Object
778 779 780 781 782 |
# File 'lib/cnvrg/dataset.rb', line 778 def update_idx(idx) File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx.to_yaml} return true end |
#update_idx_with_commit!(commit) ⇒ Object
785 786 787 788 789 790 791 |
# File 'lib/cnvrg/dataset.rb', line 785 def update_idx_with_commit!(commit) idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") idx_hash[:commit] = commit File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml} return true end |
#update_idx_with_files_commits!(files, commit_time) ⇒ Object
764 765 766 767 768 769 770 771 772 773 774 775 776 |
# File 'lib/cnvrg/dataset.rb', line 764 def update_idx_with_files_commits!(files, commit_time) # files.flatten! idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") # idx_hash[:commit] = commit files.each do |path| idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time end idx_hash[:next_commit] = idx_hash[:next_commit] File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml} return true end |
#update_ignore_list(new_ignore) ⇒ Object
229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 |
# File 'lib/cnvrg/dataset.rb', line 229 def update_ignore_list(new_ignore) if new_ignore.nil? or new_ignore.empty? return true end begin File.open(self.local_path + "/.cnvrgignore", "a+") do |f| f.puts("\n") new_ignore.each do |i| f.puts("#{i}\n") end end return true rescue return false end end |
#upload_tags_via_yml(tag_file = nil) ⇒ Object
192 193 194 195 196 197 198 199 200 201 |
# File 'lib/cnvrg/dataset.rb', line 192 def (tag_file = nil) records_yml = YAML.load_file(tag_file) tag_file.close response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/data_tags_create", 'POST', {records_yml: records_yml}) if response["status"] == 200 return true else return false end end |
#url ⇒ Object
203 204 205 206 |
# File 'lib/cnvrg/dataset.rb', line 203 def url url = Cnvrg::Helpers.remote_url "#{url}/#{self.owner}/projects/#{self.slug}" end |
#write_idx(tree = nil, commit = nil) ⇒ Object
533 534 535 536 537 538 539 540 |
# File 'lib/cnvrg/dataset.rb', line 533 def write_idx(tree = nil, commit = nil) if tree.blank? tree = self.generate_idx[:tree] tree = tree.map {|k, v| (v.present?) ? [k, {sha1: v[:sha1], commit_time: Time.now}] : [k, v]}.to_h end idx = {tree: tree, commit: commit} File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx.to_yaml} end |
#write_success(in_folder = false) ⇒ Object
429 430 431 432 433 434 435 436 |
# File 'lib/cnvrg/dataset.rb', line 429 def write_success(in_folder = false) file_path = ".cnvrg/config.yml" file_path = File.join(@local_path || @working_dir, file_path) if File.exist?(file_path) File.open(file_path, "a") {|f| f.puts(":success: true")} end rescue end |
#write_tree(tree) ⇒ Object
542 543 544 545 546 |
# File 'lib/cnvrg/dataset.rb', line 542 def write_tree(tree) idx = self.get_idx idx[:tree] = tree self.set_idx(idx) end |