Class: Cnvrg::CLI

Inherits:

Object
Thor
Cnvrg::CLI

Defined in:: lib/cnvrg/cli.rb

Constant Summary collapse

INSTALLATION_URLS =

{docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}

IP =

"localhost"

PORT =

Class Method Summary collapse

.is_thor_reserved_word?(word, type) ⇒ Boolean

Hackery.Take the run method away from Thor so that we can redefine it.

Instance Method Summary collapse

Constructor Details

#initialize(*args) ⇒ `CLI`

Returns a new instance of CLI.

# File 'lib/cnvrg/cli.rb', line 180

def initialize(*args)
  super
  self.log_handler
end

Class Method Details

.is_thor_reserved_word?(word, type) ⇒ `Boolean`

Hackery.Take the run method away from Thor so that we can redefine it.

Returns:

(Boolean)

# File 'lib/cnvrg/cli.rb', line 165

def is_thor_reserved_word?(word, type)
  return false if word == "run"
  super
end

Instance Method Details

#auth ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 465

def auth
  token = ENV['CNVRG_TOKEN'] || exit(1)
  owner = ENV['CNVRG_OWNER'] || exit(1)
  user = ENV['CNVRG_USER'] || exit(1)
  api = ENV['CNVRG_API'] || exit(1)
  email = ENV['CNVRG_EMAIL'] || exit(1)

  netrc = Netrc.read
  netrc[Cnvrg::Helpers.netrc_domain] = email, token
  netrc.save

  set_owner(owner, user, api)
end

#build(*cmd) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4300

def build(*cmd)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    working_dir = is_cnvrg_dir
    install_file = options["install"] || nil
    if !install_file.nil?
      commands = File.open(install_file).read.chop.gsub!("\n", ",").split(",")

    else
      commands = [cmd.join(" ")]
    end


    image = is_project_with_docker(working_dir)
    if image and image.is_docker
      container = image.get_container
      if !container

        say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
        exit(1)
      end
    else
      say "Project is not configured with any image", Thor::Shell::Color::RED
      exit(1)

    end
    commands.each do |c|
      if c.include? "pip"
        c.sub("pip", "/opt/ds/bin/pip")
      end
      if c.include? "pip3"
        c.sub("pip3", "/opt/ds3/bin/pip3")
      end

      say "Running #{c}", Thor::Shell::Color::BLUE
      command = ["/bin/bash", "-lc", "#{c}"]
      res = container.exec(command, tty: false)
      if res[2] != 0
        say "Could not run command: #{c}, #{res[1][0]}", Thor::Shell::Color::RED
        container.stop()
        log_end(0)

        exit(1)
      end
      say res[0].join("\n")
      image.store_image_build_commands(working_dir, c)
    end

    checks = Helpers.checkmark()
    say "Updating image", Thor::Shell::Color::BLUE
    # image.create_custom_image("",working_dir)
    container.stop()
    say "#{checks} Done", Thor::Shell::Color::GREEN

    log_end(0)
  rescue => e
    log_end(-1, e.message)
    say "Error occurred, aborting", Thor::Shell::Color::RED
    if container
      container.stop()
    end
  rescue SignalException
    log_End(-1)
    if container
      container.stop()
    end
    say "\nAborting"
    exit(1)
  end

end

#build_image(image_name) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4173

def build_image(image_name)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
                 "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
    instance_type = get_instance_type(instances)
    image_extend = options["image"]
    public = options["public"]
    base = options["base"]
    python3 = options["python3"]
    docker_path = options["docker_path"]
    owner = CLI.get_owner
    checks = Helpers.checkmark()
    tar_path = nil
    if !docker_path.nil? and !docker_path.empty?
      docker_path = File.absolute_path(docker_path)
      #create tar of the docker path: it could be a docker file, and it could be a docker folder
      tar_path = File.expand_path('~') + "/.cnvrg/tmp/docker_#{File.basename docker_path}.tar.gz"
      resp = create_docker_tar(docker_path, tar_path)
      if !resp
        log_message("Couldn't create tar from docker path", Thor::Shell::Color::RED)
        FileUtils.rm_rf tar_path
        exit(1)
      end
      files = Cnvrg::Files.new(owner, "")
      resp = Images.create_new_custom_image_with_docker(instance_type, owner, image_name, public, base, image_extend, python3, tar_path, files)
      if resp
      end
    else
      log_message("Creating machine for your custom image, this may take a few moments...", Thor::Shell::Color::BLUE)
      resp = Images.create_new_custom_image(instance_type, owner, image_name, public, base, image_extend, python3, nil)

    end

    if Cnvrg::CLI.is_response_success(resp, false)
      image_slug = resp["result"]["slug"]
      container = resp["result"]["machine_c"]
      log_message("#{checks} Created image and machine successfully", Thor::Shell::Color::GREEN)
      log_message("Connecting to machine", Thor::Shell::Color::BLUE)
      ssh = Ssh.new(resp)
      if !ssh.is_ssh
        log_message("Couldn't connect to machine,aborting", Thor::Shell::Color::RED)
        Images.revoke_custom_new_image(owner, image_slug)
      end
      log_message("run command until ctrl + c or quit is initiated", Thor::Shell::Color::BLUE)
      begin
        logs = []

        while true
          command = ask("$>")
          logs << {time: Time.now,
                   message: command,
                   type: "stdout"
          }
          if command.eql? "quit"
            log_message("Commiting Image..", Thor::Shell::Color::BLUE)
            break
          end
          res = ssh.exec_command(command)
          begin
            res_parsed = JSON.parse(res)
            res = res_parsed.join(",")
          end

          puts res
          logs << {time: Time.now,
                   message: res,
                   type: "stdout"
          }
          logs.flatten!

        end

      rescue SignalException
        log_message("Commiting Image..", Thor::Shell::Color::BLUE)

      end
      resp = Images.commit_custom_image(owner, image_slug, logs)
      if Cnvrg::CLI.is_response_success(resp, false)
        log_message("#{checks} Image commited successfuly, email will be sent when image is ready", Thor::Shell::Color::GREEN)
      else
        if image_slug
          Images.revoke_custom_new_image(owner, image_slug)
        end
        if ssh
          ssh.close_ssh()
        end
        log_message("Image couldn't be commited, rolling back changes", Thor::Shell::Color::RED)

        exit(1)
      end
      if ssh
        ssh.close_ssh()
      end


    end
  rescue => e
    log_message("Error occurd, aborting", Thor::Shell::Color::RED)

    log_error(e)
    if image_slug
      Images.revoke_custom_new_image(owner, image_slug)
    end
    if ssh
      ssh.close_ssh()
    end


  rescue SignalException
    if image_slug
      Images.revoke_custom_new_image(owner, image_slug)
    end
    if ssh
      ssh.close_ssh
    end
    say "\nAborting"
    exit(1)
  end

end

#check_spot ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 480

def check_spot
  log_start(__method__, args, options)
  project_home = get_project_home
  @project = Project.new(project_home)

  log_message('Checking Spot Instance Status', Thor::Shell::Color::YELLOW)
  will_terminate = @project.spot_will_terminate
  if not will_terminate
    log_message("Not spot termination detected", Thor::Shell::Color::YELLOW)
    return
  end
  job_type, job_id = ENV['CNVRG_JOB_TYPE'], ENV['CNVRG_JOB_ID']
  machine_activity = @project.get_machine_activity

  notify_thread = Thread.new do
    res = @project.send_restart_request(job_type: job_type, job_id: job_id, ma_id: machine_activity)
    while res.blank?
      res = @project.send_restart_request(job_type: job_type, job_id: job_id, ma_id: machine_activity)
      sleep(10)
    end
  end

  sync_force = job_type == "NotebookSession" ? false : true
  upload(false, false, true, '', true, sync_force , ENV['CNVRG_OUTPUT_DIR'], job_type, job_id)
  log_message('Spot instance is going to be terminated', Thor::Shell::Color::YELLOW)
  notify_thread.join
end

#clone(project_url) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1788

def clone(project_url)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    url_parts = project_url.split("/")
    project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
    slug = url_parts[project_index + 1]
    owner = url_parts[project_index - 1]
    remote = options["remote"] || false

    response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
    Cnvrg::CLI.is_response_success(response)
    response = JSON.parse response["result"]
    project_name = response["title"]
    git = response["git"] || false

    commit_to_clone = options["commit"] || nil

    log_message("Cloning #{project_name}", Thor::Shell::Color::BLUE)
    clone_resp = false
    project_home = Dir.pwd

    if remote and !git
      clone_resp = Project.clone_dir_remote(slug, owner, project_name,git)
    elsif git
      if remote
        clone_resp = Project.clone_dir_remote(slug, owner, project_name,git)
      else
        project_home += "/#{project_name}"
        clone_resp = Project.clone_dir(slug, owner, project_name,git)

      end
    else
      if (Dir.exists? project_name)
        # project_name = "#{project_name}_#{rand(1 .. 5000000000)}"
        # puts project_name
        log_message("Error: Conflict with dir #{project_name}", Thor::Shell::Color::RED)
        if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
          log_message("Remove dir in order to clone #{project_name}", Thor::Shell::Color::RED)
          exit(1)
        end

      end
      clone_resp = Project.clone_dir(slug, owner, project_name,git)
      project_home = Dir.pwd + "/" + project_name


    end

    if clone_resp
      @project = Project.new(project_home)
      @files = Cnvrg::Files.new(@project.owner, slug, project_home: project_home, project: @project)
      response = @project.clone(remote, commit_to_clone)
      Cnvrg::CLI.is_response_success response
      commit_sha1 = response["result"]["commit"]
      files = response["result"]["tree"].keys
      idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
      log_message("Downloading files", Thor::Shell::Color::BLUE)
      progressbar = @files.create_progressbar(files.size, "Clone Progress")
      @files.download_files(files, commit_sha1, progress: progressbar)
      progressbar.finish
      Project.verify_cnvrgignore_exist(project_name, remote)
      @project.set_idx(idx)
      log_message("Done")
      log_message("Downloaded #{files.size} files")
    end
  end
end

#clone_data(dataset_url, only_tree = false, commit = nil, query = nil, read = false, remote = false) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 861

def clone_data(dataset_url,only_tree=false,commit=nil,query=nil,read=false,remote=false)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    commit = options["commit"] || commit
    only_tree = options["only_tree"] || only_tree
    read = options["read"] || read || false
    remote = options["remote"] || remote || false
    query = options['query'].presence || query.presence
    if query.present?
      return clone_data_query(dataset_url, query)
    end

    url_parts = dataset_url.split("/")
    project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
    slug = url_parts[project_index + 1]
    owner = url_parts[project_index - 1]
    @dataset = Dataset.new(dataset_url: dataset_url)
    response = {}
    response["result"] = @dataset.get_dataset(commit: commit, query: query)
    dataset_name = response["result"]["name"]
    dataset_home = Dir.pwd+"/"+dataset_name

    check = Helpers.checkmark
    if @dataset.init_home(remote:remote)
      log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
      @files = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
      log_message("Downloading files", Thor::Shell::Color::BLUE)
      if @dataset.softlinked?
        @files.cp_ds
        Cnvrg::CLI.log_message("Clone finished successfully", Thor::Shell::Color::GREEN)
        @dataset.write_success
        return
      end

      if only_tree

        success = Dataset.clone_tree(commit: commit, dataset_home: dataset_home)
        return if success
      end
      commit = response["result"]["commit"]
      files_count = response["result"]["file_count"]
      files = @files.get_clone_chunk(commit: commit)
      downloaded_files = 0
      progressbar = ProgressBar.create(:title => "Download Progress",
                                       :progress_mark => '=',
                                       :format => "%b>>%i| %p%% %t",
                                       :starting_at => 0,
                                       :total => files_count,
                                       :autofinish => true)

      
      while files['keys'].length > 0
        Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
        @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read)

        downloaded_files += files['keys'].length
        files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
      end
      progressbar.finish
      if downloaded_files == files_count
        Dataset.verify_cnvrgignore_exist(dataset_name, false)
        if !read
          @dataset.write_idx(nil, commit) #nil means, generate idx
        end

        log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
        @dataset.write_success
      end
    else
      log_message("Error: Couldn't create directory: #{dataset_name}", Thor::Shell::Color::RED)
      exit(1)
    end
  rescue Interrupt
    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  rescue SignalException
    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  end
end

#clone_data_query(dataset_url, query = nil) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 945

def clone_data_query(dataset_url,query=nil)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    query = options["query"] || query
    if !query.present?
      log_message("Argument missing : query", Thor::Shell::Color::RED)
      exit(1)
    end

    url_parts = dataset_url.split("/")
    project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
    slug = url_parts[project_index + 1]
    owner = url_parts[project_index - 1]

    response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/search/#{query}", 'GET')
    Cnvrg::CLI.is_response_success(response,true)
    dataset_name = response["results"]["name"]
    dataset_slug = response["results"]["slug"]
    # dataset_home = Dir.pwd+"/"+dataset_name
    dataset_home = Dir.pwd

    if Dataset.blank_clone(owner, dataset_name, dataset_slug)
      dataset = Dataset.new(dataset_home)
      log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
      parallel_options = {
          :progress => {
              :title => "Download Progress",
              :progress_mark => '=',
              :format => "%b>>%i| %p%% %t",
              :starting_at => 0,
              :total => response["results"]["query_files"].size,
              :autofinish => true
          },
          in_threads: ParallelThreads
      }
      begin
        log_message("Downloading files", Thor::Shell::Color::BLUE)
        Parallel.map((response["results"]["query_files"]), parallel_options) do |f|
          relative_path = f["fullpath"].gsub(/^#{dataset_home}/, "")
          relative_path_dir = relative_path.split("/")
          file_name = relative_path_dir.pop()
          relative_path_dir = relative_path_dir.join("/")
          abs_path = dataset_home + "/" + relative_path_dir
          begin
            FileUtils.mkdir_p(abs_path) unless File.exist? (abs_path + "/" + file_name)
          rescue
            log_message("Could not create directory: #{abs_path}", Thor::Shell::Color::RED)
            exit(1)
          end
          begin
            File.write "#{abs_path}/#{file_name}", open(f["s3_url"]).read unless File.exist? (abs_path + "/" + file_name)
          rescue
            log_message("Could not download file: #{f["fullpath"]}", Thor::Shell::Color::RED)
            exit(1)
          end

        end
      rescue Interrupt
        log_message("Couldn't download", Thor::Shell::Color::RED)
        exit(1)
      end
      begin
        dataset.generate_idx()
        check = Helpers.checkmark
        log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
        dataset.write_success(in_folder=true)
      rescue
          exit(1)
      end
    end
  rescue SignalException
    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  end
end

#commit_image ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4375

def commit_image
  verify_logged_in(true)
  log_start(__method__, args, options)

  begin
    image = is_project_with_docker(Dir.pwd)
    if image and image.is_docker
      container = image.get_container
      if !container

        say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
        exit(1)
      end
    else
      say "Project is not configured with any image", Thor::Shell::Color::RED
      exit(1)

    end
    project_home = get_project_home
    @project = Project.new(project_home)
    last_local_commit = @project.last_local_commit
    say "Commiting container into image", Thor::Shell::Color::BLUE
    new_image_name = "#{@project.slug}#{last_local_commit}:latest"
    image.update_image(new_image_name, container)
    new_image = container.commit('repo' => "#{@project.slug}#{last_local_commit}", 'tag' => "lastest")
    checks = Helpers.checkmark()
    say "#{checks} Done, image was updated", Thor::Shell::Color::GREEN
    log_end(0)
    return new_image.id
  rescue => e
    log_end(-1, e.message)
    say "\nError occurred, aborting"
    exit(1)
  rescue SignalException
    log_end(-1)
    say "\nAborting"
    exit(1)
  end
end

#config_flask_remote(image_name, port = 80) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4746

def config_flask_remote(image_name, port = 80)
  local_images = Docker::Image.all

  docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.eql? "#{image_name}:latest"}.flatten
  if docker_image_local.empty?
    say "no image"
    exit(1)
  end

  begin
    login_content = options["login"]
    image_settings = {
        'Image' => "#{image_name}:latest",
        'User' => 'ds',
        'Cmd' => '/usr/local/cnvrg/start_super.sh',
        'WorkingDir' => '/home/ds/app',
        'ExposedPorts' => {
            '80/tcp' => {},
        },
        'HostConfig' => {
            'PortBindings' => {
                '80/tcp' => [
                    {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
                ],
            },
        },
    }
    container = Docker::Container.create(image_settings)
    container.start()
    command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
    container.exec(command, tty: true)
    command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
    container.exec(command, tty: true)
    command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
    container.exec(command, tty: true)
    say "#{container.id}:#{port}"
  rescue => e
    pus e
    if e.message.include? "is not running"
      return "port is taken"
    end
    puts "error"
    if container
      container.kill()
    end
    return false
  end
end

#config_flask_remote_gpu(image_name, port = 80) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4798

def config_flask_remote_gpu(image_name, port = 80)
  local_images = Docker::Image.all

  docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.eql? "#{image_name}:latest"}.flatten
  if docker_image_local.empty?
    say "no image"
    exit(1)
  end

  begin
    login_content = options["login"]
    container_id = `nvidia-docker run -itd -p 80:80 -w /home/ds/app #{image_name}:latest /usr/local/cnvrg/start_super.sh`
    container_id = container_id.gsub("\n", "")
    container = Docker::Container.get(container_id)
    command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
    container.exec(command, tty: true)
    command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
    container.exec(command, tty: true)
    command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
    container.exec(command, tty: true)
    say "#{container.id}:#{port}"
  rescue => e
    puts e
    if e.message.include? "is not running"
      return "port is taken"
    end
    puts "error"
    if container
      container.kill()
    end
    return false
  end
end

#config_netrc(container) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4664

def config_netrc(container)

  login_content = options["login"]

  container = Docker::Container.get(container)
  command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
  container.exec(command, tty: true)
  command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
  container.exec(command, tty: true)
  command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
  container.exec(command, tty: true)
  say "OK"

end

#config_remote(image_name, port = 7654, tensport = 6006) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4599

def config_remote(image_name, port = 7654, tensport = 6006)
  local_images = Docker::Image.all

  docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.eql? "#{image_name}:latest"}.flatten
  if docker_image_local.empty?
    say "no image"
    exit(1)
  end

  begin
    login_content = options["login"]
    app_dir = options["app_dir"]
    cmd = options["cmd"]
    volume_from = options["volume"]

    image_settings = {
        'Image' => "#{image_name}:latest",

        'Cmd' => cmd,
        'WorkingDir' => app_dir,
        'ExposedPorts' => {
            '8888/tcp' => {},
        },
        'HostConfig' => {
            'Binds' => ["/var/run/docker.sock:/var/run/docker.sock", "/usr/bin/docker:/usr/bin/docker"],
            'PortBindings' => {
                '8888/tcp' => [
                    {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
                ],
                '6006/tcp' => [
                    {'HostPort' => "#{tensport}", 'HostIp' => 'localhost'}
                ],
            },
        },
    }
    container = Docker::Container.create(image_settings)
    container.start()
    command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
    container.exec(command, tty: true)
    # command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
    # container.exec(command, tty: true)
    # command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
    # container.exec(command, tty: true)
    command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
    container.exec(command, tty: true)
    command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
    container.exec(command, tty: true)
    say "#{container.id}:#{port}##{tensport}"
  rescue => e
    puts e
    if e.message.include? "is not running"
      return config_remote(image_name, port - 1, tensport - 1)
    end

    if container
      container.kill()
    end
    return false
  end
end

#config_remote_gpu(image_name, port = 7654, tensport = 6006) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4685

def config_remote_gpu(image_name, port = 7654, tensport = 6006)
  local_images = Docker::Image.all

  docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.eql? "#{image_name}:latest"}.flatten
  if docker_image_local.empty?
    say "no image"
    exit(1)
  end

  begin
    login_content = options["login"]
    app_dir = options["app_dir"]
    cmd = options["cmd"]

    # image_settings = {
    #     'Image' => "#{image_name}:latest",
    #     'User' => 'ds',
    #     'Cmd' => cmd,
    #     'WorkingDir' => app_dir,
    #     'ExposedPorts' => {
    #         '8888/tcp' => {},
    #     },
    #     'HostConfig' => {
    #         'PortBindings' => {
    #             '8888/tcp' => [
    #                 {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
    #             ],
    #             '6006/tcp' => [
    #                 {'HostPort' => "6006", 'HostIp' => 'localhost'}
    #             ],
    #         },
    #     },
    # }

    container_id = `nvidia-docker run -itd -p #{port}:8888 -p #{tensport}:6006 -w #{app_dir} -v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi  -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker #{image_name}:latest #{cmd} `
    container_id = container_id.gsub("\n", "")
    container = Docker::Container.get(container_id)
    # container.start()
    command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
    container.exec(command, tty: true)
    command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
    container.exec(command, tty: true)
    command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
    container.exec(command, tty: true)
    say "#{container.id}:#{port}##{tensport}"
  rescue => e
    if e.message.include? "is not running"
      puts "running asgain with: #{port - 1} #{tensport - 1}"
      return config_remote_gpu(image_name, port - 1, tensport - 1)
    end

    if container
      container.kill()
    end
    return false
  end
end

#create_volume ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1631

def create_volume
  verify_logged_in(false)
  log_start(__method__, args, options)
  dataset_dir = is_cnvrg_dir(Dir.pwd)
  @dataset = Dataset.new(dataset_dir)
  @dataset.create_volume()

end

#data_init_container(owner, dataset_slug, dataset_name) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1072

def data_init_container(owner, dataset_slug, dataset_name)

  if Dataset.init_container(owner, dataset_slug, dataset_name)

    say "init finished successfully", Thor::Shell::Color::GREEN

  else
    say "error creating dataset, please contact support.", Thor::Shell::Color::RED
    exit(0)
  end
end

#data_jump(*commit_sha1) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2725

def data_jump(*commit_sha1)
  begin
    verify_logged_in()
    log_start(__method__, args, options)
    dataset_dir = is_cnvrg_dir(Dir.pwd)
    @dataset = Dataset.new(dataset_dir)

    @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
    if commit_sha1.nil? or commit_sha1.empty?
      commit_sha1 = @dataset.last_local_commit
    end
    response = @dataset.compare_commits(commit_sha1)
    successful_changes = []
    if !response["result"]["status"].nil?
      idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
      File.open(dataset_dir + "/.cnvrg/idx.yml", "w+") {|f| f.write idx.to_yaml}
      status = response["result"]["status"]
      (status["delete"]).each do |f|
        relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
        FileUtils.rm_rf(relative_path)
      end
      # current_tree = Dir.entries(".").reject { |file| file.start_with? '.' }
      (status["dirs"]).each do |f|
        relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
        # dir
        if @files.download_dir(dataset_dir, relative_path)
          # current_tree.delete(relative_path[0, relative_path.size-1])
          successful_changes << relative_path
        end
      end
      (status["download"]).each do |f|
        relative_path = f["name"].gsub(/^#{@dataset.local_path}/, "")
        # dir
        if @files.download_file_s3(f["name"], relative_path, dataset_dir, f["sha1"])
          successful_changes << relative_path
        end
      end


      log_message("Done. Jumped to #{commit_sha1} completed successfully", Thor::Shell::Color::GREEN)
    end
  rescue => e
    log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
    log_error(e)

  rescue SignalException
    exit(1)
  end
end

#data_put(dataset_url, files: [], dir: '', chunk_size: 1000) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1198

def data_put(dataset_url, files: [], dir: '', chunk_size: 1000)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)

    #find owner and slug in url
    url_parts = dataset_url.split("/")
    project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
    slug = url_parts[project_index + 1]
    owner = url_parts[project_index - 1]
    @dataset = Dataset.new(dataset_url: dataset_url)
    @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
    @files = @datafiles.verify_files_exists(files)

    if @files.blank?
      raise SignalException.new(1, "Cant find files to upload, exiting.")
    end
    log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
    number_of_chunks = (@files.size.to_f / chunk_size).ceil
    response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
    unless response #means we failed in the start commit.
      raise SignalException.new(1, "Cant put files into server, check the dataset slug")
    end
    @commit = response['result']['commit_sha1']
    #dir shouldnt have starting or ending slash.
    dir = dir[0..-2] if dir.end_with? '/'
    dir = dir[1..-1] if dir.start_with? '/'

    progressbar = ProgressBar.create(:title => "Upload Progress",
                                     :progress_mark => '=',
                                     :format => "%b>>%i| %p%% %t",
                                     :starting_at => 0,
                                     :total => @files.size,
                                     :autofinish => true)
    @files.each_slice(chunk_size).each do |list_files|
      temp_tree = @dataset.generate_chunked_idx(list_files, prefix: dir)
      #will throw a signal exception if something goes wrong.
      @datafiles.upload_multiple_files(@commit, temp_tree, force: true, progressbar: progressbar, prefix: dir)
    end
    res = @datafiles.put_commit(@commit)
    unless res.is_success?
      raise SignalException.new(1, res.msg)
    end
    log_message("Upload finished Successfully", Thor::Shell::Color::GREEN)
  rescue SignalException => e
    log_message(e.message, Thor::Shell::Color::RED)
    return false
  end
end

#delete_data(dataset_slug) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 811

def delete_data(dataset_slug)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
      owner = CLI.get_owner
    response = Dataset.delete(dataset_slug, owner)

    if Cnvrg::CLI.is_response_success(response)
      log_message("Successfully deleted dataset: #{dataset_slug}", Thor::Shell::Color::GREEN)
    else
      log_message("Error while tying to delete dataset: #{response["messages"]}", Thor::Shell::Color::RED)


    end

  rescue => e
    log_error(e)
  rescue SignalException

    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  end
end

#deploy(file_to_run, function) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 3441

def deploy(file_to_run, function)
  verify_logged_in(true)
  log_start(__method__, args, options)
  working_dir = is_cnvrg_dir
  begin
    instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
                 "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
    instance_type = get_instance_type(instances)

      schedule = options["schedule"] || ""
      title = options['title']

      if !schedule.nil? and !schedule.empty?
        local_timestamp = get_schedule_date
      end
      project = Project.new(working_dir)
      commit_to_run = options["commit"] || nil

    workers = options["workers"] || nil
    begin
      num_workers = workers.to_i
    rescue
      log_message("Number of workers should be a number between 1 to 10", Thor::Shell::Color::RED)
      exit(1)
    end
    file_as_input = options["file_as_input"] || false


    image = is_project_with_docker(working_dir)
    image_slug = 'cnvrg'


    invoke :sync, [false], []

      res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp, num_workers, file_as_input, title)

    if Cnvrg::CLI.is_response_success(res)

        check = Helpers.checkmark()
        log_message("#{check} Deployment process is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/endpoints/show/#{res["result"]["deploy_slug"]}", Thor::Shell::Color::GREEN)

      exit(0)
      # end
    end
  rescue => e
    log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
    log_error(e)


  rescue SignalException
    exit_status = -1
    end_commit = project.last_local_commit
    sleep(20) # end cycle

    res = @exp.end(log, exit_status, end_commit, "", "")
    say "\nAborting"

    exit(1)
  end
end

#download(sync = false, ignore_list = "", in_exp = false) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2528

def download(sync = false, ignore_list = "", in_exp=false)
  begin
    verify_logged_in(true)
    log_start(__method__, args, options)
    project_home = get_project_home
    @project = Project.new(project_home)
    @files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home, project: @project)
    git = options["git"]
    commit = options["commit"]
    if git or @project.is_git
      return download_in_git(commit)
    end
    if commit.present?
      return jump(commit)
    end
    ignore = options[:ignore] || ""
    if ignore.nil? or ignore.empty?
      ignore = ignore_list
    end
    data_ignore = data_dir_include()
    if !data_ignore.nil?
      if ignore.nil? or ignore.empty?
        ignore = data_ignore
      else
        ignore = "#{ignore},#{data_ignore}"
      end
    end
    if !@project.update_ignore_list(ignore)
      log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
    end
    new_branch = options["new_branch"] || @project.is_branch || false
    res = @project.compare_idx(new_branch, in_exp: in_exp, download: true)["result"]
    result = res["tree"]

    commit = res["commit"]

    #here im grouping the files by their current status.
    #
    # all the files that changed in the server (added + updated)
    changed_files = result['updated_on_server'] + result['added']

    # non-conflicted files, all the files that changed remotely but not locally
    updated_files = changed_files - result["update_local"]
    # conflicted - files that changed remotely and locally
    conflicted_files = changed_files & result["update_local"]

    # all deleted files
    all_deleted = result['deleted']
    # cant delete files - files that deleted on the server but changed locally
    conflicted_deleted = all_deleted & result["update_local"]
    # files to delete - files that deleted on the server and unchanged locally
    deleted_files = all_deleted - conflicted_deleted


    update_total = [all_deleted, changed_files ].flatten.size

    if update_total < 1
      if !@project.last_local_commit.eql? commit
        Cnvrg::Logger.log_info("Finish commit, updating idx with commit")
        @project.update_idx_with_commit!(commit)
      end
      log_message("Project is up to date", Thor::Shell::Color::GREEN, ((options["sync"] or sync) ? false : true))
      return true
    end
    Cnvrg::Logger.log_info("Got #{update_total} changes from server")

    successful_changes = []
    if update_total == 1
      log_message("Downloading #{update_total} file", Thor::Shell::Color::BLUE, !options["sync"])
    elsif options["verbose"]
      log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
    else
      log_message("Syncing files", Thor::Shell::Color::BLUE, !options["sync"])
    end

    progressbar = ProgressBar.create(:title => "Download Progress",
                                     :progress_mark => '=',
                                     :format => "%b>>%i| %p%% %t",
                                     :starting_at => 0,
                                     :total => update_total,
                                     :autofinish => true)


    Cnvrg::Logger.log_info("Downloading updated files:#{updated_files.join(",")}")
    @files.download_files(updated_files, commit, progress: progressbar)

    Cnvrg::Logger.log_info("Downloading conflicted files:#{conflicted_files.join(",")}")
    @files.download_files(conflicted_files, commit, postfix: ".conflict", progress: progressbar)

    Cnvrg::Logger.log_info("Delete files: #{deleted_files.join(",")}")
    @files.delete_files_local(deleted_files, conflicted: conflicted_deleted, progress: progressbar)

    # update idx with latest commit
    # the latest true its because if we define --commit in the cmd it will go to "def jump(options['commit'])"
    # so if we are downloads something, we have to stay here.
    @project.update_idx_with_commit!(commit, latest: true)
    #TODO Sync, remove idx, sync again and pray
    progressbar.finish
    check = Helpers.checkmark()
    Cnvrg::Logger.log_info("Finished downloading successfuly")
    if options["verbose"]
      log_message("#{check} Done, Downloaded:", Thor::Shell::Color::GREEN)
      log_message(successful_changes.join("\n"), Thor::Shell::Color::GREEN)
      log_message("Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN)
    else
      log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
    end
  rescue => e
    log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
    Cnvrg::Logger.log_error(e)
    exit(1)
  rescue Exception => e
    Cnvrg::Logger.log_error(e)
    log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
    exit(1)
  rescue SignalException
    log_message("\nAborting", Thor::Shell::Color::BLUE)
    exit(1)
  end
end

#download_built_image(image_name, image_slug) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4894

def download_built_image(image_name, image_slug)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    owner = Cnvrg::CLI.get_owner()
    path = File.expand_path('~') + "/.cnvrg/tmp/#{image_name}.tar.gz"
    @files = Cnvrg::Files.new(owner, "")

    log_message("Downloading image file", Thor::Shell::Color::BLUE)
    begin
      if @files.download_image(path, image_slug, owner)
        gzipRes = system("gunzip -f #{path}")
        if !gzipRes

          log_message("Couldn't create tar file from image", Thor::Shell::Color::RED)
          exit(1)
        else
          path = path.gsub(".gz", "")
          return path
        end

      else
        log_message("Couldn't download image #{image_name}", Thor::Shell::Color::RED)
        return false
      end
    rescue Interrupt
      say "The user has exited to process, aborting", Thor::Shell::Color::BLUE
      exit(1)
    end
  rescue SignalException
    say "\nAborting"
    exit(1)
  end
end

#download_cnvrg_image(image_name, secret) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4983

def download_cnvrg_image(image_name, secret)
  verify_logged_in(false)

  begin
    @files = Cnvrg::Files.new("", "")

    say "Downloading image file", Thor::Shell::Color::BLUE
    begin
      if @files.download_cnvrg_image(image_name, secret)

        say "Successfully downloaded image #{image_name}", Thor::Shell::Color::GREEN

      else
        say "Couldn't download image #{image_name}", Thor::Shell::Color::RED
        return false
      end
    rescue Interrupt
      say "The user has exited to process, aborting", Thor::Shell::Color::BLUE
      exit(1)
    end
  rescue SignalException
    say "\nAborting"
    exit(1)
  end
end

#download_data(verbose, sync, path = Dir.pwd, in_dir = true) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1088

def download_data(verbose, sync, path = Dir.pwd, in_dir = true)
  begin
    verify_logged_in(in_dir)
    log_start(__method__, args, options)
    if path.nil? or path.empty?
      path = Dir.pwd
    end
    dataset_dir = is_cnvrg_dir(path)
    @dataset = Dataset.new(dataset_dir)

    @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
    new_branch = options["new_branch"] || false

    res = @dataset.compare_idx(new_branch)["result"]

    result = res["tree"]

    commit = res["commit"]
    if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
      log_message("Project is up to date", Thor::Shell::Color::GREEN, ((options["sync"] or sync)) ? false : true)
      return true
    end
    log_message("Downloading data", Thor::Shell::Color::BLUE)

    result = @dataset.downlowd_updated_data(@dataset.last_local_commit)

    delete = result["result"]["delete"]
    commits = result["result"]["commits"]
    updated_idx = result["result"]["idx"]
    parallel_options = {
        :progress => {
            :title => "Download Progress",
            :progress_mark => '=',
            :format => "%b>>%i| %p%% %t",
            :starting_at => 0,
            :total => commits.size,
            :autofinish => true
        },
        in_processes: ParallelProcesses,
        in_thread: ParallelThreads
    }

    begin
      tar_files = []
      download_result = Parallel.map(commits, parallel_options) do |c|

        file_name = @files.download_data_file(c, dataset_dir)

        if file_name.eql? false or file_name.nil?
          count = 0
          success_download = false
          while count < 3 and !success_download
            log_message("Couldn't download data files, retrying.. ", Thor::Shell::Color::BLUE)

            file_name = @files.download_data_file(c, dataset_dir)
            success_download = (file_name.eql? false or file_name.nil?)
            count += 1

          end
          if count > 3 or !success_download
            log_message("Couldn't download data files,revoking", Thor::Shell::Color::RED)

            raise Parallel::Kill
          end
        end
        file_path = "#{dataset_dir}/#{file_name}"
        tar_files << file_path
        success = extarct_tar(file_path, dataset_dir)
        if !success
          log_message("Couldn't extract data files,revoking", Thor::Shell::Color::RED)

          raise Parallel::Kill
        end

        FileUtils.rm_rf([file_path])

      end
    rescue Interrupt
      @files.revoke_download(tar_files, updated_idx[:tree].keys)
      return false
    end


    to_delete = []
    delete.each do |d|
      to_delete << "#{dataset_dir}/#{d}"
    end
    FileUtils.rm_rf(to_delete)

    @dataset.update_idx(updated_idx)


    check = Helpers.checkmark()
    log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN)
    return true


  end
rescue => e
  log_message("Error occurd, \nAborting", Thor::Shell::Color::BLUE)
  log_error(e)
  @files.revoke_download(tar_files, updated_idx[:tree].keys)

  exit(1)
rescue SignalException
  say "\nAborting", Thor::Shell::Color::BLUE
  exit(1)
end

#download_data_new(verbose = false, new_branch = false, sync = false, commit = nil, all_files = true) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2426

def download_data_new(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
  begin
    verify_logged_in(true)
    log_start(__method__, args, options)
    dataset_dir = is_cnvrg_dir(Dir.pwd)
    @dataset = Dataset.new(dataset_dir)
    @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
    all_files = all_files
    res = @dataset.compare_idx_download(all_files: all_files, desired_commit: commit)
    unless CLI.is_response_success(res, false)
      log_message("Cant find the desired commit, please check it or try to download without it.", Thor::Shell::Color::RED)
      exit(1)
    end
    result = res["result"]["tree"]
    commit = res["result"]["commit"]
    can_commit = res["result"]["can_commit"] || false #can commit means that our next commit is newer than the latest commit
    # so if can_commit is true it means that we have to be up-to-date with the dataset.
    update_total = [result['added'], result["updated_on_server"], result["conflicts"], result["deleted"]].compact.flatten.size
    successful_changes  = 0
    if update_total == 0 or can_commit
      log_message("Dataset is up to date", Thor::Shell::Color::GREEN) if !sync
      return true
    elsif options["verbose"]
      log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
    else
      log_message("Syncing Dataset", Thor::Shell::Color::BLUE, !sync)
    end
    Cnvrg::Logger.log_info("Current commit: #{@dataset.get_current_commit}, destination commit: #{commit}")
    Cnvrg::Logger.log_info("Compare idx res: #{result}")
    progressbar = ProgressBar.create(:title => "Download Progress",
                                       :progress_mark => '=',
                                       :format => "%b>>%i| %p%% %t",
                                       :starting_at => 0,
                                       :total => update_total,
                                       :autofinish => true)
    conflicts = @files.mark_conflicts(result)
    
    log_message("Found some conflicts, check .conflict files.", Thor::Shell::Color::BLUE) if conflicts > 0
    update_res = @files.download_files_in_chunks(result["updated_on_server"], progress: progressbar) if result["updated_on_server"].present?
    added_res = @files.download_files_in_chunks(result["added"], progress: progressbar) if result["added"].present?
    # conflict_res = @files.download_files_in_chunks(result["conflicts"], conflict: true) if result["conflicts"].present?
    deleted = result["deleted"].to_a
    delete_res = @files.delete_commit_files_local(deleted)
    progressbar.progress += deleted.size if progressbar.present?
    if !delete_res
      log_message("Couldn't delete #{deleted.join(" ")}", Thor::Shell::Color::RED)
      log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
      exit(1)
    end
    success = (update_res.blank? or update_res.is_success?)
    success &= (delete_res.blank? or delete_res.is_success?)
    success &= (added_res.blank? or added_res.is_success?)
    if success
      # update idx with latest commit
      @dataset.update_idx_with_commit!(commit)
      check = Helpers.checkmark()
      if options["verbose"]
        log_message("#{check} Done, Downloaded:", Thor::Shell::Color::GREEN)
        log_message(successful_changes.join("\n"), Thor::Shell::Color::GREEN)
        log_message("Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN)
      else
        log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
      end
      return true          
    end
  rescue SignalException => e
    Cnvrg::Logger.log_error(e)
    say "\nAborting", Thor::Shell::Color::BLUE
    exit(1)
  rescue => e
    log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
    Cnvrg::Logger.log_error(e)
    exit(1)
  end
end

#download_file_data(file_path, *dataset_path) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2365

def download_file_data(file_path, *dataset_path)

  verify_logged_in(true)
  log_start(__method__, args, options)
  begin
    if dataset_path.nil? or dataset_path.empty?
      dataset_path = Dir.pwd
    elsif dataset_path.is_a? Array
      dataset_path = dataset_path[0]
    end
    dataset_dir = is_cnvrg_dir(dataset_path)
    @dataset = Dataset.new(dataset_dir)
    @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
    commit_to_download = options["commit"] || nil
    as_link  = options["link"] || false
    download_path = options["path"]
    remote = options["remote"]
    as_json = options["json"]
    if remote
      download_path = "/data/#{@dataset.title}/"

    else
      download_path = dataset_dir
    end
    if !as_json

      log_message("Downloading file", Thor::Shell::Color::BLUE)

    end
    res = @files.download_file_s3(file_path, file_path, download_path, conflict=false, commit_sha1=commit_to_download, as_link=as_link)
    if as_link
      puts res
    else
      if res
        if as_json
          response = {"status":"success"}
          puts response.to_json
        else
          log_message("#{Helpers.checkmark()} File #{file_path} was successfully downloaded", Thor::Shell::Color::GREEN)

        end
      end


    end

  rescue =>e
    log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
    log_error(e)
    exit(1)
  end
end

#download_in_git(*commit_sha1) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2503

def download_in_git(*commit_sha1)
  begin
    verify_logged_in(true)
    log_start(__method__, args, options)
    project_home = get_project_home
    @project = Project.new(project_home)
    @files = Cnvrg::Files.new(@project.owner, @project.slug, project: @project, cli: self, options: options)
    commit_sha1 = commit_sha1.try(:first) if commit_sha1.is_a? Array
    @files.download_commit(commit_sha1)
  rescue => e
      log_error(e)
      log_message("Error while trying to download ", Thor::Shell::Color::RED)
      return
  end
end

#download_tags_yaml ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1675

def download_tags_yaml
  verify_logged_in(false)
  log_start(__method__, args, options)
  dataset_dir = is_cnvrg_dir(Dir.pwd)
  @dataset = Dataset.new(dataset_dir)
  status = @dataset.download_tags_yaml()
  if status
    log_message("Downloaded tags yaml successfully", Thor::Shell::Color::GREEN)
  else
    log_message("Unable to download", Thor::Shell::Color::RED)
  end
end

#end_commit_data(commit, success: true, uploaded_files: 0) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2053

def end_commit_data(commit, success: true, uploaded_files: 0)
  begin
    verify_logged_in(true)
    log_start(__method__, args, options)
    dataset_dir = is_cnvrg_dir(Dir.pwd)
    @dataset = Dataset.new(dataset_dir)
    @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
    force = options["force"] || false
    resp = @files.end_commit(commit, force, success: success, uploaded_files: uploaded_files)
    if (resp.present? and resp["result"])
      check = Helpers.checkmark
      if resp["result"]["new_commit"].blank?
        @dataset.revert_next_commit #removes the next commit
        log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN)
      else
        log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
        @dataset.remove_next_commit #takes the next commit and put it as current commit
        @dataset.set_partial_commit(nil)
        @dataset.backup_idx
      end
    end
  end
end

#exec(*cmd) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2947

def exec(*cmd)
  log = []
  verify_logged_in(true)
  log_start(__method__, args, options)
  working_dir = is_cnvrg_dir
  script_path = get_cmd_path_in_dir(working_dir, Dir.pwd)

  sync_before = options["sync_before"]
  sync_after = options["sync_after"]
  print_log = options["log"]
  title = options["title"]
  commit = options["commit"] || nil
  image = options["image"] || nil
  indocker = options["indocker"] || false
  ignore = options[:ignore] || ""
  force = options[:force]
  sync_before_terminate = options["sync_before_terminate"]
  periodic_sync = options["periodic_sync"]
  email_notification = options["email_notification"]
  output_dir = options['output_dir'] || "output"
  project_home = get_project_home
  data_query = options["data_query"]
  @project = Project.new(project_home)
  if @project.is_git
    sync_before = false
  end
  # is_new_branch = @project.compare_commit(commit)
  is_new_branch = false
  begin
    if !commit.nil? and !commit.empty?
      invoke :jump, [commit], []
    else
      if sync_before
        # Sync before run
        invoke :sync, [false], :new_branch => is_new_branch, :ignore => ignore, :force => force

      end
    end
    #set image for the project
    if !image.nil? and !image.empty?
      invoke :set_image, [image]
    end
    if !indocker
      image_proj = is_project_with_docker(working_dir)
      if image_proj and image_proj.is_docker
        container = image_proj.get_container
        if !container
          log_message("Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED)
          exit(1)
        end


        exec_args = args.flatten.join(" ")
        options_hash = Hash[options]
        options_hash.except!("image", "indocker")
        exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
        command_to_run = copy_args.join(" ")
        command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
        puts container.exec(command, tty: true)
        container.stop()
        exit(0)
      end
    end
    remote = options["remote"]
    if remote
      docker_id = `cat /etc/hostname`
      docker_id = docker_id.strip()
    end
    is_on_gpu = options["gpu"]
    start_commit = @project.last_local_commit
    cmd = cmd.join("\s")

    @exp = Experiment.new(@project.owner, @project.slug)

    platform = RUBY_PLATFORM
    machine_name = Socket.gethostname
    begin
      machine_activity = @exp.get_machine_activity(working_dir)
      @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, script_path, sync_before_terminate, periodic_sync)
      log_message("Experiment's live results: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN)
      log_message("Running: #{cmd}\n", Thor::Shell::Color::BLUE)
      unless @exp.slug.nil?
        real = Time.now
        exp_success = true
        memory_total = []
        cpu_total = []
        start_loop = Time.now
        stdout, stderr = '', ''
        begin

          if remote
            if @exp.sync_before_terminate
              spot_status_thread = Thread.new do
                begin
                  loop do
                    log_message('Checking Spot Instance Status', Thor::Shell::Color::YELLOW)
                    restart = @exp.restart_spot_instance()

                    if restart
                      log_message('Spot instance is going to be terminated', Thor::Shell::Color::YELLOW)
                      # sync
                      if @project.is_git
                        output_dir = @exp.output_dir

                        if output_dir.blank?
                          output_dir = "output"
                        end
                        upload(false, false, true, ignore, true, true,output_dir,"Experiment",@exp.slug  )
                      else
                        upload(false, false, true, ignore, true, true,nil,"Experiment",@exp.slug  )
                      end
                        res = @exp.send_restart_request(@project.get_idx.try(:fetch, :commit))
                      while !Cnvrg::CLI.is_response_success(res, false) do
                        sleep(5)
                        res = @exp.send_restart_request(@project.get_idx.try(:fetch, :commit))
                      end
                      exit(0)
                    end
                    sleep(10)
                  end
                rescue => e
                  log_error(e)
                end
              end
            end
          end
          process_running = true
          stats_thread = Thread.new do
            while process_running do
              sleep 30
              begin
                stats = remote ? usage_metrics_in_docker(docker_id) : Helpers.ubuntu? ? {memory: memory_usage, cpu: cpu_usage} : {}
                if is_on_gpu
                  gu = gpu_util
                  stats['gpu_util'] = gu[0]
                  stats['gpu'] = gu[1]
                end
                @exp.send_machine_stats [stats] unless stats.empty?
              rescue => e
                log_error(e)
                log_message("Failed to upload ongoing stats, continuing with experiment", Thor::Shell::Color::YELLOW)
              end
            end
          end
          start_time = Time.now
          PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
            begin
              stdout.each do |line|
                cur_time = Time.now
                real_time = Time.now - real
                cur_log = {time: cur_time,
                           message: line,
                           type: "stdout",
                           real: real_time
                }
                if print_log
                  puts line
                end
                log << cur_log
                if log.size >= 5
                  @exp.upload_temp_log(log) unless log.empty?
                  log = []
                  elsif (start_time + 15.seconds) <= Time.now
                  @exp.upload_temp_log(log) unless log.empty?
                  log = []
                  start_time = Time.now
                end
              end
              if stderr
                stderr.each do |err|
                  log << {time: Time.now, message: err, type: "stderr"}
                end
              end
            rescue Errno::EIO => e
              log_error(e)
              if !log.empty?
                temp_log = log
                @exp.upload_temp_log(temp_log) unless temp_log.empty?
                log -= temp_log
              end
            rescue Errno::ENOENT => e
              exp_success = false
              log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
              log_error(e)
            rescue => e
              res = @exp.end(log, 1, start_commit, 0, 0)
              log_message("Error occurred,aborting", Thor::Shell::Color::RED)
              log_error(e)
              exit(0)
            end
            ::Process.wait pid
          end
          end_time = Time.now
          process_running = false

          if !log.empty?

            temp_log = log
              @exp.upload_temp_log(temp_log) unless temp_log.empty?
            log -= temp_log
          end

          cpu_average = cpu_total.inject(0) {|sum, el| sum + el}.to_f / cpu_total.size
          memory_average = memory_total.inject(0) {|sum, el| sum + el}.to_f / memory_total.size
          exit_status = $?.exitstatus
          if $?.exitstatus != 0
            exp_success = false
          end

            if sync_after
            # Sync after run
              if @project.is_git
                output_dir = output_dir || @exp.output_dir
                if output_dir.present?
                  upload(false, false, true, ignore, true, true,output_dir,"Experiment",@exp.slug  )
                  # invoke :upload, [false, false, true, ignore, true, true], :output_dir => output_dir, :force=>true, :job_type=>'Experiment', :job_slug=>@exp.slug
                end
              else
                upload(false, false, true, ignore, true, true,nil,"Experiment",@exp.slug  )

                # invoke :upload, [false, false, true, ignore,true, true], :job_type=>'Experiment', :job_slug=>@exp.slug, :force=>true
              end

            end
            end_commit = @project.last_local_commit

            # log_thread.join
            stats_thread.join

            res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average, end_time: end_time)


            if !exp_success

              log_message("Experiment has failed, look at the log for more details or run cnvrg exec --log", Thor::Shell::Color::RED)
            else
              check = Helpers.checkmark()
              log_message("#{check} Done. Experiment's results were updated!", Thor::Shell::Color::GREEN)
            end

        rescue => e
          if container
            container.stop()
          end
          log_message("Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED)
          if @exp
            # log_thread.join
            Thread.kill(stats_thread)
            exit_status = $?.exitstatus
            if exit_status.blank?
              exit_status = "-1"
            end
            res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)

          end
          log_error(e)
          # Thread.kill(log_thread)
          # Thread.kill(stats_thread)

          exit(1)
        end


      end

    end
  rescue SignalException
    exit_status = -1
    end_commit = @project.last_local_commit
    process_running = false
    # log_thread.join
    stats_thread.join

    res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
    if container
      container.stop()
    end
    say "\nAborting"

    exit(1)
  end
end

#exec_container(container_id, *cmd) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4561

def exec_container(container_id, *cmd)
  container = Docker::Container.get(container_id)
  container.start()
  cnvrg_command = cmd.join(" ")
  command = ["/bin/bash", "-lc", "#{cnvrg_command}"]
  res = container.exec(command, tty: true, wait: 5400)[0]
  say res
end

#exec_remote(*cmd) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 3257

def exec_remote(*cmd)

  verify_logged_in(true)
  log_start(__method__, args, options)
  working_dir = is_cnvrg_dir
  path_to_cmd = get_cmd_path_in_dir(working_dir, Dir.pwd)

  begin
    title = options["title"] || nil
    grid = options["grid"] || nil
    data = options["data"] || nil
    datasets = options["datasets"] || nil
    data_commit = options["data_commit"] || nil
    data_query = options["data_query"] || nil
    sync_before = options["sync_before"]
    force = options["force"]
    max_time = options["max_time"]
    if !max_time.nil? and !max_time.empty?
      max_time = max_time.to_i
      if max_time <=0
        log_message("Max time for experiment should be more than 0 minutess", Thor::Shell::Color::RED)
        exit(1)
      end
    end
    periodic_sync = options["periodic_sync"]
    sync_before_terminate = options["sync_before_terminate"]
    dataset_only_tree = options["dataset_only_tree"]
    ds_sync_options = 0
    if dataset_only_tree
      ds_sync_options = 1
    end
    restart_if_stuck = options["restart_if_stuck"]
    instance_type = options["machine_type"] || nil
    schedule = options["schedule"] || ""
    if schedule.start_with? 'in'
      time = schedule.split(" ")

      local = Time.now.localtime
      if time[2].downcase().start_with? "min"
        new = local + (time[1].to_i * 60)
      elsif time[2].downcase().start_with? "hours"
        new = local + (time[1].to_i * 3600)
      elsif time[2].downcase().start_with? "days"
        new = local + (time[1].to_i * 3600 * 24)
      else
        log_message("Could not undersatnd when to schedule experiment", Thor::Shell::Color::RED)
        exit(1)
      end
      new_time = new.to_s
      new_time = new_time[0, new_time.size - 6] #remove timezone
      schedule = "at #{new_time}"
    end
    upload_output = options["upload_output"]
    time_to_upload = calc_output_time(upload_output)
    if time_to_upload == 0 or time_to_upload == -1
      upload_output_option = "--upload_output=1m"
    else
      upload_output_option = "--upload_output=#{upload_output}"
    end
    remote = "--remote=true"
    if !instance_type.nil? and instance_type.include? "gpu"
      remote = "#{remote} --gpu=true"
    end

    output_dir = options["output_dir"] || nil
    git_commit = options["git_commit"]
    git_branch = options["git_branch"]
    options_hash = Hash[options]
    local_folders_options = options["local_folders"]
    options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
                         "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
                         "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets" )
    exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
    command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
    commit_to_run = options["commit"] || nil
    if !schedule.nil? and !schedule.empty?

      local_timestamp = get_schedule_date

    end
    project = Project.new(working_dir)

    if project.is_git and output_dir.blank?
      output_dir  = "output"
    end
    image = options["image"] || nil
    if image.blank?
      image  = "cnvrg"
    end

    forced_commit = nil
    if sync_before and !project.is_git
      if force
      sync_result = invoke :sync, [false], :force => force, :return_id=> true
      begin
        forced_commit = JSON(sync_result)["commit_sha1"]
      rescue
        forced_commit = nil
      end
      else
        sync_result = invoke :sync, [false], :force => false
      end
    end
    #handle grid if it's git project
    if project.is_git and grid.present?
      if !File.exist? "#{project.local_path}/#{grid}"
        log_message("Hyper Search File:#{grid} couldn't be found", Thor::Shell::Color::RED)
        return
      end
      grid_content  = YAML.load_file("#{project.local_path}/#{grid}")
      if grid_content.present?
        grid = grid_content
      else
        log_message("Hyper Search file:#{grid} has no content", Thor::Shell::Color::RED)
        return
      end
    end

    if command.include? "'"
      oc = command.to_enum(:scan, /'/).map {Regexp.last_match}
      pairs = oc.enum_for(:each_slice, 2).to_a
      pairs.each_with_index do |p, i|
        add = 0
        if i != 0
          add = 2 * i
        end
        total_loc = command[p[0].offset(0)[0] + add..p[1].offset(0)[0] + add]
        command[p[0].offset(0)[0] + add..p[1].offset(0)[0] + add] = "\"#{total_loc}\""
      end


    end
    log_message("Running remote experiment", Thor::Shell::Color::BLUE)
    exp = Experiment.new(project.owner, project.slug)
    if forced_commit and (commit_to_run.nil? or commit_to_run.empty?)
      commit_to_run = forced_commit
    end

    commit_to_run = commit_to_run.presence || project.last_local_commit

    res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
                          periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
                          data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets)
    if Cnvrg::CLI.is_response_success(res)
      check = Helpers.checkmark()
      str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"

      if res["result"]["grid"]
        str = "Running grid search, follow here: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments?grid=#{res["result"]["exp_url"]}"
      end

      log_message(str, Thor::Shell::Color::GREEN)

      exit(0)
      # end
    end

  rescue => e
    log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
    log_error(e)

  rescue SignalException
    exit_status = -1
    end_commit = project.last_local_commit
    sleep(20) # end cycle

    res = @exp.end(log, exit_status, end_commit, "", "")
    say "\nAborting"

    exit(1)
  end
end

#experiments ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 5065

def experiments
  verify_logged_in(true)
  log_start(__method__, args, options)

  project_dir = is_cnvrg_dir(Dir.pwd)
  @project = Project.new(project_dir)
  unless options['id'].to_s.size > 5
    result = @project.get_experiments()
    list = result["result"]["experiments"]
    if list and list.size > 1
      print_table(list)
    else
      say "No experiments"
    end
  else
    result = @project.get_experiment(options['id'])
    result = result.to_h['experiment']
    if result
      if options["tag"].to_s.size == 0
        list = []
        list << result.keys
        list << result.values
        print_table(list)
      else
        if result.keys.include? options["tag"]
          say result[options["tag"]]
        else
          say "No such tag"
        end
      end
    else
      say "No such experiment"
    end


  end

end

#get_machine ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 5106

def get_machine()
  begin
    verify_logged_in(true)
    log_start(__method__, args, options)
    owner = Cnvrg::CLI.get_owner()
    working_dir = is_cnvrg_dir
    @image = Images.new(working_dir)
    if @image.nil? or !@image.is_docker
      say "Couldn't find image related to this project", Thor::Shell::Color::RED
      exit(0)
    end
    res = Cnvrg::API.request("users/#{owner}/machines/list", 'GET')
    if Cnvrg::CLI.is_response_success(res)
      if res["result"]["machines"].empty?
        create = yes? "No machines available, create new machine?", Thor::Shell::Color::YELLOW
        if create
          instance_type = machine_options(res["result"]["aws_options"])
        else
          exit(0)
        end
      end
      printf "%-20s %-20s %-20s\n", "name", "created by", "last_used", "instance_type"
      res["result"]["machines"].each do |u|
        time = Time.parse(u["last_used"])
        update_at = get_local_time(time)
        printf "%-20s %-20s %-20s\n", u["name"], u["created_by"], update_at, u["instance type"]
      end
    end

  rescue SignalException
    log_end(-1)
    say "\nAborting"
    exit(1)
  end
end

#git_clone(slug, owner) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1742

def git_clone(slug, owner)
  verify_logged_in(false)
  log_start(__method__, args, options)

  clone_resp = Project.clone_dir_remote(slug, owner, slug,true)
  idx_status = Project.new(get_project_home).generate_idx
end

#init_data(public, bucket: nil) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 774

def init_data(public, bucket: nil)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    dataset_name = File.basename(Dir.getwd)
    if File.directory?(Dir.getwd + "/.cnvrg")
      config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
      log_message("Directory is already linked to #{config[:dataset_slug]}", Thor::Shell::Color::RED)

      exit(0)
    end
    log_message("Init dataset: #{dataset_name}", Thor::Shell::Color::BLUE)

    working_dir = Dir.getwd
    owner = CLI.get_owner
    if Dataset.init(owner, dataset_name, options["public"], bucket: bucket)
      path = Dir.pwd
      @dataset = Dataset.new(path)

      url = @dataset.url
      check = Helpers.checkmark
      log_message("#{check} Link finished successfully", Thor::Shell::Color::GREEN)
      log_message("#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN)

    else
      @dataset.revert(working_dir) unless @dataset.nil?
      log_message("Error creating dataset, please contact support.", Thor::Shell::Color::RED)
      exit(0)
    end
  rescue => e
    log_error(e)
  rescue SignalException
    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  end
end

#init_data_container(container) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1025

def init_data_container(container)
  begin
    login_content = options["login_content"]

    container = Docker::Container.get(container)
    command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
    container.exec(command, tty: true)
    command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
    container.exec(command, tty: true)
    command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
    container.exec(command, tty: true)
    command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
    container.exec(command, tty: true)
    command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
    container.exec(command, tty: true)

  rescue SignalException

    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  end
end

#install_python_libraries(*lib) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4100

def install_python_libraries(*lib)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    image = is_project_with_docker(Dir.pwd)
    if image and image.is_docker
      container = image.get_container
      if !container

        say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
        exit(1)
      end
    else
      say "Project is not configured with any image", Thor::Shell::Color::RED
      exit(1)

    end
    req_file = options["requirement"] || nil
    if lib.nil? and not req_file.nil?
      if not File.exist? req_file
        say "Couldn't find #{req_file}", Thor::Shell::Color::RED
        exit(1)

      end
      command_to_run = "pip install -r #{req_file}"

    else
      command_to_run = lib.join(" ")

    end
    say "Running #{command_to_run} in container", Thor::Shell::Color::BLUE
    command = ["/bin/bash", "-lc", "#{command_to_run}"]
    res = container.exec(command, tty: false)
    say res[0].join("\n")
    checks = Helpers.checkmark()
    say "Updating image", Thor::Shell::Color::BLUE

    image.create_custom_image("")
    say "#{checks} Done, installing libraries completed", Thor::Shell::Color::GREEN
    container.stop()

    log_end(0)
  rescue => e
    log_end(-1, e.message)
    say "Error occurred, aborting"
    if container
      container.stop()
    end
  rescue SignalException
    if container
      container.stop()
    end
    say "\nAborting"
    exit(1)
  end

end

#install_system_libraries(*command_to_run) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4049

def install_system_libraries(*command_to_run)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    image = is_project_with_docker(Dir.pwd)
    if image and image.is_docker
      container = image.get_container
      if !container

        say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
        exit(1)
      end
    else
      say "Project is not configured with any image", Thor::Shell::Color::RED
      exit(1)

    end

    command_to_run = command_to_run.join(" ")
    say "Running #{command_to_run} in container", Thor::Shell::Color::BLUE
    command = ["/bin/bash", "-lc", "#{command_to_run}"]
    res = container.exec(command, tty: false)
    say res[0].join("\n")
    checks = Helpers.checkmark()
    say "Updating image", Thor::Shell::Color::BLUE

    image.create_custom_image("")
    say "#{checks} Done, installing libraries completed", Thor::Shell::Color::GREEN
    container.stop()

    log_end(0)
  rescue => e
    log_end(-1, e.message)
    say "Error occurred, aborting"
    if container
      container.stop()
    end
  rescue SignalException
    log_End(-1)
    if container
      container.stop()
    end
    say "\nAborting"
    exit(1)
  end

end

#jump(commit_sha1) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2653

def jump(commit_sha1)
  begin
    verify_logged_in()
    log_start(__method__, args, options)
    project_home = get_project_home
    @project = Project.new(project_home)
    current_commit = @project.last_local_commit
    if current_commit.start_with? commit_sha1 #commit_sha1 can be partial.
      log_message("Project is already updated", Thor::Shell::Color::GREEN)
      exit(0)
    end
    log_message("Jumping to commit #{commit_sha1}")
    @files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home, project: @project)
    resp = @project.jump_idx(destination: commit_sha1)
    if resp.blank?
      log_message("Cant find the given commit", Thor::Shell::Color::RED)
      exit(0)
    end
    compare = resp['result']['compare']
    latest = resp['result']['latest']
    commit = resp['result']['commit']
    updated_files = compare['updated_on_server'] + compare['added']
    conflicted_files = compare['conflicts']
    conflicted_deleted = compare['delete_conflicts'] || []
    deleted_files = compare['deleted']
    overall_changes = [updated_files, conflicted_files, deleted_files].flatten.size

    progressbar = @files.create_progressbar(overall_changes, "Download Progress")
    @files.download_files(updated_files, commit_sha1, progress: progressbar)
    @files.download_files(conflicted_files, commit_sha1, progress: progressbar, postfix: '.conflicted')
    @files.delete_files_local(deleted_files, progress: progressbar, conflicted: conflicted_deleted)
    progressbar.finish
    @project.update_idx_with_commit!(commit, latest: latest)
    @project.generate_idx

    log_message("Jumped successfuly!", Thor::Shell::Color::GREEN)
  rescue => e
    Logger::log_error(e)
    log_message("Cant jump to the specified commit", Thor::Shell::Color::RED)
  end
end

#link ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 719

def link
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    docker_image = options["docker_image"]
    bucket = options["bucket"]
    if docker_image.present?
      docker = true
    else
      docker = false
    end

    sync = options["sync"]
    git = options["git"] ||  false
    project_name = options['title']
    project_name ||= File.basename(Dir.getwd)
    log_message("Linking #{project_name}", Thor::Shell::Color::BLUE)
    if File.directory?(Dir.getwd + "/.cnvrg")
      config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
      log_message("Directory is already linked to #{config[:project_slug]}", Thor::Shell::Color::RED)
      exit(0)
    end
    working_dir = Dir.getwd
    owner = CLI.get_owner
    if Project.link(owner, project_name, docker,git, bucket: bucket)
      path = Dir.pwd
      @project = Project.new(path)
      if sync
        @project.generate_idx() #DEV-741
        log_message("Syncing project", Thor::Shell::Color::BLUE)
        upload(true)
      end

      url = @project.url
      check = Helpers.checkmark
      log_message("#{check} Link finished successfully", Thor::Shell::Color::GREEN)
      log_message("#{project_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN)

    else
      @project.revert(working_dir) unless @project.nil?
      log_message("Error linking project, please contact support.", Thor::Shell::Color::RED)
      exit(0)
    end
  rescue => e
    log_error(e)
  rescue SignalException

    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  end
end

#link_git(project_url) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1753

def link_git(project_url)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    url_parts = project_url.split("/")
    project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
    slug = url_parts[project_index + 1]
    owner = url_parts[project_index - 1]
    response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
    Cnvrg::CLI.is_response_success(response)
    response = JSON.parse response["result"]
    project_name = response["title"]

    log_message("Linking #{project_name}", Thor::Shell::Color::BLUE)
    clone_resp = Project.clone_dir_remote(slug, owner, project_name, true)
    idx_status = Project.new(get_project_home).generate_idx
    log_message("Linking project #{project_name} successfully", Thor::Shell::Color::GREEN)

    return
  rescue => e
    log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
    log_error(e)
    return
  rescue SignalException

    say "\nAborting", Thor::Shell::Color::BLUE
    return
  end

end

#list_commits ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1716

def list_commits()
  verify_logged_in(true)
  log_start(__method__, args, options)

  project_dir = is_cnvrg_dir(Dir.pwd)
  @project = Project.new(project_dir)
  result = @project.list_commits()
  list = result["result"]["list"]
  print_table(list)

end

#list_dataset ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1642

def list_dataset
  verify_logged_in(false)
  log_start(__method__, args, options)
  dataset_dir = is_cnvrg_dir(Dir.pwd)
  @dataset = Dataset.new(dataset_dir)
  owner = @dataset.owner
  if owner.nil? or owner.empty?
    owner = CLI.get_owner()
  end

  result = @dataset.list(owner)
  Cnvrg::CLI.is_response_success(result)

    list = result["result"]["list"]

    print_table(list)


end

#list_dataset_commits ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1701

def list_dataset_commits()
  verify_logged_in(true)
  log_start(__method__, args, options)

  dataset_dir = is_cnvrg_dir(Dir.pwd)
  @dataset = Dataset.new(dataset_dir)
  result = @dataset.list_commits()
  list = result["result"]["list"]

  print_table(list)

end

#list_files_dataset ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2080

def list_files_dataset()
  begin
    verify_logged_in(true)
    log_start(__method__, args, options)
    commit = options[:commit]
    as_json = options[:json]
    dataset_dir = is_cnvrg_dir(Dir.pwd)
    @dataset = Dataset.new(dataset_dir)

    resp = @dataset.list_files(commit, as_json)
    if (resp and resp["result"])
      if as_json
        puts resp["result"]
      else
        print_table(resp["result"])

      end

    end


  rescue => e
    puts e
  rescue SignalException
    @dataset.set_next_commit(commit)
    log_message("Aborting", Thor::Shell::Color::YELLOW)
    exit(1)
  end

end

#list_images ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 5011

def list_images
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    owner = Cnvrg::CLI.get_owner()
    res = Cnvrg::API.request("users/#{owner}/images/list", 'GET')
    if Cnvrg::CLI.is_response_success(res)
      printf "%-20s %-20s  %-30s %-20s %-20s\n", "name", "project", "created by", "is_public", "last updated"
      res["result"]["images"].each do |u|
        time = Time.parse(u["created_at"])
        update_at = get_local_time(time)
        created_by = u["created_by"]

        printf "%-20s %-20s  %-30s %-20s %-20s\n", u["name"], u["project"], created_by, u["is_public"], update_at
      end
    end
    return res["result"]["images"]
  rescue SignalException
    say "\nAborting"
    exit(1)
  end


end

#list_machines ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 5038

def list_machines
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    owner = Cnvrg::CLI.get_owner()
    res = Cnvrg::API.request("users/#{owner}/machines/list", 'GET')
    if Cnvrg::CLI.is_response_success(res)
      printf "%-20s %-20s %-20s\n", "name", "created by", "last_used"
      res["result"]["machines"].each do |u|
        time = Time.parse(u["last_used"])
        update_at = get_local_time(time)
        printf "%-20s %-20s %-20s\n", u["name"], u["created_by"], update_at
      end
    end
    return res["result"]["images"]
  rescue SignalException
    say "\nAborting"
    exit(1)
  end


end

# File 'lib/cnvrg/cli.rb', line 511

def login
  begin
    log_handler()
    log_start(__method__, args, options)

    cmd = HighLine.new

    say 'Authenticating with cnvrg', Thor::Shell::Color::YELLOW

    @auth = Cnvrg::Auth.new
    netrc = Netrc.read
    @email, token = netrc[Cnvrg::Helpers.netrc_domain]

    if @email and token
      log_message('Seems you\'re already logged in', Thor::Shell::Color::BLUE)
      exit(0)
    end
    @email = ask("Enter your email:")
    password = cmd.ask("Enter your password (hidden):") {|q| q.echo = "*"}
    result = @auth.sign_in(@email, password)

    if !result["token"].nil?
      netrc[Cnvrg::Helpers.netrc_domain] = @email, result["token"]
      netrc.save

      log_message("Authenticated successfully as #{@email}", Thor::Shell::Color::GREEN)

      owners = result["owners"]
      urls = result["urls"]
      choose_owner = result["username"]
      ow_index = 0
      if owners.empty?
        choose_owner = result["username"]
      else
        choose_owner = owners[0]
      end


      if set_owner(choose_owner, result["username"], urls[ow_index])
        log_message("Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN)

      else
        log_message("Setting default owenr has failed, logging out", Thor::Shell::Color::RED)

        return logout()
      end

    else
      log_message("Failed to authenticate, wrong email/password", Thor::Shell::Color::RED)

      exit(1)
    end
  rescue => e

    log_message("Error Occurred, aborting", Thor::Shell::Color::RED)
    log_error(e)

    logout()
    exit(1)
  rescue SignalException

    say "\nAborting",Thor::Shell::Color::RED
    logout()
    exit(1)
  end
end

#logout ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 581

def logout
  begin
    log_handler()
    log_start(__method__, args, options)
    netrc = Netrc.read
    netrc.delete(Cnvrg::Helpers.netrc_domain)
    netrc.save
    log_message("Logged out successfully.\n", Thor::Shell::Color::GREEN)
  rescue => e
    puts e.message
    puts e.backtrace
  rescue SignalException
    say "\nAborting",Thor::Shell::Color::RED
    exit(1)
  end

end

#logs ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 636

def logs()
  begin
    lines = options["lines"]
    if lines.present?
      puts open($LOG.device.io.path).readlines.last(lines)
    else
      puts open($LOG.device.io.path).readlines.last(100)
    end


  rescue SignalException
    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  end
end

#me ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 602

def me()
  begin

    home_dir = File.expand_path('~')
    config = YAML.load_file(home_dir+"/.cnvrg/config.yml")

    api = config[:api]
    verify_ssl = config[:verify_ssl]
    if api.present?
      log_message("API: #{api}", Thor::Shell::Color::BLUE)
    end
    if verify_ssl.present?
      log_message("SSL Verification: #{verify_ssl}", Thor::Shell::Color::BLUE)
    end
    log_message("Logs file located at: #{$LOG.device.io.path}", Thor::Shell::Color::BLUE)

    verify_logged_in(false)
    log_start(__method__, args, options)
    auth = Cnvrg::Auth.new
    if (email = auth.get_email)
      log_message("Logged in as: #{email}", Thor::Shell::Color::GREEN)
    else
      log_message("You're not logged in.", Thor::Shell::Color::RED)
    end


  rescue SignalException
    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  end
end

#new(project_name) ⇒ `Object`

Projects

654	# File 'lib/cnvrg/cli.rb', line 654 desc 'new', 'Create a new cnvrg project'

#notebook ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 3520

def notebook
  verify_logged_in(true)
  log_start(__method__, args, options)
  local = options["local"]
  notebook_dir = options["notebook_dir"]
  datasets = options["datasets"]
  kernel = options["kernel"]
  image = options["image"] || nil
  data = options["data"]
  data_commit = options["data_commit"]
  dataset_only_tree = options["dataset_only_tree"]
  data_query = options["data_query"]
  if !data.present? and data_query.present?
    log_message("Please provide data with data_query", Thor::Shell::Color::RED)
    exit(1)
  end
  if data_query.present? and (data_commit.present? or dataset_only_tree.present?)
    log_message("Please use only one option: --query(-q) or #{data_commit.present? ? '--data_commit' : '--dataset_only_tree'} ", Thor::Shell::Color::RED)
    exit(1)
  end

  if local
    invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel, :image => image
    return
  else
    instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
                 "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
    instance_type = get_instance_type(instances)

    invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image,
           :data => data, :data_commit => data_commit , :dataset_only_tree => dataset_only_tree, :data_query => data_query, :datasets => datasets
    return

  end


end

#notebook_stop(notebook_slug) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4011

def notebook_stop(notebook_slug)
  begin
    verify_logged_in(true)
    log_start(__method__, args, options)
    project_dir = is_cnvrg_dir()


    @project = Project.new(project_dir)


    @note = Experiment.new(@project.owner, @project.slug)
    log_message("Stoping notebook session: #{notebook_slug}", Thor::Shell::Color::BLUE)

    res = @note.end_notebook_session(notebook_slug)
    if res
      check = Helpers.checkmark()
      log_message("#{check} Notebook session has stopped successfully", Thor::Shell::Color::GREEN)

      exit(0)
    else

      log_message("Couldn't stop notebook session, try stopping via cnvrg web", Thor::Shell::Color::RED)
      exit(1)
    end
  rescue => e
    log_message("Error occurd, aborting", Thor::Shell::Color::RED)
    log_error(e)
  rescue SignalException
    log_message("Aborting", Thor::Shell::Color::BLUE)
    exit(1)
  end


end

#port_container(container_id) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4572

def port_container(container_id)
  container = Docker::Container.get(container_id)
  say container.json["HostConfig"]["PortBindings"]["8888/tcp"][0]["HostPort"]
end

#pull_image(image_name) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 5145

def pull_image(image_name)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    owner = Cnvrg::CLI.get_owner()
    image = Cnvrg::Images.image_exist(owner, image_name)
    if !image
      log_message("Couldn't find image in cnvrg repository", Thor::Shell::Color::RED)
      exit(1)
    end
    path = download_image(image_name, image["slug"])
    if path
      log_message("Building image", Thor::Shell::Color::BLUE)
      Docker.options[:read_timeout] = 216000
      image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile.cpu', 't' => "#{image_name}:latest"}) do |v|
        begin
          if (log = JSON.parse(v)) && log.has_key?("stream")
            next if log["stream"].starts_with? "Step"
            $stdout.puts log["stream"]
          end
        rescue
        end

      end

      if not image.nil?
        FileUtils.rm_rf(path)
        checks = Helpers.checkmark()
        log_message("#{checks} Image built successfully", Thor::Shell::Color::GREEN)
        return image
      else

        log_message("Could not build image", Thor::Shell::Color::RED)
        return false
      end
    else

      log_message("Could not download image", Thor::Shell::Color::RED)
      return false


    end

      # else
      #   path = download_image(image_name,image["slug"])
      #   if path
      #     image = Docker::Image.import(path)
      #     image.tag('repo' => image_name, 'tag' => 'latest')
      #     if not image.nil?
      #       say "Finished downloading image, cleaning up..", Thor::Shell::Color::GREEN
      #       FileUtils.rm(path)
      #       checks = Helpers.checkmark()
      #       say "#{checks} Done", Thor::Shell::Color::GREEN
      #       log_end(0)
      #       return image
      #       log_end(0)
      #     else
      #       say "Could not download image", Thor::Shell::Color::RED
      #       return false
      #     end
      #
      #   end
      # end
  rescue => e

    log_message "Error: couldn't build image", Thor::Shell::Color::RED
    log_error(e)

  rescue SignalException
    say "\nAborting"
    exit(1)
  ensure
    if path
      FileUtils.rm_rf(path)

    end
  end


end

#push(*name) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4451

def push(*name)
  verify_logged_in(true)
  log_start(__method__, args, options)
  working_dir = is_cnvrg_dir()
  if !name.empty? and name == "cnvrg"
    log_message("can't create image with the name cnvrg", Thor::Shell::Color::RED)
    exit(1)
  end
  begin
    image = is_project_with_docker(working_dir)
    if !image or !image.is_docker
      log_message("Couldn't find image related to project", Thor::Shell::Color::RED)
      exit(0)
    end
    if !name.nil? and !name.empty?
      if name.include? " "
        name.gsub!(" ", "_")
      end
    end
    stored_commands = File.open(working_dir + "/.cnvrg/custom_image.txt").read.chop.gsub("\n", ",")
    if stored_commands.nil? or stored_commands.empty?
      log_message("Nothing to push", Thor::Shell::Color::BLUE)
      exit(0)
    end

    log_message("Pushing new image", Thor::Shell::Color::BLUE)
    if image.create_custom_image(name, working_dir, stored_commands)

      log_message("Image was updated successfully", Thor::Shell::Color::GREEN)
    end
  rescue => e
    log_message("error occurred, aborting", Thor::Shell::Color::RED)
    log_error(e)

  end
end

#queries ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1664

def queries
  verify_logged_in(false)
  log_start(__method__, args, options)
  dataset_dir = is_cnvrg_dir(Dir.pwd)
  @dataset = Dataset.new(dataset_dir)
  result = @dataset.search_queries()
  print_table(result)
end

#query_files(query) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1689

def query_files(query)
  verify_logged_in(false)
  log_start(__method__, args, options)
  dataset_dir = is_cnvrg_dir(Dir.pwd)
  @dataset = Dataset.new(dataset_dir)
  query = options["query"] || query
  result = @dataset.get_query_file(query)
  print_table(result)
end

#remote_notebook ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 3657

def remote_notebook()
  verify_logged_in(true)
  log_start(__method__, args, options)

  working_dir = is_cnvrg_dir()
  instance_type = options["machine_type"] || nil
  datasets = options["datasets"]
  data = options["data"]
  data_commit = options["data_commit"]
  commit = options["commit"]
  notebook_type = options["notebook_type"]
  dataset_only_tree = options["dataset_only_tree"]
  image = options["image"]
  ds_sync_options = 0
  if dataset_only_tree
    ds_sync_options = 1
  end

  data_query = nil
  if data.present?
    data_query = options["data_query"]
  end

  if data_commit.present? and data_query.present?
    log_message("Please use only one option: --query(-q) or --data_commit ", Thor::Shell::Color::RED)
    exit(1)
  end

  begin
    project = Project.new(working_dir)
    exp = Experiment.new(project.owner, project.slug)

    if !notebook_type.nil? and !notebook_type.empty?
      notebook_type = "jupyter"
    end
    invoke :sync, [false], []
    slug = ""
    res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options,data_query, image, datasets)
    if Cnvrg::CLI.is_response_success(res)
      slug = res["result"]["notebook_url"]
      log_message("#{Helpers.checkmark} Notebook is ready: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/notebook_sessions/show/#{slug}", Thor::Shell::Color::GREEN)

    end
  rescue => e
    log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
    log_error(e)

  rescue SignalException
    log_message("Aborting", Thor::Shell::Color::BLUE)
    notebook_stop(slug) unless slug.nil? or slug.empty?

    exit(1)
  end
end

#remote_notebook_old ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 3565

def remote_notebook_old()
  verify_logged_in(true)
  log_start(__method__, args, options)

  working_dir = is_cnvrg_dir()
  notebook_dir = options["notebook_dir"]
  instance_type = options["machine_type"] || nil
  kernel = options["kernel"] || nil
  data = options["data"]
  data_commit = options["data_commit"]


  begin
    choose_image = options["image"]

    if !choose_image.nil? and !choose_image.empty?
      invoke :set_image, [choose_image]
    end
    invoke :sync, [false], []


    res = @image.remote_notebook(notebook_dir, instance_type, kernel, data, data_commit)
    if Cnvrg::CLI.is_response_success(res)
      if res["result"]["machine"] == -1
        log_message("There are no available machines", Thor::Shell::Color::BLUE)
        create = yes? "create new machine?", Thor::Shell::Color::YELLOW
        if create
          res = Cnvrg::API.request("users/#{@image.owner}/machines/list", 'GET')
          if Cnvrg::CLI.is_response_success(res)
            instance_type = machine_options(res["result"]["aws_options"])
            if @image.new_machine(instance_type)
              res = @image.remote_notebook(notebook_dir, instance_type, kernel)
              if Cnvrg::CLI.is_response_success(res)
                url = res["result"]["url"]
                if !url.nil? and !url.empty?
                  check = Helpers.checkmark()

                  log_message("#{check} Notebook server started successfully", Thor::Shell::Color::GREEN)
                  Launchy.open(url)
                else
                  log_message("Couldn't run notebook server", Thor::Shell::Color::RED)
                end
                exit(0)
              end
            end
          else
            log_message("No machines are avilable", Thor::Shell::Color::RED)
            exit(0)
          end


        else
          log_message("Can't execute command on remote machine with local image", Thor::Shell::Color::RED)
          exit(1)

        end
      else
        note_url = res["result"]["notebook_url"]
        @image.set_note_url(note_url)
        check = Helpers.checkmark()
        log_message("#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@image.owner}/projects/#{@image.project_slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN)
        # Launchy.open(url)

        exit(0)
      end
    end
  rescue => e
    log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
    log_error(e)

  rescue SignalException
    exit_status = -1
    end_commit = @project.last_local_commit

    res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
    say "\nAborting"

    exit(1)
  end
end

#revert_exp ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1921

def revert_exp
  begin
    log_start(__method__, args, options)
    @project = Project.new(get_project_home)
    ignore_list = @project.get_ignore_list()

    result = @project.compare_idx(false)["result"]
    result = result["tree"]
    if result["added"].size > 0
      FileUtils.rm_rf(result["added"])
    end
    say "Changes were removed successfully", Thor::Shell::Color::GREEN


  rescue SignalException
    log_end(-1)
    say "\nAborting"
    exit(1)
  end
end

#run(*cmd) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2845

def run(*cmd)
  verify_logged_in(true)
  log_start(__method__, args, options)
  datasets = options["datasets"]
  sync_before = options["sync_before"]
  sync_after = options["sync_after"]
  log = options["log"]
  title = options["title"]
  commit = options["commit"] || nil
  email_notification = options["email_notification"]
  upload_output = options["upload_output"]
  local = options["local"]
  schedule = options["schedule"]
  image = options["image"] || nil
  grid = options["grid"]
  data = options["data"]
  data_commit = options["data_commit"]
  ignore = options["ignore"]
  sync_before_terminate = options["sync_before_terminate"]
  periodic_sync = options["periodic_sync"]
  force = options["force"]
  max_time = options["max_time"]
  dataset_only_tree = options["dataset_only_tree"]
  custom_machine = options["machine"]
  output_dir = options["output_dir"]
  data_query = options["data_query"]
  local_folders = options["local_folders"]

  if !data.present? and data_query.present?
    log_message("Please provide data with data_query", Thor::Shell::Color::RED)
    exit(1)
  end
  if data_query.present? and (data_commit.present? or dataset_only_tree.present?)
    log_message("Please use only one option: --query(-q) or #{data_commit.present? ? '--data_commit' : '--dataset_only_tree'} ", Thor::Shell::Color::RED)
    exit(1)
  end
  git_commit = options["git_commit"]
  git_branch = options["git_branch"]
  restart_if_stuck = options["restart_if_stuck"]

  options_hash = Hash[options]

  if local
    if Cnvrg::Helpers.windows?
        say "Windows is currently not supported for running experiments locally"
        return
    else
      invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
             :log => log, :email_notification => email_notification, :upload_output => upload_output,
             :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :output_dir=>output_dir, :data_query=>data_query
      return
    end
  else
    if !periodic_sync.nil? and !periodic_sync.empty?
      if /^\d{2}$/ === periodic_sync
        if !["15","30","45","60"].include? periodic_sync
          log_message("periodic sync can only be every 15m, 30m, 45m or 60m", Thor::Shell::Color::RED)
          exit(1)
        end
      else
        log_message("periodic sync has to be one of the following values: 15m, 30m, 45m or 60m", Thor::Shell::Color::RED)
        exit(1)

      end
    end
    instances = { "small" => options["small"], "medium" => options["medium"], "large" => options["large"],
                  "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"],
                 options["machine"] => !options["machine"].blank? }
    instance_type = get_instance_type(instances)
    invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
           :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
           :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :sync_before_terminate => sync_before_terminate,
            :max_time => max_time,
           :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
           :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
           :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets
    return
  end

end

#run_notebook ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 3840

def run_notebook

  begin
    verify_logged_in(true)
    log_start(__method__, args, options)

    project_dir = is_cnvrg_dir()

    notebook_dir = options["notebook_dir"]
    remote = options["remote"] || false
    kernel = options["kernel"] || ""
    notebooks_pid = nil

    if notebook_dir.empty?
      notebook_dir = project_dir
    else

      notebook_dir = project_dir + notebook_dir
    end
    choose_image = options["image"]

    if !choose_image.nil? and !choose_image.empty?
      invoke :set_image, [choose_image]
    end

    image = is_project_with_docker(Dir.pwd)
    if !image
      jupyter_installed = `which jupyter`
      if !$?.success?
        say "Could not find jupyter, Is it installed?", Thor::Shell::Color::RED
        exit(1)
      end


      cmd = "jupyter-notebook --port=8888"
      PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
        begin
          notebooks_pid = pid
          stdout.each do |line|
            puts line

          end

        rescue Errno::EIO => e
          # break
        rescue Errno::ENOENT
          log_end(1, "command #{cmd} isn't valid")


          say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
        rescue PTY::ChildExited
          log_end(1, "proccess exited")
          say "The process exited!", Thor::Shell::Color::RED
        rescue => e
          log_end(-1, e.message)
          say "Error occurred,aborting", Thor::Shell::Color::RED
          exit(0)

        end


      end

    end

    if image and image.is_docker and !remote
      container = image.get_container
      if !container
        say "Couldn't start docker container", Thor::Shell::Color::RED
        exit(1)

      end

      if options["verbose"]
        say "Syncing project before running", Thor::Shell::Color::BLUE
        say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
      end
      @project = Project.new(project_dir)

      start_commit = @project.last_local_commit

      if (note_slug = image.note_slug)
        say "There is a running notebook session in: https://cnvrg.io/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_slug}", Thor::Shell::Color::BLUE
        new = yes? "Create a new session?", Thor::Shell::Color::YELLOW
        if !new
          exit(0)
        end

      end
      invoke :sync, [false], :verbose => options["verbose"]
      say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
      #replace url
      base_url = get_base_url()

      local_url = "/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local"
      command = ["/bin/bash", "-lc", "sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
      container.exec(command, tty: true)
      container.stop()
      container.start()
      sleep(7)
      @note = Experiment.new(@project.owner, @project.slug)
      port = image.container_port()

      command = ["/bin/bash", "-lc", "jupyter notebook list"]
      list = container.exec(command, tty: true)[0]
      if list.empty? or list.nil?
        say "Couldn't start notebook server", Thor::Shell::Color::RED
        log_end(1, "can't start notebook server")
        exit(1)
      end

      result = ""
      list.each do |r|
        if r.include? "http"
          result = r
        end
      end
      token = result.to_s.split("::")[0].to_s.match(/(token=)(.+)\s/)[2]

      # machine_activity = @note.get_machine_activity(project_dir)


      slug = @note.start_notebook_session(kernel, start_commit, token, port, false, notebook_dir)
      image.set_note_url(slug)
      note_url = "http://localhost:#{port}/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local/?token=#{token}"


      if !note_url.empty?
        check = Helpers.checkmark()

        say "#{check} Notebook server started successfully", Thor::Shell::Color::GREEN
        Launchy.open(note_url)
      else
        say "Couldn't start notebook server", Thor::Shell::Color::RED
        log_end(1, "can't start notebook server")
        exit(1)
      end
    end


  rescue => e
    log_end(-1, e.message)
    say "Error occurred, aborting", Thor::Shell::Color::RED
    if container
      container.stop()
    end
  rescue SignalException
    if !notebooks_pid.nil?
      ::Process.kill(0, notebooks_pid)
      say "#{check} Notebook has stopped successfully", Thor::Shell::Color::GREEN


      invoke :sync, [false], []
    else
      log_end(-1)

      if container
        container.stop()
      end
    end

  end


end

#search_libraries(library) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 3714

def search_libraries(library)
  begin
    verify_logged_in(true)
    log_start(__method__, args, options)
    project_dir = is_cnvrg_dir()

    image = is_project_with_docker(project_dir)
    if image and image.is_docker
      container = image.get_container
      if !container
        say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
        exit(1)
      end
    else
      say "Project is not configured with any image", Thor::Shell::Color::RED
      exit(1)

    end

    say "Searching for #{library}", Thor::Shell::Color::BLUE
    pip_arr = image.get_installed_packages("python")
    pip_arr = pip_arr.map(&:downcase)
    check = Helpers.checkmark()
    if !(p = pip_arr.map {|x| x.split("==")[0]}.index(library.downcase)).nil?

      say "#{check} Found it!", Thor::Shell::Color::GREEN


      printf "%-40s %-30s\n", "#{pip_arr[p].split("==")[0]}", "#{pip_arr[p].split("==")[1]}"

    else
      dpkg_arr = image.get_installed_packages("system")
      dpkg_arr = dpkg_arr.map(&:downcase)
      if !(p = dpkg_arr.map {|x| x.split("==")[0]}.index(library.downcase)).nil?
        say "#{check} Found!", Thor::Shell::Color::GREEN

        printf "%-40s %-30s\n", "#{dpkg_arr[p].split("==")[0]}", "#{dpkg_arr[p].split("==")[1]}"
      else
        say "Couldn't find #{library}, run cnvrg install_libraries to install", Thor::Shell::Color::RED
        exit(1)
      end


    end
  rescue => e
    log_end(-1, e.message)
    say "Error occurred, aborting", Thor::Shell::Color::RED
    if container
      container.stop()
    end
  rescue SignalException
    log_end(-1)
    if container
      container.stop()
    end
    say "Aborting"
    exit(1)
  end
end

#set_api_url(url) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 248

def set_api_url(url)
  log_handler()
  log_start(__method__, args, options)
  home_dir = File.expand_path('~')
  if url.end_with? "/"
    url = url.chomp("/")
  end
  if !url.end_with? "/api"
    url = url + "/api"
  end
  verify_ssl = options["verify_ssl"]
  begin
    if !File.directory? home_dir + "/.cnvrg"
      FileUtils.mkdir_p([home_dir + "/.cnvrg", home_dir + "/.cnvrg/tmp"])
    end
    if !File.exist?(home_dir + "/.cnvrg/config.yml")
      FileUtils.touch [home_dir + "/.cnvrg/config.yml"]
    end
    compression_path = "#{File.expand_path('~')}/.cnvrg/tmp"
    begin
      config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
      if !config
        config = {owner: "", username: "", version_last_check: get_start_day(), api: url, compression_path: compression_path ,verify_ssl: verify_ssl }
      end


    rescue
      config = {owner: "", username: "", version_last_check: get_start_day(), api: url, compression_path: compression_path ,verify_ssl: verify_ssl }
    end

    say "Setting default api to be: #{url}", Thor::Shell::Color::BLUE
    if config.empty?
      config = {owner: "", username: "", version_last_check: get_start_day(), api: url, compression_path: compression_path, verify_ssl: verify_ssl }
    else
      if !config.to_h[:compression_path].nil?
        compression_path = config.to_h[:compression_path]
      end
      config = {owner: config.to_h[:owner], username: config.to_h[:username], version_last_check: config.to_h[:version_last_check], api: url, compression_path: compression_path, verify_ssl: verify_ssl}
    end

    checks = Helpers.checkmark
    File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }

    say "#{checks} Done", Thor::Shell::Color::GREEN

  rescue => e
    log_error(e)
    say "Couldn't set default api, contact [email protected]", Thor::Shell::Color::RED
  end
end

#set_compression_path(*compression_path) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 432

def set_compression_path(*compression_path)
  begin
    if (compression_path.nil? or compression_path.empty?) and options["reset"]
      compression_path = ["#{File.expand_path('~')}/.cnvrg/tmp"]
    end
    compression_path = compression_path.join(" ")
    if !Dir.exist? compression_path
      say "Couldn't find #{compression_path}, please make sure it exist", Thor::Shell::Color::RED
      exit(0)
    end

    home_dir = File.expand_path('~')
    path = "#{home_dir}/.cnvrg/config.yml"
    if !File.exist?(path)
      say "Couldn't find ~/.cnvrg/config.yml file, please logout and login again", Thor::Shell::Color::RED

      exit(0)
    end
    config = YAML.load_file(path)
    config_new = {owner: config.to_h[:owner], username: config.to_h[:username],
                  version_last_check: config.to_h[:version_last_check], api: config.to_h[:api], compression_path: compression_path}
    File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config_new.to_yaml}
    checks = Helpers.checkmark
    say "#{checks} Done", Thor::Shell::Color::GREEN

  rescue SignalException
    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  end
end

#set_data_url(dataset_url) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 219

def set_data_url(dataset_url)
  begin
  verify_logged_in(true)
  log_start(__method__, args, options)
  unless is_cnvrg_dir
    log_message("Not in cnvrg dir.", Thor::Shell::Color::RED)
  end
  url_parts = dataset_url.split("/")
  project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
  slug = url_parts[project_index + 1]
  owner = url_parts[project_index - 1]
  res = Cnvrg::API.request("users/#{owner}/datasets/#{slug}", 'GET')
  unless Cnvrg::CLI.is_response_success(res, false)
    raise SignalException.new
  end
  @dataset = Dataset.new(Dir.pwd)
  result = res['result']
  @dataset.change_url(result.symbolize_keys)
  log_message("Changed URL Succesfuly to #{dataset_url}", Thor::Shell::Color::GREEN)
  rescue => e
    log_message("Cant change the url to the given dataset url", Thor::Shell::Color::RED)
  end
end

#set_default_owner ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 366

def set_default_owner
  begin
    path = File.expand_path('~') + "/.cnvrg/config.yml"
    if !File.exist?(path)
      say "Couldn't find ~/.cnvrg/config.yml file, please logout and login again", Thor::Shell::Color::RED

      exit(0)
    end
    config = YAML.load_file(path)

    username = config.to_h[:username]
    res = Cnvrg::API.request("/users/#{username}/get_possible_owners", 'GET')
    if Cnvrg::CLI.is_response_success(res)
      owner = username
      result = res["result"]
      owners = result["owners"]
      urls = result["urls"]
      choose_owner = result["username"]
      if owners.empty?
      else
        owners << choose_owner
        chosen = false
        while !chosen
          owners_id = owners.each_with_index.map {|x, i| "#{i + 1}. #{x}"}
          choose_owner = ask("Choose default owner:\n" + owners_id.join("\n") + "\n")

          if choose_owner =~ /[[:digit:]]/
            ow_index = choose_owner.to_i - 1
            if ow_index < 0 or ow_index >= owners.size
              say "No such owner, please choose again", Thor::Shell::Color::BLUE
              chosen = false
              next
            end
            choose_owner = owners[choose_owner.to_i - 1]
            chosen = true

          else

            owners_lower = owners.map {|o| o.downcase}
            ow_index = owners_lower.index(choose_owner.downcase)
            if ow_index.nil?
              say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
            else
              chosen = true
            end
          end

        end


      end
      if set_owner(choose_owner, result["username"], urls[ow_index])
        say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
      else
        say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
      end
    end
  rescue SignalException
    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  end
end

#set_image(docker_image) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 5228

def set_image(docker_image)
  verify_logged_in(true)
  log_start(__method__, args, options)
  working_dir = is_cnvrg_dir
  project = Project.new(working_dir)

  local_images = Docker::Image.all
  docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.include? docker_image}.flatten
  if docker_image_local.size == 0

    if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
      image = pull(docker_image)
      if image
        log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
        @image = Images.new(working_dir, docker_image)
      else
        log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
        exit(1)
      end
    else
      log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
      exit(1)

    end
  elsif docker_image_local.size == 1
    log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
    @image = Images.new(working_dir, docker_image_local[0])
  elsif docker_image_local.size > 1
    log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
    image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
    image_name = image_name.strip
    @image = Images.new(working_dir, image_name)
  end
  @image.update_image_activity(project.last_local_commit, nil)
end

#set_remote_api_url(owner, current_user, url) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 339

def set_remote_api_url(owner, current_user, url)
  home_dir = File.expand_path('~')
  if !url.end_with? "/api"
    url = url + "/api"
  end
  begin
    if !File.directory? home_dir + "/.cnvrg"
      FileUtils.mkdir_p([home_dir + "/.cnvrg", home_dir + "/.cnvrg/tmp"])
    end
    if !File.exist?(home_dir + "/.cnvrg/config.yml")
      FileUtils.touch [home_dir + "/.cnvrg/config.yml"]
    end
    config = YAML.load_file(home_dir + "/.cnvrg/config.yml")

    compression_path = "#{home_dir}/.cnvrg/tmp"
    config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
    File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
    say "Done", Thor::Shell::Color::GREEN
  rescue
    say "ERROR", Thor::Shell::Color::RED
  end
end

#set_remote_login(current_user, owner, url, email, secret) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 301

def set_remote_login(current_user, owner, url, email, secret)
  netrc = Netrc.read
  netrc[Cnvrg::Helpers.netrc_domain] = email, secret
  netrc.save
  home_dir = File.expand_path('~')
  if !url.end_with? "/api"
    url = url + "/api"
  end
  begin
    if !File.directory? home_dir + "/.cnvrg"
      FileUtils.mkdir_p([home_dir + "/.cnvrg", home_dir + "/.cnvrg/tmp"])
    end
    if !File.exist?(home_dir + "/.cnvrg/config.yml")
      FileUtils.touch [home_dir + "/.cnvrg/config.yml"]
    end
    config = YAML.load_file(home_dir + "/.cnvrg/config.yml")

    compression_path = "#{home_dir}/.cnvrg/tmp"
    config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
    File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
    say "Done", Thor::Shell::Color::GREEN
  rescue
    say "ERROR", Thor::Shell::Color::RED
      File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
      say "#{checks} Done", Thor::Shell::Color::GREEN
    # else
    #   say "Couldn't set default api, contact [email protected]", Thor::Shell::Color::RED
    #   exit(1)
    #
    # end

  rescue => e
    say "Couldn't set default api, contact [email protected]", Thor::Shell::Color::RED
  end
end

#show ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2699

def show
  path = options['path']
  commit = options['commit']
    verify_logged_in(true)
    log_start(__method__, args, options)
    project_home = get_project_home
    @project = Project.new(project_home)



  @files = Cnvrg::Files.new(@project.owner, @project.slug, project: @project)
  begin

    file = @files.show_file_s3(path, commit)

    if file
      puts file
    else
      say "Couldn't find file"
    end
  end
end

#show_libraries ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 3777

def show_libraries
  begin
    verify_logged_in(true)
    log_start(__method__, args, options)
    system = options["system"] || false


    project_dir = is_cnvrg_dir()

    image = is_project_with_docker(project_dir)
    if image and image.is_docker
      container = image.get_container
      if !container
        say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
        exit(1)
      end
    else
      say "Project is not configured with any image", Thor::Shell::Color::RED
      exit(1)

    end

    say "Showing python installed libraries", Thor::Shell::Color::BLUE
    pip_arr = image.get_installed_packages("python")
    printf "%-40s %-30s\n", "name", "version"
    printf "%-40s %-30s\n", "====", "======="

    pip_arr.each do |p|

      printf "%-40s %-30s\n", "#{p.split("==")[0]}", "#{p.split("==")[1]}"
    end
    if system

      say "Showing system installed libraries", Thor::Shell::Color::BLUE
      dpkg_arr = image.get_installed_packages("system")
      printf "%-40s %-30s\n", "name", "version"
      printf "%-40s %-30s\n", "====", "======="
      dpkg_arr.each do |p|
        printf "%-40s %-30s\n", "#{p.split("==")[0]}", "#{p.split("==")[1]}"

      end
    end
  rescue => e
    log_end(-1, e.message)
    say "Error occurred, aborting"
    if container
      container.stop()
    end
  rescue SignalException
    log_end(-1)
    say "Aborting"
    exit(1)
  end
end

#snap_data ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1051

def snap_data
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)

    owner = CLI.get_owner
    path = Dir.pwd
    @dataset = Dataset.new(path)

    log_end(0)

  rescue SignalException
    log_end(-1)

    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  end
end

#start_commit_data ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2021

def start_commit_data()
  verify_logged_in(true)
  log_start(__method__, args, options)
  dataset_dir = is_cnvrg_dir(Dir.pwd)
  direct = options[:direct]
  new_branch = options["new_branch"] || false
  force = options["force"] || false
  chunk_size = options["chunk_size"] || false
  commit_sha1 = nil
  @dataset = Dataset.new(dataset_dir)
  @dataset.backup_idx
  @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
  next_commit = @dataset.get_next_commit #if there was a partial commit..
  chunks = (@dataset.list_all_files.length.to_f / chunk_size).ceil
  resp = @files.start_commit(new_branch, force, chunks: chunks, dataset: @dataset)
  if !resp['result']['can_commit']
    log_message("Cant upload files because a new version of this dataset exists, please download it or upload with --force", Thor::Shell::Color::RED)
    exit(1)
  end
  commit_sha1 = resp["result"]["commit_sha1"]
  unless commit_sha1.eql? next_commit
    @dataset.set_partial_commit(next_commit)
  end
  @dataset.set_next_commit(commit_sha1)
  return commit_sha1
end

#status ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1862

def status
  begin
    verify_logged_in()
    log_start(__method__, args, options)
    @project = Project.new(get_project_home)

    new_branch = options["new_branch"] || false
    force = options["force"] || false

    result = @project.compare_idx(new_branch,force:force)["result"]
    commit = result["commit"]
    result = result["tree"]
    log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE)

    if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
      log_message("Project is up to date", Thor::Shell::Color::GREEN)
      return true
    end
    if result["added"].size > 0
      log_message("Added files:\n", Thor::Shell::Color::BLUE)
      result["added"].each do |a|
        log_message("\t\tA:\t#{a}", Thor::Shell::Color::GREEN)
      end
    end

    if result["deleted"].size > 0
      log_message("Deleted files:\n", Thor::Shell::Color::BLUE)
      result["deleted"].each do |a|
        log_message("\t\tD:\t#{a}", Thor::Shell::Color::GREEN)
      end
    end
    if result["updated_on_local"].size > 0
      log_message("Local changes:\n", Thor::Shell::Color::BLUE)
      result["updated_on_local"].each do |a|
        log_message("\t\tM:\t#{a}", Thor::Shell::Color::GREEN)
      end
    end

    if result["updated_on_server"].size > 0
      log_message("Remote changes:\n", Thor::Shell::Color::BLUE)
      result["updated_on_server"].each do |a|
        log_message("\t\tM:\t#{a}", Thor::Shell::Color::GREEN)
      end
    end

    if result["conflicts"].size > 0
      log_message("Conflicted changes:\n", Thor::Shell::Color::BLUE)
      result["conflicts"].each do |a|
        log_message("\t\tC:\t#{a}", Thor::Shell::Color::RED)
      end
    end
  rescue SignalException
    say "\nAborting"
    exit(1)
  end
end

#stop_container(container_id) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4586

def stop_container(container_id)
  container = Docker::Container.get(container_id)
  container.stop()
  container.remove()

end

#sync(direct = true) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2788

def sync(direct = true)
  verify_logged_in(true) if direct
  @project = Project.new(get_project_home)
  log_start(__method__, args, options)
  log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
  log_message('Syncing project', Thor::Shell::Color::BLUE, !options["verbose"])
  job_slug = options['job_slug']
  job_type = options['job_type']
  in_exp = options["in_exp"] || (job_slug.present? and job_type.present?)
  in_exp = false if job_type.present? and job_type == "NotebookSession"
  run_download = true
  if options[:force] or options[:files].present? or options[:output_dir].present? or in_exp or @project.is_branch
    run_download = false
  end
  if run_download
    invoke :download, [true, "", in_exp ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
  end
  invoke :upload, [false, true,  direct, "",in_exp,options[:force],  options["output_dir"],job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
         :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
         :files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type
end

#sync_data_new(new_branch, force, verbose, commit, all_files, tags, parallel, chunk_size, init) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1952

def sync_data_new(new_branch, force, verbose, commit, all_files, tags ,parallel, chunk_size, init)
  verify_logged_in(true)
  log_start(__method__, args, options)
  log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
  if !force and !init
    # w(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
    invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
  end

  # w(new_branch,  verbose,sync,force, tags, chunk_size)
  invoke :upload_data_new,[new_branch,  verbose,true,force, tags, chunk_size], :new_branch=>new_branch,
         :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel

end

#sync_image(docker = false) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4420

def sync_image(docker = false)
  verify_logged_in(true)
  log_start(__method__, args, options)
  is_public = options["is_public"] || false
  is_base = options["is_base"] || false
  message = options["message"] || ""
  image_id = commit_image

  if docker
    message = "before running experiment"
    image = is_project_with_docker(Dir.pwd)
    if image and image.is_docker
      container = image.get_container
      if !container

        upload_image(image_id, is_public, is_base, message)
      else
        command = ["/bin/bash", "-lc", "cnvrg upload_image #{image_id} #{is_public} #{is_base} #{message}"]
        puts "Running in contianer"
        container.exec(command, detach: false)
      end
    end

  else
    upload_image(image_id, is_public, is_base, message)
  end

end

#tensor_port_container(container_id) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4579

def tensor_port_container(container_id)
  container = Docker::Container.get(container_id)
  say container.json["HostConfig"]["PortBindings"]["6006/tcp"][0]["HostPort"]
end

#test ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 187

def test
  # image_settings = {
  #     'Image' => "cnvrg:latest",
  #     'User' => 'ds',
  #     'Cmd' => '/home/ds/run_ipython.sh',
  #     'ExposedPorts' => {``
  #         '80/tcp' => {},
  #     },
  #     'HostConfig' => {
  #         'PortBindings' => {
  #             '80/tcp' => [
  #                 {'HostPort' => "7654", 'HostIp' => 'localhost'}
  #             ],
  #         },
  #     },
  # }
  # container = Docker::Container.get('b4d64bf83f41')
  # s = "/leah/1/2/3/4/5"
  # command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
  # puts container.exec(command, tty: true)
end

#unlink ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1731

def unlink
  verify_logged_in(false)
  log_start(__method__, args, options)
  working_dir = is_cnvrg_dir()
  list_to_del = [working_dir + "/.cnvrgignore", working_dir + "/.cnvrg"]
  FileUtils.rm_rf list_to_del
end

#upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2187

def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil)
  begin
    # we are passing "force" twice.. doesnt really make sense :\\
    verify_logged_in(true)
    log_start(__method__, args, options)
    @project = Project.new(get_project_home)
    commit_msg = options["message"]
    if commit_msg.nil? or commit_msg.empty?
      commit_msg = ""
    end
    return_id = options["return_id"]
    @files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: get_project_home, project: @project)
    ignore = options[:ignore] || ""
    force = options[:force] || force || false
    spec_files_to_upload = options["files"]
    check = Helpers.checkmark()

    if !spec_files_to_upload.blank?
      spec_files_to_upload = spec_files_to_upload.split(",")
    end
    if @project.is_git
      git_output_dir = options["output_dir"] || output_dir
      if git_output_dir.present?
        if git_output_dir.ends_with? "/"
          git_output_dir = git_output_dir[0..-2]
        end
        list = @project.generate_output_dir(git_output_dir)
        spec_files_to_upload = list
        if spec_files_to_upload.blank?
          log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
          return true
        end
        force = true
      end
    end

    if ignore.nil? or ignore.empty?
      ignore = ignore_list
    end
    data_ignore = data_dir_include()
    if !data_ignore.nil?
      if ignore.nil? or ignore.empty?
        ignore = data_ignore
      else
        ignore = "#{ignore},#{data_ignore}"
      end
    end
    if !@project.update_ignore_list(ignore)
      log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
    end
    new_branch = options["new_branch"] || @project.is_branch

    result = @project.compare_idx(new_branch, force: force, deploy: options["deploy"], in_exp: in_exp, specific_files: spec_files_to_upload)
    commit = result["result"]["commit"]

    if !link
      if (result["result"]["new_version_exist"] and !force) or ((commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?) and !force)
        log_message("Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::BLUE)
        return false
      end

      log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, (options["verbose"]))
    end
    result = result["result"]["tree"]
    if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
      msg = "#{check} Project is up to date"
      if return_id
        Cnvrg::Logger.jsonify_message(msg: msg, success: true)
      else
        log_message(msg, Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
      end
      return true
    end
    update_count = 0
    update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
    successful_updates = []
    successful_deletions = []
    if options["verbose"]
      if update_total == 1
        log_message("Updating #{update_total} file", Thor::Shell::Color::BLUE)
      else
        log_message("Updating #{update_total} files", Thor::Shell::Color::BLUE)
      end
    else
      log_message("Syncing files", Thor::Shell::Color::BLUE, ((options["sync"] or sync)) ? false : true)
    end
    # Start commit
    current_commit = nil
    exp_start_commit = nil
    if in_exp || (job_slug.present? and job_type.present?)
      exp_start_commit = @project.last_local_commit
    else
      current_commit = @project.last_local_commit
    end
    job_type = options['job_type'] || job_type
    job_slug = options['job_slug'] || job_slug
    commit_sha1 = @files.start_commit(new_branch, force: force, exp_start_commit: exp_start_commit,
                                      job_type: job_type, job_slug: job_slug, start_commit: current_commit, message: options["message"])["result"]["commit_sha1"]
    # upload / update
    # delete
    to_upload = result["added"] + result["updated_on_local"]
    deleted = result["deleted"]
    progressbar = ProgressBar.create(:title => "Upload Progress",
                                     :progress_mark => '=',
                                     :format => "%b>>%i| %p%% %t",
                                     :starting_at => 0,
                                     :total => (to_upload.size + deleted.size),
                                     :autofinish => true)
    @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar)
    @files.delete_files_from_server(deleted, commit_sha1)
    progressbar.finish
    res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
    unless Cnvrg::CLI.is_response_success(res, false)
      raise StandardError.new("Cant end commit")
    end
    # save idx
    @project.update_idx_with_files_commits!((to_upload + deleted), res["result"]["commit_time"])
    @project.update_idx_with_commit!(commit_sha1)
    if options["verbose"]
      log_message("#{check} Done", Thor::Shell::Color::BLUE)
      if successful_updates.size > 0
        successful_updates.flatten!
        log_message("Updated:", Thor::Shell::Color::GREEN)
        suc = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
        log_message(suc.join("\n"), Thor::Shell::Color::GREEN)
      end
      if successful_deletions.size > 0
        successful_deletions.flatten!
        log_message("Deleted:", Thor::Shell::Color::GREEN)
        del = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
        log_message(del.join("\n"), Thor::Shell::Color::GREEN)
      end
      log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
    else
      if return_id
        puts "\n"
        print_res = {
            'success' => "true",
            'commit_sha1' => res["result"]["commit_id"]
        }
        puts JSON[print_res]
        return JSON[print_res]
      end
      if (options["sync"] or sync) and direct
        log_message("#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN)
        return true
      else
        log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)
        return true
      end
    end
  rescue => e
    if e.is_a? SignalException
      say "\nAborting", Thor::Shell::Color::BLUE
      say "\nRolling back all changes", Thor::Shell::Color::BLUE
    else
      log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
      log_error(e)
    end
    @files.rollback_commit(commit_sha1) unless commit_sha1.nil?
    print_res = {
        'success' => "false",
        'message' => 'couldn\'t commit changes,  Rolling Back all changes.'
    }
    puts "\n"
    puts JSON[print_res] if return_id
    return false
  end
end

#upload_cnvrg_image(image_path, image_name, secret) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4834

def upload_cnvrg_image(image_path, image_name, secret)
  begin
    verify_logged_in(false)

    @files = Cnvrg::Files.new("", "")
    say "Uploading cnvrg  image file", Thor::Shell::Color::BLUE

    res = @files.upload_cnvrg_image(image_path, image_name, secret)
    if res
      say "Successfully uploaded cnvrg image file", Thor::Shell::Color::GREEN

    else
      say "Couldn't upload cnvrg image file", Thor::Shell::Color::RED
    end
  rescue => e
    puts e
    puts e.backtrace
  end


end

#upload_data(sync = false, direct = false) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1257

def upload_data(sync = false, direct = false)

  begin
    verify_logged_in(true)
    log_start(__method__, args, options)
    dataset_dir = is_cnvrg_dir(Dir.pwd)
    @dataset = Dataset.new(dataset_dir)

    @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
    ignore = options[:ignore] || []
    if !@dataset.update_ignore_list(ignore)
      log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
    end
    result = @dataset.compare_idx(false)

    commit = result["result"]["commit"]
    if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?

      log_message("Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::BLUE)
      exit(1)
    end

    log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, options["verbose"] ? true : false)
    result = result["result"]["tree"]
    # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
    #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
    #   if all.size == 1
    #     num = "conflict"
    #   else
    #     num =  "conflicts"
    #   end
    #   say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
    #   say "#{all.join("\n")}"
    #   say "Please fix #{num}, and retry", Thor::Shell::Color::RED
    #   exit(1)
    #
    # en
    check = Helpers.checkmark()

    if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
      log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
      return true
    end
    update_count = 0
    update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
    successful_updates = []
    successful_deletions = []
    if options["verbose"]
      if update_total == 1
        log_message("Updating #{update_total} file", Thor::Shell::Color::BLUE)
      else
        log_message("Updating #{update_total} files", Thor::Shell::Color::BLUE)
      end
    else
      log_message("Syncing files", Thor::Shell::Color::BLUE, ((options["sync"] or sync)) ? false : true)

    end

    # Start commit

    commit_sha1 = @files.start_commit(false)["result"]["commit_sha1"]
    # upload / update
    begin
      (result["added"] + result["updated_on_local"]).each do |f|
        absolute_path = "#{@dataset.local_path}/#{f}"
        relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
        if File.directory?(absolute_path)
          resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
          if resDir
            update_count += 1
            successful_updates << relative_path
          end
        else
          res = @files.upload_file(absolute_path, relative_path, commit_sha1)

          if res
            update_count += 1
            successful_updates << relative_path
          else
            @files.rollback_commit(commit_sha1)
            log_message("Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
            exit(0)
          end
        end
      end

      # delete
      deleted = update_deleted(result["deleted"])
      deleted.each do |f|
        relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
        if relative_path.end_with?("/")
          if @files.delete_dir(f, relative_path, commit_sha1)
            # update_count += 1
            successful_updates << relative_path
          end
        else
          if @files.delete_file(f, relative_path, commit_sha1)
            # update_count += 1
            successful_updates << relative_path
          end
        end
      end

    rescue SignalException
      @files.rollback_commit(commit_sha1)
      say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
      exit(0)
    rescue => e
      log_message("Exception while trying to upload, Rolling back", Thor::Shell::Color::RED)
      log_error(e)
      @files.rollback_commit(commit_sha1)
      exit(0)
    end
    if !result["deleted"].nil? and !result["deleted"].empty?
      update_count += result["deleted"].size
    end
    if update_count == update_total
      res = @files.end_commit(commit_sha1,false)
      if (Cnvrg::CLI.is_response_success(res, false))
        # save idx
        begin
          list_files = []
          list_files.concat successful_deletions
          list_files.concat successful_updates

          @dataset.update_idx_with_files_commits!(list_files, res["result"]["commit_time"])

          @dataset.update_idx_with_commit!(commit_sha1)
        rescue => e
          log_message("Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED)
          log_error(e)
          @files.rollback_commit(commit_sha1)
          exit(1)

        end
        if options["verbose"]
          log_message("#{check} Done", Thor::Shell::Color::BLUE)
          if successful_updates.size > 0
            log_message("Updated:", Thor::Shell::Color::GREEN)
            suc = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
            log_message(suc.join("\n"), Thor::Shell::Color::GREEN)
          end
          if successful_deletions.size > 0
            log_message("Deleted:", Thor::Shell::Color::GREEN)
            del = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
            log_message(del.join("\n"), Thor::Shell::Color::GREEN)
          end
          log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
        else
          if (options["sync"] or sync) and direct
            log_message("#{check} Syncing dataset completed successfully", Thor::Shell::Color::GREEN)

          else
            log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)

          end

        end

      else
        @files.rollback_commit(commit_sha1)
        log_message("Error: Couldn't commit. \nRolling Back all changes.", Thor::Shell::Color::RED)
      end
    else
      log_message("Error: Uploaded only #{update_count}/#{update_total} files, \nRolling back", Thor::Shell::Color::RED)

      @files.rollback_commit(commit_sha1)
    end
  rescue => e
    log_message("Error occurd, \nAborting", Thor::Shell::Color::RED)
    log_error(e)

    @files.rollback_commit(commit_sha1)
    exit(1)
  rescue SignalException

    say "\nAborting", Thor::Shell::Color::BLUE
    say "\nRolling back all changes", Thor::Shell::Color::BLUE
    @files.rollback_commit(commit_sha1)
    exit(1)
  end

end

#upload_data_files(new_commit, *files) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 2121

def upload_data_files(new_commit, *files)
  begin
  verify_logged_in(true)
  log_start(__method__, args, options)
  dataset_dir = is_cnvrg_dir(Dir.pwd)
  @dataset = Dataset.new(dataset_dir)
  @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
  new_commit ||= @dataset.get_next_commit
  partial_commit = @dataset.get_partial_commit
  if new_commit.blank?
    log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
    return false
  end
  force = options[:force] || false
  chunk_size = options[:chunk_size]
  chunk_size = [chunk_size, 1].max
  new_branch = options["new_branch"] || false
  new_tree = {}
  force = options["force"] || false
  parallel_threads = options["parallel"] || ParallelThreads
  all_files = @dataset.list_all_files
  progressbar = ProgressBar.create(:title => "Upload Progress",
                                   :progress_mark => '=',
                                   :format => "%b>>%i| %p%% %t",
                                   :starting_at => 0,
                                   :total => all_files.length,
                                   :autofinish => true)
  files_uploaded = 0
  all_files.each_slice(chunk_size).each do |list_files|
    Cnvrg::Logger.log_info("Uploading files into #{@dataset.slug}, #{files_uploaded} files uploaded")
    temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
    upload_resp = @files.upload_multiple_files(new_commit, temp_tree,
                                               threads: parallel_threads,
                                               force: force,
                                               new_branch: new_branch,
                                               progressbar: progressbar,
                                               partial_commit: partial_commit)
    files_uploaded += upload_resp
    temp_tree.each do |k, v|
      new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
    end
  end
  @dataset.write_tree(new_tree) #we dont want to re-run it every time so just on finish.
  rescue => e
    Cnvrg::Logger.log_error(e)
    raise e
  end
  return files_uploaded
end

#upload_data_new(new_branch, verbose, sync, force, tags, chunk_size) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1973

def upload_data_new(new_branch,  verbose,sync,force, tags, chunk_size)
  begin
  commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size
  upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
  if upload_res < 0
    return
  end
  invoke :end_commit_data,[commit, success: true, uploaded_files: upload_res] , :new_branch=>new_branch,  :force =>force
  if tags
    log_message('Uploading Tags', Thor::Shell::Color::BLUE)
    dataset_dir = is_cnvrg_dir(Dir.pwd)
    @dataset = Dataset.new(dataset_dir)
    begin
      tag_file = File.open(options[:tags], "r+")
      status = @dataset.upload_tags_via_yml(tag_file)
    rescue
      log_message('Tags file not found', Thor::Shell::Color::RED)
      return
    end
    if status
      log_message('Tags are successfully uploaded', Thor::Shell::Color::GREEN)
    else
      log_message('There was some error in uploading Tags', Thor::Shell::Color::RED)
    end
  end
  rescue => e
    Cnvrg::Logger.log_error(e)
    say "\nAborting", Thor::Shell::Color::BLUE
    dataset_dir = is_cnvrg_dir(Dir.pwd)
    return false if dataset_dir.blank?
    @dataset = Dataset.new(dataset_dir)
    return false
  rescue SignalException => e
    Cnvrg::Logger.log_error(e)
    say "\nAborting", Thor::Shell::Color::BLUE
    dataset_dir = is_cnvrg_dir(Dir.pwd)
    return false if dataset_dir.blank?
    @dataset = Dataset.new(dataset_dir)
    return false
  end
end

#upload_data_tar(ignore, verbose, sync, no_compression) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 1448

def upload_data_tar(ignore, verbose, sync, no_compression)

  begin
    verify_logged_in(true)
    log_start(__method__, args, options)
    dataset_dir = is_cnvrg_dir(Dir.pwd)

    @dataset = Dataset.new(dataset_dir)

    @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
    if !@dataset.update_ignore_list(ignore)
      log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::RED)
      exit(1)
    end
    log_message("Checking dataset", Thor::Shell::Color::BLUE)
    ignore_list = @dataset.get_ignore_list()
    local_idx = @dataset.generate_idx(ignore_list)

    result = @dataset.compare_idx(false, commit = @dataset.last_local_commit, local_idx = local_idx)

    commit = result["result"]["commit"]
    if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?

      log_message("Remote server has an updated version, please run `cnvrg data download` first", Thor::Shell::Color::BLUE)
      exit(1)
    end

    log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, verbose)
    result = result["result"]["tree"]
    check = Helpers.checkmark()

    if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
      log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN, (sync ? false : true))
      return true
    end
    update_count = 0
    update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
    successful_updates = []
    successful_deletions = []

    # Start commit
    res = @files.start_commit(false)["result"]
    commit_sha1 = res["commit_sha1"]
    commit_time = res["commit_time"]
    # upload / update
    begin
      (result["added"] + result["updated_on_local"]).each do |f|
        relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
        successful_updates << relative_path
        update_count += 1
      end

      # delete
      deleted = update_deleted(result["deleted"])
      deleted.each do |f|
        relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
        successful_updates << relative_path
      end
      @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)

      log_message("Compressing data", Thor::Shell::Color::BLUE)

      home_dir = File.expand_path('~')
      compression_path = get_compression_path
      tar_path = "#{compression_path}#{@dataset.slug}_#{commit_sha1}.tar.gz"
      tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
      files_to_upload = result["added"] + result["updated_on_local"]
      if File.exist? (@dataset.local_path + "/.cnvrgignore")
        files_to_upload << ".cnvrgignore"
      end
      tar_files = (files_to_upload).join("\n")
      File.open(tar_files_path, 'w') {|f| f.write tar_files}
      ignore_files_path = nil
      if !ignore_list.nil?
        ignore_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}_ignore.txt"
        File.open(ignore_files_path, 'w') {|f| f.write ignore_list.join("\n")}
      end
      is_tar = create_tar(dataset_dir, tar_path, tar_files_path, no_compression, ignore_files_path)
      if !is_tar
        log_message("ERROR: Couldn't compress data", Thor::Shell::Color::RED)
        FileUtils.rm_rf([tar_path]) if File.exist? tar_path
        FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
        FileUtils.rm_rf([ignore_files_path]) if !ignore_files_path.nil? and File.exist? ignore_files_path

        @files.rollback_commit(commit_sha1)
        log_message("Rolling Back all changes.", Thor::Shell::Color::RED)
        exit(1)
      end
      log_message("Uploading data", Thor::Shell::Color::BLUE)
      log_file = "#{home_dir}/.cnvrg/tmp/upload_#{File.basename(tar_path)}.log"
      res = false
      res = @files.upload_tar_file(tar_path, tar_path, commit_sha1)

      if res
        log_message("Commiting data", Thor::Shell::Color::BLUE)

        cur_idx = @dataset.get_idx.to_h

        res = @files.end_commit_tar(commit_sha1, cur_idx)
        if !Cnvrg::CLI.is_response_success(res, false)
          FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
          FileUtils.rm_rf([tar_path]) if File.exist? tar_path


          @files.rollback_commit(commit_sha1)
          log_message("Can't commit, Rolling Back all changes.", Thor::Shell::Color::RED)
          exit(1)
        end

      else
        if File.exist? log_file
          @files.upload_data_log_file(log_file, log_file, commit_sha1)
        end


        FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
        FileUtils.rm_rf([tar_path]) if File.exist? tar_path


        @files.rollback_commit(commit_sha1)
        log_message("Can't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
        log_message("Upload error log: #{log_file}", Thor::Shell::Color::RED)

        exit(1)
      end


      # delete
      FileUtils.rm_rf([tar_path, tar_files_path])

    rescue SignalException
      FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
      FileUtils.rm_rf([tar_path]) if File.exist? tar_path
      if File.exist? log_file
        @files.upload_data_log_file(log_file, log_file, commit_sha1)
      end


      @files.rollback_commit(commit_sha1)
      say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
      exit(0)
    rescue => e
      log_error(e)
      # if !Cnvrg::Helpers.internet_connection?
      #   say "Seems there is no internet connection", Thor::Shell::Color::RED
      #
      # end
      if File.exist? log_file
        @files.upload_data_log_file(log_file, log_file, commit_sha1)
      end
      FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
      FileUtils.rm_rf([tar_path]) if File.exist? tar_path

      @files.rollback_commit(commit_sha1)
      log_message("Exception while trying to upload, \nRolling back,\n look at the log for more details", Thor::Shell::Color::RED)
      log_message("Error log: #{log_file}", Thor::Shell::Color::RED)


      exit(0)
    end
    log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)


  rescue => e

    log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
    log_error(e)
    @files.rollback_commit(commit_sha1)
    exit(1)
  rescue SignalException

    say "\nAborting", Thor::Shell::Color::BLUE
    say "\nRolling back all changes", Thor::Shell::Color::BLUE
    @files.rollback_commit(commit_sha1)
    exit(1)
  end


end

#upload_image(image_name, image_path) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4861

def upload_image(image_name,image_path)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)

    @image = Cnvrg::Images.new()
    say "Uploading new docker image file", Thor::Shell::Color::BLUE
    workdir = options[:workdir]
    description = options[:description]
    user = options[:user]
    is_gpu = options[:gpu]
    res = @image.upload_docker_image(image_path, image_name, workdir, user, description, is_gpu)
    if res["status"] == 200
      image_slug = res["id"]
      owner = CLI.get_owner
      image_url = "#{Cnvrg::Helpers.remote_url}/#{owner}/settings/images/#{image_slug}"
      log_message("Successfully uploaded image: #{image_url}", Thor::Shell::Color::GREEN, true)


    else
      log_message("Couldn't upload image: #{image_name}", Thor::Shell::Color::RED, true)

    end
  rescue => e
    log_error(e)
  end


end

#upload_image_old(image_id, is_public, is_base, *message) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4491

def upload_image_old(image_id, is_public, is_base, *message)
  verify_logged_in(true)
  log_start(__method__, args, options)
  image = Docker::Image.get(image_id)
  project_home = get_project_home
  @project = Project.new(project_home)
  last_local_commit = @project.last_local_commit
  image_name = @project.slug + "#{last_local_commit}"
  path = File.expand_path('~') + "/.cnvrg/tmp/#{image_name}.tar"
  owner = Cnvrg::CLI.get_owner()
  if !message.nil? or !message.empty?
    message = message.join(" ")
  end

  log_message("Saving image's current state", Thor::Shell::Color::BLUE)
  image.save(path)

  begin
    log_message("Compressing image file to upload", Thor::Shell::Color::BLUE)
    gzipRes = system("gzip -f #{path}")
    if !gzipRes

      log_message("Couldn't create tar file from image", Thor::Shell::Color::RED)
      exit(1)
    end
    path = path + ".gz"
    @files = Cnvrg::Files.new(owner, "")

    exit_status = $?.exitstatus
    if exit_status == 0
      log_message("Uploading image file", Thor::Shell::Color::BLUE)

      diff = container_changes(Dir.pwd)
      res = @files.upload_image(path, image_name, owner, is_public, is_base, diff[1], diff[0], diff[2], message, image.commit_id)
      if res
        File.delete(path)
        image_loc = is_project_with_docker(Dir.pwd)
        image_loc.update_slug(res["result"]["id"])

        checks = Helpers.checkmark()
        log_message("#{checks} Done", Thor::Shell::Color::GREEN)
      else
        log_message("Couldn't upload image", Thor::Shell::Color::RED)

      end
    else
      log_message("Couldn't create image file for: #{image_name}", Thor::Shell::Color::RED)
      exit(1)
    end
  rescue => e
    log_message("Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED)
    log_error(e)
  rescue SignalException

    say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
    exit(1)
  end
end

#upload_log ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 4552

def upload_log()
  log_path = '/home/ds/app/uwsgi.log'
  loglines = File.new(log_path).readlines
  logs = loglines.select {|x| x.start_with? "cnvrg_app:"}.collect {|x| x.strip}

end

#verify_datasets(dataset_titles, timeout = 0) ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 838

def verify_datasets(dataset_titles, timeout=0)
  begin
    verify_logged_in(false)
    log_start(__method__, args, options)
    log_message("Verifying datasets #{dataset_titles}", Thor::Shell::Color::BLUE)
    verified = Dataset.verify_datasets(dataset_titles, timeout)
    log_message("All datasets are verified", Thor::Shell::Color::BLUE) if verified
    log_message("Failed to verify datasets", Thor::Shell::Color::RED) if !verified
    exit(1) if !verified

  rescue SignalException
    say "\nAborting", Thor::Shell::Color::RED
    exit(1)
  end
end

#version ⇒ `Object`

# File 'lib/cnvrg/cli.rb', line 212

def version
  puts Cnvrg::VERSION

end

Class: Cnvrg::CLI

Constant Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(*args) ⇒ CLI

Class Method Details

.is_thor_reserved_word?(word, type) ⇒ Boolean

Instance Method Details

#auth ⇒ Object

#build(*cmd) ⇒ Object

#build_image(image_name) ⇒ Object

#check_spot ⇒ Object

#clone(project_url) ⇒ Object

#clone_data(dataset_url, only_tree = false, commit = nil, query = nil, read = false, remote = false) ⇒ Object

#clone_data_query(dataset_url, query = nil) ⇒ Object

#commit_image ⇒ Object

#config_flask_remote(image_name, port = 80) ⇒ Object

#config_flask_remote_gpu(image_name, port = 80) ⇒ Object

#config_netrc(container) ⇒ Object

#config_remote(image_name, port = 7654, tensport = 6006) ⇒ Object

#config_remote_gpu(image_name, port = 7654, tensport = 6006) ⇒ Object

#create_volume ⇒ Object

#data_init_container(owner, dataset_slug, dataset_name) ⇒ Object

#data_jump(*commit_sha1) ⇒ Object

#data_put(dataset_url, files: [], dir: '', chunk_size: 1000) ⇒ Object

#delete_data(dataset_slug) ⇒ Object

#deploy(file_to_run, function) ⇒ Object

#download(sync = false, ignore_list = "", in_exp = false) ⇒ Object

#download_built_image(image_name, image_slug) ⇒ Object

#download_cnvrg_image(image_name, secret) ⇒ Object

#download_data(verbose, sync, path = Dir.pwd, in_dir = true) ⇒ Object

#download_data_new(verbose = false, new_branch = false, sync = false, commit = nil, all_files = true) ⇒ Object

#download_file_data(file_path, *dataset_path) ⇒ Object

#download_in_git(*commit_sha1) ⇒ Object

#download_tags_yaml ⇒ Object

#end_commit_data(commit, success: true, uploaded_files: 0) ⇒ Object

#exec(*cmd) ⇒ Object

#exec_container(container_id, *cmd) ⇒ Object

#exec_remote(*cmd) ⇒ Object

#experiments ⇒ Object

#get_machine ⇒ Object

#git_clone(slug, owner) ⇒ Object

#init_data(public, bucket: nil) ⇒ Object

#init_data_container(container) ⇒ Object

#install_python_libraries(*lib) ⇒ Object

#install_system_libraries(*command_to_run) ⇒ Object

#jump(commit_sha1) ⇒ Object

#link ⇒ Object

#link_git(project_url) ⇒ Object

#list_commits ⇒ Object

#list_dataset ⇒ Object

#list_dataset_commits ⇒ Object

#list_files_dataset ⇒ Object

#list_images ⇒ Object

#list_machines ⇒ Object

#login ⇒ Object

#logout ⇒ Object

#logs ⇒ Object

#me ⇒ Object

#new(project_name) ⇒ Object

#notebook ⇒ Object

#notebook_stop(notebook_slug) ⇒ Object

#port_container(container_id) ⇒ Object

#pull_image(image_name) ⇒ Object

#push(*name) ⇒ Object

#queries ⇒ Object

#query_files(query) ⇒ Object

#remote_notebook ⇒ Object

#remote_notebook_old ⇒ Object

#revert_exp ⇒ Object

#run(*cmd) ⇒ Object

#run_notebook ⇒ Object

#search_libraries(library) ⇒ Object

#set_api_url(url) ⇒ Object

#set_compression_path(*compression_path) ⇒ Object

#set_data_url(dataset_url) ⇒ Object

#set_default_owner ⇒ Object

#set_image(docker_image) ⇒ Object

#set_remote_api_url(owner, current_user, url) ⇒ Object

#initialize(*args) ⇒ `CLI`

.is_thor_reserved_word?(word, type) ⇒ `Boolean`

#auth ⇒ `Object`

#build(*cmd) ⇒ `Object`

#build_image(image_name) ⇒ `Object`

#check_spot ⇒ `Object`

#clone(project_url) ⇒ `Object`

#clone_data(dataset_url, only_tree = false, commit = nil, query = nil, read = false, remote = false) ⇒ `Object`

#clone_data_query(dataset_url, query = nil) ⇒ `Object`

#commit_image ⇒ `Object`

#config_flask_remote(image_name, port = 80) ⇒ `Object`

#config_flask_remote_gpu(image_name, port = 80) ⇒ `Object`

#config_netrc(container) ⇒ `Object`

#config_remote(image_name, port = 7654, tensport = 6006) ⇒ `Object`

#config_remote_gpu(image_name, port = 7654, tensport = 6006) ⇒ `Object`

#create_volume ⇒ `Object`

#data_init_container(owner, dataset_slug, dataset_name) ⇒ `Object`

#data_jump(*commit_sha1) ⇒ `Object`

#data_put(dataset_url, files: [], dir: '', chunk_size: 1000) ⇒ `Object`

#delete_data(dataset_slug) ⇒ `Object`

#deploy(file_to_run, function) ⇒ `Object`

#download(sync = false, ignore_list = "", in_exp = false) ⇒ `Object`

#download_built_image(image_name, image_slug) ⇒ `Object`

#download_cnvrg_image(image_name, secret) ⇒ `Object`

#download_data(verbose, sync, path = Dir.pwd, in_dir = true) ⇒ `Object`

#download_data_new(verbose = false, new_branch = false, sync = false, commit = nil, all_files = true) ⇒ `Object`

#download_file_data(file_path, *dataset_path) ⇒ `Object`

#download_in_git(*commit_sha1) ⇒ `Object`

#download_tags_yaml ⇒ `Object`

#end_commit_data(commit, success: true, uploaded_files: 0) ⇒ `Object`

#exec(*cmd) ⇒ `Object`

#exec_container(container_id, *cmd) ⇒ `Object`

#exec_remote(*cmd) ⇒ `Object`

#experiments ⇒ `Object`

#get_machine ⇒ `Object`

#git_clone(slug, owner) ⇒ `Object`

#init_data(public, bucket: nil) ⇒ `Object`

#init_data_container(container) ⇒ `Object`

#install_python_libraries(*lib) ⇒ `Object`

#install_system_libraries(*command_to_run) ⇒ `Object`

#jump(commit_sha1) ⇒ `Object`

#link ⇒ `Object`

#link_git(project_url) ⇒ `Object`

#list_commits ⇒ `Object`

#list_dataset ⇒ `Object`

#list_dataset_commits ⇒ `Object`

#list_files_dataset ⇒ `Object`

#list_images ⇒ `Object`

#list_machines ⇒ `Object`

#login ⇒ `Object`

#logout ⇒ `Object`

#logs ⇒ `Object`

#me ⇒ `Object`

#new(project_name) ⇒ `Object`

#notebook ⇒ `Object`

#notebook_stop(notebook_slug) ⇒ `Object`

#port_container(container_id) ⇒ `Object`

#pull_image(image_name) ⇒ `Object`

#push(*name) ⇒ `Object`

#queries ⇒ `Object`

#query_files(query) ⇒ `Object`

#remote_notebook ⇒ `Object`

#remote_notebook_old ⇒ `Object`

#revert_exp ⇒ `Object`

#run(*cmd) ⇒ `Object`

#run_notebook ⇒ `Object`

#search_libraries(library) ⇒ `Object`

#set_api_url(url) ⇒ `Object`

#set_compression_path(*compression_path) ⇒ `Object`

#set_data_url(dataset_url) ⇒ `Object`

#set_default_owner ⇒ `Object`

#set_image(docker_image) ⇒ `Object`

#set_remote_api_url(owner, current_user, url) ⇒ `Object`

#set_remote_login(current_user, owner, url, email, secret) ⇒ `Object`

#show ⇒ `Object`

#show_libraries ⇒ `Object`

#snap_data ⇒ `Object`

#start_commit_data ⇒ `Object`

#status ⇒ `Object`

#stop_container(container_id) ⇒ `Object`