Class: Cnvrg::Experiment

Inherits:
Object
  • Object
show all
Defined in:
lib/cnvrg/experiment.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(owner, project_slug) ⇒ Experiment

Returns a new instance of Experiment.



9
10
11
12
13
14
15
16
17
 
# File 'lib/cnvrg/experiment.rb', line 9

def initialize(owner, project_slug)
  @project_slug = project_slug
  @owner = owner
  @base_resource = "users/#{owner}/projects/#{project_slug}/"
  @slug = nil
  @sync_before_terminate = nil
  @sync_delay_time = nil
  @output_dir = nil
end

Instance Attribute Details

#output_dir ⇒ Object (readonly)

Returns the value of attribute output_dir.



7
8
9
 
# File 'lib/cnvrg/experiment.rb', line 7

def output_dir
  @output_dir
end

#slug ⇒ Object (readonly)

Returns the value of attribute slug.



4
5
6
 
# File 'lib/cnvrg/experiment.rb', line 4

def slug
  @slug
end

#sync_before_terminate ⇒ Object (readonly)

Returns the value of attribute sync_before_terminate.



5
6
7
 
# File 'lib/cnvrg/experiment.rb', line 5

def sync_before_terminate
  @sync_before_terminate
end

#sync_delay_time ⇒ Object (readonly)

Returns the value of attribute sync_delay_time.



6
7
8
 
# File 'lib/cnvrg/experiment.rb', line 6

def sync_delay_time
  @sync_delay_time
end

Instance Method Details

#end(output, exit_status, end_commit, cpu_average, memory_average, end_time: nil) ⇒ Object 



130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
 
# File 'lib/cnvrg/experiment.rb', line 130

def end(output, exit_status, end_commit, cpu_average, memory_average, end_time: nil)
  #if remote try to remove
  tries = 0
  success = false
  end_time ||= Time.now
  while tries < 10 and success.blank?
    sleep (tries*rand) ** 2 ### exponential backoff
                            ## this call is super important so we cant let it crash.

    tries += 1
    response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', {output: output, exp_slug: @slug,
                                                                            exit_status: exit_status, end_commit: end_commit,
                                                                            cpu_average: cpu_average, memory_average: memory_average, end_time: end_time})
    success = Cnvrg::CLI.is_response_success(response,false)
  end

  begin
    FileUtils.rm_rf(["/home/ds/.cnvrg/tmp/exec.log"])
  rescue

  end
end

#end_notebook_session(notebook_slug) ⇒ Object 



48
49
50
51
52
53
54
 
# File 'lib/cnvrg/experiment.rb', line 48

def end_notebook_session(notebook_slug)
  res = Cnvrg::API.request(@base_resource + "notebook/#{notebook_slug}/stop", 'GET')
  Cnvrg::CLI.is_response_success(res,false)

  return res

end

#exec_remote(command, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp, grid, path_to_cmd, data, data_commit, periodic_sync, sync_before_terminate, max_time, ds_sync_options = 0, output_dir = nil, data_query = nil, git_commit = nil, git_branch = nil, restart_if_stuck = nil, local_folders = nil, title = nil, datasets = nil) ⇒ Object 



91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/cnvrg/experiment.rb', line 91

def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
                sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
                git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil)
  response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
                                                                                                        commit_sha1: commit_to_run,
                                                                                                        instance_type: instance_type,
                                                                                                        scheduling_query:scheduling_query,
                                                                                                        local_timestamp:local_timestamp,
                                                                                                        datasets: datasets,
                                                                                                        grid: grid,
                                                                                                        path_to_cmd:path_to_cmd,dataset_slug:data,
                                                                                                        dataset_commit: data_commit,max_time:max_time,
                                                                                                        periodic_sync:periodic_sync, sync_before_terminate:sync_before_terminate,
                                                                                                        dataset_sync_options:ds_sync_options,output_dir:output_dir,
                                                                                                        dataset_query:data_query,git_commit:git_commit,git_branch:git_branch,
                                                                                                        restart_if_stuck:restart_if_stuck, local_folders: local_folders, title:title })

  return response
end

#get_machine_activity(working_dir) ⇒ Object 



78
79
80
81
82
83
84
85
86
87
88
89
 
# File 'lib/cnvrg/experiment.rb', line 78

def get_machine_activity(working_dir)
  begin
    machine_activity = File.open("#{working_dir}/.cnvrg/machine_activity", "rb").read
    machine_activity = machine_activity.to_s.strip
    ma_id = machine_activity.to_i
    return ma_id
  rescue
    return nil
  end


end

#get_notebook_slug(proj_dir) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/cnvrg/experiment.rb', line 66

def get_notebook_slug(proj_dir)
  begin
    notebook_slug = File.open(proj_dir + "/.cnvrg/notebook_slug", "rb").read
    notebook_slug = notebook_slug.gsub("/n", "")
    notebook_slug = notebook_slug.to_s.strip
    return notebook_slug
  rescue
    return nil
  end


end

#remote_notebook(instance_type, commit, data, data_commit, notebook_type, ds_sync_options = 0, data_query = nil, image = nil, datasets = nil) ⇒ Object 



110
111
112
113
114
115
116
117
 
# File 'lib/cnvrg/experiment.rb', line 110

def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil, image = nil, datasets = nil)
  response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', {instance_type: instance_type,dataset_slug:data,
                                                                                                      dataset_commit: data_commit,image_slug:image,
                                                                                                      datasets: datasets,
                                                                                                      commit:commit,notebook_type:notebook_type,dataset_sync_options:ds_sync_options,
                                                                                                      dataset_query:data_query})
  return response
end

#restart_spot_instance ⇒ Object 



153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/cnvrg/experiment.rb', line 153

def restart_spot_instance

  restart = false
  begin
  url = URI.parse('http://169.254.169.254/latest/meta-data/spot/termination-time')
  req = Net::HTTP::Get.new(url.to_s)
  res = Net::HTTP.start(url.host, url.port) {|http|
    http.request(req)
  }
  unless res.body.include? "404"
    restart = true
  end
  if res.body.include? "Empty reply from server"
    restart = false
  end
  rescue
    restart = false

  end

  return restart

end

#send_machine_stats(stats) ⇒ Object 



125
126
127
128
 
# File 'lib/cnvrg/experiment.rb', line 125

def send_machine_stats(stats)
  response = Cnvrg::API.request(@base_resource + "experiment/upload_stats", "POST", {exp_slug: @slug, stats: stats.map{|s| s.merge!({time: Time.now})}})
  Cnvrg::CLI.is_response_success(response,false)
end

#send_restart_request(sha1 = nil) ⇒ Object 



177
178
179
 
# File 'lib/cnvrg/experiment.rb', line 177

def send_restart_request(sha1=nil)
  Cnvrg::API.request(@base_resource + "experiment/check_spot_instance", 'POST', {exp_slug: @slug, end_commit: sha1})
end

#start(input, platform, machine_name, start_commit, name, email_notification, machine_activity, script_path, sync_before_terminate, periodic_sync) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/cnvrg/experiment.rb', line 19

def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity,script_path,
          sync_before_terminate, periodic_sync)

  res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
                           {input: input, platform: platform, machine_name: machine_name, start_commit: start_commit,
                            title: name, email_notification: email_notification, machine_activity: machine_activity,script_path:script_path})
  Cnvrg::CLI.is_response_success(res,false)

  @slug = res.to_h["result"].to_h["slug"]
  @sync_before_terminate = res.to_h["result"].to_h["sync_before_terminate"]
  @sync_delay_time = res.to_h["result"].to_h["sync_delay_time"]
  @output_dir = res.to_h["result"].to_h["output_dir"]
  return res

end

#start_notebook_session(kernel, start_commit, token, port, remote, notebook_path) ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/cnvrg/experiment.rb', line 35

def start_notebook_session(kernel, start_commit, token, port, remote, notebook_path)
  res = Cnvrg::API.request(@base_resource + "notebook/start_session", 'POST',
                           {kernel: kernel, start_commit: start_commit,
                             token: token, port: port, remote: remote, notebook_path: notebook_path})
  Cnvrg::CLI.is_response_success(res)

  @slug = res["result"]["id"]


  return @slug

end

#update_notebook_slug(proj_dir, slug) ⇒ Object 



55
56
57
58
59
60
61
62
63
64
 
# File 'lib/cnvrg/experiment.rb', line 55

def update_notebook_slug(proj_dir, slug)
  begin
  file = proj_dir+"/.cnvrg/notebook_slug"
  FileUtils.touch file

  File.open(file, "w+") { |f| f.write slug }
  rescue
  end

end

#upload_temp_log(temp_log) ⇒ Object 



119
120
121
122
123
 
# File 'lib/cnvrg/experiment.rb', line 119

def upload_temp_log(temp_log)
  response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', {output: temp_log,
                                                                                        exp_slug: @slug})
  Cnvrg::CLI.is_response_success(response,false)
end