Module: CodeRunner::LoadLeveler

Included in:
BlueJoule
Defined in:
lib/coderunner/system_modules/load_leveler.rb

Instance Method Summary collapse

Instance Method Details

#batch_script ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 66

def batch_script

	nodes, ppn = @nprocs.split(/x/)
	eputs "Warning: You must use 128 nodes or greater on a BlueGene Q: This may fail." if  nodes.to_i < 128
	eputs "Warning: Underuse of nodes (#{ppn} cores per node instead of #{max_ppn})" if ppn.to_i < max_ppn 
	eputs "Warning: processes per node excedes cores per node: #{max_ppn}" if ppn.to_i > max_ppn
	ppn ||= max_ppn
	if @wall_mins
		ep @wall_mins
		hours = (@wall_mins / 60).floor
		mins = @wall_mins.to_i % 60
		secs = ((@wall_mins - @wall_mins.to_i) * 60).to_i
		eputs "Allotted wall time is " + sprintf("%02d:%02d:%02d", hours, mins, secs)
	else
		raise "Error: no wall clock time specified."
	end

	return <<EOF
#!/bin/bash

## job requirements for load leveler
######################################################################
\#@bg_size=#{nodes}
\#@job_type=bluegene
\#@class=prod
\#@executable=#{batch_script_file}
\#@environment=COPY_ALL
\#@output=#{executable_name}.#{job_identifier}.$(jobid).output.txt
\#@error=#{executable_name}.#{job_identifier}.$(jobid).error.txt
\#@wall_clock_limit=#{sprintf("%02d:%02d:%02d", hours, mins, secs)}
\#@notification=complete
\#@queue

## commands to be executed
######################################################################
## This is any custom configuration required for the code:
#{code_run_environment}
printenv
echo "Submitting #{nodes}x#{ppn} job on #{CodeRunner::SYS} for project (#@project)..."


EOF

end

#batch_script_file ⇒ Object 



58
59
60
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 58

def batch_script_file
	"#{executable_name}_#{job_identifier}.sh"
end

#cancel_job ⇒ Object 



111
112
113
114
115
116
117
118
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 111

def cancel_job
	if ((prefix = ENV['CODE_RUNNER_LAUNCHER']).size > 0 rescue false)
		 fname = ENV['HOME'] + "/.coderunner_to_launch_#{prefix}/#{$$}.stop"
		 File.open(fname, 'w'){|file| file.puts "\n"}
	else
		`llcancel #{@job_no}`
	end
end

#error_file ⇒ Object 



120
121
122
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 120

def error_file
	return "#{executable_name}.#{job_identifier}.#@job_no.error.txt"
end

#execute ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 42

def execute
	if ((prefix = ENV['CODE_RUNNER_LAUNCHER']).size > 0 rescue false)
		launch_id = "#{Time.now.to_i}#{$$}"
		fname = ENV['HOME'] + "/.coderunner_to_launch_#{prefix}/#{launch_id}"
		File.open(fname + '.start', 'w'){|file| file.puts "cd #{Dir.pwd};#{run_command}"}
		sleep 1 until FileTest.exist? fname + '.pid'
		pid = File.read(fname + '.pid').to_i
		FileUtils.rm fname + '.pid'
		return pid
	else
		File.open(batch_script_file, 'w'){|file| file.puts batch_script + "\n" + run_command + "\n"}
		%x[llsubmit #{batch_script_file}].to_i
		return nil
	end
end

#get_run_status(job_no, current_status) ⇒ Object 



128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 128

def get_run_status(job_no, current_status)
	if ((prefix = ENV['CODE_RUNNER_LAUNCHER']).size > 0 rescue false)
		return :Unknown
	end
	line = current_status.split(/\n/).grep(Regexp.new(job_no.to_s))[0]
	unless line
		return :Unknown
	else 
		if line =~ /\sS|H\s/
			return :Held
		elsif line =~ /\sI\s/
			return :Queueing
		elsif line =~ /\sR\s/
			return :Running
		else
			ep 'line', line
			raise 'Could not get run status'
		end
	end
end

#max_ppn ⇒ Object 



62
63
64
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 62

def max_ppn
	raise "Please define max_ppn for your system"
end

#nodes ⇒ Object

end



19
20
21
22
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 19

def nodes
		nodes, ppn = @nprocs.split(/x/)
		nodes
end

#nprocstot ⇒ Object 



27
28
29
30
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 27

def nprocstot
		nodes, ppn = @nprocs.split(/x/)
		nprocstot = nodes.to_i * ppn.to_i
end

#output_file ⇒ Object 



124
125
126
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 124

def output_file
	return "#{executable_name}.#{job_identifier}.#@job_no.output.txt"
end

#ppn ⇒ Object 



23
24
25
26
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 23

def ppn
		nodes, ppn = @nprocs.split(/x/)
		ppn
end

#queue_status ⇒ Object 



6
7
8
9
10
11
12
13
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 6

def queue_status
	if ((prefix = ENV['CODE_RUNNER_LAUNCHER']).size > 0 rescue false)
		%x[cat #{ENV['HOME']}/.coderunner_to_launch_#{prefix}/queue_status.txt]  +
		%x[cat #{ENV['HOME']}/.coderunner_to_launch_#{prefix}/queue_status2.txt] 
	else
		%x[llq -WX all | grep $USER].gsub(/^bglogin\d\./, '')
	end
end

#queue_wait_attempts ⇒ Object

Here we therefore wait 40 seconds!



152
153
154
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 152

def queue_wait_attempts
	200
end

#run_command ⇒ Object 



31
32
33
34
35
36
37
38
39
40
 
# File 'lib/coderunner/system_modules/load_leveler.rb', line 31

def run_command
# 		"qsub #{batch_script_file}"
	if (ENV['CODE_RUNNER_LAUNCHER'].size > 0 rescue false)
		return %[mpiexec -np #{@nprocs} #{executable_location}/#{executable_name} #{parameter_string} > #{output_file} 2> #{error_file}]
	else
		#nodes, ppn = @nprocs.split(/x/)
		#nprocstot = nodes.to_i * ppn.to_i
		"runjob --env-all --exe #{executable_location}/#{executable_name} --np #{nprocstot} --ranks-per-node #{ppn} --args \"#{parameter_string}\""
	end
end