Class: Stack

Inherits:
Object
  • Object
show all
Defined in:
lib/autoflow/stack.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(exec_folder, options) ⇒ Stack

PARSE TEMPLATE



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/autoflow/stack.rb', line 12

def initialize(exec_folder, options)
	Batch.set_general_attrib({
		:cpu => options[:cpus], 
		:mem => options[:memory], 
		:time => options[:time],
		:node => options[:node_type],
		:multinode => options[:use_multinode],
		:ntask => options[:use_ntasks],
		:additional_job_options => options[:additional_job_options]
	})
	@@folder_name = :program_name
	@@folder_name = :job_name if options[:key_name]
	@commands = {}	
	@variables = {}
	@persist_variables = {}
	@@all_jobs_relations = {}
	@exec_folder = exec_folder #TODO move this to queue_manager
	@do_retry = options[:retry]
	@options = options
	@workflow = options[:workflow]
	@external_variables= options[:Variables] 
	@jobs = {}
end

Instance Attribute Details

#exec_folder ⇒ Object

Returns the value of attribute exec_folder.



7
8
9
 
# File 'lib/autoflow/stack.rb', line 7

def exec_folder
  @exec_folder
end

#jobs ⇒ Object

Returns the value of attribute jobs.



7
8
9
 
# File 'lib/autoflow/stack.rb', line 7

def jobs
  @jobs
end

#persist_variables ⇒ Object

Returns the value of attribute persist_variables.



7
8
9
 
# File 'lib/autoflow/stack.rb', line 7

def persist_variables
  @persist_variables
end

Instance Method Details

#asign_folder(job) ⇒ Object

TODO move this to queue_manager



180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
 
# File 'lib/autoflow/stack.rb', line 180

def asign_folder(job) #TODO move this to queue_manager
	folder = nil
	if job.attrib[:folder]
		if @@folder_name == :program_name
			program = File.join(job.attrib[:exec_folder], job.parameters.split(' ', 2).first) 			
			count = 0
			folder = program + "_#{"%04d" % count}"
			while @@all_jobs_relations.values.include?(folder)
				folder = program + "_#{"%04d" % count}"
				count += 1
			end
		elsif @@folder_name == :job_name
			folder = File.join(job.attrib[:exec_folder], job.name)
		end
	else
		folder = job.attrib[:exec_folder]
	end
	@@all_jobs_relations[job.name.gsub(')','')] = folder
	job.attrib[:exec_folder] = folder 
end

#create_ids(nodes) ⇒ Object 



165
166
167
168
169
170
 
# File 'lib/autoflow/stack.rb', line 165

def create_ids(nodes)
	nodes.each_with_index do |node, i|
		node << i
	end
	return nodes
end

#draw(name, name_type) ⇒ Object 



238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
 
# File 'lib/autoflow/stack.rb', line 238

def draw(name, name_type)
	representation_type = '_structural'
	representation_type = '_semantic' if name_type.include?('t')
	if name_type.include?('b')
		representation_type << '_simplified'
		set = @commands
	else
		set = @jobs
	end
	name.gsub!(/\.\S+/,'')
	file = File.open(name+representation_type+'.dot','w')
	file.puts 'digraph G {', 'node[shape=box]'
	all_dependencies = []
	all_tag = []
	set.each do |id, tag|
		if name_type.include?('b')
			tag_name = tag.main_command.split(' ').first+"_#{tag.id}"
		else 
			tag_name = File.basename(tag.attrib[:exec_folder])
		end
		tag_name = id if name_type.include?('t')
		tag_name = tag_name + '(*)' if name_type.include?('b') && tag.iterator.length > 1
		
		all_tag << tag_name
		if tag.dependencies.length > 0
			tag.dependencies.each do |dependencie, type, string|
				if name_type.include?('b')
					dependencie_name = set[dependencie].main_command.split(' ').first+"_#{set[dependencie].id}"
				else 
					dependencie_name = File.basename(set[dependencie].attrib[:exec_folder])				
				end
				dependencie_name = dependencie if name_type.include?('t')

				dependencie_name = dependencie_name + '(*)' if name_type.include?('b') && set[dependencie].iterator.length > 1
				all_dependencies << dependencie_name

				file.puts "\"#{dependencie_name}\"-> \"#{tag_name}\""
			end
		else
			file.puts "\"#{tag_name}\"[color=black, peripheries=2, style=filled, fillcolor=yellow]"
		end
	end
	all_tag.keep_if{|tag| !all_dependencies.include?(tag)}
	all_tag.each do |tag|
		if name_type.include?('b')
			if !name_type.include?('f')
				tag = tag + '(*)'  if !tag.include?('(*)') && set[tag].iterator.length > 1
			else
				id = tag.reverse.split('_',2).first.reverse.to_i
				batch = nil
				set.each do |id,tag|
					batch = tag if tag.id = id 
				end
				tag = tag + '(*)'  if batch.iterator.length > 1
			end
		end
		file.puts "\"#{tag}\"[fontcolor=white, color=black, style=filled]"
	end
	file.puts '}'
	file.close
	system('dot -Tpdf '+name+representation_type+'.dot -o '+name+representation_type+'.pdf')
end

#get_jobs ⇒ Object 



201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/autoflow/stack.rb', line 201

def get_jobs
	jobs =[]
	@commands.each do |name, batch|
		next if batch.has_jobs? #parent batch (intermediates)
		batch.get_jobs.each do |j|
			folder = asign_folder(j) #TODO move this to queue_manager
			jobs << [j.name, j]
		end

	end
	jobs.each do |j_name, job| #TODO move this to queue_manager
		set_dependencies_path(job)
		j_name.gsub!(')','') #Clean function characters on name
		job.name.gsub!(')','')
	end
	return jobs
end

#get_jobs_relations ⇒ Object 



220
221
222
223
224
225
226
 
# File 'lib/autoflow/stack.rb', line 220

def get_jobs_relations
	hash = {}
	get_jobs.each do |name, job|
		hash[name] = job
	end
	return hash
end

#inspect ⇒ Object

WORKFLOW REPRESENTATION



232
233
234
235
236
 
# File 'lib/autoflow/stack.rb', line 232

def inspect
	@jobs.each do |id, job|
		puts "#{id} > #{job.inspect}\t#{job.attrib[:done]}\n\t\e[32m#{job.dependencies.join("\n\t")}\e[0m"
	end
end

#load_variables(variables_lines, variable_type) ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/autoflow/stack.rb', line 73

def load_variables(variables_lines, variable_type)
	if !variables_lines.nil?
		variables_lines.each do |line|
			line.chomp!
			line.gsub!(/\s/,'')
			pairs = line.split(',')
			pairs.each do |pair|
				#pair =~ /(.+)=(.+)/
				#variable_type[$1] = $2
				var, value = pair.split('=', 2)
				variable_type[var] = value
			end
		end
	end
end

#parse! ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/autoflow/stack.rb', line 36

def parse!
	#Clean template
	@workflow.gsub!(/\#.+$/,'')	#Delete comments
	@workflow.gsub!("\t",'')		#Drop tabs
	@workflow.gsub!(/\n+/,"\n")	#Drop empty lines
	@workflow.gsub!(/^\s*/,'')

	#Parse template
	variables_lines = []
	persist_variables_lines = []
	node_lines = []

	node_beg = false		
	@workflow.each_line do |line|
		node_beg = true if line.include?('{') 	# This check the context of a variable
		if line.include?('}')					# if a variable is within a node,
			if node_beg							# we consider tha is a bash variable not a static autoflow variable
				node_beg = false
			else
				node_beg = true
			end
		end
		if line =~ /^\$/ && !node_beg
			variables_lines << line
		elsif line =~ /^\@/
			persist_variables_lines << line.gsub('@','')
		else
			node_lines << line
		end
	end
	load_variables(variables_lines, @variables)
	load_variables(@external_variables, @variables)
	load_variables(persist_variables_lines, @persist_variables)
	parse_nodes(node_lines)
	@jobs = get_jobs_relations
end

#parse_nodes(execution_lines) ⇒ Object 



143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
 
# File 'lib/autoflow/stack.rb', line 143

def parse_nodes(execution_lines)
	dinamic_variables = []
	nodes = scan_nodes(execution_lines)
	nodes = create_ids(nodes)
	
	#nodes.each do |tag, init, command, index|
	#	puts "#{tag.colorize(:red)}\t#{index}\n#{('-'*tag.length).colorize(:red)}\n#{init.chomp.colorize(:blue)}\n#{command.to_s.colorize(:green)}"
	#end

	nodes.each do |tag, init, command, index| #Takes the info of each node of workflow to create the job
		# Set batch	
		new_batch = Batch.new(tag, init, command, index, @exec_folder)
		dinamic_variables.concat(new_batch.handle_dependencies(dinamic_variables))
		@commands[new_batch.name] = new_batch
	end

	# link each parent batch to a child batch
	@commands.each do |name, batch|
		batch.asign_child_batch
	end
end

#replace_variables(string) ⇒ Object 



137
138
139
140
141
 
# File 'lib/autoflow/stack.rb', line 137

def replace_variables(string)
	@variables.each do |name, value|
		string.gsub!(name, value)
	end
end

#scan_nodes(execution_lines) ⇒ Object 



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/autoflow/stack.rb', line 89

def scan_nodes(execution_lines)
	template_executions = execution_lines.join('')
	replace_variables(template_executions)
	# $1 => tag, $2 => initialize, $3 => main command
	#executions = template_executions.scan(/(^.+\))\s{0,}\{\s{0,}([^\?]{0,})\s{0,}\?\s([^\}]{1,})\s{0,}\}/)
#=begin
	executions = [] # tag, initialize, main_command
	states = {} #name => [state, id(position)]
				#t => tag, i => initialize , c => command
	open_nodes = []
	template_executions.each_line do |line|
		line.strip! #Clean al whitespaces at beginning and the end of string
		node = states[open_nodes.last] if !open_nodes.empty?
		if line.empty?
			next
		# Create nodes and asign nodes states
		#----------------------------------------
		elsif line =~ /(\S*\)){$/ #Check tag and create node
			name = $1
			executions << [name, '', ''] # create node
			states[name] = [:i, executions.length - 1]
			open_nodes << name
		elsif line == '?' #Check command
			node[0] = :c
		elsif line == '}' #Close node
			finished_node = open_nodes.pop
			if !open_nodes.empty?
				parent_node = states[open_nodes.last].last #position
				child_node = states[finished_node].last
				parent_execution = executions[parent_node]
				if parent_execution[2].class == String   
					parent_execution[2] = [child_node]
				else
					parent_execution[2] << child_node
				end
			end
		# Add lines to nodes
		#----------------------
		elsif states[open_nodes.last].first == :i #Add initialize line
			executions[node.last][1] << line +"\n"
		elsif states[open_nodes.last].first == :c #Add command line
			executions[node.last][2] << line +"\n"			
		end
	end
#=end
	return executions
end

#set_dependencies_path(job) ⇒ Object

TODO move this to queue_manager



172
173
174
175
176
177
178
 
# File 'lib/autoflow/stack.rb', line 172

def set_dependencies_path(job) #TODO move this to queue_manager
	job.dependencies.sort{|d1, d2| d2.length <=> d1.length}.each do |dep|
		path =  @@all_jobs_relations[dep]
		job.initialization.gsub!(dep+')', path) if !job.initialization.nil?
		job.parameters.gsub!(dep+')', path)
	end
end