Class: SeqtrimWorker

Inherits:
ScbiMapreduce::Worker
  • Object
show all
Defined in:
lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb

Instance Method Summary collapse

Instance Method Details

#add_output_data(obj) ⇒ Object 



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 107

def add_output_data(obj)
  obj.output_text=[]

  if @tuple_size>1
    obj.each_slice(@tuple_size) do |seqs|

      write_seq_to_files_tuple(obj.output_files,seqs, obj.stats)

      seqs.each do |seq|
        obj.output_text << seq.to_text
      end
    end

  else
    obj.each do |seq|
      write_seq_to_files_normal(obj.output_files,seq, obj.stats)
      obj.output_text << seq.to_text
    end
  end

  # @remove seqs since they are not needed anymore to write output files
  obj.remove_all_seqs
end

#add_stat(stats, key, subkey, value, count = 1) ⇒ Object 



131
132
133
134
135
136
137
138
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 131

def add_stat(stats,key,subkey,value,count=1)

  stats[key]={} if !stats[key]
  stats[key][subkey]={} if !stats[key][subkey]
  stats[key][subkey][value]=0 if !stats[key][subkey][value]

  stats[key][subkey][value]+=count
end

#closing_worker ⇒ Object 



102
103
104
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 102

def closing_worker

end

#get_file(files, fn) ⇒ Object 



425
426
427
428
429
430
431
432
433
434
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 425

def get_file(files,fn)
  res=files[fn]

  if !res
    files[fn]=[]
    res=files[fn]
  end

  return res
end

#get_mid_message(mid) ⇒ Object 



256
257
258
259
260
261
262
263
264
265
266
267
268
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 256

def get_mid_message(mid)
  if (mid.nil? || (mid.message=='no_MID') ) # without mid
    mid_id = 'no_MID'
    mid_message = ' No MID found'
  else
    mid_id = mid.tag_id
    mid_message=''
    if mid_id != mid_message
      mid_message = ' '+mid.message
    end
  end
  return mid_id,mid_message
end

#json_file(files) ⇒ Object

ACCESS TO FILES



387
388
389
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 387

def json_file(files)
  return get_file(files,File.join(OUTPUT_PATH,'results.json'))
end

#low_complexity_file(files, dir_name, file_name) ⇒ Object 



413
414
415
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 413

def low_complexity_file(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'low_complexity_'+file_name+'.fastq'))
end

#low_sffinfo_file(files, dir_name, file_name) ⇒ Object 



421
422
423
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 421

def low_sffinfo_file(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'low_complexity_sff_info_'+file_name+'.txt'))
end

#paired_file(files, dir_name, file_name) ⇒ Object 



400
401
402
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 400

def paired_file(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'paired_'+file_name+'.fastq'))
end

#paired_file_ilu1(files, dir_name, file_name) ⇒ Object 



404
405
406
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 404

def paired_file_ilu1(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'paired_1_'+file_name+'.fastq'))
end

#paired_file_ilu2(files, dir_name, file_name) ⇒ Object 



408
409
410
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 408

def paired_file_ilu2(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'paired_2_'+file_name+'.fastq'))
end

#process_object(obj) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 56

def process_object(obj)

  running_seqs=SequenceGroup.new(obj.flatten)

  # execute plugins
  @plugin_manager.execute_plugins(running_seqs)

  # add output data
  add_output_data(running_seqs)

  return running_seqs
end

#receive_initial_config(obj) ⇒ Object 



69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 69

def receive_initial_config(obj)

  # Reads the parameters
  $WORKER_LOG.info "Params received"
  #				@params = Params.new(params_path)
  @params = obj
  @tuple_size=@params.get_param('tuple_size')

  @use_qual=@params.get_param('use_qual')
  @use_json=@params.get_param('use_json')
end

#rejected_output_file(files) ⇒ Object 



391
392
393
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 391

def rejected_output_file(files)
  return get_file(files,File.join(OUTPUT_PATH,'rejected.txt'))
end

#save_empty_insert(files, seq, stats) ⇒ Object 



243
244
245
246
247
248
249
250
251
252
253
254
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 243

def save_empty_insert(files,seq, stats)
  seq.seq_rejected=true
  seq.seq_rejected_by_message='short insert'

  message = 'No valid inserts found'

  rejected_output_file(files)<<('>'+seq.seq_name+ ' ' + message)

  add_stat(stats,'sequences','rejected',message)
  add_stat(stats,'sequences','count','rejected')
  
end

#save_one_insert(files, seq, stats, inserts, qual_inserts, mid, dir_name, file_name) ⇒ Object 



299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 299

def save_one_insert(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
  mid_id,mid_message=get_mid_message(mid)

  # save fasta and qual in no MID file
  has_low_complexity = seq.get_actions(ActionLowComplexity)

  if has_low_complexity.empty?
    add_stat(stats,'sequences','count','output_seqs')

    fasta_file=sequence_file(files,dir_name,file_name)
    sff_file=sffinfo_file(files,dir_name,file_name)
  else
    add_stat(stats,'sequences','count','output_seqs_low_complexity')

    fasta_file=low_complexity_file(files,dir_name,file_name)
    sff_file=low_sffinfo_file(files,dir_name,file_name)
  end

  q=[]
  if @use_qual
    q=qual_inserts[0]
  end

  n=seq.seq_name
  c=mid_message

  seq_comments=seq.get_comment_line
  if !seq_comments.strip.empty?
    c=seq_comments + c
  end

  f=inserts[0]

  fasta_file << FastqFile.to_fastq(n,f,q,c)

  inserts_pos = seq.get_actions(ActionInsert)

  sff_file<< "#{n} #{inserts_pos[0].start_pos+1} #{inserts_pos[0].end_pos+1}"
  
  
end

#save_rejected_empty_or_single(files, seq, stats, inserts, qual_inserts, mid, dir_name, file_name) ⇒ Object 



222
223
224
225
226
227
228
229
230
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 222

def save_rejected_empty_or_single(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
  if (seq.seq_rejected) # save to rejected sequences
    save_rejected_seq(files,seq, stats)
  elsif (inserts.empty?)  #sequence with no inserts
    save_empty_insert(files,seq, stats)
  elsif (inserts.count == 1) # sequence with one insert
    save_one_insert(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
  end
end

#save_rejected_seq(files, seq, stats) ⇒ Object

SAVE NORMAL ===============================



234
235
236
237
238
239
240
241
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 234

def save_rejected_seq(files,seq, stats)
  # message = seq.seq_rejected_by_message
  message= seq.get_comment_line
  rejected_output_file(files)<<('>'+seq.seq_name+ ' ' + message)

  add_stat(stats,'sequences','rejected',seq.seq_rejected_by_message)
  add_stat(stats,'sequences','count','rejected')
end

#save_two_inserts(files, seq, stats, inserts, qual_inserts, mid, dir_name, file_name) ⇒ Object 



270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 270

def save_two_inserts(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
  add_stat(stats,'sequences','count','output_seqs_paired')

  mid_id,mid_message=get_mid_message(mid)

  # save left read
  n="#{seq.seq_name}_left"
  c="template=#{seq.seq_name} dir=R library=#{mid_id} #{seq.get_comment_line}"
  f=inserts[0].reverse.tr('actgACTG','tgacTGAC')
  q=[]
  if @use_qual
    q=qual_inserts[0].reverse
  end
  
  paired_file(files,dir_name,file_name)<<FastqFile.to_fastq(n,f,q,c)
  
  # save right read
  n="#{seq.seq_name}_right"
  c="template=#{seq.seq_name} dir=F library=#{mid_id}  #{seq.get_comment_line}"
  f=inserts[1]
  q=[]
  if @use_qual
    q=qual_inserts[1]
  end

  paired_file(files,dir_name,file_name)<<FastqFile.to_fastq(n,f,q,c)
  
end

#save_two_inserts_tuple(files, seq1, seq2, stats, inserts1, inserts2, qual_inserts1, qual_inserts2, mid, dir_name, file_name) ⇒ Object 



190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 190

def save_two_inserts_tuple(files,seq1,seq2, stats,inserts1,inserts2,qual_inserts1,qual_inserts2,mid,dir_name,file_name)
  
  add_stat(stats,'sequences','count','output_seqs_paired')
  add_stat(stats,'sequences','count','output_seqs_paired')

  mid_id,mid_message=get_mid_message(mid)

  # save left read
  n="#{seq1.seq_name}"
  c=seq1.get_comment_line # "template=#{seq1.seq_name} dir=R library=#{mid_id}"
  f=inserts1[0]#.reverse.tr('actgACTG','tgacTGAC')
  q=[]
  if @use_qual
    q=qual_inserts1[0] #.reverse
  end
  
  paired_file_ilu1(files,dir_name,file_name)<<FastqFile.to_fastq(n,f,q,c)
  
  # save right read
  n="#{seq2.seq_name}"
  c=seq2.get_comment_line # "template=#{seq2.seq_name} dir=F library=#{mid_id}"
  f=inserts2[0]
  q=[]
  if @use_qual
    q=qual_inserts2[0]
  end

  paired_file_ilu2(files,dir_name,file_name)<<FastqFile.to_fastq(n,f,q,c)
  
end

#sequence_file(files, dir_name, file_name) ⇒ Object 



396
397
398
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 396

def sequence_file(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'sequences_'+file_name+'.fastq'))
end

#sffinfo_file(files, dir_name, file_name) ⇒ Object 



417
418
419
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 417

def sffinfo_file(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'sff_info_'+file_name+'.txt'))
end

#starting_worker ⇒ Object 



81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 81

def starting_worker

  # $WORKER_LOG.level = Logger::ERROR
  #$WORKER_LOG.level = Logger::WARN
  $WORKER_LOG.level = Logger::INFO
  $WORKER_LOG.info "Loading actions"

  @action_manager = ActionManager.new

  $WORKER_LOG.info "Loading plugins"
  @plugin_list = @params.get_param('plugin_list') # puts in plugin_list the plugins's array
  $WORKER_LOG.info "PLUGIN LIST:" + @plugin_list

  @plugin_manager = PluginManager.new(@plugin_list,@params) # creates an instance from PluginManager. This must storage the plugins and load it

rescue Exception => e
  puts (e.message+ e.backtrace.join("\n"))

end

#write_seq_to_files_normal(files, seq, stats) ⇒ Object 



342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 342

def write_seq_to_files_normal(files,seq, stats)

  # puts stats.to_json

  dir_name,file_name,priority=seq.get_file_tag_path
  # puts File.join(dir_name,'sequences_'+file_name)

  # get current inserts
  inserts = seq.get_inserts

  # qualities are optional
  if @use_qual
    qual_inserts = seq.get_qual_inserts
  end

  # save json if necessary
  if @use_json
    json_file(files)<< seq.to_json
  end

  # find mids
  mid = seq.get_actions(ActionMid).first


  if (seq.seq_rejected) # save to rejected sequences
    save_rejected_seq(files,seq, stats)
    
  elsif (inserts.empty?)  #sequence with no inserts
    save_empty_insert(files,seq, stats)
    
  elsif (inserts.count == 2) # sequence with two inserts  = PAIRED SEQUENCES
    save_two_inserts(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
    
  elsif (inserts.count == 1) # sequence with one insert
    save_one_insert(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
  end

end

#write_seq_to_files_tuple(files, seqs, stats) ⇒ Object 



140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 140

def write_seq_to_files_tuple(files,seqs, stats)

  
  seq1=seqs[0]
  seq2=seqs[1]
  
  dir_name,file_name,priority=seq1.get_file_tag_path
  dir_name2,file_name2,priority2=seq2.get_file_tag_path
  
  # both paired sequences must go in same file, there are priorities
  if (dir_name!=dir_name2) || (file_name!=file_name2)
    if priority2>priority
      dir_name=dir_name2
      file_name=file_name2
    end
  end
  
  # get current inserts
  inserts1 = seq1.get_inserts
  inserts2 = seq2.get_inserts

  # qualities are optional
  if @use_qual
    qual_inserts1 = seq1.get_qual_inserts
    qual_inserts2 = seq2.get_qual_inserts
  end
  
  
  
  # save json if necessary
  if @use_json
    json_file(files)<< seq1.to_json
    json_file(files)<< seq2.to_json
  end

  # find mids
  mid1 = seq1.get_actions(ActionMid).first
  mid2 = seq2.get_actions(ActionMid).first
  
  
  if !inserts1.empty? && !inserts2.empty? # both have inserts
    # save_two_inserts(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
    save_two_inserts_tuple(files,seq1,seq2, stats,inserts1,inserts2,qual_inserts1,qual_inserts2,mid1,dir_name,file_name)
  else
    save_rejected_empty_or_single(files,seq1, stats,inserts1,qual_inserts1,mid1,dir_name,file_name)
    save_rejected_empty_or_single(files,seq2, stats,inserts2,qual_inserts2,mid2,dir_name,file_name)
  end
  
end