Class: SeqtrimWorker

Inherits:
ScbiMapreduce::Worker
  • Object
show all
Defined in:
lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb

Instance Method Summary collapse

Instance Method Details

#add_output_data(obj) ⇒ Object 



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 106

def add_output_data(obj)
  obj.output_text=[]

  if @tuple_size>1
    obj.each_slice(@tuple_size) do |seqs|

      write_seq_to_files_tuple(obj.output_files,seqs, obj.stats)

      seqs.each do |seq|
        obj.output_text << seq.to_text
      end
    end

  else
    obj.each do |seq|
      write_seq_to_files_normal(obj.output_files,seq, obj.stats)
      obj.output_text << seq.to_text
    end
  end

  # @remove seqs since they are not needed anymore to write output files
  obj.remove_all_seqs
end

#add_stat(stats, key, subkey, value, count = 1) ⇒ Object 



130
131
132
133
134
135
136
137
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 130

def add_stat(stats,key,subkey,value,count=1)

  stats[key]={} if !stats[key]
  stats[key][subkey]={} if !stats[key][subkey]
  stats[key][subkey][value]=0 if !stats[key][subkey][value]

  stats[key][subkey][value]+=count
end

#closing_worker ⇒ Object 



101
102
103
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 101

def closing_worker

end

#get_file(files, fn) ⇒ Object 



424
425
426
427
428
429
430
431
432
433
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 424

def get_file(files,fn)
  res=files[fn]

  if !res
    files[fn]=[]
    res=files[fn]
  end

  return res
end

#get_mid_message(mid) ⇒ Object 



255
256
257
258
259
260
261
262
263
264
265
266
267
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 255

def get_mid_message(mid)
  if (mid.nil? || (mid.message=='no_MID') ) # without mid
    mid_id = 'no_MID'
    mid_message = ' No MID found'
  else
    mid_id = mid.tag_id
    mid_message=''
    if mid_id != mid_message
      mid_message = ' '+mid.message
    end
  end
  return mid_id,mid_message
end

#json_file(files) ⇒ Object

ACCESS TO FILES



386
387
388
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 386

def json_file(files)
  return get_file(files,File.join(OUTPUT_PATH,'results.json'))
end

#low_complexity_file(files, dir_name, file_name) ⇒ Object 



412
413
414
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 412

def low_complexity_file(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'low_complexity_'+file_name+'.fastq'))
end

#low_sffinfo_file(files, dir_name, file_name) ⇒ Object 



420
421
422
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 420

def low_sffinfo_file(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'low_complexity_sff_info_'+file_name+'.txt'))
end

#paired_file(files, dir_name, file_name) ⇒ Object 



399
400
401
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 399

def paired_file(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'paired_'+file_name+'.fastq'))
end

#paired_file_ilu1(files, dir_name, file_name) ⇒ Object 



403
404
405
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 403

def paired_file_ilu1(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'paired_1_'+file_name+'.fastq'))
end

#paired_file_ilu2(files, dir_name, file_name) ⇒ Object 



407
408
409
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 407

def paired_file_ilu2(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'paired_2_'+file_name+'.fastq'))
end

#process_object(obj) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 56

def process_object(obj)

  running_seqs=SequenceGroup.new(obj.flatten)

  # execute plugins
  @plugin_manager.execute_plugins(running_seqs)

  # add output data
  add_output_data(running_seqs)

  return running_seqs
end

#receive_initial_config(obj) ⇒ Object 



69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 69

def receive_initial_config(obj)

  # Reads the parameters
  $WORKER_LOG.info "Params received"
  #       @params = Params.new(params_path)
  @params = obj
  @tuple_size=@params.get_param('tuple_size')

  @use_qual=@params.get_param('use_qual')
  @use_json=@params.get_param('use_json')
end

#rejected_output_file(files) ⇒ Object 



390
391
392
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 390

def rejected_output_file(files)
  return get_file(files,File.join(OUTPUT_PATH,'rejected.txt'))
end

#save_empty_insert(files, seq, stats) ⇒ Object 



242
243
244
245
246
247
248
249
250
251
252
253
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 242

def save_empty_insert(files,seq, stats)
  seq.seq_rejected=true
  seq.seq_rejected_by_message='short insert'

  message = 'No valid inserts found'

  rejected_output_file(files)<<('>'+seq.seq_name+ ' ' + message)

  add_stat(stats,'sequences','rejected',message)
  add_stat(stats,'sequences','count','rejected')
  
end

#save_one_insert(files, seq, stats, inserts, qual_inserts, mid, dir_name, file_name) ⇒ Object 



298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 298

def save_one_insert(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
  mid_id,mid_message=get_mid_message(mid)

  # save fasta and qual in no MID file
  has_low_complexity = seq.get_actions(ActionLowComplexity)

  if has_low_complexity.empty?
    add_stat(stats,'sequences','count','output_seqs')

    fasta_file=sequence_file(files,dir_name,file_name)
    sff_file=sffinfo_file(files,dir_name,file_name)
  else
    add_stat(stats,'sequences','count','output_seqs_low_complexity')

    fasta_file=low_complexity_file(files,dir_name,file_name)
    sff_file=low_sffinfo_file(files,dir_name,file_name)
  end

  q=[]
  if @use_qual
    q=qual_inserts[0]
  end

  n=seq.seq_name
  c=mid_message

  seq_comments=seq.get_comment_line
  if !seq_comments.strip.empty?
    c=seq_comments + c
  end

  f=inserts[0]

  fasta_file << FastqFile.to_fastq(n,f,q,c)

  inserts_pos = seq.get_actions(ActionInsert)

  sff_file<< "#{n} #{inserts_pos[0].start_pos+1} #{inserts_pos[0].end_pos+1}"
  
  
end

#save_rejected_empty_or_single(files, seq, stats, inserts, qual_inserts, mid, dir_name, file_name) ⇒ Object 



221
222
223
224
225
226
227
228
229
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 221

def save_rejected_empty_or_single(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
  if (seq.seq_rejected) # save to rejected sequences
    save_rejected_seq(files,seq, stats)
  elsif (inserts.empty?)  #sequence with no inserts
    save_empty_insert(files,seq, stats)
  elsif (inserts.count == 1) # sequence with one insert
    save_one_insert(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
  end
end

#save_rejected_seq(files, seq, stats) ⇒ Object

SAVE NORMAL ===============================



233
234
235
236
237
238
239
240
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 233

def save_rejected_seq(files,seq, stats)
  # message = seq.seq_rejected_by_message
  message= seq.get_comment_line
  rejected_output_file(files)<<('>'+seq.seq_name+ ' ' + message)

  add_stat(stats,'sequences','rejected',seq.seq_rejected_by_message)
  add_stat(stats,'sequences','count','rejected')
end

#save_two_inserts(files, seq, stats, inserts, qual_inserts, mid, dir_name, file_name) ⇒ Object 



269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 269

def save_two_inserts(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
  add_stat(stats,'sequences','count','output_seqs_paired')

  mid_id,mid_message=get_mid_message(mid)

  # save left read
  n="#{seq.seq_name}_left"
  c="template=#{seq.seq_name} dir=R library=#{mid_id} #{seq.get_comment_line}"
  f=inserts[0].reverse.tr('actgACTG','tgacTGAC')
  q=[]
  if @use_qual
    q=qual_inserts[0].reverse
  end
  
  paired_file(files,dir_name,file_name)<<FastqFile.to_fastq(n,f,q,c)
  
  # save right read
  n="#{seq.seq_name}_right"
  c="template=#{seq.seq_name} dir=F library=#{mid_id}  #{seq.get_comment_line}"
  f=inserts[1]
  q=[]
  if @use_qual
    q=qual_inserts[1]
  end

  paired_file(files,dir_name,file_name)<<FastqFile.to_fastq(n,f,q,c)
  
end

#save_two_inserts_tuple(files, seq1, seq2, stats, inserts1, inserts2, qual_inserts1, qual_inserts2, mid, dir_name, file_name) ⇒ Object 



189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 189

def save_two_inserts_tuple(files,seq1,seq2, stats,inserts1,inserts2,qual_inserts1,qual_inserts2,mid,dir_name,file_name)
  
  add_stat(stats,'sequences','count','output_seqs_paired')
  add_stat(stats,'sequences','count','output_seqs_paired')

  mid_id,mid_message=get_mid_message(mid)

  # save left read
  n="#{seq1.seq_name}"
  c=seq1.get_comment_line # "template=#{seq1.seq_name} dir=R library=#{mid_id}"
  f=inserts1[0]#.reverse.tr('actgACTG','tgacTGAC')
  q=[]
  if @use_qual
    q=qual_inserts1[0] #.reverse
  end
  
  paired_file_ilu1(files,dir_name,file_name)<<FastqFile.to_fastq(n,f,q,c)
  
  # save right read
  n="#{seq2.seq_name}"
  c=seq2.get_comment_line # "template=#{seq2.seq_name} dir=F library=#{mid_id}"
  f=inserts2[0]
  q=[]
  if @use_qual
    q=qual_inserts2[0]
  end

  paired_file_ilu2(files,dir_name,file_name)<<FastqFile.to_fastq(n,f,q,c)
  
end

#sequence_file(files, dir_name, file_name) ⇒ Object 



395
396
397
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 395

def sequence_file(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'sequences_'+file_name+'.fastq'))
end

#sffinfo_file(files, dir_name, file_name) ⇒ Object 



416
417
418
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 416

def sffinfo_file(files, dir_name, file_name)
  return get_file(files,File.join(OUTPUT_PATH,dir_name,'sff_info_'+file_name+'.txt'))
end

#starting_worker ⇒ Object 



81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 81

def starting_worker

  # $WORKER_LOG.level = Logger::ERROR
  $WORKER_LOG.level = Logger::WARN
  $WORKER_LOG.info "Loading actions"

  @action_manager = ActionManager.new

  $WORKER_LOG.info "Loading plugins"
  @plugin_list = @params.get_param('plugin_list') # puts in plugin_list the plugins's array
  $WORKER_LOG.info "PLUGIN LIST:" + @plugin_list

  @plugin_manager = PluginManager.new(@plugin_list,@params) # creates an instance from PluginManager. This must storage the plugins and load it

rescue Exception => e
  puts (e.message+ e.backtrace.join("\n"))

end

#write_seq_to_files_normal(files, seq, stats) ⇒ Object 



341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 341

def write_seq_to_files_normal(files,seq, stats)

  # puts stats.to_json

  dir_name,file_name,priority=seq.get_file_tag_path
  # puts File.join(dir_name,'sequences_'+file_name)

  # get current inserts
  inserts = seq.get_inserts

  # qualities are optional
  if @use_qual
    qual_inserts = seq.get_qual_inserts
  end

  # save json if necessary
  if @use_json
    json_file(files)<< seq.to_json
  end

  # find mids
  mid = seq.get_actions(ActionMid).first


  if (seq.seq_rejected) # save to rejected sequences
    save_rejected_seq(files,seq, stats)
    
  elsif (inserts.empty?)  #sequence with no inserts
    save_empty_insert(files,seq, stats)
    
  elsif (inserts.count == 2) # sequence with two inserts  = PAIRED SEQUENCES
    save_two_inserts(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
    
  elsif (inserts.count == 1) # sequence with one insert
    save_one_insert(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
  end

end

#write_seq_to_files_tuple(files, seqs, stats) ⇒ Object 



139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
 
# File 'lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb', line 139

def write_seq_to_files_tuple(files,seqs, stats)

  
  seq1=seqs[0]
  seq2=seqs[1]
  
  dir_name,file_name,priority=seq1.get_file_tag_path
  dir_name2,file_name2,priority2=seq2.get_file_tag_path
  
  # both paired sequences must go in same file, there are priorities
  if (dir_name!=dir_name2) || (file_name!=file_name2)
    if priority2>priority
      dir_name=dir_name2
      file_name=file_name2
    end
  end
  
  # get current inserts
  inserts1 = seq1.get_inserts
  inserts2 = seq2.get_inserts

  # qualities are optional
  if @use_qual
    qual_inserts1 = seq1.get_qual_inserts
    qual_inserts2 = seq2.get_qual_inserts
  end
  
  
  
  # save json if necessary
  if @use_json
    json_file(files)<< seq1.to_json
    json_file(files)<< seq2.to_json
  end

  # find mids
  mid1 = seq1.get_actions(ActionMid).first
  mid2 = seq2.get_actions(ActionMid).first
  
  
  if !inserts1.empty? && !inserts2.empty? # both have inserts
    # save_two_inserts(files,seq, stats,inserts,qual_inserts,mid,dir_name,file_name)
    save_two_inserts_tuple(files,seq1,seq2, stats,inserts1,inserts2,qual_inserts1,qual_inserts2,mid1,dir_name,file_name)
  else
    save_rejected_empty_or_single(files,seq1, stats,inserts1,qual_inserts1,mid1,dir_name,file_name)
    save_rejected_empty_or_single(files,seq2, stats,inserts2,qual_inserts2,mid2,dir_name,file_name)
  end
  
end