Class: Fairy::PWC
Defined Under Namespace
Classes: PPostFilter
Constant Summary collapse
- ST_ALL_IMPORTED =
:ST_ALL_IMPORTED
- ST_WAIT_EXPORT_FINISH =
:ST_WAIT_EXPORT_FINISH
- ST_EXPORT_FINISH =
:ST_EXPORT_FINISH
Constants inherited from PIOFilter
Fairy::PIOFilter::ST_WAIT_IMPORT
Constants inherited from PFilter
Fairy::PFilter::END_OF_STREAM, Fairy::PFilter::ST_ACTIVATE, Fairy::PFilter::ST_FINISH, Fairy::PFilter::ST_INIT
Instance Attribute Summary
Attributes inherited from PFilter
#IGNORE_EXCEPTION, #id, #log_id, #ntask
Class Method Summary collapse
Instance Method Summary collapse
- #add_export(key, export) ⇒ Object
- #hash_key(e) ⇒ Object
-
#initialize(id, ntask, bjob, opts = nil) ⇒ PWC
constructor
A new instance of PWC.
- #open(nfileplace) ⇒ Object
- #start_export ⇒ Object
- #terminate ⇒ Object
- #wait_export_finish ⇒ Object
Methods inherited from PIOFilter
Methods inherited from PFilter
#abort_running, #basic_start, #break_running, #each, #global_break, #global_break_from_other, #handle_exception, #key, #key=, #next, #no, #no=, #notice_status, #processor, #start, #start_watch_status, #status=, #terminate_proc
Constructor Details
#initialize(id, ntask, bjob, opts = nil) ⇒ PWC
Returns a new instance of PWC.
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
# File 'lib/fairy/node/p-wc.rb', line 23 def initialize(id, ntask, bjob, opts=nil) super @file = nil @exports = {} @exports_queue = Queue.new @counter = {} @mod = opts[:no_segment] @mod ||= CONF.GROUP_BY_NO_SEGMENT mod = opts[:hash_module] mod ||= CONF.GROUP_BY_HASH_MODULE require mod @hash_generator = Fairy::HValueGenerator.new(bjob.hash_seed) @hash_optimize = CONF.GROUP_BY_GROUPING_OPTIMIZE @hash_optimize = opts[:grouping_optimize] if opts.key?(:grouping_optimize) end |
Class Method Details
Instance Method Details
#add_export(key, export) ⇒ Object
62 63 64 65 66 67 |
# File 'lib/fairy/node/p-wc.rb', line 62 def add_export(key, export) @exports[key] = export # @exports_queue.push [key, export] # [BUG#171]同期処理でないとまずい. @bjob.add_exports(key, export, self) end |
#hash_key(e) ⇒ Object
121 122 123 |
# File 'lib/fairy/node/p-wc.rb', line 121 def hash_key(e) @hash_generator.value(@key_proc.yield(e)) % @mod end |
#open(nfileplace) ⇒ Object
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
# File 'lib/fairy/node/p-wc.rb', line 44 def open(nfileplace) @file_name = nfileplace.path self.no = nfileplace.no begin @file = File.open(@file_name) rescue e = $!.exception($!.+ "(vfile entry##{nfileplace.no}: #{nfileplace.url})") e.set_backtrace($!.backtrace) Log::error_exception(e) handle_exception(e) raise e end # start self end |
#start_export ⇒ Object
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 |
# File 'lib/fairy/node/p-wc.rb', line 69 def start_export Log::debug(self, "START_EXPORT") start do hash_opt = @opts[:grouping_optimize] hash_opt = CONF.GROUP_BY_GROUPING_OPTIMIZE if hash_opt.nil? # if hash_opt @key_proc = eval("proc{|w| w}", @context.binding) # else # @key_proc = BBlock.new("|w| w", @context, self) # end policy = @opts[:postqueuing_policy] begin @file.each do |ln| (begin ln.chomp.split rescue [] end).each do |e| key = hash_key(e) export = @exports[key] unless export export = Export.new(policy) export.njob_id = @id export.add_key(key) add_export(key, export) @counter[key] = 0 end export.push e @counter[key] += 1 end end rescue Log::debug_exception(self) raise ensure @exports_queue.push nil @exports.each{|key, export| Log::debug(self, "G0 #{key} => #{@counter[key]}") export.push END_OF_STREAM} end end end |
#terminate ⇒ Object
115 116 117 118 119 |
# File 'lib/fairy/node/p-wc.rb', line 115 def terminate @wait_cv = @terminate_mon.new_cv wait_export_finish super end |
#wait_export_finish ⇒ Object
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 |
# File 'lib/fairy/node/p-wc.rb', line 126 def wait_export_finish Log::debug(self, "G1") self.status = ST_ALL_IMPORTED Log::debug(self, "G2") # すべての, exportのoutputが設定されるまで待っている # かなりイマイチ # for key, export in @exports #Log::debug(self, "G2.key = #{export.key}: WAIT") # export.output #Log::debug(self, "G2.key = #{export.key}: RESUME") # end Log::debug(self, "G3") # ここの位置が重要 self.status = ST_WAIT_EXPORT_FINISH # ここもいまいち Log::debug(self, "G4") for key, export in @exports Log::debug(self, "G4.WAIT #{key}") export.fib_wait_finish(@wait_cv) end Log::debug(self, "G5") self.status = ST_EXPORT_FINISH end |