Class: FileMiner

Inherits:
Object
  • Object
show all
Defined in:
lib/fileminer.rb,
lib/fileminer/version.rb

Constant Summary collapse

DEFAULT_SETTINGS = 
{
  refresh_files_time_trigger: '30s',
  max_time_of_each_mining: '5s',
  max_lines_of_each_mining: -1,
  max_lines_of_each_file: -1,
  sleep_time_when_no_more_data: '5s',
}
VERSION = 
'1.2.1'

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(conf) ⇒ FileMiner

Create a new FileMiner instance

Parameters:



25
26
27
28
29
30
31
32
33
 
# File 'lib/fileminer.rb', line 25

def initialize(conf)
  init_settings conf['fileminer.settings']
  @output = init_output conf
  raise 'Missing config fileminer.inputs' unless conf.key? 'fileminer.inputs'
  @miner = Miner.new conf['fileminer.inputs'].keys_to_sym
  @miner.refresh_files
  @miner.save_registry
  @running = false
end

Instance Attribute Details

#miner ⇒ Object (readonly)

Returns the value of attribute miner.



20
21
22
 
# File 'lib/fileminer.rb', line 20

def miner
  @miner
end

#output ⇒ Object (readonly)

Returns the value of attribute output.



20
21
22
 
# File 'lib/fileminer.rb', line 20

def output
  @output
end

#running ⇒ Object (readonly)

Returns the value of attribute running.



20
21
22
 
# File 'lib/fileminer.rb', line 20

def running
  @running
end

Instance Method Details

#check_files ⇒ Object 



221
222
223
224
225
 
# File 'lib/fileminer.rb', line 221

def check_files
  if @miner.files_need_refresh? @refresh_files_time_trigger
    @miner.refresh_files
  end
end

#mine_once ⇒ Object 



171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/fileminer.rb', line 171

def mine_once
  start_time = Time.now
  full_lines = 0
  miner = @miner
  miner.active_files.all? do |record|
    mining_next = true
    if record[:pos] < File.size(record[:path])
      file_lines = 0
      loop do
        lines = miner.read_lines record
        break if lines.empty?
        send_lines record, lines
        file_lines += lines.size
        full_lines += lines.size
        if mining_break? start_time, full_lines
          mining_next = false
          break
        end
        break if file_break? file_lines
      end
    end
    mining_next
  end
  full_lines
end

#start_mining ⇒ Object 



197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
 
# File 'lib/fileminer.rb', line 197

def start_mining
  unless @running
    @running = true
    sleep_seconds = @sleep_seconds_when_no_more_data
    while @running
      begin
        files_refreshed = check_files
        sent_lines = mine_once
        # sleep 5 seconds if no more data
        # TODO using settings instead in future
        if sent_lines == 0
          @miner.save_work_status if files_refreshed
          sleep sleep_seconds
        end
      rescue => e
        @logger.error e
        # sleep for a little while to wait output recover
        sleep sleep_seconds if @running
      end
    end
    @miner.save_registry
  end
end

#stop_mining ⇒ Object 



227
228
229
 
# File 'lib/fileminer.rb', line 227

def stop_mining
  @running = false if @running
end