Class: ProxyDaemon::Daemon

Inherits:
Object
  • Object
show all
Defined in:
lib/proxy_daemon/daemon.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(script_or_parser, options = nil) ⇒ Daemon

Returns a new instance of Daemon.



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/proxy_daemon/daemon.rb', line 10

def initialize(script_or_parser, options = nil)
  if script_or_parser.is_a? String then @script = script_or_parser
  else
    @script = '-'
    
    if script_or_parser.is_a? Hash then options = script_or_parser
    else @parser = script_or_parser end
  end
  
  @proxies = ((options[:proxies].map { |proxy| proxy.gsub /\s+/, ' ' }) || [])#.shuffle
  @urls = options[:urls] || []
  @worker_processes = (options[:worker_processes] || 10)
  @tries = options[:tries] || 4

  @threads = []
  @list = {}
  @semaphore = Mutex.new
end

Instance Attribute Details

#list ⇒ Object

Returns the value of attribute list.



8
9
10
 
# File 'lib/proxy_daemon/daemon.rb', line 8

def list
  @list
end

Instance Method Details

#add_urls(urls) ⇒ Object 



131
132
133
 
# File 'lib/proxy_daemon/daemon.rb', line 131

def add_urls(urls)
  @semaphore.synchronize { @urls |= [*urls] }
end

#command(cmd, pipe, params = {}) ⇒ Object 



29
30
31
32
33
34
 
# File 'lib/proxy_daemon/daemon.rb', line 29

def command(cmd, pipe, params = {})
  command = {command: cmd}.merge(params)
  
  Thread.current[:command] = cmd
  pipe.puts(command.to_json)
end

#listen(pipe, try = 0) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/proxy_daemon/daemon.rb', line 36

def listen(pipe, try = 0)
  packet = JSON.parse((pipe.gets || '').strip)
  raise "Empty answer from worker process" if packet.empty?
  answer = packet['answer']
  raise "Process: '#{answer}'" if answer == 'timeout' || answer == 'error'
  
  if answer == 'proxy'
    raise "Broken url: #{Thread.current[:url]}" if try > @tries

    proxy = getProxy
    log "Answer: #{answer}, data: #{packet['data']}".yellow, pipe
    log "Choosing new proxy: #{(proxy || 'nil').yellow}", pipe
    command :proxy, pipe, proxy: proxy
#        command :url, pipe, url: Thread.current[:url]
    listen pipe, try+1
  elsif answer == 'ok'
    @semaphore.synchronize { @list.merge! packet['data'] } if packet.key? 'data'
    log "Process: #{Thread.current[:url].cyan}: '#{answer.green}', data: #{packet['data'].to_json}", pipe
  else
    log "Answer: #{answer}, data: #{packet['data'].to_json}".green, pipe
  end
end

#start(options = nil, &block) ⇒ Object 



117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/proxy_daemon/daemon.rb', line 117

def start(options = nil, &block)
  @block = block if block_given?
  @parse_method = options[:parse] if options && options.key?(:parse)
  worker_processes = [@worker_processes, @urls.count].min
  
  begin
    puts "[main] Starting " + "#{worker_processes}".yellow + " workers:"
    worker_processes.times { |i| @threads << Thread.new(&(->{worker})) }
    @threads.each { |t| t.join }
  rescue Interrupt => e
    puts "[main] Interrupted by user".yellow
  end
end

#worker ⇒ Object 



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/proxy_daemon/daemon.rb', line 59

def worker
  IO.popen("#{@script}", 'r+') { |p|
    if (@script == '-' && p.nil?) # child process
      if @block.nil? && @parser.nil?
        $stderr.syswrite "[#{Process.pid}]".magenta + ' The parser is undefined and block wasn\'t given for parsing the content'.red + "\n"
        $stderr.flush

        Kernel.exit!
      end
      
      if @parser
        worker = ProxyDaemon::Worker.new(@parser, @parse_method)
        worker.call
      elsif @block
        worker = ProxyDaemon::Worker.new
        worker.call(&@block)
      end
    else
      p.sync = true
      proxy = getProxy
      log "Starting loop with new proxy: ".green + "#{(proxy || 'nil').yellow}", p
      command :proxy, p, proxy: proxy

      begin
        loop do
          sleep(0.1)
          url = getUrl

          if url.nil?
            finished "Links are finished! exitting...".green, p
            command :exit, p
            break
          else
            log 'Urls count: ' + "#{@urls.length}".green + ", #{url.green}"
            Thread.current[:url] = url
            command :url, p, url: url
            listen p
          end
        end

        log "Finishing loop".green, p
      rescue Exception => e
        @semaphore.synchronize {
          log "Exception in main:".red + " '#{Thread.current[:url]}'".yellow + " #{e.message.red}".red + "\n#{e.backtrace.join("\n").red}\n", p
          command :exit, p
          #puts e.backtrace
        }
      end
    end
  }

  if @urls.length > 0
    # @threads << Thread.new(&(->{worker}))
    # @threads.last.join
    worker
  end
end