Class: Apollo::Agent::CrawlerAgent

Inherits:
BaseAgent
  • Object
show all
Defined in:
lib/apollo_crawler/agent/crawler_agent.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods inherited from BaseAgent

#run

Constructor Details

#initialize(amqp, opts = {}) ⇒ CrawlerAgent

Returns a new instance of CrawlerAgent.



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/apollo_crawler/agent/crawler_agent.rb', line 33

def initialize(amqp, opts={})
  if(opts[:verbose])
    puts "Initializing crawler agent..."
  end

  # Declarations
  channel = amqp.create_channel
  self.declarations = Apollo::Agent.declare_entities(channel, opts)# Binding
  
  # Binding
  queue = self.declarations[:queues]["crawler.queue"]
  exchange = self.declarations[:exchanges]["crawler"]

  queue.bind(exchange).subscribe do |delivery_info, metadata, payload|
    msg = JSON.parse(payload)

    request = msg['request']
    response = msg['response']
    url = request["url"]

    puts "CrawlerAgent: Received - '#{url}', metadata #{metadata.inspect}" if opts[:verbose]

    doc = Nokogiri::HTML(response['body'])
    crawler = request['crawler_name'].constantize.new
    data = crawler.extract_data(doc)
    links = crawler.extract_links(doc)

    # puts crawler.to_s
    # puts res.inspect

    if(metadata[:reply_to] != nil)
      x = self.declarations[:exchanges][metadata[:reply_to]]

      msg = {
        :request => request,
        :response => response,
        :data => data,
        :links => links
      }

      x.publish(msg.to_json)
    end
  end
end

Instance Attribute Details

#declarations ⇒ Object

Returns the value of attribute declarations.



31
32
33
 
# File 'lib/apollo_crawler/agent/crawler_agent.rb', line 31

def declarations
  @declarations
end