Class: Gandalf::Worker
- Inherits:
-
Object
- Object
- Gandalf::Worker
- Includes:
- DataMapper::Resource
- Defined in:
- lib/gandalf/worker.rb
Overview
A magical slave
Instance Method Summary collapse
- #crawl(jobs) ⇒ Object
- #handle_error(job) ⇒ Object
- #jobs_to_do ⇒ Object
- #new_jobs(count) ⇒ Object
- #push(jobs) ⇒ Object
- #run ⇒ Object
- #save_posts(feed, channel_id) ⇒ Object
- #setup(options = {:post_class => Post}) ⇒ Object
- #stop ⇒ Object
Instance Method Details
#crawl(jobs) ⇒ Object
35 36 37 38 39 40 41 42 43 44 45 |
# File 'lib/gandalf/worker.rb', line 35 def crawl(jobs) urls = jobs.keys feeds = Feedzirra::Feed.fetch_and_parse(urls) jobs.each do |url, job| if feeds[url].is_a? Feedzirra::Parser::RSS save_posts(feeds[url], job[:id]) else handle_error(job) end end end |
#handle_error(job) ⇒ Object
60 61 62 63 64 65 66 67 68 69 70 71 72 |
# File 'lib/gandalf/worker.rb', line 60 def handle_error(job) if job[:errors].is_a? Fixnum job[:errors] += 1 else job[:errors] = 1 end if job[:errors] >= max_errors puts job else @queue.push(job) end end |
#jobs_to_do ⇒ Object
74 75 76 |
# File 'lib/gandalf/worker.rb', line 74 def jobs_to_do @queue.length end |
#new_jobs(count) ⇒ Object
84 85 86 87 88 89 90 91 |
# File 'lib/gandalf/worker.rb', line 84 def new_jobs(count) jobs = @queue.pop_first(count) hash = {} jobs.each do |job| hash[job[:url]] = job end hash end |
#push(jobs) ⇒ Object
78 79 80 81 82 |
# File 'lib/gandalf/worker.rb', line 78 def push(jobs) jobs.each do |job| @queue.push(job) end end |
#run ⇒ Object
20 21 22 23 24 25 |
# File 'lib/gandalf/worker.rb', line 20 def run @crawl_scheduler = Rufus::Scheduler.start_new unless @crawl_scheduler @crawl_scheduler.every interval do crawl new_jobs(max_jobs) end end |
#save_posts(feed, channel_id) ⇒ Object
47 48 49 50 51 52 53 54 55 56 57 58 |
# File 'lib/gandalf/worker.rb', line 47 def save_posts(feed, channel_id) posts = @Post.parse(feed) posts.each do |p| p.channel_id = channel_id p.clean! begin p.save rescue MysqlError => err break end end end |
#setup(options = {:post_class => Post}) ⇒ Object
15 16 17 18 |
# File 'lib/gandalf/worker.rb', line 15 def setup( = {:post_class => Post}) @queue = RedisQueue.new(:key => self.id, :redis => [:redis]) unless @queue @Post = ([:post_class]) end |
#stop ⇒ Object
27 28 29 |
# File 'lib/gandalf/worker.rb', line 27 def stop @crawl_scheduler.stop end |