Method: CobwebModule::Crawl#update_queues

Defined in:
lib/crawl.rb

#update_queues ⇒ Object 



162
163
164
165
166
167
168
169
170
171
 
# File 'lib/crawl.rb', line 162

def update_queues
  lock("update_queues") do
    #@redis.incr "inprogress"
    # move the url from the queued list to the crawled list - for both the original url, and the content url (to handle redirects)
    @redis.srem "queued", @options[:url]
    @redis.sadd "crawled", @options[:url]

    # increment the counter if we are not limiting by page only || we are limiting count by page and it is a page
  end
end