Method: CobwebModule::Crawl#process

Defined in:
lib/crawl.rb

#process(&block) ⇒ Object 



188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
 
# File 'lib/crawl.rb', line 188

def process(&block)
  lock("process-count") do
    if @options[:crawl_limit_by_page]
      if content.mime_type.match("text/html")
        increment_process_counter
      end
    else
      increment_process_counter
    end
    #@redis.sadd "queued", @options[:url]
  end

  yield if block_given?
  @redis.incr("crawl_job_enqueued_count")
end