Method: Cobweb#initialize

Defined in:
lib/cobweb.rb

#initialize(options = {}) ⇒ Cobweb

See readme for more information on options available



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/cobweb.rb', line 32

def initialize(options = {})
  @options = options
  default_use_encoding_safe_process_job_to  false
  default_follow_redirects_to               true
  default_redirect_limit_to                 10
  default_queue_system_to                   :resque
  if @options[:queue_system] == :resque
    default_processing_queue_to               "CobwebProcessJob"
    default_crawl_finished_queue_to           "CobwebFinishedJob"
  else
    default_processing_queue_to               "CrawlProcessWorker"
    default_crawl_finished_queue_to           "CrawlFinishedWorker"
  end
  default_quiet_to                          true
  default_debug_to                          false
  default_cache_to                          300
  default_cache_type_to                     :crawl_based # other option is :full
  default_timeout_to                        10
  default_redis_options_to                  Hash.new
  default_internal_urls_to                  []
  default_external_urls_to                  []
  default_seed_urls_to                  []
  default_first_page_redirect_internal_to   true
  default_text_mime_types_to                ["text/*", "application/xhtml+xml"]
  default_obey_robots_to                    false
  default_user_agent_to                     "cobweb/#{Cobweb.version} (ruby/#{RUBY_VERSION} nokogiri/#{Nokogiri::VERSION})"
  default_valid_mime_types_to                ["*/*"]
  default_raise_exceptions_to               false
  default_store_inbound_links_to            false
  default_proxy_addr_to                     nil
  default_proxy_port_to                     nil
  default_additional_tags_to                nil
  default_treat_https_as_http_to            true


end