Method: Scruber::Core::Crawler#initialize

Defined in:
lib/scruber/core/crawler.rb

#initialize(*args) ⇒ Scruber::Core::Crawler

Initialize crawler with scraper name and/or with options

Crawler.new(:sample, fetcher_adapter: :custom)
Crawler.new(:sample)
Crawler.new(fetcher_adapter: :custom)

Raises:



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/scruber/core/crawler.rb', line 31

def initialize(*args)
  if args.first.is_a?(Hash)
    scraper_name = nil
    @options = args.first
  else
    scraper_name, @options = args
    @options ||= {}
  end
  @scraper_name = scraper_name.present? ? scraper_name : ENV['SCRUBER_SCRAPER_NAME']
  raise Scruber::ArgumentError.new("Scraper name is empty. Pass it to `Scruber.run :name do` or through ENV['SCRUBER_SCRAPER_NAME']") if @scraper_name.blank?
  @scraper_name = @scraper_name.to_sym
  @callbacks_options = {}
  @callbacks = {}
  @on_page_error_callback = nil
  @on_complete_callbacks = []

  Scruber.configuration.merge_options(@options)
  ActiveSupport::Dependencies.autoload_paths = Scruber.configuration.autoload_paths

  @queue = Scruber::Queue.new(scraper_name: @scraper_name)
  @fetcher = Scruber::Fetcher.new
  initialize_progressbar
  load_extenstions
end