Module: Scruber

Defined in:
lib/scruber.rb,
lib/scruber/cli.rb,
lib/scruber/queue.rb,
lib/scruber/fetcher.rb,
lib/scruber/version.rb,
lib/scruber/app_searcher.rb,
lib/scruber/core/crawler.rb,
lib/scruber/cli/generators.rb,
lib/scruber/core/page_format.rb,
lib/scruber/core/configuration.rb,
lib/scruber/core/extensions/log.rb,
lib/scruber/core/extensions/base.rb,
lib/scruber/core/extensions/loop.rb,
lib/scruber/core/extensions/seed.rb,
lib/scruber/core/page_format/xml.rb,
lib/scruber/cli/project_generator.rb,
lib/scruber/core/page_format/base.rb,
lib/scruber/core/page_format/html.rb,
lib/scruber/core/page_format/json.rb,
lib/scruber/helpers/fetcher_agent.rb,
lib/scruber/helpers/proxy_rotator.rb,
lib/scruber/queue_adapters/memory.rb,
lib/scruber/helpers/dictionary_reader.rb,
lib/scruber/core/extensions/csv_output.rb,
lib/scruber/helpers/user_agent_rotator.rb,
lib/scruber/core/extensions/queue_aliases.rb,
lib/scruber/helpers/dictionary_reader/csv.rb,
lib/scruber/helpers/dictionary_reader/xml.rb,
lib/scruber/core/extensions/parser_aliases.rb,
lib/scruber/queue_adapters/abstract_adapter.rb,
lib/scruber/fetcher_adapters/abstract_adapter.rb,
lib/scruber/fetcher_adapters/typhoeus_fetcher.rb,
lib/scruber/helpers/fetcher_agent_adapters/memory.rb,
lib/scruber/helpers/fetcher_agent_adapters/abstract_adapter.rb

Defined Under Namespace

Modules: AppSearcher, CLI, Core, Fetcher, FetcherAdapters, Helpers, Queue, QueueAdapters Classes: ArgumentError

Constant Summary collapse

VERSION = 
"0.1.9"

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.configuration ⇒ Object 



66
67
68
 
# File 'lib/scruber.rb', line 66

def configuration
  @configuration ||= Core::Configuration.new
end

.logger ⇒ Object 



70
71
72
 
# File 'lib/scruber.rb', line 70

def logger
  @logger ||= Scruber.root.nil? ? nil : Logger.new(Scruber.root.join('log', 'crawler.log'))
end

Class Method Details

.configure {|configuration| ... } ⇒ Object

Yields:



74
75
76
 
# File 'lib/scruber.rb', line 74

def configure(&block)
  yield configuration
end

.root ⇒ Object 



78
79
80
81
82
83
84
 
# File 'lib/scruber.rb', line 78

def root
  if defined?(APP_PATH)
    Pathname.new(File.expand_path('../../', APP_PATH))
  else
    nil
  end
end

.run(*args, &block) ⇒ Object 



60
61
62
63
64
 
# File 'lib/scruber.rb', line 60

def run(*args, &block)
  raise "You need a block to build!" unless block_given?

  Core::Crawler.new(*args).run(&block)
end