Module: Scruber
- Defined in:
- lib/scruber.rb,
lib/scruber/cli.rb,
lib/scruber/queue.rb,
lib/scruber/fetcher.rb,
lib/scruber/version.rb,
lib/scruber/app_searcher.rb,
lib/scruber/core/crawler.rb,
lib/scruber/cli/generators.rb,
lib/scruber/core/page_format.rb,
lib/scruber/core/configuration.rb,
lib/scruber/core/extensions/log.rb,
lib/scruber/core/extensions/base.rb,
lib/scruber/core/extensions/loop.rb,
lib/scruber/core/extensions/seed.rb,
lib/scruber/core/page_format/xml.rb,
lib/scruber/cli/project_generator.rb,
lib/scruber/core/page_format/base.rb,
lib/scruber/core/page_format/html.rb,
lib/scruber/helpers/fetcher_agent.rb,
lib/scruber/helpers/proxy_rotator.rb,
lib/scruber/queue_adapters/memory.rb,
lib/scruber/helpers/dictionary_reader.rb,
lib/scruber/core/extensions/csv_output.rb,
lib/scruber/helpers/user_agent_rotator.rb,
lib/scruber/core/extensions/queue_aliases.rb,
lib/scruber/helpers/dictionary_reader/csv.rb,
lib/scruber/helpers/dictionary_reader/xml.rb,
lib/scruber/core/extensions/parser_aliases.rb,
lib/scruber/queue_adapters/abstract_adapter.rb,
lib/scruber/fetcher_adapters/abstract_adapter.rb,
lib/scruber/fetcher_adapters/typhoeus_fetcher.rb,
lib/scruber/helpers/fetcher_agent_adapters/memory.rb,
lib/scruber/helpers/fetcher_agent_adapters/abstract_adapter.rb
Defined Under Namespace
Modules: AppSearcher, CLI, Core, Fetcher, FetcherAdapters, Helpers, Queue, QueueAdapters
Classes: ArgumentError
Constant Summary
collapse
- VERSION =
"0.1.6"
Class Attribute Summary collapse
Class Method Summary
collapse
Class Attribute Details
.configuration ⇒ Object
65
66
67
|
# File 'lib/scruber.rb', line 65
def configuration
@configuration ||= Core::Configuration.new
end
|
69
70
71
|
# File 'lib/scruber.rb', line 69
def logger
@logger ||= Scruber.root.nil? ? nil : Logger.new(Scruber.root.join('log', 'crawler.log'))
end
|
Class Method Details
73
74
75
|
# File 'lib/scruber.rb', line 73
def configure(&block)
yield configuration
end
|
77
78
79
80
81
82
83
|
# File 'lib/scruber.rb', line 77
def root
if defined?(APP_PATH)
Pathname.new(File.expand_path('../../', APP_PATH))
else
nil
end
end
|
.run(*args, &block) ⇒ Object
59
60
61
62
63
|
# File 'lib/scruber.rb', line 59
def run(*args, &block)
raise "You need a block to build!" unless block_given?
Core::Crawler.new(*args).run(&block)
end
|