Module: Spidr

Extended by:
Settings::Proxy, Settings::Timeouts, Settings::UserAgent
Defined in:
lib/spidr/page.rb,
lib/spidr/agent.rb,
lib/spidr/proxy.rb,
lib/spidr/rules.rb,
lib/spidr/spidr.rb,
lib/spidr/version.rb,
lib/spidr/page/html.rb,
lib/spidr/auth_store.rb,
lib/spidr/cookie_jar.rb,
lib/spidr/agent/events.rb,
lib/spidr/agent/robots.rb,
lib/spidr/page/cookies.rb,
lib/spidr/agent/actions.rb,
lib/spidr/agent/filters.rb,
lib/spidr/session_cache.rb,
lib/spidr/settings/proxy.rb,
lib/spidr/auth_credential.rb,
lib/spidr/agent/sanitizers.rb,
lib/spidr/page/status_codes.rb,
lib/spidr/settings/timeouts.rb,
lib/spidr/page/content_types.rb,
lib/spidr/settings/user_agent.rb

Defined Under Namespace

Modules: Settings Classes: Agent, AuthCredential, AuthStore, CookieJar, Page, Proxy, Rules, SessionCache

Constant Summary collapse

VERSION =

Spidr version

'0.6.0'

Instance Attribute Summary

Attributes included from Settings::UserAgent

#user_agent

Attributes included from Settings::Timeouts

#continue_timeout, #keep_alive_timeout, #open_timeout, #read_timeout, #ssl_timeout

Class Method Summary collapse

Methods included from Settings::Proxy

disable_proxy!, proxy, proxy=

Class Method Details

.host(name, options = {}, &block) ⇒ Object

See Also:



46
47
48
# File 'lib/spidr/spidr.rb', line 46

def self.host(name,options={},&block)
  Agent.host(name,options,&block)
end

.robotsObject

This method is abstract.


60
61
# File 'lib/spidr/spidr.rb', line 60

def self.robots
end

.robots=(mode) ⇒ Boolean

Enables or disables robots.txt globally.

Parameters:

  • mode (Boolean)

Returns:

  • (Boolean)

Since:

  • 0.5.0



32
33
34
# File 'lib/spidr/spidr.rb', line 32

def self.robots=(mode)
  @robots = mode
end

.robots?Boolean

Specifies whether robots.txt should be honored globally.

Returns:

  • (Boolean)

Since:

  • 0.5.0



18
19
20
21
# File 'lib/spidr/spidr.rb', line 18

def self.robots?
  @robots ||= false
  @robots
end

.site(url, options = {}, &block) ⇒ Object

See Also:



53
54
55
# File 'lib/spidr/spidr.rb', line 53

def self.site(url,options={},&block)
  Agent.site(url,options,&block)
end

.start_at(url, options = {}, &block) ⇒ Object



39
40
41
# File 'lib/spidr/spidr.rb', line 39

def self.start_at(url,options={},&block)
  Agent.start_at(url,options,&block)
end