Method: Robots#initialize

Defined in:
lib/robots.rb

#initialize(options) ⇒ Robots

Processes the robots.txt file



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/robots.rb', line 5

def initialize(options)
  @options = options
  raise "options should be a hash" unless options.kind_of? Hash
  raise ":url is required" unless @options.has_key? :url
  @options[:file] = "robots.txt" unless @options.has_key? :file
  @options[:user_agent] = "cobweb" unless @options.has_key? :user_agent

  uri = URI.parse(@options[:url])
  content = Cobweb.new(:cache => nil, :text_mime_types => ["text/html", "application/xhtml+xml", "text/plain"]).get([uri.scheme, "://", uri.host, ":", uri.port, "/", @options[:file]].join)
  if content[:mime_type][0..4] == "text/"
    @raw_data = parse_data(content[:body])

    if @options.has_key?(:user_agent) && @raw_data.has_key?(@options[:user_agent].to_s.downcase.to_sym)
      @params = @raw_data[@options[:user_agent].to_s.downcase.to_sym]
    else
      raise "Wildcard user-agent is not present" unless @raw_data.has_key? :*
      @params = @raw_data[:*]
    end
  else
    raise "Invalid mime type: #{content[:content_type]}"
  end
end