Class: Kudzu::Agent::Robots::Parser

Inherits:
Object
  • Object
show all
Defined in:
lib/kudzu/agent/robots/parser.rb

Constant Summary collapse

UNMATCH_REGEXP =
/^$/

Class Method Summary collapse

Class Method Details

.parse(body) ⇒ Object


8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# File 'lib/kudzu/agent/robots/parser.rb', line 8

def parse(body)
  txt = Txt.new
  sets = []
  prev_key = nil

  parse_body(body).each do |key, value|
    case key
    when 'user-agent'
      new_set = RuleSet.new(user_agent: ua_regexp(value))
      txt.sets << new_set
      if prev_key == 'user-agent'
        sets << new_set
      else
        sets = [new_set]
      end
    when 'allow'
      re = path_regexp(value)
      sets.each { |set| set.rules << Rule.new(path: re, allow: true) }
    when 'disallow'
      re = path_regexp(value)
      sets.each { |set| set.rules << Rule.new(path: re, allow: false) }
    when 'crawl-delay'
      sets.each { |set| set.crawl_delay = value.to_i }
    when 'sitemap'
      txt.sitemaps << value
    end

    prev_key = key
  end

  sort(txt)
end