10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
|
# File 'lib/kudzu/agent/robots/parser.rb', line 10
def parse(body)
txt = Txt.new
sets = []
prev_key = nil
parse_body(body).each do |key, value|
case key
when 'user-agent'
new_set = RuleSet.new(user_agent: ua_regexp(value))
txt.sets << new_set
if prev_key == 'user-agent'
sets << new_set
else
sets = [new_set]
end
when 'allow'
re = path_regexp(value)
sets.each { |set| set.rules << Rule.new(path: re, allow: true) }
when 'disallow'
re = path_regexp(value)
sets.each { |set| set.rules << Rule.new(path: re, allow: false) }
when 'crawl-delay'
sets.each { |set| set.crawl_delay = value.to_i }
when 'sitemap'
txt.sitemaps << value
end
prev_key = key
end
sort(txt)
end
|