Class: BetterSeo::Generators::RobotsTxtGenerator

Inherits:
Object
  • Object
show all
Defined in:
lib/better_seo/generators/robots_txt_generator.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ RobotsTxtGenerator

Returns a new instance of RobotsTxtGenerator.



10
11
12
13
 
# File 'lib/better_seo/generators/robots_txt_generator.rb', line 10

def initialize
  @rules = []
  @sitemaps = []
end

Instance Attribute Details

#rules ⇒ Object (readonly)

Returns the value of attribute rules.



8
9
10
 
# File 'lib/better_seo/generators/robots_txt_generator.rb', line 8

def rules
  @rules
end

#sitemaps ⇒ Object (readonly)

Returns the value of attribute sitemaps.



8
9
10
 
# File 'lib/better_seo/generators/robots_txt_generator.rb', line 8

def sitemaps
  @sitemaps
end

Instance Method Details

#add_rule(user_agent, allow: nil, disallow: nil) ⇒ Object

Add rule for user agent



16
17
18
19
20
21
22
 
# File 'lib/better_seo/generators/robots_txt_generator.rb', line 16

def add_rule(user_agent, allow: nil, disallow: nil)
  rule = { user_agent: user_agent }
  rule[:allow] = Array(allow).compact if allow
  rule[:disallow] = Array(disallow).compact if disallow
  @rules << rule
  self
end

#add_sitemap(url) ⇒ Object

Add sitemap URL



38
39
40
41
 
# File 'lib/better_seo/generators/robots_txt_generator.rb', line 38

def add_sitemap(url)
  @sitemaps << url
  self
end

#clear ⇒ Object

Clear all rules and sitemaps



44
45
46
47
48
 
# File 'lib/better_seo/generators/robots_txt_generator.rb', line 44

def clear
  @rules = []
  @sitemaps = []
  self
end

#set_crawl_delay(user_agent, seconds) ⇒ Object

Set crawl delay for user agent



25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/better_seo/generators/robots_txt_generator.rb', line 25

def set_crawl_delay(user_agent, seconds)
  rule = @rules.find { |r| r[:user_agent] == user_agent }

  if rule
    rule[:crawl_delay] = seconds
  else
    @rules << { user_agent: user_agent, crawl_delay: seconds }
  end

  self
end

#to_text ⇒ Object

Generate robots.txt content



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/better_seo/generators/robots_txt_generator.rb', line 51

def to_text
  return "" if @rules.empty? && @sitemaps.empty?

  lines = []

  # Generate rules for each user agent
  @rules.each_with_index do |rule, index|
    lines << "User-agent: #{rule[:user_agent]}"

    # Allow directives
    rule[:allow]&.each do |path|
      lines << "Allow: #{path}"
    end

    # Disallow directives
    rule[:disallow]&.each do |path|
      lines << "Disallow: #{path}"
    end

    # Crawl delay
    lines << "Crawl-delay: #{rule[:crawl_delay]}" if rule[:crawl_delay]

    # Add blank line between user agent sections (except last)
    lines << "" if index < @rules.size - 1
  end

  # Add blank line before sitemaps if there are rules
  lines << "" if @rules.any? && @sitemaps.any?

  # Add sitemaps
  @sitemaps.each do |sitemap|
    lines << "Sitemap: #{sitemap}"
  end

  lines.join("\n")
end

#write_to_file(path) ⇒ Object

Write robots.txt to file



89
90
91
92
93
 
# File 'lib/better_seo/generators/robots_txt_generator.rb', line 89

def write_to_file(path)
  directory = File.dirname(path)
  FileUtils.mkdir_p(directory) unless File.directory?(directory)
  File.write(path, to_text)
end