Class: WebRobots::RobotsTxt::Record

Inherits:
Object
  • Object
show all
Defined in:
lib/webrobots/robotstxt.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(agentlines, rulelines) ⇒ Record

Returns a new instance of Record.



617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
 
# File 'lib/webrobots/robotstxt.rb', line 617

def initialize(agentlines, rulelines)
  @patterns = agentlines.map { |agentline| agentline.pattern }
  @acls = []
  @delay = nil
  @options = {}
  rulelines.each { |ruleline|
    case ruleline
    when AccessControlLine
      @acls << ruleline
    when CrawlDelayLine
      @delay = ruleline.delay
    else
      @options[ruleline.token.downcase] = ruleline.value
    end
  } if rulelines
  @acls.replace @acls.sort_by { |x|
    [-x.value.length, x.is_a?(AllowLine) ? -1 : 0]
  }
end

Instance Attribute Details

#delay ⇒ Object (readonly)

Returns the value of attribute delay.



637
638
639
 
# File 'lib/webrobots/robotstxt.rb', line 637

def delay
  @delay
end

#options ⇒ Object (readonly)

Returns the value of attribute options.



637
638
639
 
# File 'lib/webrobots/robotstxt.rb', line 637

def options
  @options
end

Instance Method Details

#allow?(request_uri) ⇒ Boolean

Returns:

  • (Boolean) 


649
650
651
652
653
654
655
656
 
# File 'lib/webrobots/robotstxt.rb', line 649

def allow?(request_uri)
  @acls.each { |acl|
    if acl.match?(request_uri)
      return acl.allow?
    end
  }
  return true
end

#default? ⇒ Boolean

Returns:

  • (Boolean) 


645
646
647
 
# File 'lib/webrobots/robotstxt.rb', line 645

def default?
  @patterns.include?(//)
end

#match?(user_agent) ⇒ Boolean

Returns:

  • (Boolean) 


639
640
641
642
643
 
# File 'lib/webrobots/robotstxt.rb', line 639

def match?(user_agent)
  @patterns.any? { |pattern|
    pattern.match(user_agent)
  }
end