Class: WebRobots::RobotsTxt::Record

Inherits:
Object
  • Object
show all
Defined in:
lib/webrobots/robotstxt.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(agentlines, rulelines) ⇒ Record



604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
# File 'lib/webrobots/robotstxt.rb', line 604

def initialize(agentlines, rulelines)
  @patterns = agentlines.map { |agentline| agentline.pattern }
  @acls = []
  @delay = nil
  @options = {}
  rulelines.each { |ruleline|
    case ruleline
    when AccessControlLine
      @acls << ruleline
    when CrawlDelayLine
      @delay = ruleline.delay
    else
      @options[ruleline.token.downcase] = ruleline.value
    end
  }
  @acls.replace @acls.sort_by { |x|
    [-x.value.length, x.is_a?(AllowLine) ? -1 : 0]
  }
end

Instance Attribute Details

#delayObject (readonly)

Returns the value of attribute delay



624
625
626
# File 'lib/webrobots/robotstxt.rb', line 624

def delay
  @delay
end

#optionsObject (readonly)

Returns the value of attribute options



624
625
626
# File 'lib/webrobots/robotstxt.rb', line 624

def options
  @options
end

Instance Method Details

#allow?(request_uri) ⇒ Boolean



636
637
638
639
640
641
642
643
# File 'lib/webrobots/robotstxt.rb', line 636

def allow?(request_uri)
  @acls.each { |acl|
    if acl.match?(request_uri)
      return acl.allow?
    end
  }
  return true
end

#default?Boolean



632
633
634
# File 'lib/webrobots/robotstxt.rb', line 632

def default?
  @patterns.include?(//)
end

#match?(user_agent) ⇒ Boolean



626
627
628
629
630
# File 'lib/webrobots/robotstxt.rb', line 626

def match?(user_agent)
  @patterns.any? { |pattern|
    pattern.match(user_agent)
  }
end