Class: Benchmark::HTTP::Command::Spider
- Inherits:
-
Samovar::Command
- Object
- Samovar::Command
- Benchmark::HTTP::Command::Spider
- Includes:
- Async::Await
- Defined in:
- lib/benchmark/http/command/spider.rb
Instance Method Summary collapse
Instance Method Details
#extract_links(url, response) ⇒ Object
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 |
# File 'lib/benchmark/http/command/spider.rb', line 56 def extract_links(url, response) base = url body = response.read begin filter = LinksFilter.parse(body) rescue Async.logger.error($!) return [] end if filter.base base = base + filter.base end filter.links.collect do |href| next if href.nil? or href.empty? begin full_url = base + href if full_url.host == url.host && full_url.kind_of?(URI::HTTP) yield full_url end rescue ArgumentError, URI::InvalidURIError puts "Could not fetch #{href}, relative to #{base}." end end.compact end |
#log(method, url, response) ⇒ Object
48 49 50 51 52 53 54 |
# File 'lib/benchmark/http/command/spider.rb', line 48 def log(method, url, response) puts "#{method} #{url} -> #{response.version} #{response.status} (#{response.body&.length || 'unspecified'} bytes)" response.headers.each do |key, value| puts "\t#{key}: #{value}" end if @options[:headers] end |