Class: Benchmark::HTTP::Command::Spider
- Inherits:
-
Samovar::Command
- Object
- Samovar::Command
- Benchmark::HTTP::Command::Spider
- Includes:
- Async::Await
- Defined in:
- lib/benchmark/http/command/spider.rb
Instance Method Summary collapse
Instance Method Details
#extract_links(url, response) ⇒ Object
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
# File 'lib/benchmark/http/command/spider.rb', line 56 def extract_links(url, response) base = url body = response.read begin filter = LinksFilter.parse(body) rescue Async.logger.error($!) return [] end if filter.base base = base + filter.base end filter.links.collect do |href| begin full_url = base + href if full_url.host == url.host && full_url.kind_of?(URI::HTTP) yield full_url end rescue ArgumentError, URI::InvalidURIError puts "Could not fetch #{href}, relative to #{base}." end end.compact end |
#log(method, url, response) ⇒ Object
48 49 50 51 52 53 54 |
# File 'lib/benchmark/http/command/spider.rb', line 48 def log(method, url, response) puts "#{method} #{url} -> #{response.version} #{response.status} (#{response.body&.length || 'unspecified'} bytes)" response.headers.each do |key, value| puts "\t#{key}: #{value}" end if @options[:headers] end |