Class: Ferrum::Network

Inherits:
Object
  • Object
show all
Defined in:
lib/ferrum/network.rb,
lib/ferrum/network/error.rb,
lib/ferrum/network/request.rb,
lib/ferrum/network/exchange.rb,
lib/ferrum/network/response.rb,
lib/ferrum/network/auth_request.rb,
lib/ferrum/network/request_params.rb,
lib/ferrum/network/intercepted_request.rb

Defined Under Namespace

Modules: RequestParams Classes: AuthRequest, Error, Exchange, InterceptedRequest, Request, Response

Constant Summary collapse

CLEAR_TYPE = 
i[traffic cache].freeze
AUTHORIZE_TYPE = 
i[server proxy].freeze
REQUEST_STAGES = 
i[Request Response].freeze
RESOURCE_TYPES = 
i[Document Stylesheet Image Media Font Script TextTrack
XHR Fetch Prefetch EventSource WebSocket Manifest
SignedExchange Ping CSPViolationReport Preflight Other].freeze
AUTHORIZE_BLOCK_MISSING = 
"Block is missing, call `authorize(...) { |r| r.continue } " \
"or subscribe to `on(:request)` events before calling it"
AUTHORIZE_TYPE_WRONG = 
":type should be in #{AUTHORIZE_TYPE}"
ALLOWED_CONNECTION_TYPE = 
%w[none cellular2g cellular3g cellular4g bluetooth ethernet wifi wimax other].freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(page) ⇒ Network

Returns a new instance of Network.



35
36
37
38
39
40
41
 
# File 'lib/ferrum/network.rb', line 35

def initialize(page)
  @page = page
  @traffic = []
  @exchange = nil
  @blacklist = nil
  @whitelist = nil
end

Instance Attribute Details

#traffic ⇒ Array<Exchange> (readonly)

Network traffic.

Examples:

browser.go_to("https://github.com/")
browser.network.traffic # => [#<Ferrum::Network::Exchange, ...]

Returns:

  • (Array<Exchange>) —

    Returns all information about network traffic as Exchange instance which in general is a wrapper around request, response and error.



33
34
35
 
# File 'lib/ferrum/network.rb', line 33

def traffic
  @traffic
end

Instance Method Details

#authorize(user:, password:, type: :server) {|request| ... } ⇒ Object

Sets HTTP Basic-Auth credentials.

Examples:

browser.network.authorize(user: "login", password: "pass") { |req| req.continue }
browser.go_to("http://example.com/authenticated")
puts browser.network.status # => 200
puts browser.body # => Welcome, authenticated client

Parameters:

  • user (String) —

    The username to send.

  • password (String) —

    The password to send.

  • type (:server, :proxy) (defaults to: :server) —

    Specifies whether the credentials are for a website or a proxy.

Yields:

  • (request) —

    The given block will be passed each authenticated request and can allow or deny the request.

Yield Parameters:

  • request (Request) —

    An HTTP request.

Raises:

  • (ArgumentError) 


244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
 
# File 'lib/ferrum/network.rb', line 244

def authorize(user:, password:, type: :server, &block)
  raise ArgumentError, AUTHORIZE_TYPE_WRONG unless AUTHORIZE_TYPE.include?(type)
  raise ArgumentError, AUTHORIZE_BLOCK_MISSING if !block_given? && !@page.subscribed?("Fetch.requestPaused")

  @authorized_ids ||= {}
  @authorized_ids[type] ||= []

  intercept

  @page.on(:request, &block)

  @page.on(:auth) do |request, index, total|
    if request.auth_challenge?(type)
      response = authorized_response(@authorized_ids[type],
                                     request.request_id,
                                     user, password)

      @authorized_ids[type] << request.request_id
      request.continue(authChallengeResponse: response)
    elsif index + 1 < total
      next # There are other callbacks that can handle this
    else
      request.abort
    end
  end
end

#authorized_response(ids, request_id, username, password) ⇒ Object 



279
280
281
282
283
284
285
286
287
 
# File 'lib/ferrum/network.rb', line 279

def authorized_response(ids, request_id, username, password)
  if ids.include?(request_id)
    { response: "CancelAuth" }
  elsif username && password
    { response: "ProvideCredentials",
      username: username,
      password: password }
  end
end

#blacklist=(patterns) ⇒ Object Also known as: blocklist= 



167
168
169
170
 
# File 'lib/ferrum/network.rb', line 167

def blacklist=(patterns)
  @blacklist = Array(patterns)
  blacklist_subscribe
end

#build_exchange(id) ⇒ Object 



293
294
295
 
# File 'lib/ferrum/network.rb', line 293

def build_exchange(id)
  Network::Exchange.new(@page, id).tap { |e| @traffic << e }
end

#cache(disable:) ⇒ Object

Toggles ignoring cache for each request. If true, cache will not be used.

Examples:

browser.network.cache(disable: true)



361
362
363
 
# File 'lib/ferrum/network.rb', line 361

def cache(disable:)
  @page.command("Network.setCacheDisabled", cacheDisabled: disable)
end

#clear(type) ⇒ true

Clear browser’s cache or collected traffic.

Examples:

traffic = browser.network.traffic # => []
browser.go_to("https://github.com/")
traffic.size # => 51
browser.network.clear(:traffic)
traffic.size # => 0

Parameters:

  • type (:traffic, :cache) —

    The type of traffic to clear.

Returns:

  • (true)

Raises:

  • (ArgumentError) 


155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/ferrum/network.rb', line 155

def clear(type)
  raise ArgumentError, ":type should be in #{CLEAR_TYPE}" unless CLEAR_TYPE.include?(type)

  if type == :traffic
    @traffic.clear
  else
    @page.command("Network.clearBrowserCache")
  end

  true
end

#emulate_network_conditions(offline: false, latency: 0, download_throughput: -1,, upload_throughput: -1,, connection_type: nil) ⇒ Object

Activates emulation of network conditions.

Examples:

browser.network.emulate_network_conditions(connection_type: "cellular2g")
browser.go_to("https://github.com/")

Parameters:

  • offline (Boolean) (defaults to: false) —

    Emulate internet disconnection,

  • latency (Integer) (defaults to: 0) —

    Minimum latency from request sent to response headers received (ms).

  • download_throughput (Integer) (defaults to: -1,) —

    Maximal aggregated download throughput (bytes/sec).

  • upload_throughput (Integer) (defaults to: -1,) —

    Maximal aggregated upload throughput (bytes/sec).

  • connection_type (String, nil) (defaults to: nil) —

    Connection type if known:

    • ‘“none”`

    • ‘“cellular2g”`

    • ‘“cellular3g”`

    • ‘“cellular4g”`

    • ‘“bluetooth”`

    • ‘“ethernet”`

    • ‘“wifi”`

    • ‘“wimax”`

    • ‘“other”`



328
329
330
331
332
333
334
335
336
337
338
339
340
341
 
# File 'lib/ferrum/network.rb', line 328

def emulate_network_conditions(offline: false, latency: 0,
                               download_throughput: -1, upload_throughput: -1,
                               connection_type: nil)
  params = {
    offline: offline, latency: latency,
    downloadThroughput: download_throughput,
    uploadThroughput: upload_throughput
  }

  params[:connectionType] = connection_type if connection_type && ALLOWED_CONNECTION_TYPE.include?(connection_type)

  @page.command("Network.emulateNetworkConditions", **params)
  true
end

#finished_connections ⇒ Object 



92
93
94
 
# File 'lib/ferrum/network.rb', line 92

def finished_connections
  @traffic.count(&:finished?)
end

#idle?(connections = 0) ⇒ Boolean

Returns:

  • (Boolean) 


84
85
86
 
# File 'lib/ferrum/network.rb', line 84

def idle?(connections = 0)
  pending_connections <= connections
end

#intercept(pattern: "*", resource_type: nil, request_stage: nil, handle_auth_requests: true) ⇒ Object

Set request interception for given options. This method is only sets request interception, you should use on callback to catch requests and abort or continue them.

Examples:

browser = Ferrum::Browser.new
browser.network.intercept
browser.on(:request) do |request|
  if request.match?(/bla-bla/)
    request.abort
  elsif request.match?(/lorem/)
    request.respond(body: "Lorem ipsum")
  else
    request.continue
  end
end
browser.go_to("https://google.com")

Parameters:



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/ferrum/network.rb', line 203

def intercept(pattern: "*", resource_type: nil, request_stage: nil, handle_auth_requests: true)
  pattern = { urlPattern: pattern }

  if resource_type && RESOURCE_TYPES.none?(resource_type.to_sym)
    raise ArgumentError, "Unknown resource type '#{resource_type}' must be #{RESOURCE_TYPES.join(' | ')}"
  end

  if request_stage && REQUEST_STAGES.none?(request_stage.to_sym)
    raise ArgumentError, "Unknown request stage '#{request_stage}' must be #{REQUEST_STAGES.join(' | ')}"
  end

  pattern[:resourceType] = resource_type if resource_type
  pattern[:requestStage] = request_stage if request_stage
  @page.command("Fetch.enable", patterns: [pattern], handleAuthRequests: handle_auth_requests)
end

#offline_mode ⇒ Object

Activates offline mode for a page.

Examples:

browser.network.offline_mode
browser.go_to("https://github.com/")
  # => Request to https://github.com/ failed (net::ERR_INTERNET_DISCONNECTED) (Ferrum::StatusError)



351
352
353
 
# File 'lib/ferrum/network.rb', line 351

def offline_mode
  emulate_network_conditions(offline: true, latency: 0, download_throughput: 0, upload_throughput: 0)
end

#pending_connections ⇒ Object 



96
97
98
 
# File 'lib/ferrum/network.rb', line 96

def pending_connections
  total_connections - finished_connections
end

#request ⇒ Request?

Page request of the main frame.

Examples:

browser.go_to("https://github.com/")
browser.network.request # => #<Ferrum::Network::Request...

Returns:



109
110
111
 
# File 'lib/ferrum/network.rb', line 109

def request
  @exchange&.request
end

#response ⇒ Response?

Page response of the main frame.

Examples:

browser.go_to("https://github.com/")
browser.network.response # => #<Ferrum::Network::Response...

Returns:



122
123
124
 
# File 'lib/ferrum/network.rb', line 122

def response
  @exchange&.response
end

#select(request_id) ⇒ Object 



289
290
291
 
# File 'lib/ferrum/network.rb', line 289

def select(request_id)
  @traffic.select { |e| e.id == request_id }
end

#status ⇒ Integer?

Contains the status code of the main page response (e.g., 200 for a success). This is just a shortcut for response.status.

Examples:

browser.go_to("https://github.com/")
browser.network.status # => 200

Returns:

  • (Integer, nil) 


136
137
138
 
# File 'lib/ferrum/network.rb', line 136

def status
  response&.status
end

#subscribe ⇒ Object 



271
272
273
274
275
276
277
 
# File 'lib/ferrum/network.rb', line 271

def subscribe
  subscribe_request_will_be_sent
  subscribe_response_received
  subscribe_loading_finished
  subscribe_loading_failed
  subscribe_log_entry_added
end

#total_connections ⇒ Object 



88
89
90
 
# File 'lib/ferrum/network.rb', line 88

def total_connections
  @traffic.size
end

#wait_for_idle(connections: 0, duration: 0.05, timeout: @page.timeout) ⇒ Boolean

Waits for network idle.

Examples:

browser.go_to("https://example.com/")
browser.at_xpath("//a[text() = 'No UI changes button']").click
browser.network.wait_for_idle # => false

Parameters:

  • connections (Integer) (defaults to: 0) —

    how many connections are allowed for network to be idling,

  • duration (Float) (defaults to: 0.05) —

    Sleep for given amount of time and check again.

  • timeout (Float) (defaults to: @page.timeout) —

    During what time we try to check idle.

Returns:

  • (Boolean) 


62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/ferrum/network.rb', line 62

def wait_for_idle(connections: 0, duration: 0.05, timeout: @page.timeout)
  start = Utils::ElapsedTime.monotonic_time

  until idle?(connections)
    return false if Utils::ElapsedTime.timeout?(start, timeout)

    sleep(duration)
  end

  true
end

#wait_for_idle! ⇒ Object

Waits for network idle or raises TimeoutError error. Accepts same arguments as wait_for_idle.

Raises:



79
80
81
82
 
# File 'lib/ferrum/network.rb', line 79

def wait_for_idle!(...)
  result = wait_for_idle(...)
  raise TimeoutError unless result
end

#whitelist=(patterns) ⇒ Object Also known as: allowlist= 



173
174
175
176
 
# File 'lib/ferrum/network.rb', line 173

def whitelist=(patterns)
  @whitelist = Array(patterns)
  whitelist_subscribe
end