Module: Wayfarer

Defined in:

lib/wayfarer/routing/serializable.rb,
lib/wayfarer.rb,
lib/wayfarer/gc.rb,
lib/wayfarer/kv.rb,
lib/wayfarer/cli.rb,
lib/wayfarer/base.rb,
lib/wayfarer/page.rb,
lib/wayfarer/task.rb,
lib/wayfarer/handler.rb,
lib/wayfarer/logging.rb,
lib/wayfarer/parsing.rb,
lib/wayfarer/callbacks.rb,
lib/wayfarer/stringify.rb,
lib/wayfarer/redis/pool.rb,
lib/wayfarer/serializer.rb,
lib/wayfarer/parsing/xml.rb,
lib/wayfarer/routing/dsl.rb,
lib/wayfarer/parsing/json.rb,
lib/wayfarer/redis/barrier.rb,
lib/wayfarer/redis/counter.rb,
lib/wayfarer/routing/route.rb,
lib/wayfarer/routing/result.rb,
lib/wayfarer/middleware/base.rb,
lib/wayfarer/middleware/lazy.rb,
lib/wayfarer/networking/http.rb,
lib/wayfarer/networking/pool.rb,
lib/wayfarer/batch_completion.rb,
lib/wayfarer/middleware/chain.rb,
lib/wayfarer/middleware/dedup.rb,
lib/wayfarer/middleware/redis.rb,
lib/wayfarer/middleware/stage.rb,
lib/wayfarer/redis/resettable.rb,
lib/wayfarer/cli/route_printer.rb,
lib/wayfarer/middleware/router.rb,
lib/wayfarer/networking/ferrum.rb,
lib/wayfarer/networking/follow.rb,
lib/wayfarer/networking/result.rb,
lib/wayfarer/routing/sub_route.rb,
lib/wayfarer/uri/normalization.rb,
lib/wayfarer/networking/context.rb,
lib/wayfarer/routing/hash_stack.rb,
lib/wayfarer/routing/root_route.rb,
lib/wayfarer/middleware/dispatch.rb,
lib/wayfarer/networking/capybara.rb,
lib/wayfarer/networking/selenium.rb,
lib/wayfarer/networking/strategy.rb,
lib/wayfarer/routing/path_finder.rb,
lib/wayfarer/middleware/normalize.rb,
lib/wayfarer/routing/matchers/url.rb,
lib/wayfarer/routing/target_route.rb,
lib/wayfarer/middleware/controller.rb,
lib/wayfarer/middleware/uri_parser.rb,
lib/wayfarer/middleware/user_agent.rb,
lib/wayfarer/routing/matchers/host.rb,
lib/wayfarer/routing/matchers/path.rb,
lib/wayfarer/routing/path_consumer.rb,
lib/wayfarer/routing/matchers/query.rb,
lib/wayfarer/middleware/content_type.rb,
lib/wayfarer/routing/matchers/custom.rb,
lib/wayfarer/routing/matchers/result.rb,
lib/wayfarer/routing/matchers/scheme.rb,
lib/wayfarer/middleware/batch_completion.rb,
lib/wayfarer/routing/matchers/empty_params.rb

Overview

lib/wayfarer/routing/hash_serialisable.rb frozen_string_literal: true

Defined Under Namespace

Modules: Base, BatchCompletion, Callbacks, GC, Handler, KV, Logging, Middleware, Networking, Parsing, Redis, Routing, Stringify, URI, VERSION Classes: CLI, Page, Serializer, Task

Constant Summary

DEFAULT_CONFIG =

{
  redis: {
    url: "redis://localhost:6379/0",
    factory: ->(redis_config) { ::Redis.new(url: redis_config.fetch(:url)) }
  },
  network: {
    agent: :http,
    pool: {
      size: 1,
      timeout: 10
    },
    http_headers: {},
    renew_on: []
  },
  parsing: {
    registry: {
      "application/json" => Wayfarer::Parsing::JSON,
      "text/html" => [Wayfarer::Parsing::XML, :html],
      "application/xml" => [Wayfarer::Parsing::XML, :xml]
    }
  },
  normalization: {
    remove_www: true,
    remove_trailing_slash: true,
    remove_fragment: true,
    remove_tracking_parameters: true,
    order_query_parameters: true,
    schemes: %w[
      http
      https
    ].to_set,
    tracking_params: %w[
      utm_source
      utm_medium
      utm_term
      utm_content
      utm_campaign
      gclid
      fbclid
      msclkid
      sms_ss
      awesm
      xtor
      PHPSESSID
    ].to_set
  },
  deduplication: {
    key: ->(task) { task[:uri].to_s }
  },
  capybara: {
    driver: nil
  },
  ferrum: {
    options: {}
  },
  selenium: {
    driver: :chrome,
    options: {}
  }
}.freeze

UserAgentTimeoutError =

TODO: Move to Networking namespace

Class.new(StandardError)