Class: Html2rss::Config

Inherits:
Object
  • Object
show all
Defined in:
lib/html2rss/config.rb,
lib/html2rss/config/schema.rb,
lib/html2rss/config/validator.rb,
lib/html2rss/config/dynamic_params.rb,
lib/html2rss/config/request_headers.rb,
lib/html2rss/config/request_controls.rb,
lib/html2rss/config/selectors_validator.rb,
lib/html2rss/config/auto_source_contract.rb,
lib/html2rss/config/multiple_feeds_config.rb

Overview

The provided configuration is used to generate the RSS feed. This class provides methods to load and process configuration from a YAML file, supporting both single and multiple feed configurations.

Configuration is validated during initialization.

Defined Under Namespace

Modules: Schema Classes: DynamicParams, InvalidConfig, MultipleFeedsConfig, Preparer, RequestControls, RequestHeaders, SelectorsValidator, ValidationResult, Validator

Constant Summary collapse

UNSET =

Sentinel to differentiate omitted params from explicit nil.

Object.new.freeze
AutoSourceContract =

Runtime source of truth for validating auto-source config values.

Dry::Schema.Params do # rubocop:disable Metrics/BlockLength
  optional(:limit).filled(:integer, gt?: 0)
  optional(:entry_resolution).hash do
    optional(:enabled).filled(:bool)
    optional(:max_probes).filled(:integer, gt?: 0)
  end

  optional(:scraper).hash do # rubocop:disable Metrics/BlockLength
    optional(:native_feed).hash do
      optional(:enabled).filled(:bool)
    end
    optional(:wordpress_api).hash do
      optional(:enabled).filled(:bool)
    end
    optional(:sitemap).hash do
      optional(:enabled).filled(:bool)
      optional(:min_priority).filled(:float)
      optional(:max_age_days).filled(:integer, gt?: 0)
    end
    optional(:schema).hash do
      optional(:enabled).filled(:bool)
    end
    optional(:microdata).hash do
      optional(:enabled).filled(:bool)
    end
    optional(:microformats2).hash do
      optional(:enabled).filled(:bool)
    end
    optional(:json_state).hash do
      optional(:enabled).filled(:bool)
    end
    optional(:xhr_articles).hash do
      optional(:enabled).filled(:bool)
    end
    optional(:meta_oembed).hash do
      optional(:enabled).filled(:bool)
    end
    optional(:semantic_html).hash do
      optional(:enabled).filled(:bool)
      optional(:fallback_anchorless).filled(:bool)
    end
    optional(:html).hash do
      optional(:enabled).filled(:bool)
      optional(:minimum_selector_frequency).filled(:integer, gt?: 0)
      optional(:use_top_selectors).filled(:integer, gt?: 0)
      optional(:fallback_anchorless).filled(:bool)
    end
  end

  optional(:cleanup).hash do
    optional(:keep_different_domain).filled(:bool)
  end
end

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config) ⇒ Config

Initializes the configuration object.

Applies default values and validates the configuration.

Parameters:

  • the configuration hash.

Raises:

  • if the configuration fails validation.



283
284
285
286
287
288
289
290
291
292
293
294
295
 
# File 'lib/html2rss/config.rb', line 283

def initialize(config)
  @request_controls = RequestControls.from_config(config)
  prepared_config = Preparer.new.call(config)
  validated_config = validated_config_for(prepared_config)

  @config = validated_config.freeze
  @request_controls = request_controls.with_effective_values(
    strategy: validated_config[:strategy],
    max_redirects: validated_config.dig(:request, :max_redirects),
    max_requests: validated_config.dig(:request, :max_requests),
    total_timeout_seconds: validated_config.dig(:request, :total_timeout_seconds)
  )
end

Instance Attribute Details

#request_controls ⇒ Html2rss::Config::RequestControls (readonly)

Returns request controls with provenance.

Returns:

  • request controls with provenance



316
317
318
 
# File 'lib/html2rss/config.rb', line 316

def request_controls
  @request_controls
end

Class Method Details

.auto_source_config(url:, items_selector: nil, request_controls: nil, limit: nil) ⇒ Hash{Symbol => Object}

Builds a top-level auto-source feed config for the public shortcut APIs.

Parameters:

  • source page URL

  • (defaults to: nil)

    optional selector hint for item extraction

  • (defaults to: nil)

    explicit request controls to write

  • (defaults to: nil)

    max articles to keep in the auto-sourced feed

Returns:



194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
 
# File 'lib/html2rss/config.rb', line 194

def auto_source_config(url:, items_selector: nil, request_controls: nil, limit: nil)
  auto_source = AutoSource::DEFAULT_CONFIG
  auto_source = auto_source.merge(limit:) unless limit.nil?

  config = {
    channel: default_config[:channel].merge(url:),
    auto_source:
  }

  request_controls ||= RequestControls.new
  request_controls.apply_to(config)

  config[:selectors] = { items: { selector: items_selector, enhance: true } } if items_selector
  config
end

.default_config ⇒ Hash{Symbol => Object}

Provides a default configuration.

Returns:

  • a hash with default configuration values.



214
215
216
217
218
219
220
221
222
 
# File 'lib/html2rss/config.rb', line 214

def default_config
  {
    strategy: default_strategy_name,
    request: default_request_config,
    channel: { time_zone: 'UTC' },
    headers: RequestHeaders.browser_defaults,
    stylesheets: Html2rss.defaults.stylesheets || []
  }
end

.default_strategy_name ⇒ Symbol

Returns the default feed-level strategy plan (+:auto+ or concrete).

Returns:

  • the default feed-level strategy plan (+:auto+ or concrete)



225
226
227
 
# File 'lib/html2rss/config.rb', line 225

def default_strategy_name
  Html2rss.defaults.default_strategy || :auto
end

.from_hash(config, params: UNSET) ⇒ Html2rss::Config

Processes the provided configuration hash, applying dynamic parameters if given, and returns a new configuration object.

Parameters:

  • the configuration hash.

  • (defaults to: UNSET)

    dynamic parameters for string formatting.

Returns:

  • the configuration object.



182
183
184
 
# File 'lib/html2rss/config.rb', line 182

def from_hash(config, params: UNSET)
  new(resolve_effective_config(config, params:))
end

.from_yaml(string) ⇒ Hash{Symbol => Object}

Parses a YAML configuration string into a symbol-keyed hash.

Does not validate. Call validate or from_hash after this.

Parameters:

  • YAML document

Returns:

  • configuration hash

Raises:

  • if string is not a String or does not deserialize to a Hash



135
136
137
138
139
140
141
142
 
# File 'lib/html2rss/config.rb', line 135

def from_yaml(string)
  raise ArgumentError, 'YAML must be a String' unless string.is_a?(String)

  parsed = YAML.safe_load(string)
  raise ArgumentError, 'YAML must deserialize to a Hash' unless parsed.is_a?(Hash)

  HashUtil.deep_symbolize_keys(parsed, context: 'config')
end

.json_schema ⇒ Hash{String => Object}

Returns the exported JSON Schema for html2rss configuration.

Returns:

  • JSON Schema represented as a Ruby hash



50
51
52
 
# File 'lib/html2rss/config.rb', line 50

def json_schema
  Schema.json_schema
end

.json_schema_json(pretty: true) ⇒ String

Returns the exported JSON Schema as JSON.

Parameters:

  • (defaults to: true)

    whether to pretty-print the JSON output

Returns:

  • serialized JSON Schema



59
60
61
 
# File 'lib/html2rss/config.rb', line 59

def json_schema_json(pretty: true)
  pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
end

.load_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS) ⇒ Hash{Symbol => Object}

Loads the feed configuration from a YAML file.

Supports multiple feeds defined under the specified key (default :feeds).

rubocop:disable-next Metrics/MethodLength

Parameters:

  • the YAML file to load.

  • (defaults to: nil)

    the feed name when using multiple feeds.

  • (defaults to: MultipleFeedsConfig::CONFIG_KEY_FEEDS)

    the key under which multiple feeds are defined.

Returns:

  • the configuration hash.

Raises:

  • if the file doesn't exist or feed is not found.



155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/html2rss/config.rb', line 155

def load_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS)
  raise ArgumentError, "File '#{file}' does not exist" unless File.exist?(file)
  raise ArgumentError, "`#{multiple_feeds_key}` is a reserved feed name" if feed_name == multiple_feeds_key

  yaml = YAML.safe_load_file(file, symbolize_names: true)

  return yaml unless yaml.key?(multiple_feeds_key)

  unless feed_name
    available_feeds = yaml.fetch(multiple_feeds_key).keys.join(', ')
    raise ArgumentError,
          "Feed name is required under `#{multiple_feeds_key}`. Available feeds: #{available_feeds}"
  end

  config = yaml.dig(multiple_feeds_key, feed_name.to_sym)
  raise ArgumentError, "Feed '#{feed_name}' not found under `#{multiple_feeds_key}` key." unless config

  MultipleFeedsConfig.to_single_feed(config, yaml, multiple_feeds_key:)
end

.resolve_and_validate(config_input, feed_name: nil, params: {}) ⇒ Array(Hash, Dry::Validation::Result, Html2rss::Config::ValidationResult)

Resolves a Hash, YAML file path, or YAML string to a working config Hash and validates it. The returned Hash is a deep copy — callers may stamp strategy/params without mutating input.

Parameters:

  • config hash, YAML string, or file path

  • (defaults to: nil)

    optional feed name for multi-feed files

  • (defaults to: {})

    dynamic feed params

Returns:



71
72
73
74
75
76
77
 
# File 'lib/html2rss/config.rb', line 71

def resolve_and_validate(config_input, feed_name: nil, params: {})
  param_arg = params.empty? ? UNSET : params
  working = HashUtil.deep_dup(resolve_raw_hash(config_input, feed_name))
  [working, validate(working, params: param_arg)]
rescue StandardError => error
  [{}, ValidationResult.parse_failure(error.message)]
end

.schema_path ⇒ String

Returns the packaged JSON Schema file path.

Returns:

  • absolute path to the packaged JSON Schema file



112
113
114
 
# File 'lib/html2rss/config.rb', line 112

def schema_path
  Schema.path
end

.to_yaml(hash) ⇒ String

Serializes a configuration hash to string-key YAML.

This is the single serializer for CLI capture and MCP capture.

Parameters:

  • configuration hash (symbol or string keys)

Returns:

  • YAML document without Ruby symbol-key prefixes



123
124
125
 
# File 'lib/html2rss/config.rb', line 123

def to_yaml(hash)
  YAML.dump(HashUtil.deep_stringify_keys(hash))
end

.validate(config, params: UNSET) ⇒ Dry::Validation::Result

Validates a configuration hash with the runtime validator.

Parameters:

  • the configuration hash

  • (defaults to: UNSET)

    dynamic parameters for string formatting

Returns:

  • validation result after defaults are applied



85
86
87
88
89
90
91
92
93
94
 
# File 'lib/html2rss/config.rb', line 85

def validate(config, params: UNSET)
  prepared_config = prepare_for_validation(resolve_effective_config(config, params:))

  Validator.new.call(prepared_config)
rescue DynamicParams::ParamsMissing => error
  prepared_config = prepare_for_validation(HashUtil.deep_symbolize_keys(config, context: 'config'))
  prepared_config[:dynamic_params_error] = error.message

  Validator.new.call(prepared_config)
end

.validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET) ⇒ Dry::Validation::Result, Html2rss::Config::ValidationResult

Loads and validates a YAML configuration file.

Parameters:

  • the YAML file to load

  • (defaults to: nil)

    optional feed name for multi-feed files

  • (defaults to: MultipleFeedsConfig::CONFIG_KEY_FEEDS)

    key under which multiple feeds are defined

  • (defaults to: UNSET)

    dynamic parameters for string formatting

Returns:



104
105
106
 
# File 'lib/html2rss/config.rb', line 104

def validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET)
  validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
end

Instance Method Details

#auto_source ⇒ Hash{Symbol => Object, nil}

Returns auto-source configuration.

Returns:

  • auto-source configuration



337
 
# File 'lib/html2rss/config.rb', line 337

def auto_source = config[:auto_source]

#channel ⇒ Hash{Symbol => Object}

Returns channel configuration.

Returns:

  • channel configuration



321
322
323
324
325
 
# File 'lib/html2rss/config.rb', line 321

def channel = config[:channel]
##
# Source channel URL (also the default scrape URL).
#
# @return [String]

#explicit_max_requests? ⇒ Boolean

Returns whether max_requests was explicitly configured by the caller.

Returns:

  • whether max_requests was explicitly configured by the caller



310
311
312
 
# File 'lib/html2rss/config.rb', line 310

def explicit_max_requests?
  request_controls.explicit?(:max_requests)
end

#headers ⇒ Hash{String => String}

Returns normalized HTTP headers.

Returns:

  • normalized HTTP headers



319
320
 
# File 'lib/html2rss/config.rb', line 319

def headers = config[:headers]
# @return [Hash{Symbol => Object}] channel configuration

#max_redirects ⇒ Integer?

Returns configured redirect budget.

Returns:

  • configured redirect budget



300
301
 
# File 'lib/html2rss/config.rb', line 300

def max_redirects = request_controls.max_redirects
# @return [Integer, nil] configured request budget

#max_requests ⇒ Integer?

Returns configured request budget.

Returns:

  • configured request budget



302
303
 
# File 'lib/html2rss/config.rb', line 302

def max_requests = request_controls.max_requests
# @return [Integer, nil] configured request timeout

#request ⇒ Hash{Symbol => Object}

Returns request envelope configuration.

Returns:

  • request envelope configuration



332
 
# File 'lib/html2rss/config.rb', line 332

def request = config[:request]

#selectors ⇒ Hash{Symbol => Object, nil}

Returns selectors configuration.

Returns:

  • selectors configuration



335
336
 
# File 'lib/html2rss/config.rb', line 335

def selectors = config[:selectors]
# @return [Hash{Symbol => Object, nil}] auto-source configuration

#strategy ⇒ Symbol?

Returns selected request strategy.

Returns:

  • selected request strategy



298
299
 
# File 'lib/html2rss/config.rb', line 298

def strategy = request_controls.strategy
# @return [Integer, nil] configured redirect budget

#stylesheets ⇒ Array<Hash>

Returns stylesheet definitions.

Returns:

  • stylesheet definitions



306
 
# File 'lib/html2rss/config.rb', line 306

def stylesheets = config[:stylesheets]

#time_zone ⇒ String?

Returns configured channel time zone.

Returns:

  • configured channel time zone



329
 
# File 'lib/html2rss/config.rb', line 329

def time_zone = config.dig(:channel, :time_zone)

#total_timeout_seconds ⇒ Integer?

Returns configured request timeout.

Returns:

  • configured request timeout



304
305
 
# File 'lib/html2rss/config.rb', line 304

def total_timeout_seconds = request_controls.total_timeout_seconds
# @return [Array<Hash>] stylesheet definitions

#url ⇒ String

Source channel URL (also the default scrape URL).

Returns:



326
 
# File 'lib/html2rss/config.rb', line 326

def url = config.dig(:channel, :url)