Module: Html2rss
- Defined in:
- lib/html2rss.rb,
lib/html2rss/cli.rb,
lib/html2rss/mcp.rb,
lib/html2rss/sst.rb,
lib/html2rss/url.rb,
lib/html2rss/html.rb,
lib/html2rss/test.rb,
lib/html2rss/batch.rb,
lib/html2rss/error.rb,
lib/html2rss/recon.rb,
lib/html2rss/config.rb,
lib/html2rss/status.rb,
lib/html2rss/article.rb,
lib/html2rss/capture.rb,
lib/html2rss/channel.rb,
lib/html2rss/scoring.rb,
lib/html2rss/version.rb,
lib/html2rss/defaults.rb,
lib/html2rss/sst/node.rb,
lib/html2rss/sst/tags.rb,
lib/html2rss/sst/text.rb,
lib/html2rss/hash_util.rb,
lib/html2rss/selectors.rb,
lib/html2rss/sst/attrs.rb,
lib/html2rss/sst/index.rb,
lib/html2rss/cli/render.rb,
lib/html2rss/html/probe.rb,
lib/html2rss/mcp/server.rb,
lib/html2rss/page_recon.rb,
lib/html2rss/auto_source.rb,
lib/html2rss/feed_result.rb,
lib/html2rss/mcp/outcome.rb,
lib/html2rss/mcp/runtime.rb,
lib/html2rss/syndication.rb,
lib/html2rss/cli/validate.rb,
lib/html2rss/feed_builder.rb,
lib/html2rss/mcp/contract.rb,
lib/html2rss/sst/document.rb,
lib/html2rss/config/schema.rb,
lib/html2rss/feed_pipeline.rb,
lib/html2rss/scoring/score.rb,
lib/html2rss/scrape_target.rb,
lib/html2rss/cli/probe_view.rb,
lib/html2rss/html/card_walk.rb,
lib/html2rss/html/feed_link.rb,
lib/html2rss/html/navigator.rb,
lib/html2rss/html/rendering.rb,
lib/html2rss/scoring/engine.rb,
lib/html2rss/sst/normalizer.rb,
lib/html2rss/feed_resolution.rb,
lib/html2rss/request_service.rb,
lib/html2rss/request_session.rb,
lib/html2rss/config/validator.rb,
lib/html2rss/feed_builder/rss.rb,
lib/html2rss/link_destination.rb,
lib/html2rss/mcp/server/tools.rb,
lib/html2rss/surface_category.rb,
lib/html2rss/article/enclosure.rb,
lib/html2rss/doctor/botasaurus.rb,
lib/html2rss/html/article_rules.rb,
lib/html2rss/syndication/parser.rb,
lib/html2rss/test/enhance_audit.rb,
lib/html2rss/auto_source/cleanup.rb,
lib/html2rss/auto_source/scraper.rb,
lib/html2rss/auto_source/segment.rb,
lib/html2rss/mcp/config_argument.rb,
lib/html2rss/scoring/observation.rb,
lib/html2rss/article/deduplicator.rb,
lib/html2rss/feed_resolution/diag.rb,
lib/html2rss/mcp/outcome/playbook.rb,
lib/html2rss/selectors/extractors.rb,
lib/html2rss/selectors/item_scope.rb,
lib/html2rss/selectors/schema_doc.rb,
lib/html2rss/auto_source/segmenter.rb,
lib/html2rss/config/dynamic_params.rb,
lib/html2rss/feed_resolution/probe.rb,
lib/html2rss/request_session/pager.rb,
lib/html2rss/scoring/link_resolver.rb,
lib/html2rss/syndication/discovery.rb,
lib/html2rss/config/request_headers.rb,
lib/html2rss/feed_builder/json_feed.rb,
lib/html2rss/feed_resolution/policy.rb,
lib/html2rss/feed_resolution/scorer.rb,
lib/html2rss/html/article_extractor.rb,
lib/html2rss/page_recon/diagnostics.rb,
lib/html2rss/request_service/budget.rb,
lib/html2rss/request_service/policy.rb,
lib/html2rss/scoring/cluster_scorer.rb,
lib/html2rss/scoring/ranked_segment.rb,
lib/html2rss/config/request_controls.rb,
lib/html2rss/feed_resolution/options.rb,
lib/html2rss/html/article_rules/date.rb,
lib/html2rss/request_service/context.rb,
lib/html2rss/auto_source/scraper/html.rb,
lib/html2rss/html/article_rules/image.rb,
lib/html2rss/request_service/response.rb,
lib/html2rss/request_service/strategy.rb,
lib/html2rss/selectors/extractors/href.rb,
lib/html2rss/selectors/extractors/html.rb,
lib/html2rss/selectors/extractors/text.rb,
lib/html2rss/selectors/post_processors.rb,
lib/html2rss/auto_source/scraper/schema.rb,
lib/html2rss/auto_source/segmenter/list.rb,
lib/html2rss/config/selectors_validator.rb,
lib/html2rss/html/sst_article_extractor.rb,
lib/html2rss/request_session/pager/base.rb,
lib/html2rss/scoring/container_assessor.rb,
lib/html2rss/auto_source/scraper/sitemap.rb,
lib/html2rss/config/auto_source_contract.rb,
lib/html2rss/feed_builder/json_feed/item.rb,
lib/html2rss/feed_builder/rss/stylesheet.rb,
lib/html2rss/feed_pipeline/auto_fallback.rb,
lib/html2rss/feed_pipeline/strategy_plan.rb,
lib/html2rss/html/article_rules/category.rb,
lib/html2rss/html/rendering/pdf_renderer.rb,
lib/html2rss/selectors/extractors/static.rb,
lib/html2rss/config/multiple_feeds_config.rb,
lib/html2rss/feed_pipeline/runtime_policy.rb,
lib/html2rss/html/article_rules/enclosure.rb,
lib/html2rss/request_session/pager/offset.rb,
lib/html2rss/auto_source/scraper/microdata.rb,
lib/html2rss/auto_source/segmenter/cluster.rb,
lib/html2rss/html/navigator/text_extractor.rb,
lib/html2rss/html/rendering/audio_renderer.rb,
lib/html2rss/html/rendering/image_renderer.rb,
lib/html2rss/html/rendering/media_renderer.rb,
lib/html2rss/html/rendering/video_renderer.rb,
lib/html2rss/link_destination/noise_policy.rb,
lib/html2rss/request_service/network_guard.rb,
lib/html2rss/syndication/candidate_catalog.rb,
lib/html2rss/auto_source/scraper/json_state.rb,
lib/html2rss/auto_source/segmenter/semantic.rb,
lib/html2rss/feed_builder/item_presentation.rb,
lib/html2rss/html/article_rules/description.rb,
lib/html2rss/request_service/httpx_strategy.rb,
lib/html2rss/request_service/response_guard.rb,
lib/html2rss/request_session/pager/rel_next.rb,
lib/html2rss/selectors/extractors/attribute.rb,
lib/html2rss/selectors/post_processors/base.rb,
lib/html2rss/selectors/post_processors/gsub.rb,
lib/html2rss/auto_source/scraper/meta_oembed.rb,
lib/html2rss/auto_source/scraper/native_feed.rb,
lib/html2rss/request_service/blocked_surface.rb,
lib/html2rss/request_service/compressed_body.rb,
lib/html2rss/auto_source/scraper/schema/thing.rb,
lib/html2rss/auto_source/scraper/xhr_articles.rb,
lib/html2rss/link_destination/path_classifier.rb,
lib/html2rss/link_destination/text_classifier.rb,
lib/html2rss/auto_source/scraper/microformats2.rb,
lib/html2rss/auto_source/scraper/semantic_html.rb,
lib/html2rss/auto_source/scraper/wordpress_api.rb,
lib/html2rss/html/rendering/escaped_attributes.rb,
lib/html2rss/request_session/pager/json_cursor.rb,
lib/html2rss/selectors/object_to_xml_converter.rb,
lib/html2rss/auto_source/scraper/sitemap/parser.rb,
lib/html2rss/auto_source/segmenter/primary_link.rb,
lib/html2rss/html/rendering/description_builder.rb,
lib/html2rss/link_destination/destination_facts.rb,
lib/html2rss/request_session/pager/url_template.rb,
lib/html2rss/selectors/post_processors/template.rb,
lib/html2rss/feed_resolution/candidate_generator.rb,
lib/html2rss/html/article_extractor/id_generator.rb,
lib/html2rss/request_service/botasaurus_contract.rb,
lib/html2rss/request_service/botasaurus_strategy.rb,
lib/html2rss/request_service/local_file_strategy.rb,
lib/html2rss/selectors/post_processors/parse_uri.rb,
lib/html2rss/selectors/post_processors/substring.rb,
lib/html2rss/auto_source/scraper/schema/item_list.rb,
lib/html2rss/selectors/post_processors/parse_time.rb,
lib/html2rss/html/article_extractor/date_extractor.rb,
lib/html2rss/request_session/pager/custom_selector.rb,
lib/html2rss/html/article_extractor/image_extractor.rb,
lib/html2rss/selectors/post_processors/sanitize_html.rb,
lib/html2rss/html/article_extractor/heading_extractor.rb,
lib/html2rss/html/article_extractor/category_extractor.rb,
lib/html2rss/html/article_extractor/enclosure_extractor.rb,
lib/html2rss/selectors/post_processors/markdown_to_html.rb,
lib/html2rss/auto_source/scraper/json_state/value_finder.rb,
lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb,
lib/html2rss/auto_source/scraper/schema/category_extractor.rb,
lib/html2rss/auto_source/scraper/json_state/document_scanner.rb,
lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb,
lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb,
lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb,
lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb,
lib/html2rss/selectors/post_processors/html_transformers/wrap_img_in_a.rb,
lib/html2rss/auto_source/scraper/wordpress_api/page_scope/date_archive_range.rb,
lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb
Overview
rubocop:disable Metrics/ModuleLength -- Channel value object owns extract + freeze + Marshal
Defined Under Namespace
Modules: Batch, Doctor, FeedBuilder, FeedResolution, HashUtil, Html, LinkDestination, Log, MCP, Recon, SST, Scoring, Syndication, Test Classes: Article, AutoSource, CLI, Capture, Channel, Config, Defaults, Error, FeedPipeline, FeedResult, NoFeedItemsExtracted, PageRecon, RequestService, RequestSession, ScrapeTarget, Selectors, Status, SurfaceCategory, Url
Constant Summary collapse
- VERSION =
Current application version.
'0.30.0'
Class Method Summary collapse
-
.apply(raw_config) ⇒ Html2rss::FeedResult
Ships RSS from a validated config (user-facing verb for Html2rss.feed_result).
-
.auto_feed_result(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ Html2rss::FeedResult
Scrapes the provided URL without hand-written selectors and returns FeedResult.
-
.auto_json_feed(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ Hash
Scrapes the provided URL without hand-written selectors and returns a JSONFeed 1.1 hash.
-
.auto_source(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ RSS::Rss
Scrapes the provided URL without hand-written selectors and returns an RSS object.
-
.batch_inspect(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY) ⇒ Html2rss::Batch::BatchResult
Inspects multiple URLs in parallel with per-URL error isolation.
-
.batch_recon(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY) ⇒ Html2rss::Batch::BatchResult
Runs recon across multiple URLs in parallel with per-URL error isolation.
-
.batch_scrape(urls, strategy: :auto, limit: 10, concurrency: Batch::DEFAULT_CONCURRENCY) ⇒ Html2rss::Batch::BatchResult
Scrapes multiple URLs in parallel using auto-source article discovery.
-
.capture(url, strategy: :auto) ⇒ Html2rss::Capture::CaptureResult
Derives a reusable YAML-ready feed config from a URL.
-
.config_from_yaml_file(file, feed_name = nil) ⇒ Hash{Symbol => Object}
Loads a feed configuration from YAML.
-
.configure {|config| ... } ⇒ Html2rss::Defaults
Configures global library defaults.
-
.defaults ⇒ Html2rss::Defaults
The global defaults instance.
-
.feed(raw_config) ⇒ RSS::Rss
Returns an RSS object generated from the provided configuration.
-
.feed_result(raw_config) ⇒ Html2rss::FeedResult
Returns an opaque, Marshal-cacheable result of one scrape.
-
.inspect(url, strategy: :auto, deep: false) ⇒ Html2rss::PageRecon::Diagnostics::Report
Cheap diagnostics for a URL (final URL, status, alternates, surface).
-
.json_feed(raw_config, feed_url: nil) ⇒ Hash
Returns a JSONFeed 1.1 hash generated from the provided configuration.
-
.logger ⇒ Object
The logger.
- .logger=(logger) ⇒ Object
-
.recon(url, strategy: :auto) ⇒ Html2rss::Recon::Result
Curation verdict and native_feed preference for a URL.
-
.schema_json(pretty: true) ⇒ String
Exports the configuration JSON Schema as JSON string.
-
.scrape(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ Html2rss::FeedResult
One-shot auto-source scrape from a URL (user-facing verb for Html2rss.auto_feed_result).
-
.test(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, strict_quality: false, compare_enhance: false) ⇒ Html2rss::Test::Result
Validates schema and asserts live item extraction.
-
.validate(config_input, feed_name = nil, params: {}) ⇒ Dry::Validation::Result, Html2rss::Config::ValidationResult
Validates a config hash, YAML string, or file path against the schema.
Class Method Details
.apply(raw_config) ⇒ Html2rss::FeedResult
Ships RSS from a validated config (user-facing verb for feed_result). Golden path step 5.
111 112 113 |
# File 'lib/html2rss.rb', line 111 def self.apply(raw_config) feed_result(raw_config) end |
.auto_feed_result(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ Html2rss::FeedResult
Scrapes the provided URL without hand-written selectors and returns FeedResult.
Prefer this when callers need Html2rss::FeedResult#status (MCP envelope payload, CLI --explain).
240 241 242 243 244 245 246 247 248 249 |
# File 'lib/html2rss.rb', line 240 def self.auto_feed_result(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) feed_result(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path:, limit:)) end |
.auto_json_feed(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ Hash
Scrapes the provided URL without hand-written selectors and returns a JSONFeed 1.1 hash.
Same auto_source pipeline as auto_source; see lib/html2rss/auto_source/README.md.
289 290 291 292 293 294 295 296 297 298 |
# File 'lib/html2rss.rb', line 289 def self.auto_json_feed(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) auto_feed_result(url, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path:, limit:).to_json_feed end |
.auto_source(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ RSS::Rss
Scrapes the provided URL without hand-written selectors and returns an RSS object.
Builds an auto_source config, then FeedPipeline runs structured scrapers and (when needed) the SST heuristic path — see lib/html2rss/auto_source/README.md.
265 266 267 268 269 270 271 272 273 274 |
# File 'lib/html2rss.rb', line 265 def self.auto_source(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) auto_feed_result(url, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path:, limit:).to_rss end |
.batch_inspect(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY) ⇒ Html2rss::Batch::BatchResult
Inspects multiple URLs in parallel with per-URL error isolation.
158 159 160 |
# File 'lib/html2rss.rb', line 158 def self.batch_inspect(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY) Batch.batch_inspect(urls:, strategy:, concurrency:) end |
.batch_recon(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY) ⇒ Html2rss::Batch::BatchResult
Runs recon across multiple URLs in parallel with per-URL error isolation.
170 171 172 |
# File 'lib/html2rss.rb', line 170 def self.batch_recon(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY, **) Batch.batch_recon(urls:, strategy:, concurrency:, **) end |
.batch_scrape(urls, strategy: :auto, limit: 10, concurrency: Batch::DEFAULT_CONCURRENCY) ⇒ Html2rss::Batch::BatchResult
Scrapes multiple URLs in parallel using auto-source article discovery.
147 148 149 |
# File 'lib/html2rss.rb', line 147 def self.batch_scrape(urls, strategy: :auto, limit: 10, concurrency: Batch::DEFAULT_CONCURRENCY) Batch.batch_scrape(urls:, strategy:, limit:, concurrency:) end |
.capture(url, strategy: :auto) ⇒ Html2rss::Capture::CaptureResult
Derives a reusable YAML-ready feed config from a URL. Golden path step 3.
71 72 73 |
# File 'lib/html2rss.rb', line 71 def self.capture(url, strategy: :auto, **) Capture.build(url, strategy:, **) end |
.config_from_yaml_file(file, feed_name = nil) ⇒ Hash{Symbol => Object}
Loads a feed configuration from YAML.
191 192 193 |
# File 'lib/html2rss.rb', line 191 def self.config_from_yaml_file(file, feed_name = nil) Config.load_yaml(file, feed_name) end |
.configure {|config| ... } ⇒ Html2rss::Defaults
Configures global library defaults.
315 316 317 318 319 |
# File 'lib/html2rss.rb', line 315 def configure config = defaults.dup yield config @defaults = config.freeze end |
.defaults ⇒ Html2rss::Defaults
Returns the global defaults instance.
306 307 308 |
# File 'lib/html2rss.rb', line 306 def defaults @defaults ||= Defaults.new.freeze end |
.feed(raw_config) ⇒ RSS::Rss
Returns an RSS object generated from the provided configuration.
211 212 213 |
# File 'lib/html2rss.rb', line 211 def self.feed(raw_config) feed_result(raw_config).to_rss end |
.feed_result(raw_config) ⇒ Html2rss::FeedResult
Returns an opaque, Marshal-cacheable result of one scrape.
Prefer this when the same scrape must render as RSS and JSON Feed (e.g. web cache).
202 203 204 |
# File 'lib/html2rss.rb', line 202 def self.feed_result(raw_config) FeedPipeline.new(raw_config).to_result end |
.inspect(url, strategy: :auto, deep: false) ⇒ Html2rss::PageRecon::Diagnostics::Report
Cheap diagnostics for a URL (final URL, status, alternates, surface). Golden path step 1 (optional); use recon for verdict and native_feed.
39 40 41 |
# File 'lib/html2rss.rb', line 39 def self.inspect(url, strategy: :auto, deep: false, **) PageRecon::Diagnostics.call(url:, strategy:, deep:, **) end |
.json_feed(raw_config, feed_url: nil) ⇒ Hash
Returns a JSONFeed 1.1 hash generated from the provided configuration.
221 222 223 |
# File 'lib/html2rss.rb', line 221 def self.json_feed(raw_config, feed_url: nil) feed_result(raw_config).to_json_feed(feed_url:) end |
.logger ⇒ Object
Returns the logger.
323 324 325 |
# File 'lib/html2rss.rb', line 323 def logger defaults.logger end |
.logger=(logger) ⇒ Object
329 330 331 |
# File 'lib/html2rss.rb', line 329 def logger=(logger) configure { |config| config.logger = logger } end |
.recon(url, strategy: :auto) ⇒ Html2rss::Recon::Result
Curation verdict and native_feed preference for a URL. Golden path step 2 (optional); adds verdict beyond inspect.
50 51 52 |
# File 'lib/html2rss.rb', line 50 def self.recon(url, strategy: :auto, **) Recon.call(url, strategy:, **) end |
.schema_json(pretty: true) ⇒ String
Exports the configuration JSON Schema as JSON string.
181 182 183 |
# File 'lib/html2rss.rb', line 181 def self.schema_json(pretty: true) Config.json_schema_json(pretty:) end |
.scrape(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ Html2rss::FeedResult
One-shot auto-source scrape from a URL (user-facing verb for auto_feed_result).
128 129 130 131 132 133 134 135 136 137 |
# File 'lib/html2rss.rb', line 128 def self.scrape(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) auto_feed_result(url, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path:, limit:) end |
.test(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, strict_quality: false, compare_enhance: false) ⇒ Html2rss::Test::Result
Validates schema and asserts live item extraction. Golden path step 4.
100 101 102 103 |
# File 'lib/html2rss.rb', line 100 def self.test(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, strict_quality: false, # rubocop:disable Metrics/ParameterLists compare_enhance: false) Test.call(config_input, feed_name, min_items:, params:, strategy:, strict_quality:, compare_enhance:) end |
.validate(config_input, feed_name = nil, params: {}) ⇒ Dry::Validation::Result, Html2rss::Config::ValidationResult
Validates a config hash, YAML string, or file path against the schema. Side door: schema-only check without live extraction.
83 84 85 86 |
# File 'lib/html2rss.rb', line 83 def self.validate(config_input, feed_name = nil, params: {}) _raw, validation = Config.resolve_and_validate(config_input, feed_name:, params:) validation end |