Module: Nous

Defined in:
lib/nous.rb,
lib/nous/cli.rb,
lib/nous/command.rb,
lib/nous/crawler.rb,
lib/nous/fetcher.rb,
lib/nous/version.rb,
lib/nous/converter.rb,
lib/nous/serializer.rb,
lib/nous/url_resolver.rb,
lib/nous/extractor/jina.rb,
lib/nous/primitives/url.rb,
lib/nous/primitives/page.rb,
lib/nous/extractor/default.rb,
lib/nous/crawler/url_filter.rb,
lib/nous/configuration_builder.rb,
lib/nous/extractor/jina/client.rb,
lib/nous/crawler/link_extractor.rb,
lib/nous/fetcher/page_extractor.rb,
lib/nous/primitives/fetch_record.rb,
lib/nous/primitives/fetch_result.rb,
lib/nous/extractor/default/client.rb,
lib/nous/primitives/configuration.rb,
lib/nous/crawler/redirect_follower.rb,
lib/nous/fetcher/extraction_runner.rb,
lib/nous/crawler/async_page_fetcher.rb,
lib/nous/crawler/single_page_fetcher.rb,
lib/nous/primitives/extracted_content.rb,
lib/nous/crawler/recursive_page_fetcher.rb,
sig/nous.rbs

Defined Under Namespace

Modules: Extractor Classes: Cli, Command, Configuration, ConfigurationBuilder, Converter, Crawler, ExtractedContent, FetchRecord, FetchResult, Fetcher, Page, Serializer, Url, UrlResolver

Constant Summary collapse

VERSION =

Returns:

  • (String) 
"0.4.0"

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.configuration ⇒ Object (readonly)

Returns the value of attribute configuration.



11
12
13
 
# File 'lib/nous.rb', line 11

def configuration
  @configuration
end

Class Method Details

.configure ⇒ Object 



13
14
15
 
# File 'lib/nous.rb', line 13

def configure(...)
  @configuration = ConfigurationBuilder.call(...)
end

.fetch(seed_url, extractor: Extractor::Default.new, http_client: nil, details: false, **options) ⇒ Object 



21
22
23
24
25
26
27
28
 
# File 'lib/nous.rb', line 21

def fetch(seed_url, extractor: Extractor::Default.new, http_client: nil, details: false, **options)
  configure(**options)

  result = Fetcher.call(seed_url:, extractor:, http_client:, details:)
  raise result.error if result.failure?

  result.payload
end

.reset_configuration! ⇒ Object 



17
18
19
 
# File 'lib/nous.rb', line 17

def reset_configuration!
  @configuration = nil
end

.serialize(pages, format: :text) ⇒ Object 



30
31
32
33
34
35
 
# File 'lib/nous.rb', line 30

def serialize(pages, format: :text)
  result = Serializer.call(pages:, format:)
  raise result.error if result.failure?

  result.payload
end