Class: Xberg::CrawlConfig

Inherits:
Object
  • Object
show all
Defined in:
sig/types.rbs

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ CrawlConfig

Returns a new instance of CrawlConfig.

Parameters:

  • max_depth: (Integer)
  • max_pages: (Integer)
  • max_links_per_page: (Integer)
  • max_concurrent: (Integer)
  • crawl_strategy: (enum_CrawlStrategyKind)
  • content_filter: (enum_ContentFilterKind)
  • bm25_query: (String)
  • bm25_threshold: (Float)
  • respect_robots_txt: (Boolean)
  • soft_http_errors: (Boolean)
  • user_agent: (String)
  • stay_on_domain: (Boolean)
  • allow_subdomains: (Boolean)
  • include_paths: (Array[String])
  • exclude_paths: (Array[String])
  • custom_headers: (Hash[String, String])
  • request_timeout: (Integer)
  • rate_limit_ms: (Integer)
  • max_redirects: (Integer)
  • retry_count: (Integer)
  • retry_codes: (Array[Integer])
  • cookies_enabled: (Boolean)
  • auth: (AuthConfig)
  • max_body_size: (Integer)
  • remove_tags: (Array[String])
  • content: (ContentConfig)
  • map_limit: (Integer)
  • map_search: (String)
  • download_assets: (Boolean)
  • asset_types: (Array[enum_AssetCategory])
  • max_asset_size: (Integer)
  • browser: (BrowserConfig)
  • proxy: (ProxyConfig)
  • user_agents: (Array[String])
  • capture_screenshot: (Boolean)
  • follow_document_urls: (Boolean)
  • document_url_depth: (Integer)
  • download_documents: (Boolean)
  • document_max_size: (Integer)
  • document_mime_types: (Array[String])
  • document_output_dir: (String)
  • document_content_encoding: (enum_DocumentContentEncoding)
  • warc_output: (String)
  • browser_profile: (String)
  • save_browser_profile: (Boolean)
  • ssrf: (SsrfPolicy)
  • ssrf_deny_private_explicit: (Boolean) 


437
 
# File 'sig/types.rbs', line 437

def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void

Instance Attribute Details

#allow_subdomains ⇒ Boolean (readonly)

Returns the value of attribute allow_subdomains.

Returns:

  • (Boolean) 


401
402
403
 
# File 'sig/types.rbs', line 401

def allow_subdomains
  @allow_subdomains
end

#asset_types ⇒ Array[enum_AssetCategory] (readonly)

Returns the value of attribute asset_types.

Returns:

  • (Array[enum_AssetCategory]) 


418
419
420
 
# File 'sig/types.rbs', line 418

def asset_types
  @asset_types
end

#auth ⇒ AuthConfig? (readonly)

Returns the value of attribute auth.

Returns:



411
412
413
 
# File 'sig/types.rbs', line 411

def auth
  @auth
end

#bm25_query ⇒ String? (readonly)

Returns the value of attribute bm25_query.

Returns:

  • (String, nil) 


395
396
397
 
# File 'sig/types.rbs', line 395

def bm25_query
  @bm25_query
end

#bm25_threshold ⇒ Float? (readonly)

Returns the value of attribute bm25_threshold.

Returns:

  • (Float, nil) 


396
397
398
 
# File 'sig/types.rbs', line 396

def bm25_threshold
  @bm25_threshold
end

#browser ⇒ BrowserConfig (readonly)

Returns the value of attribute browser.

Returns:



420
421
422
 
# File 'sig/types.rbs', line 420

def browser
  @browser
end

#browser_profile ⇒ String? (readonly)

Returns the value of attribute browser_profile.

Returns:

  • (String, nil) 


432
433
434
 
# File 'sig/types.rbs', line 432

def browser_profile
  @browser_profile
end

#capture_screenshot ⇒ Boolean (readonly)

Returns the value of attribute capture_screenshot.

Returns:

  • (Boolean) 


423
424
425
 
# File 'sig/types.rbs', line 423

def capture_screenshot
  @capture_screenshot
end

#content ⇒ ContentConfig (readonly)

Returns the value of attribute content.

Returns:



414
415
416
 
# File 'sig/types.rbs', line 414

def content
  @content
end

#content_filter ⇒ enum_ContentFilterKind? (readonly)

Returns the value of attribute content_filter.

Returns:

  • (enum_ContentFilterKind, nil) 


394
395
396
 
# File 'sig/types.rbs', line 394

def content_filter
  @content_filter
end

#cookies_enabled ⇒ Boolean (readonly)

Returns the value of attribute cookies_enabled.

Returns:

  • (Boolean) 


410
411
412
 
# File 'sig/types.rbs', line 410

def cookies_enabled
  @cookies_enabled
end

#crawl_strategy ⇒ enum_CrawlStrategyKind (readonly)

Returns the value of attribute crawl_strategy.

Returns:

  • (enum_CrawlStrategyKind) 


393
394
395
 
# File 'sig/types.rbs', line 393

def crawl_strategy
  @crawl_strategy
end

#custom_headers ⇒ Hash[String, String] (readonly)

Returns the value of attribute custom_headers.

Returns:

  • (Hash[String, String]) 


404
405
406
 
# File 'sig/types.rbs', line 404

def custom_headers
  @custom_headers
end

#document_content_encoding ⇒ enum_DocumentContentEncoding? (readonly)

Returns the value of attribute document_content_encoding.

Returns:

  • (enum_DocumentContentEncoding, nil) 


430
431
432
 
# File 'sig/types.rbs', line 430

def document_content_encoding
  @document_content_encoding
end

#document_max_size ⇒ Integer? (readonly)

Returns the value of attribute document_max_size.

Returns:

  • (Integer, nil) 


427
428
429
 
# File 'sig/types.rbs', line 427

def document_max_size
  @document_max_size
end

#document_mime_types ⇒ Array[String] (readonly)

Returns the value of attribute document_mime_types.

Returns:

  • (Array[String]) 


428
429
430
 
# File 'sig/types.rbs', line 428

def document_mime_types
  @document_mime_types
end

#document_output_dir ⇒ String? (readonly)

Returns the value of attribute document_output_dir.

Returns:

  • (String, nil) 


429
430
431
 
# File 'sig/types.rbs', line 429

def document_output_dir
  @document_output_dir
end

#document_url_depth ⇒ Integer? (readonly)

Returns the value of attribute document_url_depth.

Returns:

  • (Integer, nil) 


425
426
427
 
# File 'sig/types.rbs', line 425

def document_url_depth
  @document_url_depth
end

#download_assets ⇒ Boolean (readonly)

Returns the value of attribute download_assets.

Returns:

  • (Boolean) 


417
418
419
 
# File 'sig/types.rbs', line 417

def download_assets
  @download_assets
end

#download_documents ⇒ Boolean (readonly)

Returns the value of attribute download_documents.

Returns:

  • (Boolean) 


426
427
428
 
# File 'sig/types.rbs', line 426

def download_documents
  @download_documents
end

#exclude_paths ⇒ Array[String] (readonly)

Returns the value of attribute exclude_paths.

Returns:

  • (Array[String]) 


403
404
405
 
# File 'sig/types.rbs', line 403

def exclude_paths
  @exclude_paths
end

#follow_document_urls ⇒ Boolean (readonly)

Returns the value of attribute follow_document_urls.

Returns:

  • (Boolean) 


424
425
426
 
# File 'sig/types.rbs', line 424

def follow_document_urls
  @follow_document_urls
end

#include_paths ⇒ Array[String] (readonly)

Returns the value of attribute include_paths.

Returns:

  • (Array[String]) 


402
403
404
 
# File 'sig/types.rbs', line 402

def include_paths
  @include_paths
end

#map_limit ⇒ Integer? (readonly)

Returns the value of attribute map_limit.

Returns:

  • (Integer, nil) 


415
416
417
 
# File 'sig/types.rbs', line 415

def map_limit
  @map_limit
end

#map_search ⇒ String? (readonly)

Returns the value of attribute map_search.

Returns:

  • (String, nil) 


416
417
418
 
# File 'sig/types.rbs', line 416

def map_search
  @map_search
end

#max_asset_size ⇒ Integer? (readonly)

Returns the value of attribute max_asset_size.

Returns:

  • (Integer, nil) 


419
420
421
 
# File 'sig/types.rbs', line 419

def max_asset_size
  @max_asset_size
end

#max_body_size ⇒ Integer? (readonly)

Returns the value of attribute max_body_size.

Returns:

  • (Integer, nil) 


412
413
414
 
# File 'sig/types.rbs', line 412

def max_body_size
  @max_body_size
end

#max_concurrent ⇒ Integer? (readonly)

Returns the value of attribute max_concurrent.

Returns:

  • (Integer, nil) 


392
393
394
 
# File 'sig/types.rbs', line 392

def max_concurrent
  @max_concurrent
end

#max_depth ⇒ Integer? (readonly)

Returns the value of attribute max_depth.

Returns:

  • (Integer, nil) 


389
390
391
 
# File 'sig/types.rbs', line 389

def max_depth
  @max_depth
end

#max_links_per_page ⇒ Integer? (readonly)

Returns the value of attribute max_links_per_page.

Returns:

  • (Integer, nil) 


391
392
393
 
# File 'sig/types.rbs', line 391

def max_links_per_page
  @max_links_per_page
end

#max_pages ⇒ Integer? (readonly)

Returns the value of attribute max_pages.

Returns:

  • (Integer, nil) 


390
391
392
 
# File 'sig/types.rbs', line 390

def max_pages
  @max_pages
end

#max_redirects ⇒ Integer (readonly)

Returns the value of attribute max_redirects.

Returns:

  • (Integer) 


407
408
409
 
# File 'sig/types.rbs', line 407

def max_redirects
  @max_redirects
end

#proxy ⇒ ProxyConfig? (readonly)

Returns the value of attribute proxy.

Returns:



421
422
423
 
# File 'sig/types.rbs', line 421

def proxy
  @proxy
end

#rate_limit_ms ⇒ Integer? (readonly)

Returns the value of attribute rate_limit_ms.

Returns:

  • (Integer, nil) 


406
407
408
 
# File 'sig/types.rbs', line 406

def rate_limit_ms
  @rate_limit_ms
end

#remove_tags ⇒ Array[String] (readonly)

Returns the value of attribute remove_tags.

Returns:

  • (Array[String]) 


413
414
415
 
# File 'sig/types.rbs', line 413

def remove_tags
  @remove_tags
end

#request_timeout ⇒ Integer (readonly)

Returns the value of attribute request_timeout.

Returns:

  • (Integer) 


405
406
407
 
# File 'sig/types.rbs', line 405

def request_timeout
  @request_timeout
end

#respect_robots_txt ⇒ Boolean (readonly)

Returns the value of attribute respect_robots_txt.

Returns:

  • (Boolean) 


397
398
399
 
# File 'sig/types.rbs', line 397

def respect_robots_txt
  @respect_robots_txt
end

#retry_codes ⇒ Array[Integer] (readonly)

Returns the value of attribute retry_codes.

Returns:

  • (Array[Integer]) 


409
410
411
 
# File 'sig/types.rbs', line 409

def retry_codes
  @retry_codes
end

#retry_count ⇒ Integer (readonly)

Returns the value of attribute retry_count.

Returns:

  • (Integer) 


408
409
410
 
# File 'sig/types.rbs', line 408

def retry_count
  @retry_count
end

#save_browser_profile ⇒ Boolean (readonly)

Returns the value of attribute save_browser_profile.

Returns:

  • (Boolean) 


433
434
435
 
# File 'sig/types.rbs', line 433

def save_browser_profile
  @save_browser_profile
end

#soft_http_errors ⇒ Boolean (readonly)

Returns the value of attribute soft_http_errors.

Returns:

  • (Boolean) 


398
399
400
 
# File 'sig/types.rbs', line 398

def soft_http_errors
  @soft_http_errors
end

#ssrf ⇒ SsrfPolicy (readonly)

Returns the value of attribute ssrf.

Returns:



434
435
436
 
# File 'sig/types.rbs', line 434

def ssrf
  @ssrf
end

#ssrf_deny_private_explicit ⇒ Boolean? (readonly)

Returns the value of attribute ssrf_deny_private_explicit.

Returns:

  • (Boolean, nil) 


435
436
437
 
# File 'sig/types.rbs', line 435

def ssrf_deny_private_explicit
  @ssrf_deny_private_explicit
end

#stay_on_domain ⇒ Boolean (readonly)

Returns the value of attribute stay_on_domain.

Returns:

  • (Boolean) 


400
401
402
 
# File 'sig/types.rbs', line 400

def stay_on_domain
  @stay_on_domain
end

#user_agent ⇒ String? (readonly)

Returns the value of attribute user_agent.

Returns:

  • (String, nil) 


399
400
401
 
# File 'sig/types.rbs', line 399

def user_agent
  @user_agent
end

#user_agents ⇒ Array[String] (readonly)

Returns the value of attribute user_agents.

Returns:

  • (Array[String]) 


422
423
424
 
# File 'sig/types.rbs', line 422

def user_agents
  @user_agents
end

#warc_output ⇒ String? (readonly)

Returns the value of attribute warc_output.

Returns:

  • (String, nil) 


431
432
433
 
# File 'sig/types.rbs', line 431

def warc_output
  @warc_output
end