Class: RagRuby::Loaders::URL

Inherits:
Base
  • Object
show all
Defined in:
lib/rag_ruby/loaders/url.rb

Instance Method Summary collapse

Constructor Details

#initialize(timeout: 30) ⇒ URL

Returns a new instance of URL.



9
10
11
 
# File 'lib/rag_ruby/loaders/url.rb', line 9

def initialize(timeout: 30)
  @timeout = timeout
end

Instance Method Details

#load(url) ⇒ Object

Raises:

  • (ArgumentError) 


13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/rag_ruby/loaders/url.rb', line 13

def load(url)
  uri = URI.parse(url)
  raise ArgumentError, "Invalid URL: #{url}" unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)

  response = fetch(uri)
  content = response.body.force_encoding("UTF-8")

  [Document.new(
    content: content,
    metadata: {
      source: url,
      content_type: response["content-type"],
      status: response.code.to_i
    },
    source: url
  )]
end