Class: SwarmSDK::Tools::WebFetch

Inherits:
Base
  • Object
show all
Defined in:
lib/swarm_sdk/tools/web_fetch.rb

Overview

WebFetch tool for fetching and processing web content

Fetches content from URLs, converts HTML to markdown, and processes it using an AI model to extract information based on a provided prompt.

Constant Summary collapse

USER_AGENT =

NOTE: Content length and timeout now accessed via SwarmSDK.config

"SwarmSDK WebFetch Tool (https://github.com/parruda/claude-swarm)"

Instance Method Summary collapse

Methods inherited from Base

removable, removable?, #removable?

Constructor Details

#initialize ⇒ WebFetch

Returns a new instance of WebFetch.



10
11
12
13
14
 
# File 'lib/swarm_sdk/tools/web_fetch.rb', line 10

def initialize
  super()
  @cache = {}
  @cache_ttl = 900 # 15 minutes in seconds
end

Instance Method Details

#execute(url:, prompt: nil) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/swarm_sdk/tools/web_fetch.rb', line 56

def execute(url:, prompt: nil)
  # Validate inputs
  return validation_error("url is required") if url.nil? || url.empty?

  # Check if LLM processing is enabled (lazy check)
  llm_enabled = SwarmSDK.config.webfetch_llm_enabled?

  # Validate prompt when LLM processing is enabled
  if llm_enabled && (prompt.nil? || prompt.empty?)
    return validation_error("prompt is required when LLM processing is configured")
  end

  # Validate and normalize URL
  normalized_url = normalize_url(url)
  return validation_error("Invalid URL format: #{url}") unless normalized_url

  # Check cache first (cache key includes prompt if LLM is enabled)
  cache_key = llm_enabled ? "#{normalized_url}:#{prompt}" : normalized_url
  cached = get_from_cache(cache_key)
  return cached if cached

  # Fetch the URL
  fetch_result = fetch_url(normalized_url)
  return fetch_result if fetch_result.is_a?(String) && fetch_result.start_with?("Error")

  # Check for redirects to different hosts
  if fetch_result[:redirect_url] && different_host?(normalized_url, fetch_result[:redirect_url])
    return format_redirect_message(fetch_result[:redirect_url])
  end

  # Convert HTML to markdown
  markdown_content = html_to_markdown(fetch_result[:body])

  # Truncate if too long
  max_content = SwarmSDK.config.web_fetch_character_limit
  if markdown_content.length > max_content
    markdown_content = markdown_content[0...max_content]
    markdown_content += "\n\n[Content truncated due to length]"
  end

  # Process with AI model if LLM is enabled, otherwise return markdown
  result = if llm_enabled
    process_with_llm(markdown_content, prompt, normalized_url)
  else
    markdown_content
  end

  # Cache the result
  store_in_cache(cache_key, result)

  result
rescue StandardError => e
  error("Unexpected error fetching URL: #{e.class.name} - #{e.message}")
end

#name ⇒ Object 



16
17
18
 
# File 'lib/swarm_sdk/tools/web_fetch.rb', line 16

def name
  "WebFetch"
end