Module: LanguageOperator::CLI::Commands::System::Helpers::LlmSynthesis

Defined in:
lib/language_operator/cli/commands/system/helpers/llm_synthesis.rb

Overview

LLM synthesis utilities

Instance Method Summary collapse

Instance Method Details

#call_llm_for_synthesis(prompt, model_name) ⇒ Object

Call LLM to generate code from synthesis prompt using cluster model



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/language_operator/cli/commands/system/helpers/llm_synthesis.rb', line 11

def call_llm_for_synthesis(prompt, model_name)
  require 'json'
  require 'faraday'

  # Get model resource
  model = get_resource_or_exit('LanguageModel', model_name)
  model_id = model.dig('spec', 'modelName')

  # Get the model's pod
  pod = get_model_pod(model_name)
  pod_name = pod.dig('metadata', 'name')

  # Set up port-forward to access the model pod
  port_forward_pid = nil
  local_port = find_available_port

  begin
    # Start kubectl port-forward in background
    port_forward_pid = start_port_forward(pod_name, local_port, 4000)

    # Wait for port-forward to be ready
    wait_for_port(local_port)

    # Build the JSON payload for the chat completion request
    payload = {
      model: model_id,
      messages: [{ role: 'user', content: prompt }],
      max_tokens: 4000,
      temperature: 0.3
    }

    # Make HTTP request using Faraday
    conn = Faraday.new(url: "http://localhost:#{local_port}") do |f|
      f.request :json
      f.response :json
      f.adapter Faraday.default_adapter
      f.options.timeout = 120
      f.options.open_timeout = 10
    end

    response = conn.post('/v1/chat/completions', payload)

    # Parse response
    result = response.body

    if result['error']
      error_msg = result['error']['message'] || result['error']
      raise "Model error: #{error_msg}"
    elsif !result['choices'] || result['choices'].empty?
      raise "Unexpected response format: #{result.inspect}"
    end

    # Extract the content from the first choice
    result.dig('choices', 0, 'message', 'content')
  rescue Faraday::TimeoutError
    raise 'LLM request timed out after 120 seconds'
  rescue Faraday::ConnectionFailed => e
    raise "Failed to connect to model: #{e.message}"
  rescue StandardError => e
    Formatters::ProgressFormatter.error("LLM call failed: #{e.message}")
    puts
    puts "Make sure the model '#{model_name}' is running: kubectl get pods -n #{ctx.namespace}"
    exit 1
  ensure
    # Clean up port-forward process
    cleanup_port_forward(port_forward_pid) if port_forward_pid
  end
end

#cleanup_port_forward(pid) ⇒ Object

Clean up port-forward process



151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/language_operator/cli/commands/system/helpers/llm_synthesis.rb', line 151

def cleanup_port_forward(pid)
  return unless pid

  begin
    Process.kill('TERM', pid)
    Process.wait(pid, Process::WNOHANG)
  rescue Errno::ESRCH
    # Process already gone
  rescue Errno::ECHILD
    # Process already reaped
  end
end

#extract_ruby_code(response) ⇒ Object

Extract Ruby code from LLM response Looks for ruby ... blocks



166
167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/language_operator/cli/commands/system/helpers/llm_synthesis.rb', line 166

def extract_ruby_code(response)
  # Match ```ruby ... ``` blocks
  match = response.match(/```ruby\n(.*?)```/m)
  return match[1].strip if match

  # Try without language specifier
  match = response.match(/```\n(.*?)```/m)
  return match[1].strip if match

  # If no code blocks, return nil
  nil
end

#find_available_port ⇒ Object

Find an available local port for port-forwarding



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/language_operator/cli/commands/system/helpers/llm_synthesis.rb', line 106

def find_available_port
  require 'socket'

  # Try ports in the range 14000-14999
  (14_000..14_999).each do |port|
    server = TCPServer.new('127.0.0.1', port)
    server.close
    return port
  rescue Errno::EADDRINUSE
    # Port in use, try next
    next
  end

  raise 'No available ports found in range 14000-14999'
end

#get_model_pod(model_name) ⇒ Object

Get the pod for a model



81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/language_operator/cli/commands/system/helpers/llm_synthesis.rb', line 81

def get_model_pod(model_name)
  # Get the deployment for the model
  deployment = ctx.client.get_resource('Deployment', model_name, ctx.namespace)
  labels = deployment.dig('spec', 'selector', 'matchLabels')

  # Find matching pods using centralized utility
  pods = CLI::Helpers::LabelUtils.find_pods_by_deployment_labels(ctx, model_name, labels)
  raise "No pods found for model '#{model_name}'" if pods.empty?

  running_pod = pods.find do |pod|
    pod.dig('status', 'phase') == 'Running' &&
      pod.dig('status', 'conditions')&.any? { |c| c['type'] == 'Ready' && c['status'] == 'True' }
  end

  if running_pod.nil?
    pod_phases = pods.map { |p| p.dig('status', 'phase') }.join(', ')
    raise "No running pods found. Pod phases: #{pod_phases}"
  end

  running_pod
rescue K8s::Error::NotFound
  raise "Model deployment '#{model_name}' not found"
end

#start_port_forward(pod_name, local_port, remote_port) ⇒ Object

Start kubectl port-forward in background



123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/language_operator/cli/commands/system/helpers/llm_synthesis.rb', line 123

def start_port_forward(pod_name, local_port, remote_port)
  require 'English'

  cmd = "kubectl port-forward -n #{ctx.namespace} #{pod_name} #{local_port}:#{remote_port}"
  pid = spawn(cmd, out: '/dev/null', err: '/dev/null')

  # Detach so it runs in background
  Process.detach(pid)

  pid
end

#wait_for_port(port, max_attempts: 30) ⇒ Object

Wait for port-forward to be ready



136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/language_operator/cli/commands/system/helpers/llm_synthesis.rb', line 136

def wait_for_port(port, max_attempts: 30)
  require 'socket'

  max_attempts.times do
    socket = TCPSocket.new('127.0.0.1', port)
    socket.close
    return true
  rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
    sleep 0.1
  end

  raise "Port-forward to localhost:#{port} failed to become ready after #{max_attempts} attempts"
end