Class: RubyLLM::Protocols::Gemini::LiveTranscription

Inherits:
RubyLLM::Protocol show all
Defined in:
lib/ruby_llm/protocols/gemini/live_transcription.rb

Overview

:nodoc: all

Instance Attribute Summary

Attributes inherited from RubyLLM::Protocol

#config, #connection, #model, #provider

Instance Method Summary collapse

Methods inherited from RubyLLM::Protocol

abstract, #animate_later, #apply_compaction, #apply_compaction_headers, #apply_end_user, #cache_content, #compact, #complete, #count_tokens, #delete_cache, #embed, #extend_cache, #find_cache, #initialize, #list_models, #moderate, #ocr, #paint, #parse_error, #parse_image_responses, #post_image, #post_video, #preprocess_message, #raise_transcription_streaming_unsupported, #refresh_video_job, #render, #render_embedding, #render_transcription_options, #render_video_extension_payload, #rerank, #server_tool_aliases, #speak, #stream_speech, #stream_speech_response, #stream_transcription, #supports_embedding_media?, #tokenize, #tool_approval_response, #video_extension_attachment, #video_extension_url, #video_request_url

Methods included from RubyLLM::Protocol::BinaryStreaming

#stream_binary

Methods included from RubyLLM::Protocol::Streaming

build_on_data_handler, build_stream_error_response, error_chunk?, failed_http_status, faraday_1?, handle_data, handle_error_chunk, handle_error_event, handle_failed_response, handle_json_error_chunk, handle_sse, handle_stream, json_error_payload?, parse_error_from_json, parse_streaming_error, process_stream_chunk, raise_stream_error, stream_events, stream_response, stream_state

Constructor Details

This class inherits a constructor from RubyLLM::Protocol

Instance Method Details

#collect_transcription(audio, setup, &block) ⇒ Object 



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/ruby_llm/protocols/gemini/live_transcription.rb', line 76

def collect_transcription(audio, setup, &block)
  events = []
  ready = Queue.new
  Transport::WebsocketConnection.open(
    transcription_websocket_url, headers: @provider.headers.transform_keys(&:to_s), config: @config
  ) do |socket|
    socket.send_text(JSON.generate(setup))
    write = lambda { |connection|
      ready.pop
      send_transcription_audio(connection, audio)
    }
    socket.each_message(write:) do |message|
      event = JSON.parse(message)
      events << event
      ready << true if event.key?('setupComplete')
      process_transcription_event(event, &block)
      socket.close if event.dig('serverContent', 'generationComplete')
    end
  end
  unless events.last&.dig('serverContent', 'generationComplete')
    raise Error, 'Google Live transcription ended before generation completed'
  end

  events
end

#parse_transcription_events(events, audio:, model:) ⇒ Object 



129
130
131
132
133
134
135
136
 
# File 'lib/ruby_llm/protocols/gemini/live_transcription.rb', line 129

def parse_transcription_events(events, audio:, model:)
  text = events.filter_map { |event| event.dig('serverContent', 'inputTranscription', 'text') }.join
  usage = events.reverse.find { |event| event['usageMetadata'] }&.fetch('usageMetadata') || {}
  RubyLLM::Transcription.new(
    text:, model:, duration: audio.duration,
    input_tokens: usage['promptTokenCount'], output_tokens: usage['candidatesTokenCount']
  )
end

#process_transcription_event(event) ⇒ Object

Raises:



115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/ruby_llm/protocols/gemini/live_transcription.rb', line 115

def process_transcription_event(event)
  raise Error, event.dig('error', 'message') || 'Google Live transcription failed' if event['error']
  return unless block_given?

  content = event['serverContent'] || {}
  if content['interimInputTranscription']
    yield TranscriptionChunk.new(type: TranscriptionChunk::PARTIAL,
                                 text: content.dig('interimInputTranscription', 'text'), raw: event)
  elsif content['inputTranscription']
    yield TranscriptionChunk.new(type: TranscriptionChunk::DELTA,
                                 delta: content.dig('inputTranscription', 'text'), raw: event)
  end
end

#render_transcription_setup(model:, language:, prompt:, provider_options:) ⇒ Object 



41
42
43
44
45
46
47
48
49
 
# File 'lib/ruby_llm/protocols/gemini/live_transcription.rb', line 41

def render_transcription_setup(model:, language:, prompt:, provider_options:)
  payload = { model: transcription_model_name(model), generationConfig: { responseModalities: ['TEXT'] },
              inputAudioTranscription: { languageCodes: language && Array(language),
                                         customVocabulary: prompt && Array(prompt) }.compact,
              realtimeInputConfig: { automaticActivityDetection: { disabled: true } } }
  payload = Support::Utils.deep_merge(payload, provider_options)
  validate_transcription_setup(payload)
  { setup: payload }
end

#send_transcription_audio(socket, audio) ⇒ Object 



102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/ruby_llm/protocols/gemini/live_transcription.rb', line 102

def send_transcription_audio(socket, audio)
  socket.send_text(JSON.generate(realtimeInput: { activityStart: {} }))
  bytes = [audio.sample_rate / 10, 1].max * 2
  offset = 0
  while offset < audio.data.bytesize
    chunk = audio.data.byteslice(offset, bytes)
    input = { audio: { mimeType: "audio/pcm;rate=#{audio.sample_rate}", data: Base64.strict_encode64(chunk) } }
    socket.send_text(JSON.generate(realtimeInput: input))
    offset += chunk.bytesize
  end
  socket.send_text(JSON.generate(realtimeInput: { activityEnd: {} }))
end

#transcribe(audio_file, model:, language:, format: nil, speaker_names: nil, speaker_references: nil, provider_options: {}, prompt: nil, temperature: nil, &block) ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/ruby_llm/protocols/gemini/live_transcription.rb', line 7

def transcribe(audio_file, model:, language:, format: nil, speaker_names: nil,
               speaker_references: nil, provider_options: {}, prompt: nil, temperature: nil, &block)
  validate_transcription_request(format:, speaker_names:, speaker_references:, temperature:)
  audio = transcription_audio(audio_file)
  setup = render_transcription_setup(model:, language:, prompt:, provider_options:)

  track_usage(:transcription) do
    @usage_tracker.start
    events = collect_transcription(audio, setup, &block)
    result = parse_transcription_events(events, audio:, model:)
    block&.call(TranscriptionChunk.new(type: TranscriptionChunk::DONE, text: result.text, raw: events.last))
    result
  end
end

#transcription_audio(file) ⇒ Object

Raises:

  • (ArgumentError) 


28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/ruby_llm/protocols/gemini/live_transcription.rb', line 28

def transcription_audio(file)
  attachments = Attachment.wrap(file, config: @config)
  raise ArgumentError, 'Transcription requires exactly one audio file' unless attachments.one?

  audio = RubyLLM::Transcription::WavAudio.new(attachments.first.content)
  unless [audio.encoding, audio.channels, audio.bits_per_sample] == [1, 1, 16] &&
         audio.data.bytesize.positive? && audio.data.bytesize.even?
    raise ArgumentError, 'Google Live transcription requires non-empty mono 16-bit PCM WAV audio'
  end

  audio
end

#transcription_model_name(model) ⇒ Object 



61
62
63
 
# File 'lib/ruby_llm/protocols/gemini/live_transcription.rb', line 61

def transcription_model_name(model)
  "models/#{model}"
end

#transcription_websocket_url ⇒ Object 



69
70
71
72
73
74
 
# File 'lib/ruby_llm/protocols/gemini/live_transcription.rb', line 69

def transcription_websocket_url
  uri = URI.parse(@provider.api_base)
  uri.scheme = uri.scheme == 'https' ? 'wss' : 'ws'
  uri.path = "/ws/#{websocket_service}"
  uri.to_s
end

#validate_transcription_request(format:, speaker_names:, speaker_references:, temperature:) ⇒ Object

Raises:

  • (ArgumentError) 


22
23
24
25
26
 
# File 'lib/ruby_llm/protocols/gemini/live_transcription.rb', line 22

def validate_transcription_request(format:, speaker_names:, speaker_references:, temperature:)
  return unless format || speaker_names || speaker_references || temperature

  raise ArgumentError, 'Google Live transcription does not accept format, diarization, or temperature'
end

#validate_transcription_setup(payload) ⇒ Object

Raises:

  • (ArgumentError) 


51
52
53
54
55
56
57
58
59
 
# File 'lib/ruby_llm/protocols/gemini/live_transcription.rb', line 51

def validate_transcription_setup(payload)
  config = payload.fetch(:inputAudioTranscription)
  if config[:diarization] || config[:wordTimestamp]
    raise ArgumentError, 'Google Live transcription does not support diarization or word timestamps'
  end
  return if payload.dig(:realtimeInputConfig, :automaticActivityDetection, :disabled) == true

  raise ArgumentError, 'Google file transcription requires manual activity boundaries'
end

#websocket_service ⇒ Object 



65
66
67
 
# File 'lib/ruby_llm/protocols/gemini/live_transcription.rb', line 65

def websocket_service
  'google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent'
end