Module: RubyLLM::Protocols::ElevenLabs::StreamingTranscription

Defined in:
lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb

Overview

:nodoc: all

Constant Summary collapse

SAMPLE_RATES = 
[8000, 16_000, 22_050, 24_000, 44_100, 48_000].freeze

Instance Method Summary collapse

Instance Method Details

#build_streaming_transcription(transcripts, audio, model:) ⇒ Object 



48
49
50
51
52
53
54
 
# File 'lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb', line 48

def build_streaming_transcription(transcripts, audio, model:)
  RubyLLM::Transcription.new(
    text: transcripts.map { |item| item.fetch('text') }.join(' '), model:,
    duration: audio.duration, language: transcripts.last['language_code'],
    words: transcripts.flat_map { |item| item['words'] || [] }
  )
end

#collect_transcription(url, audio, expected_commits, &block) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb', line 26

def collect_transcription(url, audio, expected_commits, &block)
  transcripts = []
  commits = Queue.new
  Transport::WebsocketConnection.open(url, headers: @provider.headers, config: @config) do |socket|
    write = ->(connection) { send_transcription_audio(connection, audio, commits) }
    socket.each_message(write:) do |message|
      event = JSON.parse(message)
      transcript = process_transcription_event(event, prefix: transcripts.any?, &block)
      next unless transcript

      transcripts << transcript
      commits << true
      socket.close if transcripts.size == expected_commits
    end
  end
  unless transcripts.size == expected_commits && transcripts.any?
    raise Error, 'ElevenLabs transcription ended before its committed transcript'
  end

  transcripts
end

#pcm_audio?(audio) ⇒ Boolean

Returns:

  • (Boolean) 


76
77
78
79
 
# File 'lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb', line 76

def pcm_audio?(audio)
  [audio.channels, audio.encoding, audio.bits_per_sample] == [1, 1, 16] &&
    SAMPLE_RATES.include?(audio.sample_rate)
end

#process_transcription_event(event, prefix: false) ⇒ Object 



110
111
112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb', line 110

def process_transcription_event(event, prefix: false)
  case event['message_type']
  when 'partial_transcript'
    yield TranscriptionChunk.new(type: TranscriptionChunk::PARTIAL, text: event['text'], raw: event)
  when 'committed_transcript'
    text = event['text']
    yield TranscriptionChunk.new(type: TranscriptionChunk::DELTA, delta: prefix ? " #{text}" : text, raw: event)
  when 'committed_transcript_with_timestamps'
    return event
  else
    raise Error, event['error'] if event['error']
  end
  nil
end

#send_transcription_audio(socket, audio, commits) ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb', line 95

def send_transcription_audio(socket, audio, commits)
  offset = 0
  segment_bytes = transcription_segment_bytes(audio)
  while offset < audio.data.bytesize
    length = [16_384, segment_bytes - (offset % segment_bytes)].min
    data = audio.data.byteslice(offset, length)
    offset += data.bytesize
    commit = (offset % segment_bytes).zero? || offset == audio.data.bytesize
    socket.send_text(JSON.generate(message_type: 'input_audio_chunk',
                                   'audio_base_64' => Base64.strict_encode64(data),
                                   sample_rate: audio.sample_rate, commit:))
    commits.pop if commit
  end
end

#stream_transcription(payload, model:, &block) ⇒ Object 



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb', line 9

def stream_transcription(payload, model:, &block)
  validate_streaming_transcription(payload)
  audio = RubyLLM::Transcription::WavAudio.new(payload.fetch(:file).io.read)
  raise ArgumentError, 'Streaming transcription requires non-empty audio' if audio.data.empty?

  audio_format = streaming_audio_format(audio)
  url = streaming_transcription_url(payload, audio_format:)
  @usage_tracker.start
  expected_commits = (audio.data.bytesize.to_f / transcription_segment_bytes(audio)).ceil
  transcripts = collect_transcription(url, audio, expected_commits, &block)
  result = build_streaming_transcription(transcripts, audio, model:)
  block.call(TranscriptionChunk.new(type: TranscriptionChunk::DONE, text: result.text, raw: transcripts.last))
  result
ensure
  payload[:file]&.io&.close
end

#streaming_audio_format(audio) ⇒ Object

Raises:

  • (ArgumentError) 


68
69
70
71
72
73
74
 
# File 'lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb', line 68

def streaming_audio_format(audio)
  return "pcm_#{audio.sample_rate}" if pcm_audio?(audio)
  return 'ulaw_8000' if [audio.channels, audio.encoding, audio.sample_rate,
                         audio.bits_per_sample] == [1, 7, 8000, 8]

  raise ArgumentError, 'ElevenLabs streaming requires mono 16-bit PCM WAV or 8 kHz mu-law WAV audio'
end

#streaming_transcription_url(payload, audio_format:) ⇒ Object 



81
82
83
84
85
86
87
88
89
 
# File 'lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb', line 81

def streaming_transcription_url(payload, audio_format:)
  language_detection = payload.fetch(:include_language_detection, true)
  params = payload.except(:file).merge(audio_format:, commit_strategy: 'manual', include_timestamps: true,
                                       include_language_detection: language_detection)
  uri = URI.join("#{@provider.api_base.sub(%r{/+\z}, '')}/",
                 "v1/speech-to-text/realtime?#{URI.encode_www_form(params)}")
  uri.scheme = uri.scheme == 'https' ? 'wss' : 'ws'
  uri.to_s
end

#transcription_segment_bytes(audio) ⇒ Object 



91
92
93
 
# File 'lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb', line 91

def transcription_segment_bytes(audio)
  audio.sample_rate * audio.channels * audio.bits_per_sample / 8 * 20
end

#validate_streaming_transcription(payload) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb', line 56

def validate_streaming_transcription(payload)
  unsupported = payload.keys & i[diarize num_speakers temperature timestamps_granularity]
  unless unsupported.empty?
    raise ArgumentError, "ElevenLabs streaming transcription does not accept #{unsupported.join(', ')}"
  end

  if payload.fetch(:commit_strategy, 'manual') != 'manual' || payload[:include_timestamps] == false ||
     payload[:filter_background_audio]
    raise ArgumentError, 'ElevenLabs file streaming requires manual commits and word timestamps'
  end
end