Class: Fastlane::Helper::TranslateGptHelper

Inherits:
Object
  • Object
show all
Defined in:
lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb

Instance Method Summary collapse

Constructor Details

#initialize(params) ⇒ TranslateGptHelper

Returns a new instance of TranslateGptHelper.



12
13
14
15
16
17
18
19
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 12

def initialize(params)
  @params = params
  @client = OpenAI::Client.new(
    access_token: params[:api_token],
    request_timeout: params[:request_timeout]
  )
  @timeout = params[:request_timeout]
end

Instance Method Details

#check_value_for_translate(string, orignal_string) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 36

def check_value_for_translate(string, orignal_string)
  return true unless string 
  if string.is_a? LocoStrings::LocoString
    return false if orignal_string.value.nil? || orignal_string.value.empty?
    return string.value.empty?
  elsif string.is_a? LocoStrings::LocoVariantions
    orignal_string.strings.each do |key, _|
      return true unless string.strings.has_key?(key)
      return true if string.strings[key].value.empty?
    end
  end
  return false
end

#filter_translated(need_to_skip, base, target) ⇒ Object 



382
383
384
385
386
387
388
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 382

def filter_translated(need_to_skip, base, target) 
  if need_to_skip
    return base.reject { |k, v| target[k] }
  else 
    return base
  end
end

#get_context(localization_file, localization_key) ⇒ String

Get the context associated with a localization key

Parameters:

  • localization_file (String) —

    The path to the strings file

  • localization_key (String) —

    The localization key

Returns:

  • (String) —

    The context associated with the localization key



376
377
378
379
380
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 376

def get_context(localization_file, localization_key)
  file = LocoStrings.load(localization_file)
  string = file.read[localization_key]
  return string.comment
end

#get_strings(localization_file) ⇒ Hash

Read the strings file into a hash

Parameters:

  • localization_file (String) —

    The path to the strings file

Returns:

  • (Hash) —

    The strings file as a hash



367
368
369
370
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 367

def get_strings(localization_file)
  file = LocoStrings.load(localization_file)
  return file.read
end

#log_input(bunch_size) ⇒ Object

Log information about the input strings



66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 66

def log_input(bunch_size) 
  @translation_count = @to_translate.size
  number_of_strings = Colorizer::colorize("#{@translation_count}", :blue)
  UI.message "Translating #{number_of_strings} strings..."
  if bunch_size.nil? || bunch_size < 1
    estimated_string = Colorizer::colorize("#{@translation_count * @params[:request_timeout]}", :white)
    UI.message "Estimated time: #{estimated_string} seconds"
  else 
    number_of_bunches = (@translation_count / bunch_size.to_f).ceil
    estimated_string = Colorizer::colorize("#{number_of_bunches * @params[:request_timeout]}", :white)
    UI.message "Estimated time: #{estimated_string} seconds"
  end
end

#prepare_bunch_prompt(strings) ⇒ Object 



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 199

def prepare_bunch_prompt(strings)
  prompt = "I want you to act as a translator for a mobile application strings. " + \
      "Try to keep length of the translated text. " + \
      "You need to response with a JSON only with the translation and nothing else until I say to stop it. "
  if @params[:context] && !@params[:context].empty?
    prompt += "This app is #{@params[:context]}. "
  end
  prompt += "Translate next text from #{@params[:source_language]} to #{@params[:target_language]}:\n"

  json_hash = []
  strings.each do |key, string|
    # UI.message "Translating #{key} - #{string.value}"
    next if string.nil?

    string_hash = {}
    context = string.comment
    string_hash["context"] = context if context && !context.empty?

    key = transform_string(string.key)
    @keys_associations[key] = string.key
    string_hash["key"] = key

    if string.is_a? LocoStrings::LocoString
      next if string.value.nil? || string.value.empty?
      string_hash["string_to_translate"] = string.value
    elsif string.is_a? LocoStrings::LocoVariantions
      variants = {}
      string.strings.each do |key, variant|
        next if variant.nil? || variant.value.nil? || variant.value.empty?
        variants[key] = variant.value
      end
      string_hash["strings_to_translate"] = variants
    else 
      UI.warning "Unknown type of string: #{string.key}"
    end
    json_hash << string_hash
  end
  return '' if json_hash.empty?
  prompt += "'''\n"
  prompt += json_hash.to_json
  prompt += "\n'''"
  return prompt
end

#prepare_hashes ⇒ Object

Get the strings from a file



57
58
59
60
61
62
63
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 57

def prepare_hashes() 
  if File.extname(@params[:source_file]) == ".xcstrings"
    prepare_xcstrings() 
  else
    prepare_strings() 
  end
end

#prepare_prompt(string) ⇒ Object

Prepare the prompt for the GPT API



183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 183

def prepare_prompt(string) 
  prompt = "I want you to act as a translator for a mobile application strings. " + \
      "Try to keep length of the translated text. " + \
      "You need to answer only with the translation and nothing else until I say to stop it.  No commentaries." 
  if @params[:context] && !@params[:context].empty?
    prompt += "This app is #{@params[:context]}. "
  end 
  context = string.comment
  if context && !context.empty?
    prompt += "Additional context is #{context}. "
  end
  prompt += "Translate next text from #{@params[:source_language]} to #{@params[:target_language]}:\n" +
    "#{string.value}"
  return prompt
end

#prepare_strings ⇒ Object 



50
51
52
53
54
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 50

def prepare_strings() 
  @input_hash = get_strings(@params[:source_file])
  @output_hash = get_strings(@params[:target_file])
  @to_translate = filter_translated(@params[:skip_translated], @input_hash, @output_hash)
end

#prepare_xcstrings ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 21

def prepare_xcstrings() 
  @xcfile = LocoStrings::XCStringsFile.new @params[:source_file]
  @output_hash = {}
  @to_translate = @xcfile.read
  
  if @params[:skip_translated] == true
    @to_translate = @to_translate.reject { |k, original| 
      !check_value_for_translate(
        @xcfile.unit(k, @params[:target_language]),
        original
      )
    }
  end 
end

#request_bunch(bunch, progress) ⇒ Object 



157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 157

def request_bunch(bunch, progress) 
  UI.message "[#{progress}%] Translating #{bunch.size} strings..."
  prompt = prepare_bunch_prompt bunch
  if prompt.empty?
    UI.important "Empty prompt, skipping bunch"
    raise "Empty prompt"
  end
  max_retries = 10
  times_retried = 0

  # translate the source string to the target language
  begin
    request_bunch_translate(bunch, prompt, progress)
  rescue Net::ReadTimeout => error
    if times_retried < max_retries
      times_retried += 1
      UI.important "Failed to request translation, retry #{times_retried}/#{max_retries}"
      wait 1
      retry
    else
      UI.error "Can't translate the bunch: #{error}"
    end
  end
end

#request_bunch_translate(strings, prompt, progress) ⇒ Object 



278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 278

def request_bunch_translate(strings, prompt, progress)
  response = @client.chat(
    parameters: {
      model: @params[:model_name],
      messages: [
        { role: "user", content: prompt }
      ],
      temperature: @params[:temperature],
    }
  )
  # extract the translated string from the response
  error = response.dig("error", "message")
  
  #key_log = Colorizer::colorize(key, :blue)
  index_log = Colorizer::colorize("[#{progress}%]", :white)
  if error
    UI.error "#{index_log} Error translating: #{error}"
  else
    target_string = response.dig("choices", 0, "message", "content")
    json_string = target_string[/\[[^\[\]]*\]/m]
    begin
      json_hash = JSON.parse(json_string)
    rescue => error
      UI.error "#{index_log} Error parsing JSON: #{error}"
      UI.error "#{index_log} JSON: \"#{json_string}\""
      return
    end
    keys_to_translate = json_hash.map { |string_hash| string_hash["key"] }
    json_hash.each do |string_hash|
      key = string_hash["key"]
      context = string_hash["context"]
      string_hash.delete("key")
      string_hash.delete("context")
      translated_string = string_hash.values.first
      return unless key && !key.empty? 
      real_key = @keys_associations[key]
      if translated_string.is_a? Hash
        strings = {}
        translated_string.each do |pl_key, value|
          UI.message "#{index_log} #{real_key}: #{pl_key} - #{value}"
          strings[pl_key] = LocoStrings::LocoString.new(pl_key, value, context)
        end
        string = LocoStrings::LocoVariantions.new(real_key, strings, context)
      elsif translated_string && !translated_string.empty?
        UI.message "#{index_log} #{real_key}: #{translated_string}"
        string = LocoStrings::LocoString.new(real_key, translated_string, context)
      end
      @output_hash[real_key] = string
      keys_to_translate.delete(key)
    end

    if keys_to_translate.length > 0
      UI.important "#{index_log} Unable to translate #{keys_to_translate.join(", ")}"
    end
  end
end

#request_translate(key, string, prompt, index) ⇒ Object

Request a translation from the GPT API



250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 250

def request_translate(key, string, prompt, index)
  response = @client.chat(
    parameters: {
      model: @params[:model_name], 
      messages: [
        { role: "user", content: prompt }
      ], 
      temperature: @params[:temperature],
    }
  )
  # extract the translated string from the response
  error = response.dig("error", "message")
  key_log = Colorizer::colorize(key, :blue)
  index_log = Colorizer::colorize("[#{index + 1}/#{@translation_count}]", :white)
  if error
    UI.error "#{index_log} Error translating #{key_log}: #{error}"
  else
    target_string = response.dig("choices", 0, "message", "content")
    if target_string && !target_string.empty?
      UI.message "#{index_log} Translating #{key_log} - #{string.value} -> #{target_string}"
      string.value = target_string
      @output_hash[key] = string
    else
      UI.important "#{index_log} Unable to translate #{key_log} - #{string.value}"
    end
  end
end

#transform_string(input_string) ⇒ Object 



243
244
245
246
247
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 243

def transform_string(input_string)
  uppercased_string = input_string.upcase
  escaped_string = uppercased_string.gsub(/[^0-9a-zA-Z]+/, '_')
  return escaped_string
end

#translate_bunch_of_strings(bunch_size) ⇒ Object 



105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 105

def translate_bunch_of_strings(bunch_size)
  bunch_index = 0
  number_of_bunches = (@translation_count / bunch_size.to_f).ceil
  @keys_associations = {}
  @to_translate.each_slice(bunch_size) do |bunch|
    begin 
      progress = (bunch_index / number_of_bunches.to_f * 100).round
      request_bunch(bunch, progress)
      bunch_index += 1
    rescue "Empty prompt"
      next
    end
    if bunch_index < number_of_bunches - 1 then wait end
  end
end

#translate_bunch_with_tokenizer(max_tokens) ⇒ Object 



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 121

def translate_bunch_with_tokenizer(max_tokens)
  string_index = 0 
  @keys_associations = {}
  current_bunch = {}
  @to_translate.each do |key, string|
    string_index += 1
    tmp_bunch = current_bunch.clone
    tmp_bunch[key] = string

    prompt = prepare_bunch_prompt tmp_bunch
    tokens = OpenAI.rough_token_count(prompt)
    if tokens > max_tokens
      if current_bunch.empty?
        string_index -= 1
        UI.error "Can't translate #{key}: string is too long"
        next
      end
      prompt = prepare_bunch_prompt current_bunch
      progress = (string_index / @translation_count.to_f * 100).round
      request_bunch(bunch, progress)
      current_bunch = {}
      current_bunch[key] = string
      if progress < 100 then wait end
    else 
      current_bunch = tmp_bunch
    end
  end

  if !current_bunch.empty?
    prompt = prepare_bunch_prompt current_bunch
    progress = (string_index / @translation_count.to_f * 100).round
    request_bunch(current_bunch, progress)
  end

end

#translate_strings ⇒ Object

Cycle through the input strings and translate them



81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 81

def translate_strings()
  @to_translate.each_with_index do |(key, string), index|
    prompt = prepare_prompt string

    max_retries = 10
    times_retried = 0

    # translate the source string to the target language
    begin
      request_translate(key, string, prompt, index)
    rescue Net::ReadTimeout => error
      if times_retried < max_retries
        times_retried += 1
        UI.important "Failed to request translation, retry #{times_retried}/#{max_retries}"
        wait 1
        retry
      else
        UI.error "Can't translate #{key}: #{error}"
      end
    end
    if index < @translation_count - 1 then wait end
  end
end

#wait(seconds = @timeout) ⇒ Object

Sleep for a specified number of seconds, displaying a progress bar

Parameters:

  • seconds (Integer) (defaults to: @timeout) —

    The number of seconds to sleep



392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 392

def wait(seconds = @timeout)
  sleep_time = 0
  while sleep_time < seconds
    percent_complete = (sleep_time.to_f / seconds.to_f) * 100.0
    progress_bar_width = 20
    completed_width = (progress_bar_width * percent_complete / 100.0).round
    remaining_width = progress_bar_width - completed_width
    print "\rTimeout [" 
    print Colorizer::code(:green)
    print "=" * completed_width
    print " " * remaining_width
    print Colorizer::code(:reset)
    print "]"
    print " %.2f%%" % percent_complete
    $stdout.flush
    sleep(1)
    sleep_time += 1
  end
  print "\r"
  $stdout.flush
end

#write_output ⇒ Object

Write the translated strings to the target file



336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
 
# File 'lib/fastlane/plugin/translate_gpt/helper/translate_gpt_helper.rb', line 336

def write_output()
  number_of_strings = Colorizer::colorize("#{@output_hash.size}", :blue)  
  target_string = Colorizer::colorize(@params[:target_file], :white)
  UI.message "Writing #{number_of_strings} strings to #{target_string}..."

  if @xcfile.nil?
    file = LocoStrings.load(@params[:target_file])
    file.read
    @output_hash.each do |key, value|
      file.update(key, value.value, value.comment)
    end
    file.write
  else
    default_state = if :mark_for_review then "needs_review" else "translated" end
    @xcfile.update_file_path(@params[:target_file])
    @output_hash.each do |key, value|
      if value.is_a? LocoStrings::LocoString
        @xcfile.update(key, value.value, value.comment, default_state, @params[:target_language])
      elsif value.is_a? LocoStrings::LocoVariantions
        value.strings.each do |pl_key, variant|
          @xcfile.update_variation(key, pl_key, variant.value, variant.comment, default_state, @params[:target_language])
        end
      end
    end
    @xcfile.write
  end
end