Module: Gitlab::EncodingHelper

Extended by:
EncodingHelper
Included in:
AbstractPathValidator, BlobPresenter, Ci::ParseAnnotationsArtifactService, Ci::ParseDotenvArtifactService, Ci::Ansi2html::Converter, Ci::Ansi2json::Line::Segment, EncodingHelper, Git, Git::Blame, Git::Blob, Git::Commit, Git::Diff, Git::Ref, Git::Repository, Git::Tag, Git::Tree, GitalyClient::AnalysisService, GitalyClient::BlobService, GitalyClient::CleanupService, GitalyClient::CommitService, GitalyClient::ConflictsService, GitalyClient::OperationService, GitalyClient::RefService, GitalyClient::RemoteService, GitalyClient::RepositoryService, GitalyClient::WikiPage, GithubImport::MarkdownText, GrapeLogging::Formatters::LogrageWithTimestamp, Search::FoundBlob, Search::Query, MergeRequestContextCommitDiffFile, MergeRequestDiffFile, NamespacePathValidator, NoteDiffFile, ProjectPathValidator
Defined in:
lib/gitlab/encoding_helper.rb

Constant Summary collapse

ENCODING_CONFIDENCE_THRESHOLD =

This threshold is carefully tweaked to prevent usage of encodings detected by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, we’re better off sticking with utf8 encoding. Reason: git diff can return strings with invalid utf8 byte sequences if it truncates a diff in the middle of a multibyte character. In this case CharlockHolmes will try to guess the encoding and will likely suggest an obscure encoding with low confidence. There is a lot more info with this merge request: gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193

50
UNICODE_REPLACEMENT_CHARACTER = 
"�"
BOM_UTF8 = 
"\xEF\xBB\xBF"
ESCAPED_CHARS = 
{
  "a" => "\a", "b" => "\b", "e" => "\e", "f" => "\f",
  "n" => "\n", "r" => "\r", "t" => "\t", "v" => "\v",
  "\"" => "\""
}.freeze

Instance Method Summary collapse

Instance Method Details

#binary_io(str_or_io) ⇒ Object 



115
116
117
118
119
120
 
# File 'lib/gitlab/encoding_helper.rb', line 115

def binary_io(str_or_io)
  io = str_or_io.to_io.dup if str_or_io.respond_to?(:to_io)
  io ||= StringIO.new(str_or_io.to_s.freeze)

  io.tap { |io| io.set_encoding(Encoding::ASCII_8BIT) }
end

#detect_binary?(data, detect = nil) ⇒ Boolean

Returns:

  • (Boolean) 


49
50
51
52
 
# File 'lib/gitlab/encoding_helper.rb', line 49

def detect_binary?(data, detect = nil)
  detect ||= detect_encoding(data)
  detect && detect[:type] == :binary && detect[:confidence] == 100
end

#detect_encoding(data, limit: CharlockHolmes::EncodingDetector::DEFAULT_BINARY_SCAN_LEN) ⇒ Object 



43
44
45
46
47
 
# File 'lib/gitlab/encoding_helper.rb', line 43

def detect_encoding(data, limit: CharlockHolmes::EncodingDetector::DEFAULT_BINARY_SCAN_LEN)
  return if data.nil?

  CharlockHolmes::EncodingDetector.new(limit).detect(data)
end

#detect_libgit2_binary?(data) ⇒ Boolean

EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks only the first 8000 (github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), which is what we use below to keep a consistent behavior.

Returns:

  • (Boolean) 


57
58
59
60
 
# File 'lib/gitlab/encoding_helper.rb', line 57

def detect_libgit2_binary?(data)
  detect = detect_encoding(data, limit: 8000)
  detect && detect[:type] == :binary
end

#encode!(message) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/gitlab/encoding_helper.rb', line 21

def encode!(message)
  message = force_encode_utf8(message)
  return message if message.valid_encoding?

  # return message if message type is binary
  detect = detect_encoding(message)
  return message.force_encoding("BINARY") if detect_binary?(message, detect)

  if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
    # force detected encoding if we have sufficient confidence.
    message.force_encoding(detect[:encoding])
  end

  # encode and clean the bad chars
  message.replace clean(message)
rescue ArgumentError => e
  return unless e.message.include?('unknown encoding name')

  encoding = detect ? detect[:encoding] : "unknown"
  "--broken encoding: #{encoding}"
end

#encode_binary(str) ⇒ Object 



109
110
111
112
113
 
# File 'lib/gitlab/encoding_helper.rb', line 109

def encode_binary(str)
  return "" if str.nil?

  str.dup.force_encoding(Encoding::ASCII_8BIT)
end

#encode_utf8(message, replace: "") ⇒ Object 



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/gitlab/encoding_helper.rb', line 88

def encode_utf8(message, replace: "")
  message = force_encode_utf8(message)
  return message if message.valid_encoding?

  detect = detect_encoding(message)

  if detect && detect[:encoding]
    begin
      CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
    rescue ArgumentError => e
      Gitlab::AppLogger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}")

      ''
    end
  else
    clean(message, replace: replace)
  end
rescue ArgumentError
  nil
end

#encode_utf8_no_detect(message) ⇒ Object

This is like encode_utf8 except we skip autodetection of the encoding. We assume the data must be interpreted as UTF-8.



64
65
66
67
68
69
 
# File 'lib/gitlab/encoding_helper.rb', line 64

def encode_utf8_no_detect(message)
  message = force_encode_utf8(message)
  return message if message.valid_encoding?

  message.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
end

#encode_utf8_with_escaping!(message) ⇒ Object

This method escapes unsupported UTF-8 characters instead of deleting them



76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/gitlab/encoding_helper.rb', line 76

def encode_utf8_with_escaping!(message)
  message = force_encode_utf8(message)
  return message if message.valid_encoding?

  unless message.valid_encoding?
    message = message.chars.map { |char| char.valid_encoding? ? char : escape_chars(char) }.join
  end

  # encode and clean the bad chars
  message.replace clean(message)
end

#encode_utf8_with_replacement_character(data) ⇒ Object 



71
72
73
 
# File 'lib/gitlab/encoding_helper.rb', line 71

def encode_utf8_with_replacement_character(data)
  encode_utf8(data, replace: UNICODE_REPLACEMENT_CHARACTER)
end

#force_encode_utf8(message) ⇒ Object

Raises:

  • (ArgumentError) 


155
156
157
158
159
160
161
162
 
# File 'lib/gitlab/encoding_helper.rb', line 155

def force_encode_utf8(message)
  raise ArgumentError unless message.respond_to?(:force_encoding)
  return message if message.encoding == Encoding::UTF_8 && message.valid_encoding?

  message = message.dup if message.respond_to?(:frozen?) && message.frozen?

  message.force_encoding("UTF-8")
end

#strip_bom(message) ⇒ Object 



151
152
153
 
# File 'lib/gitlab/encoding_helper.rb', line 151

def strip_bom(message)
  message.delete_prefix(BOM_UTF8)
end

#unquote_path(filename) ⇒ Object

rubocop:disable Style/AsciiComments ‘unquote_path` decode filepaths that are returned by some git commands. The path may be returned in double-quotes if it contains special characters, that are encoded in octal. Also, some characters (see `ESCAPED_CHARS`) are escaped. eg. “311240304253305247305200310247306200” (quotes included) is decoded as ɠīŧŀȧƀ

Based on ‘unquote_c_style` from git source github.com/git/git/blob/v2.35.1/quote.c#L399 rubocop:enable Style/AsciiComments



137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/gitlab/encoding_helper.rb', line 137

def unquote_path(filename)
  return filename unless filename[0] == '"'

  filename = filename[1..-2].gsub(/\\(?:([#{ESCAPED_CHARS.keys.join}\\])|(\d{3}))/) do
    if c = Regexp.last_match(1)
      c == "\\" ? "\\" : ESCAPED_CHARS[c]
    elsif c = Regexp.last_match(2)
      c.to_i(8).chr
    end
  end

  filename.force_encoding("UTF-8")
end