Module: Menkar

Defined in:
lib/menkar.rb,
lib/menkar.rb,
lib/menkar/version.rb,
lib/menkar/detector.rb,
lib/menkar/transcoder.rb,
lib/menkar/encoding_scorer.rb,
sig/menkar.rbs

Defined Under Namespace

Classes: Detection, Error, Indent

Constant Summary collapse

VERSION =

Returns:

  • (String) 
"0.1.0"
DEFAULT_SAMPLE =

Returns:

  • (Integer) 
64 * 1024
MAX_SAMPLE =

Returns:

  • (Integer) 
16 * 1024 * 1024

Class Method Summary collapse

Class Method Details

.binary?(bytes) ⇒ Boolean

Parameters:

  • bytes (String)

Returns:

  • (Boolean) 


73
74
75
 
# File 'lib/menkar/detector.rb', line 73

def binary?(bytes)
  detect(bytes).binary
end

.decode(bytes, detection) ⇒ String

Parameters:

Returns:

  • (String) 


8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/menkar/transcoder.rb', line 8

def decode(bytes, detection)
  validate_detection!(detection)
  raise Error, "cannot decode binary input" if detection.binary
  raise Error, "bytes must be a String" unless bytes.is_a?(String)

  source = bytes.b
  unless detection.bom.empty?
    raise Error, "input does not start with the detected BOM" unless source.start_with?(detection.bom)

    source = source.byteslice(detection.bom.bytesize..) || "".b
  end
  source.force_encoding(detection.encoding)
  raise Error, "invalid #{detection.encoding.name} input" unless source.valid_encoding?

  source.encode(Encoding::UTF_8)
rescue EncodingError => error
  raise Error, "cannot decode #{detection.encoding.name}: #{error.message}"
end

.detect(bytes, hint: nil, sample: DEFAULT_SAMPLE) ⇒ Detection

Parameters:

  • bytes (String)
  • hint: (Object) (defaults to: nil)
  • sample: (Integer) (defaults to: DEFAULT_SAMPLE)

Returns:



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/menkar/detector.rb', line 20

def detect(bytes, hint: nil, sample: DEFAULT_SAMPLE)
  validate_bytes!(bytes)
  limit = validate_sample!(sample)
  hint_encoding = normalize_hint(hint)
  chunk = (bytes.byteslice(0, limit) || "").b
  truncated = bytes.bytesize > chunk.bytesize
  bom, bom_encoding = BOMS.find { |mark, _encoding| chunk.start_with?(mark) }

  if bom_encoding
    text = EncodingScorer.decode(chunk.byteslice(bom.bytesize..) || "".b, bom_encoding, truncated: truncated)
    return detection(bom_encoding, 1.0, bom, text, binary: false)
  end

  if chunk.include?("\0")
    encoding, text = nul_encoding(chunk, truncated: truncated)
    return detection(encoding, 0.95, "".b.freeze, text, binary: false) if encoding

    return detection(nil, 1.0, "".b.freeze, nil, binary: true)
  end

  return detection(nil, 1.0, "".b.freeze, nil, binary: true) if binary_sample?(chunk)

  if chunk.match?(/\e\$(?:@|B)|\e\([BJ]/n)
    text = EncodingScorer.decode(chunk, Encoding::ISO_2022_JP, truncated: truncated)
    return detection(Encoding::ISO_2022_JP, 0.99, "".b.freeze, text, binary: false) if text
  end

  utf8 = EncodingScorer.decode(chunk, Encoding::UTF_8, truncated: truncated)
  if utf8
    if utf8.ascii_only? && hint_encoding && hint_encoding != Encoding::UTF_8
      return detection(hint_encoding, 0.6, "".b.freeze, utf8, binary: false)
    end
    return detection(Encoding::UTF_8, utf8.ascii_only? ? 0.5 : 0.9, "".b.freeze, utf8, binary: false)
  end

  encoding, confidence, text = EncodingScorer.legacy(chunk, hint_encoding, truncated: truncated)
  return detection(nil, 1.0, "".b.freeze, nil, binary: true) unless encoding

  detection(encoding, confidence, "".b.freeze, text, binary: false)
end

.detect_file(path, sample: DEFAULT_SAMPLE) ⇒ Detection

Parameters:

  • path (String)
  • sample: (Integer) (defaults to: DEFAULT_SAMPLE)

Returns:



61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/menkar/detector.rb', line 61

def detect_file(path, sample: DEFAULT_SAMPLE)
  limit = validate_sample!(sample)
  unless path.is_a?(String) && path.valid_encoding? && !path.include?("\0")
    raise Error, "path must be a valid String without NUL bytes"
  end

  bytes = File.open(path, "rb") { |file| file.read(limit + 1) || "".b }
  detect(bytes, sample: limit)
rescue SystemCallError => error
  raise Error, "cannot read #{path}: #{error.message}"
end

.encode(string, detection) ⇒ String

Parameters:

Returns:

  • (String) 


27
28
29
30
31
32
33
34
35
36
 
# File 'lib/menkar/transcoder.rb', line 27

def encode(string, detection)
  validate_detection!(detection)
  raise Error, "cannot encode binary input" if detection.binary
  raise Error, "string must be valid text" unless string.is_a?(String) && string.valid_encoding?

  utf8 = string.encode(Encoding::UTF_8)
  detection.bom + utf8.encode(detection.encoding).b
rescue EncodingError => error
  raise Error, "cannot encode #{detection.encoding.name}: #{error.message}"
end

.normalize_newlines(string, to: :lf) ⇒ [String, Symbol]

Parameters:

  • string (String)
  • to: (:lf, :crlf, :cr) (defaults to: :lf)

Returns:

  • ([String, Symbol])

Raises:



50
51
52
53
54
55
56
 
# File 'lib/menkar/transcoder.rb', line 50

def normalize_newlines(string, to: :lf)
  raise Error, "string must be valid text" unless string.is_a?(String) && string.valid_encoding?
  replacement = NEWLINES.fetch(to) { raise Error, "newline must be :lf, :crlf, or :cr" }
  utf8 = string.encode(Encoding::UTF_8)
  original = newline_kind(utf8)
  [utf8.gsub(/\r\n|\r|\n/, replacement), original]
end

.roundtrip?(value, detection) ⇒ Boolean

Parameters:

Returns:

  • (Boolean) 


38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/menkar/transcoder.rb', line 38

def roundtrip?(value, detection)
  return false unless value.is_a?(String)

  if value.encoding == Encoding::UTF_8 && value.valid_encoding?
    decode(encode(value, detection), detection) == value
  else
    encode(decode(value, detection), detection) == value.b
  end
rescue Error
  false
end