Class: WhatLanguage

Inherits:
Object
  • Object
show all
Defined in:
lib/whatlanguage/languages.rb,
lib/whatlanguage.rb,
lib/whatlanguage/version.rb

Overview

AUTO-GENERATED from the ISO 639-3 registry + whatlang dataset codes. Maps ISO 639-3 code => [language name symbol, ISO 639-1 (or 639-3 fallback) symbol]. Original gem languages keep their historical symbols and 2-letter codes.

Defined Under Namespace

Classes: Result

Constant Summary collapse

VERSION = 
'2.1.0'

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(*selection, only: nil, min_chars: DEFAULT_MIN_CHARS) ⇒ WhatLanguage

Returns a new instance of WhatLanguage.



97
98
99
100
101
 
# File 'lib/whatlanguage.rb', line 97

def initialize(*selection, only: nil, min_chars: DEFAULT_MIN_CHARS)
  @selection = Array(only || (selection.empty? ? [:all] : selection))
  validate_selection!
  @min_chars = min_chars
end

Class Method Details

.detect(text) ⇒ Object 



57
58
59
 
# File 'lib/whatlanguage.rb', line 57

def detect(text)
  default_detector.detect(text)
end

.language(text) ⇒ Object 



61
62
63
 
# File 'lib/whatlanguage.rb', line 61

def language(text)
  default_detector.language(text)
end

.language_iso(text) ⇒ Object 



65
66
67
 
# File 'lib/whatlanguage.rb', line 65

def language_iso(text)
  default_detector.language_iso(text)
end

.languages ⇒ Object 



80
81
82
 
# File 'lib/whatlanguage.rb', line 80

def languages
  NAME_TO_CODE.keys
end

.process_text ⇒ Object 



78
79
80
 
# File 'lib/whatlanguage.rb', line 78

def score_hash(text)
  default_detector.score_hash(text)
end

.profiles ⇒ Object

script name => [[code, [trigram, ...]], ...], loaded once and memoized.



85
86
87
88
 
# File 'lib/whatlanguage.rb', line 85

def profiles
  @profiles ||= JSON.parse(File.read(File.join(__dir__, 'whatlanguage', 'trigrams.json')))
                    .transform_values { |langs| langs.map { |code, str| [code, str.split('|')] } }
end

.ranked(text) ⇒ Object 



69
70
71
 
# File 'lib/whatlanguage.rb', line 69

def ranked(text)
  default_detector.ranked(text)
end

.score_hash(text) ⇒ Object 



73
74
75
 
# File 'lib/whatlanguage.rb', line 73

def score_hash(text)
  default_detector.score_hash(text)
end

.scores ⇒ Object 



77
78
79
 
# File 'lib/whatlanguage.rb', line 77

def score_hash(text)
  default_detector.score_hash(text)
end

Instance Method Details

#detect(text) ⇒ Object

Detection result with the winning language, ISO code, winning score, and full ranked scores. Returns nil when the text is too short or unrecognized.



158
159
160
161
162
163
164
 
# File 'lib/whatlanguage.rb', line 158

def detect(text)
  ranked_scores = ranked(text)
  return nil if ranked_scores.empty?

  name, score = ranked_scores.first
  Result.new(language: name, iso: ISO_CODES[name], score: score, ranked: ranked_scores)
end

#language(text) ⇒ Object

Most likely language as a name symbol, or nil when no language is detected.



167
168
169
 
# File 'lib/whatlanguage.rb', line 167

def language(text)
  detect(text)&.language
end

#language_iso(text) ⇒ Object

Most likely language as an ISO 639-1 symbol (639-3 fallback), or nil.



172
173
174
 
# File 'lib/whatlanguage.rb', line 172

def language_iso(text)
  detect(text)&.iso
end

#languages ⇒ Object

Language-name symbols this instance scores against: every supported language for :all, otherwise the requested selection intersected with the supported set (legacy aliases such as :pinyin resolved to their modern names).



106
107
108
109
110
111
112
113
114
 
# File 'lib/whatlanguage.rb', line 106

def languages
  @languages ||=
    if @selection.include?(:all)
      self.class.languages
    else
      wanted = @selection.map { |s| NAME_ALIASES.fetch(s, s) }
      self.class.languages & wanted
    end
end

#ranked(text) ⇒ Object

Per-language scores as an array sorted from most likely to least likely.



152
153
154
 
# File 'lib/whatlanguage.rb', line 152

def ranked(text)
  score_hash(text).sort_by { |_name, score| -score }
end

#score_hash(text) ⇒ Object Also known as: scores, process_text

Per-language scores for the text (higher = more likely). Languages outside the current selection, or not under the detected script, are absent; the hash defaults to 0. Only the relative ranking is meaningful.



119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/whatlanguage.rb', line 119

def score_hash(text)
  results = Hash.new(0)
  text = normalize_text(text)
  script = detect_script(text)
  return results unless script

  if (code = DETERMINISTIC[script])
    name = CODE_INFO[code].first
    results[name] = MAX_TOTAL_DISTANCE if allowed?(name)
    return results
  end

  candidates = self.class.profiles[script]
  return results unless candidates
  char_count = significant_char_count(text)
  return results if char_count < @min_chars

  bonus = short_text_bonus(char_count)
  positions = trigram_positions(text)
  candidates.each do |code, trigrams|
    name = CODE_INFO[code].first
    next unless allowed?(name)

    results[name] = MAX_TOTAL_DISTANCE - distance(trigrams, positions)
    results[name] += bonus if SHORT_TEXT_PREFERENCES.include?(name)
  end
  results
end