mastodon/app/lib/language_detector.rb
dependabot-preview[bot] 7088633ae1
Bump cld3 from 3.2.6 to 3.3.0 (#13107)
* Bump cld3 from 3.2.6 to 3.3.0

Bumps [cld3](https://github.com/akihikodaki/cld3-ruby) from 3.2.6 to 3.3.0.
- [Release notes](https://github.com/akihikodaki/cld3-ruby/releases)
- [Commits](https://github.com/akihikodaki/cld3-ruby/compare/v3.2.6...v3.3.0)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>

* Fix compatibility with cld3 3.3.0

Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com>
Co-authored-by: Eugen Rochko <eugen@zeonfederated.com>
2020-03-09 00:12:52 +01:00

102 lines
2.4 KiB
Ruby

# frozen_string_literal: true
class LanguageDetector
include Singleton
WORDS_THRESHOLD = 4
RELIABLE_CHARACTERS_RE = /[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}\p{Han}\p{Katakana}\p{Hiragana}\p{Hangul}]+/m
def initialize
@identifier = CLD3::NNetLanguageIdentifier.new(1, 2048)
end
def detect(text, account)
input_text = prepare_text(text)
return if input_text.blank?
detect_language_code(input_text) || default_locale(account)
end
def language_names
@language_names = CLD3::TaskContextParams::LANGUAGE_NAMES.map { |name| iso6391(name.to_s).to_sym }.uniq
end
private
def prepare_text(text)
simplify_text(text).strip
end
def unreliable_input?(text)
!reliable_input?(text)
end
def reliable_input?(text)
sufficient_text_length?(text) || language_specific_character_set?(text)
end
def sufficient_text_length?(text)
text.split(/\s+/).size >= WORDS_THRESHOLD
end
def language_specific_character_set?(text)
words = text.scan(RELIABLE_CHARACTERS_RE)
if words.present?
words.reduce(0) { |acc, elem| acc + elem.size }.to_f / text.size > 0.3
else
false
end
end
def detect_language_code(text)
return if unreliable_input?(text)
result = @identifier.find_language(text)
iso6391(result.language.to_s).to_sym if result&.reliable?
end
def iso6391(bcp47)
iso639 = bcp47.split('-').first
# CLD3 returns grandfathered language code for Hebrew
return 'he' if iso639 == 'iw'
ISO_639.find(iso639).alpha2
end
def simplify_text(text)
new_text = remove_html(text)
new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
new_text.gsub!(Account::MENTION_RE, '')
new_text.gsub!(Tag::HASHTAG_RE) { |string| string.gsub(/[#_]/, '#' => '', '_' => ' ').gsub(/[a-z][A-Z]|[a-zA-Z][\d]/) { |s| s.insert(1, ' ') }.downcase }
new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '')
new_text.gsub!(/\s+/, ' ')
new_text
end
def new_scrubber
scrubber = Rails::Html::PermitScrubber.new
scrubber.tags = %w(br p)
scrubber
end
def scrubber
@scrubber ||= new_scrubber
end
def remove_html(text)
text = Loofah.fragment(text).scrub!(scrubber).to_s
text.gsub!('<br>', "\n")
text.gsub!('</p><p>', "\n\n")
text.gsub!(/(^<p>|<\/p>$)/, '')
text
end
def default_locale(account)
account.user_locale&.to_sym || I18n.default_locale if account.local?
end
end