asonix/mastodon
1
1
Fork 1
Code Issues Pull requests Projects Releases Wiki Activity

v3.4.5-branch #1

Merged
asonix merged 60 commits from v3.4.5-branch into asonix/changes 2022-01-31 18:33:23 +00:00
Conversation 0 Commits 60 Files changed 154 +5 -30
2 changed files with 5 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 9a468c895b - Show all commits

31
app/lib/formatter.rb
View file

@ -214,39 +214,10 @@ class Formatter
result.flatten.join
end
UNICODE_ESCAPE_BLACKLIST_RE = /\p{Z}|\p{P}/
def utf8_friendly_extractor(text, options = {})
old_to_new_index = [0]
escaped = text.chars.map do |c|
output = begin
if c.ord.to_s(16).length > 2 && !UNICODE_ESCAPE_BLACKLIST_RE.match?(c)
CGI.escape(c)
else
c
end
end
old_to_new_index << old_to_new_index.last + output.length
output
end.join
# Note: I couldn't obtain list_slug with @user/list-name format
# for mention so this requires additional check
special = Extractor.extract_urls_with_indices(escaped, options).map do |extract|
new_indices = [
old_to_new_index.find_index(extract[:indices].first),
old_to_new_index.find_index(extract[:indices].last),
]
next extract.merge(
indices: new_indices,
url: text[new_indices.first..new_indices.last - 1]
)
end
special = Extractor.extract_urls_with_indices(text, options)
standard = Extractor.extract_entities_with_indices(text, options)
extra = Extractor.extract_extra_uris_with_indices(text, options)

4
config/initializers/twitter_regex.rb
View file

@ -24,6 +24,10 @@ module Twitter::TwitterText
)
\)
/iox
REGEXEN[:valid_iri_ucschar] = /[\u{A0}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFEF}\u{10000}-\u{1FFFD}\u{20000}-\u{2FFFD}\u{30000}-\u{3FFFD}\u{40000}-\u{4FFFD}\u{50000}-\u{5FFFD}\u{60000}-\u{6FFFD}\u{70000}-\u{7FFFD}\u{80000}-\u{8FFFD}\u{90000}-\u{9FFFD}\u{A0000}-\u{AFFFD}\u{B0000}-\u{BFFFD}\u{C0000}-\u{CFFFD}\u{D0000}-\u{DFFFD}\u{E1000}-\u{EFFFD}]/iou
REGEXEN[:valid_iri_iprivate] = /[\u{E000}-\u{F8FF}\u{F0000}-\u{FFFFD}\u{100000}-\u{10FFFD}]/iou
REGEXEN[:valid_url_query_chars] = /(?:#{REGEXEN[:valid_iri_ucschar]})|(?:#{REGEXEN[:valid_iri_iprivate]})|[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@]/iou
REGEXEN[:valid_url_query_ending_chars] = /(?:#{REGEXEN[:valid_iri_ucschar]})|(?:#{REGEXEN[:valid_iri_iprivate]})|[a-z0-9_&=#\/\-]/iou
REGEXEN[:valid_url_path] = /(?:
(?:
#{REGEXEN[:valid_general_url_path_chars]}*