monorail
/
mastodon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve encoding detection for link cards (#30780)

This commit is contained in:
David Roetzel 2024-06-21 14:51:10 +02:00 committed by GitHub
parent 4e6db5e284
commit 2cab1c7b09
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 36 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
app/lib/link_details_extractor.rb
View File

@ -269,16 +269,21 @@ class LinkDetailsExtractor
end end
def document def document
@document ||= Nokogiri::HTML(@html, nil, encoding) @document ||= detect_encoding_and_parse_document
end end
def encoding def detect_encoding_and_parse_document
@encoding ||= begin [detect_encoding, nil, @html_charset, 'UTF-8'].uniq.each do |encoding|
guess = detector.detect(@html, @html_charset) document = Nokogiri::HTML(@html, nil, encoding)
guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil return document if document.to_s.valid_encoding?
end end
end end
def detect_encoding
guess = detector.detect(@html, @html_charset)
guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
end
def detector def detector
@detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector| @detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector|
detector.strip_tags = true detector.strip_tags = true

17
spec/fixtures/requests/low_confidence_latin1.txt vendored Normal file
View File

@ -0,0 +1,17 @@
HTTP/1.1 200 OK
server: nginx
date: Thu, 13 Jun 2024 14:33:13 GMT
content-type: text/html; charset=ISO-8859-1
content-length: 158
accept-ranges: bytes
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Tofu á l'orange</title>
</head>
<body>
<h2>Tofu á l'orange</h2>
</body>
</html>

9
spec/services/fetch_link_card_service_spec.rb
View File

@ -26,6 +26,7 @@ RSpec.describe FetchLinkCardService do
stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt')) stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt'))
stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt')) stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt'))
stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt')) stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
stub_request(:get, 'http://example.com/low_confidence_latin1').to_return(request_fixture('low_confidence_latin1.txt'))
Rails.cache.write('oembed_endpoint:example.com', oembed_cache) if oembed_cache Rails.cache.write('oembed_endpoint:example.com', oembed_cache) if oembed_cache
@ -148,6 +149,14 @@ RSpec.describe FetchLinkCardService do
end end
end end
context 'with a URL of a page in ISO-8859-1 encoding, that charlock_holmes cannot detect' do
let(:status) { Fabricate(:status, text: 'Check out http://example.com/low_confidence_latin1') }
it 'decodes the HTML' do
expect(status.preview_card.title).to eq("Tofu á l'orange")
end
end
context 'with a Japanese path URL' do context 'with a Japanese path URL' do
let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') } let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') }