Sha256: 311a26b1f168f0cb032b32d6e1c763a7b35657ba737fd74d57244b8572bc4e6a
Contents?: true
Size: 1.86 KB
Versions: 33
Compression:
Stored size: 1.86 KB
Contents
module EncodingHelper
extend self
# This threshold is carefully tweaked to prevent usage of encodings detected
# by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
# we're better off sticking with utf8 encoding.
# Reason: git diff can return strings with invalid utf8 byte sequences if it
# truncates a diff in the middle of a multibyte character. In this case
# CharlockHolmes will try to guess the encoding and will likely suggest an
# obscure encoding with low confidence.
# There is a lot more info with this merge request:
# https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
ENCODING_CONFIDENCE_THRESHOLD = 40
def encode!(message)
return nil unless message.respond_to? :force_encoding
# if message is utf-8 encoding, just return it
message.force_encoding("UTF-8")
return message if message.valid_encoding?
# return message if message type is binary
detect = CharlockHolmes::EncodingDetector.detect(message)
return message.force_encoding("BINARY") if detect && detect[:type] == :binary
# force detected encoding if we have sufficient confidence.
if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
message.force_encoding(detect[:encoding])
end
# encode and clean the bad chars
message.replace clean(message)
rescue
encoding = detect ? detect[:encoding] : "unknown"
"--broken encoding: #{encoding}"
end
def encode_utf8(message)
detect = CharlockHolmes::EncodingDetector.detect(message)
if detect
CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
else
clean(message)
end
end
private
def clean(message)
message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
.encode("UTF-8")
.gsub("\0".encode("UTF-8"), "")
end
end
Version data entries
33 entries across 33 versions & 1 rubygems