From 267323cad5ca4888dae11c9e08ccef175f7d72c7 Mon Sep 17 00:00:00 2001 From: duerst Date: Tue, 6 Jun 2017 10:34:10 +0000 Subject: add test_ill_formed_utf_8_replace to test recommended number of \uFFFD Add test method test_ill_formed_utf_8_replace to test/ruby/test_transcode.rb to check for the recommended number of \uFFFD replacement characters. This is the first part, using ill-formed prefixes, with suffixes up to the length of the original UTF-8 structure (including overlongs and the full 31-bit space.) For more details, see Unicode 9.0.0, Section 3.9, Best Practices for Using U+FFFD. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59026 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- test/ruby/test_transcode.rb | 74 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) (limited to 'test/ruby/test_transcode.rb') diff --git a/test/ruby/test_transcode.rb b/test/ruby/test_transcode.rb index b65a8c2b98..1561468b54 100644 --- a/test/ruby/test_transcode.rb +++ b/test/ruby/test_transcode.rb @@ -998,6 +998,80 @@ class TestTranscode < Test::Unit::TestCase check_both_ways("\u00A0", "\xFF", 'CP855') # non-breaking space end + def test_ill_formed_utf_8_replace + fffd1 = "\uFFFD".encode 'UTF-16BE' + fffd2 = "\uFFFD\uFFFD".encode 'UTF-16BE' + fffd3 = "\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE' + fffd4 = "\uFFFD\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE' + fffd5 = "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE' + fffd6 = "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE' + + assert_equal fffd2, "\xC0\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xC0\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xC1\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xC1\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xE0\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xE0\x9F".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xE0\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xE0\x9F\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xED\xA0".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xED\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xED\xA0\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xED\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xF0\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xF0\x8F".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xF0\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xF0\x8F\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd4, "\xF0\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd4, "\xF0\x8F\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xF4\x90".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xF4\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xF4\x90\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xF4\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd4, "\xF4\x90\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd4, "\xF4\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xF5\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xF7\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xF5\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xF7\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd4, "\xF5\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd4, "\xF7\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd1, "\xF8".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd1, "\xFB".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xF8\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xFB\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xF8\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xFB\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd4, "\xF8\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd4, "\xFB\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd5, "\xF8\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd5, "\xFB\xBF\xBF\xBF\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd1, "\xFC".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd1, "\xFD".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xFC\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xFD\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xFC\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xFD\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd4, "\xFC\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd4, "\xFD\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd5, "\xFC\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd5, "\xFD\xBF\xBF\xBF\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd6, "\xFC\x80\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd6, "\xFD\xBF\xBF\xBF\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd1, "\xFE".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd1, "\xFF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xFE\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd2, "\xFF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xFE\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd3, "\xFF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd4, "\xFE\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd4, "\xFF\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd5, "\xFE\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd5, "\xFF\xBF\xBF\xBF\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd6, "\xFE\x80\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + assert_equal fffd6, "\xFF\xBF\xBF\xBF\x80\x80".encode("utf-16be", "utf-8", invalid: :replace) + end + def check_utf_16_both_ways(utf8, raw) copy = raw.dup 0.step(copy.length-1, 2) { |i| copy[i+1], copy[i] = copy[i], copy[i+1] } -- cgit v1.2.3