[ruby/prism] Provide flags for changing encodings

https://github.com/ruby/prism/commit/e838eaff6f

18 files changed, 176 insertions, 89 deletions

diff --git a/test/prism/encoding_test.rb b/test/prism/encoding_test.rb
index 94ba3a6c2a..e4678c6f82 100644
--- a/test/prism/encoding_test.rb
+++ b/test/prism/encoding_test.rb
@@ -7,90 +7,16 @@ require_relative "test_helper"
 module Prism
   class EncodingTest < TestCase
     codepoints_1byte = 0...0x100
-    codepoints_2bytes = 0...0x10000
-
     encodings = {
-      Encoding::ASCII =>         codepoints_1byte,
-      Encoding::ASCII_8BIT =>    codepoints_1byte,
-      Encoding::CP850 =>         codepoints_1byte,
-      Encoding::CP852 =>         codepoints_1byte,
-      Encoding::CP855 =>         codepoints_1byte,
-      Encoding::GB1988 =>        codepoints_1byte,
-      Encoding::IBM437 =>        codepoints_1byte,
-      Encoding::IBM720 =>        codepoints_1byte,
-      Encoding::IBM737 =>        codepoints_1byte,
-      Encoding::IBM775 =>        codepoints_1byte,
-      Encoding::IBM852 =>        codepoints_1byte,
-      Encoding::IBM855 =>        codepoints_1byte,
-      Encoding::IBM857 =>        codepoints_1byte,
-      Encoding::IBM860 =>        codepoints_1byte,
-      Encoding::IBM861 =>        codepoints_1byte,
-      Encoding::IBM862 =>        codepoints_1byte,
-      Encoding::IBM863 =>        codepoints_1byte,
-      Encoding::IBM864 =>        codepoints_1byte,
-      Encoding::IBM865 =>        codepoints_1byte,
-      Encoding::IBM866 =>        codepoints_1byte,
-      Encoding::IBM869 =>        codepoints_1byte,
-      Encoding::ISO_8859_1 =>    codepoints_1byte,
-      Encoding::ISO_8859_2 =>    codepoints_1byte,
-      Encoding::ISO_8859_3 =>    codepoints_1byte,
-      Encoding::ISO_8859_4 =>    codepoints_1byte,
-      Encoding::ISO_8859_5 =>    codepoints_1byte,
-      Encoding::ISO_8859_6 =>    codepoints_1byte,
-      Encoding::ISO_8859_7 =>    codepoints_1byte,
-      Encoding::ISO_8859_8 =>    codepoints_1byte,
-      Encoding::ISO_8859_9 =>    codepoints_1byte,
-      Encoding::ISO_8859_10 =>   codepoints_1byte,
-      Encoding::ISO_8859_11 =>   codepoints_1byte,
-      Encoding::ISO_8859_13 =>   codepoints_1byte,
-      Encoding::ISO_8859_14 =>   codepoints_1byte,
-      Encoding::ISO_8859_15 =>   codepoints_1byte,
-      Encoding::ISO_8859_16 =>   codepoints_1byte,
-      Encoding::KOI8_R =>        codepoints_1byte,
-      Encoding::KOI8_U =>        codepoints_1byte,
-      Encoding::MACCENTEURO =>   codepoints_1byte,
-      Encoding::MACCROATIAN =>   codepoints_1byte,
-      Encoding::MACCYRILLIC =>   codepoints_1byte,
-      Encoding::MACGREEK =>      codepoints_1byte,
-      Encoding::MACICELAND =>    codepoints_1byte,
-      Encoding::MACROMAN =>      codepoints_1byte,
-      Encoding::MACROMANIA =>    codepoints_1byte,
-      Encoding::MACTHAI =>       codepoints_1byte,
-      Encoding::MACTURKISH =>    codepoints_1byte,
-      Encoding::MACUKRAINE =>    codepoints_1byte,
-      Encoding::TIS_620 =>       codepoints_1byte,
-      Encoding::Windows_1250 =>  codepoints_1byte,
-      Encoding::Windows_1251 =>  codepoints_1byte,
-      Encoding::Windows_1252 =>  codepoints_1byte,
-      Encoding::Windows_1253 =>  codepoints_1byte,
-      Encoding::Windows_1254 =>  codepoints_1byte,
-      Encoding::Windows_1255 =>  codepoints_1byte,
-      Encoding::Windows_1256 =>  codepoints_1byte,
-      Encoding::Windows_1257 =>  codepoints_1byte,
-      Encoding::Windows_1258 =>  codepoints_1byte,
-      Encoding::Windows_874 =>   codepoints_1byte,
-      Encoding::Big5 =>          codepoints_2bytes,
-      Encoding::Big5_HKSCS =>    codepoints_2bytes,
-      Encoding::Big5_UAO =>      codepoints_2bytes,
-      Encoding::CP949 =>         codepoints_2bytes,
-      Encoding::CP950 =>         codepoints_2bytes,
-      Encoding::CP951 =>         codepoints_2bytes,
-      Encoding::EUC_KR =>        codepoints_2bytes,
-      Encoding::GBK =>           codepoints_2bytes,
-      Encoding::GB12345 =>       codepoints_2bytes,
-      Encoding::GB2312 =>        codepoints_2bytes,
-      Encoding::MACJAPANESE =>   codepoints_2bytes,
-      Encoding::Shift_JIS =>     codepoints_2bytes,
-      Encoding::SJIS_DoCoMo =>   codepoints_2bytes,
-      Encoding::SJIS_KDDI =>     codepoints_2bytes,
-      Encoding::SJIS_SoftBank => codepoints_2bytes,
-      Encoding::Windows_31J =>   codepoints_2bytes
+      Encoding::ASCII_8BIT =>   codepoints_1byte,
+      Encoding::US_ASCII =>     codepoints_1byte,
+      Encoding::Windows_1253 => codepoints_1byte
     }
 
-    # By default we don't test every codepoint in these encodings because they
-    # are 3 and 4 byte representations so it can drastically slow down the test
-    # suite.
+    # By default we don't test every codepoint in these encodings because it
+    # takes a very long time.
     if ENV["PRISM_TEST_ALL_ENCODINGS"]
+      codepoints_2bytes = 0...0x10000
       codepoints_unicode = (0...0x110000)
 
       codepoints_eucjp = [
@@ -118,6 +44,78 @@ module Prism
       ]
 
       encodings.merge!(
+        Encoding::CP850 =>                      codepoints_1byte,
+        Encoding::CP852 =>                      codepoints_1byte,
+        Encoding::CP855 =>                      codepoints_1byte,
+        Encoding::GB1988 =>                     codepoints_1byte,
+        Encoding::IBM437 =>                     codepoints_1byte,
+        Encoding::IBM720 =>                     codepoints_1byte,
+        Encoding::IBM737 =>                     codepoints_1byte,
+        Encoding::IBM775 =>                     codepoints_1byte,
+        Encoding::IBM852 =>                     codepoints_1byte,
+        Encoding::IBM855 =>                     codepoints_1byte,
+        Encoding::IBM857 =>                     codepoints_1byte,
+        Encoding::IBM860 =>                     codepoints_1byte,
+        Encoding::IBM861 =>                     codepoints_1byte,
+        Encoding::IBM862 =>                     codepoints_1byte,
+        Encoding::IBM863 =>                     codepoints_1byte,
+        Encoding::IBM864 =>                     codepoints_1byte,
+        Encoding::IBM865 =>                     codepoints_1byte,
+        Encoding::IBM866 =>                     codepoints_1byte,
+        Encoding::IBM869 =>                     codepoints_1byte,
+        Encoding::ISO_8859_1 =>                 codepoints_1byte,
+        Encoding::ISO_8859_2 =>                 codepoints_1byte,
+        Encoding::ISO_8859_3 =>                 codepoints_1byte,
+        Encoding::ISO_8859_4 =>                 codepoints_1byte,
+        Encoding::ISO_8859_5 =>                 codepoints_1byte,
+        Encoding::ISO_8859_6 =>                 codepoints_1byte,
+        Encoding::ISO_8859_7 =>                 codepoints_1byte,
+        Encoding::ISO_8859_8 =>                 codepoints_1byte,
+        Encoding::ISO_8859_9 =>                 codepoints_1byte,
+        Encoding::ISO_8859_10 =>                codepoints_1byte,
+        Encoding::ISO_8859_11 =>                codepoints_1byte,
+        Encoding::ISO_8859_13 =>                codepoints_1byte,
+        Encoding::ISO_8859_14 =>                codepoints_1byte,
+        Encoding::ISO_8859_15 =>                codepoints_1byte,
+        Encoding::ISO_8859_16 =>                codepoints_1byte,
+        Encoding::KOI8_R =>                     codepoints_1byte,
+        Encoding::KOI8_U =>                     codepoints_1byte,
+        Encoding::MACCENTEURO =>                codepoints_1byte,
+        Encoding::MACCROATIAN =>                codepoints_1byte,
+        Encoding::MACCYRILLIC =>                codepoints_1byte,
+        Encoding::MACGREEK =>                   codepoints_1byte,
+        Encoding::MACICELAND =>                 codepoints_1byte,
+        Encoding::MACROMAN =>                   codepoints_1byte,
+        Encoding::MACROMANIA =>                 codepoints_1byte,
+        Encoding::MACTHAI =>                    codepoints_1byte,
+        Encoding::MACTURKISH =>                 codepoints_1byte,
+        Encoding::MACUKRAINE =>                 codepoints_1byte,
+        Encoding::TIS_620 =>                    codepoints_1byte,
+        Encoding::Windows_1250 =>               codepoints_1byte,
+        Encoding::Windows_1251 =>               codepoints_1byte,
+        Encoding::Windows_1252 =>               codepoints_1byte,
+        Encoding::Windows_1254 =>               codepoints_1byte,
+        Encoding::Windows_1255 =>               codepoints_1byte,
+        Encoding::Windows_1256 =>               codepoints_1byte,
+        Encoding::Windows_1257 =>               codepoints_1byte,
+        Encoding::Windows_1258 =>               codepoints_1byte,
+        Encoding::Windows_874 =>                codepoints_1byte,
+        Encoding::Big5 =>                       codepoints_2bytes,
+        Encoding::Big5_HKSCS =>                 codepoints_2bytes,
+        Encoding::Big5_UAO =>                   codepoints_2bytes,
+        Encoding::CP949 =>                      codepoints_2bytes,
+        Encoding::CP950 =>                      codepoints_2bytes,
+        Encoding::CP951 =>                      codepoints_2bytes,
+        Encoding::EUC_KR =>                     codepoints_2bytes,
+        Encoding::GBK =>                        codepoints_2bytes,
+        Encoding::GB12345 =>                    codepoints_2bytes,
+        Encoding::GB2312 =>                     codepoints_2bytes,
+        Encoding::MACJAPANESE =>                codepoints_2bytes,
+        Encoding::Shift_JIS =>                  codepoints_2bytes,
+        Encoding::SJIS_DoCoMo =>                codepoints_2bytes,
+        Encoding::SJIS_KDDI =>                  codepoints_2bytes,
+        Encoding::SJIS_SoftBank =>              codepoints_2bytes,
+        Encoding::Windows_31J =>                codepoints_2bytes,
         Encoding::UTF_8 =>                      codepoints_unicode,
         Encoding::UTF8_MAC =>                   codepoints_unicode,
         Encoding::UTF8_DoCoMo =>                codepoints_unicode,
@@ -136,6 +134,8 @@ module Prism
       )
     end
 
+    # These test that we're correctly parsing codepoints for each alias of each
+    # encoding that prism supports.
     encodings.each do |encoding, range|
       encoding.names.each do |name|
         next if name == "locale"
@@ -146,6 +146,17 @@ module Prism
       end
     end
 
+    # These test that we're correctly setting the flags on strings for each
+    # encoding that prism supports.
+    escapes = ["\\x00", "\\x7F", "\\x80", "\\xFF", "\\u{00}", "\\u{7F}", "\\u{80}", "\\M-\\C-?"]
+    escapes = escapes.concat(escapes.product(escapes).map(&:join))
+
+    encodings.each_key do |encoding|
+      define_method(:"test_encoding_flags_#{encoding.name}") do
+        assert_encoding_flags(encoding, escapes)
+      end
+    end
+
     def test_coding
       result = Prism.parse("# coding: utf-8\n'string'")
       actual = result.value.statements.body.first.unescaped.encoding
@@ -292,5 +303,47 @@ module Prism
         refute Prism.parse(source).success?
       end
     end
+
+    def assert_encoding_flags(encoding, escapes)
+      escapes.each do |escaped|
+        source = "# encoding: #{encoding.name}\n\"#{escaped}\""
+
+        expected =
+          begin
+            eval(source).encoding
+          rescue SyntaxError => error
+            if error.message.include?("UTF-8 mixed within")
+              error.message[/: (.+?)\n/, 1]
+            else
+              raise
+            end
+          end
+
+        actual =
+          Prism.parse(source).then do |result|
+            if result.success?
+              string = result.value.statements.body.first
+
+              if string.forced_utf8_encoding?
+                Encoding::UTF_8
+              elsif string.forced_binary_encoding?
+                Encoding::ASCII_8BIT
+              else
+                encoding
+              end
+            else
+              error = result.errors.first
+
+              if error.message.include?("mixed")
+                error.message
+              else
+                raise error.message
+              end
+            end
+          end
+
+        assert_equal expected, actual
+      end
+    end
   end
 end
diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb
index 54b710b146..58bb2e3218 100644
--- a/test/prism/errors_test.rb
+++ b/test/prism/errors_test.rb
@@ -659,7 +659,7 @@ module Prism
     end
 
     def test_do_not_allow_multiple_codepoints_in_a_single_character_literal
-      expected = StringNode(0, Location(), Location(), nil, "\u0001\u0002")
+      expected = StringNode(StringFlags::FORCED_UTF8_ENCODING, Location(), Location(), nil, "\u0001\u0002")
 
       assert_errors expected, '?\u{0001 0002}', [
         ["invalid Unicode escape sequence; multiple codepoints are not allowed in a character literal", 9..12]
diff --git a/test/prism/snapshots/arrays.txt b/test/prism/snapshots/arrays.txt
index a9adea9627..c3d4ba1e6c 100644
--- a/test/prism/snapshots/arrays.txt
+++ b/test/prism/snapshots/arrays.txt
@@ -796,6 +796,7 @@
         │   ├── closing_loc: (64,16)-(64,17) = "#"
         │   └── flags: ∅
         ├── @ XStringNode (location: (66,0)-(66,17))
+        │   ├── flags: ∅
         │   ├── opening_loc: (66,0)-(66,3) = "%x#"
         │   ├── content_loc: (66,3)-(66,16) = "one two three"
         │   ├── closing_loc: (66,16)-(66,17) = "#"
@@ -844,6 +845,7 @@
         │   ├── closing_loc: (71,16)-(71,17) = "@"
         │   └── flags: ∅
         ├── @ XStringNode (location: (73,0)-(73,17))
+        │   ├── flags: ∅
         │   ├── opening_loc: (73,0)-(73,3) = "%x@"
         │   ├── content_loc: (73,3)-(73,16) = "one two three"
         │   ├── closing_loc: (73,16)-(73,17) = "@"
@@ -892,6 +894,7 @@
         │   ├── closing_loc: (78,16)-(78,17) = "}"
         │   └── flags: ∅
         ├── @ XStringNode (location: (80,0)-(80,17))
+        │   ├── flags: ∅
         │   ├── opening_loc: (80,0)-(80,3) = "%x{"
         │   ├── content_loc: (80,3)-(80,16) = "one two three"
         │   ├── closing_loc: (80,16)-(80,17) = "}"
diff --git a/test/prism/snapshots/patterns.txt b/test/prism/snapshots/patterns.txt
index ee18448206..05f558d7cb 100644
--- a/test/prism/snapshots/patterns.txt
+++ b/test/prism/snapshots/patterns.txt
@@ -178,6 +178,7 @@
         │   │   └── flags: variable_call
         │   ├── pattern:
         │   │   @ XStringNode (location: (10,7)-(10,12))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (10,7)-(10,8) = "`"
         │   │   ├── content_loc: (10,8)-(10,11) = "foo"
         │   │   ├── closing_loc: (10,11)-(10,12) = "`"
@@ -197,6 +198,7 @@
         │   │   └── flags: variable_call
         │   ├── pattern:
         │   │   @ XStringNode (location: (11,7)-(11,14))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (11,7)-(11,10) = "%x["
         │   │   ├── content_loc: (11,10)-(11,13) = "foo"
         │   │   ├── closing_loc: (11,13)-(11,14) = "]"
@@ -725,12 +727,14 @@
         │   │   @ RangeNode (location: (36,7)-(36,21))
         │   │   ├── left:
         │   │   │   @ XStringNode (location: (36,7)-(36,12))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: (36,7)-(36,8) = "`"
         │   │   │   ├── content_loc: (36,8)-(36,11) = "foo"
         │   │   │   ├── closing_loc: (36,11)-(36,12) = "`"
         │   │   │   └── unescaped: "foo"
         │   │   ├── right:
         │   │   │   @ XStringNode (location: (36,16)-(36,21))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: (36,16)-(36,17) = "`"
         │   │   │   ├── content_loc: (36,17)-(36,20) = "foo"
         │   │   │   ├── closing_loc: (36,20)-(36,21) = "`"
@@ -754,12 +758,14 @@
         │   │   @ RangeNode (location: (37,7)-(37,25))
         │   │   ├── left:
         │   │   │   @ XStringNode (location: (37,7)-(37,14))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: (37,7)-(37,10) = "%x["
         │   │   │   ├── content_loc: (37,10)-(37,13) = "foo"
         │   │   │   ├── closing_loc: (37,13)-(37,14) = "]"
         │   │   │   └── unescaped: "foo"
         │   │   ├── right:
         │   │   │   @ XStringNode (location: (37,18)-(37,25))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: (37,18)-(37,21) = "%x["
         │   │   │   ├── content_loc: (37,21)-(37,24) = "foo"
         │   │   │   ├── closing_loc: (37,24)-(37,25) = "]"
@@ -2483,6 +2489,7 @@
         │   │   └── flags: variable_call
         │   ├── pattern:
         │   │   @ XStringNode (location: (109,7)-(109,12))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (109,7)-(109,8) = "`"
         │   │   ├── content_loc: (109,8)-(109,11) = "foo"
         │   │   ├── closing_loc: (109,11)-(109,12) = "`"
@@ -2502,6 +2509,7 @@
         │   │   └── flags: variable_call
         │   ├── pattern:
         │   │   @ XStringNode (location: (110,7)-(110,14))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (110,7)-(110,10) = "%x["
         │   │   ├── content_loc: (110,10)-(110,13) = "foo"
         │   │   ├── closing_loc: (110,13)-(110,14) = "]"
@@ -3038,6 +3046,7 @@
         │   │   └── @ InNode (location: (136,10)-(136,23))
         │   │       ├── pattern:
         │   │       │   @ XStringNode (location: (136,13)-(136,18))
+        │   │       │   ├── flags: ∅
         │   │       │   ├── opening_loc: (136,13)-(136,14) = "`"
         │   │       │   ├── content_loc: (136,14)-(136,17) = "foo"
         │   │       │   ├── closing_loc: (136,17)-(136,18) = "`"
@@ -3064,6 +3073,7 @@
         │   │   └── @ InNode (location: (137,10)-(137,25))
         │   │       ├── pattern:
         │   │       │   @ XStringNode (location: (137,13)-(137,20))
+        │   │       │   ├── flags: ∅
         │   │       │   ├── opening_loc: (137,13)-(137,16) = "%x["
         │   │       │   ├── content_loc: (137,16)-(137,19) = "foo"
         │   │       │   ├── closing_loc: (137,19)-(137,20) = "]"
@@ -3828,6 +3838,7 @@
         │   │       │   │   @ StatementsNode (location: (163,13)-(163,18))
         │   │       │   │   └── body: (length: 1)
         │   │       │   │       └── @ XStringNode (location: (163,13)-(163,18))
+        │   │       │   │           ├── flags: ∅
         │   │       │   │           ├── opening_loc: (163,13)-(163,14) = "`"
         │   │       │   │           ├── content_loc: (163,14)-(163,17) = "foo"
         │   │       │   │           ├── closing_loc: (163,17)-(163,18) = "`"
@@ -3866,6 +3877,7 @@
         │   │       │   │   @ StatementsNode (location: (164,13)-(164,20))
         │   │       │   │   └── body: (length: 1)
         │   │       │   │       └── @ XStringNode (location: (164,13)-(164,20))
+        │   │       │   │           ├── flags: ∅
         │   │       │   │           ├── opening_loc: (164,13)-(164,16) = "%x["
         │   │       │   │           ├── content_loc: (164,16)-(164,19) = "foo"
         │   │       │   │           ├── closing_loc: (164,19)-(164,20) = "]"
diff --git a/test/prism/snapshots/seattlerb/case_in.txt b/test/prism/snapshots/seattlerb/case_in.txt
index 9134e2cb52..e66b4597b2 100644
--- a/test/prism/snapshots/seattlerb/case_in.txt
+++ b/test/prism/snapshots/seattlerb/case_in.txt
@@ -806,6 +806,7 @@
         │   │   └── @ InNode (location: (98,0)-(98,12))
         │   │       ├── pattern:
         │   │       │   @ XStringNode (location: (98,3)-(98,12))
+        │   │       │   ├── flags: ∅
         │   │       │   ├── opening_loc: (98,3)-(98,4) = "`"
         │   │       │   ├── content_loc: (98,4)-(98,11) = "echo hi"
         │   │       │   ├── closing_loc: (98,11)-(98,12) = "`"
diff --git a/test/prism/snapshots/seattlerb/heredoc_bad_hex_escape.txt b/test/prism/snapshots/seattlerb/heredoc_bad_hex_escape.txt
index e97c1fd7f4..2b1d776404 100644
--- a/test/prism/snapshots/seattlerb/heredoc_bad_hex_escape.txt
+++ b/test/prism/snapshots/seattlerb/heredoc_bad_hex_escape.txt
@@ -9,7 +9,7 @@
             ├── name_loc: (1,0)-(1,1) = "s"
             ├── value:
             │   @ StringNode (location: (1,4)-(1,9))
-            │   ├── flags: ∅
+            │   ├── flags: forced_utf8_encoding
             │   ├── opening_loc: (1,4)-(1,9) = "<<eos"
             │   ├── content_loc: (2,0)-(3,0) = "a\\xE9b\n"
             │   ├── closing_loc: (3,0)-(4,0) = "eos\n"
diff --git a/test/prism/snapshots/seattlerb/heredoc_bad_oct_escape.txt b/test/prism/snapshots/seattlerb/heredoc_bad_oct_escape.txt
index 223a32e298..7a01f8d6d1 100644
--- a/test/prism/snapshots/seattlerb/heredoc_bad_oct_escape.txt
+++ b/test/prism/snapshots/seattlerb/heredoc_bad_oct_escape.txt
@@ -9,7 +9,7 @@
             ├── name_loc: (1,0)-(1,1) = "s"
             ├── value:
             │   @ StringNode (location: (1,4)-(1,10))
-            │   ├── flags: ∅
+            │   ├── flags: forced_utf8_encoding
             │   ├── opening_loc: (1,4)-(1,10) = "<<-EOS"
             │   ├── content_loc: (2,0)-(4,0) = "a\\247b\ncöd\n"
             │   ├── closing_loc: (4,0)-(5,0) = "EOS\n"
diff --git a/test/prism/snapshots/seattlerb/read_escape_unicode_curlies.txt b/test/prism/snapshots/seattlerb/read_escape_unicode_curlies.txt
index 4c4b6b9528..3ea5604b69 100644
--- a/test/prism/snapshots/seattlerb/read_escape_unicode_curlies.txt
+++ b/test/prism/snapshots/seattlerb/read_escape_unicode_curlies.txt
@@ -4,7 +4,7 @@
     @ StatementsNode (location: (1,0)-(1,9))
     └── body: (length: 1)
         └── @ StringNode (location: (1,0)-(1,9))
-            ├── flags: ∅
+            ├── flags: forced_utf8_encoding
             ├── opening_loc: (1,0)-(1,1) = "?"
             ├── content_loc: (1,1)-(1,9) = "\\u{00a0}"
             ├── closing_loc: ∅
diff --git a/test/prism/snapshots/seattlerb/read_escape_unicode_h4.txt b/test/prism/snapshots/seattlerb/read_escape_unicode_h4.txt
index 4bdf2cd2f9..1eba1396fd 100644
--- a/test/prism/snapshots/seattlerb/read_escape_unicode_h4.txt
+++ b/test/prism/snapshots/seattlerb/read_escape_unicode_h4.txt
@@ -4,7 +4,7 @@
     @ StatementsNode (location: (1,0)-(1,7))
     └── body: (length: 1)
         └── @ StringNode (location: (1,0)-(1,7))
-            ├── flags: ∅
+            ├── flags: forced_utf8_encoding
             ├── opening_loc: (1,0)-(1,1) = "?"
             ├── content_loc: (1,1)-(1,7) = "\\u00a0"
             ├── closing_loc: ∅
diff --git a/test/prism/snapshots/seattlerb/str_evstr_escape.txt b/test/prism/snapshots/seattlerb/str_evstr_escape.txt
index 49891a9e96..a71ab57cf5 100644
--- a/test/prism/snapshots/seattlerb/str_evstr_escape.txt
+++ b/test/prism/snapshots/seattlerb/str_evstr_escape.txt
@@ -29,7 +29,7 @@
             │   │   │           └── flags: variable_call
             │   │   └── closing_loc: (1,6)-(1,7) = "}"
             │   └── @ StringNode (location: (1,7)-(1,15))
-            │       ├── flags: ∅
+            │       ├── flags: forced_utf8_encoding
             │       ├── opening_loc: ∅
             │       ├── content_loc: (1,7)-(1,15) = "\\302\\275"
             │       ├── closing_loc: ∅
diff --git a/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt b/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt
index f1226f34e2..b841407cd8 100644
--- a/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt
+++ b/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt
@@ -7,13 +7,13 @@
             ├── opening_loc: ∅
             ├── parts: (length: 2)
             │   ├── @ StringNode (location: (1,0)-(1,62))
-            │   │   ├── flags: ∅
+            │   │   ├── flags: forced_utf8_encoding
             │   │   ├── opening_loc: (1,0)-(1,1) = "\""
             │   │   ├── content_loc: (1,1)-(1,61) = "\\xE3\\xD3\\x8B\\xE3\\x83\\xBC\\x83\\xE3\\x83\\xE3\\x82\\xB3\\xA3\\x82\\x99"
             │   │   ├── closing_loc: (1,61)-(1,62) = "\""
             │   │   └── unescaped: "\xE3Ӌー\x83\xE3\x83コ\xA3\x82\x99"
             │   └── @ StringNode (location: (2,8)-(2,66))
-            │       ├── flags: ∅
+            │       ├── flags: forced_utf8_encoding
             │       ├── opening_loc: (2,8)-(2,9) = "\""
             │       ├── content_loc: (2,9)-(2,65) = "\\xE3\\x83\\xB3\\xE3\\x83\\x8F\\xE3\\x82\\x9A\\xC3\\xBD;foo@bar.com"
             │       ├── closing_loc: (2,65)-(2,66) = "\""
diff --git a/test/prism/snapshots/unparser/corpus/literal/literal.txt b/test/prism/snapshots/unparser/corpus/literal/literal.txt
index e019b65229..eb7e734787 100644
--- a/test/prism/snapshots/unparser/corpus/literal/literal.txt
+++ b/test/prism/snapshots/unparser/corpus/literal/literal.txt
@@ -472,6 +472,7 @@
         │   ├── closing_loc: (37,9)-(37,10) = "\""
         │   └── unescaped: "foo\nbar"
         ├── @ XStringNode (location: (38,0)-(38,5))
+        │   ├── flags: ∅
         │   ├── opening_loc: (38,0)-(38,1) = "`"
         │   ├── content_loc: (38,1)-(38,4) = "foo"
         │   ├── closing_loc: (38,4)-(38,5) = "`"
@@ -495,16 +496,19 @@
         │   │       └── closing_loc: (39,10)-(39,11) = "}"
         │   └── closing_loc: (39,11)-(39,12) = "`"
         ├── @ XStringNode (location: (40,0)-(40,3))
+        │   ├── flags: ∅
         │   ├── opening_loc: (40,0)-(40,1) = "`"
         │   ├── content_loc: (40,1)-(40,2) = ")"
         │   ├── closing_loc: (40,2)-(40,3) = "`"
         │   └── unescaped: ")"
         ├── @ XStringNode (location: (41,0)-(41,4))
+        │   ├── flags: ∅
         │   ├── opening_loc: (41,0)-(41,1) = "`"
         │   ├── content_loc: (41,1)-(41,3) = "\\`"
         │   ├── closing_loc: (41,3)-(41,4) = "`"
         │   └── unescaped: "`"
         ├── @ XStringNode (location: (42,0)-(42,3))
+        │   ├── flags: ∅
         │   ├── opening_loc: (42,0)-(42,1) = "`"
         │   ├── content_loc: (42,1)-(42,2) = "\""
         │   ├── closing_loc: (42,2)-(42,3) = "`"
diff --git a/test/prism/snapshots/whitequark/bug_ascii_8bit_in_literal.txt b/test/prism/snapshots/whitequark/bug_ascii_8bit_in_literal.txt
index 65d733166a..7aa8694f66 100644
--- a/test/prism/snapshots/whitequark/bug_ascii_8bit_in_literal.txt
+++ b/test/prism/snapshots/whitequark/bug_ascii_8bit_in_literal.txt
@@ -4,7 +4,7 @@
     @ StatementsNode (location: (2,9)-(2,75))
     └── body: (length: 1)
         └── @ StringNode (location: (2,9)-(2,75))
-            ├── flags: ∅
+            ├── flags: forced_utf8_encoding
             ├── opening_loc: (2,9)-(2,10) = "\""
             ├── content_loc: (2,10)-(2,74) = "\\xD0\\xBF\\xD1\\x80\\xD0\\xBE\\xD0\\xB2\\xD0\\xB5\\xD1\\x80\\xD0\\xBA\\xD0\\xB0"
             ├── closing_loc: (2,74)-(2,75) = "\""
diff --git a/test/prism/snapshots/whitequark/heredoc.txt b/test/prism/snapshots/whitequark/heredoc.txt
index 0d718c6945..86543097ee 100644
--- a/test/prism/snapshots/whitequark/heredoc.txt
+++ b/test/prism/snapshots/whitequark/heredoc.txt
@@ -16,6 +16,7 @@
         │   ├── closing_loc: (9,0)-(10,0) = "HERE\n"
         │   └── unescaped: "foo\nbar\n"
         └── @ XStringNode (location: (11,0)-(11,8))
+            ├── flags: ∅
             ├── opening_loc: (11,0)-(11,8) = "<<`HERE`"
             ├── content_loc: (12,0)-(14,0) = "foo\nbar\n"
             ├── closing_loc: (14,0)-(15,0) = "HERE\n"
diff --git a/test/prism/snapshots/whitequark/interp_digit_var.txt b/test/prism/snapshots/whitequark/interp_digit_var.txt
index 5796357fd3..d493027e44 100644
--- a/test/prism/snapshots/whitequark/interp_digit_var.txt
+++ b/test/prism/snapshots/whitequark/interp_digit_var.txt
@@ -146,11 +146,13 @@
         │   ├── closing_loc: (35,10)-(35,11) = "]"
         │   └── flags: ∅
         ├── @ XStringNode (location: (37,1)-(37,8))
+        │   ├── flags: ∅
         │   ├── opening_loc: (37,1)-(37,4) = "%x{"
         │   ├── content_loc: (37,4)-(37,7) = "\#@1"
         │   ├── closing_loc: (37,7)-(37,8) = "}"
         │   └── unescaped: "\#@1"
         ├── @ XStringNode (location: (39,1)-(39,9))
+        │   ├── flags: ∅
         │   ├── opening_loc: (39,1)-(39,4) = "%x{"
         │   ├── content_loc: (39,4)-(39,8) = "\#@@1"
         │   ├── closing_loc: (39,8)-(39,9) = "}"
@@ -212,11 +214,13 @@
         │   ├── closing_loc: (59,7)-(59,8) = "'"
         │   └── unescaped: "\#@@1"
         ├── @ XStringNode (location: (61,1)-(61,6))
+        │   ├── flags: ∅
         │   ├── opening_loc: (61,1)-(61,2) = "`"
         │   ├── content_loc: (61,2)-(61,5) = "\#@1"
         │   ├── closing_loc: (61,5)-(61,6) = "`"
         │   └── unescaped: "\#@1"
         ├── @ XStringNode (location: (63,1)-(63,7))
+        │   ├── flags: ∅
         │   ├── opening_loc: (63,1)-(63,2) = "`"
         │   ├── content_loc: (63,2)-(63,6) = "\#@@1"
         │   ├── closing_loc: (63,6)-(63,7) = "`"
@@ -246,11 +250,13 @@
         │   ├── closing_loc: (79,0)-(80,0) = "HERE\n"
         │   └── unescaped: "\#@@1\n"
         ├── @ XStringNode (location: (81,0)-(81,9))
+        │   ├── flags: ∅
         │   ├── opening_loc: (81,0)-(81,9) = "<<-`HERE`"
         │   ├── content_loc: (82,0)-(83,0) = "\#@1\n"
         │   ├── closing_loc: (83,0)-(84,0) = "HERE\n"
         │   └── unescaped: "\#@1\n"
         └── @ XStringNode (location: (85,0)-(85,9))
+            ├── flags: ∅
             ├── opening_loc: (85,0)-(85,9) = "<<-`HERE`"
             ├── content_loc: (86,0)-(87,0) = "\#@@1\n"
             ├── closing_loc: (87,0)-(88,0) = "HERE\n"
diff --git a/test/prism/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt b/test/prism/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt
index e86098e7ba..5b4b9863db 100644
--- a/test/prism/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt
+++ b/test/prism/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt
@@ -75,6 +75,7 @@
         │   ├── closing_loc: (26,1)-(26,2) = "}"
         │   └── flags: ∅
         ├── @ XStringNode (location: (28,0)-(29,2))
+        │   ├── flags: ∅
         │   ├── opening_loc: (28,0)-(28,3) = "%x{"
         │   ├── content_loc: (28,3)-(29,1) = "a\\\nb"
         │   ├── closing_loc: (29,1)-(29,2) = "}"
@@ -120,11 +121,13 @@
         │   ├── closing_loc: (54,0)-(55,0) = "HERE\n"
         │   └── unescaped: "a\\\nb\n"
         ├── @ XStringNode (location: (56,0)-(56,9))
+        │   ├── flags: ∅
         │   ├── opening_loc: (56,0)-(56,9) = "<<-`HERE`"
         │   ├── content_loc: (57,0)-(59,0) = "a\\\nb\n"
         │   ├── closing_loc: (59,0)-(60,0) = "HERE\n"
         │   └── unescaped: "ab\n"
         └── @ XStringNode (location: (61,0)-(62,2))
+            ├── flags: ∅
             ├── opening_loc: (61,0)-(61,1) = "`"
             ├── content_loc: (61,1)-(62,1) = "a\\\nb"
             ├── closing_loc: (62,1)-(62,2) = "`"
diff --git a/test/prism/snapshots/whitequark/xstring_plain.txt b/test/prism/snapshots/whitequark/xstring_plain.txt
index 2546f9829f..97084286d9 100644
--- a/test/prism/snapshots/whitequark/xstring_plain.txt
+++ b/test/prism/snapshots/whitequark/xstring_plain.txt
@@ -4,6 +4,7 @@
     @ StatementsNode (location: (1,0)-(1,8))
     └── body: (length: 1)
         └── @ XStringNode (location: (1,0)-(1,8))
+            ├── flags: ∅
             ├── opening_loc: (1,0)-(1,1) = "`"
             ├── content_loc: (1,1)-(1,7) = "foobar"
             ├── closing_loc: (1,7)-(1,8) = "`"
diff --git a/test/prism/snapshots/xstring.txt b/test/prism/snapshots/xstring.txt
index 6cfa9a350e..c8dba2a652 100644
--- a/test/prism/snapshots/xstring.txt
+++ b/test/prism/snapshots/xstring.txt
@@ -4,6 +4,7 @@
     @ StatementsNode (location: (1,0)-(7,5))
     └── body: (length: 4)
         ├── @ XStringNode (location: (1,0)-(1,7))
+        │   ├── flags: ∅
         │   ├── opening_loc: (1,0)-(1,3) = "%x["
         │   ├── content_loc: (1,3)-(1,6) = "foo"
         │   ├── closing_loc: (1,6)-(1,7) = "]"
@@ -41,11 +42,13 @@
         │   │       └── unescaped: " baz"
         │   └── closing_loc: (3,15)-(3,16) = "`"
         ├── @ XStringNode (location: (5,0)-(5,6))
+        │   ├── flags: ∅
         │   ├── opening_loc: (5,0)-(5,1) = "`"
         │   ├── content_loc: (5,1)-(5,5) = "f\\oo"
         │   ├── closing_loc: (5,5)-(5,6) = "`"
         │   └── unescaped: "foo"
         └── @ XStringNode (location: (7,0)-(7,5))
+            ├── flags: ∅
             ├── opening_loc: (7,0)-(7,1) = "`"
             ├── content_loc: (7,1)-(7,4) = "foo"
             ├── closing_loc: (7,4)-(7,5) = "`"