diff options
author | Koichi ITO <koic.ito@gmail.com> | 2024-03-13 02:05:57 +0900 |
---|---|---|
committer | git <svn-admin@ruby-lang.org> | 2024-03-12 17:33:14 +0000 |
commit | f8cab4ef8e214dc05efe8a0d4178864bce3b790a (patch) | |
tree | 27708629dd6957888f51fe91dc02f016afe51614 | |
parent | 83790e5fe1e254c315552a6927891ed697da590a (diff) | |
download | ruby-f8cab4ef8e214dc05efe8a0d4178864bce3b790a.tar.gz |
[ruby/prism] Fix a token incompatibility for `Prism::Translation::Parser::Lexer`
In practice, the `BACKTICK` is mapped either as `:tXSTRING_BEG` or `:tBACK_REF2`.
The former is used in xstrings like `` `foo` ``, while the latter is utilized as
a back reference in contexts like `` A::` ``.
This PR will make corrections to differentiate the use of `BACKTICK`.
This mistake was discovered through the investigation of xstring.txt file.
The PR will run tests from xstring.txt file except for `` `f\oo` ``, which will still fail,
hence it will be separated into xstring_with_backslash.txt file.
This separation will facilitate addressing the correction at a different time.
https://github.com/ruby/prism/commit/49ad8df40a
-rw-r--r-- | lib/prism/translation/parser/lexer.rb | 6 | ||||
-rw-r--r-- | test/prism/fixtures/xstring.txt | 2 | ||||
-rw-r--r-- | test/prism/fixtures/xstring_with_backslash.txt | 1 | ||||
-rw-r--r-- | test/prism/parser_test.rb | 2 | ||||
-rw-r--r-- | test/prism/snapshots/xstring.txt | 20 | ||||
-rw-r--r-- | test/prism/snapshots/xstring_with_backslash.txt | 11 |
6 files changed, 25 insertions, 17 deletions
diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb index 054e56a4ba..92495ab3d2 100644 --- a/lib/prism/translation/parser/lexer.rb +++ b/lib/prism/translation/parser/lexer.rb @@ -24,7 +24,7 @@ module Prism AMPERSAND_DOT: :tANDDOT, AMPERSAND_EQUAL: :tOP_ASGN, BACK_REFERENCE: :tBACK_REF, - BACKTICK: :tBACK_REF2, + BACKTICK: :tXSTRING_BEG, BANG: :tBANG, BANG_EQUAL: :tNEQ, BANG_TILDE: :tNMATCH, @@ -325,6 +325,10 @@ module Prism if !tokens.empty? && tokens.dig(-1, 0) == :kDEF type = :tIDENTIFIER end + when :tXSTRING_BEG + if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT + type = :tBACK_REF2 + end end tokens << [type, [value, location]] diff --git a/test/prism/fixtures/xstring.txt b/test/prism/fixtures/xstring.txt index 01bcc09fc5..623afd9797 100644 --- a/test/prism/fixtures/xstring.txt +++ b/test/prism/fixtures/xstring.txt @@ -2,6 +2,4 @@ `foo #{bar} baz` -`f\oo` - `foo` diff --git a/test/prism/fixtures/xstring_with_backslash.txt b/test/prism/fixtures/xstring_with_backslash.txt new file mode 100644 index 0000000000..b51bb0f6f9 --- /dev/null +++ b/test/prism/fixtures/xstring_with_backslash.txt @@ -0,0 +1 @@ +`f\oo` diff --git a/test/prism/parser_test.rb b/test/prism/parser_test.rb index 60fe0e0f47..518cb36c42 100644 --- a/test/prism/parser_test.rb +++ b/test/prism/parser_test.rb @@ -77,7 +77,7 @@ module Prism "heredocs_nested.txt", "indented_file_end.txt", "strings.txt", - "xstring.txt" + "xstring_with_backslash.txt" ] Dir["*.txt", base: base].each do |name| diff --git a/test/prism/snapshots/xstring.txt b/test/prism/snapshots/xstring.txt index 325a39afa5..0a9d075818 100644 --- a/test/prism/snapshots/xstring.txt +++ b/test/prism/snapshots/xstring.txt @@ -1,8 +1,8 @@ -@ ProgramNode (location: (1,0)-(7,5)) +@ ProgramNode (location: (1,0)-(5,5)) ├── locals: [] └── statements: - @ StatementsNode (location: (1,0)-(7,5)) - └── body: (length: 4) + @ StatementsNode (location: (1,0)-(5,5)) + └── body: (length: 3) ├── @ XStringNode (location: (1,0)-(1,7)) │ ├── flags: ∅ │ ├── opening_loc: (1,0)-(1,3) = "%x[" @@ -41,15 +41,9 @@ │ │ ├── closing_loc: ∅ │ │ └── unescaped: " baz" │ └── closing_loc: (3,15)-(3,16) = "`" - ├── @ XStringNode (location: (5,0)-(5,6)) - │ ├── flags: ∅ - │ ├── opening_loc: (5,0)-(5,1) = "`" - │ ├── content_loc: (5,1)-(5,5) = "f\\oo" - │ ├── closing_loc: (5,5)-(5,6) = "`" - │ └── unescaped: "foo" - └── @ XStringNode (location: (7,0)-(7,5)) + └── @ XStringNode (location: (5,0)-(5,5)) ├── flags: ∅ - ├── opening_loc: (7,0)-(7,1) = "`" - ├── content_loc: (7,1)-(7,4) = "foo" - ├── closing_loc: (7,4)-(7,5) = "`" + ├── opening_loc: (5,0)-(5,1) = "`" + ├── content_loc: (5,1)-(5,4) = "foo" + ├── closing_loc: (5,4)-(5,5) = "`" └── unescaped: "foo" diff --git a/test/prism/snapshots/xstring_with_backslash.txt b/test/prism/snapshots/xstring_with_backslash.txt new file mode 100644 index 0000000000..7e0fa1ab5f --- /dev/null +++ b/test/prism/snapshots/xstring_with_backslash.txt @@ -0,0 +1,11 @@ +@ ProgramNode (location: (1,0)-(1,6)) +├── locals: [] +└── statements: + @ StatementsNode (location: (1,0)-(1,6)) + └── body: (length: 1) + └── @ XStringNode (location: (1,0)-(1,6)) + ├── flags: ∅ + ├── opening_loc: (1,0)-(1,1) = "`" + ├── content_loc: (1,1)-(1,5) = "f\\oo" + ├── closing_loc: (1,5)-(1,6) = "`" + └── unescaped: "foo" |