From 7df1e45bb6b4326da5c799dcf58d38f0a14362af Mon Sep 17 00:00:00 2001 From: nobu Date: Thu, 14 Sep 2017 10:53:47 +0000 Subject: ripper: add states of scanner * parse.y (ripper_state): add states of scanner to tokens from Ripper.lex and Ripper::Filter#on_*. based on the patch by aycabta (Code Ahss) at [ruby-core:81789]. [Feature #13686] * ext/ripper/tools/preproc.rb (prelude, usercode): generate EXPR_* constants from enums. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59896 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- test/ripper/test_filter.rb | 11 +++++ test/ripper/test_ripper.rb | 4 ++ test/ripper/test_scanner_events.rb | 98 +++++++++++++++++++------------------- 3 files changed, 64 insertions(+), 49 deletions(-) (limited to 'test/ripper') diff --git a/test/ripper/test_filter.rb b/test/ripper/test_filter.rb index d025cf5a98..c39820c321 100644 --- a/test/ripper/test_filter.rb +++ b/test/ripper/test_filter.rb @@ -15,6 +15,7 @@ class TestRipper::Filter < Test::Unit::TestCase data[:filename] = filename rescue nil data[:lineno] = lineno data[:column] = column + data[:state] = state data[:token] = token end data @@ -75,6 +76,16 @@ class TestRipper::Filter < Test::Unit::TestCase assert_equal(last_columns, filter.column) end + def test_filter_state + data = {} + src = File.read(filename) + filter = Filter.new(src) + assert_equal(nil, filter.state) + filter.parse(data) + assert_not_nil(data[:state]) + assert_not_nil(filter.state) + end + def test_filter_token data = {} filter = Filter.new("begin; puts 1; end") diff --git a/test/ripper/test_ripper.rb b/test/ripper/test_ripper.rb index 79cbf88a15..e7d20a66a0 100644 --- a/test/ripper/test_ripper.rb +++ b/test/ripper/test_ripper.rb @@ -17,6 +17,10 @@ class TestRipper::Ripper < Test::Unit::TestCase assert_nil @ripper.column end + def test_state + assert_nil @ripper.state + end + def test_encoding assert_equal Encoding::UTF_8, @ripper.encoding ripper = Ripper.new('# coding: iso-8859-15') diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb index ef49fc18ba..90fd599a31 100644 --- a/test/ripper/test_scanner_events.rb +++ b/test/ripper/test_scanner_events.rb @@ -48,70 +48,70 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase def test_lex assert_equal [], Ripper.lex('') - assert_equal [[[1,0], :on_ident, "a"]], + assert_equal [[[1,0], :on_ident, "a", Ripper::EXPR_CMDARG]], Ripper.lex('a') - assert_equal [[[1, 0], :on_kw, "nil"]], + assert_equal [[[1, 0], :on_kw, "nil", Ripper::EXPR_END]], Ripper.lex("nil") - assert_equal [[[1, 0], :on_kw, "def"], - [[1, 3], :on_sp, " "], - [[1, 4], :on_ident, "m"], - [[1, 5], :on_lparen, "("], - [[1, 6], :on_ident, "a"], - [[1, 7], :on_rparen, ")"], - [[1, 8], :on_kw, "end"]], + assert_equal [[[1, 0], :on_kw, "def", Ripper::EXPR_FNAME], + [[1, 3], :on_sp, " ", Ripper::EXPR_FNAME], + [[1, 4], :on_ident, "m", Ripper::EXPR_ENDFN], + [[1, 5], :on_lparen, "(", Ripper::EXPR_BEG | Ripper::EXPR_LABEL], + [[1, 6], :on_ident, "a", Ripper::EXPR_ARG], + [[1, 7], :on_rparen, ")", Ripper::EXPR_ENDFN], + [[1, 8], :on_kw, "end", Ripper::EXPR_END]], Ripper.lex("def m(a)end") - assert_equal [[[1, 0], :on_int, "1"], - [[1, 1], :on_nl, "\n"], - [[2, 0], :on_int, "2"], - [[2, 1], :on_nl, "\n"], - [[3, 0], :on_int, "3"]], + assert_equal [[[1, 0], :on_int, "1", Ripper::EXPR_END | Ripper::EXPR_ENDARG], + [[1, 1], :on_nl, "\n", Ripper::EXPR_BEG], + [[2, 0], :on_int, "2", Ripper::EXPR_END | Ripper::EXPR_ENDARG], + [[2, 1], :on_nl, "\n", Ripper::EXPR_BEG], + [[3, 0], :on_int, "3", Ripper::EXPR_END | Ripper::EXPR_ENDARG]], Ripper.lex("1\n2\n3") - assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS"], - [[1, 5], :on_nl, "\n"], - [[2, 0], :on_tstring_content, "heredoc\n"], - [[3, 0], :on_heredoc_end, "EOS"]], + assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS", Ripper::EXPR_BEG], + [[1, 5], :on_nl, "\n", Ripper::EXPR_BEG], + [[2, 0], :on_tstring_content, "heredoc\n", Ripper::EXPR_BEG], + [[3, 0], :on_heredoc_end, "EOS", Ripper::EXPR_BEG]], Ripper.lex("<<""EOS\nheredoc\nEOS") - assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS"], - [[1, 5], :on_nl, "\n"], - [[2, 0], :on_heredoc_end, "EOS"]], + assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS", Ripper::EXPR_BEG], + [[1, 5], :on_nl, "\n", Ripper::EXPR_BEG], + [[2, 0], :on_heredoc_end, "EOS", Ripper::EXPR_BEG]], Ripper.lex("<<""EOS\nEOS"), "bug#4543" - assert_equal [[[1, 0], :on_regexp_beg, "/"], - [[1, 1], :on_tstring_content, "foo\nbar"], - [[2, 3], :on_regexp_end, "/"]], + assert_equal [[[1, 0], :on_regexp_beg, "/", Ripper::EXPR_BEG], + [[1, 1], :on_tstring_content, "foo\nbar", Ripper::EXPR_BEG], + [[2, 3], :on_regexp_end, "/", Ripper::EXPR_BEG]], Ripper.lex("/foo\nbar/") - assert_equal [[[1, 0], :on_regexp_beg, "/"], - [[1, 1], :on_tstring_content, "foo\n\u3020"], - [[2, 3], :on_regexp_end, "/"]], + assert_equal [[[1, 0], :on_regexp_beg, "/", Ripper::EXPR_BEG], + [[1, 1], :on_tstring_content, "foo\n\u3020", Ripper::EXPR_BEG], + [[2, 3], :on_regexp_end, "/", Ripper::EXPR_BEG]], Ripper.lex("/foo\n\u3020/") - assert_equal [[[1, 0], :on_tstring_beg, "'"], - [[1, 1], :on_tstring_content, "foo\n\xe3\x80\xa0"], - [[2, 3], :on_tstring_end, "'"]], + assert_equal [[[1, 0], :on_tstring_beg, "'", Ripper::EXPR_BEG], + [[1, 1], :on_tstring_content, "foo\n\xe3\x80\xa0", Ripper::EXPR_BEG], + [[2, 3], :on_tstring_end, "'", Ripper::EXPR_END | Ripper::EXPR_ENDARG]], Ripper.lex("'foo\n\xe3\x80\xa0'") - assert_equal [[[1, 0], :on_tstring_beg, "'"], - [[1, 1], :on_tstring_content, "\u3042\n\u3044"], - [[2, 3], :on_tstring_end, "'"]], + assert_equal [[[1, 0], :on_tstring_beg, "'", Ripper::EXPR_BEG], + [[1, 1], :on_tstring_content, "\u3042\n\u3044", Ripper::EXPR_BEG], + [[2, 3], :on_tstring_end, "'", Ripper::EXPR_END | Ripper::EXPR_ENDARG]], Ripper.lex("'\u3042\n\u3044'") - assert_equal [[[1, 0], :on_rational, "1r"], - [[1, 2], :on_nl, "\n"], - [[2, 0], :on_imaginary, "2i"], - [[2, 2], :on_nl, "\n"], - [[3, 0], :on_imaginary, "3ri"], - [[3, 3], :on_nl, "\n"], - [[4, 0], :on_rational, "4.2r"], - [[4, 4], :on_nl, "\n"], - [[5, 0], :on_imaginary, "5.6ri"], + assert_equal [[[1, 0], :on_rational, "1r", Ripper::EXPR_END | Ripper::EXPR_ENDARG], + [[1, 2], :on_nl, "\n", Ripper::EXPR_BEG], + [[2, 0], :on_imaginary, "2i", Ripper::EXPR_END | Ripper::EXPR_ENDARG], + [[2, 2], :on_nl, "\n", Ripper::EXPR_BEG], + [[3, 0], :on_imaginary, "3ri", Ripper::EXPR_END | Ripper::EXPR_ENDARG], + [[3, 3], :on_nl, "\n", Ripper::EXPR_BEG], + [[4, 0], :on_rational, "4.2r", Ripper::EXPR_END | Ripper::EXPR_ENDARG], + [[4, 4], :on_nl, "\n", Ripper::EXPR_BEG], + [[5, 0], :on_imaginary, "5.6ri", Ripper::EXPR_END | Ripper::EXPR_ENDARG], ], Ripper.lex("1r\n2i\n3ri\n4.2r\n5.6ri") - assert_equal [[[1, 0], :on_heredoc_beg, "<<~EOS"], - [[1, 6], :on_nl, "\n"], - [[2, 0], :on_ignored_sp, " "], - [[2, 2], :on_tstring_content, "heredoc\n"], - [[3, 0], :on_heredoc_end, "EOS"] + assert_equal [[[1, 0], :on_heredoc_beg, "<<~EOS", Ripper::EXPR_BEG], + [[1, 6], :on_nl, "\n", Ripper::EXPR_BEG], + [[2, 0], :on_ignored_sp, " ", Ripper::EXPR_BEG], + [[2, 2], :on_tstring_content, "heredoc\n", Ripper::EXPR_BEG], + [[3, 0], :on_heredoc_end, "EOS", Ripper::EXPR_BEG] ], Ripper.lex("<<~EOS\n heredoc\nEOS") - assert_equal [[[1, 0], :on_tstring_beg, "'"], - [[1, 1], :on_tstring_content, "foo"]], + assert_equal [[[1, 0], :on_tstring_beg, "'", Ripper::EXPR_BEG], + [[1, 1], :on_tstring_content, "foo", Ripper::EXPR_BEG]], Ripper.lex("'foo") end -- cgit v1.2.3