Upgrade to RDoc 3.5.3. Fixes [Bug #4376]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@30815 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

11 files changed, 204 insertions, 34 deletions

diff --git a/ChangeLog b/ChangeLog
index 8d839b0853..fedacf12c3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+Mon Feb  7 16:05:32 2011  Eric Hodel  <drbrain@segment7.net>
+
+	* lib/rdoc: Upgrade to RDoc 3.5.3  Fixes [Bug #4376]
+
 Mon Feb  7 11:46:59 2011  NARUSE, Yui  <naruse@ruby-lang.org>
 
 	* common.mk (rdoc): add --encoding=UTF-8; ruby's rdoc must be UTF-8.
diff --git a/NEWS b/NEWS
index feca91e46f..7d206779f5 100644
--- a/NEWS
+++ b/NEWS
@@ -92,7 +92,7 @@ with all sufficient information, see the ChangeLog file.
   * support for bash/zsh completion.
 
 * RDoc
-  * RDoc has been upgraded to RDoc 3.5.2.  For full release notes see
+  * RDoc has been upgraded to RDoc 3.5.3.  For full release notes see
     http://docs.seattlerb.org/rdoc/History_txt.html
 
 * rexml
diff --git a/lib/rdoc.rb b/lib/rdoc.rb
index 0b521a5d2d..d9948deebb 100644
--- a/lib/rdoc.rb
+++ b/lib/rdoc.rb
@@ -95,7 +95,7 @@ module RDoc
   ##
   # RDoc version you are using
 
-  VERSION = '3.5.2'
+  VERSION = '3.5.3'
 
   ##
   # Method visibilities
diff --git a/lib/rdoc/encoding.rb b/lib/rdoc/encoding.rb
index 8483aabbc6..cfe1ba7725 100644
--- a/lib/rdoc/encoding.rb
+++ b/lib/rdoc/encoding.rb
@@ -12,8 +12,11 @@ module RDoc::Encoding
   #
   # The content will be converted to the +encoding+.  If the file cannot be
   # converted a warning will be printed and nil will be returned.
+  #
+  # If +force_transcode+ is true the document will be transcoded and any
+  # unknown character in the target encoding will be replaced with '?'
 
-  def self.read_file filename, encoding
+  def self.read_file filename, encoding, force_transcode = false
     content = open filename, "rb" do |f| f.read end
 
     utf8 = content.sub!(/\A\xef\xbb\xbf/, '')
@@ -50,8 +53,14 @@ module RDoc::Encoding
     warn "unknown encoding name \"#{$1}\" for #{filename}, skipping"
     nil
   rescue Encoding::UndefinedConversionError => e
-    warn "unable to convert #{e.message} for #{filename}, skipping"
-    nil
+    if force_transcode then
+      content.force_encoding orig_encoding
+      content.encode! encoding, :undef => :replace, :replace => '?'
+      content
+    else
+      warn "unable to convert #{e.message} for #{filename}, skipping"
+      nil
+    end
   rescue Errno::EISDIR, Errno::ENOENT
     nil
   end
diff --git a/lib/rdoc/generator/darkfish.rb b/lib/rdoc/generator/darkfish.rb
index 57873468e4..299e6b7dec 100644
--- a/lib/rdoc/generator/darkfish.rb
+++ b/lib/rdoc/generator/darkfish.rb
@@ -46,7 +46,7 @@ require 'rdoc/generator/markup'
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
+
 class RDoc::Generator::Darkfish
 
   RDoc::RDoc.add_generator self
@@ -162,9 +162,9 @@ class RDoc::Generator::Darkfish
     generate_class_files
     generate_file_files
 
-  rescue StandardError => err
+  rescue => e
     debug_msg "%s: %s\n  %s" % [
-      err.class.name, err.message, err.backtrace.join("\n  ")
+      e.class.name, e.message, e.backtrace.join("\n  ")
     ]
 
     raise
@@ -208,6 +208,12 @@ class RDoc::Generator::Darkfish
     out_file = @basedir + @options.op_dir + 'index.html'
 
     render_template template_file, out_file do |io| binding end
+  rescue => e
+    error = RDoc::Error.new \
+      "error generating index.html: #{e.message} (#{e.class})"
+    error.set_backtrace e.backtrace
+
+    raise error
   end
 
   ##
@@ -216,9 +222,12 @@ class RDoc::Generator::Darkfish
   def generate_class_files
     template_file = @template_dir + 'classpage.rhtml'
     return unless template_file.exist?
-    debug_msg "Generating class documentation in #@outputdir"
+    debug_msg "Generating class documentation in #{@outputdir}"
+
+    current = nil
 
     @classes.each do |klass|
+      current = klass
       debug_msg "  working on %s (%s)" % [klass.full_name, klass.path]
       out_file   = @outputdir + klass.path
       # suppress 1.9.3 warning
@@ -228,6 +237,12 @@ class RDoc::Generator::Darkfish
       debug_msg "  rendering #{out_file}"
       render_template template_file, out_file do |io| binding end
     end
+  rescue => e
+    error = RDoc::Error.new \
+      "error generating #{current.path}: #{e.message} (#{e.class})"
+    error.set_backtrace e.backtrace
+
+    raise error
   end
 
   ##
@@ -236,17 +251,25 @@ class RDoc::Generator::Darkfish
   def generate_file_files
     template_file = @template_dir + 'filepage.rhtml'
     return unless template_file.exist?
-    debug_msg "Generating file documentation in #@outputdir"
+    debug_msg "Generating file documentation in #{@outputdir}"
+
+    out_file = nil
 
     @files.each do |file|
       out_file     = @outputdir + file.path
-      debug_msg "  working on %s (%s)" % [ file.full_name, out_file ]
+      debug_msg "  working on %s (%s)" % [file.full_name, out_file]
       # suppress 1.9.3 warning
       rel_prefix = rel_prefix  = @outputdir.relative_path_from(out_file.dirname)
 
       debug_msg "  rendering #{out_file}"
       render_template template_file, out_file do |io| binding end
     end
+  rescue => e
+    error =
+      RDoc::Error.new "error generating #{out_file}: #{e.message} (#{e.class})"
+    error.set_backtrace e.backtrace
+
+    raise error
   end
 
   ##
diff --git a/lib/rdoc/markup/pre_process.rb b/lib/rdoc/markup/pre_process.rb
index e59bd227b7..ccc2688050 100644
--- a/lib/rdoc/markup/pre_process.rb
+++ b/lib/rdoc/markup/pre_process.rb
@@ -120,7 +120,7 @@ class RDoc::Markup::PreProcess
       return ''
     end
 
-    content = RDoc::Encoding.read_file full_name, encoding
+    content = RDoc::Encoding.read_file full_name, encoding, true
 
     # strip magic comment
     content = content.sub(/\A# .*coding[=:].*$/, '').lstrip
diff --git a/lib/rdoc/text.rb b/lib/rdoc/text.rb
index 501871c8df..adbc630cf1 100644
--- a/lib/rdoc/text.rb
+++ b/lib/rdoc/text.rb
@@ -46,7 +46,9 @@ module RDoc::Text
 
     text.each_line do |line|
       line.gsub!(/^(.{8}*?)([^\t\r\n]{0,7})\t/) do
-        "#{$1}#{$2}#{' ' * (8 - $2.size)}"
+        r = "#{$1}#{$2}#{' ' * (8 - $2.size)}"
+        r.force_encoding text.encoding if Object.const_defined? :Encoding
+        r
       end until line !~ /\t/
 
       expanded << line
@@ -69,8 +71,11 @@ module RDoc::Text
 
     flush = []
 
+    empty = ''
+    empty.force_encoding text.encoding if Object.const_defined? :Encoding
+
     text.each_line do |line|
-      line[/^ {0,#{indent}}/] = ''
+      line[/^ {0,#{indent}}/] = empty
       flush << line
     end
 
@@ -158,11 +163,20 @@ http://rubyforge.org/tracker/?atid=2472&group_id=627&func=browse
   # Strips /* */ style comments
 
   def strip_stars text
+    encoding = text.encoding if Object.const_defined? :Encoding
+
     text = text.gsub %r%Document-method:\s+[\w:.#]+%, ''
-    text.sub!  %r%/\*+%       do " " * $&.length end
-    text.sub!  %r%\*+/%       do " " * $&.length end
-    text.gsub! %r%^[ \t]*\*%m do " " * $&.length end
-    text.gsub(/^\s+$/, '')
+
+    space = ' '
+    space.force_encoding encoding if encoding
+
+    text.sub!  %r%/\*+%       do space * $&.length end
+    text.sub!  %r%\*+/%       do space * $&.length end
+    text.gsub! %r%^[ \t]*\*%m do space * $&.length end
+
+    empty = ''
+    empty.force_encoding encoding if encoding
+    text.gsub(/^\s+$/, empty)
   end
 
   ##
diff --git a/test/rdoc/test_rdoc_encoding.rb b/test/rdoc/test_rdoc_encoding.rb
index de6c10f678..f6c113affc 100644
--- a/test/rdoc/test_rdoc_encoding.rb
+++ b/test/rdoc/test_rdoc_encoding.rb
@@ -49,6 +49,26 @@ class TestRDocEncoding < MiniTest::Unit::TestCase
     assert_equal "hi \u00e9verybody", contents.sub("\r", '')
   end
 
+  def test_class_read_file_encoding_fail
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
+
+    @tempfile.write "# coding: utf-8\n\317\200" # pi
+    @tempfile.flush
+
+    # FIXME 1.9 fix on windoze
+    expected.gsub!("\n", "\r\n") if RUBY_VERSION =~ /^1.9/ && RUBY_PLATFORM =~ /mswin|mingw/
+
+    contents = :junk
+
+    _, err = capture_io do
+      contents = RDoc::Encoding.read_file @tempfile.path, Encoding::US_ASCII
+    end
+
+    assert_nil contents
+
+    assert_match %r%^unable to convert%, err
+  end
+
   def test_class_read_file_encoding_fancy
     skip "Encoding not implemented" unless Object.const_defined? :Encoding
 
@@ -66,6 +86,21 @@ class TestRDocEncoding < MiniTest::Unit::TestCase
     assert_equal Encoding::UTF_8, contents.encoding
   end
 
+  def test_class_read_file_encoding_force_transcode
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
+
+    @tempfile.write "# coding: utf-8\n\317\200" # pi
+    @tempfile.flush
+
+    # FIXME 1.9 fix on windoze
+    expected.gsub!("\n", "\r\n") if RUBY_VERSION =~ /^1.9/ && RUBY_PLATFORM =~ /mswin|mingw/
+
+    contents = RDoc::Encoding.read_file @tempfile.path, Encoding::US_ASCII, true
+
+    assert_equal '?', contents
+    assert_equal Encoding::US_ASCII, contents.encoding
+  end
+
   def test_class_read_file_encoding_guess
     skip "Encoding not implemented" unless Object.const_defined? :Encoding
 
diff --git a/test/rdoc/test_rdoc_markup_pre_process.rb b/test/rdoc/test_rdoc_markup_pre_process.rb
index b4db512ce3..0d2e8565be 100644
--- a/test/rdoc/test_rdoc_markup_pre_process.rb
+++ b/test/rdoc/test_rdoc_markup_pre_process.rb
@@ -1,3 +1,5 @@
+# coding: utf-8
+
 require 'tempfile'
 require 'rubygems'
 require 'minitest/autorun'
@@ -46,6 +48,30 @@ contents of a string.
     assert_equal expected, content
   end
 
+  def test_include_file_encoding_incompatible
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
+
+    @tempfile.write <<-INCLUDE
+# -*- mode: rdoc; coding: utf-8; fill-column: 74; -*-
+
+π
+    INCLUDE
+
+    @tempfile.flush
+    @tempfile.rewind
+
+    content = @pp.include_file @file_name, '', Encoding::US_ASCII
+
+    expected = "?\n"
+
+    # FIXME 1.9 fix on windoze
+    # preprocessor uses binread, so line endings are \r\n
+    expected.gsub!("\n", "\r\n") if
+      RUBY_VERSION =~ /^1.9/ && RUBY_PLATFORM =~ /mswin|mingw/
+
+    assert_equal expected, content
+  end
+
   def test_handle
     text = "# :x: y\n"
     out = @pp.handle text
diff --git a/test/rdoc/test_rdoc_options.rb b/test/rdoc/test_rdoc_options.rb
index eaa10586fd..763f50b5f0 100644
--- a/test/rdoc/test_rdoc_options.rb
+++ b/test/rdoc/test_rdoc_options.rb
@@ -17,28 +17,17 @@ class TestRDocOptions < MiniTest::Unit::TestCase
   end
 
   def test_check_files
-    expected = ''
+    skip "assumes UNIX permission model" if /mswin|mingw/ =~ RUBY_PLATFORM
     out, err = capture_io do
       Dir.mktmpdir do |dir|
-        if RUBY_PLATFORM =~ /mswin|mingw/ then
-          @options.files = %w[nonexistent]
-
-          expected = <<-EXPECTED
-file 'nonexistent' not found
-          EXPECTED
-        else
+        Dir.chdir dir do
           FileUtils.touch 'unreadable'
           FileUtils.chmod 0, 'unreadable'
 
           @options.files = %w[nonexistent unreadable]
 
-          expected = <<-EXPECTED
-file 'nonexistent' not found
-file 'unreadable' not readable
-          EXPECTED
+          @options.check_files
         end
-
-        @options.check_files
       end
     end
 
@@ -46,6 +35,11 @@ file 'unreadable' not readable
 
     assert_equal '', out
 
+    expected = <<-EXPECTED
+file 'nonexistent' not found
+file 'unreadable' not readable
+    EXPECTED
+
     assert_equal expected, err
   end
 
diff --git a/test/rdoc/test_rdoc_text.rb b/test/rdoc/test_rdoc_text.rb
index ebc92dc1a5..96635aaa8e 100644
--- a/test/rdoc/test_rdoc_text.rb
+++ b/test/rdoc/test_rdoc_text.rb
@@ -55,6 +55,18 @@ class TestRDocText < MiniTest::Unit::TestCase
                  expand_tabs(".\t\t."), 'dot tab tab dot')
   end
 
+  def test_expand_tabs_encoding
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
+
+    inn = "hello\ns\tdave"
+    inn.force_encoding Encoding::BINARY
+
+    out = expand_tabs inn
+
+    assert_equal "hello\ns       dave", out
+    assert_equal Encoding::BINARY, out.encoding
+  end
+
   def test_flush_left
     text = <<-TEXT
 
@@ -73,6 +85,31 @@ The comments associated with
     assert_equal expected, flush_left(text)
   end
 
+  def test_flush_left_encoding
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
+
+    text = <<-TEXT
+
+  we don't worry too much.
+
+  The comments associated with
+    TEXT
+
+    text.force_encoding Encoding::US_ASCII
+
+    expected = <<-EXPECTED
+
+we don't worry too much.
+
+The comments associated with
+    EXPECTED
+
+    result = flush_left text
+
+    assert_equal expected, result
+    assert_equal Encoding::US_ASCII, result.encoding
+  end
+
   def test_markup
     def formatter() RDoc::Markup::ToHtml.new end
 
@@ -223,8 +260,36 @@ The comments associated with
    The comments associated with
     EXPECTED
 
-    assert_equal expected, strip_stars(text)
-    assert_equal Encoding::CP852, text.encoding
+    result = strip_stars text
+
+    assert_equal expected, result
+    assert_equal Encoding::CP852, result.encoding
+  end
+
+  def test_strip_stars_encoding2
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
+
+    text = <<-TEXT
+/*
+ * * we don't worry too much.
+ *
+ * The comments associated with
+ */
+    TEXT
+
+    text.force_encoding Encoding::BINARY
+
+    expected = <<-EXPECTED
+
+   * we don't worry too much.
+
+   The comments associated with
+    EXPECTED
+
+    result = strip_stars text
+
+    assert_equal expected, result
+    assert_equal Encoding::BINARY, result.encoding
   end
 
   def test_to_html_apostrophe