Set external encoding correctly for File.open('f', FILE::BINARY) on Windows

Previously, the external encoding was only set correctly for
File::BINARY if keyword arguments were provided.  This copies
the logic for the keyword arguments case to the no keyword
arguments case.  Possibly it should be refactored into a
separate function.

Fixes [Bug #16737]

1 files changed, 12 insertions, 0 deletions

diff --git a/io.c b/io.c
index 9634e2c57d..adf5c17f97 100644
--- a/io.c
+++ b/io.c
@@ -5955,6 +5955,18 @@ rb_io_extract_modeenc(VALUE *vmode_p, VALUE *vperm_p, VALUE opthash,
 #endif
 	SET_UNIVERSAL_NEWLINE_DECORATOR_IF_ENC2(enc2, ecflags);
         ecopts = Qnil;
+        if (fmode & FMODE_BINMODE) {
+#ifdef O_BINARY
+            oflags |= O_BINARY;
+#endif
+            if (!has_enc)
+                rb_io_ext_int_to_encs(rb_ascii8bit_encoding(), NULL, &enc, &enc2, fmode);
+        }
+#if DEFAULT_TEXTMODE
+        else if (NIL_P(vmode)) {
+            fmode |= DEFAULT_TEXTMODE;
+        }
+#endif
     }
     else {
 	VALUE v;