io.c: fix stack smashing

* io.c (parse_mode_enc): fix buffer overflow.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@53083 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,7 @@
+Sun Dec 13 18:45:12 2015  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* io.c (parse_mode_enc): fix buffer overflow.
+
 Sun Dec 13 18:35:57 2015  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* ext/fiddle/function.c (initialize): check all arguments first.

io.c

@@ -5090,9 +5090,11 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p, int
 	    fmode |= FMODE_SETENC_BY_BOM;
 	    estr += 4;
             len -= 4;
-	    memcpy(encname, estr, len);
-	    encname[len] = '\0';
-	    estr = encname;
+	    if (len > 0 && len <= ENCODING_MAXNAMELEN) {
+		memcpy(encname, estr, len);
+		encname[len] = '\0';
+		estr = encname;
+	    }
 	}
 	idx = rb_enc_find_index(estr);
     }

test/ruby/test_io_m17n.rb

@@ -2082,6 +2082,19 @@ EOT
     }
   end
 
+  def test_bom_too_long_utfname
+    assert_separately([], <<-'end;') # do
+      assert_warn(/Unsupported encoding/) {
+        open(IO::NULL, "r:bom|utf-" + "x" * 10000) {}
+      }
+    end;
+    assert_separately([], <<-'end;') # do
+      assert_warn(/Unsupported encoding/) {
+        open(IO::NULL, encoding: "bom|utf-" + "x" * 10000) {}
+      }
+    end;
+  end
+
   def test_cbuf
     with_tmpdir {
       fn = "tst"