From 3ba5ffc8ba976fa45d12791c94a8d9641c956b63 Mon Sep 17 00:00:00 2001 From: shugo Date: Sat, 19 Nov 2016 02:29:23 +0000 Subject: Support TLS and hash styles options for Net::FTP.new. If the :ssl options is specified, the control connection is protected with TLS in the manner described in RFC 4217. Data connections are also protected with TLS unless the :private_data_connection is set to false. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@56834 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- NEWS | 5 + lib/net/ftp.rb | 162 ++++++++++++++--- test/net/ftp/test_ftp.rb | 456 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 601 insertions(+), 22 deletions(-) diff --git a/NEWS b/NEWS index 8d09fd71e1..050e8b83d7 100644 --- a/NEWS +++ b/NEWS @@ -201,6 +201,11 @@ with all sufficient information, see the ChangeLog file or Redmine * New method: Net::HTTP.post [Feature #12375] +* Net::FTP + + * Support TLS (RFC 4217). + * Support hash style options for Net::FTP.new. + * OpenSSL * OpenSSL is extracted as a gem and the upstream has been migrated to diff --git a/lib/net/ftp.rb b/lib/net/ftp.rb index cb786e8360..18dda61566 100644 --- a/lib/net/ftp.rb +++ b/lib/net/ftp.rb @@ -19,6 +19,10 @@ require "socket" require "monitor" require "net/protocol" require "time" +begin + require "openssl" +rescue LoadError +end module Net @@ -75,6 +79,10 @@ module Net # class FTP include MonitorMixin + if defined?(OpenSSL::SSL) + include OpenSSL + include SSL + end # :stopdoc: FTP_PORT = 21 @@ -143,38 +151,108 @@ module Net # If a block is given, it is passed the +FTP+ object, which will be closed # when the block finishes, or when an exception is raised. # - def FTP.open(host, user = nil, passwd = nil, acct = nil) + def FTP.open(host, *args) if block_given? - ftp = new(host, user, passwd, acct) + ftp = new(host, *args) begin yield ftp ensure ftp.close end else - new(host, user, passwd, acct) + new(host, *args) end end + # :call-seq: + # Net::FTP.new(host = nil, options = {}) # # Creates and returns a new +FTP+ object. If a +host+ is given, a connection - # is made. Additionally, if the +user+ is given, the given user name, - # password, and (optionally) account are used to log in. See #login. - # - def initialize(host = nil, user = nil, passwd = nil, acct = nil) + # is made. + # + # +options+ is an option hash, each key of which is a symbol. + # + # The available options are: + # + # port:: Port number (default value is 21) + # ssl:: If options[:ssl] is true, then an attempt will be made + # to use SSL (now TLS) to connect to the server. For this to + # work OpenSSL [OSSL] and the Ruby OpenSSL [RSSL] extensions + # need to be installed. If options[:ssl] is a hash, it's + # passed to OpenSSL::SSL::SSLContext#set_params as parameters. + # private_data_connection:: If true, TLS is used for data connections. + # Default: +true+ when options[:ssl] is true. + # user:: Username for login. If options[:user] is the string + # "anonymous" and the options[:password] is +nil+, + # "anonymous@" is used as a password. If options[:user] is + # +nil+, + # passwd:: Password for login. + # acct:: Account information for ACCT. + # passive:: When +true+, the connection is in passive mode. Default: +true+. + # debug_mode:: When +true+, all traffic to and from the server is + # written to +$stdout+. Default: +false+. + # + def initialize(host = nil, user_or_options = {}, passwd = nil, acct = nil) super() + begin + options = user_or_options.to_hash + rescue NoMethodError + # for backward compatibility + options = {} + options[:user] = user_or_options + options[:passwd] = passwd + options[:acct] = acct + end + @host = nil + if options[:ssl] + unless defined?(OpenSSL::SSL) + raise "SSL extension not installed" + end + ssl_params = options[:ssl] == true ? {} : options[:ssl] + @ssl_context = SSLContext.new + @ssl_context.set_params(ssl_params) + if defined?(VerifyCallbackProc) + @ssl_context.verify_callback = VerifyCallbackProc + end + @ssl_session = nil + if options[:private_data_connection].nil? + @private_data_connection = true + else + @private_data_connection = options[:private_data_connection] + end + else + @ssl_context = nil + if options[:private_data_connection] + raise ArgumentError, + "private_data_connection can be set to true only when ssl is enabled" + end + end @binary = true - @passive = @@default_passive - @debug_mode = false + if options[:passive].nil? + @passive = @@default_passive + else + @passive = options[:passive] + end + if options[:debug_mode].nil? + @debug_mode = false + else + @debug_mode = options[:debug_mode] + end @resume = false - @sock = NullSocket.new + @bare_sock = @sock = NullSocket.new @logged_in = false @open_timeout = nil @read_timeout = 60 if host - connect(host) - if user - login(user, passwd, acct) + if options[:port] + connect(host, options[:port] || FTP_PORT) + else + # spec/rubyspec/library/net/ftp/initialize_spec.rb depends on + # the number of arguments passed to connect.... + connect(host) + end + if options[:user] + login(options[:user], options[:passwd], options[:acct]) end end end @@ -242,11 +320,28 @@ module Net else sock = TCPSocket.open(host, port) end - BufferedSocket.new(sock, read_timeout: @read_timeout) } end private :open_socket + def start_tls_session(sock) + ssl_sock = SSLSocket.new(sock, @ssl_context) + ssl_sock.sync_close = true + ssl_sock.hostname = @host if ssl_sock.respond_to? :hostname= + if @ssl_session && + Process.clock_gettime(Process::CLOCK_REALTIME) < @ssl_session.time.to_f + @ssl_session.timeout + # ProFTPD returns 425 for data connections if session is not reused. + ssl_sock.session = @ssl_session + end + ssl_sock.connect + if @ssl_context.verify_mode != VERIFY_NONE + ssl_sock.post_connection_check(@host) + end + @ssl_session = ssl_sock.session + return ssl_sock + end + private :start_tls_session + # # Establishes an FTP connection to host, optionally overriding the default # port. If the environment variable +SOCKS_SERVER+ is set, sets up the @@ -258,8 +353,24 @@ module Net print "connect: ", host, ", ", port, "\n" end synchronize do - @sock = open_socket(host, port) + @host = host + @bare_sock = open_socket(host, port) + @sock = BufferedSocket.new(@bare_sock, read_timeout: @read_timeout) voidresp + if @ssl_context + begin + voidcmd("AUTH TLS") + ssl_sock = start_tls_session(@bare_sock) + @sock = BufferedSocket.new(ssl_sock, read_timeout: @read_timeout) + if @private_data_connection + voidcmd("PBSZ 0") + voidcmd("PROT P") + end + rescue OpenSSL::SSL::SSLError + close + raise + end + end end end @@ -381,7 +492,7 @@ module Net # Constructs and send the appropriate PORT (or EPRT) command def sendport(host, port) # :nodoc: - remote_address = @sock.remote_address + remote_address = @bare_sock.remote_address if remote_address.ipv4? cmd = "PORT " + (host.split(".") + port.divmod(256)).join(",") elsif remote_address.ipv6? @@ -395,13 +506,13 @@ module Net # Constructs a TCPServer socket def makeport # :nodoc: - TCPServer.open(@sock.local_address.ip_address, 0) + TCPServer.open(@bare_sock.local_address.ip_address, 0) end private :makeport # sends the appropriate command to enable a passive connection def makepasv # :nodoc: - if @sock.remote_address.ipv4? + if @bare_sock.remote_address.ipv4? host, port = parse227(sendcmd("PASV")) else host, port = parse229(sendcmd("EPSV")) @@ -445,14 +556,17 @@ module Net if !resp.start_with?("1") raise FTPReplyError, resp end - conn = BufferedSocket.new(sock.accept, read_timeout: @read_timeout) + conn = sock.accept sock.shutdown(Socket::SHUT_WR) rescue nil sock.read rescue nil ensure sock.close end end - return conn + if @private_data_connection + conn = start_tls_session(conn) + end + return BufferedSocket.new(conn, read_timeout: @read_timeout) end private :transfercmd @@ -1168,7 +1282,7 @@ module Net def close if @sock and not @sock.closed? begin - @sock.shutdown(Socket::SHUT_WR) rescue nil + @bare_sock.shutdown(Socket::SHUT_WR) rescue nil orig, self.read_timeout = self.read_timeout, 3 @sock.read rescue nil ensure @@ -1284,12 +1398,16 @@ module Net end class BufferedSocket < BufferedIO - [:local_address, :remote_address, :addr, :peeraddr, :send, :shutdown].each do |method| + [:local_address, :remote_address, :addr, :peeraddr, :send].each do |method| define_method(method) { |*args| @io.__send__(method, *args) } end + def shutdown(*args) + @io.to_io.shutdown(*args) + end + def read(len = nil) if len s = super(len, String.new, true) diff --git a/test/net/ftp/test_ftp.rb b/test/net/ftp/test_ftp.rb index 8290b0168a..4394808075 100644 --- a/test/net/ftp/test_ftp.rb +++ b/test/net/ftp/test_ftp.rb @@ -8,6 +8,9 @@ require "tempfile" class FTPTest < Test::Unit::TestCase SERVER_ADDR = "127.0.0.1" + CA_FILE = File.expand_path("../imap/cacert.pem", __dir__) + SERVER_KEY = File.expand_path("../imap/server.key", __dir__) + SERVER_CERT = File.expand_path("../imap/server.crt", __dir__) def setup @thread = nil @@ -219,6 +222,62 @@ class FTPTest < Test::Unit::TestCase end end + def test_implicit_login + commands = [] + server = create_ftp_server { |sock| + sock.print("220 (test_ftp).\r\n") + commands.push(sock.gets) + sock.print("331 Please specify the password.\r\n") + commands.push(sock.gets) + sock.print("332 Need account for login.\r\n") + commands.push(sock.gets) + sock.print("230 Login successful.\r\n") + commands.push(sock.gets) + sock.print("200 Switching to Binary mode.\r\n") + } + begin + begin + ftp = Net::FTP.new(SERVER_ADDR, + port: server.port, + user: "foo", + passwd: "bar", + acct: "baz") + assert_equal("USER foo\r\n", commands.shift) + assert_equal("PASS bar\r\n", commands.shift) + assert_equal("ACCT baz\r\n", commands.shift) + assert_equal("TYPE I\r\n", commands.shift) + assert_equal(nil, commands.shift) + ensure + ftp.close if ftp + end + ensure + server.close + end + end + + def test_s_open + commands = [] + server = create_ftp_server { |sock| + sock.print("220 (test_ftp).\r\n") + commands.push(sock.gets) + sock.print("331 Please specify the password.\r\n") + commands.push(sock.gets) + sock.print("230 Login successful.\r\n") + commands.push(sock.gets) + sock.print("200 Switching to Binary mode.\r\n") + } + begin + Net::FTP.open(SERVER_ADDR, port: server.port, user: "anonymous") do + end + assert_equal("USER anonymous\r\n", commands.shift) + assert_equal("PASS anonymous@\r\n", commands.shift) + assert_equal("TYPE I\r\n", commands.shift) + assert_equal(nil, commands.shift) + ensure + server.close + end + end + # TODO: How can we test open_timeout? sleep before accept cannot delay # connections. def _test_open_timeout_exceeded @@ -1644,6 +1703,362 @@ EOF end end + if defined?(OpenSSL::SSL) + def test_tls_unknown_ca + assert_raise(OpenSSL::SSL::SSLError) do + tls_test do |port| + begin + Net::FTP.new("localhost", + :port => port, + :ssl => true) + rescue SystemCallError + skip $! + end + end + end + end + + def test_tls_with_ca_file + assert_nothing_raised do + tls_test do |port| + begin + Net::FTP.new("localhost", + :port => port, + :ssl => { :ca_file => CA_FILE }) + rescue SystemCallError + skip $! + end + end + end + end + + def test_tls_verify_none + assert_nothing_raised do + tls_test do |port| + Net::FTP.new(SERVER_ADDR, + :port => port, + :ssl => { :verify_mode => OpenSSL::SSL::VERIFY_NONE }) + end + end + end + + def test_tls_post_connection_check + assert_raise(OpenSSL::SSL::SSLError) do + tls_test do |port| + # SERVER_ADDR is different from the hostname in the certificate, + # so the following code should raise a SSLError. + Net::FTP.new(SERVER_ADDR, + :port => port, + :ssl => { :ca_file => CA_FILE }) + end + end + end + + def test_active_private_data_connection + server = TCPServer.new(SERVER_ADDR, 0) + port = server.addr[1] + commands = [] + binary_data = (0..0xff).map {|i| i.chr}.join * 4 * 3 + @thread = Thread.start do + sock = server.accept + begin + sock.print("220 (test_ftp).\r\n") + commands.push(sock.gets) + sock.print("234 AUTH success.\r\n") + ctx = OpenSSL::SSL::SSLContext.new + ctx.ca_file = CA_FILE + ctx.key = File.open(SERVER_KEY) { |f| + OpenSSL::PKey::RSA.new(f) + } + ctx.cert = File.open(SERVER_CERT) { |f| + OpenSSL::X509::Certificate.new(f) + } + sock = OpenSSL::SSL::SSLSocket.new(sock, ctx) + sock.sync_close = true + begin + sock.accept + commands.push(sock.gets) + sock.print("200 PSBZ success.\r\n") + commands.push(sock.gets) + sock.print("200 PROT success.\r\n") + commands.push(sock.gets) + sock.print("331 Please specify the password.\r\n") + commands.push(sock.gets) + sock.print("230 Login successful.\r\n") + commands.push(sock.gets) + sock.print("200 Switching to Binary mode.\r\n") + line = sock.gets + commands.push(line) + port_args = line.slice(/\APORT (.*)/, 1).split(/,/) + host = port_args[0, 4].join(".") + port = port_args[4, 2].map(&:to_i).inject {|x, y| (x << 8) + y} + sock.print("200 PORT command successful.\r\n") + commands.push(sock.gets) + sock.print("150 Opening BINARY mode data connection for foo (#{binary_data.size} bytes)\r\n") + conn = TCPSocket.new(host, port) + conn = OpenSSL::SSL::SSLSocket.new(conn, ctx) + conn.sync_close = true + conn.accept + binary_data.scan(/.{1,1024}/nm) do |s| + conn.print(s) + end + conn.close + sock.print("226 Transfer complete.\r\n") + rescue OpenSSL::SSL::SSLError + end + ensure + sock.close + server.close + end + end + ftp = Net::FTP.new("localhost", + port: port, + ssl: { ca_file: CA_FILE }, + passive: false) + begin + assert_equal("AUTH TLS\r\n", commands.shift) + assert_equal("PBSZ 0\r\n", commands.shift) + assert_equal("PROT P\r\n", commands.shift) + ftp.login + assert_match(/\AUSER /, commands.shift) + assert_match(/\APASS /, commands.shift) + assert_equal("TYPE I\r\n", commands.shift) + buf = ftp.getbinaryfile("foo", nil) + assert_equal(binary_data, buf) + assert_equal(Encoding::ASCII_8BIT, buf.encoding) + assert_match(/\APORT /, commands.shift) + assert_equal("RETR foo\r\n", commands.shift) + assert_equal(nil, commands.shift) + ensure + ftp.close + end + end + + def test_passive_private_data_connection + server = TCPServer.new(SERVER_ADDR, 0) + port = server.addr[1] + commands = [] + binary_data = (0..0xff).map {|i| i.chr}.join * 4 * 3 + @thread = Thread.start do + sock = server.accept + begin + sock.print("220 (test_ftp).\r\n") + commands.push(sock.gets) + sock.print("234 AUTH success.\r\n") + ctx = OpenSSL::SSL::SSLContext.new + ctx.ca_file = CA_FILE + ctx.key = File.open(SERVER_KEY) { |f| + OpenSSL::PKey::RSA.new(f) + } + ctx.cert = File.open(SERVER_CERT) { |f| + OpenSSL::X509::Certificate.new(f) + } + sock = OpenSSL::SSL::SSLSocket.new(sock, ctx) + sock.sync_close = true + begin + sock.accept + commands.push(sock.gets) + sock.print("200 PSBZ success.\r\n") + commands.push(sock.gets) + sock.print("200 PROT success.\r\n") + commands.push(sock.gets) + sock.print("331 Please specify the password.\r\n") + commands.push(sock.gets) + sock.print("230 Login successful.\r\n") + commands.push(sock.gets) + sock.print("200 Switching to Binary mode.\r\n") + commands.push(sock.gets) + data_server = TCPServer.new(SERVER_ADDR, 0) + port = data_server.local_address.ip_port + sock.printf("227 Entering Passive Mode (127,0,0,1,%s).\r\n", + port.divmod(256).join(",")) + commands.push(sock.gets) + sock.print("150 Opening BINARY mode data connection for foo (#{binary_data.size} bytes)\r\n") + conn = data_server.accept + conn = OpenSSL::SSL::SSLSocket.new(conn, ctx) + conn.sync_close = true + conn.accept + binary_data.scan(/.{1,1024}/nm) do |s| + conn.print(s) + end + conn.close + data_server.close + sock.print("226 Transfer complete.\r\n") + rescue OpenSSL::SSL::SSLError + end + ensure + sock.close + server.close + end + end + ftp = Net::FTP.new("localhost", + port: port, + ssl: { ca_file: CA_FILE }, + passive: true) + begin + assert_equal("AUTH TLS\r\n", commands.shift) + assert_equal("PBSZ 0\r\n", commands.shift) + assert_equal("PROT P\r\n", commands.shift) + ftp.login + assert_match(/\AUSER /, commands.shift) + assert_match(/\APASS /, commands.shift) + assert_equal("TYPE I\r\n", commands.shift) + buf = ftp.getbinaryfile("foo", nil) + assert_equal(binary_data, buf) + assert_equal(Encoding::ASCII_8BIT, buf.encoding) + assert_equal("PASV\r\n", commands.shift) + assert_equal("RETR foo\r\n", commands.shift) + assert_equal(nil, commands.shift) + ensure + ftp.close + end + end + + def test_active_clear_data_connection + server = TCPServer.new(SERVER_ADDR, 0) + port = server.addr[1] + commands = [] + binary_data = (0..0xff).map {|i| i.chr}.join * 4 * 3 + @thread = Thread.start do + sock = server.accept + begin + sock.print("220 (test_ftp).\r\n") + commands.push(sock.gets) + sock.print("234 AUTH success.\r\n") + ctx = OpenSSL::SSL::SSLContext.new + ctx.ca_file = CA_FILE + ctx.key = File.open(SERVER_KEY) { |f| + OpenSSL::PKey::RSA.new(f) + } + ctx.cert = File.open(SERVER_CERT) { |f| + OpenSSL::X509::Certificate.new(f) + } + sock = OpenSSL::SSL::SSLSocket.new(sock, ctx) + sock.sync_close = true + begin + sock.accept + commands.push(sock.gets) + sock.print("331 Please specify the password.\r\n") + commands.push(sock.gets) + sock.print("230 Login successful.\r\n") + commands.push(sock.gets) + sock.print("200 Switching to Binary mode.\r\n") + line = sock.gets + commands.push(line) + port_args = line.slice(/\APORT (.*)/, 1).split(/,/) + host = port_args[0, 4].join(".") + port = port_args[4, 2].map(&:to_i).inject {|x, y| (x << 8) + y} + sock.print("200 PORT command successful.\r\n") + commands.push(sock.gets) + sock.print("150 Opening BINARY mode data connection for foo (#{binary_data.size} bytes)\r\n") + conn = TCPSocket.new(host, port) + binary_data.scan(/.{1,1024}/nm) do |s| + conn.print(s) + end + conn.close + sock.print("226 Transfer complete.\r\n") + rescue OpenSSL::SSL::SSLError + end + ensure + sock.close + server.close + end + end + ftp = Net::FTP.new("localhost", + port: port, + ssl: { ca_file: CA_FILE }, + private_data_connection: false, + passive: false) + begin + assert_equal("AUTH TLS\r\n", commands.shift) + ftp.login + assert_match(/\AUSER /, commands.shift) + assert_match(/\APASS /, commands.shift) + assert_equal("TYPE I\r\n", commands.shift) + buf = ftp.getbinaryfile("foo", nil) + assert_equal(binary_data, buf) + assert_equal(Encoding::ASCII_8BIT, buf.encoding) + assert_match(/\APORT /, commands.shift) + assert_equal("RETR foo\r\n", commands.shift) + assert_equal(nil, commands.shift) + ensure + ftp.close + end + end + + def test_passive_clear_data_connection + server = TCPServer.new(SERVER_ADDR, 0) + port = server.addr[1] + commands = [] + binary_data = (0..0xff).map {|i| i.chr}.join * 4 * 3 + @thread = Thread.start do + sock = server.accept + begin + sock.print("220 (test_ftp).\r\n") + commands.push(sock.gets) + sock.print("234 AUTH success.\r\n") + ctx = OpenSSL::SSL::SSLContext.new + ctx.ca_file = CA_FILE + ctx.key = File.open(SERVER_KEY) { |f| + OpenSSL::PKey::RSA.new(f) + } + ctx.cert = File.open(SERVER_CERT) { |f| + OpenSSL::X509::Certificate.new(f) + } + sock = OpenSSL::SSL::SSLSocket.new(sock, ctx) + sock.sync_close = true + begin + sock.accept + commands.push(sock.gets) + sock.print("331 Please specify the password.\r\n") + commands.push(sock.gets) + sock.print("230 Login successful.\r\n") + commands.push(sock.gets) + sock.print("200 Switching to Binary mode.\r\n") + commands.push(sock.gets) + data_server = TCPServer.new(SERVER_ADDR, 0) + port = data_server.local_address.ip_port + sock.printf("227 Entering Passive Mode (127,0,0,1,%s).\r\n", + port.divmod(256).join(",")) + commands.push(sock.gets) + sock.print("150 Opening BINARY mode data connection for foo (#{binary_data.size} bytes)\r\n") + conn = data_server.accept + binary_data.scan(/.{1,1024}/nm) do |s| + conn.print(s) + end + conn.close + data_server.close + sock.print("226 Transfer complete.\r\n") + rescue OpenSSL::SSL::SSLError + end + ensure + sock.close + server.close + end + end + ftp = Net::FTP.new("localhost", + port: port, + ssl: { ca_file: CA_FILE }, + private_data_connection: false, + passive: true) + begin + assert_equal("AUTH TLS\r\n", commands.shift) + ftp.login + assert_match(/\AUSER /, commands.shift) + assert_match(/\APASS /, commands.shift) + assert_equal("TYPE I\r\n", commands.shift) + buf = ftp.getbinaryfile("foo", nil) + assert_equal(binary_data, buf) + assert_equal(Encoding::ASCII_8BIT, buf.encoding) + assert_equal("PASV\r\n", commands.shift) + assert_equal("RETR foo\r\n", commands.shift) + assert_equal(nil, commands.shift) + ensure + ftp.close + end + end + end + private def create_ftp_server(sleep_time = nil) @@ -1667,4 +2082,45 @@ EOF end return server end + + def tls_test + server = TCPServer.new(SERVER_ADDR, 0) + port = server.addr[1] + commands = [] + @thread = Thread.start do + sock = server.accept + begin + sock.print("220 (test_ftp).\r\n") + commands.push(sock.gets) + sock.print("234 AUTH success.\r\n") + ctx = OpenSSL::SSL::SSLContext.new + ctx.ca_file = CA_FILE + ctx.key = File.open(SERVER_KEY) { |f| + OpenSSL::PKey::RSA.new(f) + } + ctx.cert = File.open(SERVER_CERT) { |f| + OpenSSL::X509::Certificate.new(f) + } + sock = OpenSSL::SSL::SSLSocket.new(sock, ctx) + sock.sync_close = true + begin + sock.accept + commands.push(sock.gets) + sock.print("200 PSBZ success.\r\n") + commands.push(sock.gets) + sock.print("200 PROT success.\r\n") + rescue OpenSSL::SSL::SSLError + end + ensure + sock.close + server.close + end + end + ftp = yield(port) + ftp.close + + assert_equal("AUTH TLS\r\n", commands.shift) + assert_equal("PBSZ 0\r\n", commands.shift) + assert_equal("PROT P\r\n", commands.shift) + end end -- cgit v1.2.3