cipher: allow setting IV length when using AEAD cipherstopic/cipher-iv-len

Add OpenSSL::Cipher#iv_len=. For interoperability with other
applications, it is sometimes required. Normally 'IV' is fixed-length,
but in OpenSSL, some ciphers such as aes-128-gcm make use of it as
'nonce', which is variable-length.

Changing the IV length in Cipher#iv= is also an option but I decided not
to choose it. Because in Ruby <= 2.3 Cipher#iv= truncates the input when
the length is longer than the current IV length, changing the behavior
might cause unexpected encryption result.

[Bug #8667] [Bug #10420] [GH ruby/ruby#569]

1 files changed, 57 insertions, 3 deletions

diff --git a/ext/openssl/ossl_cipher.c b/ext/openssl/ossl_cipher.c
index be7a0321..910d9437 100644
--- a/ext/openssl/ossl_cipher.c
+++ b/ext/openssl/ossl_cipher.c
@@ -499,12 +499,17 @@ static VALUE
 ossl_cipher_set_iv(VALUE self, VALUE iv)
 {
     EVP_CIPHER_CTX *ctx;
-    int iv_len;
+    int iv_len = 0;
 
     StringValue(iv);
     GetCipher(self, ctx);
 
-    iv_len = EVP_CIPHER_CTX_iv_length(ctx);
+#if defined(HAVE_AUTHENTICATED_ENCRYPTION)
+    if (EVP_CIPHER_CTX_flags(ctx) & EVP_CIPH_FLAG_AEAD_CIPHER)
+	iv_len = (int)(VALUE)EVP_CIPHER_CTX_get_app_data(ctx);
+#endif
+    if (!iv_len)
+	iv_len = EVP_CIPHER_CTX_iv_length(ctx);
     if (RSTRING_LEN(iv) != iv_len)
 	ossl_raise(rb_eArgError, "iv must be %d bytes", iv_len);
 
@@ -638,11 +643,42 @@ ossl_cipher_is_authenticated(VALUE self)
 
     return (EVP_CIPHER_CTX_flags(ctx) & EVP_CIPH_FLAG_AEAD_CIPHER) ? Qtrue : Qfalse;
 }
+
+/*
+ * call-seq:
+ *   cipher.iv_len = integer -> integer
+ *
+ * Sets the IV/nonce length of the Cipher. Normally block ciphers don't allow
+ * changing the IV length, but some make use of IV for 'nonce'. You may need
+ * this for interoperability with other applications.
+ */
+static VALUE
+ossl_cipher_set_iv_length(VALUE self, VALUE iv_length)
+{
+    int len = NUM2INT(iv_length);
+    EVP_CIPHER_CTX *ctx;
+
+    GetCipher(self, ctx);
+    if (!(EVP_CIPHER_CTX_flags(ctx) & EVP_CIPH_FLAG_AEAD_CIPHER))
+	ossl_raise(eCipherError, "cipher does not support AEAD");
+
+    if (!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, len, NULL))
+	ossl_raise(eCipherError, "unable to set IV length");
+
+    /*
+     * EVP_CIPHER_CTX_iv_length() returns the default length. So we need to save
+     * the length somewhere. Luckily currently we aren't using app_data.
+     */
+    EVP_CIPHER_CTX_set_app_data(ctx, (void *)(VALUE)len);
+
+    return iv_length;
+}
 #else
 #define ossl_cipher_set_auth_data rb_f_notimplement
 #define ossl_cipher_get_auth_tag rb_f_notimplement
 #define ossl_cipher_set_auth_tag rb_f_notimplement
 #define ossl_cipher_is_authenticated rb_f_notimplement
+#define ossl_cipher_set_iv_length rb_f_notimplement
 #endif
 
 /*
@@ -708,13 +744,30 @@ ossl_cipher_set_padding(VALUE self, VALUE padding)
  *  Returns the key length in bytes of the Cipher.
  */
 CIPHER_0ARG_INT(key_length)
+
 /*
  *  call-seq:
  *     cipher.iv_len -> integer
  *
  *  Returns the expected length in bytes for an IV for this Cipher.
  */
-CIPHER_0ARG_INT(iv_length)
+static VALUE
+ossl_cipher_iv_length(VALUE self)
+{
+    EVP_CIPHER_CTX *ctx;
+    int len = 0;
+
+    GetCipher(self, ctx);
+#if defined(HAVE_AUTHENTICATED_ENCRYPTION)
+    if (EVP_CIPHER_CTX_flags(ctx) & EVP_CIPH_FLAG_AEAD_CIPHER)
+	len = (int)(VALUE)EVP_CIPHER_CTX_get_app_data(ctx);
+#endif
+    if (!len)
+	len = EVP_CIPHER_CTX_iv_length(ctx);
+
+    return INT2NUM(len);
+}
+
 /*
  *  call-seq:
  *     cipher.block_size -> integer
@@ -952,6 +1005,7 @@ Init_ossl_cipher(void)
     rb_define_method(cCipher, "key_len=", ossl_cipher_set_key_length, 1);
     rb_define_method(cCipher, "key_len", ossl_cipher_key_length, 0);
     rb_define_method(cCipher, "iv=", ossl_cipher_set_iv, 1);
+    rb_define_method(cCipher, "iv_len=", ossl_cipher_set_iv_length, 1);
     rb_define_method(cCipher, "iv_len", ossl_cipher_iv_length, 0);
     rb_define_method(cCipher, "block_size", ossl_cipher_block_size, 0);
     rb_define_method(cCipher, "padding=", ossl_cipher_set_padding, 1);