test: handle not a number (NaN) values in the param conversion test.

Reviewed-by: Tim Hudson <tjh@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16144)

1 files changed, 22 insertions, 2 deletions

diff --git a/test/params_conversion_test.c b/test/params_conversion_test.c
index 48d01cac68..9422ef1473 100644
--- a/test/params_conversion_test.c
+++ b/test/params_conversion_test.c
@@ -279,8 +279,28 @@ static int param_conversion_test(const PARAM_CONVERSION *pc, int line)
             return 0;
         }
     } else {
-        if (!TEST_true(OSSL_PARAM_get_double(pc->param, &d))
-            || !TEST_true(d == pc->d)) {
+        if (!TEST_true(OSSL_PARAM_get_double(pc->param, &d))) {
+            TEST_note("unable to convert to double on line %d", line);
+            return 0;
+        }
+        /*
+         * Check for not a number (NaN) without using the libm functions.
+         * When d is a NaN, the standard requires d == d to be false.
+         * It's less clear if d != d should be true even though it generally is.
+         * Hence we use the equality test and a not.
+         */
+        if (!(d == d)) {
+            /*
+             * We've encountered a NaN so check it's really meant to be a NaN.
+             * We ignore the case where the two values are both different NaN,
+             * that's not resolvable without knowing the underlying format
+             * or using libm functions.
+             */
+            if (!TEST_false(pc->d == pc->d)) {
+                TEST_note("unexpected NaN on line %d", line);
+                return 0;
+            }
+        } else if (!TEST_true(d == pc->d)) {
             TEST_note("unexpected conversion to double on line %d", line);
             return 0;
         }