TAPify testutil

With the perl test framework comes the output format TAP
(Test Anything Protocol, see http://testanything.org/) with
extra extension for subtests.  This change extends that same
output format to any test program using testutil.

In this implementation, each test program is seen as a full test that
can be used as a subtest.  The perl framework passes on the subtest
level to the test programs with the environment variable
HARNESS_OSSL_LEVEL.  Furthermore, and series of tests added with
ADD_ALL_TESTS is regarded as another subtest level.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3296)

5 files changed, 83 insertions, 25 deletions

diff --git a/test/gmdifftest.c b/test/gmdifftest.c
index 1d508c0c7c..6869300f3e 100644
--- a/test/gmdifftest.c
+++ b/test/gmdifftest.c
@@ -60,5 +60,5 @@ void register_tests(void)
     if (sizeof(time_t) < 8)
         TEST_info("Skipping; time_t is less than 64-bits");
     else
-        ADD_ALL_TESTS(test_gmtime, 1000000);
+        ADD_ALL_TESTS_NOSUBTEST(test_gmtime, 1000000);
 }
diff --git a/test/testlib/OpenSSL/Test.pm b/test/testlib/OpenSSL/Test.pm
index c4799e8648..f8fcbe906d 100644
--- a/test/testlib/OpenSSL/Test.pm
+++ b/test/testlib/OpenSSL/Test.pm
@@ -17,7 +17,7 @@ use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
 $VERSION = "0.8";
 @ISA = qw(Exporter);
 @EXPORT = (@Test::More::EXPORT, qw(setup run indir cmd app fuzz test
-                                   perlapp perltest));
+                                   perlapp perltest subtest));
 @EXPORT_OK = (@Test::More::EXPORT_OK, qw(bldtop_dir bldtop_file
                                          srctop_dir srctop_file
                                          data_file
@@ -65,6 +65,7 @@ use File::Spec::Functions qw/file_name_is_absolute curdir canonpath splitdir
 use File::Path 2.00 qw/rmtree mkpath/;
 use File::Basename;
 
+my $level = 0;
 
 # The name of the test.  This is set by setup() and is used in the other
 # functions to verify that setup() has been used.
@@ -454,6 +455,8 @@ sub run {
         open STDERR, ">", devnull();
     }
 
+    $ENV{HARNESS_OSSL_LEVEL} = $level + 1;
+
     # The dance we do with $? is the same dance the Unix shells appear to
     # do.  For example, a program that gets aborted (and therefore signals
     # SIGABRT = 6) will appear to exit with the code 134.  We mimic this
@@ -1153,4 +1156,13 @@ inspiration from Andy Polyakov E<lt>appro@openssl.org<gt>.
 
 =cut
 
+no warnings 'redefine';
+sub subtest {
+    $level++;
+
+    Test::More::subtest @_;
+
+    $level--;
+};
+
 1;
diff --git a/test/testutil.h b/test/testutil.h
index 5d96ddd41c..ecf993445d 100644
--- a/test/testutil.h
+++ b/test/testutil.h
@@ -53,7 +53,12 @@
  * Simple parameterized tests. Calls test_function(idx) for each 0 <= idx < num.
  */
 # define ADD_ALL_TESTS(test_function, num) \
-  add_all_tests(#test_function, test_function, num)
+    add_all_tests(#test_function, test_function, num, 1)
+/*
+ * A variant of the same without TAP output.
+ */
+# define ADD_ALL_TESTS_NOSUBTEST(test_function, num) \
+    add_all_tests(#test_function, test_function, num, 0)
 
 /*-
  * Test cases that share common setup should use the helper
@@ -131,7 +136,8 @@ void setup_test(void);
 __owur int finish_test(int ret);
 
 void add_test(const char *test_case_name, int (*test_fn) ());
-void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num);
+void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num,
+                   int subtest);
 __owur int run_tests(const char *test_prog_name);
 
 /*
@@ -369,3 +375,5 @@ int test_flush_stderr(void);
 
 extern BIO *bio_out;
 extern BIO *bio_err;
+
+int subtest_level(void);
diff --git a/test/testutil/driver.c b/test/testutil/driver.c
index e70fd21615..4f6b5119c3 100644
--- a/test/testutil/driver.c
+++ b/test/testutil/driver.c
@@ -23,6 +23,9 @@ typedef struct test_info {
     int (*test_fn) ();
     int (*param_test_fn)(int idx);
     int num;
+
+    /* flags */
+    int subtest:1;
 } TEST_INFO;
 
 static TEST_INFO all_tests[1024];
@@ -45,16 +48,24 @@ void add_test(const char *test_case_name, int (*test_fn) ())
 }
 
 void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
-                   int num)
+                   int num, int subtest)
 {
     assert(num_tests != OSSL_NELEM(all_tests));
     all_tests[num_tests].test_case_name = test_case_name;
     all_tests[num_tests].param_test_fn = test_fn;
     all_tests[num_tests].num = num;
+    all_tests[num_tests].subtest = subtest;
     ++num_tests;
     num_test_cases += num;
 }
 
+static int level = 0;
+
+int subtest_level(void)
+{
+    return level;
+}
+
 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
 static int should_report_leaks()
 {
@@ -71,7 +82,6 @@ static int should_report_leaks()
 }
 #endif
 
-
 static int err_cb(const char *str, size_t len, void *u)
 {
     return test_puts_stderr(str);
@@ -79,8 +89,12 @@ static int err_cb(const char *str, size_t len, void *u)
 
 void setup_test()
 {
+    char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
+
     test_open_streams();
 
+    level = TAP_levels != NULL ? 4 * atoi(TAP_levels) : 0;
+
 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
     if (should_report_leaks()) {
         CRYPTO_set_mem_debug(1);
@@ -121,47 +135,69 @@ static void helper_printf_stdout(const char *fmt, ...)
 int run_tests(const char *test_prog_name)
 {
     int num_failed = 0;
+    char *verdict = NULL;
     int i, j;
 
-    helper_printf_stdout("%s: %d test case%s\n", test_prog_name, num_test_cases,
-                         num_test_cases == 1 ? "" : "s");
+    helper_printf_stdout("%*s%d..%d\n", level, "", 1, num_tests);
     test_flush_stdout();
 
     for (i = 0; i != num_tests; ++i) {
         if (all_tests[i].num == -1) {
             int ret = all_tests[i].test_fn();
 
+            verdict = "ok";
             if (!ret) {
-                helper_printf_stdout("** %s failed **\n--------\n",
-                                     all_tests[i].test_case_name);
-                test_flush_stdout();
+                verdict = "not ok";
                 ++num_failed;
             }
+            helper_printf_stdout("%*s%s %d - %s\n", level, "", verdict, i + 1,
+                                     all_tests[i].test_case_name);
+            test_flush_stdout();
             finalize(ret);
         } else {
+            int num_failed_inner = 0;
+
+            level += 4;
+            if (all_tests[i].subtest) {
+                helper_printf_stdout("%*s# Subtest: %s\n", level, "",
+                                     all_tests[i].test_case_name);
+                helper_printf_stdout("%*s%d..%d\n", level, "", 1,
+                                     all_tests[i].num);
+                test_flush_stdout();
+            }
+
             for (j = 0; j < all_tests[i].num; j++) {
                 int ret = all_tests[i].param_test_fn(j);
 
-                if (!ret) {
-                    helper_printf_stdout("** %s failed test %d\n--------\n",
-                                         all_tests[i].test_case_name, j);
+                if (!ret)
+                    ++num_failed_inner;
+
+                finalize(ret);
+
+                if (all_tests[i].subtest) {
+                    verdict = "ok";
+                    if (!ret) {
+                        verdict = "not ok";
+                        ++num_failed_inner;
+                    }
+                    helper_printf_stdout("%*s%s %d\n", level, "", verdict, j + 1);
                     test_flush_stdout();
-                    ++num_failed;
                 }
-                finalize(ret);
             }
+
+            level -= 4;
+            verdict = "ok";
+            if (num_failed_inner) {
+                verdict = "not ok";
+                ++num_failed;
+            }
+            helper_printf_stdout("%*s%s %d - %s\n", level, "", verdict, i + 1,
+                                 all_tests[i].test_case_name);
+            test_flush_stdout();
         }
     }
-
-    if (num_failed != 0) {
-        helper_printf_stdout("%s: %d test%s failed (out of %d)\n",
-                             test_prog_name, num_failed,
-                             num_failed != 1 ? "s" : "", num_test_cases);
-        test_flush_stdout();
+    if (num_failed != 0)
         return EXIT_FAILURE;
-    }
-    helper_printf_stdout("  All tests passed.\n");
-    test_flush_stdout();
     return EXIT_SUCCESS;
 }
 
diff --git a/test/testutil/tests.c b/test/testutil/tests.c
index f00ec6c19e..67b20a572c 100644
--- a/test/testutil/tests.c
+++ b/test/testutil/tests.c
@@ -43,6 +43,7 @@
 static void test_fail_message(const char *prefix, const char *file, int line,
                               const char *type, const char *fmt, ...)
             PRINTF_FORMAT(5, 6);
+int subtest_level(void);
 
 static void helper_printf_stderr(const char *fmt, ...)
 {
@@ -56,6 +57,7 @@ static void helper_printf_stderr(const char *fmt, ...)
 static void test_fail_message_va(const char *prefix, const char *file, int line,
                                  const char *type, const char *fmt, va_list ap)
 {
+    helper_printf_stderr("%*s# ", subtest_level(), "");
     test_puts_stderr(prefix != NULL ? prefix : "ERROR");
     test_puts_stderr(":");
     if (type)