New SSL test framework

Currently, SSL tests are configured via command-line switches to
ssltest.c. This results in a lot of duplication between ssltest.c and
apps, and a complex setup. ssltest.c is also simply old and needs
maintenance.

Instead, we already have a way to configure SSL servers and clients, so
we leverage that. SSL tests can now be configured from a configuration
file. Test servers and clients are configured using the standard
ssl_conf module. Additional test settings are configured via a test
configuration.

Moreover, since the CONF language involves unnecessary boilerplate, the
test conf itself is generated from a shorter Perl syntax.

The generated testcase files are checked in to the repo to make
it easier to verify that the intended test cases are in fact run; and to
simplify debugging failures.

To demonstrate the approach, min/max protocol tests are converted to the
new format. This change also fixes MinProtocol and MaxProtocol
handling. It was previously requested that an SSL_CTX have both the
server and client flags set for these commands; this clearly can never work.

Guide to this PR:
 - test/ssl_test.c - test framework
 - test/ssl_test_ctx.* - test configuration structure
 - test/handshake_helper.* - new SSL test handshaking code
 - test/ssl-tests/ - test configurations
 - test/generate_ssl_tests.pl - script for generating CONF-style test
   configurations from perl inputs

Reviewed-by: Richard Levitte <levitte@openssl.org>

1 files changed, 217 insertions, 0 deletions

diff --git a/test/ssl_test.c b/test/ssl_test.c
new file mode 100644
index 0000000000..01ce5d5684
--- /dev/null
+++ b/test/ssl_test.c
@@ -0,0 +1,217 @@
+/*
+ * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL licenses, (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * https://www.openssl.org/source/license.html
+ * or in the file LICENSE in the source distribution.
+ */
+
+#include <stdio.h>
+
+#include <openssl/conf.h>
+#include <openssl/err.h>
+#include <openssl/ssl.h>
+
+#include "handshake_helper.h"
+#include "ssl_test_ctx.h"
+#include "testutil.h"
+
+static CONF *conf = NULL;
+
+/* Currently the section names are of the form test-<number>, e.g. test-15. */
+#define MAX_TESTCASE_NAME_LENGTH 100
+
+typedef struct ssl_test_ctx_test_fixture {
+    const char *test_case_name;
+    char test_app[MAX_TESTCASE_NAME_LENGTH];
+} SSL_TEST_FIXTURE;
+
+static SSL_TEST_FIXTURE set_up(const char *const test_case_name)
+{
+    SSL_TEST_FIXTURE fixture;
+    fixture.test_case_name = test_case_name;
+    return fixture;
+}
+
+static const char *print_alert(int alert)
+{
+    return alert ? SSL_alert_desc_string_long(alert) : "no alert";
+}
+
+static int check_result(HANDSHAKE_RESULT result, SSL_TEST_CTX *test_ctx)
+{
+    if (result.result != test_ctx->expected_result) {
+        fprintf(stderr, "ExpectedResult mismatch: expected %s, got %s.\n",
+                ssl_test_result_t_name(test_ctx->expected_result),
+                ssl_test_result_t_name(result.result));
+        return 0;
+    }
+    return 1;
+}
+
+static int check_alerts(HANDSHAKE_RESULT result, SSL_TEST_CTX *test_ctx)
+{
+    if (result.client_alert_sent != result.client_alert_received) {
+        fprintf(stderr, "Client sent alert %s but server received %s\n.",
+                print_alert(result.client_alert_sent),
+                print_alert(result.client_alert_received));
+        /*
+         * We can't bail here because the peer doesn't always get far enough
+         * to process a received alert. Specifically, in protocol version
+         * negotiation tests, we have the following scenario.
+         * Client supports TLS v1.2 only; Server supports TLS v1.1.
+         * Client proposes TLS v1.2; server responds with 1.1;
+         * Client now sends a protocol alert, using TLS v1.2 in the header.
+         * The server, however, rejects the alert because of version mismatch
+         * in the record layer; therefore, the server appears to never
+         * receive the alert.
+         */
+        /* return 0; */
+    }
+
+    if (result.server_alert_sent != result.server_alert_received) {
+        fprintf(stderr, "Server sent alert %s but client received %s\n.",
+                print_alert(result.server_alert_sent),
+                print_alert(result.server_alert_received));
+        /* return 0; */
+    }
+
+    /* Tolerate an alert if one wasn't explicitly specified in the test. */
+    if (test_ctx->client_alert
+        /*
+         * The info callback alert value is computed as
+         * (s->s3->send_alert[0] << 8) | s->s3->send_alert[1]
+         * where the low byte is the alert code and the high byte is other stuff.
+         */
+        && (result.client_alert_sent & 0xff) != test_ctx->client_alert) {
+        fprintf(stderr, "ClientAlert mismatch: expected %s, got %s.\n",
+                print_alert(test_ctx->client_alert),
+                print_alert(result.client_alert_sent));
+        return 0;
+    }
+
+    if (test_ctx->server_alert
+        && (result.server_alert_sent & 0xff) != test_ctx->server_alert) {
+        fprintf(stderr, "ServerAlert mismatch: expected %s, got %s.\n",
+                print_alert(test_ctx->server_alert),
+                print_alert(result.server_alert_sent));
+        return 0;
+    }
+
+    return 1;
+}
+
+static int check_protocol(HANDSHAKE_RESULT result, SSL_TEST_CTX *test_ctx)
+{
+    if (result.client_protocol != result.server_protocol) {
+        fprintf(stderr, "Client has protocol %s but server has %s\n.",
+                ssl_protocol_name(result.client_protocol),
+                ssl_protocol_name(result.server_protocol));
+        return 0;
+    }
+
+    if (test_ctx->protocol) {
+        if (result.client_protocol != test_ctx->protocol) {
+            fprintf(stderr, "Protocol mismatch: expected %s, got %s.\n",
+                    ssl_protocol_name(test_ctx->protocol),
+                    ssl_protocol_name(result.client_protocol));
+            return 0;
+        }
+    }
+    return 1;
+}
+
+/*
+ * This could be further simplified by constructing an expected
+ * HANDSHAKE_RESULT, and implementing comparison methods for
+ * its fields.
+ */
+static int check_test(HANDSHAKE_RESULT result, SSL_TEST_CTX *test_ctx)
+{
+    int ret = 1;
+    ret &= check_result(result, test_ctx);
+    ret &= check_alerts(result, test_ctx);
+    if (result.result == SSL_TEST_SUCCESS)
+        ret &= check_protocol(result, test_ctx);
+    return ret;
+}
+
+static int execute_test(SSL_TEST_FIXTURE fixture)
+{
+    /* TODO(emilia): this is confusing. Flip to return 1 on success. */
+    int ret = 1;
+    SSL_CTX *server_ctx = NULL, *client_ctx = NULL;
+    SSL_TEST_CTX *test_ctx = NULL;
+    HANDSHAKE_RESULT result;
+
+    server_ctx = SSL_CTX_new(TLS_server_method());
+    client_ctx = SSL_CTX_new(TLS_client_method());
+    OPENSSL_assert(server_ctx != NULL && client_ctx != NULL);
+
+    OPENSSL_assert(CONF_modules_load(conf, fixture.test_app, 0) > 0);
+
+    if (!SSL_CTX_config(server_ctx, "server")
+       || !SSL_CTX_config(client_ctx, "client")) {
+        goto err;
+    }
+
+    test_ctx = SSL_TEST_CTX_create(conf, fixture.test_app);
+    if (test_ctx == NULL)
+        goto err;
+
+    result = do_handshake(server_ctx, client_ctx);
+
+    if (check_test(result, test_ctx))
+        ret = 0;
+
+err:
+    CONF_modules_unload(0);
+    SSL_CTX_free(server_ctx);
+    SSL_CTX_free(client_ctx);
+    SSL_TEST_CTX_free(test_ctx);
+    if (ret != 0)
+        ERR_print_errors_fp(stderr);
+    return ret;
+}
+
+static void tear_down(SSL_TEST_FIXTURE fixture)
+{
+}
+
+#define SETUP_SSL_TEST_FIXTURE()                        \
+    SETUP_TEST_FIXTURE(SSL_TEST_FIXTURE, set_up)
+#define EXECUTE_SSL_TEST()             \
+    EXECUTE_TEST(execute_test, tear_down)
+
+static int test_handshake(int idx)
+{
+    SETUP_SSL_TEST_FIXTURE();
+    snprintf(fixture.test_app, sizeof(fixture.test_app),
+             "test-%d", idx);
+    EXECUTE_SSL_TEST();
+}
+
+int main(int argc, char **argv)
+{
+    int result = 0;
+    long num_tests;
+
+    if (argc != 2)
+        return 1;
+
+    conf = NCONF_new(NULL);
+    OPENSSL_assert(conf != NULL);
+
+    /* argv[1] should point to the test conf file */
+    OPENSSL_assert(NCONF_load(conf, argv[1], NULL) > 0);
+
+    OPENSSL_assert(NCONF_get_number_e(conf, NULL, "num_tests", &num_tests));
+
+    ADD_ALL_TESTS(test_handshake, (int)(num_tests));
+    result = run_tests(argv[0]);
+
+    CONF_modules_free();
+    return result;
+}