src/test/SDL_test_harness.c
author Andreas Schiffler <aschiffler@ferzkopp.net>
Tue, 11 Mar 2014 07:17:56 -0700
changeset 8605 57faccca4fab
parent 8149 681eb46b8ac4
child 8831 9326ec96c132
permissions -rw-r--r--
Fix bug/add test coverage for SDLTest_GenerateRunSeed helper; improve test harness adding output of repro steps for failures; improve negative test for SDL_GetError/SDL_SetError
aschiffler@6717
     1
/*
philipp@8139
     2
  Simple DirectMedia Layer
slouken@8149
     3
  Copyright (C) 1997-2014 Sam Lantinga <slouken@libsdl.org>
aschiffler@6717
     4
philipp@8139
     5
  This software is provided 'as-is', without any express or implied
philipp@8139
     6
  warranty.  In no event will the authors be held liable for any damages
philipp@8139
     7
  arising from the use of this software.
aschiffler@6717
     8
philipp@8139
     9
  Permission is granted to anyone to use this software for any purpose,
philipp@8139
    10
  including commercial applications, and to alter it and redistribute it
philipp@8139
    11
  freely, subject to the following restrictions:
aschiffler@6717
    12
philipp@8139
    13
  1. The origin of this software must not be misrepresented; you must not
philipp@8139
    14
     claim that you wrote the original software. If you use this software
philipp@8139
    15
     in a product, an acknowledgment in the product documentation would be
philipp@8139
    16
     appreciated but is not required.
philipp@8139
    17
  2. Altered source versions must be plainly marked as such, and must not be
philipp@8139
    18
     misrepresented as being the original software.
philipp@8139
    19
  3. This notice may not be removed or altered from any source distribution.
aschiffler@6717
    20
*/
aschiffler@6717
    21
aschiffler@6717
    22
#include "SDL_config.h"
aschiffler@6717
    23
aschiffler@6717
    24
#include "SDL_test.h"
aschiffler@6717
    25
aschiffler@6721
    26
#include <stdio.h>
aschiffler@6721
    27
#include <stdlib.h>
aschiffler@6718
    28
#include <string.h>
aschiffler@6721
    29
#include <time.h>
aschiffler@6718
    30
aschiffler@6756
    31
/* Invalid test name/description message format */
aschiffler@6756
    32
const char *SDLTest_InvalidNameFormat = "(Invalid)";
aschiffler@6721
    33
aschiffler@6756
    34
/* Log summary message format */
aschiffler@6756
    35
const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d";
aschiffler@6756
    36
aschiffler@6756
    37
/* Final result message format */
aschiffler@6756
    38
const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n";
aschiffler@6721
    39
gabomdq@7678
    40
/* ! \brief Timeout for single test case execution */
aschiffler@6721
    41
static Uint32 SDLTest_TestCaseTimeout = 3600;
aschiffler@6717
    42
aschiffler@6717
    43
/**
aschiffler@6763
    44
* Generates a random run seed string for the harness. The generated seed
aschiffler@6763
    45
* will contain alphanumeric characters (0-9A-Z).
aschiffler@6763
    46
*
aschiffler@6763
    47
* Note: The returned string needs to be deallocated by the caller.
aschiffler@6763
    48
*
aschiffler@6763
    49
* \param length The length of the seed string to generate
aschiffler@6763
    50
*
aschiffler@6763
    51
* \returns The generated seed string
aschiffler@6763
    52
*/
aschiffler@6717
    53
char *
aschiffler@6772
    54
SDLTest_GenerateRunSeed(const int length)
aschiffler@6717
    55
{
slouken@7191
    56
    char *seed = NULL;
slouken@7191
    57
    SDLTest_RandomContext randomContext;
slouken@7191
    58
    int counter;
aschiffler@6717
    59
slouken@7191
    60
    /* Sanity check input */
slouken@7191
    61
    if (length <= 0) {
slouken@7191
    62
        SDLTest_LogError("The length of the harness seed must be >0.");
slouken@7191
    63
        return NULL;
slouken@7191
    64
    }
aschiffler@6717
    65
slouken@7191
    66
    /* Allocate output buffer */
slouken@7191
    67
    seed = (char *)SDL_malloc((length + 1) * sizeof(char));
slouken@7191
    68
    if (seed == NULL) {
slouken@7191
    69
        SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
aschiffler@8605
    70
        SDL_Error(SDL_ENOMEM);
slouken@7191
    71
        return NULL;
slouken@7191
    72
    }
aschiffler@6717
    73
slouken@7191
    74
    /* Generate a random string of alphanumeric characters */
slouken@7191
    75
    SDLTest_RandomInitTime(&randomContext);
aschiffler@8605
    76
    for (counter = 0; counter < length; counter++) {
slouken@7191
    77
        unsigned int number = SDLTest_Random(&randomContext);
slouken@7191
    78
        char ch = (char) (number % (91 - 48)) + 48;
slouken@7191
    79
        if (ch >= 58 && ch <= 64) {
slouken@7191
    80
            ch = 65;
slouken@7191
    81
        }
slouken@7191
    82
        seed[counter] = ch;
slouken@7191
    83
    }
aschiffler@8605
    84
    seed[length] = '\0';
aschiffler@6717
    85
slouken@7191
    86
    return seed;
aschiffler@6717
    87
}
aschiffler@6717
    88
aschiffler@6717
    89
/**
aschiffler@6763
    90
* Generates an execution key for the fuzzer.
aschiffler@6763
    91
*
slouken@7191
    92
* \param runSeed        The run seed to use
slouken@7191
    93
* \param suiteName      The name of the test suite
slouken@7191
    94
* \param testName       The name of the test
slouken@7191
    95
* \param iteration      The iteration count
aschiffler@6763
    96
*
aschiffler@6763
    97
* \returns The generated execution key to initialize the fuzzer with.
aschiffler@6763
    98
*
aschiffler@6763
    99
*/
aschiffler@6717
   100
Uint64
aschiffler@6772
   101
SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
aschiffler@6717
   102
{
slouken@7191
   103
    SDLTest_Md5Context md5Context;
slouken@7191
   104
    Uint64 *keys;
slouken@7191
   105
    char iterationString[16];
slouken@7191
   106
    Uint32 runSeedLength;
slouken@7191
   107
    Uint32 suiteNameLength;
slouken@7191
   108
    Uint32 testNameLength;
slouken@7191
   109
    Uint32 iterationStringLength;
slouken@7191
   110
    Uint32 entireStringLength;
slouken@7191
   111
    char *buffer;
aschiffler@6717
   112
slouken@7721
   113
    if (runSeed == NULL || runSeed[0] == '\0') {
slouken@7191
   114
        SDLTest_LogError("Invalid runSeed string.");
slouken@7191
   115
        return -1;
slouken@7191
   116
    }
aschiffler@6717
   117
slouken@7721
   118
    if (suiteName == NULL || suiteName[0] == '\0') {
slouken@7191
   119
        SDLTest_LogError("Invalid suiteName string.");
slouken@7191
   120
        return -1;
slouken@7191
   121
    }
aschiffler@6717
   122
slouken@7721
   123
    if (testName == NULL || testName[0] == '\0') {
slouken@7191
   124
        SDLTest_LogError("Invalid testName string.");
slouken@7191
   125
        return -1;
slouken@7191
   126
    }
aschiffler@6717
   127
slouken@7191
   128
    if (iteration <= 0) {
slouken@7191
   129
        SDLTest_LogError("Invalid iteration count.");
slouken@7191
   130
        return -1;
slouken@7191
   131
    }
aschiffler@6717
   132
slouken@7191
   133
    /* Convert iteration number into a string */
slouken@7191
   134
    SDL_memset(iterationString, 0, sizeof(iterationString));
slouken@7191
   135
    SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
aschiffler@6717
   136
slouken@7191
   137
    /* Combine the parameters into single string */
slouken@7191
   138
    runSeedLength = SDL_strlen(runSeed);
slouken@7191
   139
    suiteNameLength = SDL_strlen(suiteName);
slouken@7191
   140
    testNameLength = SDL_strlen(testName);
slouken@7191
   141
    iterationStringLength = SDL_strlen(iterationString);
slouken@7191
   142
    entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
slouken@7191
   143
    buffer = (char *)SDL_malloc(entireStringLength);
slouken@7191
   144
    if (buffer == NULL) {
aschiffler@8605
   145
        SDLTest_LogError("Failed to allocate buffer for execKey generation.");
aschiffler@8605
   146
        SDL_Error(SDL_ENOMEM);
slouken@7191
   147
        return 0;
slouken@7191
   148
    }
slouken@7191
   149
    SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
aschiffler@6717
   150
slouken@7191
   151
    /* Hash string and use half of the digest as 64bit exec key */
slouken@7191
   152
    SDLTest_Md5Init(&md5Context);
slouken@7191
   153
    SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
slouken@7191
   154
    SDLTest_Md5Final(&md5Context);
slouken@7191
   155
    SDL_free(buffer);
slouken@7191
   156
    keys = (Uint64 *)md5Context.digest;
aschiffler@6717
   157
slouken@7191
   158
    return keys[0];
aschiffler@6717
   159
}
aschiffler@6718
   160
aschiffler@6718
   161
/**
aschiffler@6763
   162
* \brief Set timeout handler for test.
aschiffler@6763
   163
*
aschiffler@6763
   164
* Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
aschiffler@6763
   165
*
aschiffler@6763
   166
* \param timeout Timeout interval in seconds.
aschiffler@6763
   167
* \param callback Function that will be called after timeout has elapsed.
slouken@7191
   168
*
aschiffler@6763
   169
* \return Timer id or -1 on failure.
aschiffler@6763
   170
*/
aschiffler@6718
   171
SDL_TimerID
aschiffler@6772
   172
SDLTest_SetTestTimeout(int timeout, void (*callback)())
aschiffler@6718
   173
{
slouken@7191
   174
    Uint32 timeoutInMilliseconds;
slouken@7191
   175
    SDL_TimerID timerID;
aschiffler@6718
   176
slouken@7191
   177
    if (callback == NULL) {
slouken@7191
   178
        SDLTest_LogError("Timeout callback can't be NULL");
slouken@7191
   179
        return -1;
slouken@7191
   180
    }
aschiffler@6718
   181
slouken@7191
   182
    if (timeout < 0) {
slouken@7191
   183
        SDLTest_LogError("Timeout value must be bigger than zero.");
slouken@7191
   184
        return -1;
slouken@7191
   185
    }
aschiffler@6718
   186
slouken@7191
   187
    /* Init SDL timer if not initialized before */
slouken@7191
   188
    if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
slouken@7191
   189
        if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
slouken@7191
   190
            SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
slouken@7191
   191
            return -1;
slouken@7191
   192
        }
slouken@7191
   193
    }
aschiffler@6718
   194
slouken@7191
   195
    /* Set timer */
slouken@7191
   196
    timeoutInMilliseconds = timeout * 1000;
slouken@7191
   197
    timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
slouken@7191
   198
    if (timerID == 0) {
slouken@7191
   199
        SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
slouken@7191
   200
        return -1;
slouken@7191
   201
    }
aschiffler@6718
   202
slouken@7191
   203
    return timerID;
aschiffler@6718
   204
}
aschiffler@6721
   205
aschiffler@6763
   206
/**
aschiffler@6763
   207
* \brief Timeout handler. Aborts test run and exits harness process.
aschiffler@6763
   208
*/
aschiffler@6721
   209
void
slouken@7191
   210
    SDLTest_BailOut()
aschiffler@6721
   211
{
slouken@7191
   212
    SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
slouken@7191
   213
    exit(TEST_ABORTED); /* bail out from the test */
aschiffler@6721
   214
}
aschiffler@6721
   215
aschiffler@6721
   216
/**
aschiffler@6763
   217
* \brief Execute a test using the given execution key.
aschiffler@6763
   218
*
aschiffler@6763
   219
* \param testSuite Suite containing the test case.
aschiffler@6763
   220
* \param testCase Case to execute.
aschiffler@6763
   221
* \param execKey Execution key for the fuzzer.
aschiffler@6763
   222
*
aschiffler@6763
   223
* \returns Test case result.
aschiffler@6763
   224
*/
aschiffler@6721
   225
int
aschiffler@6830
   226
SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
aschiffler@6721
   227
{
slouken@7191
   228
    SDL_TimerID timer = 0;
slouken@7191
   229
    int testCaseResult = 0;
slouken@7191
   230
    int testResult = 0;
slouken@7191
   231
    int fuzzerCount;
aschiffler@6721
   232
slouken@7191
   233
    if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
slouken@7191
   234
    {
slouken@7191
   235
        SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
slouken@7191
   236
        return TEST_RESULT_SETUP_FAILURE;
slouken@7191
   237
    }
aschiffler@6721
   238
slouken@7191
   239
    if (!testCase->enabled)
slouken@7191
   240
    {
slouken@7191
   241
        SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Disabled)");
slouken@7191
   242
        return TEST_RESULT_SKIPPED;
slouken@7191
   243
    }
aschiffler@6721
   244
aschiffler@6763
   245
slouken@7191
   246
    /* Initialize fuzzer */
slouken@7191
   247
    SDLTest_FuzzerInit(execKey);
aschiffler@6721
   248
slouken@7191
   249
    /* Reset assert tracker */
slouken@7191
   250
    SDLTest_ResetAssertSummary();
aschiffler@6721
   251
slouken@7191
   252
    /* Set timeout timer */
slouken@7191
   253
    timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
aschiffler@6721
   254
slouken@7191
   255
    /* Maybe run suite initalizer function */
slouken@7191
   256
    if (testSuite->testSetUp) {
slouken@7191
   257
        testSuite->testSetUp(0x0);
slouken@7191
   258
        if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
slouken@7191
   259
            SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed");
slouken@7191
   260
            return TEST_RESULT_SETUP_FAILURE;
slouken@7191
   261
        }
slouken@7191
   262
    }
aschiffler@6721
   263
slouken@7191
   264
    /* Run test case function */
slouken@7191
   265
    testCaseResult = testCase->testCase(0x0);
aschiffler@6721
   266
slouken@7191
   267
    /* Convert test execution result into harness result */
slouken@7191
   268
    if (testCaseResult == TEST_SKIPPED) {
slouken@7191
   269
        /* Test was programatically skipped */
slouken@7191
   270
        testResult = TEST_RESULT_SKIPPED;
slouken@7191
   271
    } else if (testCaseResult == TEST_STARTED) {
slouken@7191
   272
        /* Test did not return a TEST_COMPLETED value; assume it failed */
slouken@7191
   273
        testResult = TEST_RESULT_FAILED;
slouken@7191
   274
    } else if (testCaseResult == TEST_ABORTED) {
slouken@7191
   275
        /* Test was aborted early; assume it failed */
slouken@7191
   276
        testResult = TEST_RESULT_FAILED;
slouken@7191
   277
    } else {
slouken@7191
   278
        /* Perform failure analysis based on asserts */
slouken@7191
   279
        testResult = SDLTest_AssertSummaryToTestResult();
slouken@7191
   280
    }
aschiffler@6721
   281
slouken@7191
   282
    /* Maybe run suite cleanup function (ignore failed asserts) */
slouken@7191
   283
    if (testSuite->testTearDown) {
slouken@7191
   284
        testSuite->testTearDown(0x0);
slouken@7191
   285
    }
aschiffler@6721
   286
slouken@7191
   287
    /* Cancel timeout timer */
slouken@7191
   288
    if (timer) {
slouken@7191
   289
        SDL_RemoveTimer(timer);
slouken@7191
   290
    }
aschiffler@7189
   291
slouken@7191
   292
    /* Report on asserts and fuzzer usage */
slouken@7191
   293
    fuzzerCount = SDLTest_GetFuzzerInvocationCount();
slouken@7191
   294
    if (fuzzerCount > 0) {
slouken@7191
   295
        SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
slouken@7191
   296
    }
aschiffler@6721
   297
slouken@7191
   298
    /* Final log based on test execution result */
slouken@7191
   299
    if (testCaseResult == TEST_SKIPPED) {
slouken@7191
   300
        /* Test was programatically skipped */
slouken@7191
   301
        SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)");
slouken@7191
   302
    } else if (testCaseResult == TEST_STARTED) {
slouken@7191
   303
        /* Test did not return a TEST_COMPLETED value; assume it failed */
slouken@7191
   304
        SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
slouken@7191
   305
    } else if (testCaseResult == TEST_ABORTED) {
slouken@7191
   306
        /* Test was aborted early; assume it failed */
slouken@7191
   307
        SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)");
slouken@7191
   308
    } else {
slouken@7191
   309
        SDLTest_LogAssertSummary();
slouken@7191
   310
    }
slouken@7191
   311
slouken@7191
   312
    return testResult;
aschiffler@6721
   313
}
aschiffler@6721
   314
aschiffler@6721
   315
/* Prints summary of all suites/tests contained in the given reference */
aschiffler@6721
   316
void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
aschiffler@6721
   317
{
slouken@7191
   318
    int suiteCounter;
slouken@7191
   319
    int testCounter;
slouken@7191
   320
    SDLTest_TestSuiteReference *testSuite;
slouken@7191
   321
    SDLTest_TestCaseReference *testCase;
aschiffler@6721
   322
slouken@7191
   323
    /* Loop over all suites */
slouken@7191
   324
    suiteCounter = 0;
slouken@7191
   325
    while(&testSuites[suiteCounter]) {
slouken@7191
   326
        testSuite=&testSuites[suiteCounter];
slouken@7191
   327
        suiteCounter++;
slouken@7191
   328
        SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
slouken@7191
   329
            (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
aschiffler@6721
   330
slouken@7191
   331
        /* Loop over all test cases */
slouken@7191
   332
        testCounter = 0;
slouken@7191
   333
        while(testSuite->testCases[testCounter])
slouken@7191
   334
        {
slouken@7191
   335
            testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
slouken@7191
   336
            testCounter++;
slouken@7191
   337
            SDLTest_Log("  Test Case %i - %s: %s", testCounter,
slouken@7191
   338
                (testCase->name) ? testCase->name : SDLTest_InvalidNameFormat,
slouken@7191
   339
                (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
slouken@7191
   340
        }
slouken@7191
   341
    }
aschiffler@6721
   342
}
aschiffler@6721
   343
aschiffler@6756
   344
/* Gets a timer value in seconds */
aschiffler@6756
   345
float GetClock()
aschiffler@6756
   346
{
slouken@7191
   347
    float currentClock = (float)clock();
slouken@7191
   348
    return currentClock / (float)CLOCKS_PER_SEC;
aschiffler@6756
   349
}
aschiffler@6721
   350
aschiffler@6721
   351
/**
aschiffler@8605
   352
* \brief Execute a test suite using the given run seed and execution key.
aschiffler@6763
   353
*
aschiffler@6763
   354
* The filter string is matched to the suite name (full comparison) to select a single suite,
aschiffler@6763
   355
* or if no suite matches, it is matched to the test names (full comparison) to select a single test.
aschiffler@6763
   356
*
aschiffler@6763
   357
* \param testSuites Suites containing the test case.
aschiffler@6763
   358
* \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
aschiffler@6763
   359
* \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
aschiffler@6763
   360
* \param filter Filter specification. NULL disables. Case sensitive.
aschiffler@6763
   361
* \param testIterations Number of iterations to run each test case.
aschiffler@6763
   362
*
aschiffler@6763
   363
* \returns Test run result; 0 when all tests passed, 1 if any tests failed.
aschiffler@6763
   364
*/
slouken@6768
   365
int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
aschiffler@6721
   366
{
aschiffler@8605
   367
    int totalNumberOfTests = 0;
aschiffler@8605
   368
    int failedNumberOfTests = 0;
slouken@7191
   369
    int suiteCounter;
slouken@7191
   370
    int testCounter;
slouken@7191
   371
    int iterationCounter;
slouken@7191
   372
    SDLTest_TestSuiteReference *testSuite;
slouken@7191
   373
    SDLTest_TestCaseReference *testCase;
slouken@7191
   374
    const char *runSeed = NULL;
slouken@7191
   375
    char *currentSuiteName;
slouken@7191
   376
    char *currentTestName;
slouken@7191
   377
    Uint64 execKey;
slouken@7191
   378
    float runStartSeconds;
slouken@7191
   379
    float suiteStartSeconds;
slouken@7191
   380
    float testStartSeconds;
slouken@7191
   381
    float runEndSeconds;
slouken@7191
   382
    float suiteEndSeconds;
slouken@7191
   383
    float testEndSeconds;
slouken@7191
   384
    float runtime;
slouken@7191
   385
    int suiteFilter = 0;
slouken@7191
   386
    char *suiteFilterName = NULL;
slouken@7191
   387
    int testFilter = 0;
slouken@7191
   388
    char *testFilterName = NULL;
slouken@7191
   389
    int testResult = 0;
slouken@7191
   390
    int runResult = 0;
slouken@7191
   391
    Uint32 totalTestFailedCount = 0;
slouken@7191
   392
    Uint32 totalTestPassedCount = 0;
slouken@7191
   393
    Uint32 totalTestSkippedCount = 0;
slouken@7191
   394
    Uint32 testFailedCount = 0;
slouken@7191
   395
    Uint32 testPassedCount = 0;
slouken@7191
   396
    Uint32 testSkippedCount = 0;
slouken@7191
   397
    Uint32 countSum = 0;
slouken@7191
   398
    char *logFormat = (char *)SDLTest_LogSummaryFormat;
aschiffler@8605
   399
    SDLTest_TestCaseReference **failedTests;
aschiffler@6721
   400
slouken@7191
   401
    /* Sanitize test iterations */
slouken@7191
   402
    if (testIterations < 1) {
slouken@7191
   403
        testIterations = 1;
slouken@7191
   404
    }
aschiffler@6721
   405
slouken@7191
   406
    /* Generate run see if we don't have one already */
slouken@7721
   407
    if (userRunSeed == NULL || userRunSeed[0] == '\0') {
slouken@7191
   408
        runSeed = SDLTest_GenerateRunSeed(16);
slouken@7191
   409
        if (runSeed == NULL) {
slouken@7191
   410
            SDLTest_LogError("Generating a random seed failed");
slouken@7191
   411
            return 2;
slouken@7191
   412
        }
slouken@7191
   413
    } else {
slouken@7191
   414
        runSeed = userRunSeed;
slouken@7191
   415
    }
aschiffler@6721
   416
aschiffler@6763
   417
slouken@7191
   418
    /* Reset per-run counters */
slouken@7191
   419
    totalTestFailedCount = 0;
slouken@7191
   420
    totalTestPassedCount = 0;
slouken@7191
   421
    totalTestSkippedCount = 0;
aschiffler@6721
   422
slouken@7191
   423
    /* Take time - run start */
slouken@7191
   424
    runStartSeconds = GetClock();
aschiffler@6721
   425
slouken@7191
   426
    /* Log run with fuzzer parameters */
slouken@7191
   427
    SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
aschiffler@6721
   428
aschiffler@8605
   429
	/* Count the total number of tests */
aschiffler@8605
   430
    suiteCounter = 0;
aschiffler@8605
   431
    while (testSuites[suiteCounter]) {
aschiffler@8605
   432
        testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
aschiffler@8605
   433
        suiteCounter++;
aschiffler@8605
   434
        testCounter = 0;
aschiffler@8605
   435
        while (testSuite->testCases[testCounter])
aschiffler@8605
   436
        {
aschiffler@8605
   437
            testCounter++;
aschiffler@8605
   438
			totalNumberOfTests++;
aschiffler@8605
   439
		}
aschiffler@8605
   440
	}
aschiffler@8605
   441
aschiffler@8605
   442
	/* Pre-allocate an array for tracking failed tests (potentially all test cases) */
aschiffler@8605
   443
	failedTests = (SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
aschiffler@8605
   444
	if (failedTests == NULL) {	
aschiffler@8605
   445
	   SDLTest_LogError("Unable to allocate cache for failed tests");
aschiffler@8605
   446
           SDL_Error(SDL_ENOMEM);	   
aschiffler@8605
   447
           return -1;
aschiffler@8605
   448
	}
aschiffler@8605
   449
slouken@7191
   450
    /* Initialize filtering */
slouken@7721
   451
    if (filter != NULL && filter[0] != '\0') {
slouken@7191
   452
        /* Loop over all suites to check if we have a filter match */
slouken@7191
   453
        suiteCounter = 0;
slouken@7191
   454
        while (testSuites[suiteCounter] && suiteFilter == 0) {
slouken@7191
   455
            testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
slouken@7191
   456
            suiteCounter++;
slouken@7191
   457
            if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
slouken@7191
   458
                /* Matched a suite name */
slouken@7191
   459
                suiteFilter = 1;
slouken@7191
   460
                suiteFilterName = testSuite->name;
slouken@7191
   461
                SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
slouken@7191
   462
                break;
slouken@7191
   463
            }
aschiffler@6763
   464
slouken@7191
   465
            /* Within each suite, loop over all test cases to check if we have a filter match */
slouken@7191
   466
            testCounter = 0;
slouken@7191
   467
            while (testSuite->testCases[testCounter] && testFilter == 0)
slouken@7191
   468
            {
slouken@7191
   469
                testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
slouken@7191
   470
                testCounter++;
slouken@7191
   471
                if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
slouken@7191
   472
                    /* Matched a test name */
slouken@7191
   473
                    suiteFilter = 1;
slouken@7191
   474
                    suiteFilterName = testSuite->name;
slouken@7191
   475
                    testFilter = 1;
slouken@7191
   476
                    testFilterName = testCase->name;
slouken@7191
   477
                    SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
slouken@7191
   478
                    break;
slouken@7191
   479
                }
slouken@7191
   480
            }
slouken@7191
   481
        }
aschiffler@6763
   482
slouken@7191
   483
        if (suiteFilter == 0 && testFilter == 0) {
slouken@7191
   484
            SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
slouken@7191
   485
            SDLTest_Log("Exit code: 2");
slouken@7191
   486
            return 2;
slouken@7191
   487
        }
slouken@7191
   488
    }
aschiffler@6721
   489
slouken@7191
   490
    /* Loop over all suites */
slouken@7191
   491
    suiteCounter = 0;
slouken@7191
   492
    while(testSuites[suiteCounter]) {
slouken@7191
   493
        testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
slouken@7191
   494
        currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
slouken@7191
   495
        suiteCounter++;
aschiffler@6721
   496
slouken@7191
   497
        /* Filter suite if flag set and we have a name */
slouken@7191
   498
        if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
slouken@7191
   499
            SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
slouken@7191
   500
                /* Skip suite */
slouken@7191
   501
                SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
slouken@7191
   502
                    suiteCounter,
slouken@7191
   503
                    currentSuiteName);
slouken@7191
   504
        } else {
aschiffler@6721
   505
slouken@7191
   506
            /* Reset per-suite counters */
slouken@7191
   507
            testFailedCount = 0;
slouken@7191
   508
            testPassedCount = 0;
slouken@7191
   509
            testSkippedCount = 0;
aschiffler@6721
   510
slouken@7191
   511
            /* Take time - suite start */
slouken@7191
   512
            suiteStartSeconds = GetClock();
aschiffler@6721
   513
slouken@7191
   514
            /* Log suite started */
slouken@7191
   515
            SDLTest_Log("===== Test Suite %i: '%s' started\n",
slouken@7191
   516
                suiteCounter,
slouken@7191
   517
                currentSuiteName);
aschiffler@6721
   518
slouken@7191
   519
            /* Loop over all test cases */
slouken@7191
   520
            testCounter = 0;
slouken@7191
   521
            while(testSuite->testCases[testCounter])
slouken@7191
   522
            {
slouken@7191
   523
                testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
slouken@7191
   524
                currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
slouken@7191
   525
                testCounter++;
aschiffler@6721
   526
slouken@7191
   527
                /* Filter tests if flag set and we have a name */
slouken@7191
   528
                if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
slouken@7191
   529
                    SDL_strcmp(testFilterName, testCase->name) != 0) {
slouken@7191
   530
                        /* Skip test */
slouken@7191
   531
                        SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
slouken@7191
   532
                            suiteCounter,
slouken@7191
   533
                            testCounter,
slouken@7191
   534
                            currentTestName);
slouken@7191
   535
                } else {
slouken@7191
   536
                    /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
slouken@7191
   537
                    if (testFilter == 1 && !testCase->enabled) {
slouken@7191
   538
                        SDLTest_Log("Force run of disabled test since test filter was set");
slouken@7191
   539
                        testCase->enabled = 1;
slouken@7191
   540
                    }
aschiffler@6721
   541
slouken@7191
   542
                    /* Take time - test start */
slouken@7191
   543
                    testStartSeconds = GetClock();
aschiffler@6763
   544
slouken@7191
   545
                    /* Log test started */
slouken@7191
   546
                    SDLTest_Log("----- Test Case %i.%i: '%s' started",
slouken@7191
   547
                        suiteCounter,
slouken@7191
   548
                        testCounter,
slouken@7191
   549
                        currentTestName);
slouken@7721
   550
                    if (testCase->description != NULL && testCase->description[0] != '\0') {
slouken@7191
   551
                        SDLTest_Log("Test Description: '%s'",
slouken@7191
   552
                            (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
slouken@7191
   553
                    }
aschiffler@6763
   554
slouken@7191
   555
                    /* Loop over all iterations */
slouken@7191
   556
                    iterationCounter = 0;
slouken@7191
   557
                    while(iterationCounter < testIterations)
slouken@7191
   558
                    {
slouken@7191
   559
                        iterationCounter++;
aschiffler@6763
   560
slouken@7191
   561
                        if (userExecKey != 0) {
slouken@7191
   562
                            execKey = userExecKey;
slouken@7191
   563
                        } else {
slouken@7191
   564
                            execKey = SDLTest_GenerateExecKey((char *)runSeed, testSuite->name, testCase->name, iterationCounter);
slouken@7191
   565
                        }
aschiffler@6763
   566
slouken@7191
   567
                        SDLTest_Log("Test Iteration %i: execKey %llu", iterationCounter, execKey);
slouken@7191
   568
                        testResult = SDLTest_RunTest(testSuite, testCase, execKey);
aschiffler@6763
   569
slouken@7191
   570
                        if (testResult == TEST_RESULT_PASSED) {
slouken@7191
   571
                            testPassedCount++;
slouken@7191
   572
                            totalTestPassedCount++;
slouken@7191
   573
                        } else if (testResult == TEST_RESULT_SKIPPED) {
slouken@7191
   574
                            testSkippedCount++;
slouken@7191
   575
                            totalTestSkippedCount++;
slouken@7191
   576
                        } else {
slouken@7191
   577
                            testFailedCount++;
slouken@7191
   578
                            totalTestFailedCount++;
slouken@7191
   579
                        }
slouken@7191
   580
                    }
aschiffler@6763
   581
slouken@7191
   582
                    /* Take time - test end */
slouken@7191
   583
                    testEndSeconds = GetClock();
slouken@7191
   584
                    runtime = testEndSeconds - testStartSeconds;
slouken@7191
   585
                    if (runtime < 0.0f) runtime = 0.0f;
aschiffler@6763
   586
slouken@7191
   587
                    if (testIterations > 1) {
slouken@7191
   588
                        /* Log test runtime */
slouken@7191
   589
                        SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
slouken@7191
   590
                        SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
slouken@7191
   591
                    } else {
slouken@7191
   592
                        /* Log test runtime */
slouken@7191
   593
                        SDLTest_Log("Total Test runtime: %.1f sec", runtime);
slouken@7191
   594
                    }
aschiffler@6763
   595
slouken@7191
   596
                    /* Log final test result */
slouken@7191
   597
                    switch (testResult) {
slouken@7191
   598
                    case TEST_RESULT_PASSED:
slouken@7191
   599
                        SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
slouken@7191
   600
                        break;
slouken@7191
   601
                    case TEST_RESULT_FAILED:
slouken@7191
   602
                        SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
slouken@7191
   603
                        break;
slouken@7191
   604
                    case TEST_RESULT_NO_ASSERT:
slouken@7191
   605
                        SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
slouken@7191
   606
                        break;
slouken@7191
   607
                    }
aschiffler@6721
   608
aschiffler@8605
   609
                    /* Collect failed test case references for repro-step display */
aschiffler@8605
   610
                    if (testResult == TEST_RESULT_FAILED) {
aschiffler@8605
   611
                        failedTests[failedNumberOfTests] = testCase;
aschiffler@8605
   612
                        failedNumberOfTests++;
aschiffler@8605
   613
                    }
slouken@7191
   614
                }
slouken@7191
   615
            }
aschiffler@6721
   616
slouken@7191
   617
            /* Take time - suite end */
slouken@7191
   618
            suiteEndSeconds = GetClock();
slouken@7191
   619
            runtime = suiteEndSeconds - suiteStartSeconds;
slouken@7191
   620
            if (runtime < 0.0f) runtime = 0.0f;
aschiffler@6756
   621
slouken@7191
   622
            /* Log suite runtime */
slouken@7191
   623
            SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
aschiffler@6721
   624
slouken@7191
   625
            /* Log summary and final Suite result */
slouken@7191
   626
            countSum = testPassedCount + testFailedCount + testSkippedCount;
slouken@7191
   627
            if (testFailedCount == 0)
slouken@7191
   628
            {
slouken@7191
   629
                SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
slouken@7191
   630
                SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
slouken@7191
   631
            }
slouken@7191
   632
            else
slouken@7191
   633
            {
slouken@7191
   634
                SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
slouken@7191
   635
                SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
slouken@7191
   636
            }
aschiffler@6721
   637
slouken@7191
   638
        }
slouken@7191
   639
    }
aschiffler@6721
   640
slouken@7191
   641
    /* Take time - run end */
slouken@7191
   642
    runEndSeconds = GetClock();
slouken@7191
   643
    runtime = runEndSeconds - runStartSeconds;
slouken@7191
   644
    if (runtime < 0.0f) runtime = 0.0f;
aschiffler@6721
   645
slouken@7191
   646
    /* Log total runtime */
slouken@7191
   647
    SDLTest_Log("Total Run runtime: %.1f sec", runtime);
aschiffler@6756
   648
slouken@7191
   649
    /* Log summary and final run result */
slouken@7191
   650
    countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
slouken@7191
   651
    if (totalTestFailedCount == 0)
slouken@7191
   652
    {
slouken@7191
   653
        runResult = 0;
slouken@7191
   654
        SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
slouken@7191
   655
        SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
slouken@7191
   656
    }
slouken@7191
   657
    else
slouken@7191
   658
    {
slouken@7191
   659
        runResult = 1;
slouken@7191
   660
        SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
slouken@7191
   661
        SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
slouken@7191
   662
    }
slouken@7191
   663
aschiffler@8605
   664
    /* Print repro steps for failed tests */
aschiffler@8605
   665
    if (failedNumberOfTests > 0) {
aschiffler@8605
   666
        SDLTest_Log("Harness input to repro failures:");
aschiffler@8605
   667
        for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
aschiffler@8605
   668
          SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
aschiffler@8605
   669
        }
aschiffler@8605
   670
    }
aschiffler@8605
   671
    SDL_free(failedTests);
aschiffler@8605
   672
slouken@7191
   673
    SDLTest_Log("Exit code: %d", runResult);
slouken@7191
   674
    return runResult;
aschiffler@6721
   675
}