src/test/SDL_test_harness.c
author Sam Lantinga <slouken@libsdl.org>
Sun, 13 Nov 2016 23:09:42 -0800
changeset 10611 52ba31230c02
parent 10609 d702ecbd8ba7
child 10616 d17dd08640a4
permissions -rw-r--r--
Fixed build error with missing function prototype in the SDL_test_harness.h header
aschiffler@6717
     1
/*
philipp@8139
     2
  Simple DirectMedia Layer
slouken@9998
     3
  Copyright (C) 1997-2016 Sam Lantinga <slouken@libsdl.org>
aschiffler@6717
     4
philipp@8139
     5
  This software is provided 'as-is', without any express or implied
philipp@8139
     6
  warranty.  In no event will the authors be held liable for any damages
philipp@8139
     7
  arising from the use of this software.
aschiffler@6717
     8
philipp@8139
     9
  Permission is granted to anyone to use this software for any purpose,
philipp@8139
    10
  including commercial applications, and to alter it and redistribute it
philipp@8139
    11
  freely, subject to the following restrictions:
aschiffler@6717
    12
philipp@8139
    13
  1. The origin of this software must not be misrepresented; you must not
philipp@8139
    14
     claim that you wrote the original software. If you use this software
philipp@8139
    15
     in a product, an acknowledgment in the product documentation would be
philipp@8139
    16
     appreciated but is not required.
philipp@8139
    17
  2. Altered source versions must be plainly marked as such, and must not be
philipp@8139
    18
     misrepresented as being the original software.
philipp@8139
    19
  3. This notice may not be removed or altered from any source distribution.
aschiffler@6717
    20
*/
aschiffler@6717
    21
aschiffler@6717
    22
#include "SDL_config.h"
aschiffler@6717
    23
aschiffler@6717
    24
#include "SDL_test.h"
aschiffler@6717
    25
aschiffler@6721
    26
#include <stdio.h>
aschiffler@6721
    27
#include <stdlib.h>
aschiffler@6718
    28
#include <string.h>
aschiffler@6721
    29
#include <time.h>
aschiffler@6718
    30
aschiffler@6756
    31
/* Invalid test name/description message format */
philipp@10185
    32
#define SDLTEST_INVALID_NAME_FORMAT "(Invalid)"
aschiffler@6721
    33
aschiffler@6756
    34
/* Log summary message format */
philipp@10185
    35
#define SDLTEST_LOG_SUMMARY_FORMAT "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"
aschiffler@6756
    36
aschiffler@6756
    37
/* Final result message format */
philipp@10185
    38
#define SDLTEST_FINAL_RESULT_FORMAT ">>> %s '%s': %s\n"
aschiffler@6721
    39
gabomdq@7678
    40
/* ! \brief Timeout for single test case execution */
aschiffler@6721
    41
static Uint32 SDLTest_TestCaseTimeout = 3600;
aschiffler@6717
    42
aschiffler@6717
    43
/**
aschiffler@6763
    44
* Generates a random run seed string for the harness. The generated seed
aschiffler@6763
    45
* will contain alphanumeric characters (0-9A-Z).
aschiffler@6763
    46
*
aschiffler@6763
    47
* Note: The returned string needs to be deallocated by the caller.
aschiffler@6763
    48
*
aschiffler@6763
    49
* \param length The length of the seed string to generate
aschiffler@6763
    50
*
aschiffler@6763
    51
* \returns The generated seed string
aschiffler@6763
    52
*/
slouken@10611
    53
char *
aschiffler@6772
    54
SDLTest_GenerateRunSeed(const int length)
aschiffler@6717
    55
{
slouken@7191
    56
    char *seed = NULL;
slouken@7191
    57
    SDLTest_RandomContext randomContext;
slouken@7191
    58
    int counter;
aschiffler@6717
    59
slouken@7191
    60
    /* Sanity check input */
slouken@7191
    61
    if (length <= 0) {
slouken@7191
    62
        SDLTest_LogError("The length of the harness seed must be >0.");
slouken@7191
    63
        return NULL;
slouken@7191
    64
    }
aschiffler@6717
    65
slouken@7191
    66
    /* Allocate output buffer */
slouken@7191
    67
    seed = (char *)SDL_malloc((length + 1) * sizeof(char));
slouken@7191
    68
    if (seed == NULL) {
slouken@7191
    69
        SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
aschiffler@8605
    70
        SDL_Error(SDL_ENOMEM);
slouken@7191
    71
        return NULL;
slouken@7191
    72
    }
aschiffler@6717
    73
slouken@7191
    74
    /* Generate a random string of alphanumeric characters */
slouken@7191
    75
    SDLTest_RandomInitTime(&randomContext);
aschiffler@8605
    76
    for (counter = 0; counter < length; counter++) {
slouken@7191
    77
        unsigned int number = SDLTest_Random(&randomContext);
slouken@7191
    78
        char ch = (char) (number % (91 - 48)) + 48;
slouken@7191
    79
        if (ch >= 58 && ch <= 64) {
slouken@7191
    80
            ch = 65;
slouken@7191
    81
        }
slouken@7191
    82
        seed[counter] = ch;
slouken@7191
    83
    }
aschiffler@8605
    84
    seed[length] = '\0';
aschiffler@6717
    85
slouken@7191
    86
    return seed;
aschiffler@6717
    87
}
aschiffler@6717
    88
aschiffler@6717
    89
/**
aschiffler@6763
    90
* Generates an execution key for the fuzzer.
aschiffler@6763
    91
*
slouken@7191
    92
* \param runSeed        The run seed to use
slouken@7191
    93
* \param suiteName      The name of the test suite
slouken@7191
    94
* \param testName       The name of the test
slouken@7191
    95
* \param iteration      The iteration count
aschiffler@6763
    96
*
aschiffler@6763
    97
* \returns The generated execution key to initialize the fuzzer with.
aschiffler@6763
    98
*
aschiffler@6763
    99
*/
slouken@10609
   100
static Uint64
slouken@10609
   101
SDLTest_GenerateExecKey(const char *runSeed, char *suiteName, char *testName, int iteration)
aschiffler@6717
   102
{
slouken@7191
   103
    SDLTest_Md5Context md5Context;
slouken@7191
   104
    Uint64 *keys;
slouken@7191
   105
    char iterationString[16];
slouken@7191
   106
    Uint32 runSeedLength;
slouken@7191
   107
    Uint32 suiteNameLength;
slouken@7191
   108
    Uint32 testNameLength;
slouken@7191
   109
    Uint32 iterationStringLength;
slouken@7191
   110
    Uint32 entireStringLength;
slouken@7191
   111
    char *buffer;
aschiffler@6717
   112
slouken@7721
   113
    if (runSeed == NULL || runSeed[0] == '\0') {
slouken@7191
   114
        SDLTest_LogError("Invalid runSeed string.");
slouken@7191
   115
        return -1;
slouken@7191
   116
    }
aschiffler@6717
   117
slouken@7721
   118
    if (suiteName == NULL || suiteName[0] == '\0') {
slouken@7191
   119
        SDLTest_LogError("Invalid suiteName string.");
slouken@7191
   120
        return -1;
slouken@7191
   121
    }
aschiffler@6717
   122
slouken@7721
   123
    if (testName == NULL || testName[0] == '\0') {
slouken@7191
   124
        SDLTest_LogError("Invalid testName string.");
slouken@7191
   125
        return -1;
slouken@7191
   126
    }
aschiffler@6717
   127
slouken@7191
   128
    if (iteration <= 0) {
slouken@7191
   129
        SDLTest_LogError("Invalid iteration count.");
slouken@7191
   130
        return -1;
slouken@7191
   131
    }
aschiffler@6717
   132
slouken@7191
   133
    /* Convert iteration number into a string */
slouken@7191
   134
    SDL_memset(iterationString, 0, sizeof(iterationString));
slouken@7191
   135
    SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
aschiffler@6717
   136
slouken@7191
   137
    /* Combine the parameters into single string */
slouken@7191
   138
    runSeedLength = SDL_strlen(runSeed);
slouken@7191
   139
    suiteNameLength = SDL_strlen(suiteName);
slouken@7191
   140
    testNameLength = SDL_strlen(testName);
slouken@7191
   141
    iterationStringLength = SDL_strlen(iterationString);
slouken@7191
   142
    entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
slouken@7191
   143
    buffer = (char *)SDL_malloc(entireStringLength);
slouken@7191
   144
    if (buffer == NULL) {
aschiffler@8605
   145
        SDLTest_LogError("Failed to allocate buffer for execKey generation.");
aschiffler@8605
   146
        SDL_Error(SDL_ENOMEM);
slouken@7191
   147
        return 0;
slouken@7191
   148
    }
slouken@7191
   149
    SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
aschiffler@6717
   150
slouken@7191
   151
    /* Hash string and use half of the digest as 64bit exec key */
slouken@7191
   152
    SDLTest_Md5Init(&md5Context);
slouken@7191
   153
    SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
slouken@7191
   154
    SDLTest_Md5Final(&md5Context);
slouken@7191
   155
    SDL_free(buffer);
slouken@7191
   156
    keys = (Uint64 *)md5Context.digest;
aschiffler@6717
   157
slouken@7191
   158
    return keys[0];
aschiffler@6717
   159
}
aschiffler@6718
   160
aschiffler@6718
   161
/**
aschiffler@6763
   162
* \brief Set timeout handler for test.
aschiffler@6763
   163
*
aschiffler@6763
   164
* Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
aschiffler@6763
   165
*
aschiffler@6763
   166
* \param timeout Timeout interval in seconds.
aschiffler@6763
   167
* \param callback Function that will be called after timeout has elapsed.
slouken@7191
   168
*
aschiffler@6763
   169
* \return Timer id or -1 on failure.
aschiffler@6763
   170
*/
slouken@10609
   171
static SDL_TimerID
aschiffler@6772
   172
SDLTest_SetTestTimeout(int timeout, void (*callback)())
aschiffler@6718
   173
{
slouken@7191
   174
    Uint32 timeoutInMilliseconds;
slouken@7191
   175
    SDL_TimerID timerID;
aschiffler@6718
   176
slouken@7191
   177
    if (callback == NULL) {
slouken@7191
   178
        SDLTest_LogError("Timeout callback can't be NULL");
slouken@7191
   179
        return -1;
slouken@7191
   180
    }
aschiffler@6718
   181
slouken@7191
   182
    if (timeout < 0) {
slouken@7191
   183
        SDLTest_LogError("Timeout value must be bigger than zero.");
slouken@7191
   184
        return -1;
slouken@7191
   185
    }
aschiffler@6718
   186
slouken@7191
   187
    /* Init SDL timer if not initialized before */
slouken@7191
   188
    if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
slouken@7191
   189
        if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
slouken@7191
   190
            SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
slouken@7191
   191
            return -1;
slouken@7191
   192
        }
slouken@7191
   193
    }
aschiffler@6718
   194
slouken@7191
   195
    /* Set timer */
slouken@7191
   196
    timeoutInMilliseconds = timeout * 1000;
slouken@7191
   197
    timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
slouken@7191
   198
    if (timerID == 0) {
slouken@7191
   199
        SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
slouken@7191
   200
        return -1;
slouken@7191
   201
    }
aschiffler@6718
   202
slouken@7191
   203
    return timerID;
aschiffler@6718
   204
}
aschiffler@6721
   205
aschiffler@6763
   206
/**
aschiffler@6763
   207
* \brief Timeout handler. Aborts test run and exits harness process.
aschiffler@6763
   208
*/
slouken@10609
   209
static void
slouken@10609
   210
SDLTest_BailOut()
aschiffler@6721
   211
{
slouken@7191
   212
    SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
slouken@7191
   213
    exit(TEST_ABORTED); /* bail out from the test */
aschiffler@6721
   214
}
aschiffler@6721
   215
aschiffler@6721
   216
/**
aschiffler@6763
   217
* \brief Execute a test using the given execution key.
aschiffler@6763
   218
*
aschiffler@6763
   219
* \param testSuite Suite containing the test case.
aschiffler@6763
   220
* \param testCase Case to execute.
aschiffler@6763
   221
* \param execKey Execution key for the fuzzer.
aschiffler@8975
   222
* \param forceTestRun Force test to run even if test was disabled in suite.
aschiffler@6763
   223
*
aschiffler@6763
   224
* \returns Test case result.
aschiffler@6763
   225
*/
slouken@10609
   226
static int
slouken@10609
   227
SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
aschiffler@6721
   228
{
slouken@7191
   229
    SDL_TimerID timer = 0;
slouken@7191
   230
    int testCaseResult = 0;
slouken@7191
   231
    int testResult = 0;
slouken@7191
   232
    int fuzzerCount;
aschiffler@6721
   233
slouken@7191
   234
    if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
slouken@7191
   235
    {
slouken@7191
   236
        SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
slouken@7191
   237
        return TEST_RESULT_SETUP_FAILURE;
slouken@7191
   238
    }
aschiffler@6721
   239
aschiffler@8975
   240
	if (!testCase->enabled && forceTestRun == SDL_FALSE)
slouken@7191
   241
    {
philipp@10185
   242
        SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Disabled)");
slouken@7191
   243
        return TEST_RESULT_SKIPPED;
slouken@7191
   244
    }
aschiffler@6721
   245
slouken@7191
   246
    /* Initialize fuzzer */
slouken@7191
   247
    SDLTest_FuzzerInit(execKey);
aschiffler@6721
   248
slouken@7191
   249
    /* Reset assert tracker */
slouken@7191
   250
    SDLTest_ResetAssertSummary();
aschiffler@6721
   251
slouken@7191
   252
    /* Set timeout timer */
slouken@7191
   253
    timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
aschiffler@6721
   254
slouken@7191
   255
    /* Maybe run suite initalizer function */
slouken@7191
   256
    if (testSuite->testSetUp) {
slouken@7191
   257
        testSuite->testSetUp(0x0);
slouken@7191
   258
        if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
philipp@10185
   259
            SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite Setup", testSuite->name, "Failed");
slouken@7191
   260
            return TEST_RESULT_SETUP_FAILURE;
slouken@7191
   261
        }
slouken@7191
   262
    }
aschiffler@6721
   263
slouken@7191
   264
    /* Run test case function */
slouken@7191
   265
    testCaseResult = testCase->testCase(0x0);
aschiffler@6721
   266
slouken@7191
   267
    /* Convert test execution result into harness result */
slouken@7191
   268
    if (testCaseResult == TEST_SKIPPED) {
slouken@7191
   269
        /* Test was programatically skipped */
slouken@7191
   270
        testResult = TEST_RESULT_SKIPPED;
slouken@7191
   271
    } else if (testCaseResult == TEST_STARTED) {
slouken@7191
   272
        /* Test did not return a TEST_COMPLETED value; assume it failed */
slouken@7191
   273
        testResult = TEST_RESULT_FAILED;
slouken@7191
   274
    } else if (testCaseResult == TEST_ABORTED) {
slouken@7191
   275
        /* Test was aborted early; assume it failed */
slouken@7191
   276
        testResult = TEST_RESULT_FAILED;
slouken@7191
   277
    } else {
slouken@7191
   278
        /* Perform failure analysis based on asserts */
slouken@7191
   279
        testResult = SDLTest_AssertSummaryToTestResult();
slouken@7191
   280
    }
aschiffler@6721
   281
slouken@7191
   282
    /* Maybe run suite cleanup function (ignore failed asserts) */
slouken@7191
   283
    if (testSuite->testTearDown) {
slouken@7191
   284
        testSuite->testTearDown(0x0);
slouken@7191
   285
    }
aschiffler@6721
   286
slouken@7191
   287
    /* Cancel timeout timer */
slouken@7191
   288
    if (timer) {
slouken@7191
   289
        SDL_RemoveTimer(timer);
slouken@7191
   290
    }
aschiffler@7189
   291
slouken@7191
   292
    /* Report on asserts and fuzzer usage */
slouken@7191
   293
    fuzzerCount = SDLTest_GetFuzzerInvocationCount();
slouken@7191
   294
    if (fuzzerCount > 0) {
slouken@7191
   295
        SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
slouken@7191
   296
    }
aschiffler@6721
   297
slouken@7191
   298
    /* Final log based on test execution result */
slouken@7191
   299
    if (testCaseResult == TEST_SKIPPED) {
slouken@7191
   300
        /* Test was programatically skipped */
philipp@10185
   301
        SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Programmatically)");
slouken@7191
   302
    } else if (testCaseResult == TEST_STARTED) {
slouken@7191
   303
        /* Test did not return a TEST_COMPLETED value; assume it failed */
philipp@10185
   304
        SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
slouken@7191
   305
    } else if (testCaseResult == TEST_ABORTED) {
slouken@7191
   306
        /* Test was aborted early; assume it failed */
philipp@10185
   307
        SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (Aborted)");
slouken@7191
   308
    } else {
slouken@7191
   309
        SDLTest_LogAssertSummary();
slouken@7191
   310
    }
slouken@7191
   311
slouken@7191
   312
    return testResult;
aschiffler@6721
   313
}
aschiffler@6721
   314
aschiffler@6721
   315
/* Prints summary of all suites/tests contained in the given reference */
slouken@10609
   316
#if 0 
slouken@10609
   317
static void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
aschiffler@6721
   318
{
slouken@7191
   319
    int suiteCounter;
slouken@7191
   320
    int testCounter;
slouken@7191
   321
    SDLTest_TestSuiteReference *testSuite;
slouken@7191
   322
    SDLTest_TestCaseReference *testCase;
aschiffler@6721
   323
slouken@7191
   324
    /* Loop over all suites */
slouken@7191
   325
    suiteCounter = 0;
slouken@7191
   326
    while(&testSuites[suiteCounter]) {
slouken@7191
   327
        testSuite=&testSuites[suiteCounter];
slouken@7191
   328
        suiteCounter++;
slouken@7191
   329
        SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
philipp@10185
   330
            (testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
aschiffler@6721
   331
slouken@7191
   332
        /* Loop over all test cases */
slouken@7191
   333
        testCounter = 0;
slouken@7191
   334
        while(testSuite->testCases[testCounter])
slouken@7191
   335
        {
slouken@7191
   336
            testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
slouken@7191
   337
            testCounter++;
slouken@7191
   338
            SDLTest_Log("  Test Case %i - %s: %s", testCounter,
philipp@10185
   339
                (testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT,
philipp@10185
   340
                (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
slouken@7191
   341
        }
slouken@7191
   342
    }
aschiffler@6721
   343
}
slouken@10609
   344
#endif
aschiffler@6721
   345
aschiffler@6756
   346
/* Gets a timer value in seconds */
slouken@10609
   347
static float GetClock()
aschiffler@6756
   348
{
slouken@10609
   349
    float currentClock = clock() / (float) CLOCKS_PER_SEC;
slouken@10609
   350
    return currentClock;
aschiffler@6756
   351
}
aschiffler@6721
   352
aschiffler@6721
   353
/**
aschiffler@8605
   354
* \brief Execute a test suite using the given run seed and execution key.
aschiffler@6763
   355
*
aschiffler@6763
   356
* The filter string is matched to the suite name (full comparison) to select a single suite,
aschiffler@6763
   357
* or if no suite matches, it is matched to the test names (full comparison) to select a single test.
aschiffler@6763
   358
*
aschiffler@6763
   359
* \param testSuites Suites containing the test case.
aschiffler@6763
   360
* \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
aschiffler@6763
   361
* \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
aschiffler@6763
   362
* \param filter Filter specification. NULL disables. Case sensitive.
aschiffler@6763
   363
* \param testIterations Number of iterations to run each test case.
aschiffler@6763
   364
*
aschiffler@6763
   365
* \returns Test run result; 0 when all tests passed, 1 if any tests failed.
aschiffler@6763
   366
*/
slouken@6768
   367
int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
aschiffler@6721
   368
{
aschiffler@8605
   369
    int totalNumberOfTests = 0;
aschiffler@8605
   370
    int failedNumberOfTests = 0;
slouken@7191
   371
    int suiteCounter;
slouken@7191
   372
    int testCounter;
slouken@7191
   373
    int iterationCounter;
slouken@7191
   374
    SDLTest_TestSuiteReference *testSuite;
slouken@10609
   375
    const SDLTest_TestCaseReference *testCase;
slouken@7191
   376
    const char *runSeed = NULL;
slouken@7191
   377
    char *currentSuiteName;
slouken@7191
   378
    char *currentTestName;
slouken@7191
   379
    Uint64 execKey;
slouken@7191
   380
    float runStartSeconds;
slouken@7191
   381
    float suiteStartSeconds;
slouken@7191
   382
    float testStartSeconds;
slouken@7191
   383
    float runEndSeconds;
slouken@7191
   384
    float suiteEndSeconds;
slouken@7191
   385
    float testEndSeconds;
slouken@7191
   386
    float runtime;
slouken@7191
   387
    int suiteFilter = 0;
slouken@7191
   388
    char *suiteFilterName = NULL;
slouken@7191
   389
    int testFilter = 0;
slouken@7191
   390
    char *testFilterName = NULL;
aschiffler@8975
   391
	SDL_bool forceTestRun = SDL_FALSE;
slouken@7191
   392
    int testResult = 0;
slouken@7191
   393
    int runResult = 0;
slouken@7191
   394
    Uint32 totalTestFailedCount = 0;
slouken@7191
   395
    Uint32 totalTestPassedCount = 0;
slouken@7191
   396
    Uint32 totalTestSkippedCount = 0;
slouken@7191
   397
    Uint32 testFailedCount = 0;
slouken@7191
   398
    Uint32 testPassedCount = 0;
slouken@7191
   399
    Uint32 testSkippedCount = 0;
slouken@7191
   400
    Uint32 countSum = 0;
slouken@10609
   401
    const SDLTest_TestCaseReference **failedTests;
aschiffler@6721
   402
slouken@7191
   403
    /* Sanitize test iterations */
slouken@7191
   404
    if (testIterations < 1) {
slouken@7191
   405
        testIterations = 1;
slouken@7191
   406
    }
aschiffler@6721
   407
slouken@7191
   408
    /* Generate run see if we don't have one already */
slouken@7721
   409
    if (userRunSeed == NULL || userRunSeed[0] == '\0') {
slouken@7191
   410
        runSeed = SDLTest_GenerateRunSeed(16);
slouken@7191
   411
        if (runSeed == NULL) {
slouken@7191
   412
            SDLTest_LogError("Generating a random seed failed");
slouken@7191
   413
            return 2;
slouken@7191
   414
        }
slouken@7191
   415
    } else {
slouken@7191
   416
        runSeed = userRunSeed;
slouken@7191
   417
    }
aschiffler@6721
   418
aschiffler@6763
   419
slouken@7191
   420
    /* Reset per-run counters */
slouken@7191
   421
    totalTestFailedCount = 0;
slouken@7191
   422
    totalTestPassedCount = 0;
slouken@7191
   423
    totalTestSkippedCount = 0;
aschiffler@6721
   424
slouken@7191
   425
    /* Take time - run start */
slouken@7191
   426
    runStartSeconds = GetClock();
aschiffler@6721
   427
slouken@7191
   428
    /* Log run with fuzzer parameters */
slouken@7191
   429
    SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
aschiffler@6721
   430
aschiffler@8605
   431
	/* Count the total number of tests */
aschiffler@8605
   432
    suiteCounter = 0;
aschiffler@8605
   433
    while (testSuites[suiteCounter]) {
aschiffler@8605
   434
        testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
aschiffler@8605
   435
        suiteCounter++;
aschiffler@8605
   436
        testCounter = 0;
aschiffler@8605
   437
        while (testSuite->testCases[testCounter])
aschiffler@8605
   438
        {
aschiffler@8605
   439
            testCounter++;
aschiffler@8605
   440
			totalNumberOfTests++;
aschiffler@8605
   441
		}
aschiffler@8605
   442
	}
aschiffler@8605
   443
aschiffler@8605
   444
	/* Pre-allocate an array for tracking failed tests (potentially all test cases) */
slouken@10609
   445
	failedTests = (const SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
aschiffler@8605
   446
	if (failedTests == NULL) {	
aschiffler@8605
   447
	   SDLTest_LogError("Unable to allocate cache for failed tests");
aschiffler@8605
   448
           SDL_Error(SDL_ENOMEM);	   
aschiffler@8605
   449
           return -1;
aschiffler@8605
   450
	}
aschiffler@8605
   451
slouken@7191
   452
    /* Initialize filtering */
slouken@7721
   453
    if (filter != NULL && filter[0] != '\0') {
slouken@7191
   454
        /* Loop over all suites to check if we have a filter match */
slouken@7191
   455
        suiteCounter = 0;
slouken@7191
   456
        while (testSuites[suiteCounter] && suiteFilter == 0) {
slouken@7191
   457
            testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
slouken@7191
   458
            suiteCounter++;
slouken@7191
   459
            if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
slouken@7191
   460
                /* Matched a suite name */
slouken@7191
   461
                suiteFilter = 1;
slouken@7191
   462
                suiteFilterName = testSuite->name;
slouken@7191
   463
                SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
slouken@7191
   464
                break;
slouken@7191
   465
            }
aschiffler@6763
   466
slouken@7191
   467
            /* Within each suite, loop over all test cases to check if we have a filter match */
slouken@7191
   468
            testCounter = 0;
slouken@7191
   469
            while (testSuite->testCases[testCounter] && testFilter == 0)
slouken@7191
   470
            {
slouken@10609
   471
                testCase = testSuite->testCases[testCounter];
slouken@7191
   472
                testCounter++;
slouken@7191
   473
                if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
slouken@7191
   474
                    /* Matched a test name */
slouken@7191
   475
                    suiteFilter = 1;
slouken@7191
   476
                    suiteFilterName = testSuite->name;
slouken@7191
   477
                    testFilter = 1;
slouken@7191
   478
                    testFilterName = testCase->name;
slouken@7191
   479
                    SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
slouken@7191
   480
                    break;
slouken@7191
   481
                }
slouken@7191
   482
            }
slouken@7191
   483
        }
aschiffler@6763
   484
slouken@7191
   485
        if (suiteFilter == 0 && testFilter == 0) {
slouken@7191
   486
            SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
slouken@7191
   487
            SDLTest_Log("Exit code: 2");
philipp@9798
   488
            SDL_free(failedTests);
slouken@7191
   489
            return 2;
slouken@7191
   490
        }
slouken@7191
   491
    }
aschiffler@6721
   492
slouken@7191
   493
    /* Loop over all suites */
slouken@7191
   494
    suiteCounter = 0;
slouken@7191
   495
    while(testSuites[suiteCounter]) {
slouken@7191
   496
        testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
philipp@10185
   497
        currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
slouken@7191
   498
        suiteCounter++;
aschiffler@6721
   499
slouken@7191
   500
        /* Filter suite if flag set and we have a name */
slouken@7191
   501
        if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
slouken@7191
   502
            SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
slouken@7191
   503
                /* Skip suite */
slouken@7191
   504
                SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
slouken@7191
   505
                    suiteCounter,
slouken@7191
   506
                    currentSuiteName);
slouken@7191
   507
        } else {
aschiffler@6721
   508
slouken@7191
   509
            /* Reset per-suite counters */
slouken@7191
   510
            testFailedCount = 0;
slouken@7191
   511
            testPassedCount = 0;
slouken@7191
   512
            testSkippedCount = 0;
aschiffler@6721
   513
slouken@7191
   514
            /* Take time - suite start */
slouken@7191
   515
            suiteStartSeconds = GetClock();
aschiffler@6721
   516
slouken@7191
   517
            /* Log suite started */
slouken@7191
   518
            SDLTest_Log("===== Test Suite %i: '%s' started\n",
slouken@7191
   519
                suiteCounter,
slouken@7191
   520
                currentSuiteName);
aschiffler@6721
   521
slouken@7191
   522
            /* Loop over all test cases */
slouken@7191
   523
            testCounter = 0;
slouken@7191
   524
            while(testSuite->testCases[testCounter])
slouken@7191
   525
            {
slouken@10609
   526
                testCase = testSuite->testCases[testCounter];
philipp@10185
   527
                currentTestName = (char *)((testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
slouken@7191
   528
                testCounter++;
aschiffler@6721
   529
slouken@7191
   530
                /* Filter tests if flag set and we have a name */
slouken@7191
   531
                if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
slouken@7191
   532
                    SDL_strcmp(testFilterName, testCase->name) != 0) {
slouken@7191
   533
                        /* Skip test */
slouken@7191
   534
                        SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
slouken@7191
   535
                            suiteCounter,
slouken@7191
   536
                            testCounter,
slouken@7191
   537
                            currentTestName);
slouken@7191
   538
                } else {
slouken@7191
   539
                    /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
slouken@7191
   540
                    if (testFilter == 1 && !testCase->enabled) {
slouken@7191
   541
                        SDLTest_Log("Force run of disabled test since test filter was set");
aschiffler@8975
   542
						forceTestRun = SDL_TRUE;
slouken@7191
   543
                    }
aschiffler@6721
   544
slouken@7191
   545
                    /* Take time - test start */
slouken@7191
   546
                    testStartSeconds = GetClock();
aschiffler@6763
   547
slouken@7191
   548
                    /* Log test started */
slouken@7191
   549
                    SDLTest_Log("----- Test Case %i.%i: '%s' started",
slouken@7191
   550
                        suiteCounter,
slouken@7191
   551
                        testCounter,
slouken@7191
   552
                        currentTestName);
slouken@7721
   553
                    if (testCase->description != NULL && testCase->description[0] != '\0') {
slouken@7191
   554
                        SDLTest_Log("Test Description: '%s'",
philipp@10185
   555
                            (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
slouken@7191
   556
                    }
aschiffler@6763
   557
slouken@7191
   558
                    /* Loop over all iterations */
slouken@7191
   559
                    iterationCounter = 0;
slouken@7191
   560
                    while(iterationCounter < testIterations)
slouken@7191
   561
                    {
slouken@7191
   562
                        iterationCounter++;
aschiffler@6763
   563
slouken@7191
   564
                        if (userExecKey != 0) {
slouken@7191
   565
                            execKey = userExecKey;
slouken@7191
   566
                        } else {
slouken@10609
   567
                            execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
slouken@7191
   568
                        }
aschiffler@6763
   569
slouken@8917
   570
                        SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
aschiffler@8975
   571
						testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
aschiffler@6763
   572
slouken@7191
   573
                        if (testResult == TEST_RESULT_PASSED) {
slouken@7191
   574
                            testPassedCount++;
slouken@7191
   575
                            totalTestPassedCount++;
slouken@7191
   576
                        } else if (testResult == TEST_RESULT_SKIPPED) {
slouken@7191
   577
                            testSkippedCount++;
slouken@7191
   578
                            totalTestSkippedCount++;
slouken@7191
   579
                        } else {
slouken@7191
   580
                            testFailedCount++;
slouken@7191
   581
                            totalTestFailedCount++;
slouken@7191
   582
                        }
slouken@7191
   583
                    }
aschiffler@6763
   584
slouken@7191
   585
                    /* Take time - test end */
slouken@7191
   586
                    testEndSeconds = GetClock();
slouken@7191
   587
                    runtime = testEndSeconds - testStartSeconds;
slouken@7191
   588
                    if (runtime < 0.0f) runtime = 0.0f;
aschiffler@6763
   589
slouken@7191
   590
                    if (testIterations > 1) {
slouken@7191
   591
                        /* Log test runtime */
slouken@7191
   592
                        SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
slouken@7191
   593
                        SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
slouken@7191
   594
                    } else {
slouken@7191
   595
                        /* Log test runtime */
slouken@7191
   596
                        SDLTest_Log("Total Test runtime: %.1f sec", runtime);
slouken@7191
   597
                    }
aschiffler@6763
   598
slouken@7191
   599
                    /* Log final test result */
slouken@7191
   600
                    switch (testResult) {
slouken@7191
   601
                    case TEST_RESULT_PASSED:
philipp@10185
   602
                        SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed");
slouken@7191
   603
                        break;
slouken@7191
   604
                    case TEST_RESULT_FAILED:
philipp@10185
   605
                        SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed");
slouken@7191
   606
                        break;
slouken@7191
   607
                    case TEST_RESULT_NO_ASSERT:
philipp@10185
   608
                        SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts");
slouken@7191
   609
                        break;
slouken@7191
   610
                    }
aschiffler@6721
   611
aschiffler@8605
   612
                    /* Collect failed test case references for repro-step display */
aschiffler@8605
   613
                    if (testResult == TEST_RESULT_FAILED) {
aschiffler@8605
   614
                        failedTests[failedNumberOfTests] = testCase;
aschiffler@8605
   615
                        failedNumberOfTests++;
aschiffler@8605
   616
                    }
slouken@7191
   617
                }
slouken@7191
   618
            }
aschiffler@6721
   619
slouken@7191
   620
            /* Take time - suite end */
slouken@7191
   621
            suiteEndSeconds = GetClock();
slouken@7191
   622
            runtime = suiteEndSeconds - suiteStartSeconds;
slouken@7191
   623
            if (runtime < 0.0f) runtime = 0.0f;
aschiffler@6756
   624
slouken@7191
   625
            /* Log suite runtime */
slouken@7191
   626
            SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
aschiffler@6721
   627
slouken@7191
   628
            /* Log summary and final Suite result */
slouken@7191
   629
            countSum = testPassedCount + testFailedCount + testSkippedCount;
slouken@7191
   630
            if (testFailedCount == 0)
slouken@7191
   631
            {
philipp@10185
   632
                SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
philipp@10185
   633
                SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed");
slouken@7191
   634
            }
slouken@7191
   635
            else
slouken@7191
   636
            {
philipp@10185
   637
                SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
philipp@10185
   638
                SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed");
slouken@7191
   639
            }
aschiffler@6721
   640
slouken@7191
   641
        }
slouken@7191
   642
    }
aschiffler@6721
   643
slouken@7191
   644
    /* Take time - run end */
slouken@7191
   645
    runEndSeconds = GetClock();
slouken@7191
   646
    runtime = runEndSeconds - runStartSeconds;
slouken@7191
   647
    if (runtime < 0.0f) runtime = 0.0f;
aschiffler@6721
   648
slouken@7191
   649
    /* Log total runtime */
slouken@7191
   650
    SDLTest_Log("Total Run runtime: %.1f sec", runtime);
aschiffler@6756
   651
slouken@7191
   652
    /* Log summary and final run result */
slouken@7191
   653
    countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
slouken@7191
   654
    if (totalTestFailedCount == 0)
slouken@7191
   655
    {
slouken@7191
   656
        runResult = 0;
philipp@10185
   657
        SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
philipp@10185
   658
        SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed");
slouken@7191
   659
    }
slouken@7191
   660
    else
slouken@7191
   661
    {
slouken@7191
   662
        runResult = 1;
philipp@10185
   663
        SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
philipp@10185
   664
        SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed");
slouken@7191
   665
    }
slouken@7191
   666
aschiffler@8605
   667
    /* Print repro steps for failed tests */
aschiffler@8605
   668
    if (failedNumberOfTests > 0) {
aschiffler@8605
   669
        SDLTest_Log("Harness input to repro failures:");
aschiffler@8605
   670
        for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
aschiffler@8605
   671
          SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
aschiffler@8605
   672
        }
aschiffler@8605
   673
    }
aschiffler@8605
   674
    SDL_free(failedTests);
aschiffler@8605
   675
slouken@7191
   676
    SDLTest_Log("Exit code: %d", runResult);
slouken@7191
   677
    return runResult;
aschiffler@6721
   678
}