src/test/SDL_test_harness.c
author Philipp Wiesemann <philipp.wiesemann@arcor.de>
Wed, 29 Jan 2014 00:29:14 +0100
changeset 8139 ad675d62631b
parent 7721 bdf8c2c84e14
child 8149 681eb46b8ac4
permissions -rw-r--r--
Fixed spaces in source file license comment.
aschiffler@6717
     1
/*
philipp@8139
     2
  Simple DirectMedia Layer
philipp@8139
     3
  Copyright (C) 1997-2013 Sam Lantinga <slouken@libsdl.org>
aschiffler@6717
     4
philipp@8139
     5
  This software is provided 'as-is', without any express or implied
philipp@8139
     6
  warranty.  In no event will the authors be held liable for any damages
philipp@8139
     7
  arising from the use of this software.
aschiffler@6717
     8
philipp@8139
     9
  Permission is granted to anyone to use this software for any purpose,
philipp@8139
    10
  including commercial applications, and to alter it and redistribute it
philipp@8139
    11
  freely, subject to the following restrictions:
aschiffler@6717
    12
philipp@8139
    13
  1. The origin of this software must not be misrepresented; you must not
philipp@8139
    14
     claim that you wrote the original software. If you use this software
philipp@8139
    15
     in a product, an acknowledgment in the product documentation would be
philipp@8139
    16
     appreciated but is not required.
philipp@8139
    17
  2. Altered source versions must be plainly marked as such, and must not be
philipp@8139
    18
     misrepresented as being the original software.
philipp@8139
    19
  3. This notice may not be removed or altered from any source distribution.
aschiffler@6717
    20
*/
aschiffler@6717
    21
aschiffler@6717
    22
#include "SDL_config.h"
aschiffler@6717
    23
aschiffler@6717
    24
#include "SDL_test.h"
aschiffler@6717
    25
aschiffler@6721
    26
#include <stdio.h>
aschiffler@6721
    27
#include <stdlib.h>
aschiffler@6718
    28
#include <string.h>
aschiffler@6721
    29
#include <time.h>
aschiffler@6718
    30
aschiffler@6756
    31
/* Invalid test name/description message format */
aschiffler@6756
    32
const char *SDLTest_InvalidNameFormat = "(Invalid)";
aschiffler@6721
    33
aschiffler@6756
    34
/* Log summary message format */
aschiffler@6756
    35
const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d";
aschiffler@6756
    36
aschiffler@6756
    37
/* Final result message format */
aschiffler@6756
    38
const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n";
aschiffler@6721
    39
gabomdq@7678
    40
/* ! \brief Timeout for single test case execution */
aschiffler@6721
    41
static Uint32 SDLTest_TestCaseTimeout = 3600;
aschiffler@6717
    42
aschiffler@6717
    43
/**
aschiffler@6763
    44
* Generates a random run seed string for the harness. The generated seed
aschiffler@6763
    45
* will contain alphanumeric characters (0-9A-Z).
aschiffler@6763
    46
*
aschiffler@6763
    47
* Note: The returned string needs to be deallocated by the caller.
aschiffler@6763
    48
*
aschiffler@6763
    49
* \param length The length of the seed string to generate
aschiffler@6763
    50
*
aschiffler@6763
    51
* \returns The generated seed string
aschiffler@6763
    52
*/
aschiffler@6717
    53
char *
aschiffler@6772
    54
SDLTest_GenerateRunSeed(const int length)
aschiffler@6717
    55
{
slouken@7191
    56
    char *seed = NULL;
slouken@7191
    57
    SDLTest_RandomContext randomContext;
slouken@7191
    58
    int counter;
aschiffler@6717
    59
slouken@7191
    60
    /* Sanity check input */
slouken@7191
    61
    if (length <= 0) {
slouken@7191
    62
        SDLTest_LogError("The length of the harness seed must be >0.");
slouken@7191
    63
        return NULL;
slouken@7191
    64
    }
aschiffler@6717
    65
slouken@7191
    66
    /* Allocate output buffer */
slouken@7191
    67
    seed = (char *)SDL_malloc((length + 1) * sizeof(char));
slouken@7191
    68
    if (seed == NULL) {
slouken@7191
    69
        SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
slouken@7191
    70
        return NULL;
slouken@7191
    71
    }
aschiffler@6717
    72
slouken@7191
    73
    /* Generate a random string of alphanumeric characters */
slouken@7191
    74
    SDLTest_RandomInitTime(&randomContext);
slouken@7191
    75
    for (counter = 0; counter < length - 1; ++counter) {
slouken@7191
    76
        unsigned int number = SDLTest_Random(&randomContext);
slouken@7191
    77
        char ch = (char) (number % (91 - 48)) + 48;
slouken@7191
    78
        if (ch >= 58 && ch <= 64) {
slouken@7191
    79
            ch = 65;
slouken@7191
    80
        }
slouken@7191
    81
        seed[counter] = ch;
slouken@7191
    82
    }
slouken@7191
    83
    seed[counter] = '\0';
aschiffler@6717
    84
slouken@7191
    85
    return seed;
aschiffler@6717
    86
}
aschiffler@6717
    87
aschiffler@6717
    88
/**
aschiffler@6763
    89
* Generates an execution key for the fuzzer.
aschiffler@6763
    90
*
slouken@7191
    91
* \param runSeed        The run seed to use
slouken@7191
    92
* \param suiteName      The name of the test suite
slouken@7191
    93
* \param testName       The name of the test
slouken@7191
    94
* \param iteration      The iteration count
aschiffler@6763
    95
*
aschiffler@6763
    96
* \returns The generated execution key to initialize the fuzzer with.
aschiffler@6763
    97
*
aschiffler@6763
    98
*/
aschiffler@6717
    99
Uint64
aschiffler@6772
   100
SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
aschiffler@6717
   101
{
slouken@7191
   102
    SDLTest_Md5Context md5Context;
slouken@7191
   103
    Uint64 *keys;
slouken@7191
   104
    char iterationString[16];
slouken@7191
   105
    Uint32 runSeedLength;
slouken@7191
   106
    Uint32 suiteNameLength;
slouken@7191
   107
    Uint32 testNameLength;
slouken@7191
   108
    Uint32 iterationStringLength;
slouken@7191
   109
    Uint32 entireStringLength;
slouken@7191
   110
    char *buffer;
aschiffler@6717
   111
slouken@7721
   112
    if (runSeed == NULL || runSeed[0] == '\0') {
slouken@7191
   113
        SDLTest_LogError("Invalid runSeed string.");
slouken@7191
   114
        return -1;
slouken@7191
   115
    }
aschiffler@6717
   116
slouken@7721
   117
    if (suiteName == NULL || suiteName[0] == '\0') {
slouken@7191
   118
        SDLTest_LogError("Invalid suiteName string.");
slouken@7191
   119
        return -1;
slouken@7191
   120
    }
aschiffler@6717
   121
slouken@7721
   122
    if (testName == NULL || testName[0] == '\0') {
slouken@7191
   123
        SDLTest_LogError("Invalid testName string.");
slouken@7191
   124
        return -1;
slouken@7191
   125
    }
aschiffler@6717
   126
slouken@7191
   127
    if (iteration <= 0) {
slouken@7191
   128
        SDLTest_LogError("Invalid iteration count.");
slouken@7191
   129
        return -1;
slouken@7191
   130
    }
aschiffler@6717
   131
slouken@7191
   132
    /* Convert iteration number into a string */
slouken@7191
   133
    SDL_memset(iterationString, 0, sizeof(iterationString));
slouken@7191
   134
    SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
aschiffler@6717
   135
slouken@7191
   136
    /* Combine the parameters into single string */
slouken@7191
   137
    runSeedLength = SDL_strlen(runSeed);
slouken@7191
   138
    suiteNameLength = SDL_strlen(suiteName);
slouken@7191
   139
    testNameLength = SDL_strlen(testName);
slouken@7191
   140
    iterationStringLength = SDL_strlen(iterationString);
slouken@7191
   141
    entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
slouken@7191
   142
    buffer = (char *)SDL_malloc(entireStringLength);
slouken@7191
   143
    if (buffer == NULL) {
slouken@7191
   144
        SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
slouken@7191
   145
        return 0;
slouken@7191
   146
    }
slouken@7191
   147
    SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
aschiffler@6717
   148
slouken@7191
   149
    /* Hash string and use half of the digest as 64bit exec key */
slouken@7191
   150
    SDLTest_Md5Init(&md5Context);
slouken@7191
   151
    SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
slouken@7191
   152
    SDLTest_Md5Final(&md5Context);
slouken@7191
   153
    SDL_free(buffer);
slouken@7191
   154
    keys = (Uint64 *)md5Context.digest;
aschiffler@6717
   155
slouken@7191
   156
    return keys[0];
aschiffler@6717
   157
}
aschiffler@6718
   158
aschiffler@6718
   159
/**
aschiffler@6763
   160
* \brief Set timeout handler for test.
aschiffler@6763
   161
*
aschiffler@6763
   162
* Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
aschiffler@6763
   163
*
aschiffler@6763
   164
* \param timeout Timeout interval in seconds.
aschiffler@6763
   165
* \param callback Function that will be called after timeout has elapsed.
slouken@7191
   166
*
aschiffler@6763
   167
* \return Timer id or -1 on failure.
aschiffler@6763
   168
*/
aschiffler@6718
   169
SDL_TimerID
aschiffler@6772
   170
SDLTest_SetTestTimeout(int timeout, void (*callback)())
aschiffler@6718
   171
{
slouken@7191
   172
    Uint32 timeoutInMilliseconds;
slouken@7191
   173
    SDL_TimerID timerID;
aschiffler@6718
   174
slouken@7191
   175
    if (callback == NULL) {
slouken@7191
   176
        SDLTest_LogError("Timeout callback can't be NULL");
slouken@7191
   177
        return -1;
slouken@7191
   178
    }
aschiffler@6718
   179
slouken@7191
   180
    if (timeout < 0) {
slouken@7191
   181
        SDLTest_LogError("Timeout value must be bigger than zero.");
slouken@7191
   182
        return -1;
slouken@7191
   183
    }
aschiffler@6718
   184
slouken@7191
   185
    /* Init SDL timer if not initialized before */
slouken@7191
   186
    if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
slouken@7191
   187
        if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
slouken@7191
   188
            SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
slouken@7191
   189
            return -1;
slouken@7191
   190
        }
slouken@7191
   191
    }
aschiffler@6718
   192
slouken@7191
   193
    /* Set timer */
slouken@7191
   194
    timeoutInMilliseconds = timeout * 1000;
slouken@7191
   195
    timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
slouken@7191
   196
    if (timerID == 0) {
slouken@7191
   197
        SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
slouken@7191
   198
        return -1;
slouken@7191
   199
    }
aschiffler@6718
   200
slouken@7191
   201
    return timerID;
aschiffler@6718
   202
}
aschiffler@6721
   203
aschiffler@6763
   204
/**
aschiffler@6763
   205
* \brief Timeout handler. Aborts test run and exits harness process.
aschiffler@6763
   206
*/
aschiffler@6721
   207
void
slouken@7191
   208
    SDLTest_BailOut()
aschiffler@6721
   209
{
slouken@7191
   210
    SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
slouken@7191
   211
    exit(TEST_ABORTED); /* bail out from the test */
aschiffler@6721
   212
}
aschiffler@6721
   213
aschiffler@6721
   214
/**
aschiffler@6763
   215
* \brief Execute a test using the given execution key.
aschiffler@6763
   216
*
aschiffler@6763
   217
* \param testSuite Suite containing the test case.
aschiffler@6763
   218
* \param testCase Case to execute.
aschiffler@6763
   219
* \param execKey Execution key for the fuzzer.
aschiffler@6763
   220
*
aschiffler@6763
   221
* \returns Test case result.
aschiffler@6763
   222
*/
aschiffler@6721
   223
int
aschiffler@6830
   224
SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
aschiffler@6721
   225
{
slouken@7191
   226
    SDL_TimerID timer = 0;
slouken@7191
   227
    int testCaseResult = 0;
slouken@7191
   228
    int testResult = 0;
slouken@7191
   229
    int fuzzerCount;
aschiffler@6721
   230
slouken@7191
   231
    if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
slouken@7191
   232
    {
slouken@7191
   233
        SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
slouken@7191
   234
        return TEST_RESULT_SETUP_FAILURE;
slouken@7191
   235
    }
aschiffler@6721
   236
slouken@7191
   237
    if (!testCase->enabled)
slouken@7191
   238
    {
slouken@7191
   239
        SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Disabled)");
slouken@7191
   240
        return TEST_RESULT_SKIPPED;
slouken@7191
   241
    }
aschiffler@6721
   242
aschiffler@6763
   243
slouken@7191
   244
    /* Initialize fuzzer */
slouken@7191
   245
    SDLTest_FuzzerInit(execKey);
aschiffler@6721
   246
slouken@7191
   247
    /* Reset assert tracker */
slouken@7191
   248
    SDLTest_ResetAssertSummary();
aschiffler@6721
   249
slouken@7191
   250
    /* Set timeout timer */
slouken@7191
   251
    timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
aschiffler@6721
   252
slouken@7191
   253
    /* Maybe run suite initalizer function */
slouken@7191
   254
    if (testSuite->testSetUp) {
slouken@7191
   255
        testSuite->testSetUp(0x0);
slouken@7191
   256
        if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
slouken@7191
   257
            SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed");
slouken@7191
   258
            return TEST_RESULT_SETUP_FAILURE;
slouken@7191
   259
        }
slouken@7191
   260
    }
aschiffler@6721
   261
slouken@7191
   262
    /* Run test case function */
slouken@7191
   263
    testCaseResult = testCase->testCase(0x0);
aschiffler@6721
   264
slouken@7191
   265
    /* Convert test execution result into harness result */
slouken@7191
   266
    if (testCaseResult == TEST_SKIPPED) {
slouken@7191
   267
        /* Test was programatically skipped */
slouken@7191
   268
        testResult = TEST_RESULT_SKIPPED;
slouken@7191
   269
    } else if (testCaseResult == TEST_STARTED) {
slouken@7191
   270
        /* Test did not return a TEST_COMPLETED value; assume it failed */
slouken@7191
   271
        testResult = TEST_RESULT_FAILED;
slouken@7191
   272
    } else if (testCaseResult == TEST_ABORTED) {
slouken@7191
   273
        /* Test was aborted early; assume it failed */
slouken@7191
   274
        testResult = TEST_RESULT_FAILED;
slouken@7191
   275
    } else {
slouken@7191
   276
        /* Perform failure analysis based on asserts */
slouken@7191
   277
        testResult = SDLTest_AssertSummaryToTestResult();
slouken@7191
   278
    }
aschiffler@6721
   279
slouken@7191
   280
    /* Maybe run suite cleanup function (ignore failed asserts) */
slouken@7191
   281
    if (testSuite->testTearDown) {
slouken@7191
   282
        testSuite->testTearDown(0x0);
slouken@7191
   283
    }
aschiffler@6721
   284
slouken@7191
   285
    /* Cancel timeout timer */
slouken@7191
   286
    if (timer) {
slouken@7191
   287
        SDL_RemoveTimer(timer);
slouken@7191
   288
    }
aschiffler@7189
   289
slouken@7191
   290
    /* Report on asserts and fuzzer usage */
slouken@7191
   291
    fuzzerCount = SDLTest_GetFuzzerInvocationCount();
slouken@7191
   292
    if (fuzzerCount > 0) {
slouken@7191
   293
        SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
slouken@7191
   294
    }
aschiffler@6721
   295
slouken@7191
   296
    /* Final log based on test execution result */
slouken@7191
   297
    if (testCaseResult == TEST_SKIPPED) {
slouken@7191
   298
        /* Test was programatically skipped */
slouken@7191
   299
        SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)");
slouken@7191
   300
    } else if (testCaseResult == TEST_STARTED) {
slouken@7191
   301
        /* Test did not return a TEST_COMPLETED value; assume it failed */
slouken@7191
   302
        SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
slouken@7191
   303
    } else if (testCaseResult == TEST_ABORTED) {
slouken@7191
   304
        /* Test was aborted early; assume it failed */
slouken@7191
   305
        SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)");
slouken@7191
   306
    } else {
slouken@7191
   307
        SDLTest_LogAssertSummary();
slouken@7191
   308
    }
slouken@7191
   309
slouken@7191
   310
    return testResult;
aschiffler@6721
   311
}
aschiffler@6721
   312
aschiffler@6721
   313
/* Prints summary of all suites/tests contained in the given reference */
aschiffler@6721
   314
void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
aschiffler@6721
   315
{
slouken@7191
   316
    int suiteCounter;
slouken@7191
   317
    int testCounter;
slouken@7191
   318
    SDLTest_TestSuiteReference *testSuite;
slouken@7191
   319
    SDLTest_TestCaseReference *testCase;
aschiffler@6721
   320
slouken@7191
   321
    /* Loop over all suites */
slouken@7191
   322
    suiteCounter = 0;
slouken@7191
   323
    while(&testSuites[suiteCounter]) {
slouken@7191
   324
        testSuite=&testSuites[suiteCounter];
slouken@7191
   325
        suiteCounter++;
slouken@7191
   326
        SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
slouken@7191
   327
            (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
aschiffler@6721
   328
slouken@7191
   329
        /* Loop over all test cases */
slouken@7191
   330
        testCounter = 0;
slouken@7191
   331
        while(testSuite->testCases[testCounter])
slouken@7191
   332
        {
slouken@7191
   333
            testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
slouken@7191
   334
            testCounter++;
slouken@7191
   335
            SDLTest_Log("  Test Case %i - %s: %s", testCounter,
slouken@7191
   336
                (testCase->name) ? testCase->name : SDLTest_InvalidNameFormat,
slouken@7191
   337
                (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
slouken@7191
   338
        }
slouken@7191
   339
    }
aschiffler@6721
   340
}
aschiffler@6721
   341
aschiffler@6756
   342
/* Gets a timer value in seconds */
aschiffler@6756
   343
float GetClock()
aschiffler@6756
   344
{
slouken@7191
   345
    float currentClock = (float)clock();
slouken@7191
   346
    return currentClock / (float)CLOCKS_PER_SEC;
aschiffler@6756
   347
}
aschiffler@6721
   348
aschiffler@6721
   349
/**
aschiffler@6763
   350
* \brief Execute a test suite using the given run seend and execution key.
aschiffler@6763
   351
*
aschiffler@6763
   352
* The filter string is matched to the suite name (full comparison) to select a single suite,
aschiffler@6763
   353
* or if no suite matches, it is matched to the test names (full comparison) to select a single test.
aschiffler@6763
   354
*
aschiffler@6763
   355
* \param testSuites Suites containing the test case.
aschiffler@6763
   356
* \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
aschiffler@6763
   357
* \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
aschiffler@6763
   358
* \param filter Filter specification. NULL disables. Case sensitive.
aschiffler@6763
   359
* \param testIterations Number of iterations to run each test case.
aschiffler@6763
   360
*
aschiffler@6763
   361
* \returns Test run result; 0 when all tests passed, 1 if any tests failed.
aschiffler@6763
   362
*/
slouken@6768
   363
int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
aschiffler@6721
   364
{
slouken@7191
   365
    int suiteCounter;
slouken@7191
   366
    int testCounter;
slouken@7191
   367
    int iterationCounter;
slouken@7191
   368
    SDLTest_TestSuiteReference *testSuite;
slouken@7191
   369
    SDLTest_TestCaseReference *testCase;
slouken@7191
   370
    const char *runSeed = NULL;
slouken@7191
   371
    char *currentSuiteName;
slouken@7191
   372
    char *currentTestName;
slouken@7191
   373
    Uint64 execKey;
slouken@7191
   374
    float runStartSeconds;
slouken@7191
   375
    float suiteStartSeconds;
slouken@7191
   376
    float testStartSeconds;
slouken@7191
   377
    float runEndSeconds;
slouken@7191
   378
    float suiteEndSeconds;
slouken@7191
   379
    float testEndSeconds;
slouken@7191
   380
    float runtime;
slouken@7191
   381
    int suiteFilter = 0;
slouken@7191
   382
    char *suiteFilterName = NULL;
slouken@7191
   383
    int testFilter = 0;
slouken@7191
   384
    char *testFilterName = NULL;
slouken@7191
   385
    int testResult = 0;
slouken@7191
   386
    int runResult = 0;
slouken@7191
   387
    Uint32 totalTestFailedCount = 0;
slouken@7191
   388
    Uint32 totalTestPassedCount = 0;
slouken@7191
   389
    Uint32 totalTestSkippedCount = 0;
slouken@7191
   390
    Uint32 testFailedCount = 0;
slouken@7191
   391
    Uint32 testPassedCount = 0;
slouken@7191
   392
    Uint32 testSkippedCount = 0;
slouken@7191
   393
    Uint32 countSum = 0;
slouken@7191
   394
    char *logFormat = (char *)SDLTest_LogSummaryFormat;
aschiffler@6721
   395
slouken@7191
   396
    /* Sanitize test iterations */
slouken@7191
   397
    if (testIterations < 1) {
slouken@7191
   398
        testIterations = 1;
slouken@7191
   399
    }
aschiffler@6721
   400
slouken@7191
   401
    /* Generate run see if we don't have one already */
slouken@7721
   402
    if (userRunSeed == NULL || userRunSeed[0] == '\0') {
slouken@7191
   403
        runSeed = SDLTest_GenerateRunSeed(16);
slouken@7191
   404
        if (runSeed == NULL) {
slouken@7191
   405
            SDLTest_LogError("Generating a random seed failed");
slouken@7191
   406
            return 2;
slouken@7191
   407
        }
slouken@7191
   408
    } else {
slouken@7191
   409
        runSeed = userRunSeed;
slouken@7191
   410
    }
aschiffler@6721
   411
aschiffler@6763
   412
slouken@7191
   413
    /* Reset per-run counters */
slouken@7191
   414
    totalTestFailedCount = 0;
slouken@7191
   415
    totalTestPassedCount = 0;
slouken@7191
   416
    totalTestSkippedCount = 0;
aschiffler@6721
   417
slouken@7191
   418
    /* Take time - run start */
slouken@7191
   419
    runStartSeconds = GetClock();
aschiffler@6721
   420
slouken@7191
   421
    /* Log run with fuzzer parameters */
slouken@7191
   422
    SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
aschiffler@6721
   423
slouken@7191
   424
    /* Initialize filtering */
slouken@7721
   425
    if (filter != NULL && filter[0] != '\0') {
slouken@7191
   426
        /* Loop over all suites to check if we have a filter match */
slouken@7191
   427
        suiteCounter = 0;
slouken@7191
   428
        while (testSuites[suiteCounter] && suiteFilter == 0) {
slouken@7191
   429
            testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
slouken@7191
   430
            suiteCounter++;
slouken@7191
   431
            if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
slouken@7191
   432
                /* Matched a suite name */
slouken@7191
   433
                suiteFilter = 1;
slouken@7191
   434
                suiteFilterName = testSuite->name;
slouken@7191
   435
                SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
slouken@7191
   436
                break;
slouken@7191
   437
            }
aschiffler@6763
   438
slouken@7191
   439
            /* Within each suite, loop over all test cases to check if we have a filter match */
slouken@7191
   440
            testCounter = 0;
slouken@7191
   441
            while (testSuite->testCases[testCounter] && testFilter == 0)
slouken@7191
   442
            {
slouken@7191
   443
                testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
slouken@7191
   444
                testCounter++;
slouken@7191
   445
                if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
slouken@7191
   446
                    /* Matched a test name */
slouken@7191
   447
                    suiteFilter = 1;
slouken@7191
   448
                    suiteFilterName = testSuite->name;
slouken@7191
   449
                    testFilter = 1;
slouken@7191
   450
                    testFilterName = testCase->name;
slouken@7191
   451
                    SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
slouken@7191
   452
                    break;
slouken@7191
   453
                }
slouken@7191
   454
            }
slouken@7191
   455
        }
aschiffler@6763
   456
slouken@7191
   457
        if (suiteFilter == 0 && testFilter == 0) {
slouken@7191
   458
            SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
slouken@7191
   459
            SDLTest_Log("Exit code: 2");
slouken@7191
   460
            return 2;
slouken@7191
   461
        }
slouken@7191
   462
    }
aschiffler@6721
   463
slouken@7191
   464
    /* Loop over all suites */
slouken@7191
   465
    suiteCounter = 0;
slouken@7191
   466
    while(testSuites[suiteCounter]) {
slouken@7191
   467
        testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
slouken@7191
   468
        currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
slouken@7191
   469
        suiteCounter++;
aschiffler@6721
   470
slouken@7191
   471
        /* Filter suite if flag set and we have a name */
slouken@7191
   472
        if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
slouken@7191
   473
            SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
slouken@7191
   474
                /* Skip suite */
slouken@7191
   475
                SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
slouken@7191
   476
                    suiteCounter,
slouken@7191
   477
                    currentSuiteName);
slouken@7191
   478
        } else {
aschiffler@6721
   479
slouken@7191
   480
            /* Reset per-suite counters */
slouken@7191
   481
            testFailedCount = 0;
slouken@7191
   482
            testPassedCount = 0;
slouken@7191
   483
            testSkippedCount = 0;
aschiffler@6721
   484
slouken@7191
   485
            /* Take time - suite start */
slouken@7191
   486
            suiteStartSeconds = GetClock();
aschiffler@6721
   487
slouken@7191
   488
            /* Log suite started */
slouken@7191
   489
            SDLTest_Log("===== Test Suite %i: '%s' started\n",
slouken@7191
   490
                suiteCounter,
slouken@7191
   491
                currentSuiteName);
aschiffler@6721
   492
slouken@7191
   493
            /* Loop over all test cases */
slouken@7191
   494
            testCounter = 0;
slouken@7191
   495
            while(testSuite->testCases[testCounter])
slouken@7191
   496
            {
slouken@7191
   497
                testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
slouken@7191
   498
                currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
slouken@7191
   499
                testCounter++;
aschiffler@6721
   500
slouken@7191
   501
                /* Filter tests if flag set and we have a name */
slouken@7191
   502
                if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
slouken@7191
   503
                    SDL_strcmp(testFilterName, testCase->name) != 0) {
slouken@7191
   504
                        /* Skip test */
slouken@7191
   505
                        SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
slouken@7191
   506
                            suiteCounter,
slouken@7191
   507
                            testCounter,
slouken@7191
   508
                            currentTestName);
slouken@7191
   509
                } else {
slouken@7191
   510
                    /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
slouken@7191
   511
                    if (testFilter == 1 && !testCase->enabled) {
slouken@7191
   512
                        SDLTest_Log("Force run of disabled test since test filter was set");
slouken@7191
   513
                        testCase->enabled = 1;
slouken@7191
   514
                    }
aschiffler@6721
   515
slouken@7191
   516
                    /* Take time - test start */
slouken@7191
   517
                    testStartSeconds = GetClock();
aschiffler@6763
   518
slouken@7191
   519
                    /* Log test started */
slouken@7191
   520
                    SDLTest_Log("----- Test Case %i.%i: '%s' started",
slouken@7191
   521
                        suiteCounter,
slouken@7191
   522
                        testCounter,
slouken@7191
   523
                        currentTestName);
slouken@7721
   524
                    if (testCase->description != NULL && testCase->description[0] != '\0') {
slouken@7191
   525
                        SDLTest_Log("Test Description: '%s'",
slouken@7191
   526
                            (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
slouken@7191
   527
                    }
aschiffler@6763
   528
slouken@7191
   529
                    /* Loop over all iterations */
slouken@7191
   530
                    iterationCounter = 0;
slouken@7191
   531
                    while(iterationCounter < testIterations)
slouken@7191
   532
                    {
slouken@7191
   533
                        iterationCounter++;
aschiffler@6763
   534
slouken@7191
   535
                        if (userExecKey != 0) {
slouken@7191
   536
                            execKey = userExecKey;
slouken@7191
   537
                        } else {
slouken@7191
   538
                            execKey = SDLTest_GenerateExecKey((char *)runSeed, testSuite->name, testCase->name, iterationCounter);
slouken@7191
   539
                        }
aschiffler@6763
   540
slouken@7191
   541
                        SDLTest_Log("Test Iteration %i: execKey %llu", iterationCounter, execKey);
slouken@7191
   542
                        testResult = SDLTest_RunTest(testSuite, testCase, execKey);
aschiffler@6763
   543
slouken@7191
   544
                        if (testResult == TEST_RESULT_PASSED) {
slouken@7191
   545
                            testPassedCount++;
slouken@7191
   546
                            totalTestPassedCount++;
slouken@7191
   547
                        } else if (testResult == TEST_RESULT_SKIPPED) {
slouken@7191
   548
                            testSkippedCount++;
slouken@7191
   549
                            totalTestSkippedCount++;
slouken@7191
   550
                        } else {
slouken@7191
   551
                            testFailedCount++;
slouken@7191
   552
                            totalTestFailedCount++;
slouken@7191
   553
                        }
slouken@7191
   554
                    }
aschiffler@6763
   555
slouken@7191
   556
                    /* Take time - test end */
slouken@7191
   557
                    testEndSeconds = GetClock();
slouken@7191
   558
                    runtime = testEndSeconds - testStartSeconds;
slouken@7191
   559
                    if (runtime < 0.0f) runtime = 0.0f;
aschiffler@6763
   560
slouken@7191
   561
                    if (testIterations > 1) {
slouken@7191
   562
                        /* Log test runtime */
slouken@7191
   563
                        SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
slouken@7191
   564
                        SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
slouken@7191
   565
                    } else {
slouken@7191
   566
                        /* Log test runtime */
slouken@7191
   567
                        SDLTest_Log("Total Test runtime: %.1f sec", runtime);
slouken@7191
   568
                    }
aschiffler@6763
   569
slouken@7191
   570
                    /* Log final test result */
slouken@7191
   571
                    switch (testResult) {
slouken@7191
   572
                    case TEST_RESULT_PASSED:
slouken@7191
   573
                        SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
slouken@7191
   574
                        break;
slouken@7191
   575
                    case TEST_RESULT_FAILED:
slouken@7191
   576
                        SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
slouken@7191
   577
                        break;
slouken@7191
   578
                    case TEST_RESULT_NO_ASSERT:
slouken@7191
   579
                        SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
slouken@7191
   580
                        break;
slouken@7191
   581
                    }
aschiffler@6721
   582
slouken@7191
   583
                }
slouken@7191
   584
            }
aschiffler@6721
   585
slouken@7191
   586
            /* Take time - suite end */
slouken@7191
   587
            suiteEndSeconds = GetClock();
slouken@7191
   588
            runtime = suiteEndSeconds - suiteStartSeconds;
slouken@7191
   589
            if (runtime < 0.0f) runtime = 0.0f;
aschiffler@6756
   590
slouken@7191
   591
            /* Log suite runtime */
slouken@7191
   592
            SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
aschiffler@6721
   593
slouken@7191
   594
            /* Log summary and final Suite result */
slouken@7191
   595
            countSum = testPassedCount + testFailedCount + testSkippedCount;
slouken@7191
   596
            if (testFailedCount == 0)
slouken@7191
   597
            {
slouken@7191
   598
                SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
slouken@7191
   599
                SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
slouken@7191
   600
            }
slouken@7191
   601
            else
slouken@7191
   602
            {
slouken@7191
   603
                SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
slouken@7191
   604
                SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
slouken@7191
   605
            }
aschiffler@6721
   606
slouken@7191
   607
        }
slouken@7191
   608
    }
aschiffler@6721
   609
slouken@7191
   610
    /* Take time - run end */
slouken@7191
   611
    runEndSeconds = GetClock();
slouken@7191
   612
    runtime = runEndSeconds - runStartSeconds;
slouken@7191
   613
    if (runtime < 0.0f) runtime = 0.0f;
aschiffler@6721
   614
slouken@7191
   615
    /* Log total runtime */
slouken@7191
   616
    SDLTest_Log("Total Run runtime: %.1f sec", runtime);
aschiffler@6756
   617
slouken@7191
   618
    /* Log summary and final run result */
slouken@7191
   619
    countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
slouken@7191
   620
    if (totalTestFailedCount == 0)
slouken@7191
   621
    {
slouken@7191
   622
        runResult = 0;
slouken@7191
   623
        SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
slouken@7191
   624
        SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
slouken@7191
   625
    }
slouken@7191
   626
    else
slouken@7191
   627
    {
slouken@7191
   628
        runResult = 1;
slouken@7191
   629
        SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
slouken@7191
   630
        SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
slouken@7191
   631
    }
slouken@7191
   632
slouken@7191
   633
    SDLTest_Log("Exit code: %d", runResult);
slouken@7191
   634
    return runResult;
aschiffler@6721
   635
}