src/test/SDL_test_harness.c
author Sam Lantinga <slouken@libsdl.org>
Mon, 10 Mar 2014 05:44:34 -0700
changeset 8595 c6fc94639f38
parent 8149 681eb46b8ac4
child 8605 57faccca4fab
permissions -rw-r--r--
Implemented YV12 and IYUV texture support for the D3D11 renderer
     1 /*
     2   Simple DirectMedia Layer
     3   Copyright (C) 1997-2014 Sam Lantinga <slouken@libsdl.org>
     4 
     5   This software is provided 'as-is', without any express or implied
     6   warranty.  In no event will the authors be held liable for any damages
     7   arising from the use of this software.
     8 
     9   Permission is granted to anyone to use this software for any purpose,
    10   including commercial applications, and to alter it and redistribute it
    11   freely, subject to the following restrictions:
    12 
    13   1. The origin of this software must not be misrepresented; you must not
    14      claim that you wrote the original software. If you use this software
    15      in a product, an acknowledgment in the product documentation would be
    16      appreciated but is not required.
    17   2. Altered source versions must be plainly marked as such, and must not be
    18      misrepresented as being the original software.
    19   3. This notice may not be removed or altered from any source distribution.
    20 */
    21 
    22 #include "SDL_config.h"
    23 
    24 #include "SDL_test.h"
    25 
    26 #include <stdio.h>
    27 #include <stdlib.h>
    28 #include <string.h>
    29 #include <time.h>
    30 
    31 /* Invalid test name/description message format */
    32 const char *SDLTest_InvalidNameFormat = "(Invalid)";
    33 
    34 /* Log summary message format */
    35 const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d";
    36 
    37 /* Final result message format */
    38 const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n";
    39 
    40 /* ! \brief Timeout for single test case execution */
    41 static Uint32 SDLTest_TestCaseTimeout = 3600;
    42 
    43 /**
    44 * Generates a random run seed string for the harness. The generated seed
    45 * will contain alphanumeric characters (0-9A-Z).
    46 *
    47 * Note: The returned string needs to be deallocated by the caller.
    48 *
    49 * \param length The length of the seed string to generate
    50 *
    51 * \returns The generated seed string
    52 */
    53 char *
    54 SDLTest_GenerateRunSeed(const int length)
    55 {
    56     char *seed = NULL;
    57     SDLTest_RandomContext randomContext;
    58     int counter;
    59 
    60     /* Sanity check input */
    61     if (length <= 0) {
    62         SDLTest_LogError("The length of the harness seed must be >0.");
    63         return NULL;
    64     }
    65 
    66     /* Allocate output buffer */
    67     seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    68     if (seed == NULL) {
    69         SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    70         return NULL;
    71     }
    72 
    73     /* Generate a random string of alphanumeric characters */
    74     SDLTest_RandomInitTime(&randomContext);
    75     for (counter = 0; counter < length - 1; ++counter) {
    76         unsigned int number = SDLTest_Random(&randomContext);
    77         char ch = (char) (number % (91 - 48)) + 48;
    78         if (ch >= 58 && ch <= 64) {
    79             ch = 65;
    80         }
    81         seed[counter] = ch;
    82     }
    83     seed[counter] = '\0';
    84 
    85     return seed;
    86 }
    87 
    88 /**
    89 * Generates an execution key for the fuzzer.
    90 *
    91 * \param runSeed        The run seed to use
    92 * \param suiteName      The name of the test suite
    93 * \param testName       The name of the test
    94 * \param iteration      The iteration count
    95 *
    96 * \returns The generated execution key to initialize the fuzzer with.
    97 *
    98 */
    99 Uint64
   100 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
   101 {
   102     SDLTest_Md5Context md5Context;
   103     Uint64 *keys;
   104     char iterationString[16];
   105     Uint32 runSeedLength;
   106     Uint32 suiteNameLength;
   107     Uint32 testNameLength;
   108     Uint32 iterationStringLength;
   109     Uint32 entireStringLength;
   110     char *buffer;
   111 
   112     if (runSeed == NULL || runSeed[0] == '\0') {
   113         SDLTest_LogError("Invalid runSeed string.");
   114         return -1;
   115     }
   116 
   117     if (suiteName == NULL || suiteName[0] == '\0') {
   118         SDLTest_LogError("Invalid suiteName string.");
   119         return -1;
   120     }
   121 
   122     if (testName == NULL || testName[0] == '\0') {
   123         SDLTest_LogError("Invalid testName string.");
   124         return -1;
   125     }
   126 
   127     if (iteration <= 0) {
   128         SDLTest_LogError("Invalid iteration count.");
   129         return -1;
   130     }
   131 
   132     /* Convert iteration number into a string */
   133     SDL_memset(iterationString, 0, sizeof(iterationString));
   134     SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
   135 
   136     /* Combine the parameters into single string */
   137     runSeedLength = SDL_strlen(runSeed);
   138     suiteNameLength = SDL_strlen(suiteName);
   139     testNameLength = SDL_strlen(testName);
   140     iterationStringLength = SDL_strlen(iterationString);
   141     entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
   142     buffer = (char *)SDL_malloc(entireStringLength);
   143     if (buffer == NULL) {
   144         SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
   145         return 0;
   146     }
   147     SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
   148 
   149     /* Hash string and use half of the digest as 64bit exec key */
   150     SDLTest_Md5Init(&md5Context);
   151     SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
   152     SDLTest_Md5Final(&md5Context);
   153     SDL_free(buffer);
   154     keys = (Uint64 *)md5Context.digest;
   155 
   156     return keys[0];
   157 }
   158 
   159 /**
   160 * \brief Set timeout handler for test.
   161 *
   162 * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   163 *
   164 * \param timeout Timeout interval in seconds.
   165 * \param callback Function that will be called after timeout has elapsed.
   166 *
   167 * \return Timer id or -1 on failure.
   168 */
   169 SDL_TimerID
   170 SDLTest_SetTestTimeout(int timeout, void (*callback)())
   171 {
   172     Uint32 timeoutInMilliseconds;
   173     SDL_TimerID timerID;
   174 
   175     if (callback == NULL) {
   176         SDLTest_LogError("Timeout callback can't be NULL");
   177         return -1;
   178     }
   179 
   180     if (timeout < 0) {
   181         SDLTest_LogError("Timeout value must be bigger than zero.");
   182         return -1;
   183     }
   184 
   185     /* Init SDL timer if not initialized before */
   186     if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
   187         if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
   188             SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
   189             return -1;
   190         }
   191     }
   192 
   193     /* Set timer */
   194     timeoutInMilliseconds = timeout * 1000;
   195     timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
   196     if (timerID == 0) {
   197         SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
   198         return -1;
   199     }
   200 
   201     return timerID;
   202 }
   203 
   204 /**
   205 * \brief Timeout handler. Aborts test run and exits harness process.
   206 */
   207 void
   208     SDLTest_BailOut()
   209 {
   210     SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   211     exit(TEST_ABORTED); /* bail out from the test */
   212 }
   213 
   214 /**
   215 * \brief Execute a test using the given execution key.
   216 *
   217 * \param testSuite Suite containing the test case.
   218 * \param testCase Case to execute.
   219 * \param execKey Execution key for the fuzzer.
   220 *
   221 * \returns Test case result.
   222 */
   223 int
   224 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
   225 {
   226     SDL_TimerID timer = 0;
   227     int testCaseResult = 0;
   228     int testResult = 0;
   229     int fuzzerCount;
   230 
   231     if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
   232     {
   233         SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
   234         return TEST_RESULT_SETUP_FAILURE;
   235     }
   236 
   237     if (!testCase->enabled)
   238     {
   239         SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Disabled)");
   240         return TEST_RESULT_SKIPPED;
   241     }
   242 
   243 
   244     /* Initialize fuzzer */
   245     SDLTest_FuzzerInit(execKey);
   246 
   247     /* Reset assert tracker */
   248     SDLTest_ResetAssertSummary();
   249 
   250     /* Set timeout timer */
   251     timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
   252 
   253     /* Maybe run suite initalizer function */
   254     if (testSuite->testSetUp) {
   255         testSuite->testSetUp(0x0);
   256         if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
   257             SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed");
   258             return TEST_RESULT_SETUP_FAILURE;
   259         }
   260     }
   261 
   262     /* Run test case function */
   263     testCaseResult = testCase->testCase(0x0);
   264 
   265     /* Convert test execution result into harness result */
   266     if (testCaseResult == TEST_SKIPPED) {
   267         /* Test was programatically skipped */
   268         testResult = TEST_RESULT_SKIPPED;
   269     } else if (testCaseResult == TEST_STARTED) {
   270         /* Test did not return a TEST_COMPLETED value; assume it failed */
   271         testResult = TEST_RESULT_FAILED;
   272     } else if (testCaseResult == TEST_ABORTED) {
   273         /* Test was aborted early; assume it failed */
   274         testResult = TEST_RESULT_FAILED;
   275     } else {
   276         /* Perform failure analysis based on asserts */
   277         testResult = SDLTest_AssertSummaryToTestResult();
   278     }
   279 
   280     /* Maybe run suite cleanup function (ignore failed asserts) */
   281     if (testSuite->testTearDown) {
   282         testSuite->testTearDown(0x0);
   283     }
   284 
   285     /* Cancel timeout timer */
   286     if (timer) {
   287         SDL_RemoveTimer(timer);
   288     }
   289 
   290     /* Report on asserts and fuzzer usage */
   291     fuzzerCount = SDLTest_GetFuzzerInvocationCount();
   292     if (fuzzerCount > 0) {
   293         SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
   294     }
   295 
   296     /* Final log based on test execution result */
   297     if (testCaseResult == TEST_SKIPPED) {
   298         /* Test was programatically skipped */
   299         SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)");
   300     } else if (testCaseResult == TEST_STARTED) {
   301         /* Test did not return a TEST_COMPLETED value; assume it failed */
   302         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
   303     } else if (testCaseResult == TEST_ABORTED) {
   304         /* Test was aborted early; assume it failed */
   305         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)");
   306     } else {
   307         SDLTest_LogAssertSummary();
   308     }
   309 
   310     return testResult;
   311 }
   312 
   313 /* Prints summary of all suites/tests contained in the given reference */
   314 void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
   315 {
   316     int suiteCounter;
   317     int testCounter;
   318     SDLTest_TestSuiteReference *testSuite;
   319     SDLTest_TestCaseReference *testCase;
   320 
   321     /* Loop over all suites */
   322     suiteCounter = 0;
   323     while(&testSuites[suiteCounter]) {
   324         testSuite=&testSuites[suiteCounter];
   325         suiteCounter++;
   326         SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
   327             (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   328 
   329         /* Loop over all test cases */
   330         testCounter = 0;
   331         while(testSuite->testCases[testCounter])
   332         {
   333             testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   334             testCounter++;
   335             SDLTest_Log("  Test Case %i - %s: %s", testCounter,
   336                 (testCase->name) ? testCase->name : SDLTest_InvalidNameFormat,
   337                 (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   338         }
   339     }
   340 }
   341 
   342 /* Gets a timer value in seconds */
   343 float GetClock()
   344 {
   345     float currentClock = (float)clock();
   346     return currentClock / (float)CLOCKS_PER_SEC;
   347 }
   348 
   349 /**
   350 * \brief Execute a test suite using the given run seend and execution key.
   351 *
   352 * The filter string is matched to the suite name (full comparison) to select a single suite,
   353 * or if no suite matches, it is matched to the test names (full comparison) to select a single test.
   354 *
   355 * \param testSuites Suites containing the test case.
   356 * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   357 * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   358 * \param filter Filter specification. NULL disables. Case sensitive.
   359 * \param testIterations Number of iterations to run each test case.
   360 *
   361 * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   362 */
   363 int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
   364 {
   365     int suiteCounter;
   366     int testCounter;
   367     int iterationCounter;
   368     SDLTest_TestSuiteReference *testSuite;
   369     SDLTest_TestCaseReference *testCase;
   370     const char *runSeed = NULL;
   371     char *currentSuiteName;
   372     char *currentTestName;
   373     Uint64 execKey;
   374     float runStartSeconds;
   375     float suiteStartSeconds;
   376     float testStartSeconds;
   377     float runEndSeconds;
   378     float suiteEndSeconds;
   379     float testEndSeconds;
   380     float runtime;
   381     int suiteFilter = 0;
   382     char *suiteFilterName = NULL;
   383     int testFilter = 0;
   384     char *testFilterName = NULL;
   385     int testResult = 0;
   386     int runResult = 0;
   387     Uint32 totalTestFailedCount = 0;
   388     Uint32 totalTestPassedCount = 0;
   389     Uint32 totalTestSkippedCount = 0;
   390     Uint32 testFailedCount = 0;
   391     Uint32 testPassedCount = 0;
   392     Uint32 testSkippedCount = 0;
   393     Uint32 countSum = 0;
   394     char *logFormat = (char *)SDLTest_LogSummaryFormat;
   395 
   396     /* Sanitize test iterations */
   397     if (testIterations < 1) {
   398         testIterations = 1;
   399     }
   400 
   401     /* Generate run see if we don't have one already */
   402     if (userRunSeed == NULL || userRunSeed[0] == '\0') {
   403         runSeed = SDLTest_GenerateRunSeed(16);
   404         if (runSeed == NULL) {
   405             SDLTest_LogError("Generating a random seed failed");
   406             return 2;
   407         }
   408     } else {
   409         runSeed = userRunSeed;
   410     }
   411 
   412 
   413     /* Reset per-run counters */
   414     totalTestFailedCount = 0;
   415     totalTestPassedCount = 0;
   416     totalTestSkippedCount = 0;
   417 
   418     /* Take time - run start */
   419     runStartSeconds = GetClock();
   420 
   421     /* Log run with fuzzer parameters */
   422     SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   423 
   424     /* Initialize filtering */
   425     if (filter != NULL && filter[0] != '\0') {
   426         /* Loop over all suites to check if we have a filter match */
   427         suiteCounter = 0;
   428         while (testSuites[suiteCounter] && suiteFilter == 0) {
   429             testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   430             suiteCounter++;
   431             if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
   432                 /* Matched a suite name */
   433                 suiteFilter = 1;
   434                 suiteFilterName = testSuite->name;
   435                 SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
   436                 break;
   437             }
   438 
   439             /* Within each suite, loop over all test cases to check if we have a filter match */
   440             testCounter = 0;
   441             while (testSuite->testCases[testCounter] && testFilter == 0)
   442             {
   443                 testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   444                 testCounter++;
   445                 if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
   446                     /* Matched a test name */
   447                     suiteFilter = 1;
   448                     suiteFilterName = testSuite->name;
   449                     testFilter = 1;
   450                     testFilterName = testCase->name;
   451                     SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
   452                     break;
   453                 }
   454             }
   455         }
   456 
   457         if (suiteFilter == 0 && testFilter == 0) {
   458             SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
   459             SDLTest_Log("Exit code: 2");
   460             return 2;
   461         }
   462     }
   463 
   464     /* Loop over all suites */
   465     suiteCounter = 0;
   466     while(testSuites[suiteCounter]) {
   467         testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   468         currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   469         suiteCounter++;
   470 
   471         /* Filter suite if flag set and we have a name */
   472         if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
   473             SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
   474                 /* Skip suite */
   475                 SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
   476                     suiteCounter,
   477                     currentSuiteName);
   478         } else {
   479 
   480             /* Reset per-suite counters */
   481             testFailedCount = 0;
   482             testPassedCount = 0;
   483             testSkippedCount = 0;
   484 
   485             /* Take time - suite start */
   486             suiteStartSeconds = GetClock();
   487 
   488             /* Log suite started */
   489             SDLTest_Log("===== Test Suite %i: '%s' started\n",
   490                 suiteCounter,
   491                 currentSuiteName);
   492 
   493             /* Loop over all test cases */
   494             testCounter = 0;
   495             while(testSuite->testCases[testCounter])
   496             {
   497                 testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   498                 currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   499                 testCounter++;
   500 
   501                 /* Filter tests if flag set and we have a name */
   502                 if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
   503                     SDL_strcmp(testFilterName, testCase->name) != 0) {
   504                         /* Skip test */
   505                         SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
   506                             suiteCounter,
   507                             testCounter,
   508                             currentTestName);
   509                 } else {
   510                     /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
   511                     if (testFilter == 1 && !testCase->enabled) {
   512                         SDLTest_Log("Force run of disabled test since test filter was set");
   513                         testCase->enabled = 1;
   514                     }
   515 
   516                     /* Take time - test start */
   517                     testStartSeconds = GetClock();
   518 
   519                     /* Log test started */
   520                     SDLTest_Log("----- Test Case %i.%i: '%s' started",
   521                         suiteCounter,
   522                         testCounter,
   523                         currentTestName);
   524                     if (testCase->description != NULL && testCase->description[0] != '\0') {
   525                         SDLTest_Log("Test Description: '%s'",
   526                             (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   527                     }
   528 
   529                     /* Loop over all iterations */
   530                     iterationCounter = 0;
   531                     while(iterationCounter < testIterations)
   532                     {
   533                         iterationCounter++;
   534 
   535                         if (userExecKey != 0) {
   536                             execKey = userExecKey;
   537                         } else {
   538                             execKey = SDLTest_GenerateExecKey((char *)runSeed, testSuite->name, testCase->name, iterationCounter);
   539                         }
   540 
   541                         SDLTest_Log("Test Iteration %i: execKey %llu", iterationCounter, execKey);
   542                         testResult = SDLTest_RunTest(testSuite, testCase, execKey);
   543 
   544                         if (testResult == TEST_RESULT_PASSED) {
   545                             testPassedCount++;
   546                             totalTestPassedCount++;
   547                         } else if (testResult == TEST_RESULT_SKIPPED) {
   548                             testSkippedCount++;
   549                             totalTestSkippedCount++;
   550                         } else {
   551                             testFailedCount++;
   552                             totalTestFailedCount++;
   553                         }
   554                     }
   555 
   556                     /* Take time - test end */
   557                     testEndSeconds = GetClock();
   558                     runtime = testEndSeconds - testStartSeconds;
   559                     if (runtime < 0.0f) runtime = 0.0f;
   560 
   561                     if (testIterations > 1) {
   562                         /* Log test runtime */
   563                         SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
   564                         SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
   565                     } else {
   566                         /* Log test runtime */
   567                         SDLTest_Log("Total Test runtime: %.1f sec", runtime);
   568                     }
   569 
   570                     /* Log final test result */
   571                     switch (testResult) {
   572                     case TEST_RESULT_PASSED:
   573                         SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   574                         break;
   575                     case TEST_RESULT_FAILED:
   576                         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
   577                         break;
   578                     case TEST_RESULT_NO_ASSERT:
   579                         SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
   580                         break;
   581                     }
   582 
   583                 }
   584             }
   585 
   586             /* Take time - suite end */
   587             suiteEndSeconds = GetClock();
   588             runtime = suiteEndSeconds - suiteStartSeconds;
   589             if (runtime < 0.0f) runtime = 0.0f;
   590 
   591             /* Log suite runtime */
   592             SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
   593 
   594             /* Log summary and final Suite result */
   595             countSum = testPassedCount + testFailedCount + testSkippedCount;
   596             if (testFailedCount == 0)
   597             {
   598                 SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   599                 SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
   600             }
   601             else
   602             {
   603                 SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   604                 SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
   605             }
   606 
   607         }
   608     }
   609 
   610     /* Take time - run end */
   611     runEndSeconds = GetClock();
   612     runtime = runEndSeconds - runStartSeconds;
   613     if (runtime < 0.0f) runtime = 0.0f;
   614 
   615     /* Log total runtime */
   616     SDLTest_Log("Total Run runtime: %.1f sec", runtime);
   617 
   618     /* Log summary and final run result */
   619     countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   620     if (totalTestFailedCount == 0)
   621     {
   622         runResult = 0;
   623         SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   624         SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
   625     }
   626     else
   627     {
   628         runResult = 1;
   629         SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   630         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
   631     }
   632 
   633     SDLTest_Log("Exit code: %d", runResult);
   634     return runResult;
   635 }