src/test/SDL_test_harness.c
author Ryan C. Gordon
Sun, 03 Jan 2016 20:52:44 -0500
changeset 10004 8f2f519d1e61
parent 9998 f67cf37e9cd4
child 10185 8eeab279c545
permissions -rw-r--r--
CMake: Don't make a libSDL2.so symlink on Mac OS X (do .dylib instead).
     1 /*
     2   Simple DirectMedia Layer
     3   Copyright (C) 1997-2016 Sam Lantinga <slouken@libsdl.org>
     4 
     5   This software is provided 'as-is', without any express or implied
     6   warranty.  In no event will the authors be held liable for any damages
     7   arising from the use of this software.
     8 
     9   Permission is granted to anyone to use this software for any purpose,
    10   including commercial applications, and to alter it and redistribute it
    11   freely, subject to the following restrictions:
    12 
    13   1. The origin of this software must not be misrepresented; you must not
    14      claim that you wrote the original software. If you use this software
    15      in a product, an acknowledgment in the product documentation would be
    16      appreciated but is not required.
    17   2. Altered source versions must be plainly marked as such, and must not be
    18      misrepresented as being the original software.
    19   3. This notice may not be removed or altered from any source distribution.
    20 */
    21 
    22 #include "SDL_config.h"
    23 
    24 #include "SDL_test.h"
    25 
    26 #include <stdio.h>
    27 #include <stdlib.h>
    28 #include <string.h>
    29 #include <time.h>
    30 
    31 /* Invalid test name/description message format */
    32 const char *SDLTest_InvalidNameFormat = "(Invalid)";
    33 
    34 /* Log summary message format */
    35 const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d";
    36 
    37 /* Final result message format */
    38 const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n";
    39 
    40 /* ! \brief Timeout for single test case execution */
    41 static Uint32 SDLTest_TestCaseTimeout = 3600;
    42 
    43 /**
    44 * Generates a random run seed string for the harness. The generated seed
    45 * will contain alphanumeric characters (0-9A-Z).
    46 *
    47 * Note: The returned string needs to be deallocated by the caller.
    48 *
    49 * \param length The length of the seed string to generate
    50 *
    51 * \returns The generated seed string
    52 */
    53 char *
    54 SDLTest_GenerateRunSeed(const int length)
    55 {
    56     char *seed = NULL;
    57     SDLTest_RandomContext randomContext;
    58     int counter;
    59 
    60     /* Sanity check input */
    61     if (length <= 0) {
    62         SDLTest_LogError("The length of the harness seed must be >0.");
    63         return NULL;
    64     }
    65 
    66     /* Allocate output buffer */
    67     seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    68     if (seed == NULL) {
    69         SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    70         SDL_Error(SDL_ENOMEM);
    71         return NULL;
    72     }
    73 
    74     /* Generate a random string of alphanumeric characters */
    75     SDLTest_RandomInitTime(&randomContext);
    76     for (counter = 0; counter < length; counter++) {
    77         unsigned int number = SDLTest_Random(&randomContext);
    78         char ch = (char) (number % (91 - 48)) + 48;
    79         if (ch >= 58 && ch <= 64) {
    80             ch = 65;
    81         }
    82         seed[counter] = ch;
    83     }
    84     seed[length] = '\0';
    85 
    86     return seed;
    87 }
    88 
    89 /**
    90 * Generates an execution key for the fuzzer.
    91 *
    92 * \param runSeed        The run seed to use
    93 * \param suiteName      The name of the test suite
    94 * \param testName       The name of the test
    95 * \param iteration      The iteration count
    96 *
    97 * \returns The generated execution key to initialize the fuzzer with.
    98 *
    99 */
   100 Uint64
   101 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
   102 {
   103     SDLTest_Md5Context md5Context;
   104     Uint64 *keys;
   105     char iterationString[16];
   106     Uint32 runSeedLength;
   107     Uint32 suiteNameLength;
   108     Uint32 testNameLength;
   109     Uint32 iterationStringLength;
   110     Uint32 entireStringLength;
   111     char *buffer;
   112 
   113     if (runSeed == NULL || runSeed[0] == '\0') {
   114         SDLTest_LogError("Invalid runSeed string.");
   115         return -1;
   116     }
   117 
   118     if (suiteName == NULL || suiteName[0] == '\0') {
   119         SDLTest_LogError("Invalid suiteName string.");
   120         return -1;
   121     }
   122 
   123     if (testName == NULL || testName[0] == '\0') {
   124         SDLTest_LogError("Invalid testName string.");
   125         return -1;
   126     }
   127 
   128     if (iteration <= 0) {
   129         SDLTest_LogError("Invalid iteration count.");
   130         return -1;
   131     }
   132 
   133     /* Convert iteration number into a string */
   134     SDL_memset(iterationString, 0, sizeof(iterationString));
   135     SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
   136 
   137     /* Combine the parameters into single string */
   138     runSeedLength = SDL_strlen(runSeed);
   139     suiteNameLength = SDL_strlen(suiteName);
   140     testNameLength = SDL_strlen(testName);
   141     iterationStringLength = SDL_strlen(iterationString);
   142     entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
   143     buffer = (char *)SDL_malloc(entireStringLength);
   144     if (buffer == NULL) {
   145         SDLTest_LogError("Failed to allocate buffer for execKey generation.");
   146         SDL_Error(SDL_ENOMEM);
   147         return 0;
   148     }
   149     SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
   150 
   151     /* Hash string and use half of the digest as 64bit exec key */
   152     SDLTest_Md5Init(&md5Context);
   153     SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
   154     SDLTest_Md5Final(&md5Context);
   155     SDL_free(buffer);
   156     keys = (Uint64 *)md5Context.digest;
   157 
   158     return keys[0];
   159 }
   160 
   161 /**
   162 * \brief Set timeout handler for test.
   163 *
   164 * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   165 *
   166 * \param timeout Timeout interval in seconds.
   167 * \param callback Function that will be called after timeout has elapsed.
   168 *
   169 * \return Timer id or -1 on failure.
   170 */
   171 SDL_TimerID
   172 SDLTest_SetTestTimeout(int timeout, void (*callback)())
   173 {
   174     Uint32 timeoutInMilliseconds;
   175     SDL_TimerID timerID;
   176 
   177     if (callback == NULL) {
   178         SDLTest_LogError("Timeout callback can't be NULL");
   179         return -1;
   180     }
   181 
   182     if (timeout < 0) {
   183         SDLTest_LogError("Timeout value must be bigger than zero.");
   184         return -1;
   185     }
   186 
   187     /* Init SDL timer if not initialized before */
   188     if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
   189         if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
   190             SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
   191             return -1;
   192         }
   193     }
   194 
   195     /* Set timer */
   196     timeoutInMilliseconds = timeout * 1000;
   197     timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
   198     if (timerID == 0) {
   199         SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
   200         return -1;
   201     }
   202 
   203     return timerID;
   204 }
   205 
   206 /**
   207 * \brief Timeout handler. Aborts test run and exits harness process.
   208 */
   209 void
   210     SDLTest_BailOut()
   211 {
   212     SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   213     exit(TEST_ABORTED); /* bail out from the test */
   214 }
   215 
   216 /**
   217 * \brief Execute a test using the given execution key.
   218 *
   219 * \param testSuite Suite containing the test case.
   220 * \param testCase Case to execute.
   221 * \param execKey Execution key for the fuzzer.
   222 * \param forceTestRun Force test to run even if test was disabled in suite.
   223 *
   224 * \returns Test case result.
   225 */
   226 int
   227 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
   228 {
   229     SDL_TimerID timer = 0;
   230     int testCaseResult = 0;
   231     int testResult = 0;
   232     int fuzzerCount;
   233 
   234     if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
   235     {
   236         SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
   237         return TEST_RESULT_SETUP_FAILURE;
   238     }
   239 
   240 	if (!testCase->enabled && forceTestRun == SDL_FALSE)
   241     {
   242         SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Disabled)");
   243         return TEST_RESULT_SKIPPED;
   244     }
   245 
   246     /* Initialize fuzzer */
   247     SDLTest_FuzzerInit(execKey);
   248 
   249     /* Reset assert tracker */
   250     SDLTest_ResetAssertSummary();
   251 
   252     /* Set timeout timer */
   253     timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
   254 
   255     /* Maybe run suite initalizer function */
   256     if (testSuite->testSetUp) {
   257         testSuite->testSetUp(0x0);
   258         if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
   259             SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed");
   260             return TEST_RESULT_SETUP_FAILURE;
   261         }
   262     }
   263 
   264     /* Run test case function */
   265     testCaseResult = testCase->testCase(0x0);
   266 
   267     /* Convert test execution result into harness result */
   268     if (testCaseResult == TEST_SKIPPED) {
   269         /* Test was programatically skipped */
   270         testResult = TEST_RESULT_SKIPPED;
   271     } else if (testCaseResult == TEST_STARTED) {
   272         /* Test did not return a TEST_COMPLETED value; assume it failed */
   273         testResult = TEST_RESULT_FAILED;
   274     } else if (testCaseResult == TEST_ABORTED) {
   275         /* Test was aborted early; assume it failed */
   276         testResult = TEST_RESULT_FAILED;
   277     } else {
   278         /* Perform failure analysis based on asserts */
   279         testResult = SDLTest_AssertSummaryToTestResult();
   280     }
   281 
   282     /* Maybe run suite cleanup function (ignore failed asserts) */
   283     if (testSuite->testTearDown) {
   284         testSuite->testTearDown(0x0);
   285     }
   286 
   287     /* Cancel timeout timer */
   288     if (timer) {
   289         SDL_RemoveTimer(timer);
   290     }
   291 
   292     /* Report on asserts and fuzzer usage */
   293     fuzzerCount = SDLTest_GetFuzzerInvocationCount();
   294     if (fuzzerCount > 0) {
   295         SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
   296     }
   297 
   298     /* Final log based on test execution result */
   299     if (testCaseResult == TEST_SKIPPED) {
   300         /* Test was programatically skipped */
   301         SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)");
   302     } else if (testCaseResult == TEST_STARTED) {
   303         /* Test did not return a TEST_COMPLETED value; assume it failed */
   304         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
   305     } else if (testCaseResult == TEST_ABORTED) {
   306         /* Test was aborted early; assume it failed */
   307         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)");
   308     } else {
   309         SDLTest_LogAssertSummary();
   310     }
   311 
   312     return testResult;
   313 }
   314 
   315 /* Prints summary of all suites/tests contained in the given reference */
   316 void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
   317 {
   318     int suiteCounter;
   319     int testCounter;
   320     SDLTest_TestSuiteReference *testSuite;
   321     SDLTest_TestCaseReference *testCase;
   322 
   323     /* Loop over all suites */
   324     suiteCounter = 0;
   325     while(&testSuites[suiteCounter]) {
   326         testSuite=&testSuites[suiteCounter];
   327         suiteCounter++;
   328         SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
   329             (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   330 
   331         /* Loop over all test cases */
   332         testCounter = 0;
   333         while(testSuite->testCases[testCounter])
   334         {
   335             testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   336             testCounter++;
   337             SDLTest_Log("  Test Case %i - %s: %s", testCounter,
   338                 (testCase->name) ? testCase->name : SDLTest_InvalidNameFormat,
   339                 (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   340         }
   341     }
   342 }
   343 
   344 /* Gets a timer value in seconds */
   345 float GetClock()
   346 {
   347     float currentClock = (float)clock();
   348     return currentClock / (float)CLOCKS_PER_SEC;
   349 }
   350 
   351 /**
   352 * \brief Execute a test suite using the given run seed and execution key.
   353 *
   354 * The filter string is matched to the suite name (full comparison) to select a single suite,
   355 * or if no suite matches, it is matched to the test names (full comparison) to select a single test.
   356 *
   357 * \param testSuites Suites containing the test case.
   358 * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   359 * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   360 * \param filter Filter specification. NULL disables. Case sensitive.
   361 * \param testIterations Number of iterations to run each test case.
   362 *
   363 * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   364 */
   365 int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
   366 {
   367     int totalNumberOfTests = 0;
   368     int failedNumberOfTests = 0;
   369     int suiteCounter;
   370     int testCounter;
   371     int iterationCounter;
   372     SDLTest_TestSuiteReference *testSuite;
   373     SDLTest_TestCaseReference *testCase;
   374     const char *runSeed = NULL;
   375     char *currentSuiteName;
   376     char *currentTestName;
   377     Uint64 execKey;
   378     float runStartSeconds;
   379     float suiteStartSeconds;
   380     float testStartSeconds;
   381     float runEndSeconds;
   382     float suiteEndSeconds;
   383     float testEndSeconds;
   384     float runtime;
   385     int suiteFilter = 0;
   386     char *suiteFilterName = NULL;
   387     int testFilter = 0;
   388     char *testFilterName = NULL;
   389 	SDL_bool forceTestRun = SDL_FALSE;
   390     int testResult = 0;
   391     int runResult = 0;
   392     Uint32 totalTestFailedCount = 0;
   393     Uint32 totalTestPassedCount = 0;
   394     Uint32 totalTestSkippedCount = 0;
   395     Uint32 testFailedCount = 0;
   396     Uint32 testPassedCount = 0;
   397     Uint32 testSkippedCount = 0;
   398     Uint32 countSum = 0;
   399     char *logFormat = (char *)SDLTest_LogSummaryFormat;
   400     SDLTest_TestCaseReference **failedTests;
   401 
   402     /* Sanitize test iterations */
   403     if (testIterations < 1) {
   404         testIterations = 1;
   405     }
   406 
   407     /* Generate run see if we don't have one already */
   408     if (userRunSeed == NULL || userRunSeed[0] == '\0') {
   409         runSeed = SDLTest_GenerateRunSeed(16);
   410         if (runSeed == NULL) {
   411             SDLTest_LogError("Generating a random seed failed");
   412             return 2;
   413         }
   414     } else {
   415         runSeed = userRunSeed;
   416     }
   417 
   418 
   419     /* Reset per-run counters */
   420     totalTestFailedCount = 0;
   421     totalTestPassedCount = 0;
   422     totalTestSkippedCount = 0;
   423 
   424     /* Take time - run start */
   425     runStartSeconds = GetClock();
   426 
   427     /* Log run with fuzzer parameters */
   428     SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   429 
   430 	/* Count the total number of tests */
   431     suiteCounter = 0;
   432     while (testSuites[suiteCounter]) {
   433         testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   434         suiteCounter++;
   435         testCounter = 0;
   436         while (testSuite->testCases[testCounter])
   437         {
   438             testCounter++;
   439 			totalNumberOfTests++;
   440 		}
   441 	}
   442 
   443 	/* Pre-allocate an array for tracking failed tests (potentially all test cases) */
   444 	failedTests = (SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
   445 	if (failedTests == NULL) {	
   446 	   SDLTest_LogError("Unable to allocate cache for failed tests");
   447            SDL_Error(SDL_ENOMEM);	   
   448            return -1;
   449 	}
   450 
   451     /* Initialize filtering */
   452     if (filter != NULL && filter[0] != '\0') {
   453         /* Loop over all suites to check if we have a filter match */
   454         suiteCounter = 0;
   455         while (testSuites[suiteCounter] && suiteFilter == 0) {
   456             testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   457             suiteCounter++;
   458             if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
   459                 /* Matched a suite name */
   460                 suiteFilter = 1;
   461                 suiteFilterName = testSuite->name;
   462                 SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
   463                 break;
   464             }
   465 
   466             /* Within each suite, loop over all test cases to check if we have a filter match */
   467             testCounter = 0;
   468             while (testSuite->testCases[testCounter] && testFilter == 0)
   469             {
   470                 testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   471                 testCounter++;
   472                 if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
   473                     /* Matched a test name */
   474                     suiteFilter = 1;
   475                     suiteFilterName = testSuite->name;
   476                     testFilter = 1;
   477                     testFilterName = testCase->name;
   478                     SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
   479                     break;
   480                 }
   481             }
   482         }
   483 
   484         if (suiteFilter == 0 && testFilter == 0) {
   485             SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
   486             SDLTest_Log("Exit code: 2");
   487             SDL_free(failedTests);
   488             return 2;
   489         }
   490     }
   491 
   492     /* Loop over all suites */
   493     suiteCounter = 0;
   494     while(testSuites[suiteCounter]) {
   495         testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   496         currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   497         suiteCounter++;
   498 
   499         /* Filter suite if flag set and we have a name */
   500         if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
   501             SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
   502                 /* Skip suite */
   503                 SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
   504                     suiteCounter,
   505                     currentSuiteName);
   506         } else {
   507 
   508             /* Reset per-suite counters */
   509             testFailedCount = 0;
   510             testPassedCount = 0;
   511             testSkippedCount = 0;
   512 
   513             /* Take time - suite start */
   514             suiteStartSeconds = GetClock();
   515 
   516             /* Log suite started */
   517             SDLTest_Log("===== Test Suite %i: '%s' started\n",
   518                 suiteCounter,
   519                 currentSuiteName);
   520 
   521             /* Loop over all test cases */
   522             testCounter = 0;
   523             while(testSuite->testCases[testCounter])
   524             {
   525                 testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   526                 currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   527                 testCounter++;
   528 
   529                 /* Filter tests if flag set and we have a name */
   530                 if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
   531                     SDL_strcmp(testFilterName, testCase->name) != 0) {
   532                         /* Skip test */
   533                         SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
   534                             suiteCounter,
   535                             testCounter,
   536                             currentTestName);
   537                 } else {
   538                     /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
   539                     if (testFilter == 1 && !testCase->enabled) {
   540                         SDLTest_Log("Force run of disabled test since test filter was set");
   541 						forceTestRun = SDL_TRUE;
   542                     }
   543 
   544                     /* Take time - test start */
   545                     testStartSeconds = GetClock();
   546 
   547                     /* Log test started */
   548                     SDLTest_Log("----- Test Case %i.%i: '%s' started",
   549                         suiteCounter,
   550                         testCounter,
   551                         currentTestName);
   552                     if (testCase->description != NULL && testCase->description[0] != '\0') {
   553                         SDLTest_Log("Test Description: '%s'",
   554                             (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   555                     }
   556 
   557                     /* Loop over all iterations */
   558                     iterationCounter = 0;
   559                     while(iterationCounter < testIterations)
   560                     {
   561                         iterationCounter++;
   562 
   563                         if (userExecKey != 0) {
   564                             execKey = userExecKey;
   565                         } else {
   566                             execKey = SDLTest_GenerateExecKey((char *)runSeed, testSuite->name, testCase->name, iterationCounter);
   567                         }
   568 
   569                         SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
   570 						testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
   571 
   572                         if (testResult == TEST_RESULT_PASSED) {
   573                             testPassedCount++;
   574                             totalTestPassedCount++;
   575                         } else if (testResult == TEST_RESULT_SKIPPED) {
   576                             testSkippedCount++;
   577                             totalTestSkippedCount++;
   578                         } else {
   579                             testFailedCount++;
   580                             totalTestFailedCount++;
   581                         }
   582                     }
   583 
   584                     /* Take time - test end */
   585                     testEndSeconds = GetClock();
   586                     runtime = testEndSeconds - testStartSeconds;
   587                     if (runtime < 0.0f) runtime = 0.0f;
   588 
   589                     if (testIterations > 1) {
   590                         /* Log test runtime */
   591                         SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
   592                         SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
   593                     } else {
   594                         /* Log test runtime */
   595                         SDLTest_Log("Total Test runtime: %.1f sec", runtime);
   596                     }
   597 
   598                     /* Log final test result */
   599                     switch (testResult) {
   600                     case TEST_RESULT_PASSED:
   601                         SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   602                         break;
   603                     case TEST_RESULT_FAILED:
   604                         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
   605                         break;
   606                     case TEST_RESULT_NO_ASSERT:
   607                         SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
   608                         break;
   609                     }
   610 
   611                     /* Collect failed test case references for repro-step display */
   612                     if (testResult == TEST_RESULT_FAILED) {
   613                         failedTests[failedNumberOfTests] = testCase;
   614                         failedNumberOfTests++;
   615                     }
   616                 }
   617             }
   618 
   619             /* Take time - suite end */
   620             suiteEndSeconds = GetClock();
   621             runtime = suiteEndSeconds - suiteStartSeconds;
   622             if (runtime < 0.0f) runtime = 0.0f;
   623 
   624             /* Log suite runtime */
   625             SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
   626 
   627             /* Log summary and final Suite result */
   628             countSum = testPassedCount + testFailedCount + testSkippedCount;
   629             if (testFailedCount == 0)
   630             {
   631                 SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   632                 SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
   633             }
   634             else
   635             {
   636                 SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   637                 SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
   638             }
   639 
   640         }
   641     }
   642 
   643     /* Take time - run end */
   644     runEndSeconds = GetClock();
   645     runtime = runEndSeconds - runStartSeconds;
   646     if (runtime < 0.0f) runtime = 0.0f;
   647 
   648     /* Log total runtime */
   649     SDLTest_Log("Total Run runtime: %.1f sec", runtime);
   650 
   651     /* Log summary and final run result */
   652     countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   653     if (totalTestFailedCount == 0)
   654     {
   655         runResult = 0;
   656         SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   657         SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
   658     }
   659     else
   660     {
   661         runResult = 1;
   662         SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   663         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
   664     }
   665 
   666     /* Print repro steps for failed tests */
   667     if (failedNumberOfTests > 0) {
   668         SDLTest_Log("Harness input to repro failures:");
   669         for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
   670           SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
   671         }
   672     }
   673     SDL_free(failedTests);
   674 
   675     SDLTest_Log("Exit code: %d", runResult);
   676     return runResult;
   677 }