src/test/SDL_test_harness.c
author Sam Lantinga <slouken@libsdl.org>
Wed, 04 Jun 2014 10:56:56 -0700
changeset 8820 0e935d5b193a
parent 8605 57faccca4fab
child 8831 9326ec96c132
permissions -rw-r--r--
Added annotations to help code analysis tools

CR: Bruce Dawson
     1 /*
     2   Simple DirectMedia Layer
     3   Copyright (C) 1997-2014 Sam Lantinga <slouken@libsdl.org>
     4 
     5   This software is provided 'as-is', without any express or implied
     6   warranty.  In no event will the authors be held liable for any damages
     7   arising from the use of this software.
     8 
     9   Permission is granted to anyone to use this software for any purpose,
    10   including commercial applications, and to alter it and redistribute it
    11   freely, subject to the following restrictions:
    12 
    13   1. The origin of this software must not be misrepresented; you must not
    14      claim that you wrote the original software. If you use this software
    15      in a product, an acknowledgment in the product documentation would be
    16      appreciated but is not required.
    17   2. Altered source versions must be plainly marked as such, and must not be
    18      misrepresented as being the original software.
    19   3. This notice may not be removed or altered from any source distribution.
    20 */
    21 
    22 #include "SDL_config.h"
    23 
    24 #include "SDL_test.h"
    25 
    26 #include <stdio.h>
    27 #include <stdlib.h>
    28 #include <string.h>
    29 #include <time.h>
    30 
    31 /* Invalid test name/description message format */
    32 const char *SDLTest_InvalidNameFormat = "(Invalid)";
    33 
    34 /* Log summary message format */
    35 const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d";
    36 
    37 /* Final result message format */
    38 const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n";
    39 
    40 /* ! \brief Timeout for single test case execution */
    41 static Uint32 SDLTest_TestCaseTimeout = 3600;
    42 
    43 /**
    44 * Generates a random run seed string for the harness. The generated seed
    45 * will contain alphanumeric characters (0-9A-Z).
    46 *
    47 * Note: The returned string needs to be deallocated by the caller.
    48 *
    49 * \param length The length of the seed string to generate
    50 *
    51 * \returns The generated seed string
    52 */
    53 char *
    54 SDLTest_GenerateRunSeed(const int length)
    55 {
    56     char *seed = NULL;
    57     SDLTest_RandomContext randomContext;
    58     int counter;
    59 
    60     /* Sanity check input */
    61     if (length <= 0) {
    62         SDLTest_LogError("The length of the harness seed must be >0.");
    63         return NULL;
    64     }
    65 
    66     /* Allocate output buffer */
    67     seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    68     if (seed == NULL) {
    69         SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    70         SDL_Error(SDL_ENOMEM);
    71         return NULL;
    72     }
    73 
    74     /* Generate a random string of alphanumeric characters */
    75     SDLTest_RandomInitTime(&randomContext);
    76     for (counter = 0; counter < length; counter++) {
    77         unsigned int number = SDLTest_Random(&randomContext);
    78         char ch = (char) (number % (91 - 48)) + 48;
    79         if (ch >= 58 && ch <= 64) {
    80             ch = 65;
    81         }
    82         seed[counter] = ch;
    83     }
    84     seed[length] = '\0';
    85 
    86     return seed;
    87 }
    88 
    89 /**
    90 * Generates an execution key for the fuzzer.
    91 *
    92 * \param runSeed        The run seed to use
    93 * \param suiteName      The name of the test suite
    94 * \param testName       The name of the test
    95 * \param iteration      The iteration count
    96 *
    97 * \returns The generated execution key to initialize the fuzzer with.
    98 *
    99 */
   100 Uint64
   101 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
   102 {
   103     SDLTest_Md5Context md5Context;
   104     Uint64 *keys;
   105     char iterationString[16];
   106     Uint32 runSeedLength;
   107     Uint32 suiteNameLength;
   108     Uint32 testNameLength;
   109     Uint32 iterationStringLength;
   110     Uint32 entireStringLength;
   111     char *buffer;
   112 
   113     if (runSeed == NULL || runSeed[0] == '\0') {
   114         SDLTest_LogError("Invalid runSeed string.");
   115         return -1;
   116     }
   117 
   118     if (suiteName == NULL || suiteName[0] == '\0') {
   119         SDLTest_LogError("Invalid suiteName string.");
   120         return -1;
   121     }
   122 
   123     if (testName == NULL || testName[0] == '\0') {
   124         SDLTest_LogError("Invalid testName string.");
   125         return -1;
   126     }
   127 
   128     if (iteration <= 0) {
   129         SDLTest_LogError("Invalid iteration count.");
   130         return -1;
   131     }
   132 
   133     /* Convert iteration number into a string */
   134     SDL_memset(iterationString, 0, sizeof(iterationString));
   135     SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
   136 
   137     /* Combine the parameters into single string */
   138     runSeedLength = SDL_strlen(runSeed);
   139     suiteNameLength = SDL_strlen(suiteName);
   140     testNameLength = SDL_strlen(testName);
   141     iterationStringLength = SDL_strlen(iterationString);
   142     entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
   143     buffer = (char *)SDL_malloc(entireStringLength);
   144     if (buffer == NULL) {
   145         SDLTest_LogError("Failed to allocate buffer for execKey generation.");
   146         SDL_Error(SDL_ENOMEM);
   147         return 0;
   148     }
   149     SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
   150 
   151     /* Hash string and use half of the digest as 64bit exec key */
   152     SDLTest_Md5Init(&md5Context);
   153     SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
   154     SDLTest_Md5Final(&md5Context);
   155     SDL_free(buffer);
   156     keys = (Uint64 *)md5Context.digest;
   157 
   158     return keys[0];
   159 }
   160 
   161 /**
   162 * \brief Set timeout handler for test.
   163 *
   164 * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   165 *
   166 * \param timeout Timeout interval in seconds.
   167 * \param callback Function that will be called after timeout has elapsed.
   168 *
   169 * \return Timer id or -1 on failure.
   170 */
   171 SDL_TimerID
   172 SDLTest_SetTestTimeout(int timeout, void (*callback)())
   173 {
   174     Uint32 timeoutInMilliseconds;
   175     SDL_TimerID timerID;
   176 
   177     if (callback == NULL) {
   178         SDLTest_LogError("Timeout callback can't be NULL");
   179         return -1;
   180     }
   181 
   182     if (timeout < 0) {
   183         SDLTest_LogError("Timeout value must be bigger than zero.");
   184         return -1;
   185     }
   186 
   187     /* Init SDL timer if not initialized before */
   188     if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
   189         if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
   190             SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
   191             return -1;
   192         }
   193     }
   194 
   195     /* Set timer */
   196     timeoutInMilliseconds = timeout * 1000;
   197     timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
   198     if (timerID == 0) {
   199         SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
   200         return -1;
   201     }
   202 
   203     return timerID;
   204 }
   205 
   206 /**
   207 * \brief Timeout handler. Aborts test run and exits harness process.
   208 */
   209 void
   210     SDLTest_BailOut()
   211 {
   212     SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   213     exit(TEST_ABORTED); /* bail out from the test */
   214 }
   215 
   216 /**
   217 * \brief Execute a test using the given execution key.
   218 *
   219 * \param testSuite Suite containing the test case.
   220 * \param testCase Case to execute.
   221 * \param execKey Execution key for the fuzzer.
   222 *
   223 * \returns Test case result.
   224 */
   225 int
   226 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
   227 {
   228     SDL_TimerID timer = 0;
   229     int testCaseResult = 0;
   230     int testResult = 0;
   231     int fuzzerCount;
   232 
   233     if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
   234     {
   235         SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
   236         return TEST_RESULT_SETUP_FAILURE;
   237     }
   238 
   239     if (!testCase->enabled)
   240     {
   241         SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Disabled)");
   242         return TEST_RESULT_SKIPPED;
   243     }
   244 
   245 
   246     /* Initialize fuzzer */
   247     SDLTest_FuzzerInit(execKey);
   248 
   249     /* Reset assert tracker */
   250     SDLTest_ResetAssertSummary();
   251 
   252     /* Set timeout timer */
   253     timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
   254 
   255     /* Maybe run suite initalizer function */
   256     if (testSuite->testSetUp) {
   257         testSuite->testSetUp(0x0);
   258         if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
   259             SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed");
   260             return TEST_RESULT_SETUP_FAILURE;
   261         }
   262     }
   263 
   264     /* Run test case function */
   265     testCaseResult = testCase->testCase(0x0);
   266 
   267     /* Convert test execution result into harness result */
   268     if (testCaseResult == TEST_SKIPPED) {
   269         /* Test was programatically skipped */
   270         testResult = TEST_RESULT_SKIPPED;
   271     } else if (testCaseResult == TEST_STARTED) {
   272         /* Test did not return a TEST_COMPLETED value; assume it failed */
   273         testResult = TEST_RESULT_FAILED;
   274     } else if (testCaseResult == TEST_ABORTED) {
   275         /* Test was aborted early; assume it failed */
   276         testResult = TEST_RESULT_FAILED;
   277     } else {
   278         /* Perform failure analysis based on asserts */
   279         testResult = SDLTest_AssertSummaryToTestResult();
   280     }
   281 
   282     /* Maybe run suite cleanup function (ignore failed asserts) */
   283     if (testSuite->testTearDown) {
   284         testSuite->testTearDown(0x0);
   285     }
   286 
   287     /* Cancel timeout timer */
   288     if (timer) {
   289         SDL_RemoveTimer(timer);
   290     }
   291 
   292     /* Report on asserts and fuzzer usage */
   293     fuzzerCount = SDLTest_GetFuzzerInvocationCount();
   294     if (fuzzerCount > 0) {
   295         SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
   296     }
   297 
   298     /* Final log based on test execution result */
   299     if (testCaseResult == TEST_SKIPPED) {
   300         /* Test was programatically skipped */
   301         SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)");
   302     } else if (testCaseResult == TEST_STARTED) {
   303         /* Test did not return a TEST_COMPLETED value; assume it failed */
   304         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
   305     } else if (testCaseResult == TEST_ABORTED) {
   306         /* Test was aborted early; assume it failed */
   307         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)");
   308     } else {
   309         SDLTest_LogAssertSummary();
   310     }
   311 
   312     return testResult;
   313 }
   314 
   315 /* Prints summary of all suites/tests contained in the given reference */
   316 void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
   317 {
   318     int suiteCounter;
   319     int testCounter;
   320     SDLTest_TestSuiteReference *testSuite;
   321     SDLTest_TestCaseReference *testCase;
   322 
   323     /* Loop over all suites */
   324     suiteCounter = 0;
   325     while(&testSuites[suiteCounter]) {
   326         testSuite=&testSuites[suiteCounter];
   327         suiteCounter++;
   328         SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
   329             (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   330 
   331         /* Loop over all test cases */
   332         testCounter = 0;
   333         while(testSuite->testCases[testCounter])
   334         {
   335             testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   336             testCounter++;
   337             SDLTest_Log("  Test Case %i - %s: %s", testCounter,
   338                 (testCase->name) ? testCase->name : SDLTest_InvalidNameFormat,
   339                 (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   340         }
   341     }
   342 }
   343 
   344 /* Gets a timer value in seconds */
   345 float GetClock()
   346 {
   347     float currentClock = (float)clock();
   348     return currentClock / (float)CLOCKS_PER_SEC;
   349 }
   350 
   351 /**
   352 * \brief Execute a test suite using the given run seed and execution key.
   353 *
   354 * The filter string is matched to the suite name (full comparison) to select a single suite,
   355 * or if no suite matches, it is matched to the test names (full comparison) to select a single test.
   356 *
   357 * \param testSuites Suites containing the test case.
   358 * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   359 * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   360 * \param filter Filter specification. NULL disables. Case sensitive.
   361 * \param testIterations Number of iterations to run each test case.
   362 *
   363 * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   364 */
   365 int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
   366 {
   367     int totalNumberOfTests = 0;
   368     int failedNumberOfTests = 0;
   369     int suiteCounter;
   370     int testCounter;
   371     int iterationCounter;
   372     SDLTest_TestSuiteReference *testSuite;
   373     SDLTest_TestCaseReference *testCase;
   374     const char *runSeed = NULL;
   375     char *currentSuiteName;
   376     char *currentTestName;
   377     Uint64 execKey;
   378     float runStartSeconds;
   379     float suiteStartSeconds;
   380     float testStartSeconds;
   381     float runEndSeconds;
   382     float suiteEndSeconds;
   383     float testEndSeconds;
   384     float runtime;
   385     int suiteFilter = 0;
   386     char *suiteFilterName = NULL;
   387     int testFilter = 0;
   388     char *testFilterName = NULL;
   389     int testResult = 0;
   390     int runResult = 0;
   391     Uint32 totalTestFailedCount = 0;
   392     Uint32 totalTestPassedCount = 0;
   393     Uint32 totalTestSkippedCount = 0;
   394     Uint32 testFailedCount = 0;
   395     Uint32 testPassedCount = 0;
   396     Uint32 testSkippedCount = 0;
   397     Uint32 countSum = 0;
   398     char *logFormat = (char *)SDLTest_LogSummaryFormat;
   399     SDLTest_TestCaseReference **failedTests;
   400 
   401     /* Sanitize test iterations */
   402     if (testIterations < 1) {
   403         testIterations = 1;
   404     }
   405 
   406     /* Generate run see if we don't have one already */
   407     if (userRunSeed == NULL || userRunSeed[0] == '\0') {
   408         runSeed = SDLTest_GenerateRunSeed(16);
   409         if (runSeed == NULL) {
   410             SDLTest_LogError("Generating a random seed failed");
   411             return 2;
   412         }
   413     } else {
   414         runSeed = userRunSeed;
   415     }
   416 
   417 
   418     /* Reset per-run counters */
   419     totalTestFailedCount = 0;
   420     totalTestPassedCount = 0;
   421     totalTestSkippedCount = 0;
   422 
   423     /* Take time - run start */
   424     runStartSeconds = GetClock();
   425 
   426     /* Log run with fuzzer parameters */
   427     SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   428 
   429 	/* Count the total number of tests */
   430     suiteCounter = 0;
   431     while (testSuites[suiteCounter]) {
   432         testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   433         suiteCounter++;
   434         testCounter = 0;
   435         while (testSuite->testCases[testCounter])
   436         {
   437             testCounter++;
   438 			totalNumberOfTests++;
   439 		}
   440 	}
   441 
   442 	/* Pre-allocate an array for tracking failed tests (potentially all test cases) */
   443 	failedTests = (SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
   444 	if (failedTests == NULL) {	
   445 	   SDLTest_LogError("Unable to allocate cache for failed tests");
   446            SDL_Error(SDL_ENOMEM);	   
   447            return -1;
   448 	}
   449 
   450     /* Initialize filtering */
   451     if (filter != NULL && filter[0] != '\0') {
   452         /* Loop over all suites to check if we have a filter match */
   453         suiteCounter = 0;
   454         while (testSuites[suiteCounter] && suiteFilter == 0) {
   455             testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   456             suiteCounter++;
   457             if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
   458                 /* Matched a suite name */
   459                 suiteFilter = 1;
   460                 suiteFilterName = testSuite->name;
   461                 SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
   462                 break;
   463             }
   464 
   465             /* Within each suite, loop over all test cases to check if we have a filter match */
   466             testCounter = 0;
   467             while (testSuite->testCases[testCounter] && testFilter == 0)
   468             {
   469                 testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   470                 testCounter++;
   471                 if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
   472                     /* Matched a test name */
   473                     suiteFilter = 1;
   474                     suiteFilterName = testSuite->name;
   475                     testFilter = 1;
   476                     testFilterName = testCase->name;
   477                     SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
   478                     break;
   479                 }
   480             }
   481         }
   482 
   483         if (suiteFilter == 0 && testFilter == 0) {
   484             SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
   485             SDLTest_Log("Exit code: 2");
   486             return 2;
   487         }
   488     }
   489 
   490     /* Loop over all suites */
   491     suiteCounter = 0;
   492     while(testSuites[suiteCounter]) {
   493         testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   494         currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   495         suiteCounter++;
   496 
   497         /* Filter suite if flag set and we have a name */
   498         if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
   499             SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
   500                 /* Skip suite */
   501                 SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
   502                     suiteCounter,
   503                     currentSuiteName);
   504         } else {
   505 
   506             /* Reset per-suite counters */
   507             testFailedCount = 0;
   508             testPassedCount = 0;
   509             testSkippedCount = 0;
   510 
   511             /* Take time - suite start */
   512             suiteStartSeconds = GetClock();
   513 
   514             /* Log suite started */
   515             SDLTest_Log("===== Test Suite %i: '%s' started\n",
   516                 suiteCounter,
   517                 currentSuiteName);
   518 
   519             /* Loop over all test cases */
   520             testCounter = 0;
   521             while(testSuite->testCases[testCounter])
   522             {
   523                 testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   524                 currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   525                 testCounter++;
   526 
   527                 /* Filter tests if flag set and we have a name */
   528                 if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
   529                     SDL_strcmp(testFilterName, testCase->name) != 0) {
   530                         /* Skip test */
   531                         SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
   532                             suiteCounter,
   533                             testCounter,
   534                             currentTestName);
   535                 } else {
   536                     /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
   537                     if (testFilter == 1 && !testCase->enabled) {
   538                         SDLTest_Log("Force run of disabled test since test filter was set");
   539                         testCase->enabled = 1;
   540                     }
   541 
   542                     /* Take time - test start */
   543                     testStartSeconds = GetClock();
   544 
   545                     /* Log test started */
   546                     SDLTest_Log("----- Test Case %i.%i: '%s' started",
   547                         suiteCounter,
   548                         testCounter,
   549                         currentTestName);
   550                     if (testCase->description != NULL && testCase->description[0] != '\0') {
   551                         SDLTest_Log("Test Description: '%s'",
   552                             (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   553                     }
   554 
   555                     /* Loop over all iterations */
   556                     iterationCounter = 0;
   557                     while(iterationCounter < testIterations)
   558                     {
   559                         iterationCounter++;
   560 
   561                         if (userExecKey != 0) {
   562                             execKey = userExecKey;
   563                         } else {
   564                             execKey = SDLTest_GenerateExecKey((char *)runSeed, testSuite->name, testCase->name, iterationCounter);
   565                         }
   566 
   567                         SDLTest_Log("Test Iteration %i: execKey %llu", iterationCounter, execKey);
   568                         testResult = SDLTest_RunTest(testSuite, testCase, execKey);
   569 
   570                         if (testResult == TEST_RESULT_PASSED) {
   571                             testPassedCount++;
   572                             totalTestPassedCount++;
   573                         } else if (testResult == TEST_RESULT_SKIPPED) {
   574                             testSkippedCount++;
   575                             totalTestSkippedCount++;
   576                         } else {
   577                             testFailedCount++;
   578                             totalTestFailedCount++;
   579                         }
   580                     }
   581 
   582                     /* Take time - test end */
   583                     testEndSeconds = GetClock();
   584                     runtime = testEndSeconds - testStartSeconds;
   585                     if (runtime < 0.0f) runtime = 0.0f;
   586 
   587                     if (testIterations > 1) {
   588                         /* Log test runtime */
   589                         SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
   590                         SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
   591                     } else {
   592                         /* Log test runtime */
   593                         SDLTest_Log("Total Test runtime: %.1f sec", runtime);
   594                     }
   595 
   596                     /* Log final test result */
   597                     switch (testResult) {
   598                     case TEST_RESULT_PASSED:
   599                         SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   600                         break;
   601                     case TEST_RESULT_FAILED:
   602                         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
   603                         break;
   604                     case TEST_RESULT_NO_ASSERT:
   605                         SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
   606                         break;
   607                     }
   608 
   609                     /* Collect failed test case references for repro-step display */
   610                     if (testResult == TEST_RESULT_FAILED) {
   611                         failedTests[failedNumberOfTests] = testCase;
   612                         failedNumberOfTests++;
   613                     }
   614                 }
   615             }
   616 
   617             /* Take time - suite end */
   618             suiteEndSeconds = GetClock();
   619             runtime = suiteEndSeconds - suiteStartSeconds;
   620             if (runtime < 0.0f) runtime = 0.0f;
   621 
   622             /* Log suite runtime */
   623             SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
   624 
   625             /* Log summary and final Suite result */
   626             countSum = testPassedCount + testFailedCount + testSkippedCount;
   627             if (testFailedCount == 0)
   628             {
   629                 SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   630                 SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
   631             }
   632             else
   633             {
   634                 SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   635                 SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
   636             }
   637 
   638         }
   639     }
   640 
   641     /* Take time - run end */
   642     runEndSeconds = GetClock();
   643     runtime = runEndSeconds - runStartSeconds;
   644     if (runtime < 0.0f) runtime = 0.0f;
   645 
   646     /* Log total runtime */
   647     SDLTest_Log("Total Run runtime: %.1f sec", runtime);
   648 
   649     /* Log summary and final run result */
   650     countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   651     if (totalTestFailedCount == 0)
   652     {
   653         runResult = 0;
   654         SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   655         SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
   656     }
   657     else
   658     {
   659         runResult = 1;
   660         SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   661         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
   662     }
   663 
   664     /* Print repro steps for failed tests */
   665     if (failedNumberOfTests > 0) {
   666         SDLTest_Log("Harness input to repro failures:");
   667         for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
   668           SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
   669         }
   670     }
   671     SDL_free(failedTests);
   672 
   673     SDLTest_Log("Exit code: %d", runResult);
   674     return runResult;
   675 }