src/test/SDL_test_harness.c
author Philipp Wiesemann
Tue, 30 Aug 2016 21:16:04 +0200
changeset 10298 06e7d0ccbc11
parent 10185 8eeab279c545
child 10609 d702ecbd8ba7
permissions -rw-r--r--
Fixed log message in audio capture test program.
     1 /*
     2   Simple DirectMedia Layer
     3   Copyright (C) 1997-2016 Sam Lantinga <slouken@libsdl.org>
     4 
     5   This software is provided 'as-is', without any express or implied
     6   warranty.  In no event will the authors be held liable for any damages
     7   arising from the use of this software.
     8 
     9   Permission is granted to anyone to use this software for any purpose,
    10   including commercial applications, and to alter it and redistribute it
    11   freely, subject to the following restrictions:
    12 
    13   1. The origin of this software must not be misrepresented; you must not
    14      claim that you wrote the original software. If you use this software
    15      in a product, an acknowledgment in the product documentation would be
    16      appreciated but is not required.
    17   2. Altered source versions must be plainly marked as such, and must not be
    18      misrepresented as being the original software.
    19   3. This notice may not be removed or altered from any source distribution.
    20 */
    21 
    22 #include "SDL_config.h"
    23 
    24 #include "SDL_test.h"
    25 
    26 #include <stdio.h>
    27 #include <stdlib.h>
    28 #include <string.h>
    29 #include <time.h>
    30 
    31 /* Invalid test name/description message format */
    32 #define SDLTEST_INVALID_NAME_FORMAT "(Invalid)"
    33 
    34 /* Log summary message format */
    35 #define SDLTEST_LOG_SUMMARY_FORMAT "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"
    36 
    37 /* Final result message format */
    38 #define SDLTEST_FINAL_RESULT_FORMAT ">>> %s '%s': %s\n"
    39 
    40 /* ! \brief Timeout for single test case execution */
    41 static Uint32 SDLTest_TestCaseTimeout = 3600;
    42 
    43 /**
    44 * Generates a random run seed string for the harness. The generated seed
    45 * will contain alphanumeric characters (0-9A-Z).
    46 *
    47 * Note: The returned string needs to be deallocated by the caller.
    48 *
    49 * \param length The length of the seed string to generate
    50 *
    51 * \returns The generated seed string
    52 */
    53 char *
    54 SDLTest_GenerateRunSeed(const int length)
    55 {
    56     char *seed = NULL;
    57     SDLTest_RandomContext randomContext;
    58     int counter;
    59 
    60     /* Sanity check input */
    61     if (length <= 0) {
    62         SDLTest_LogError("The length of the harness seed must be >0.");
    63         return NULL;
    64     }
    65 
    66     /* Allocate output buffer */
    67     seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    68     if (seed == NULL) {
    69         SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    70         SDL_Error(SDL_ENOMEM);
    71         return NULL;
    72     }
    73 
    74     /* Generate a random string of alphanumeric characters */
    75     SDLTest_RandomInitTime(&randomContext);
    76     for (counter = 0; counter < length; counter++) {
    77         unsigned int number = SDLTest_Random(&randomContext);
    78         char ch = (char) (number % (91 - 48)) + 48;
    79         if (ch >= 58 && ch <= 64) {
    80             ch = 65;
    81         }
    82         seed[counter] = ch;
    83     }
    84     seed[length] = '\0';
    85 
    86     return seed;
    87 }
    88 
    89 /**
    90 * Generates an execution key for the fuzzer.
    91 *
    92 * \param runSeed        The run seed to use
    93 * \param suiteName      The name of the test suite
    94 * \param testName       The name of the test
    95 * \param iteration      The iteration count
    96 *
    97 * \returns The generated execution key to initialize the fuzzer with.
    98 *
    99 */
   100 Uint64
   101 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
   102 {
   103     SDLTest_Md5Context md5Context;
   104     Uint64 *keys;
   105     char iterationString[16];
   106     Uint32 runSeedLength;
   107     Uint32 suiteNameLength;
   108     Uint32 testNameLength;
   109     Uint32 iterationStringLength;
   110     Uint32 entireStringLength;
   111     char *buffer;
   112 
   113     if (runSeed == NULL || runSeed[0] == '\0') {
   114         SDLTest_LogError("Invalid runSeed string.");
   115         return -1;
   116     }
   117 
   118     if (suiteName == NULL || suiteName[0] == '\0') {
   119         SDLTest_LogError("Invalid suiteName string.");
   120         return -1;
   121     }
   122 
   123     if (testName == NULL || testName[0] == '\0') {
   124         SDLTest_LogError("Invalid testName string.");
   125         return -1;
   126     }
   127 
   128     if (iteration <= 0) {
   129         SDLTest_LogError("Invalid iteration count.");
   130         return -1;
   131     }
   132 
   133     /* Convert iteration number into a string */
   134     SDL_memset(iterationString, 0, sizeof(iterationString));
   135     SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
   136 
   137     /* Combine the parameters into single string */
   138     runSeedLength = SDL_strlen(runSeed);
   139     suiteNameLength = SDL_strlen(suiteName);
   140     testNameLength = SDL_strlen(testName);
   141     iterationStringLength = SDL_strlen(iterationString);
   142     entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
   143     buffer = (char *)SDL_malloc(entireStringLength);
   144     if (buffer == NULL) {
   145         SDLTest_LogError("Failed to allocate buffer for execKey generation.");
   146         SDL_Error(SDL_ENOMEM);
   147         return 0;
   148     }
   149     SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
   150 
   151     /* Hash string and use half of the digest as 64bit exec key */
   152     SDLTest_Md5Init(&md5Context);
   153     SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
   154     SDLTest_Md5Final(&md5Context);
   155     SDL_free(buffer);
   156     keys = (Uint64 *)md5Context.digest;
   157 
   158     return keys[0];
   159 }
   160 
   161 /**
   162 * \brief Set timeout handler for test.
   163 *
   164 * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   165 *
   166 * \param timeout Timeout interval in seconds.
   167 * \param callback Function that will be called after timeout has elapsed.
   168 *
   169 * \return Timer id or -1 on failure.
   170 */
   171 SDL_TimerID
   172 SDLTest_SetTestTimeout(int timeout, void (*callback)())
   173 {
   174     Uint32 timeoutInMilliseconds;
   175     SDL_TimerID timerID;
   176 
   177     if (callback == NULL) {
   178         SDLTest_LogError("Timeout callback can't be NULL");
   179         return -1;
   180     }
   181 
   182     if (timeout < 0) {
   183         SDLTest_LogError("Timeout value must be bigger than zero.");
   184         return -1;
   185     }
   186 
   187     /* Init SDL timer if not initialized before */
   188     if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
   189         if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
   190             SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
   191             return -1;
   192         }
   193     }
   194 
   195     /* Set timer */
   196     timeoutInMilliseconds = timeout * 1000;
   197     timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
   198     if (timerID == 0) {
   199         SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
   200         return -1;
   201     }
   202 
   203     return timerID;
   204 }
   205 
   206 /**
   207 * \brief Timeout handler. Aborts test run and exits harness process.
   208 */
   209 void
   210     SDLTest_BailOut()
   211 {
   212     SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   213     exit(TEST_ABORTED); /* bail out from the test */
   214 }
   215 
   216 /**
   217 * \brief Execute a test using the given execution key.
   218 *
   219 * \param testSuite Suite containing the test case.
   220 * \param testCase Case to execute.
   221 * \param execKey Execution key for the fuzzer.
   222 * \param forceTestRun Force test to run even if test was disabled in suite.
   223 *
   224 * \returns Test case result.
   225 */
   226 int
   227 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
   228 {
   229     SDL_TimerID timer = 0;
   230     int testCaseResult = 0;
   231     int testResult = 0;
   232     int fuzzerCount;
   233 
   234     if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
   235     {
   236         SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
   237         return TEST_RESULT_SETUP_FAILURE;
   238     }
   239 
   240 	if (!testCase->enabled && forceTestRun == SDL_FALSE)
   241     {
   242         SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Disabled)");
   243         return TEST_RESULT_SKIPPED;
   244     }
   245 
   246     /* Initialize fuzzer */
   247     SDLTest_FuzzerInit(execKey);
   248 
   249     /* Reset assert tracker */
   250     SDLTest_ResetAssertSummary();
   251 
   252     /* Set timeout timer */
   253     timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
   254 
   255     /* Maybe run suite initalizer function */
   256     if (testSuite->testSetUp) {
   257         testSuite->testSetUp(0x0);
   258         if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
   259             SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite Setup", testSuite->name, "Failed");
   260             return TEST_RESULT_SETUP_FAILURE;
   261         }
   262     }
   263 
   264     /* Run test case function */
   265     testCaseResult = testCase->testCase(0x0);
   266 
   267     /* Convert test execution result into harness result */
   268     if (testCaseResult == TEST_SKIPPED) {
   269         /* Test was programatically skipped */
   270         testResult = TEST_RESULT_SKIPPED;
   271     } else if (testCaseResult == TEST_STARTED) {
   272         /* Test did not return a TEST_COMPLETED value; assume it failed */
   273         testResult = TEST_RESULT_FAILED;
   274     } else if (testCaseResult == TEST_ABORTED) {
   275         /* Test was aborted early; assume it failed */
   276         testResult = TEST_RESULT_FAILED;
   277     } else {
   278         /* Perform failure analysis based on asserts */
   279         testResult = SDLTest_AssertSummaryToTestResult();
   280     }
   281 
   282     /* Maybe run suite cleanup function (ignore failed asserts) */
   283     if (testSuite->testTearDown) {
   284         testSuite->testTearDown(0x0);
   285     }
   286 
   287     /* Cancel timeout timer */
   288     if (timer) {
   289         SDL_RemoveTimer(timer);
   290     }
   291 
   292     /* Report on asserts and fuzzer usage */
   293     fuzzerCount = SDLTest_GetFuzzerInvocationCount();
   294     if (fuzzerCount > 0) {
   295         SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
   296     }
   297 
   298     /* Final log based on test execution result */
   299     if (testCaseResult == TEST_SKIPPED) {
   300         /* Test was programatically skipped */
   301         SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Programmatically)");
   302     } else if (testCaseResult == TEST_STARTED) {
   303         /* Test did not return a TEST_COMPLETED value; assume it failed */
   304         SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
   305     } else if (testCaseResult == TEST_ABORTED) {
   306         /* Test was aborted early; assume it failed */
   307         SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (Aborted)");
   308     } else {
   309         SDLTest_LogAssertSummary();
   310     }
   311 
   312     return testResult;
   313 }
   314 
   315 /* Prints summary of all suites/tests contained in the given reference */
   316 void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
   317 {
   318     int suiteCounter;
   319     int testCounter;
   320     SDLTest_TestSuiteReference *testSuite;
   321     SDLTest_TestCaseReference *testCase;
   322 
   323     /* Loop over all suites */
   324     suiteCounter = 0;
   325     while(&testSuites[suiteCounter]) {
   326         testSuite=&testSuites[suiteCounter];
   327         suiteCounter++;
   328         SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
   329             (testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
   330 
   331         /* Loop over all test cases */
   332         testCounter = 0;
   333         while(testSuite->testCases[testCounter])
   334         {
   335             testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   336             testCounter++;
   337             SDLTest_Log("  Test Case %i - %s: %s", testCounter,
   338                 (testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT,
   339                 (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
   340         }
   341     }
   342 }
   343 
   344 /* Gets a timer value in seconds */
   345 float GetClock()
   346 {
   347     float currentClock = (float)clock();
   348     return currentClock / (float)CLOCKS_PER_SEC;
   349 }
   350 
   351 /**
   352 * \brief Execute a test suite using the given run seed and execution key.
   353 *
   354 * The filter string is matched to the suite name (full comparison) to select a single suite,
   355 * or if no suite matches, it is matched to the test names (full comparison) to select a single test.
   356 *
   357 * \param testSuites Suites containing the test case.
   358 * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   359 * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   360 * \param filter Filter specification. NULL disables. Case sensitive.
   361 * \param testIterations Number of iterations to run each test case.
   362 *
   363 * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   364 */
   365 int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
   366 {
   367     int totalNumberOfTests = 0;
   368     int failedNumberOfTests = 0;
   369     int suiteCounter;
   370     int testCounter;
   371     int iterationCounter;
   372     SDLTest_TestSuiteReference *testSuite;
   373     SDLTest_TestCaseReference *testCase;
   374     const char *runSeed = NULL;
   375     char *currentSuiteName;
   376     char *currentTestName;
   377     Uint64 execKey;
   378     float runStartSeconds;
   379     float suiteStartSeconds;
   380     float testStartSeconds;
   381     float runEndSeconds;
   382     float suiteEndSeconds;
   383     float testEndSeconds;
   384     float runtime;
   385     int suiteFilter = 0;
   386     char *suiteFilterName = NULL;
   387     int testFilter = 0;
   388     char *testFilterName = NULL;
   389 	SDL_bool forceTestRun = SDL_FALSE;
   390     int testResult = 0;
   391     int runResult = 0;
   392     Uint32 totalTestFailedCount = 0;
   393     Uint32 totalTestPassedCount = 0;
   394     Uint32 totalTestSkippedCount = 0;
   395     Uint32 testFailedCount = 0;
   396     Uint32 testPassedCount = 0;
   397     Uint32 testSkippedCount = 0;
   398     Uint32 countSum = 0;
   399     SDLTest_TestCaseReference **failedTests;
   400 
   401     /* Sanitize test iterations */
   402     if (testIterations < 1) {
   403         testIterations = 1;
   404     }
   405 
   406     /* Generate run see if we don't have one already */
   407     if (userRunSeed == NULL || userRunSeed[0] == '\0') {
   408         runSeed = SDLTest_GenerateRunSeed(16);
   409         if (runSeed == NULL) {
   410             SDLTest_LogError("Generating a random seed failed");
   411             return 2;
   412         }
   413     } else {
   414         runSeed = userRunSeed;
   415     }
   416 
   417 
   418     /* Reset per-run counters */
   419     totalTestFailedCount = 0;
   420     totalTestPassedCount = 0;
   421     totalTestSkippedCount = 0;
   422 
   423     /* Take time - run start */
   424     runStartSeconds = GetClock();
   425 
   426     /* Log run with fuzzer parameters */
   427     SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   428 
   429 	/* Count the total number of tests */
   430     suiteCounter = 0;
   431     while (testSuites[suiteCounter]) {
   432         testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   433         suiteCounter++;
   434         testCounter = 0;
   435         while (testSuite->testCases[testCounter])
   436         {
   437             testCounter++;
   438 			totalNumberOfTests++;
   439 		}
   440 	}
   441 
   442 	/* Pre-allocate an array for tracking failed tests (potentially all test cases) */
   443 	failedTests = (SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
   444 	if (failedTests == NULL) {	
   445 	   SDLTest_LogError("Unable to allocate cache for failed tests");
   446            SDL_Error(SDL_ENOMEM);	   
   447            return -1;
   448 	}
   449 
   450     /* Initialize filtering */
   451     if (filter != NULL && filter[0] != '\0') {
   452         /* Loop over all suites to check if we have a filter match */
   453         suiteCounter = 0;
   454         while (testSuites[suiteCounter] && suiteFilter == 0) {
   455             testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   456             suiteCounter++;
   457             if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
   458                 /* Matched a suite name */
   459                 suiteFilter = 1;
   460                 suiteFilterName = testSuite->name;
   461                 SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
   462                 break;
   463             }
   464 
   465             /* Within each suite, loop over all test cases to check if we have a filter match */
   466             testCounter = 0;
   467             while (testSuite->testCases[testCounter] && testFilter == 0)
   468             {
   469                 testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   470                 testCounter++;
   471                 if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
   472                     /* Matched a test name */
   473                     suiteFilter = 1;
   474                     suiteFilterName = testSuite->name;
   475                     testFilter = 1;
   476                     testFilterName = testCase->name;
   477                     SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
   478                     break;
   479                 }
   480             }
   481         }
   482 
   483         if (suiteFilter == 0 && testFilter == 0) {
   484             SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
   485             SDLTest_Log("Exit code: 2");
   486             SDL_free(failedTests);
   487             return 2;
   488         }
   489     }
   490 
   491     /* Loop over all suites */
   492     suiteCounter = 0;
   493     while(testSuites[suiteCounter]) {
   494         testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   495         currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
   496         suiteCounter++;
   497 
   498         /* Filter suite if flag set and we have a name */
   499         if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
   500             SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
   501                 /* Skip suite */
   502                 SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
   503                     suiteCounter,
   504                     currentSuiteName);
   505         } else {
   506 
   507             /* Reset per-suite counters */
   508             testFailedCount = 0;
   509             testPassedCount = 0;
   510             testSkippedCount = 0;
   511 
   512             /* Take time - suite start */
   513             suiteStartSeconds = GetClock();
   514 
   515             /* Log suite started */
   516             SDLTest_Log("===== Test Suite %i: '%s' started\n",
   517                 suiteCounter,
   518                 currentSuiteName);
   519 
   520             /* Loop over all test cases */
   521             testCounter = 0;
   522             while(testSuite->testCases[testCounter])
   523             {
   524                 testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   525                 currentTestName = (char *)((testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
   526                 testCounter++;
   527 
   528                 /* Filter tests if flag set and we have a name */
   529                 if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
   530                     SDL_strcmp(testFilterName, testCase->name) != 0) {
   531                         /* Skip test */
   532                         SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
   533                             suiteCounter,
   534                             testCounter,
   535                             currentTestName);
   536                 } else {
   537                     /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
   538                     if (testFilter == 1 && !testCase->enabled) {
   539                         SDLTest_Log("Force run of disabled test since test filter was set");
   540 						forceTestRun = SDL_TRUE;
   541                     }
   542 
   543                     /* Take time - test start */
   544                     testStartSeconds = GetClock();
   545 
   546                     /* Log test started */
   547                     SDLTest_Log("----- Test Case %i.%i: '%s' started",
   548                         suiteCounter,
   549                         testCounter,
   550                         currentTestName);
   551                     if (testCase->description != NULL && testCase->description[0] != '\0') {
   552                         SDLTest_Log("Test Description: '%s'",
   553                             (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
   554                     }
   555 
   556                     /* Loop over all iterations */
   557                     iterationCounter = 0;
   558                     while(iterationCounter < testIterations)
   559                     {
   560                         iterationCounter++;
   561 
   562                         if (userExecKey != 0) {
   563                             execKey = userExecKey;
   564                         } else {
   565                             execKey = SDLTest_GenerateExecKey((char *)runSeed, testSuite->name, testCase->name, iterationCounter);
   566                         }
   567 
   568                         SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
   569 						testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
   570 
   571                         if (testResult == TEST_RESULT_PASSED) {
   572                             testPassedCount++;
   573                             totalTestPassedCount++;
   574                         } else if (testResult == TEST_RESULT_SKIPPED) {
   575                             testSkippedCount++;
   576                             totalTestSkippedCount++;
   577                         } else {
   578                             testFailedCount++;
   579                             totalTestFailedCount++;
   580                         }
   581                     }
   582 
   583                     /* Take time - test end */
   584                     testEndSeconds = GetClock();
   585                     runtime = testEndSeconds - testStartSeconds;
   586                     if (runtime < 0.0f) runtime = 0.0f;
   587 
   588                     if (testIterations > 1) {
   589                         /* Log test runtime */
   590                         SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
   591                         SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
   592                     } else {
   593                         /* Log test runtime */
   594                         SDLTest_Log("Total Test runtime: %.1f sec", runtime);
   595                     }
   596 
   597                     /* Log final test result */
   598                     switch (testResult) {
   599                     case TEST_RESULT_PASSED:
   600                         SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed");
   601                         break;
   602                     case TEST_RESULT_FAILED:
   603                         SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed");
   604                         break;
   605                     case TEST_RESULT_NO_ASSERT:
   606                         SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts");
   607                         break;
   608                     }
   609 
   610                     /* Collect failed test case references for repro-step display */
   611                     if (testResult == TEST_RESULT_FAILED) {
   612                         failedTests[failedNumberOfTests] = testCase;
   613                         failedNumberOfTests++;
   614                     }
   615                 }
   616             }
   617 
   618             /* Take time - suite end */
   619             suiteEndSeconds = GetClock();
   620             runtime = suiteEndSeconds - suiteStartSeconds;
   621             if (runtime < 0.0f) runtime = 0.0f;
   622 
   623             /* Log suite runtime */
   624             SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
   625 
   626             /* Log summary and final Suite result */
   627             countSum = testPassedCount + testFailedCount + testSkippedCount;
   628             if (testFailedCount == 0)
   629             {
   630                 SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   631                 SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed");
   632             }
   633             else
   634             {
   635                 SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   636                 SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed");
   637             }
   638 
   639         }
   640     }
   641 
   642     /* Take time - run end */
   643     runEndSeconds = GetClock();
   644     runtime = runEndSeconds - runStartSeconds;
   645     if (runtime < 0.0f) runtime = 0.0f;
   646 
   647     /* Log total runtime */
   648     SDLTest_Log("Total Run runtime: %.1f sec", runtime);
   649 
   650     /* Log summary and final run result */
   651     countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   652     if (totalTestFailedCount == 0)
   653     {
   654         runResult = 0;
   655         SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   656         SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed");
   657     }
   658     else
   659     {
   660         runResult = 1;
   661         SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   662         SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed");
   663     }
   664 
   665     /* Print repro steps for failed tests */
   666     if (failedNumberOfTests > 0) {
   667         SDLTest_Log("Harness input to repro failures:");
   668         for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
   669           SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
   670         }
   671     }
   672     SDL_free(failedTests);
   673 
   674     SDLTest_Log("Exit code: %d", runResult);
   675     return runResult;
   676 }