src/test/SDL_test_harness.c
author Andreas Schiffler
Sat, 18 May 2013 09:35:09 -0700
changeset 7189 414be1d64060
parent 6885 700f1b25f77f
child 7191 75360622e65f
permissions -rw-r--r--
Update test harness to handle test return codes; fix comment format in harness; update Main test suite to handle globally disabled features
     1 /*
     2 Simple DirectMedia Layer
     3 Copyright (C) 1997-2013 Sam Lantinga <slouken@libsdl.org>
     4 
     5 This software is provided 'as-is', without any express or implied
     6 warranty.  In no event will the authors be held liable for any damages
     7 arising from the use of this software.
     8 
     9 Permission is granted to anyone to use this software for any purpose,
    10 including commercial applications, and to alter it and redistribute it
    11 freely, subject to the following restrictions:
    12 
    13 1. The origin of this software must not be misrepresented; you must not
    14 claim that you wrote the original software. If you use this software
    15 in a product, an acknowledgment in the product documentation would be
    16 appreciated but is not required.
    17 2. Altered source versions must be plainly marked as such, and must not be
    18 misrepresented as being the original software.
    19 3. This notice may not be removed or altered from any source distribution.
    20 */
    21 
    22 #include "SDL_config.h"
    23 
    24 #include "SDL_test.h"
    25 
    26 #include <stdio.h>
    27 #include <stdlib.h>
    28 #include <string.h>
    29 #include <time.h>
    30 
    31 /* Invalid test name/description message format */
    32 const char *SDLTest_InvalidNameFormat = "(Invalid)";
    33 
    34 /* Log summary message format */
    35 const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d";
    36 
    37 /* Final result message format */
    38 const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n";
    39 
    40 /*! \brief Timeout for single test case execution */
    41 static Uint32 SDLTest_TestCaseTimeout = 3600;
    42 
    43 /**
    44 * Generates a random run seed string for the harness. The generated seed
    45 * will contain alphanumeric characters (0-9A-Z).
    46 *
    47 * Note: The returned string needs to be deallocated by the caller.
    48 *
    49 * \param length The length of the seed string to generate
    50 *
    51 * \returns The generated seed string
    52 */
    53 char *
    54 SDLTest_GenerateRunSeed(const int length)
    55 {
    56 	char *seed = NULL;
    57 	SDLTest_RandomContext randomContext;
    58 	int counter;
    59 
    60 	/* Sanity check input */
    61 	if (length <= 0) {
    62 		SDLTest_LogError("The length of the harness seed must be >0.");
    63 		return NULL;
    64 	}
    65 
    66 	/* Allocate output buffer */
    67 	seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    68 	if (seed == NULL) {
    69 		SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    70 		return NULL;
    71 	}
    72 
    73 	/* Generate a random string of alphanumeric characters */
    74 	SDLTest_RandomInitTime(&randomContext);
    75 	for (counter = 0; counter < length - 1; ++counter) {
    76 		unsigned int number = SDLTest_Random(&randomContext);
    77 		char ch = (char) (number % (91 - 48)) + 48;
    78 		if (ch >= 58 && ch <= 64) {
    79 			ch = 65;
    80 		}
    81 		seed[counter] = ch;
    82 	}
    83 	seed[counter] = '\0';
    84 
    85 	return seed;
    86 }
    87 
    88 /**
    89 * Generates an execution key for the fuzzer.
    90 *
    91 * \param runSeed		The run seed to use
    92 * \param suiteName		The name of the test suite
    93 * \param testName		The name of the test
    94 * \param iteration		The iteration count
    95 *
    96 * \returns The generated execution key to initialize the fuzzer with.
    97 *
    98 */
    99 Uint64
   100 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
   101 {
   102 	SDLTest_Md5Context md5Context;
   103 	Uint64 *keys;
   104 	char iterationString[16];
   105 	Uint32 runSeedLength;
   106 	Uint32 suiteNameLength;
   107 	Uint32 testNameLength;
   108 	Uint32 iterationStringLength;
   109 	Uint32 entireStringLength;
   110 	char *buffer;
   111 
   112 	if (runSeed == NULL || SDL_strlen(runSeed)==0) {
   113 		SDLTest_LogError("Invalid runSeed string.");
   114 		return -1;
   115 	}
   116 
   117 	if (suiteName == NULL || SDL_strlen(suiteName)==0) {
   118 		SDLTest_LogError("Invalid suiteName string.");
   119 		return -1;
   120 	}
   121 
   122 	if (testName == NULL || SDL_strlen(testName)==0) {
   123 		SDLTest_LogError("Invalid testName string.");
   124 		return -1;
   125 	}
   126 
   127 	if (iteration <= 0) {
   128 		SDLTest_LogError("Invalid iteration count.");
   129 		return -1;
   130 	}
   131 
   132 	/* Convert iteration number into a string */
   133 	SDL_memset(iterationString, 0, sizeof(iterationString));
   134 	SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
   135 
   136 	/* Combine the parameters into single string */
   137 	runSeedLength = SDL_strlen(runSeed);
   138 	suiteNameLength = SDL_strlen(suiteName);
   139 	testNameLength = SDL_strlen(testName);
   140 	iterationStringLength = SDL_strlen(iterationString);
   141 	entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
   142 	buffer = (char *)SDL_malloc(entireStringLength);
   143 	if (buffer == NULL) {
   144 		SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
   145 		return 0;
   146 	}
   147 	SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
   148 
   149 	/* Hash string and use half of the digest as 64bit exec key */
   150 	SDLTest_Md5Init(&md5Context);
   151 	SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
   152 	SDLTest_Md5Final(&md5Context);
   153 	SDL_free(buffer);
   154 	keys = (Uint64 *)md5Context.digest;
   155 
   156 	return keys[0];
   157 }
   158 
   159 /**
   160 * \brief Set timeout handler for test.
   161 *
   162 * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   163 *
   164 * \param timeout Timeout interval in seconds.
   165 * \param callback Function that will be called after timeout has elapsed.
   166 * 
   167 * \return Timer id or -1 on failure.
   168 */
   169 SDL_TimerID
   170 SDLTest_SetTestTimeout(int timeout, void (*callback)())
   171 {
   172 	Uint32 timeoutInMilliseconds;
   173 	SDL_TimerID timerID;
   174 
   175 	if (callback == NULL) {
   176 		SDLTest_LogError("Timeout callback can't be NULL");
   177 		return -1;
   178 	}
   179 
   180 	if (timeout < 0) {
   181 		SDLTest_LogError("Timeout value must be bigger than zero.");
   182 		return -1;
   183 	}
   184 
   185 	/* Init SDL timer if not initialized before */
   186 	if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
   187 		if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
   188 			SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
   189 			return -1;
   190 		}
   191 	}
   192 
   193 	/* Set timer */
   194 	timeoutInMilliseconds = timeout * 1000;
   195 	timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
   196 	if (timerID == 0) {
   197 		SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
   198 		return -1;
   199 	}
   200 
   201 	return timerID;
   202 }
   203 
   204 /**
   205 * \brief Timeout handler. Aborts test run and exits harness process.
   206 */
   207 void
   208 	SDLTest_BailOut()
   209 {
   210 	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   211 	exit(TEST_ABORTED); /* bail out from the test */
   212 }
   213 
   214 /**
   215 * \brief Execute a test using the given execution key.
   216 *
   217 * \param testSuite Suite containing the test case.
   218 * \param testCase Case to execute.
   219 * \param execKey Execution key for the fuzzer.
   220 *
   221 * \returns Test case result.
   222 */
   223 int
   224 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
   225 {
   226 	SDL_TimerID timer = 0;
   227 	int testCaseResult = 0;
   228 	int testResult = 0;
   229 	int fuzzerCount;
   230 
   231 	if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
   232 	{
   233 		SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
   234 		return TEST_RESULT_SETUP_FAILURE;
   235 	}
   236 
   237 	if (!testCase->enabled)
   238 	{
   239 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Disabled)");
   240 		return TEST_RESULT_SKIPPED;
   241 	}
   242 
   243 
   244 	/* Initialize fuzzer */
   245 	SDLTest_FuzzerInit(execKey);
   246 
   247 	/* Reset assert tracker */
   248 	SDLTest_ResetAssertSummary();
   249 
   250 	/* Set timeout timer */
   251 	timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
   252 
   253 	/* Maybe run suite initalizer function */
   254 	if (testSuite->testSetUp) {
   255 		testSuite->testSetUp(0x0);
   256 		if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
   257 			SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed");
   258 			return TEST_RESULT_SETUP_FAILURE;
   259 		}
   260 	}
   261 
   262 	/* Run test case function */
   263 	testCaseResult = testCase->testCase(0x0);
   264 	
   265 	/* Convert test execution result into harness result */
   266 	if (testCaseResult == TEST_SKIPPED) {
   267 		/* Test was programatically skipped */
   268 		testResult = TEST_RESULT_SKIPPED;
   269 	} else if (testCaseResult == TEST_STARTED) {
   270 		/* Test did not return a TEST_COMPLETED value; assume it failed */
   271 		testResult = TEST_RESULT_FAILED;
   272 	} else if (testCaseResult == TEST_ABORTED) {
   273 		/* Test was aborted early; assume it failed */
   274 		testResult = TEST_RESULT_FAILED;
   275 	} else {
   276 		/* Perform failure analysis based on asserts */
   277 		testResult = SDLTest_AssertSummaryToTestResult();
   278 	}
   279 
   280 	/* Maybe run suite cleanup function (ignore failed asserts) */
   281 	if (testSuite->testTearDown) {
   282 		testSuite->testTearDown(0x0);
   283 	}
   284 
   285 	/* Cancel timeout timer */
   286 	if (timer) {
   287 		SDL_RemoveTimer(timer);
   288 	}
   289 
   290 	/* Report on asserts and fuzzer usage */
   291 	fuzzerCount = SDLTest_GetFuzzerInvocationCount();
   292 	if (fuzzerCount > 0) {
   293 		SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
   294 	}
   295 
   296 	/* Final log based on test execution result */
   297 	if (testCaseResult == TEST_SKIPPED) {
   298 		/* Test was programatically skipped */
   299 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)");
   300 	} else if (testCaseResult == TEST_STARTED) {
   301 		/* Test did not return a TEST_COMPLETED value; assume it failed */
   302 		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
   303 	} else if (testCaseResult == TEST_ABORTED) {
   304 		/* Test was aborted early; assume it failed */
   305 		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)");
   306 	} else {
   307 		SDLTest_LogAssertSummary();
   308 	}
   309 
   310 	return testResult;
   311 }
   312 
   313 /* Prints summary of all suites/tests contained in the given reference */
   314 void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
   315 {
   316 	int suiteCounter;
   317 	int testCounter;
   318 	SDLTest_TestSuiteReference *testSuite;
   319 	SDLTest_TestCaseReference *testCase;
   320 
   321 	/* Loop over all suites */
   322 	suiteCounter = 0;
   323 	while(&testSuites[suiteCounter]) {
   324 		testSuite=&testSuites[suiteCounter];
   325 		suiteCounter++;
   326 		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   327 			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   328 
   329 		/* Loop over all test cases */
   330 		testCounter = 0;
   331 		while(testSuite->testCases[testCounter])
   332 		{
   333 			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   334 			testCounter++;
   335 			SDLTest_Log("  Test Case %i - %s: %s", testCounter, 
   336 				(testCase->name) ? testCase->name : SDLTest_InvalidNameFormat, 
   337 				(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   338 		}
   339 	}
   340 }
   341 
   342 /* Gets a timer value in seconds */
   343 float GetClock()
   344 {
   345 	float currentClock = (float)clock();
   346 	return currentClock / (float)CLOCKS_PER_SEC;
   347 }
   348 
   349 /**
   350 * \brief Execute a test suite using the given run seend and execution key.
   351 *
   352 * The filter string is matched to the suite name (full comparison) to select a single suite,
   353 * or if no suite matches, it is matched to the test names (full comparison) to select a single test.
   354 *
   355 * \param testSuites Suites containing the test case.
   356 * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   357 * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   358 * \param filter Filter specification. NULL disables. Case sensitive.
   359 * \param testIterations Number of iterations to run each test case.
   360 *
   361 * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   362 */
   363 int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
   364 {
   365 	int suiteCounter;
   366 	int testCounter;
   367 	int iterationCounter;
   368 	SDLTest_TestSuiteReference *testSuite;
   369 	SDLTest_TestCaseReference *testCase;
   370 	const char *runSeed = NULL;
   371 	char *currentSuiteName;
   372 	char *currentTestName;
   373 	Uint64 execKey;
   374 	float runStartSeconds;
   375 	float suiteStartSeconds;
   376 	float testStartSeconds;
   377 	float runEndSeconds;
   378 	float suiteEndSeconds;
   379 	float testEndSeconds;
   380 	float runtime;
   381 	int suiteFilter = 0;
   382 	char *suiteFilterName = NULL;
   383 	int testFilter = 0;
   384 	char *testFilterName = NULL;
   385 	int testResult = 0;
   386 	int runResult = 0;
   387 	Uint32 totalTestFailedCount = 0;
   388 	Uint32 totalTestPassedCount = 0;
   389 	Uint32 totalTestSkippedCount = 0;
   390 	Uint32 testFailedCount = 0;
   391 	Uint32 testPassedCount = 0;
   392 	Uint32 testSkippedCount = 0;
   393 	Uint32 countSum = 0;
   394 	char *logFormat = (char *)SDLTest_LogSummaryFormat;
   395 
   396 	/* Sanitize test iterations */
   397 	if (testIterations < 1) {
   398 		testIterations = 1;
   399 	}
   400 
   401 	/* Generate run see if we don't have one already */
   402 	if (userRunSeed == NULL || SDL_strlen(userRunSeed) == 0) {
   403 		runSeed = SDLTest_GenerateRunSeed(16);
   404 		if (runSeed == NULL) {
   405 			SDLTest_LogError("Generating a random seed failed");
   406 			return 2;
   407 		}
   408 	} else {
   409 		runSeed = userRunSeed;
   410 	}
   411 
   412 
   413 	/* Reset per-run counters */
   414 	totalTestFailedCount = 0;
   415 	totalTestPassedCount = 0;
   416 	totalTestSkippedCount = 0;
   417 
   418 	/* Take time - run start */
   419 	runStartSeconds = GetClock();
   420 
   421 	/* Log run with fuzzer parameters */
   422 	SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   423 
   424 	/* Initialize filtering */
   425 	if (filter != NULL && SDL_strlen(filter) > 0) {
   426 		/* Loop over all suites to check if we have a filter match */
   427 		suiteCounter = 0;
   428 		while (testSuites[suiteCounter] && suiteFilter == 0) {
   429 			testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   430 			suiteCounter++;
   431 			if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
   432 				/* Matched a suite name */
   433 				suiteFilter = 1;
   434 				suiteFilterName = testSuite->name;
   435 				SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
   436 				break;
   437 			}
   438 
   439 			/* Within each suite, loop over all test cases to check if we have a filter match */
   440 			testCounter = 0;
   441 			while (testSuite->testCases[testCounter] && testFilter == 0)
   442 			{
   443 				testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   444 				testCounter++;
   445 				if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
   446 					/* Matched a test name */
   447 					suiteFilter = 1;
   448 					suiteFilterName = testSuite->name;
   449 					testFilter = 1;
   450 					testFilterName = testCase->name;
   451 					SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);					
   452 					break;
   453 				}
   454 			}						
   455 		}
   456 		
   457 		if (suiteFilter == 0 && testFilter == 0) {
   458 			SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
   459 			SDLTest_Log("Exit code: 2");	
   460 			return 2;
   461 		}		
   462 	}
   463 
   464 	/* Loop over all suites */
   465 	suiteCounter = 0;
   466 	while(testSuites[suiteCounter]) {
   467 		testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   468 		currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   469 		suiteCounter++;
   470 
   471 		/* Filter suite if flag set and we have a name */
   472 		if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
   473 			SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
   474 				/* Skip suite */
   475 				SDLTest_Log("===== Test Suite %i: '%s' skipped\n", 
   476 					suiteCounter, 
   477 					currentSuiteName);
   478 		} else {
   479 
   480 			/* Reset per-suite counters */
   481 			testFailedCount = 0;
   482 			testPassedCount = 0;
   483 			testSkippedCount = 0;
   484 
   485 			/* Take time - suite start */
   486 			suiteStartSeconds = GetClock();
   487 
   488 			/* Log suite started */
   489 			SDLTest_Log("===== Test Suite %i: '%s' started\n", 
   490 				suiteCounter, 
   491 				currentSuiteName);
   492 
   493 			/* Loop over all test cases */
   494 			testCounter = 0;
   495 			while(testSuite->testCases[testCounter])
   496 			{
   497 				testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   498 				currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   499 				testCounter++;
   500 
   501 				/* Filter tests if flag set and we have a name */
   502 				if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
   503 					SDL_strcmp(testFilterName, testCase->name) != 0) {
   504 						/* Skip test */
   505 						SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n", 
   506 							suiteCounter,
   507 							testCounter,
   508 							currentTestName);
   509 				} else {
   510 					/* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
   511 					if (testFilter == 1 && !testCase->enabled) {
   512 						SDLTest_Log("Force run of disabled test since test filter was set");
   513 						testCase->enabled = 1;
   514 					}
   515 
   516 					/* Take time - test start */
   517 					testStartSeconds = GetClock();
   518 
   519 					/* Log test started */
   520 					SDLTest_Log("----- Test Case %i.%i: '%s' started",
   521 						suiteCounter,
   522 						testCounter, 
   523 						currentTestName);
   524 					if (testCase->description != NULL && SDL_strlen(testCase->description)>0) {
   525 						SDLTest_Log("Test Description: '%s'", 
   526 							(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   527 					}
   528 
   529 					/* Loop over all iterations */
   530 					iterationCounter = 0;
   531 					while(iterationCounter < testIterations)
   532 					{
   533 						iterationCounter++;
   534 
   535 						if (userExecKey != 0) {
   536 							execKey = userExecKey;
   537 						} else {
   538 							execKey = SDLTest_GenerateExecKey((char *)runSeed, testSuite->name, testCase->name, iterationCounter);
   539 						}
   540 
   541 						SDLTest_Log("Test Iteration %i: execKey %llu", iterationCounter, execKey);
   542 						testResult = SDLTest_RunTest(testSuite, testCase, execKey);
   543 
   544 						if (testResult == TEST_RESULT_PASSED) {
   545 							testPassedCount++;
   546 							totalTestPassedCount++;
   547 						} else if (testResult == TEST_RESULT_SKIPPED) {
   548 							testSkippedCount++;
   549 							totalTestSkippedCount++;
   550 						} else {
   551 							testFailedCount++;
   552 							totalTestFailedCount++;
   553 						}
   554 					}
   555 
   556 					/* Take time - test end */
   557 					testEndSeconds = GetClock();
   558 					runtime = testEndSeconds - testStartSeconds;
   559 					if (runtime < 0.0f) runtime = 0.0f;
   560 
   561 					if (testIterations > 1) {
   562 						/* Log test runtime */
   563 						SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
   564 						SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
   565 					} else {
   566 						/* Log test runtime */
   567 						SDLTest_Log("Total Test runtime: %.1f sec", runtime);
   568 					}
   569 
   570 					/* Log final test result */
   571 					switch (testResult) {
   572 					case TEST_RESULT_PASSED:
   573 						SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   574 						break;
   575 					case TEST_RESULT_FAILED:
   576 						SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
   577 						break;
   578 					case TEST_RESULT_NO_ASSERT:
   579 						SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
   580 						break;
   581 					}
   582 
   583 				}
   584 			}
   585 
   586 			/* Take time - suite end */
   587 			suiteEndSeconds = GetClock();
   588 			runtime = suiteEndSeconds - suiteStartSeconds;
   589 			if (runtime < 0.0f) runtime = 0.0f;
   590 
   591 			/* Log suite runtime */
   592 			SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
   593 
   594 			/* Log summary and final Suite result */
   595 			countSum = testPassedCount + testFailedCount + testSkippedCount;
   596 			if (testFailedCount == 0)
   597 			{
   598 				SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   599 				SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
   600 			} 
   601 			else 
   602 			{
   603 				SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   604 				SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
   605 			}
   606 
   607 		}
   608 	}
   609 
   610 	/* Take time - run end */
   611 	runEndSeconds = GetClock();
   612 	runtime = runEndSeconds - runStartSeconds;
   613 	if (runtime < 0.0f) runtime = 0.0f;
   614 
   615 	/* Log total runtime */
   616 	SDLTest_Log("Total Run runtime: %.1f sec", runtime);
   617 
   618 	/* Log summary and final run result */
   619 	countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   620 	if (totalTestFailedCount == 0)
   621 	{
   622 		runResult = 0;
   623 		SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   624 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
   625 	} 
   626 	else 
   627 	{
   628 		runResult = 1;
   629 		SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   630 		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
   631 	}
   632 
   633 	SDLTest_Log("Exit code: %d", runResult);	
   634 	return runResult;
   635 }