src/test/SDL_test_harness.c
changeset 6763 9cbd31a3450b
parent 6760 04dcce3081e6
child 6768 22c37bf0afbc
equal deleted inserted replaced
6762:c680e085a5f2 6763:9cbd31a3450b
     1 /*
     1 /*
     2   Simple DirectMedia Layer
     2 Simple DirectMedia Layer
     3   Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
     3 Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
     4 
     4 
     5   This software is provided 'as-is', without any express or implied
     5 This software is provided 'as-is', without any express or implied
     6   warranty.  In no event will the authors be held liable for any damages
     6 warranty.  In no event will the authors be held liable for any damages
     7   arising from the use of this software.
     7 arising from the use of this software.
     8 
     8 
     9   Permission is granted to anyone to use this software for any purpose,
     9 Permission is granted to anyone to use this software for any purpose,
    10   including commercial applications, and to alter it and redistribute it
    10 including commercial applications, and to alter it and redistribute it
    11   freely, subject to the following restrictions:
    11 freely, subject to the following restrictions:
    12 
    12 
    13   1. The origin of this software must not be misrepresented; you must not
    13 1. The origin of this software must not be misrepresented; you must not
    14      claim that you wrote the original software. If you use this software
    14 claim that you wrote the original software. If you use this software
    15      in a product, an acknowledgment in the product documentation would be
    15 in a product, an acknowledgment in the product documentation would be
    16      appreciated but is not required.
    16 appreciated but is not required.
    17   2. Altered source versions must be plainly marked as such, and must not be
    17 2. Altered source versions must be plainly marked as such, and must not be
    18      misrepresented as being the original software.
    18 misrepresented as being the original software.
    19   3. This notice may not be removed or altered from any source distribution.
    19 3. This notice may not be removed or altered from any source distribution.
    20 */
    20 */
    21 
    21 
    22 #include "SDL_config.h"
    22 #include "SDL_config.h"
    23 
    23 
    24 #include "SDL_test.h"
    24 #include "SDL_test.h"
    39 
    39 
    40 /*! \brief Timeout for single test case execution */
    40 /*! \brief Timeout for single test case execution */
    41 static Uint32 SDLTest_TestCaseTimeout = 3600;
    41 static Uint32 SDLTest_TestCaseTimeout = 3600;
    42 
    42 
    43 /**
    43 /**
    44  * Generates a random run seed string for the harness. The generated seed
    44 * Generates a random run seed string for the harness. The generated seed
    45  * will contain alphanumeric characters (0-9A-Z).
    45 * will contain alphanumeric characters (0-9A-Z).
    46  *
    46 *
    47  * Note: The returned string needs to be deallocated by the caller.
    47 * Note: The returned string needs to be deallocated by the caller.
    48  *
    48 *
    49  * \param length The length of the seed string to generate
    49 * \param length The length of the seed string to generate
    50  *
    50 *
    51  * \returns The generated seed string
    51 * \returns The generated seed string
    52  */
    52 */
    53 char *
    53 char *
    54 SDLTest_GenerateRunSeed(const int length)
    54 	SDLTest_GenerateRunSeed(const int length)
    55 {
    55 {
    56 	char *seed = NULL;
    56 	char *seed = NULL;
    57 	SDLTest_RandomContext randomContext;
    57 	SDLTest_RandomContext randomContext;
    58 	int counter;
    58 	int counter;
    59 
    59 
    84 
    84 
    85 	return seed;
    85 	return seed;
    86 }
    86 }
    87 
    87 
    88 /**
    88 /**
    89  * Generates an execution key for the fuzzer.
    89 * Generates an execution key for the fuzzer.
    90  *
    90 *
    91  * \param runSeed		The run seed to use
    91 * \param runSeed		The run seed to use
    92  * \param suiteName		The name of the test suite
    92 * \param suiteName		The name of the test suite
    93  * \param testName		The name of the test
    93 * \param testName		The name of the test
    94  * \param iteration		The iteration count
    94 * \param iteration		The iteration count
    95  *
    95 *
    96  * \returns The generated execution key to initialize the fuzzer with.
    96 * \returns The generated execution key to initialize the fuzzer with.
    97  *
    97 *
    98  */
    98 */
    99 Uint64
    99 Uint64
   100 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
   100 	SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
   101 {
   101 {
   102 	SDLTest_Md5Context md5Context;
   102 	SDLTest_Md5Context md5Context;
   103 	Uint64 *keys;
   103 	Uint64 *keys;
   104 	char iterationString[16];
   104 	char iterationString[16];
   105 	Uint32 runSeedLength;
   105 	Uint32 runSeedLength;
   155 
   155 
   156 	return keys[0];
   156 	return keys[0];
   157 }
   157 }
   158 
   158 
   159 /**
   159 /**
   160  * \brief Set timeout handler for test.
   160 * \brief Set timeout handler for test.
   161  *
   161 *
   162  * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   162 * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   163  *
   163 *
   164  * \param timeout Timeout interval in seconds.
   164 * \param timeout Timeout interval in seconds.
   165  * \param callback Function that will be called after timeout has elapsed.
   165 * \param callback Function that will be called after timeout has elapsed.
   166  * 
   166 * 
   167  * \return Timer id or -1 on failure.
   167 * \return Timer id or -1 on failure.
   168  */
   168 */
   169 SDL_TimerID
   169 SDL_TimerID
   170 SDLTest_SetTestTimeout(int timeout, void (*callback)())
   170 	SDLTest_SetTestTimeout(int timeout, void (*callback)())
   171 {
   171 {
   172 	Uint32 timeoutInMilliseconds;
   172 	Uint32 timeoutInMilliseconds;
   173 	SDL_TimerID timerID;
   173 	SDL_TimerID timerID;
   174 
   174 
   175 	if (callback == NULL) {
   175 	if (callback == NULL) {
   199 	}
   199 	}
   200 
   200 
   201 	return timerID;
   201 	return timerID;
   202 }
   202 }
   203 
   203 
       
   204 /**
       
   205 * \brief Timeout handler. Aborts test run and exits harness process.
       
   206 */
   204 void
   207 void
   205 SDLTest_BailOut()
   208 	SDLTest_BailOut()
   206 {
   209 {
   207 	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   210 	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   208 	exit(TEST_ABORTED); // bail out from the test
   211 	exit(TEST_ABORTED); // bail out from the test
   209 }
   212 }
   210 
   213 
   211 /**
   214 /**
   212  * \brief Execute a test using the given execution key.
   215 * \brief Execute a test using the given execution key.
   213  *
   216 *
   214  * \param testSuite Suite containing the test case.
   217 * \param testSuite Suite containing the test case.
   215  * \param testCase Case to execute.
   218 * \param testCase Case to execute.
   216  * \param execKey Execution key for the fuzzer.
   219 * \param execKey Execution key for the fuzzer.
   217  *
   220 *
   218  * \returns Test case result.
   221 * \returns Test case result.
   219  */
   222 */
   220 int
   223 int
   221 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
   224 	SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
   222 {
   225 {
   223 	SDL_TimerID timer = 0;
   226 	SDL_TimerID timer = 0;
   224 	int testResult = 0;
   227 	int testResult = 0;
   225 	int fuzzerCount;
   228 	int fuzzerCount;
   226 
   229 
   230 		return TEST_RESULT_SETUP_FAILURE;
   233 		return TEST_RESULT_SETUP_FAILURE;
   231 	}
   234 	}
   232 
   235 
   233 	if (!testCase->enabled)
   236 	if (!testCase->enabled)
   234 	{
   237 	{
   235 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped");
   238 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Disabled)");
   236 		return TEST_RESULT_SKIPPED;
   239 		return TEST_RESULT_SKIPPED;
   237 	}
   240 	}
   238 
   241 
   239         // Initialize fuzzer
   242 
       
   243 	// Initialize fuzzer
   240 	SDLTest_FuzzerInit(execKey);
   244 	SDLTest_FuzzerInit(execKey);
   241 
   245 
   242 	// Reset assert tracker
   246 	// Reset assert tracker
   243 	SDLTest_ResetAssertSummary();
   247 	SDLTest_ResetAssertSummary();
   244 
   248 
   313 	float currentClock = (float)clock();
   317 	float currentClock = (float)clock();
   314 	return currentClock / (float)CLOCKS_PER_SEC;
   318 	return currentClock / (float)CLOCKS_PER_SEC;
   315 }
   319 }
   316 
   320 
   317 /**
   321 /**
   318  * \brief Execute a test suite using the given run seend and execution key.
   322 * \brief Execute a test suite using the given run seend and execution key.
   319  *
   323 *
   320  * \param testSuites Suites containing the test case.
   324 * The filter string is matched to the suite name (full comparison) to select a single suite,
   321  * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   325 * or if no suite matches, it is matched to the test names (full comparison) to select a single test.
   322  * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   326 *
   323  * \param testIterations Number of iterations to run each test case.
   327 * \param testSuites Suites containing the test case.
   324  *
   328 * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   325  * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   329 * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   326  */
   330 * \param filter Filter specification. NULL disables. Case sensitive.
       
   331 * \param testIterations Number of iterations to run each test case.
       
   332 *
       
   333 * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
       
   334 */
   327 int
   335 int
   328 SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, Uint64 userExecKey, int testIterations)
   336 	SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, Uint64 userExecKey, char *filter, int testIterations)
   329 {
   337 {
   330 	int suiteCounter;
   338 	int suiteCounter;
   331 	int testCounter;
   339 	int testCounter;
   332 	int iterationCounter;
   340 	int iterationCounter;
   333 	SDLTest_TestSuiteReference *testSuite;
   341 	SDLTest_TestSuiteReference *testSuite;
   341 	float testStartSeconds;
   349 	float testStartSeconds;
   342 	float runEndSeconds;
   350 	float runEndSeconds;
   343 	float suiteEndSeconds;
   351 	float suiteEndSeconds;
   344 	float testEndSeconds;
   352 	float testEndSeconds;
   345 	float runtime;
   353 	float runtime;
       
   354 	int suiteFilter = 0;
       
   355 	char *suiteFilterName = NULL;
       
   356 	int testFilter = 0;
       
   357 	char *testFilterName = NULL;
   346 	int testResult = 0;
   358 	int testResult = 0;
   347 	int runResult = 0;
   359 	int runResult = 0;
   348 	Uint32 totalTestFailedCount = 0;
   360 	Uint32 totalTestFailedCount = 0;
   349 	Uint32 totalTestPassedCount = 0;
   361 	Uint32 totalTestPassedCount = 0;
   350 	Uint32 totalTestSkippedCount = 0;
   362 	Uint32 totalTestSkippedCount = 0;
   368 		}
   380 		}
   369 	} else {
   381 	} else {
   370 		runSeed = userRunSeed;
   382 		runSeed = userRunSeed;
   371 	}
   383 	}
   372 
   384 
       
   385 
   373 	// Reset per-run counters
   386 	// Reset per-run counters
   374 	totalTestFailedCount = 0;
   387 	totalTestFailedCount = 0;
   375 	totalTestPassedCount = 0;
   388 	totalTestPassedCount = 0;
   376 	totalTestSkippedCount = 0;
   389 	totalTestSkippedCount = 0;
   377 
   390 
   379 	runStartSeconds = GetClock();
   392 	runStartSeconds = GetClock();
   380 
   393 
   381 	// Log run with fuzzer parameters
   394 	// Log run with fuzzer parameters
   382 	SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   395 	SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   383 
   396 
       
   397 	// Initialize filtering
       
   398 	if (filter != NULL && SDL_strlen(filter) > 0) {
       
   399 		/* Loop over all suites to check if we have a filter match */
       
   400 		suiteCounter = 0;
       
   401 		while (testSuites[suiteCounter] && suiteFilter == 0) {
       
   402 			testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
       
   403 			suiteCounter++;
       
   404 			if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
       
   405 				/* Matched a suite name */
       
   406 				suiteFilter = 1;
       
   407 				suiteFilterName = testSuite->name;
       
   408 				SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
       
   409 				break;
       
   410 			}
       
   411 
       
   412 			/* Within each suite, loop over all test cases to check if we have a filter match */
       
   413 			testCounter = 0;
       
   414 			while (testSuite->testCases[testCounter] && testFilter == 0)
       
   415 			{
       
   416 				testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
       
   417 				testCounter++;
       
   418 				if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
       
   419 					/* Matched a test name */
       
   420 					suiteFilter = 1;
       
   421 					suiteFilterName = testSuite->name;
       
   422 					testFilter = 1;
       
   423 					testFilterName = testCase->name;
       
   424 					SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);					
       
   425 					break;
       
   426 				}
       
   427 			}						
       
   428 		}
       
   429 		
       
   430 		if (suiteFilter == 0 && testFilter == 0) {
       
   431 			SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
       
   432 			SDLTest_Log("Exit code: 2");	
       
   433 			return 2;
       
   434 		}		
       
   435 	}
       
   436 
   384 	// Loop over all suites
   437 	// Loop over all suites
   385 	suiteCounter = 0;
   438 	suiteCounter = 0;
   386 	while(testSuites[suiteCounter]) {
   439 	while(testSuites[suiteCounter]) {
   387 		testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   440 		testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
       
   441 		currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   388 		suiteCounter++;
   442 		suiteCounter++;
   389 
   443 
   390 		// Reset per-suite counters
   444 		// Filter suite if flag set and we have a name
   391 		testFailedCount = 0;
   445 		if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
   392 		testPassedCount = 0;
   446 			SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
   393 		testSkippedCount = 0;
   447 				// Skip suite
   394 
   448 				SDLTest_Log("===== Test Suite %i: '%s' skipped\n", 
   395 		// Take time - suite start
   449 					suiteCounter, 
   396 		suiteStartSeconds = GetClock();
   450 					currentSuiteName);
   397 
   451 		} else {
   398 		// Log suite started
   452 
   399 		currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   453 			// Reset per-suite counters
   400 		SDLTest_Log("===== Test Suite %i: '%s' started\n", 
   454 			testFailedCount = 0;
   401 			suiteCounter, 
   455 			testPassedCount = 0;
   402 			currentSuiteName);
   456 			testSkippedCount = 0;
   403 
   457 
   404 		// Loop over all test cases
   458 			// Take time - suite start
   405 		testCounter = 0;
   459 			suiteStartSeconds = GetClock();
   406 		while(testSuite->testCases[testCounter])
   460 
   407 		{
   461 			// Log suite started
   408 			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   462 			SDLTest_Log("===== Test Suite %i: '%s' started\n", 
   409 			testCounter++;
   463 				suiteCounter, 
   410 			
   464 				currentSuiteName);
   411 			// Take time - test start
   465 
   412 			testStartSeconds = GetClock();
   466 			// Loop over all test cases
   413 
   467 			testCounter = 0;
   414 			// Log test started
   468 			while(testSuite->testCases[testCounter])
   415 			currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
       
   416 			SDLTest_Log("----- Test Case %i.%i: '%s' started",
       
   417 			        suiteCounter,
       
   418 				testCounter, 
       
   419 				currentTestName);
       
   420 			if (testCase->description != NULL && strlen(testCase->description)>0) {
       
   421 				SDLTest_Log("Test Description: '%s'", 
       
   422 					(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
       
   423 			}
       
   424 			
       
   425 			// Loop over all iterations
       
   426 			iterationCounter = 0;
       
   427 			while(iterationCounter < testIterations)
       
   428 			{
   469 			{
   429 				iterationCounter++;
   470 				testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   430 
   471 				currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   431 				if (userExecKey != 0) {
   472 				testCounter++;
   432 					execKey = userExecKey;
   473 
       
   474 				// Filter tests if flag set and we have a name
       
   475 				if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
       
   476 					SDL_strcmp(testFilterName, testCase->name) != 0) {
       
   477 						// Skip test
       
   478 						SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n", 
       
   479 							suiteCounter,
       
   480 							testCounter,
       
   481 							currentTestName);
   433 				} else {
   482 				} else {
   434 					execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
   483 
   435 				}
   484 					// Take time - test start
   436 
   485 					testStartSeconds = GetClock();
   437 				SDLTest_Log("Test Iteration %i: execKey %llu", iterationCounter, execKey);
   486 
   438 				testResult = SDLTest_RunTest(testSuite, testCase, execKey);
   487 					// Log test started
   439 
   488 					SDLTest_Log("----- Test Case %i.%i: '%s' started",
   440 				if (testResult == TEST_RESULT_PASSED) {
   489 						suiteCounter,
   441 					testPassedCount++;
   490 						testCounter, 
   442 					totalTestPassedCount++;
   491 						currentTestName);
   443 				} else if (testResult == TEST_RESULT_SKIPPED) {
   492 					if (testCase->description != NULL && strlen(testCase->description)>0) {
   444 					testSkippedCount++;
   493 						SDLTest_Log("Test Description: '%s'", 
   445 					totalTestSkippedCount++;
   494 							(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   446 				} else {
   495 					}
   447 					testFailedCount++;
   496 
   448 					totalTestFailedCount++;
   497 					// Loop over all iterations
       
   498 					iterationCounter = 0;
       
   499 					while(iterationCounter < testIterations)
       
   500 					{
       
   501 						iterationCounter++;
       
   502 
       
   503 						if (userExecKey != 0) {
       
   504 							execKey = userExecKey;
       
   505 						} else {
       
   506 							execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
       
   507 						}
       
   508 
       
   509 						SDLTest_Log("Test Iteration %i: execKey %llu", iterationCounter, execKey);
       
   510 						testResult = SDLTest_RunTest(testSuite, testCase, execKey);
       
   511 
       
   512 						if (testResult == TEST_RESULT_PASSED) {
       
   513 							testPassedCount++;
       
   514 							totalTestPassedCount++;
       
   515 						} else if (testResult == TEST_RESULT_SKIPPED) {
       
   516 							testSkippedCount++;
       
   517 							totalTestSkippedCount++;
       
   518 						} else {
       
   519 							testFailedCount++;
       
   520 							totalTestFailedCount++;
       
   521 						}
       
   522 					}
       
   523 
       
   524 					// Take time - test end
       
   525 					testEndSeconds = GetClock();
       
   526 					runtime = testEndSeconds - testStartSeconds;
       
   527 					if (runtime < 0.0f) runtime = 0.0f;
       
   528 
       
   529 					if (testIterations > 1) {
       
   530 						// Log test runtime
       
   531 						SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
       
   532 						SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
       
   533 					} else {
       
   534 						// Log test runtime
       
   535 						SDLTest_Log("Total Test runtime: %.1f sec", runtime);
       
   536 					}
       
   537 
       
   538 					// Log final test result
       
   539 					switch (testResult) {
       
   540 					case TEST_RESULT_PASSED:
       
   541 						SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
       
   542 						break;
       
   543 					case TEST_RESULT_FAILED:
       
   544 						SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
       
   545 						break;
       
   546 					case TEST_RESULT_NO_ASSERT:
       
   547 						SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
       
   548 						break;
       
   549 					}
       
   550 
   449 				}
   551 				}
   450 			}
   552 			}
   451 
   553 
   452 			// Take time - test end
   554 			// Take time - suite end
   453 			testEndSeconds = GetClock();
   555 			suiteEndSeconds = GetClock();
   454 			runtime = testEndSeconds - testStartSeconds;
   556 			runtime = suiteEndSeconds - suiteStartSeconds;
   455 			if (runtime < 0.0f) runtime = 0.0f;
   557 			if (runtime < 0.0f) runtime = 0.0f;
   456 
   558 
   457 			if (testIterations > 1) {
   559 			// Log suite runtime
   458         			// Log test runtime
   560 			SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
   459 	        		SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
   561 
   460 	        		SDLTest_Log("Test runtime: %.5f sec", runtime / (float)testIterations);
   562 			// Log summary and final Suite result
   461                         } else {
   563 			countSum = testPassedCount + testFailedCount + testSkippedCount;
   462         			// Log test runtime
   564 			if (testFailedCount == 0)
   463 	        		SDLTest_Log("Test runtime: %.1f sec", runtime);
   565 			{
   464                         }
   566 				SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   465 
   567 				SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
   466 			// Log final test result
   568 			} 
   467 			switch (testResult) {
   569 			else 
   468 				case TEST_RESULT_PASSED:
   570 			{
   469 					SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   571 				SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   470 					break;
   572 				SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
   471 				case TEST_RESULT_FAILED:
       
   472 					SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
       
   473 					break;
       
   474 				case TEST_RESULT_NO_ASSERT:
       
   475 					SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
       
   476 					break;
       
   477 			}
   573 			}
   478 		}
   574 
   479 
       
   480 		// Take time - suite end
       
   481 		suiteEndSeconds = GetClock();
       
   482 		runtime = suiteEndSeconds - suiteStartSeconds;
       
   483 		if (runtime < 0.0f) runtime = 0.0f;
       
   484 
       
   485 		// Log suite runtime
       
   486 		SDLTest_Log("Suite runtime: %.1f sec", runtime);
       
   487 
       
   488 		// Log summary and final Suite result
       
   489 		countSum = testPassedCount + testFailedCount + testSkippedCount;
       
   490 		if (testFailedCount == 0)
       
   491 		{
       
   492 			SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
       
   493 			SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
       
   494 		} 
       
   495 		else 
       
   496 		{
       
   497 			SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
       
   498 			SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
       
   499 		}
   575 		}
   500 	}
   576 	}
   501 
   577 
   502 	// Take time - run end
   578 	// Take time - run end
   503 	runEndSeconds = GetClock();
   579 	runEndSeconds = GetClock();
   504 	runtime = runEndSeconds - runStartSeconds;
   580 	runtime = runEndSeconds - runStartSeconds;
   505 	if (runtime < 0.0f) runtime = 0.0f;
   581 	if (runtime < 0.0f) runtime = 0.0f;
   506 
   582 
   507 	// Log total runtime
   583 	// Log total runtime
   508 	SDLTest_Log("Total runtime: %.1f sec", runtime);
   584 	SDLTest_Log("Total Run runtime: %.1f sec", runtime);
   509 
   585 
   510 	// Log summary and final run result
   586 	// Log summary and final run result
   511 	countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   587 	countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   512 	if (testFailedCount == 0)
   588 	if (totalTestFailedCount == 0)
   513 	{
   589 	{
   514 		runResult = 0;
   590 		runResult = 0;
   515 		SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   591 		SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   516 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
   592 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
   517 	} 
   593 	}