src/test/SDL_test_harness.c
author Sam Lantinga
Fri, 15 Feb 2013 08:47:44 -0800
changeset 6885 700f1b25f77f
parent 6830 0cd3e2e9b2c5
child 7189 414be1d64060
permissions -rw-r--r--
Happy New Year!
     1 /*
     2 Simple DirectMedia Layer
     3 Copyright (C) 1997-2013 Sam Lantinga <slouken@libsdl.org>
     4 
     5 This software is provided 'as-is', without any express or implied
     6 warranty.  In no event will the authors be held liable for any damages
     7 arising from the use of this software.
     8 
     9 Permission is granted to anyone to use this software for any purpose,
    10 including commercial applications, and to alter it and redistribute it
    11 freely, subject to the following restrictions:
    12 
    13 1. The origin of this software must not be misrepresented; you must not
    14 claim that you wrote the original software. If you use this software
    15 in a product, an acknowledgment in the product documentation would be
    16 appreciated but is not required.
    17 2. Altered source versions must be plainly marked as such, and must not be
    18 misrepresented as being the original software.
    19 3. This notice may not be removed or altered from any source distribution.
    20 */
    21 
    22 #include "SDL_config.h"
    23 
    24 #include "SDL_test.h"
    25 
    26 #include <stdio.h>
    27 #include <stdlib.h>
    28 #include <string.h>
    29 #include <time.h>
    30 
    31 /* Invalid test name/description message format */
    32 const char *SDLTest_InvalidNameFormat = "(Invalid)";
    33 
    34 /* Log summary message format */
    35 const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d";
    36 
    37 /* Final result message format */
    38 const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n";
    39 
    40 /*! \brief Timeout for single test case execution */
    41 static Uint32 SDLTest_TestCaseTimeout = 3600;
    42 
    43 /**
    44 * Generates a random run seed string for the harness. The generated seed
    45 * will contain alphanumeric characters (0-9A-Z).
    46 *
    47 * Note: The returned string needs to be deallocated by the caller.
    48 *
    49 * \param length The length of the seed string to generate
    50 *
    51 * \returns The generated seed string
    52 */
    53 char *
    54 SDLTest_GenerateRunSeed(const int length)
    55 {
    56 	char *seed = NULL;
    57 	SDLTest_RandomContext randomContext;
    58 	int counter;
    59 
    60 	// Sanity check input
    61 	if (length <= 0) {
    62 		SDLTest_LogError("The length of the harness seed must be >0.");
    63 		return NULL;
    64 	}
    65 
    66 	// Allocate output buffer
    67 	seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    68 	if (seed == NULL) {
    69 		SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    70 		return NULL;
    71 	}
    72 
    73 	// Generate a random string of alphanumeric characters
    74 	SDLTest_RandomInitTime(&randomContext);
    75 	for (counter = 0; counter < length - 1; ++counter) {
    76 		unsigned int number = SDLTest_Random(&randomContext);
    77 		char ch = (char) (number % (91 - 48)) + 48;
    78 		if (ch >= 58 && ch <= 64) {
    79 			ch = 65;
    80 		}
    81 		seed[counter] = ch;
    82 	}
    83 	seed[counter] = '\0';
    84 
    85 	return seed;
    86 }
    87 
    88 /**
    89 * Generates an execution key for the fuzzer.
    90 *
    91 * \param runSeed		The run seed to use
    92 * \param suiteName		The name of the test suite
    93 * \param testName		The name of the test
    94 * \param iteration		The iteration count
    95 *
    96 * \returns The generated execution key to initialize the fuzzer with.
    97 *
    98 */
    99 Uint64
   100 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
   101 {
   102 	SDLTest_Md5Context md5Context;
   103 	Uint64 *keys;
   104 	char iterationString[16];
   105 	Uint32 runSeedLength;
   106 	Uint32 suiteNameLength;
   107 	Uint32 testNameLength;
   108 	Uint32 iterationStringLength;
   109 	Uint32 entireStringLength;
   110 	char *buffer;
   111 
   112 	if (runSeed == NULL || SDL_strlen(runSeed)==0) {
   113 		SDLTest_LogError("Invalid runSeed string.");
   114 		return -1;
   115 	}
   116 
   117 	if (suiteName == NULL || SDL_strlen(suiteName)==0) {
   118 		SDLTest_LogError("Invalid suiteName string.");
   119 		return -1;
   120 	}
   121 
   122 	if (testName == NULL || SDL_strlen(testName)==0) {
   123 		SDLTest_LogError("Invalid testName string.");
   124 		return -1;
   125 	}
   126 
   127 	if (iteration <= 0) {
   128 		SDLTest_LogError("Invalid iteration count.");
   129 		return -1;
   130 	}
   131 
   132 	// Convert iteration number into a string
   133 	SDL_memset(iterationString, 0, sizeof(iterationString));
   134 	SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
   135 
   136 	// Combine the parameters into single string
   137 	runSeedLength = SDL_strlen(runSeed);
   138 	suiteNameLength = SDL_strlen(suiteName);
   139 	testNameLength = SDL_strlen(testName);
   140 	iterationStringLength = SDL_strlen(iterationString);
   141 	entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
   142 	buffer = (char *)SDL_malloc(entireStringLength);
   143 	if (buffer == NULL) {
   144 		SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
   145 		return 0;
   146 	}
   147 	SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
   148 
   149 	// Hash string and use half of the digest as 64bit exec key
   150 	SDLTest_Md5Init(&md5Context);
   151 	SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
   152 	SDLTest_Md5Final(&md5Context);
   153 	SDL_free(buffer);
   154 	keys = (Uint64 *)md5Context.digest;
   155 
   156 	return keys[0];
   157 }
   158 
   159 /**
   160 * \brief Set timeout handler for test.
   161 *
   162 * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   163 *
   164 * \param timeout Timeout interval in seconds.
   165 * \param callback Function that will be called after timeout has elapsed.
   166 * 
   167 * \return Timer id or -1 on failure.
   168 */
   169 SDL_TimerID
   170 SDLTest_SetTestTimeout(int timeout, void (*callback)())
   171 {
   172 	Uint32 timeoutInMilliseconds;
   173 	SDL_TimerID timerID;
   174 
   175 	if (callback == NULL) {
   176 		SDLTest_LogError("Timeout callback can't be NULL");
   177 		return -1;
   178 	}
   179 
   180 	if (timeout < 0) {
   181 		SDLTest_LogError("Timeout value must be bigger than zero.");
   182 		return -1;
   183 	}
   184 
   185 	/* Init SDL timer if not initialized before */
   186 	if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
   187 		if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
   188 			SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
   189 			return -1;
   190 		}
   191 	}
   192 
   193 	/* Set timer */
   194 	timeoutInMilliseconds = timeout * 1000;
   195 	timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
   196 	if (timerID == 0) {
   197 		SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
   198 		return -1;
   199 	}
   200 
   201 	return timerID;
   202 }
   203 
   204 /**
   205 * \brief Timeout handler. Aborts test run and exits harness process.
   206 */
   207 void
   208 	SDLTest_BailOut()
   209 {
   210 	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   211 	exit(TEST_ABORTED); // bail out from the test
   212 }
   213 
   214 /**
   215 * \brief Execute a test using the given execution key.
   216 *
   217 * \param testSuite Suite containing the test case.
   218 * \param testCase Case to execute.
   219 * \param execKey Execution key for the fuzzer.
   220 *
   221 * \returns Test case result.
   222 */
   223 int
   224 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
   225 {
   226 	SDL_TimerID timer = 0;
   227 	int testResult = 0;
   228 	int fuzzerCount;
   229 
   230 	if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
   231 	{
   232 		SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
   233 		return TEST_RESULT_SETUP_FAILURE;
   234 	}
   235 
   236 	if (!testCase->enabled)
   237 	{
   238 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Disabled)");
   239 		return TEST_RESULT_SKIPPED;
   240 	}
   241 
   242 
   243 	// Initialize fuzzer
   244 	SDLTest_FuzzerInit(execKey);
   245 
   246 	// Reset assert tracker
   247 	SDLTest_ResetAssertSummary();
   248 
   249 	// Set timeout timer
   250 	timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
   251 
   252 	// Maybe run suite initalizer function
   253 	if (testSuite->testSetUp) {
   254 		testSuite->testSetUp(0x0);
   255 		if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
   256 			SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed");
   257 			return TEST_RESULT_SETUP_FAILURE;
   258 		}
   259 	}
   260 
   261 	// Run test case function
   262 	testCase->testCase(0x0);
   263 	testResult = SDLTest_AssertSummaryToTestResult();
   264 
   265 	// Maybe run suite cleanup function (ignore failed asserts)
   266 	if (testSuite->testTearDown) {
   267 		testSuite->testTearDown(0x0);
   268 	}
   269 
   270 	// Cancel timeout timer
   271 	if (timer) {
   272 		SDL_RemoveTimer(timer);
   273 	}
   274 
   275 	// Report on asserts and fuzzer usage
   276 	fuzzerCount = SDLTest_GetFuzzerInvocationCount();
   277 	if (fuzzerCount > 0) {
   278 		SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
   279 	}
   280 	SDLTest_LogAssertSummary();
   281 
   282 	return testResult;
   283 }
   284 
   285 /* Prints summary of all suites/tests contained in the given reference */
   286 void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
   287 {
   288 	int suiteCounter;
   289 	int testCounter;
   290 	SDLTest_TestSuiteReference *testSuite;
   291 	SDLTest_TestCaseReference *testCase;
   292 
   293 	// Loop over all suites
   294 	suiteCounter = 0;
   295 	while(&testSuites[suiteCounter]) {
   296 		testSuite=&testSuites[suiteCounter];
   297 		suiteCounter++;
   298 		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   299 			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   300 
   301 		// Loop over all test cases
   302 		testCounter = 0;
   303 		while(testSuite->testCases[testCounter])
   304 		{
   305 			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   306 			testCounter++;
   307 			SDLTest_Log("  Test Case %i - %s: %s", testCounter, 
   308 				(testCase->name) ? testCase->name : SDLTest_InvalidNameFormat, 
   309 				(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   310 		}
   311 	}
   312 }
   313 
   314 /* Gets a timer value in seconds */
   315 float GetClock()
   316 {
   317 	float currentClock = (float)clock();
   318 	return currentClock / (float)CLOCKS_PER_SEC;
   319 }
   320 
   321 /**
   322 * \brief Execute a test suite using the given run seend and execution key.
   323 *
   324 * The filter string is matched to the suite name (full comparison) to select a single suite,
   325 * or if no suite matches, it is matched to the test names (full comparison) to select a single test.
   326 *
   327 * \param testSuites Suites containing the test case.
   328 * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   329 * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   330 * \param filter Filter specification. NULL disables. Case sensitive.
   331 * \param testIterations Number of iterations to run each test case.
   332 *
   333 * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   334 */
   335 int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
   336 {
   337 	int suiteCounter;
   338 	int testCounter;
   339 	int iterationCounter;
   340 	SDLTest_TestSuiteReference *testSuite;
   341 	SDLTest_TestCaseReference *testCase;
   342 	const char *runSeed = NULL;
   343 	char *currentSuiteName;
   344 	char *currentTestName;
   345 	Uint64 execKey;
   346 	float runStartSeconds;
   347 	float suiteStartSeconds;
   348 	float testStartSeconds;
   349 	float runEndSeconds;
   350 	float suiteEndSeconds;
   351 	float testEndSeconds;
   352 	float runtime;
   353 	int suiteFilter = 0;
   354 	char *suiteFilterName = NULL;
   355 	int testFilter = 0;
   356 	char *testFilterName = NULL;
   357 	int testResult = 0;
   358 	int runResult = 0;
   359 	Uint32 totalTestFailedCount = 0;
   360 	Uint32 totalTestPassedCount = 0;
   361 	Uint32 totalTestSkippedCount = 0;
   362 	Uint32 testFailedCount = 0;
   363 	Uint32 testPassedCount = 0;
   364 	Uint32 testSkippedCount = 0;
   365 	Uint32 countSum = 0;
   366 	char *logFormat = (char *)SDLTest_LogSummaryFormat;
   367 
   368 	// Sanitize test iterations
   369 	if (testIterations < 1) {
   370 		testIterations = 1;
   371 	}
   372 
   373 	// Generate run see if we don't have one already
   374 	if (userRunSeed == NULL || SDL_strlen(userRunSeed) == 0) {
   375 		runSeed = SDLTest_GenerateRunSeed(16);
   376 		if (runSeed == NULL) {
   377 			SDLTest_LogError("Generating a random seed failed");
   378 			return 2;
   379 		}
   380 	} else {
   381 		runSeed = userRunSeed;
   382 	}
   383 
   384 
   385 	// Reset per-run counters
   386 	totalTestFailedCount = 0;
   387 	totalTestPassedCount = 0;
   388 	totalTestSkippedCount = 0;
   389 
   390 	// Take time - run start
   391 	runStartSeconds = GetClock();
   392 
   393 	// Log run with fuzzer parameters
   394 	SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   395 
   396 	// Initialize filtering
   397 	if (filter != NULL && SDL_strlen(filter) > 0) {
   398 		/* Loop over all suites to check if we have a filter match */
   399 		suiteCounter = 0;
   400 		while (testSuites[suiteCounter] && suiteFilter == 0) {
   401 			testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   402 			suiteCounter++;
   403 			if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
   404 				/* Matched a suite name */
   405 				suiteFilter = 1;
   406 				suiteFilterName = testSuite->name;
   407 				SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
   408 				break;
   409 			}
   410 
   411 			/* Within each suite, loop over all test cases to check if we have a filter match */
   412 			testCounter = 0;
   413 			while (testSuite->testCases[testCounter] && testFilter == 0)
   414 			{
   415 				testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   416 				testCounter++;
   417 				if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
   418 					/* Matched a test name */
   419 					suiteFilter = 1;
   420 					suiteFilterName = testSuite->name;
   421 					testFilter = 1;
   422 					testFilterName = testCase->name;
   423 					SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);					
   424 					break;
   425 				}
   426 			}						
   427 		}
   428 		
   429 		if (suiteFilter == 0 && testFilter == 0) {
   430 			SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
   431 			SDLTest_Log("Exit code: 2");	
   432 			return 2;
   433 		}		
   434 	}
   435 
   436 	// Loop over all suites
   437 	suiteCounter = 0;
   438 	while(testSuites[suiteCounter]) {
   439 		testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   440 		currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   441 		suiteCounter++;
   442 
   443 		// Filter suite if flag set and we have a name
   444 		if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
   445 			SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
   446 				// Skip suite
   447 				SDLTest_Log("===== Test Suite %i: '%s' skipped\n", 
   448 					suiteCounter, 
   449 					currentSuiteName);
   450 		} else {
   451 
   452 			// Reset per-suite counters
   453 			testFailedCount = 0;
   454 			testPassedCount = 0;
   455 			testSkippedCount = 0;
   456 
   457 			// Take time - suite start
   458 			suiteStartSeconds = GetClock();
   459 
   460 			// Log suite started
   461 			SDLTest_Log("===== Test Suite %i: '%s' started\n", 
   462 				suiteCounter, 
   463 				currentSuiteName);
   464 
   465 			// Loop over all test cases
   466 			testCounter = 0;
   467 			while(testSuite->testCases[testCounter])
   468 			{
   469 				testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   470 				currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   471 				testCounter++;
   472 
   473 				// Filter tests if flag set and we have a name
   474 				if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
   475 					SDL_strcmp(testFilterName, testCase->name) != 0) {
   476 						// Skip test
   477 						SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n", 
   478 							suiteCounter,
   479 							testCounter,
   480 							currentTestName);
   481 				} else {
   482 					// Override 'disabled' flag if we specified a test filter (i.e. force run for debugging)
   483 					if (testFilter == 1 && !testCase->enabled) {
   484 						SDLTest_Log("Force run of disabled test since test filter was set");
   485 						testCase->enabled = 1;
   486 					}
   487 
   488 					// Take time - test start
   489 					testStartSeconds = GetClock();
   490 
   491 					// Log test started
   492 					SDLTest_Log("----- Test Case %i.%i: '%s' started",
   493 						suiteCounter,
   494 						testCounter, 
   495 						currentTestName);
   496 					if (testCase->description != NULL && SDL_strlen(testCase->description)>0) {
   497 						SDLTest_Log("Test Description: '%s'", 
   498 							(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   499 					}
   500 
   501 					// Loop over all iterations
   502 					iterationCounter = 0;
   503 					while(iterationCounter < testIterations)
   504 					{
   505 						iterationCounter++;
   506 
   507 						if (userExecKey != 0) {
   508 							execKey = userExecKey;
   509 						} else {
   510 							execKey = SDLTest_GenerateExecKey((char *)runSeed, testSuite->name, testCase->name, iterationCounter);
   511 						}
   512 
   513 						SDLTest_Log("Test Iteration %i: execKey %llu", iterationCounter, execKey);
   514 						testResult = SDLTest_RunTest(testSuite, testCase, execKey);
   515 
   516 						if (testResult == TEST_RESULT_PASSED) {
   517 							testPassedCount++;
   518 							totalTestPassedCount++;
   519 						} else if (testResult == TEST_RESULT_SKIPPED) {
   520 							testSkippedCount++;
   521 							totalTestSkippedCount++;
   522 						} else {
   523 							testFailedCount++;
   524 							totalTestFailedCount++;
   525 						}
   526 					}
   527 
   528 					// Take time - test end
   529 					testEndSeconds = GetClock();
   530 					runtime = testEndSeconds - testStartSeconds;
   531 					if (runtime < 0.0f) runtime = 0.0f;
   532 
   533 					if (testIterations > 1) {
   534 						// Log test runtime
   535 						SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
   536 						SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
   537 					} else {
   538 						// Log test runtime
   539 						SDLTest_Log("Total Test runtime: %.1f sec", runtime);
   540 					}
   541 
   542 					// Log final test result
   543 					switch (testResult) {
   544 					case TEST_RESULT_PASSED:
   545 						SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   546 						break;
   547 					case TEST_RESULT_FAILED:
   548 						SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
   549 						break;
   550 					case TEST_RESULT_NO_ASSERT:
   551 						SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
   552 						break;
   553 					}
   554 
   555 				}
   556 			}
   557 
   558 			// Take time - suite end
   559 			suiteEndSeconds = GetClock();
   560 			runtime = suiteEndSeconds - suiteStartSeconds;
   561 			if (runtime < 0.0f) runtime = 0.0f;
   562 
   563 			// Log suite runtime
   564 			SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
   565 
   566 			// Log summary and final Suite result
   567 			countSum = testPassedCount + testFailedCount + testSkippedCount;
   568 			if (testFailedCount == 0)
   569 			{
   570 				SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   571 				SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
   572 			} 
   573 			else 
   574 			{
   575 				SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   576 				SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
   577 			}
   578 
   579 		}
   580 	}
   581 
   582 	// Take time - run end
   583 	runEndSeconds = GetClock();
   584 	runtime = runEndSeconds - runStartSeconds;
   585 	if (runtime < 0.0f) runtime = 0.0f;
   586 
   587 	// Log total runtime
   588 	SDLTest_Log("Total Run runtime: %.1f sec", runtime);
   589 
   590 	// Log summary and final run result
   591 	countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   592 	if (totalTestFailedCount == 0)
   593 	{
   594 		runResult = 0;
   595 		SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   596 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
   597 	} 
   598 	else 
   599 	{
   600 		runResult = 1;
   601 		SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   602 		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
   603 	}
   604 
   605 	SDLTest_Log("Exit code: %d", runResult);	
   606 	return runResult;
   607 }