src/test/SDL_test_harness.c
author Andreas Schiffler
Sat, 22 Dec 2012 16:06:55 -0800
changeset 6763 9cbd31a3450b
parent 6760 04dcce3081e6
child 6768 22c37bf0afbc
permissions -rw-r--r--
Test lib updates: updated harness to support filtering, added surface comparer, updated interface to test images; added Render test suite from GSOC project
     1 /*
     2 Simple DirectMedia Layer
     3 Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
     4 
     5 This software is provided 'as-is', without any express or implied
     6 warranty.  In no event will the authors be held liable for any damages
     7 arising from the use of this software.
     8 
     9 Permission is granted to anyone to use this software for any purpose,
    10 including commercial applications, and to alter it and redistribute it
    11 freely, subject to the following restrictions:
    12 
    13 1. The origin of this software must not be misrepresented; you must not
    14 claim that you wrote the original software. If you use this software
    15 in a product, an acknowledgment in the product documentation would be
    16 appreciated but is not required.
    17 2. Altered source versions must be plainly marked as such, and must not be
    18 misrepresented as being the original software.
    19 3. This notice may not be removed or altered from any source distribution.
    20 */
    21 
    22 #include "SDL_config.h"
    23 
    24 #include "SDL_test.h"
    25 
    26 #include <stdio.h>
    27 #include <stdlib.h>
    28 #include <string.h>
    29 #include <time.h>
    30 
    31 /* Invalid test name/description message format */
    32 const char *SDLTest_InvalidNameFormat = "(Invalid)";
    33 
    34 /* Log summary message format */
    35 const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d";
    36 
    37 /* Final result message format */
    38 const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n";
    39 
    40 /*! \brief Timeout for single test case execution */
    41 static Uint32 SDLTest_TestCaseTimeout = 3600;
    42 
    43 /**
    44 * Generates a random run seed string for the harness. The generated seed
    45 * will contain alphanumeric characters (0-9A-Z).
    46 *
    47 * Note: The returned string needs to be deallocated by the caller.
    48 *
    49 * \param length The length of the seed string to generate
    50 *
    51 * \returns The generated seed string
    52 */
    53 char *
    54 	SDLTest_GenerateRunSeed(const int length)
    55 {
    56 	char *seed = NULL;
    57 	SDLTest_RandomContext randomContext;
    58 	int counter;
    59 
    60 	// Sanity check input
    61 	if (length <= 0) {
    62 		SDLTest_LogError("The length of the harness seed must be >0.");
    63 		return NULL;
    64 	}
    65 
    66 	// Allocate output buffer
    67 	seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    68 	if (seed == NULL) {
    69 		SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    70 		return NULL;
    71 	}
    72 
    73 	// Generate a random string of alphanumeric characters
    74 	SDLTest_RandomInitTime(&randomContext);
    75 	for (counter = 0; counter < length - 1; ++counter) {
    76 		unsigned int number = SDLTest_Random(&randomContext);
    77 		char ch = (char) (number % (91 - 48)) + 48;
    78 		if (ch >= 58 && ch <= 64) {
    79 			ch = 65;
    80 		}
    81 		seed[counter] = ch;
    82 	}
    83 	seed[counter] = '\0';
    84 
    85 	return seed;
    86 }
    87 
    88 /**
    89 * Generates an execution key for the fuzzer.
    90 *
    91 * \param runSeed		The run seed to use
    92 * \param suiteName		The name of the test suite
    93 * \param testName		The name of the test
    94 * \param iteration		The iteration count
    95 *
    96 * \returns The generated execution key to initialize the fuzzer with.
    97 *
    98 */
    99 Uint64
   100 	SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
   101 {
   102 	SDLTest_Md5Context md5Context;
   103 	Uint64 *keys;
   104 	char iterationString[16];
   105 	Uint32 runSeedLength;
   106 	Uint32 suiteNameLength;
   107 	Uint32 testNameLength;
   108 	Uint32 iterationStringLength;
   109 	Uint32 entireStringLength;
   110 	char *buffer;
   111 
   112 	if (runSeed == NULL || strlen(runSeed)==0) {
   113 		SDLTest_LogError("Invalid runSeed string.");
   114 		return -1;
   115 	}
   116 
   117 	if (suiteName == NULL || strlen(suiteName)==0) {
   118 		SDLTest_LogError("Invalid suiteName string.");
   119 		return -1;
   120 	}
   121 
   122 	if (testName == NULL || strlen(testName)==0) {
   123 		SDLTest_LogError("Invalid testName string.");
   124 		return -1;
   125 	}
   126 
   127 	if (iteration <= 0) {
   128 		SDLTest_LogError("Invalid iteration count.");
   129 		return -1;
   130 	}
   131 
   132 	// Convert iteration number into a string
   133 	memset(iterationString, 0, sizeof(iterationString));
   134 	SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
   135 
   136 	// Combine the parameters into single string
   137 	runSeedLength = strlen(runSeed);
   138 	suiteNameLength = strlen(suiteName);
   139 	testNameLength = strlen(testName);
   140 	iterationStringLength = strlen(iterationString);
   141 	entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
   142 	buffer = (char *)SDL_malloc(entireStringLength);
   143 	if (buffer == NULL) {
   144 		SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
   145 		return 0;
   146 	}
   147 	SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
   148 
   149 	// Hash string and use half of the digest as 64bit exec key
   150 	SDLTest_Md5Init(&md5Context);
   151 	SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
   152 	SDLTest_Md5Final(&md5Context);
   153 	SDL_free(buffer);
   154 	keys = (Uint64 *)md5Context.digest;
   155 
   156 	return keys[0];
   157 }
   158 
   159 /**
   160 * \brief Set timeout handler for test.
   161 *
   162 * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   163 *
   164 * \param timeout Timeout interval in seconds.
   165 * \param callback Function that will be called after timeout has elapsed.
   166 * 
   167 * \return Timer id or -1 on failure.
   168 */
   169 SDL_TimerID
   170 	SDLTest_SetTestTimeout(int timeout, void (*callback)())
   171 {
   172 	Uint32 timeoutInMilliseconds;
   173 	SDL_TimerID timerID;
   174 
   175 	if (callback == NULL) {
   176 		SDLTest_LogError("Timeout callback can't be NULL");
   177 		return -1;
   178 	}
   179 
   180 	if (timeout < 0) {
   181 		SDLTest_LogError("Timeout value must be bigger than zero.");
   182 		return -1;
   183 	}
   184 
   185 	/* Init SDL timer if not initialized before */
   186 	if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
   187 		if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
   188 			SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
   189 			return -1;
   190 		}
   191 	}
   192 
   193 	/* Set timer */
   194 	timeoutInMilliseconds = timeout * 1000;
   195 	timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
   196 	if (timerID == 0) {
   197 		SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
   198 		return -1;
   199 	}
   200 
   201 	return timerID;
   202 }
   203 
   204 /**
   205 * \brief Timeout handler. Aborts test run and exits harness process.
   206 */
   207 void
   208 	SDLTest_BailOut()
   209 {
   210 	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   211 	exit(TEST_ABORTED); // bail out from the test
   212 }
   213 
   214 /**
   215 * \brief Execute a test using the given execution key.
   216 *
   217 * \param testSuite Suite containing the test case.
   218 * \param testCase Case to execute.
   219 * \param execKey Execution key for the fuzzer.
   220 *
   221 * \returns Test case result.
   222 */
   223 int
   224 	SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
   225 {
   226 	SDL_TimerID timer = 0;
   227 	int testResult = 0;
   228 	int fuzzerCount;
   229 
   230 	if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
   231 	{
   232 		SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
   233 		return TEST_RESULT_SETUP_FAILURE;
   234 	}
   235 
   236 	if (!testCase->enabled)
   237 	{
   238 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Disabled)");
   239 		return TEST_RESULT_SKIPPED;
   240 	}
   241 
   242 
   243 	// Initialize fuzzer
   244 	SDLTest_FuzzerInit(execKey);
   245 
   246 	// Reset assert tracker
   247 	SDLTest_ResetAssertSummary();
   248 
   249 	// Set timeout timer
   250 	timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
   251 
   252 	// Maybe run suite initalizer function
   253 	if (testSuite->testSetUp) {
   254 		testSuite->testSetUp(0x0);
   255 		if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
   256 			SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed");
   257 			return TEST_RESULT_SETUP_FAILURE;
   258 		}
   259 	}
   260 
   261 	// Run test case function
   262 	testCase->testCase(0x0);
   263 	testResult = SDLTest_AssertSummaryToTestResult();
   264 
   265 	// Maybe run suite cleanup function (ignore failed asserts)
   266 	if (testSuite->testTearDown) {
   267 		testSuite->testTearDown(0x0);
   268 	}
   269 
   270 	// Cancel timeout timer
   271 	if (timer) {
   272 		SDL_RemoveTimer(timer);
   273 	}
   274 
   275 	// Report on asserts and fuzzer usage
   276 	fuzzerCount = SDLTest_GetFuzzerInvocationCount();
   277 	if (fuzzerCount > 0) {
   278 		SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
   279 	}
   280 	SDLTest_LogAssertSummary();
   281 
   282 	return testResult;
   283 }
   284 
   285 /* Prints summary of all suites/tests contained in the given reference */
   286 void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
   287 {
   288 	int suiteCounter;
   289 	int testCounter;
   290 	SDLTest_TestSuiteReference *testSuite;
   291 	SDLTest_TestCaseReference *testCase;
   292 
   293 	// Loop over all suites
   294 	suiteCounter = 0;
   295 	while(&testSuites[suiteCounter]) {
   296 		testSuite=&testSuites[suiteCounter];
   297 		suiteCounter++;
   298 		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   299 			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   300 
   301 		// Loop over all test cases
   302 		testCounter = 0;
   303 		while(testSuite->testCases[testCounter])
   304 		{
   305 			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   306 			testCounter++;
   307 			SDLTest_Log("  Test Case %i - %s: %s", testCounter, 
   308 				(testCase->name) ? testCase->name : SDLTest_InvalidNameFormat, 
   309 				(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   310 		}
   311 	}
   312 }
   313 
   314 /* Gets a timer value in seconds */
   315 float GetClock()
   316 {
   317 	float currentClock = (float)clock();
   318 	return currentClock / (float)CLOCKS_PER_SEC;
   319 }
   320 
   321 /**
   322 * \brief Execute a test suite using the given run seend and execution key.
   323 *
   324 * The filter string is matched to the suite name (full comparison) to select a single suite,
   325 * or if no suite matches, it is matched to the test names (full comparison) to select a single test.
   326 *
   327 * \param testSuites Suites containing the test case.
   328 * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   329 * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   330 * \param filter Filter specification. NULL disables. Case sensitive.
   331 * \param testIterations Number of iterations to run each test case.
   332 *
   333 * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   334 */
   335 int
   336 	SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, Uint64 userExecKey, char *filter, int testIterations)
   337 {
   338 	int suiteCounter;
   339 	int testCounter;
   340 	int iterationCounter;
   341 	SDLTest_TestSuiteReference *testSuite;
   342 	SDLTest_TestCaseReference *testCase;
   343 	char *runSeed = NULL;
   344 	char *currentSuiteName;
   345 	char *currentTestName;
   346 	Uint64 execKey;
   347 	float runStartSeconds;
   348 	float suiteStartSeconds;
   349 	float testStartSeconds;
   350 	float runEndSeconds;
   351 	float suiteEndSeconds;
   352 	float testEndSeconds;
   353 	float runtime;
   354 	int suiteFilter = 0;
   355 	char *suiteFilterName = NULL;
   356 	int testFilter = 0;
   357 	char *testFilterName = NULL;
   358 	int testResult = 0;
   359 	int runResult = 0;
   360 	Uint32 totalTestFailedCount = 0;
   361 	Uint32 totalTestPassedCount = 0;
   362 	Uint32 totalTestSkippedCount = 0;
   363 	Uint32 testFailedCount = 0;
   364 	Uint32 testPassedCount = 0;
   365 	Uint32 testSkippedCount = 0;
   366 	Uint32 countSum = 0;
   367 	char *logFormat = (char *)SDLTest_LogSummaryFormat;
   368 
   369 	// Sanitize test iterations
   370 	if (testIterations < 1) {
   371 		testIterations = 1;
   372 	}
   373 
   374 	// Generate run see if we don't have one already
   375 	if (userRunSeed == NULL || strlen(userRunSeed) == 0) {
   376 		runSeed = SDLTest_GenerateRunSeed(16);
   377 		if (runSeed == NULL) {
   378 			SDLTest_LogError("Generating a random seed failed");
   379 			return 2;
   380 		}
   381 	} else {
   382 		runSeed = userRunSeed;
   383 	}
   384 
   385 
   386 	// Reset per-run counters
   387 	totalTestFailedCount = 0;
   388 	totalTestPassedCount = 0;
   389 	totalTestSkippedCount = 0;
   390 
   391 	// Take time - run start
   392 	runStartSeconds = GetClock();
   393 
   394 	// Log run with fuzzer parameters
   395 	SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   396 
   397 	// Initialize filtering
   398 	if (filter != NULL && SDL_strlen(filter) > 0) {
   399 		/* Loop over all suites to check if we have a filter match */
   400 		suiteCounter = 0;
   401 		while (testSuites[suiteCounter] && suiteFilter == 0) {
   402 			testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   403 			suiteCounter++;
   404 			if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
   405 				/* Matched a suite name */
   406 				suiteFilter = 1;
   407 				suiteFilterName = testSuite->name;
   408 				SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
   409 				break;
   410 			}
   411 
   412 			/* Within each suite, loop over all test cases to check if we have a filter match */
   413 			testCounter = 0;
   414 			while (testSuite->testCases[testCounter] && testFilter == 0)
   415 			{
   416 				testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   417 				testCounter++;
   418 				if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
   419 					/* Matched a test name */
   420 					suiteFilter = 1;
   421 					suiteFilterName = testSuite->name;
   422 					testFilter = 1;
   423 					testFilterName = testCase->name;
   424 					SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);					
   425 					break;
   426 				}
   427 			}						
   428 		}
   429 		
   430 		if (suiteFilter == 0 && testFilter == 0) {
   431 			SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
   432 			SDLTest_Log("Exit code: 2");	
   433 			return 2;
   434 		}		
   435 	}
   436 
   437 	// Loop over all suites
   438 	suiteCounter = 0;
   439 	while(testSuites[suiteCounter]) {
   440 		testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   441 		currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   442 		suiteCounter++;
   443 
   444 		// Filter suite if flag set and we have a name
   445 		if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
   446 			SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
   447 				// Skip suite
   448 				SDLTest_Log("===== Test Suite %i: '%s' skipped\n", 
   449 					suiteCounter, 
   450 					currentSuiteName);
   451 		} else {
   452 
   453 			// Reset per-suite counters
   454 			testFailedCount = 0;
   455 			testPassedCount = 0;
   456 			testSkippedCount = 0;
   457 
   458 			// Take time - suite start
   459 			suiteStartSeconds = GetClock();
   460 
   461 			// Log suite started
   462 			SDLTest_Log("===== Test Suite %i: '%s' started\n", 
   463 				suiteCounter, 
   464 				currentSuiteName);
   465 
   466 			// Loop over all test cases
   467 			testCounter = 0;
   468 			while(testSuite->testCases[testCounter])
   469 			{
   470 				testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   471 				currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   472 				testCounter++;
   473 
   474 				// Filter tests if flag set and we have a name
   475 				if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
   476 					SDL_strcmp(testFilterName, testCase->name) != 0) {
   477 						// Skip test
   478 						SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n", 
   479 							suiteCounter,
   480 							testCounter,
   481 							currentTestName);
   482 				} else {
   483 
   484 					// Take time - test start
   485 					testStartSeconds = GetClock();
   486 
   487 					// Log test started
   488 					SDLTest_Log("----- Test Case %i.%i: '%s' started",
   489 						suiteCounter,
   490 						testCounter, 
   491 						currentTestName);
   492 					if (testCase->description != NULL && strlen(testCase->description)>0) {
   493 						SDLTest_Log("Test Description: '%s'", 
   494 							(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   495 					}
   496 
   497 					// Loop over all iterations
   498 					iterationCounter = 0;
   499 					while(iterationCounter < testIterations)
   500 					{
   501 						iterationCounter++;
   502 
   503 						if (userExecKey != 0) {
   504 							execKey = userExecKey;
   505 						} else {
   506 							execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
   507 						}
   508 
   509 						SDLTest_Log("Test Iteration %i: execKey %llu", iterationCounter, execKey);
   510 						testResult = SDLTest_RunTest(testSuite, testCase, execKey);
   511 
   512 						if (testResult == TEST_RESULT_PASSED) {
   513 							testPassedCount++;
   514 							totalTestPassedCount++;
   515 						} else if (testResult == TEST_RESULT_SKIPPED) {
   516 							testSkippedCount++;
   517 							totalTestSkippedCount++;
   518 						} else {
   519 							testFailedCount++;
   520 							totalTestFailedCount++;
   521 						}
   522 					}
   523 
   524 					// Take time - test end
   525 					testEndSeconds = GetClock();
   526 					runtime = testEndSeconds - testStartSeconds;
   527 					if (runtime < 0.0f) runtime = 0.0f;
   528 
   529 					if (testIterations > 1) {
   530 						// Log test runtime
   531 						SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
   532 						SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
   533 					} else {
   534 						// Log test runtime
   535 						SDLTest_Log("Total Test runtime: %.1f sec", runtime);
   536 					}
   537 
   538 					// Log final test result
   539 					switch (testResult) {
   540 					case TEST_RESULT_PASSED:
   541 						SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   542 						break;
   543 					case TEST_RESULT_FAILED:
   544 						SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
   545 						break;
   546 					case TEST_RESULT_NO_ASSERT:
   547 						SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
   548 						break;
   549 					}
   550 
   551 				}
   552 			}
   553 
   554 			// Take time - suite end
   555 			suiteEndSeconds = GetClock();
   556 			runtime = suiteEndSeconds - suiteStartSeconds;
   557 			if (runtime < 0.0f) runtime = 0.0f;
   558 
   559 			// Log suite runtime
   560 			SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
   561 
   562 			// Log summary and final Suite result
   563 			countSum = testPassedCount + testFailedCount + testSkippedCount;
   564 			if (testFailedCount == 0)
   565 			{
   566 				SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   567 				SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
   568 			} 
   569 			else 
   570 			{
   571 				SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   572 				SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
   573 			}
   574 
   575 		}
   576 	}
   577 
   578 	// Take time - run end
   579 	runEndSeconds = GetClock();
   580 	runtime = runEndSeconds - runStartSeconds;
   581 	if (runtime < 0.0f) runtime = 0.0f;
   582 
   583 	// Log total runtime
   584 	SDLTest_Log("Total Run runtime: %.1f sec", runtime);
   585 
   586 	// Log summary and final run result
   587 	countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   588 	if (totalTestFailedCount == 0)
   589 	{
   590 		runResult = 0;
   591 		SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   592 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
   593 	} 
   594 	else 
   595 	{
   596 		runResult = 1;
   597 		SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   598 		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
   599 	}
   600 
   601 	SDLTest_Log("Exit code: %d", runResult);	
   602 	return runResult;
   603 }