1.1 --- a/include/SDL_test_harness.h	Sat May 18 14:48:19 2013 +0200
     1.2 +++ b/include/SDL_test_harness.h	Sat May 18 09:35:09 2013 -0700
     1.3 @@ -51,8 +51,9 @@
     1.4  
     1.5  //! Definition of all the possible test return values of the test case method
     1.6  #define TEST_ABORTED		-1
     1.7 -#define TEST_COMPLETED		 0
     1.8 -#define TEST_SKIPPED		 1
     1.9 +#define TEST_STARTED		 0
    1.10 +#define TEST_COMPLETED		 1
    1.11 +#define TEST_SKIPPED		 2
    1.12  
    1.13  //! Definition of all the possible test results for the harness
    1.14  #define TEST_RESULT_PASSED				0
    1.15 @@ -65,7 +66,7 @@
    1.16  typedef void (*SDLTest_TestCaseSetUpFp)(void *arg);
    1.17  
    1.18  //!< Function pointer to a test case function
    1.19 -typedef void (*SDLTest_TestCaseFp)(void *arg);
    1.20 +typedef int (*SDLTest_TestCaseFp)(void *arg);
    1.21  
    1.22  //!< Function pointer to a test case teardown function (run after every test)
    1.23  typedef void  (*SDLTest_TestCaseTearDownFp)(void *arg);

     2.1 --- a/src/test/SDL_test_harness.c	Sat May 18 14:48:19 2013 +0200
     2.2 +++ b/src/test/SDL_test_harness.c	Sat May 18 09:35:09 2013 -0700
     2.3 @@ -57,20 +57,20 @@
     2.4  	SDLTest_RandomContext randomContext;
     2.5  	int counter;
     2.6  
     2.7 -	// Sanity check input
     2.8 +	/* Sanity check input */
     2.9  	if (length <= 0) {
    2.10  		SDLTest_LogError("The length of the harness seed must be >0.");
    2.11  		return NULL;
    2.12  	}
    2.13  
    2.14 -	// Allocate output buffer
    2.15 +	/* Allocate output buffer */
    2.16  	seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    2.17  	if (seed == NULL) {
    2.18  		SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    2.19  		return NULL;
    2.20  	}
    2.21  
    2.22 -	// Generate a random string of alphanumeric characters
    2.23 +	/* Generate a random string of alphanumeric characters */
    2.24  	SDLTest_RandomInitTime(&randomContext);
    2.25  	for (counter = 0; counter < length - 1; ++counter) {
    2.26  		unsigned int number = SDLTest_Random(&randomContext);
    2.27 @@ -129,11 +129,11 @@
    2.28  		return -1;
    2.29  	}
    2.30  
    2.31 -	// Convert iteration number into a string
    2.32 +	/* Convert iteration number into a string */
    2.33  	SDL_memset(iterationString, 0, sizeof(iterationString));
    2.34  	SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
    2.35  
    2.36 -	// Combine the parameters into single string
    2.37 +	/* Combine the parameters into single string */
    2.38  	runSeedLength = SDL_strlen(runSeed);
    2.39  	suiteNameLength = SDL_strlen(suiteName);
    2.40  	testNameLength = SDL_strlen(testName);
    2.41 @@ -146,7 +146,7 @@
    2.42  	}
    2.43  	SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
    2.44  
    2.45 -	// Hash string and use half of the digest as 64bit exec key
    2.46 +	/* Hash string and use half of the digest as 64bit exec key */
    2.47  	SDLTest_Md5Init(&md5Context);
    2.48  	SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
    2.49  	SDLTest_Md5Final(&md5Context);
    2.50 @@ -208,7 +208,7 @@
    2.51  	SDLTest_BailOut()
    2.52  {
    2.53  	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
    2.54 -	exit(TEST_ABORTED); // bail out from the test
    2.55 +	exit(TEST_ABORTED); /* bail out from the test */
    2.56  }
    2.57  
    2.58  /**
    2.59 @@ -224,6 +224,7 @@
    2.60  SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
    2.61  {
    2.62  	SDL_TimerID timer = 0;
    2.63 +	int testCaseResult = 0;
    2.64  	int testResult = 0;
    2.65  	int fuzzerCount;
    2.66  
    2.67 @@ -240,16 +241,16 @@
    2.68  	}
    2.69  
    2.70  
    2.71 -	// Initialize fuzzer
    2.72 +	/* Initialize fuzzer */
    2.73  	SDLTest_FuzzerInit(execKey);
    2.74  
    2.75 -	// Reset assert tracker
    2.76 +	/* Reset assert tracker */
    2.77  	SDLTest_ResetAssertSummary();
    2.78  
    2.79 -	// Set timeout timer
    2.80 +	/* Set timeout timer */
    2.81  	timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
    2.82  
    2.83 -	// Maybe run suite initalizer function
    2.84 +	/* Maybe run suite initalizer function */
    2.85  	if (testSuite->testSetUp) {
    2.86  		testSuite->testSetUp(0x0);
    2.87  		if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
    2.88 @@ -258,26 +259,53 @@
    2.89  		}
    2.90  	}
    2.91  
    2.92 -	// Run test case function
    2.93 -	testCase->testCase(0x0);
    2.94 -	testResult = SDLTest_AssertSummaryToTestResult();
    2.95 +	/* Run test case function */
    2.96 +	testCaseResult = testCase->testCase(0x0);
    2.97 +	
    2.98 +	/* Convert test execution result into harness result */
    2.99 +	if (testCaseResult == TEST_SKIPPED) {
   2.100 +		/* Test was programatically skipped */
   2.101 +		testResult = TEST_RESULT_SKIPPED;
   2.102 +	} else if (testCaseResult == TEST_STARTED) {
   2.103 +		/* Test did not return a TEST_COMPLETED value; assume it failed */
   2.104 +		testResult = TEST_RESULT_FAILED;
   2.105 +	} else if (testCaseResult == TEST_ABORTED) {
   2.106 +		/* Test was aborted early; assume it failed */
   2.107 +		testResult = TEST_RESULT_FAILED;
   2.108 +	} else {
   2.109 +		/* Perform failure analysis based on asserts */
   2.110 +		testResult = SDLTest_AssertSummaryToTestResult();
   2.111 +	}
   2.112  
   2.113 -	// Maybe run suite cleanup function (ignore failed asserts)
   2.114 +	/* Maybe run suite cleanup function (ignore failed asserts) */
   2.115  	if (testSuite->testTearDown) {
   2.116  		testSuite->testTearDown(0x0);
   2.117  	}
   2.118  
   2.119 -	// Cancel timeout timer
   2.120 +	/* Cancel timeout timer */
   2.121  	if (timer) {
   2.122  		SDL_RemoveTimer(timer);
   2.123  	}
   2.124  
   2.125 -	// Report on asserts and fuzzer usage
   2.126 +	/* Report on asserts and fuzzer usage */
   2.127  	fuzzerCount = SDLTest_GetFuzzerInvocationCount();
   2.128  	if (fuzzerCount > 0) {
   2.129  		SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
   2.130  	}
   2.131 -	SDLTest_LogAssertSummary();
   2.132 +
   2.133 +	/* Final log based on test execution result */
   2.134 +	if (testCaseResult == TEST_SKIPPED) {
   2.135 +		/* Test was programatically skipped */
   2.136 +		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)");
   2.137 +	} else if (testCaseResult == TEST_STARTED) {
   2.138 +		/* Test did not return a TEST_COMPLETED value; assume it failed */
   2.139 +		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
   2.140 +	} else if (testCaseResult == TEST_ABORTED) {
   2.141 +		/* Test was aborted early; assume it failed */
   2.142 +		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)");
   2.143 +	} else {
   2.144 +		SDLTest_LogAssertSummary();
   2.145 +	}
   2.146  
   2.147  	return testResult;
   2.148  }
   2.149 @@ -290,7 +318,7 @@
   2.150  	SDLTest_TestSuiteReference *testSuite;
   2.151  	SDLTest_TestCaseReference *testCase;
   2.152  
   2.153 -	// Loop over all suites
   2.154 +	/* Loop over all suites */
   2.155  	suiteCounter = 0;
   2.156  	while(&testSuites[suiteCounter]) {
   2.157  		testSuite=&testSuites[suiteCounter];
   2.158 @@ -298,7 +326,7 @@
   2.159  		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   2.160  			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   2.161  
   2.162 -		// Loop over all test cases
   2.163 +		/* Loop over all test cases */
   2.164  		testCounter = 0;
   2.165  		while(testSuite->testCases[testCounter])
   2.166  		{
   2.167 @@ -365,12 +393,12 @@
   2.168  	Uint32 countSum = 0;
   2.169  	char *logFormat = (char *)SDLTest_LogSummaryFormat;
   2.170  
   2.171 -	// Sanitize test iterations
   2.172 +	/* Sanitize test iterations */
   2.173  	if (testIterations < 1) {
   2.174  		testIterations = 1;
   2.175  	}
   2.176  
   2.177 -	// Generate run see if we don't have one already
   2.178 +	/* Generate run see if we don't have one already */
   2.179  	if (userRunSeed == NULL || SDL_strlen(userRunSeed) == 0) {
   2.180  		runSeed = SDLTest_GenerateRunSeed(16);
   2.181  		if (runSeed == NULL) {
   2.182 @@ -382,18 +410,18 @@
   2.183  	}
   2.184  
   2.185  
   2.186 -	// Reset per-run counters
   2.187 +	/* Reset per-run counters */
   2.188  	totalTestFailedCount = 0;
   2.189  	totalTestPassedCount = 0;
   2.190  	totalTestSkippedCount = 0;
   2.191  
   2.192 -	// Take time - run start
   2.193 +	/* Take time - run start */
   2.194  	runStartSeconds = GetClock();
   2.195  
   2.196 -	// Log run with fuzzer parameters
   2.197 +	/* Log run with fuzzer parameters */
   2.198  	SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   2.199  
   2.200 -	// Initialize filtering
   2.201 +	/* Initialize filtering */
   2.202  	if (filter != NULL && SDL_strlen(filter) > 0) {
   2.203  		/* Loop over all suites to check if we have a filter match */
   2.204  		suiteCounter = 0;
   2.205 @@ -433,36 +461,36 @@
   2.206  		}		
   2.207  	}
   2.208  
   2.209 -	// Loop over all suites
   2.210 +	/* Loop over all suites */
   2.211  	suiteCounter = 0;
   2.212  	while(testSuites[suiteCounter]) {
   2.213  		testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   2.214  		currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   2.215  		suiteCounter++;
   2.216  
   2.217 -		// Filter suite if flag set and we have a name
   2.218 +		/* Filter suite if flag set and we have a name */
   2.219  		if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
   2.220  			SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
   2.221 -				// Skip suite
   2.222 +				/* Skip suite */
   2.223  				SDLTest_Log("===== Test Suite %i: '%s' skipped\n", 
   2.224  					suiteCounter, 
   2.225  					currentSuiteName);
   2.226  		} else {
   2.227  
   2.228 -			// Reset per-suite counters
   2.229 +			/* Reset per-suite counters */
   2.230  			testFailedCount = 0;
   2.231  			testPassedCount = 0;
   2.232  			testSkippedCount = 0;
   2.233  
   2.234 -			// Take time - suite start
   2.235 +			/* Take time - suite start */
   2.236  			suiteStartSeconds = GetClock();
   2.237  
   2.238 -			// Log suite started
   2.239 +			/* Log suite started */
   2.240  			SDLTest_Log("===== Test Suite %i: '%s' started\n", 
   2.241  				suiteCounter, 
   2.242  				currentSuiteName);
   2.243  
   2.244 -			// Loop over all test cases
   2.245 +			/* Loop over all test cases */
   2.246  			testCounter = 0;
   2.247  			while(testSuite->testCases[testCounter])
   2.248  			{
   2.249 @@ -470,25 +498,25 @@
   2.250  				currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   2.251  				testCounter++;
   2.252  
   2.253 -				// Filter tests if flag set and we have a name
   2.254 +				/* Filter tests if flag set and we have a name */
   2.255  				if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
   2.256  					SDL_strcmp(testFilterName, testCase->name) != 0) {
   2.257 -						// Skip test
   2.258 +						/* Skip test */
   2.259  						SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n", 
   2.260  							suiteCounter,
   2.261  							testCounter,
   2.262  							currentTestName);
   2.263  				} else {
   2.264 -					// Override 'disabled' flag if we specified a test filter (i.e. force run for debugging)
   2.265 +					/* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
   2.266  					if (testFilter == 1 && !testCase->enabled) {
   2.267  						SDLTest_Log("Force run of disabled test since test filter was set");
   2.268  						testCase->enabled = 1;
   2.269  					}
   2.270  
   2.271 -					// Take time - test start
   2.272 +					/* Take time - test start */
   2.273  					testStartSeconds = GetClock();
   2.274  
   2.275 -					// Log test started
   2.276 +					/* Log test started */
   2.277  					SDLTest_Log("----- Test Case %i.%i: '%s' started",
   2.278  						suiteCounter,
   2.279  						testCounter, 
   2.280 @@ -498,7 +526,7 @@
   2.281  							(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   2.282  					}
   2.283  
   2.284 -					// Loop over all iterations
   2.285 +					/* Loop over all iterations */
   2.286  					iterationCounter = 0;
   2.287  					while(iterationCounter < testIterations)
   2.288  					{
   2.289 @@ -525,21 +553,21 @@
   2.290  						}
   2.291  					}
   2.292  
   2.293 -					// Take time - test end
   2.294 +					/* Take time - test end */
   2.295  					testEndSeconds = GetClock();
   2.296  					runtime = testEndSeconds - testStartSeconds;
   2.297  					if (runtime < 0.0f) runtime = 0.0f;
   2.298  
   2.299  					if (testIterations > 1) {
   2.300 -						// Log test runtime
   2.301 +						/* Log test runtime */
   2.302  						SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
   2.303  						SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
   2.304  					} else {
   2.305 -						// Log test runtime
   2.306 +						/* Log test runtime */
   2.307  						SDLTest_Log("Total Test runtime: %.1f sec", runtime);
   2.308  					}
   2.309  
   2.310 -					// Log final test result
   2.311 +					/* Log final test result */
   2.312  					switch (testResult) {
   2.313  					case TEST_RESULT_PASSED:
   2.314  						SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   2.315 @@ -555,15 +583,15 @@
   2.316  				}
   2.317  			}
   2.318  
   2.319 -			// Take time - suite end
   2.320 +			/* Take time - suite end */
   2.321  			suiteEndSeconds = GetClock();
   2.322  			runtime = suiteEndSeconds - suiteStartSeconds;
   2.323  			if (runtime < 0.0f) runtime = 0.0f;
   2.324  
   2.325 -			// Log suite runtime
   2.326 +			/* Log suite runtime */
   2.327  			SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
   2.328  
   2.329 -			// Log summary and final Suite result
   2.330 +			/* Log summary and final Suite result */
   2.331  			countSum = testPassedCount + testFailedCount + testSkippedCount;
   2.332  			if (testFailedCount == 0)
   2.333  			{
   2.334 @@ -579,15 +607,15 @@
   2.335  		}
   2.336  	}
   2.337  
   2.338 -	// Take time - run end
   2.339 +	/* Take time - run end */
   2.340  	runEndSeconds = GetClock();
   2.341  	runtime = runEndSeconds - runStartSeconds;
   2.342  	if (runtime < 0.0f) runtime = 0.0f;
   2.343  
   2.344 -	// Log total runtime
   2.345 +	/* Log total runtime */
   2.346  	SDLTest_Log("Total Run runtime: %.1f sec", runtime);
   2.347  
   2.348 -	// Log summary and final run result
   2.349 +	/* Log summary and final run result */
   2.350  	countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   2.351  	if (totalTestFailedCount == 0)
   2.352  	{

     3.1 --- a/test/testautomation_main.c	Sat May 18 14:48:19 2013 +0200
     3.2 +++ b/test/testautomation_main.c	Sat May 18 09:35:09 2013 -0700
     3.3 @@ -11,13 +11,16 @@
     3.4  
     3.5  
     3.6  /*!
     3.7 - * \brief Tests SDL_Init() and SDL_Quit()
     3.8 + * \brief Tests SDL_Init() and SDL_Quit() of Joystick and Haptic subsystems
     3.9   * \sa
    3.10   * http://wiki.libsdl.org/moin.cgi/SDL_Init
    3.11   * http://wiki.libsdl.org/moin.cgi/SDL_Quit
    3.12   */
    3.13 -static int main_testInitQuit (void *arg)
    3.14 +static int main_testInitQuitJoystickHaptic (void *arg)
    3.15  {
    3.16 +#if defined SDL_JOYSTICK_DISABLED || defined SDL_HAPTIC_DISABLED
    3.17 +    return TEST_SKIPPED;
    3.18 +#else
    3.19      int enabled_subsystems;
    3.20      int initialized_subsystems = SDL_INIT_JOYSTICK | SDL_INIT_HAPTIC;
    3.21  
    3.22 @@ -32,6 +35,7 @@
    3.23      SDLTest_AssertCheck( enabled_subsystems == 0, "SDL_Quit should shut down everything (%i)", enabled_subsystems );
    3.24  
    3.25      return TEST_COMPLETED;
    3.26 +#endif
    3.27  }
    3.28  
    3.29  /*!
    3.30 @@ -42,6 +46,9 @@
    3.31   */
    3.32  static int main_testInitQuitSubSystem (void *arg)
    3.33  {
    3.34 +#if defined SDL_JOYSTICK_DISABLED || defined SDL_HAPTIC_DISABLED || defined SDL_GAMECONTROLLER_DISABLED
    3.35 +    return TEST_SKIPPED;
    3.36 +#else
    3.37      int i;
    3.38      int subsystems[] = { SDL_INIT_JOYSTICK, SDL_INIT_HAPTIC, SDL_INIT_GAMECONTROLLER };
    3.39  
    3.40 @@ -61,11 +68,15 @@
    3.41      }
    3.42  
    3.43      return TEST_COMPLETED;
    3.44 +#endif
    3.45  }
    3.46  
    3.47  const int joy_and_controller = SDL_INIT_JOYSTICK | SDL_INIT_GAMECONTROLLER;
    3.48  static int main_testImpliedJoystickInit (void *arg)
    3.49  {
    3.50 +#if defined SDL_JOYSTICK_DISABLED || defined SDL_GAMECONTROLLER_DISABLED
    3.51 +    return TEST_SKIPPED;
    3.52 +#else
    3.53      int initialized_system;
    3.54  
    3.55      // First initialize the controller
    3.56 @@ -82,11 +93,15 @@
    3.57      initialized_system = SDL_WasInit(joy_and_controller);
    3.58      SDLTest_AssertCheck( (initialized_system & joy_and_controller) == 0, "SDL_WasInit() should be false for joystick & controller (%x)", initialized_system );
    3.59  
    3.60 -	return TEST_COMPLETED;
    3.61 +    return TEST_COMPLETED;
    3.62 +#endif
    3.63  }
    3.64  
    3.65  static int main_testImpliedJoystickQuit (void *arg)
    3.66  {
    3.67 +#if defined SDL_JOYSTICK_DISABLED || defined SDL_GAMECONTROLLER_DISABLED
    3.68 +    return TEST_SKIPPED;
    3.69 +#else
    3.70      int initialized_system;
    3.71  
    3.72      // First initialize the controller and the joystick (explicitly)
    3.73 @@ -106,11 +121,12 @@
    3.74  
    3.75      SDL_QuitSubSystem(SDL_INIT_JOYSTICK);
    3.76  
    3.77 -	return TEST_COMPLETED;
    3.78 +    return TEST_COMPLETED;
    3.79 +#endif
    3.80  }
    3.81  
    3.82  static const SDLTest_TestCaseReference mainTest1 =
    3.83 -		{ (SDLTest_TestCaseFp)main_testInitQuit, "main_testInitQuit", "Tests SDL_Init/Quit", TEST_ENABLED};
    3.84 +		{ (SDLTest_TestCaseFp)main_testInitQuitJoystickHaptic, "main_testInitQuitJoystickHaptic", "Tests SDL_Init/Quit of Joystick and Haptic subsystem", TEST_ENABLED};
    3.85  
    3.86  static const SDLTest_TestCaseReference mainTest2 =
    3.87  		{ (SDLTest_TestCaseFp)main_testInitQuitSubSystem, "main_testInitQuitSubSystem", "Tests SDL_InitSubSystem/QuitSubSystem", TEST_ENABLED};