Update test harness to handle test return codes; fix comment format in harness; update Main test suite to handle globally disabled features
1.1 --- a/include/SDL_test_harness.h Sat May 18 14:48:19 2013 +0200
1.2 +++ b/include/SDL_test_harness.h Sat May 18 09:35:09 2013 -0700
1.3 @@ -51,8 +51,9 @@
1.4
1.5 //! Definition of all the possible test return values of the test case method
1.6 #define TEST_ABORTED -1
1.7 -#define TEST_COMPLETED 0
1.8 -#define TEST_SKIPPED 1
1.9 +#define TEST_STARTED 0
1.10 +#define TEST_COMPLETED 1
1.11 +#define TEST_SKIPPED 2
1.12
1.13 //! Definition of all the possible test results for the harness
1.14 #define TEST_RESULT_PASSED 0
1.15 @@ -65,7 +66,7 @@
1.16 typedef void (*SDLTest_TestCaseSetUpFp)(void *arg);
1.17
1.18 //!< Function pointer to a test case function
1.19 -typedef void (*SDLTest_TestCaseFp)(void *arg);
1.20 +typedef int (*SDLTest_TestCaseFp)(void *arg);
1.21
1.22 //!< Function pointer to a test case teardown function (run after every test)
1.23 typedef void (*SDLTest_TestCaseTearDownFp)(void *arg);
2.1 --- a/src/test/SDL_test_harness.c Sat May 18 14:48:19 2013 +0200
2.2 +++ b/src/test/SDL_test_harness.c Sat May 18 09:35:09 2013 -0700
2.3 @@ -57,20 +57,20 @@
2.4 SDLTest_RandomContext randomContext;
2.5 int counter;
2.6
2.7 - // Sanity check input
2.8 + /* Sanity check input */
2.9 if (length <= 0) {
2.10 SDLTest_LogError("The length of the harness seed must be >0.");
2.11 return NULL;
2.12 }
2.13
2.14 - // Allocate output buffer
2.15 + /* Allocate output buffer */
2.16 seed = (char *)SDL_malloc((length + 1) * sizeof(char));
2.17 if (seed == NULL) {
2.18 SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
2.19 return NULL;
2.20 }
2.21
2.22 - // Generate a random string of alphanumeric characters
2.23 + /* Generate a random string of alphanumeric characters */
2.24 SDLTest_RandomInitTime(&randomContext);
2.25 for (counter = 0; counter < length - 1; ++counter) {
2.26 unsigned int number = SDLTest_Random(&randomContext);
2.27 @@ -129,11 +129,11 @@
2.28 return -1;
2.29 }
2.30
2.31 - // Convert iteration number into a string
2.32 + /* Convert iteration number into a string */
2.33 SDL_memset(iterationString, 0, sizeof(iterationString));
2.34 SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
2.35
2.36 - // Combine the parameters into single string
2.37 + /* Combine the parameters into single string */
2.38 runSeedLength = SDL_strlen(runSeed);
2.39 suiteNameLength = SDL_strlen(suiteName);
2.40 testNameLength = SDL_strlen(testName);
2.41 @@ -146,7 +146,7 @@
2.42 }
2.43 SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
2.44
2.45 - // Hash string and use half of the digest as 64bit exec key
2.46 + /* Hash string and use half of the digest as 64bit exec key */
2.47 SDLTest_Md5Init(&md5Context);
2.48 SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
2.49 SDLTest_Md5Final(&md5Context);
2.50 @@ -208,7 +208,7 @@
2.51 SDLTest_BailOut()
2.52 {
2.53 SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
2.54 - exit(TEST_ABORTED); // bail out from the test
2.55 + exit(TEST_ABORTED); /* bail out from the test */
2.56 }
2.57
2.58 /**
2.59 @@ -224,6 +224,7 @@
2.60 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
2.61 {
2.62 SDL_TimerID timer = 0;
2.63 + int testCaseResult = 0;
2.64 int testResult = 0;
2.65 int fuzzerCount;
2.66
2.67 @@ -240,16 +241,16 @@
2.68 }
2.69
2.70
2.71 - // Initialize fuzzer
2.72 + /* Initialize fuzzer */
2.73 SDLTest_FuzzerInit(execKey);
2.74
2.75 - // Reset assert tracker
2.76 + /* Reset assert tracker */
2.77 SDLTest_ResetAssertSummary();
2.78
2.79 - // Set timeout timer
2.80 + /* Set timeout timer */
2.81 timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
2.82
2.83 - // Maybe run suite initalizer function
2.84 + /* Maybe run suite initalizer function */
2.85 if (testSuite->testSetUp) {
2.86 testSuite->testSetUp(0x0);
2.87 if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
2.88 @@ -258,26 +259,53 @@
2.89 }
2.90 }
2.91
2.92 - // Run test case function
2.93 - testCase->testCase(0x0);
2.94 - testResult = SDLTest_AssertSummaryToTestResult();
2.95 + /* Run test case function */
2.96 + testCaseResult = testCase->testCase(0x0);
2.97 +
2.98 + /* Convert test execution result into harness result */
2.99 + if (testCaseResult == TEST_SKIPPED) {
2.100 + /* Test was programatically skipped */
2.101 + testResult = TEST_RESULT_SKIPPED;
2.102 + } else if (testCaseResult == TEST_STARTED) {
2.103 + /* Test did not return a TEST_COMPLETED value; assume it failed */
2.104 + testResult = TEST_RESULT_FAILED;
2.105 + } else if (testCaseResult == TEST_ABORTED) {
2.106 + /* Test was aborted early; assume it failed */
2.107 + testResult = TEST_RESULT_FAILED;
2.108 + } else {
2.109 + /* Perform failure analysis based on asserts */
2.110 + testResult = SDLTest_AssertSummaryToTestResult();
2.111 + }
2.112
2.113 - // Maybe run suite cleanup function (ignore failed asserts)
2.114 + /* Maybe run suite cleanup function (ignore failed asserts) */
2.115 if (testSuite->testTearDown) {
2.116 testSuite->testTearDown(0x0);
2.117 }
2.118
2.119 - // Cancel timeout timer
2.120 + /* Cancel timeout timer */
2.121 if (timer) {
2.122 SDL_RemoveTimer(timer);
2.123 }
2.124
2.125 - // Report on asserts and fuzzer usage
2.126 + /* Report on asserts and fuzzer usage */
2.127 fuzzerCount = SDLTest_GetFuzzerInvocationCount();
2.128 if (fuzzerCount > 0) {
2.129 SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
2.130 }
2.131 - SDLTest_LogAssertSummary();
2.132 +
2.133 + /* Final log based on test execution result */
2.134 + if (testCaseResult == TEST_SKIPPED) {
2.135 + /* Test was programatically skipped */
2.136 + SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)");
2.137 + } else if (testCaseResult == TEST_STARTED) {
2.138 + /* Test did not return a TEST_COMPLETED value; assume it failed */
2.139 + SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
2.140 + } else if (testCaseResult == TEST_ABORTED) {
2.141 + /* Test was aborted early; assume it failed */
2.142 + SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)");
2.143 + } else {
2.144 + SDLTest_LogAssertSummary();
2.145 + }
2.146
2.147 return testResult;
2.148 }
2.149 @@ -290,7 +318,7 @@
2.150 SDLTest_TestSuiteReference *testSuite;
2.151 SDLTest_TestCaseReference *testCase;
2.152
2.153 - // Loop over all suites
2.154 + /* Loop over all suites */
2.155 suiteCounter = 0;
2.156 while(&testSuites[suiteCounter]) {
2.157 testSuite=&testSuites[suiteCounter];
2.158 @@ -298,7 +326,7 @@
2.159 SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
2.160 (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
2.161
2.162 - // Loop over all test cases
2.163 + /* Loop over all test cases */
2.164 testCounter = 0;
2.165 while(testSuite->testCases[testCounter])
2.166 {
2.167 @@ -365,12 +393,12 @@
2.168 Uint32 countSum = 0;
2.169 char *logFormat = (char *)SDLTest_LogSummaryFormat;
2.170
2.171 - // Sanitize test iterations
2.172 + /* Sanitize test iterations */
2.173 if (testIterations < 1) {
2.174 testIterations = 1;
2.175 }
2.176
2.177 - // Generate run see if we don't have one already
2.178 + /* Generate run see if we don't have one already */
2.179 if (userRunSeed == NULL || SDL_strlen(userRunSeed) == 0) {
2.180 runSeed = SDLTest_GenerateRunSeed(16);
2.181 if (runSeed == NULL) {
2.182 @@ -382,18 +410,18 @@
2.183 }
2.184
2.185
2.186 - // Reset per-run counters
2.187 + /* Reset per-run counters */
2.188 totalTestFailedCount = 0;
2.189 totalTestPassedCount = 0;
2.190 totalTestSkippedCount = 0;
2.191
2.192 - // Take time - run start
2.193 + /* Take time - run start */
2.194 runStartSeconds = GetClock();
2.195
2.196 - // Log run with fuzzer parameters
2.197 + /* Log run with fuzzer parameters */
2.198 SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
2.199
2.200 - // Initialize filtering
2.201 + /* Initialize filtering */
2.202 if (filter != NULL && SDL_strlen(filter) > 0) {
2.203 /* Loop over all suites to check if we have a filter match */
2.204 suiteCounter = 0;
2.205 @@ -433,36 +461,36 @@
2.206 }
2.207 }
2.208
2.209 - // Loop over all suites
2.210 + /* Loop over all suites */
2.211 suiteCounter = 0;
2.212 while(testSuites[suiteCounter]) {
2.213 testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
2.214 currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
2.215 suiteCounter++;
2.216
2.217 - // Filter suite if flag set and we have a name
2.218 + /* Filter suite if flag set and we have a name */
2.219 if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
2.220 SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
2.221 - // Skip suite
2.222 + /* Skip suite */
2.223 SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
2.224 suiteCounter,
2.225 currentSuiteName);
2.226 } else {
2.227
2.228 - // Reset per-suite counters
2.229 + /* Reset per-suite counters */
2.230 testFailedCount = 0;
2.231 testPassedCount = 0;
2.232 testSkippedCount = 0;
2.233
2.234 - // Take time - suite start
2.235 + /* Take time - suite start */
2.236 suiteStartSeconds = GetClock();
2.237
2.238 - // Log suite started
2.239 + /* Log suite started */
2.240 SDLTest_Log("===== Test Suite %i: '%s' started\n",
2.241 suiteCounter,
2.242 currentSuiteName);
2.243
2.244 - // Loop over all test cases
2.245 + /* Loop over all test cases */
2.246 testCounter = 0;
2.247 while(testSuite->testCases[testCounter])
2.248 {
2.249 @@ -470,25 +498,25 @@
2.250 currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
2.251 testCounter++;
2.252
2.253 - // Filter tests if flag set and we have a name
2.254 + /* Filter tests if flag set and we have a name */
2.255 if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
2.256 SDL_strcmp(testFilterName, testCase->name) != 0) {
2.257 - // Skip test
2.258 + /* Skip test */
2.259 SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
2.260 suiteCounter,
2.261 testCounter,
2.262 currentTestName);
2.263 } else {
2.264 - // Override 'disabled' flag if we specified a test filter (i.e. force run for debugging)
2.265 + /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
2.266 if (testFilter == 1 && !testCase->enabled) {
2.267 SDLTest_Log("Force run of disabled test since test filter was set");
2.268 testCase->enabled = 1;
2.269 }
2.270
2.271 - // Take time - test start
2.272 + /* Take time - test start */
2.273 testStartSeconds = GetClock();
2.274
2.275 - // Log test started
2.276 + /* Log test started */
2.277 SDLTest_Log("----- Test Case %i.%i: '%s' started",
2.278 suiteCounter,
2.279 testCounter,
2.280 @@ -498,7 +526,7 @@
2.281 (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
2.282 }
2.283
2.284 - // Loop over all iterations
2.285 + /* Loop over all iterations */
2.286 iterationCounter = 0;
2.287 while(iterationCounter < testIterations)
2.288 {
2.289 @@ -525,21 +553,21 @@
2.290 }
2.291 }
2.292
2.293 - // Take time - test end
2.294 + /* Take time - test end */
2.295 testEndSeconds = GetClock();
2.296 runtime = testEndSeconds - testStartSeconds;
2.297 if (runtime < 0.0f) runtime = 0.0f;
2.298
2.299 if (testIterations > 1) {
2.300 - // Log test runtime
2.301 + /* Log test runtime */
2.302 SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
2.303 SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
2.304 } else {
2.305 - // Log test runtime
2.306 + /* Log test runtime */
2.307 SDLTest_Log("Total Test runtime: %.1f sec", runtime);
2.308 }
2.309
2.310 - // Log final test result
2.311 + /* Log final test result */
2.312 switch (testResult) {
2.313 case TEST_RESULT_PASSED:
2.314 SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
2.315 @@ -555,15 +583,15 @@
2.316 }
2.317 }
2.318
2.319 - // Take time - suite end
2.320 + /* Take time - suite end */
2.321 suiteEndSeconds = GetClock();
2.322 runtime = suiteEndSeconds - suiteStartSeconds;
2.323 if (runtime < 0.0f) runtime = 0.0f;
2.324
2.325 - // Log suite runtime
2.326 + /* Log suite runtime */
2.327 SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
2.328
2.329 - // Log summary and final Suite result
2.330 + /* Log summary and final Suite result */
2.331 countSum = testPassedCount + testFailedCount + testSkippedCount;
2.332 if (testFailedCount == 0)
2.333 {
2.334 @@ -579,15 +607,15 @@
2.335 }
2.336 }
2.337
2.338 - // Take time - run end
2.339 + /* Take time - run end */
2.340 runEndSeconds = GetClock();
2.341 runtime = runEndSeconds - runStartSeconds;
2.342 if (runtime < 0.0f) runtime = 0.0f;
2.343
2.344 - // Log total runtime
2.345 + /* Log total runtime */
2.346 SDLTest_Log("Total Run runtime: %.1f sec", runtime);
2.347
2.348 - // Log summary and final run result
2.349 + /* Log summary and final run result */
2.350 countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
2.351 if (totalTestFailedCount == 0)
2.352 {
3.1 --- a/test/testautomation_main.c Sat May 18 14:48:19 2013 +0200
3.2 +++ b/test/testautomation_main.c Sat May 18 09:35:09 2013 -0700
3.3 @@ -11,13 +11,16 @@
3.4
3.5
3.6 /*!
3.7 - * \brief Tests SDL_Init() and SDL_Quit()
3.8 + * \brief Tests SDL_Init() and SDL_Quit() of Joystick and Haptic subsystems
3.9 * \sa
3.10 * http://wiki.libsdl.org/moin.cgi/SDL_Init
3.11 * http://wiki.libsdl.org/moin.cgi/SDL_Quit
3.12 */
3.13 -static int main_testInitQuit (void *arg)
3.14 +static int main_testInitQuitJoystickHaptic (void *arg)
3.15 {
3.16 +#if defined SDL_JOYSTICK_DISABLED || defined SDL_HAPTIC_DISABLED
3.17 + return TEST_SKIPPED;
3.18 +#else
3.19 int enabled_subsystems;
3.20 int initialized_subsystems = SDL_INIT_JOYSTICK | SDL_INIT_HAPTIC;
3.21
3.22 @@ -32,6 +35,7 @@
3.23 SDLTest_AssertCheck( enabled_subsystems == 0, "SDL_Quit should shut down everything (%i)", enabled_subsystems );
3.24
3.25 return TEST_COMPLETED;
3.26 +#endif
3.27 }
3.28
3.29 /*!
3.30 @@ -42,6 +46,9 @@
3.31 */
3.32 static int main_testInitQuitSubSystem (void *arg)
3.33 {
3.34 +#if defined SDL_JOYSTICK_DISABLED || defined SDL_HAPTIC_DISABLED || defined SDL_GAMECONTROLLER_DISABLED
3.35 + return TEST_SKIPPED;
3.36 +#else
3.37 int i;
3.38 int subsystems[] = { SDL_INIT_JOYSTICK, SDL_INIT_HAPTIC, SDL_INIT_GAMECONTROLLER };
3.39
3.40 @@ -61,11 +68,15 @@
3.41 }
3.42
3.43 return TEST_COMPLETED;
3.44 +#endif
3.45 }
3.46
3.47 const int joy_and_controller = SDL_INIT_JOYSTICK | SDL_INIT_GAMECONTROLLER;
3.48 static int main_testImpliedJoystickInit (void *arg)
3.49 {
3.50 +#if defined SDL_JOYSTICK_DISABLED || defined SDL_GAMECONTROLLER_DISABLED
3.51 + return TEST_SKIPPED;
3.52 +#else
3.53 int initialized_system;
3.54
3.55 // First initialize the controller
3.56 @@ -82,11 +93,15 @@
3.57 initialized_system = SDL_WasInit(joy_and_controller);
3.58 SDLTest_AssertCheck( (initialized_system & joy_and_controller) == 0, "SDL_WasInit() should be false for joystick & controller (%x)", initialized_system );
3.59
3.60 - return TEST_COMPLETED;
3.61 + return TEST_COMPLETED;
3.62 +#endif
3.63 }
3.64
3.65 static int main_testImpliedJoystickQuit (void *arg)
3.66 {
3.67 +#if defined SDL_JOYSTICK_DISABLED || defined SDL_GAMECONTROLLER_DISABLED
3.68 + return TEST_SKIPPED;
3.69 +#else
3.70 int initialized_system;
3.71
3.72 // First initialize the controller and the joystick (explicitly)
3.73 @@ -106,11 +121,12 @@
3.74
3.75 SDL_QuitSubSystem(SDL_INIT_JOYSTICK);
3.76
3.77 - return TEST_COMPLETED;
3.78 + return TEST_COMPLETED;
3.79 +#endif
3.80 }
3.81
3.82 static const SDLTest_TestCaseReference mainTest1 =
3.83 - { (SDLTest_TestCaseFp)main_testInitQuit, "main_testInitQuit", "Tests SDL_Init/Quit", TEST_ENABLED};
3.84 + { (SDLTest_TestCaseFp)main_testInitQuitJoystickHaptic, "main_testInitQuitJoystickHaptic", "Tests SDL_Init/Quit of Joystick and Haptic subsystem", TEST_ENABLED};
3.85
3.86 static const SDLTest_TestCaseReference mainTest2 =
3.87 { (SDLTest_TestCaseFp)main_testInitQuitSubSystem, "main_testInitQuitSubSystem", "Tests SDL_InitSubSystem/QuitSubSystem", TEST_ENABLED};