1.1 --- a/src/test/SDL_test_harness.c Sat May 18 14:48:19 2013 +0200
1.2 +++ b/src/test/SDL_test_harness.c Sat May 18 09:35:09 2013 -0700
1.3 @@ -57,20 +57,20 @@
1.4 SDLTest_RandomContext randomContext;
1.5 int counter;
1.6
1.7 - // Sanity check input
1.8 + /* Sanity check input */
1.9 if (length <= 0) {
1.10 SDLTest_LogError("The length of the harness seed must be >0.");
1.11 return NULL;
1.12 }
1.13
1.14 - // Allocate output buffer
1.15 + /* Allocate output buffer */
1.16 seed = (char *)SDL_malloc((length + 1) * sizeof(char));
1.17 if (seed == NULL) {
1.18 SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
1.19 return NULL;
1.20 }
1.21
1.22 - // Generate a random string of alphanumeric characters
1.23 + /* Generate a random string of alphanumeric characters */
1.24 SDLTest_RandomInitTime(&randomContext);
1.25 for (counter = 0; counter < length - 1; ++counter) {
1.26 unsigned int number = SDLTest_Random(&randomContext);
1.27 @@ -129,11 +129,11 @@
1.28 return -1;
1.29 }
1.30
1.31 - // Convert iteration number into a string
1.32 + /* Convert iteration number into a string */
1.33 SDL_memset(iterationString, 0, sizeof(iterationString));
1.34 SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
1.35
1.36 - // Combine the parameters into single string
1.37 + /* Combine the parameters into single string */
1.38 runSeedLength = SDL_strlen(runSeed);
1.39 suiteNameLength = SDL_strlen(suiteName);
1.40 testNameLength = SDL_strlen(testName);
1.41 @@ -146,7 +146,7 @@
1.42 }
1.43 SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
1.44
1.45 - // Hash string and use half of the digest as 64bit exec key
1.46 + /* Hash string and use half of the digest as 64bit exec key */
1.47 SDLTest_Md5Init(&md5Context);
1.48 SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
1.49 SDLTest_Md5Final(&md5Context);
1.50 @@ -208,7 +208,7 @@
1.51 SDLTest_BailOut()
1.52 {
1.53 SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
1.54 - exit(TEST_ABORTED); // bail out from the test
1.55 + exit(TEST_ABORTED); /* bail out from the test */
1.56 }
1.57
1.58 /**
1.59 @@ -224,6 +224,7 @@
1.60 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
1.61 {
1.62 SDL_TimerID timer = 0;
1.63 + int testCaseResult = 0;
1.64 int testResult = 0;
1.65 int fuzzerCount;
1.66
1.67 @@ -240,16 +241,16 @@
1.68 }
1.69
1.70
1.71 - // Initialize fuzzer
1.72 + /* Initialize fuzzer */
1.73 SDLTest_FuzzerInit(execKey);
1.74
1.75 - // Reset assert tracker
1.76 + /* Reset assert tracker */
1.77 SDLTest_ResetAssertSummary();
1.78
1.79 - // Set timeout timer
1.80 + /* Set timeout timer */
1.81 timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
1.82
1.83 - // Maybe run suite initalizer function
1.84 + /* Maybe run suite initalizer function */
1.85 if (testSuite->testSetUp) {
1.86 testSuite->testSetUp(0x0);
1.87 if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
1.88 @@ -258,26 +259,53 @@
1.89 }
1.90 }
1.91
1.92 - // Run test case function
1.93 - testCase->testCase(0x0);
1.94 - testResult = SDLTest_AssertSummaryToTestResult();
1.95 + /* Run test case function */
1.96 + testCaseResult = testCase->testCase(0x0);
1.97 +
1.98 + /* Convert test execution result into harness result */
1.99 + if (testCaseResult == TEST_SKIPPED) {
1.100 + /* Test was programatically skipped */
1.101 + testResult = TEST_RESULT_SKIPPED;
1.102 + } else if (testCaseResult == TEST_STARTED) {
1.103 + /* Test did not return a TEST_COMPLETED value; assume it failed */
1.104 + testResult = TEST_RESULT_FAILED;
1.105 + } else if (testCaseResult == TEST_ABORTED) {
1.106 + /* Test was aborted early; assume it failed */
1.107 + testResult = TEST_RESULT_FAILED;
1.108 + } else {
1.109 + /* Perform failure analysis based on asserts */
1.110 + testResult = SDLTest_AssertSummaryToTestResult();
1.111 + }
1.112
1.113 - // Maybe run suite cleanup function (ignore failed asserts)
1.114 + /* Maybe run suite cleanup function (ignore failed asserts) */
1.115 if (testSuite->testTearDown) {
1.116 testSuite->testTearDown(0x0);
1.117 }
1.118
1.119 - // Cancel timeout timer
1.120 + /* Cancel timeout timer */
1.121 if (timer) {
1.122 SDL_RemoveTimer(timer);
1.123 }
1.124
1.125 - // Report on asserts and fuzzer usage
1.126 + /* Report on asserts and fuzzer usage */
1.127 fuzzerCount = SDLTest_GetFuzzerInvocationCount();
1.128 if (fuzzerCount > 0) {
1.129 SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
1.130 }
1.131 - SDLTest_LogAssertSummary();
1.132 +
1.133 + /* Final log based on test execution result */
1.134 + if (testCaseResult == TEST_SKIPPED) {
1.135 + /* Test was programatically skipped */
1.136 + SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)");
1.137 + } else if (testCaseResult == TEST_STARTED) {
1.138 + /* Test did not return a TEST_COMPLETED value; assume it failed */
1.139 + SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
1.140 + } else if (testCaseResult == TEST_ABORTED) {
1.141 + /* Test was aborted early; assume it failed */
1.142 + SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)");
1.143 + } else {
1.144 + SDLTest_LogAssertSummary();
1.145 + }
1.146
1.147 return testResult;
1.148 }
1.149 @@ -290,7 +318,7 @@
1.150 SDLTest_TestSuiteReference *testSuite;
1.151 SDLTest_TestCaseReference *testCase;
1.152
1.153 - // Loop over all suites
1.154 + /* Loop over all suites */
1.155 suiteCounter = 0;
1.156 while(&testSuites[suiteCounter]) {
1.157 testSuite=&testSuites[suiteCounter];
1.158 @@ -298,7 +326,7 @@
1.159 SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
1.160 (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
1.161
1.162 - // Loop over all test cases
1.163 + /* Loop over all test cases */
1.164 testCounter = 0;
1.165 while(testSuite->testCases[testCounter])
1.166 {
1.167 @@ -365,12 +393,12 @@
1.168 Uint32 countSum = 0;
1.169 char *logFormat = (char *)SDLTest_LogSummaryFormat;
1.170
1.171 - // Sanitize test iterations
1.172 + /* Sanitize test iterations */
1.173 if (testIterations < 1) {
1.174 testIterations = 1;
1.175 }
1.176
1.177 - // Generate run see if we don't have one already
1.178 + /* Generate run see if we don't have one already */
1.179 if (userRunSeed == NULL || SDL_strlen(userRunSeed) == 0) {
1.180 runSeed = SDLTest_GenerateRunSeed(16);
1.181 if (runSeed == NULL) {
1.182 @@ -382,18 +410,18 @@
1.183 }
1.184
1.185
1.186 - // Reset per-run counters
1.187 + /* Reset per-run counters */
1.188 totalTestFailedCount = 0;
1.189 totalTestPassedCount = 0;
1.190 totalTestSkippedCount = 0;
1.191
1.192 - // Take time - run start
1.193 + /* Take time - run start */
1.194 runStartSeconds = GetClock();
1.195
1.196 - // Log run with fuzzer parameters
1.197 + /* Log run with fuzzer parameters */
1.198 SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
1.199
1.200 - // Initialize filtering
1.201 + /* Initialize filtering */
1.202 if (filter != NULL && SDL_strlen(filter) > 0) {
1.203 /* Loop over all suites to check if we have a filter match */
1.204 suiteCounter = 0;
1.205 @@ -433,36 +461,36 @@
1.206 }
1.207 }
1.208
1.209 - // Loop over all suites
1.210 + /* Loop over all suites */
1.211 suiteCounter = 0;
1.212 while(testSuites[suiteCounter]) {
1.213 testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
1.214 currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
1.215 suiteCounter++;
1.216
1.217 - // Filter suite if flag set and we have a name
1.218 + /* Filter suite if flag set and we have a name */
1.219 if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
1.220 SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
1.221 - // Skip suite
1.222 + /* Skip suite */
1.223 SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
1.224 suiteCounter,
1.225 currentSuiteName);
1.226 } else {
1.227
1.228 - // Reset per-suite counters
1.229 + /* Reset per-suite counters */
1.230 testFailedCount = 0;
1.231 testPassedCount = 0;
1.232 testSkippedCount = 0;
1.233
1.234 - // Take time - suite start
1.235 + /* Take time - suite start */
1.236 suiteStartSeconds = GetClock();
1.237
1.238 - // Log suite started
1.239 + /* Log suite started */
1.240 SDLTest_Log("===== Test Suite %i: '%s' started\n",
1.241 suiteCounter,
1.242 currentSuiteName);
1.243
1.244 - // Loop over all test cases
1.245 + /* Loop over all test cases */
1.246 testCounter = 0;
1.247 while(testSuite->testCases[testCounter])
1.248 {
1.249 @@ -470,25 +498,25 @@
1.250 currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
1.251 testCounter++;
1.252
1.253 - // Filter tests if flag set and we have a name
1.254 + /* Filter tests if flag set and we have a name */
1.255 if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
1.256 SDL_strcmp(testFilterName, testCase->name) != 0) {
1.257 - // Skip test
1.258 + /* Skip test */
1.259 SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
1.260 suiteCounter,
1.261 testCounter,
1.262 currentTestName);
1.263 } else {
1.264 - // Override 'disabled' flag if we specified a test filter (i.e. force run for debugging)
1.265 + /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
1.266 if (testFilter == 1 && !testCase->enabled) {
1.267 SDLTest_Log("Force run of disabled test since test filter was set");
1.268 testCase->enabled = 1;
1.269 }
1.270
1.271 - // Take time - test start
1.272 + /* Take time - test start */
1.273 testStartSeconds = GetClock();
1.274
1.275 - // Log test started
1.276 + /* Log test started */
1.277 SDLTest_Log("----- Test Case %i.%i: '%s' started",
1.278 suiteCounter,
1.279 testCounter,
1.280 @@ -498,7 +526,7 @@
1.281 (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
1.282 }
1.283
1.284 - // Loop over all iterations
1.285 + /* Loop over all iterations */
1.286 iterationCounter = 0;
1.287 while(iterationCounter < testIterations)
1.288 {
1.289 @@ -525,21 +553,21 @@
1.290 }
1.291 }
1.292
1.293 - // Take time - test end
1.294 + /* Take time - test end */
1.295 testEndSeconds = GetClock();
1.296 runtime = testEndSeconds - testStartSeconds;
1.297 if (runtime < 0.0f) runtime = 0.0f;
1.298
1.299 if (testIterations > 1) {
1.300 - // Log test runtime
1.301 + /* Log test runtime */
1.302 SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
1.303 SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
1.304 } else {
1.305 - // Log test runtime
1.306 + /* Log test runtime */
1.307 SDLTest_Log("Total Test runtime: %.1f sec", runtime);
1.308 }
1.309
1.310 - // Log final test result
1.311 + /* Log final test result */
1.312 switch (testResult) {
1.313 case TEST_RESULT_PASSED:
1.314 SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
1.315 @@ -555,15 +583,15 @@
1.316 }
1.317 }
1.318
1.319 - // Take time - suite end
1.320 + /* Take time - suite end */
1.321 suiteEndSeconds = GetClock();
1.322 runtime = suiteEndSeconds - suiteStartSeconds;
1.323 if (runtime < 0.0f) runtime = 0.0f;
1.324
1.325 - // Log suite runtime
1.326 + /* Log suite runtime */
1.327 SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
1.328
1.329 - // Log summary and final Suite result
1.330 + /* Log summary and final Suite result */
1.331 countSum = testPassedCount + testFailedCount + testSkippedCount;
1.332 if (testFailedCount == 0)
1.333 {
1.334 @@ -579,15 +607,15 @@
1.335 }
1.336 }
1.337
1.338 - // Take time - run end
1.339 + /* Take time - run end */
1.340 runEndSeconds = GetClock();
1.341 runtime = runEndSeconds - runStartSeconds;
1.342 if (runtime < 0.0f) runtime = 0.0f;
1.343
1.344 - // Log total runtime
1.345 + /* Log total runtime */
1.346 SDLTest_Log("Total Run runtime: %.1f sec", runtime);
1.347
1.348 - // Log summary and final run result
1.349 + /* Log summary and final run result */
1.350 countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
1.351 if (totalTestFailedCount == 0)
1.352 {