1.1 --- a/src/test/SDL_test_harness.c Sat Dec 01 23:42:17 2012 -0800
1.2 +++ b/src/test/SDL_test_harness.c Tue Dec 04 19:21:10 2012 -0800
1.3 @@ -23,10 +23,19 @@
1.4
1.5 #include "SDL_test.h"
1.6
1.7 -#include <stdio.h>
1.8 +#include <stdio.h>
1.9 +#include <stdlib.h>
1.10 #include <string.h>
1.11 +#include <time.h>
1.12
1.13 -// TODO: port over remaining harness
1.14 +/* Assert check message format */
1.15 +const char *SDLTest_TestCheckFmt = "Test '%s': %s";
1.16 +
1.17 +/* Invalid test name/description message format */
1.18 +const char *SDLTest_InvalidNameFmt = "(Invalid)";
1.19 +
1.20 +/*! \brief Timeout for single test case execution */
1.21 +static Uint32 SDLTest_TestCaseTimeout = 3600;
1.22
1.23 /**
1.24 * Generates a random run seed string for the harness. The generated seed
1.25 @@ -155,7 +164,7 @@
1.26 * \return Timer id or -1 on failure.
1.27 */
1.28 SDL_TimerID
1.29 -SetTestTimeout(int timeout, void (*callback)())
1.30 +SDLTest_SetTestTimeout(int timeout, void (*callback)())
1.31 {
1.32 Uint32 timeoutInMilliseconds;
1.33 SDL_TimerID timerID;
1.34 @@ -188,3 +197,258 @@
1.35
1.36 return timerID;
1.37 }
1.38 +
1.39 +void
1.40 +SDLTest_BailOut()
1.41 +{
1.42 + SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
1.43 + exit(TEST_ABORTED); // bail out from the test
1.44 +}
1.45 +
1.46 +/**
1.47 + * \brief Execute a test using the given execution key.
1.48 + *
1.49 + * \param testSuite Suite containing the test case.
1.50 + * \param testCase Case to execute.
1.51 + * \param execKey Execution key for the fuzzer.
1.52 + *
1.53 + * \returns Test case result.
1.54 + */
1.55 +int
1.56 +SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
1.57 +{
1.58 + SDL_TimerID timer = 0;
1.59 +
1.60 + if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
1.61 + {
1.62 + SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
1.63 + return TEST_RESULT_SETUP_FAILURE;
1.64 + }
1.65 +
1.66 + if (!testCase->enabled)
1.67 + {
1.68 + SDLTest_Log((char *)SDLTest_TestCheckFmt, testCase->name, "Skipped");
1.69 + return TEST_RESULT_SKIPPED;
1.70 + }
1.71 +
1.72 + // Initialize fuzzer
1.73 + SDLTest_FuzzerInit(execKey);
1.74 +
1.75 + // Reset assert tracker
1.76 + SDLTest_ResetAssertSummary();
1.77 +
1.78 + // Set timeout timer
1.79 + timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
1.80 +
1.81 + // Maybe run suite initalizer function
1.82 + if (testSuite->testSetUp) {
1.83 + testSuite->testSetUp(0x0);
1.84 + if (SDLTest_AssertsFailed > 0) {
1.85 + SDLTest_LogError((char *)SDLTest_TestCheckFmt, testSuite->name, "Failed");
1.86 + return TEST_RESULT_SETUP_FAILURE;
1.87 + }
1.88 + }
1.89 +
1.90 + // Run test case function
1.91 + testCase->testCase(0x0);
1.92 +
1.93 + // Maybe run suite cleanup function
1.94 + if (testSuite->testTearDown) {
1.95 + testSuite->testTearDown(0x0);
1.96 + }
1.97 +
1.98 + // Cancel timeout timer
1.99 + if (timer) {
1.100 + SDL_RemoveTimer(timer);
1.101 + }
1.102 +
1.103 + // Report on asserts and fuzzer usage
1.104 + SDLTest_Log("Fuzzer invocations: %d", SDLTest_GetFuzzerInvocationCount());
1.105 + SDLTest_LogAssertSummary();
1.106 +
1.107 + // Analyze assert count to determine test case result
1.108 + if (SDLTest_AssertsFailed > 0) {
1.109 + SDLTest_LogError((char *)SDLTest_TestCheckFmt, testCase->name, "Failed");
1.110 + return TEST_RESULT_FAILED;
1.111 + } else {
1.112 + if (SDLTest_AssertsPassed > 0) {
1.113 + SDLTest_Log((char *)SDLTest_TestCheckFmt, testCase->name, "Passed");
1.114 + return TEST_RESULT_PASSED;
1.115 + } else {
1.116 + SDLTest_LogError((char *)SDLTest_TestCheckFmt, testCase->name, "No Asserts");
1.117 + return TEST_RESULT_NO_ASSERT;
1.118 + }
1.119 + }
1.120 +}
1.121 +
1.122 +/* Prints summary of all suites/tests contained in the given reference */
1.123 +void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
1.124 +{
1.125 + int suiteCounter;
1.126 + int testCounter;
1.127 + SDLTest_TestSuiteReference *testSuite;
1.128 + SDLTest_TestCaseReference *testCase;
1.129 +
1.130 + // Loop over all suites
1.131 + suiteCounter = 0;
1.132 + while(&testSuites[suiteCounter]) {
1.133 + testSuite=&testSuites[suiteCounter];
1.134 + suiteCounter++;
1.135 + SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
1.136 + (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFmt);
1.137 +
1.138 + // Loop over all test cases
1.139 + testCounter = 0;
1.140 + while(testSuite->testCases[testCounter])
1.141 + {
1.142 + testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
1.143 + testCounter++;
1.144 + SDLTest_Log(" Test Case %i - %s: %s", testCounter,
1.145 + (testCase->name) ? testCase->name : SDLTest_InvalidNameFmt,
1.146 + (testCase->description) ? testCase->description : SDLTest_InvalidNameFmt);
1.147 + }
1.148 + }
1.149 +}
1.150 +
1.151 +
1.152 +/**
1.153 + * \brief Execute a test using the given execution key.
1.154 + *
1.155 + * \param testSuites Suites containing the test case.
1.156 + * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
1.157 + * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
1.158 + * \param testIterations Number of iterations to run each test case.
1.159 + *
1.160 + * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
1.161 + */
1.162 +int
1.163 +SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites, char *userRunSeed, Uint64 userExecKey, int testIterations)
1.164 +{
1.165 + int suiteCounter;
1.166 + int testCounter;
1.167 + int iterationCounter;
1.168 + SDLTest_TestSuiteReference *testSuite;
1.169 + SDLTest_TestCaseReference *testCase;
1.170 + char *runSeed;
1.171 + Uint64 execKey;
1.172 + Uint32 runStartTicks;
1.173 + time_t runStartTimestamp;
1.174 + Uint32 suiteStartTicks;
1.175 + time_t suiteStartTimestamp;
1.176 + Uint32 testStartTicks;
1.177 + time_t testStartTimestamp;
1.178 + Uint32 runEndTicks;
1.179 + time_t runEndTimestamp;
1.180 + Uint32 suiteEndTicks;
1.181 + time_t suiteEndTimestamp;
1.182 + Uint32 testEndTicks;
1.183 + time_t testEndTimestamp;
1.184 + int testResult;
1.185 + int totalTestFailedCount, totalTestPassedCount, totalTestSkippedCount;
1.186 + int testFailedCount, testPassedCount, testSkippedCount;
1.187 +
1.188 + // Sanitize test iterations
1.189 + if (testIterations < 1) {
1.190 + testIterations = 1;
1.191 + }
1.192 +
1.193 + // Generate run see if we don't have one already
1.194 + if (userRunSeed == NULL || strlen(userRunSeed) == 0) {
1.195 + runSeed = SDLTest_GenerateRunSeed(16);
1.196 + if (runSeed == NULL) {
1.197 + SDLTest_LogError("Generating a random run seed failed");
1.198 + return 2;
1.199 + }
1.200 + }
1.201 +
1.202 + // Reset per-run counters
1.203 + totalTestFailedCount = totalTestPassedCount = totalTestSkippedCount = 0;
1.204 +
1.205 + // Take time - run start
1.206 + runStartTicks = SDL_GetTicks();
1.207 + runStartTimestamp = time(0);
1.208 +
1.209 + // TODO log run started
1.210 +
1.211 + // Loop over all suites
1.212 + suiteCounter = 0;
1.213 + while(&testSuites[suiteCounter]) {
1.214 + testSuite=&testSuites[suiteCounter];
1.215 + suiteCounter++;
1.216 +
1.217 + // Reset per-suite counters
1.218 + testFailedCount = testPassedCount = testSkippedCount = 0;
1.219 +
1.220 + // Take time - suite start
1.221 + suiteStartTicks = SDL_GetTicks();
1.222 + suiteStartTimestamp = time(0);
1.223 +
1.224 + // TODO log suite started
1.225 + SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
1.226 + (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFmt);
1.227 +
1.228 + // Loop over all test cases
1.229 + testCounter = 0;
1.230 + while(testSuite->testCases[testCounter])
1.231 + {
1.232 + testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
1.233 + testCounter++;
1.234 +
1.235 + // Take time - test start
1.236 + testStartTicks = SDL_GetTicks();
1.237 + testStartTimestamp = time(0);
1.238 +
1.239 + // TODO log test started
1.240 + SDLTest_Log("Test Case %i - %s: %s", testCounter,
1.241 + (testCase->name) ? testCase->name : SDLTest_InvalidNameFmt,
1.242 + (testCase->description) ? testCase->description : SDLTest_InvalidNameFmt);
1.243 +
1.244 + // Loop over all iterations
1.245 + iterationCounter = 0;
1.246 + while(iterationCounter < testIterations)
1.247 + {
1.248 + iterationCounter++;
1.249 +
1.250 + if(userExecKey != 0) {
1.251 + execKey = userExecKey;
1.252 + } else {
1.253 + execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
1.254 + }
1.255 +
1.256 + SDLTest_Log("Test Iteration %i: execKey %d", iterationCounter, execKey);
1.257 + testResult = SDLTest_RunTest(testSuite, testCase, execKey);
1.258 +
1.259 + if (testResult == TEST_RESULT_PASSED) {
1.260 + testPassedCount++;
1.261 + totalTestPassedCount++;
1.262 + } else if (testResult == TEST_RESULT_SKIPPED) {
1.263 + testSkippedCount++;
1.264 + totalTestSkippedCount++;
1.265 + } else {
1.266 + testFailedCount++;
1.267 + totalTestFailedCount++;
1.268 + }
1.269 + }
1.270 +
1.271 + // Take time - test end
1.272 + testEndTicks = SDL_GetTicks();
1.273 + testEndTimestamp = time(0);
1.274 +
1.275 + // TODO log test ended
1.276 + }
1.277 +
1.278 + // Take time - suite end
1.279 + suiteEndTicks = SDL_GetTicks();
1.280 + suiteEndTimestamp = time(0);
1.281 +
1.282 + // TODO log suite ended
1.283 + }
1.284 +
1.285 + // Take time - run end
1.286 + runEndTicks = SDL_GetTicks();
1.287 + runEndTimestamp = time(0);
1.288 +
1.289 + // TODO log run ended
1.290 +
1.291 + return (totalTestFailedCount ? 1 : 0);
1.292 +}