src/test/SDL_test_harness.c
changeset 6721 53b71f45a53a
parent 6718 918ba414168b
child 6727 1b5280cd5885
     1.1 --- a/src/test/SDL_test_harness.c	Sat Dec 01 23:42:17 2012 -0800
     1.2 +++ b/src/test/SDL_test_harness.c	Tue Dec 04 19:21:10 2012 -0800
     1.3 @@ -23,10 +23,19 @@
     1.4  
     1.5  #include "SDL_test.h"
     1.6  
     1.7 -#include <stdio.h>
     1.8 +#include <stdio.h>
     1.9 +#include <stdlib.h>
    1.10  #include <string.h>
    1.11 +#include <time.h>
    1.12  
    1.13 -// TODO: port over remaining harness
    1.14 +/* Assert check message format */
    1.15 +const char *SDLTest_TestCheckFmt = "Test '%s': %s";
    1.16 +
    1.17 +/* Invalid test name/description message format */
    1.18 +const char *SDLTest_InvalidNameFmt = "(Invalid)";
    1.19 +
    1.20 +/*! \brief Timeout for single test case execution */
    1.21 +static Uint32 SDLTest_TestCaseTimeout = 3600;
    1.22  
    1.23  /**
    1.24   * Generates a random run seed string for the harness. The generated seed
    1.25 @@ -155,7 +164,7 @@
    1.26   * \return Timer id or -1 on failure.
    1.27   */
    1.28  SDL_TimerID
    1.29 -SetTestTimeout(int timeout, void (*callback)())
    1.30 +SDLTest_SetTestTimeout(int timeout, void (*callback)())
    1.31  {
    1.32  	Uint32 timeoutInMilliseconds;
    1.33  	SDL_TimerID timerID;
    1.34 @@ -188,3 +197,258 @@
    1.35  
    1.36  	return timerID;
    1.37  }
    1.38 +
    1.39 +void
    1.40 +SDLTest_BailOut()
    1.41 +{
    1.42 +	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
    1.43 +	exit(TEST_ABORTED); // bail out from the test
    1.44 +}
    1.45 +
    1.46 +/**
    1.47 + * \brief Execute a test using the given execution key.
    1.48 + *
    1.49 + * \param testSuite Suite containing the test case.
    1.50 + * \param testCase Case to execute.
    1.51 + * \param execKey Execution key for the fuzzer.
    1.52 + *
    1.53 + * \returns Test case result.
    1.54 + */
    1.55 +int
    1.56 +SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
    1.57 +{
    1.58 +	SDL_TimerID timer = 0;
    1.59 +
    1.60 +	if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
    1.61 +	{
    1.62 +		SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
    1.63 +		return TEST_RESULT_SETUP_FAILURE;
    1.64 +	}
    1.65 +
    1.66 +	if (!testCase->enabled)
    1.67 +	{
    1.68 +		SDLTest_Log((char *)SDLTest_TestCheckFmt, testCase->name, "Skipped");
    1.69 +		return TEST_RESULT_SKIPPED;
    1.70 +	}
    1.71 +
    1.72 +    // Initialize fuzzer
    1.73 +	SDLTest_FuzzerInit(execKey);
    1.74 +
    1.75 +	// Reset assert tracker
    1.76 +	SDLTest_ResetAssertSummary();
    1.77 +
    1.78 +	// Set timeout timer
    1.79 +	timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
    1.80 +
    1.81 +	// Maybe run suite initalizer function
    1.82 +	if (testSuite->testSetUp) {
    1.83 +		testSuite->testSetUp(0x0);
    1.84 +		if (SDLTest_AssertsFailed > 0) {
    1.85 +			SDLTest_LogError((char *)SDLTest_TestCheckFmt, testSuite->name, "Failed");
    1.86 +			return TEST_RESULT_SETUP_FAILURE;
    1.87 +		}
    1.88 +	}
    1.89 +
    1.90 +	// Run test case function
    1.91 +	testCase->testCase(0x0);
    1.92 +
    1.93 +	// Maybe run suite cleanup function
    1.94 +	if (testSuite->testTearDown) {
    1.95 +		testSuite->testTearDown(0x0);
    1.96 +	}
    1.97 +
    1.98 +	// Cancel timeout timer
    1.99 +	if (timer) {
   1.100 +		SDL_RemoveTimer(timer);
   1.101 +	}
   1.102 +
   1.103 +	// Report on asserts and fuzzer usage
   1.104 +	SDLTest_Log("Fuzzer invocations: %d", SDLTest_GetFuzzerInvocationCount());
   1.105 +	SDLTest_LogAssertSummary();
   1.106 +
   1.107 +	// Analyze assert count to determine test case result
   1.108 +	if (SDLTest_AssertsFailed > 0) {
   1.109 +		SDLTest_LogError((char *)SDLTest_TestCheckFmt, testCase->name, "Failed");
   1.110 +		return TEST_RESULT_FAILED;
   1.111 +	} else {
   1.112 +		if (SDLTest_AssertsPassed > 0) {
   1.113 +			SDLTest_Log((char *)SDLTest_TestCheckFmt, testCase->name, "Passed");
   1.114 +			return TEST_RESULT_PASSED;
   1.115 +		} else {
   1.116 +			SDLTest_LogError((char *)SDLTest_TestCheckFmt, testCase->name, "No Asserts");
   1.117 +			return TEST_RESULT_NO_ASSERT;
   1.118 +		}
   1.119 +	}
   1.120 +}
   1.121 +
   1.122 +/* Prints summary of all suites/tests contained in the given reference */
   1.123 +void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
   1.124 +{
   1.125 +	int suiteCounter;
   1.126 +	int testCounter;
   1.127 +	SDLTest_TestSuiteReference *testSuite;
   1.128 +	SDLTest_TestCaseReference *testCase;
   1.129 +
   1.130 +	// Loop over all suites
   1.131 +	suiteCounter = 0;
   1.132 +	while(&testSuites[suiteCounter]) {
   1.133 +		testSuite=&testSuites[suiteCounter];
   1.134 +		suiteCounter++;
   1.135 +		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   1.136 +			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFmt);
   1.137 +
   1.138 +		// Loop over all test cases
   1.139 +		testCounter = 0;
   1.140 +		while(testSuite->testCases[testCounter])
   1.141 +		{
   1.142 +			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   1.143 +			testCounter++;
   1.144 +			SDLTest_Log("  Test Case %i - %s: %s", testCounter, 
   1.145 +				(testCase->name) ? testCase->name : SDLTest_InvalidNameFmt, 
   1.146 +				(testCase->description) ? testCase->description : SDLTest_InvalidNameFmt);
   1.147 +		}
   1.148 +	}
   1.149 +}
   1.150 +
   1.151 +
   1.152 +/**
   1.153 + * \brief Execute a test using the given execution key.
   1.154 + *
   1.155 + * \param testSuites Suites containing the test case.
   1.156 + * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   1.157 + * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   1.158 + * \param testIterations Number of iterations to run each test case.
   1.159 + *
   1.160 + * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   1.161 + */
   1.162 +int
   1.163 +SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites, char *userRunSeed, Uint64 userExecKey, int testIterations)
   1.164 +{
   1.165 +	int suiteCounter;
   1.166 +	int testCounter;
   1.167 +	int iterationCounter;
   1.168 +	SDLTest_TestSuiteReference *testSuite;
   1.169 +	SDLTest_TestCaseReference *testCase;
   1.170 +	char *runSeed;
   1.171 +	Uint64 execKey;
   1.172 +	Uint32 runStartTicks;
   1.173 +	time_t runStartTimestamp;
   1.174 +	Uint32 suiteStartTicks;
   1.175 +	time_t suiteStartTimestamp;
   1.176 +	Uint32 testStartTicks;
   1.177 +	time_t testStartTimestamp;
   1.178 +	Uint32 runEndTicks;
   1.179 +	time_t runEndTimestamp;
   1.180 +	Uint32 suiteEndTicks;
   1.181 +	time_t suiteEndTimestamp;
   1.182 +	Uint32 testEndTicks;
   1.183 +	time_t testEndTimestamp;
   1.184 +	int testResult;
   1.185 +	int totalTestFailedCount, totalTestPassedCount, totalTestSkippedCount;
   1.186 +	int testFailedCount, testPassedCount, testSkippedCount;
   1.187 +
   1.188 +	// Sanitize test iterations
   1.189 +	if (testIterations < 1) {
   1.190 +		testIterations = 1;
   1.191 +	}
   1.192 +
   1.193 +	// Generate run see if we don't have one already
   1.194 +	if (userRunSeed == NULL || strlen(userRunSeed) == 0) {
   1.195 +		runSeed = SDLTest_GenerateRunSeed(16);
   1.196 +		if (runSeed == NULL) {
   1.197 +			SDLTest_LogError("Generating a random run seed failed");
   1.198 +			return 2;
   1.199 +		}
   1.200 +	}
   1.201 +
   1.202 +	// Reset per-run counters
   1.203 +	totalTestFailedCount = totalTestPassedCount = totalTestSkippedCount = 0;
   1.204 +
   1.205 +	// Take time - run start
   1.206 +	runStartTicks = SDL_GetTicks();
   1.207 +	runStartTimestamp = time(0);
   1.208 +
   1.209 +	// TODO log run started
   1.210 +
   1.211 +	// Loop over all suites
   1.212 +	suiteCounter = 0;
   1.213 +	while(&testSuites[suiteCounter]) {
   1.214 +		testSuite=&testSuites[suiteCounter];
   1.215 +		suiteCounter++;
   1.216 +
   1.217 +		// Reset per-suite counters
   1.218 +		testFailedCount = testPassedCount = testSkippedCount = 0;
   1.219 +
   1.220 +		// Take time - suite start
   1.221 +		suiteStartTicks = SDL_GetTicks();
   1.222 +		suiteStartTimestamp = time(0);
   1.223 +
   1.224 +		// TODO log suite started
   1.225 +		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   1.226 +			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFmt);
   1.227 +
   1.228 +		// Loop over all test cases
   1.229 +		testCounter = 0;
   1.230 +		while(testSuite->testCases[testCounter])
   1.231 +		{
   1.232 +			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   1.233 +			testCounter++;
   1.234 +			
   1.235 +			// Take time - test start
   1.236 +			testStartTicks = SDL_GetTicks();
   1.237 +			testStartTimestamp = time(0);
   1.238 +
   1.239 +			// TODO log test started
   1.240 +			SDLTest_Log("Test Case %i - %s: %s", testCounter, 
   1.241 +				(testCase->name) ? testCase->name : SDLTest_InvalidNameFmt, 
   1.242 +				(testCase->description) ? testCase->description : SDLTest_InvalidNameFmt);
   1.243 +
   1.244 +			// Loop over all iterations
   1.245 +			iterationCounter = 0;
   1.246 +			while(iterationCounter < testIterations)
   1.247 +			{
   1.248 +				iterationCounter++;
   1.249 +
   1.250 +				if(userExecKey != 0) {
   1.251 +					execKey = userExecKey;
   1.252 +				} else {
   1.253 +					execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
   1.254 +				}
   1.255 +
   1.256 +				SDLTest_Log("Test Iteration %i: execKey %d", iterationCounter, execKey);
   1.257 +				testResult = SDLTest_RunTest(testSuite, testCase, execKey);
   1.258 +
   1.259 +				if (testResult == TEST_RESULT_PASSED) {
   1.260 +					testPassedCount++;
   1.261 +					totalTestPassedCount++;
   1.262 +				} else if (testResult == TEST_RESULT_SKIPPED) {
   1.263 +					testSkippedCount++;
   1.264 +					totalTestSkippedCount++;
   1.265 +				} else {
   1.266 +					testFailedCount++;
   1.267 +					totalTestFailedCount++;
   1.268 +				}
   1.269 +			}
   1.270 +
   1.271 +			// Take time - test end
   1.272 +			testEndTicks = SDL_GetTicks();
   1.273 +			testEndTimestamp = time(0);
   1.274 +
   1.275 +			// TODO log test ended
   1.276 +		}
   1.277 +
   1.278 +		// Take time - suite end
   1.279 +		suiteEndTicks = SDL_GetTicks();
   1.280 +		suiteEndTimestamp = time(0);
   1.281 +
   1.282 +		// TODO log suite ended
   1.283 +	}
   1.284 +
   1.285 +	// Take time - run end
   1.286 +	runEndTicks = SDL_GetTicks();
   1.287 +	runEndTimestamp = time(0);
   1.288 +
   1.289 +	// TODO log run ended
   1.290 +
   1.291 +	return (totalTestFailedCount ? 1 : 0);
   1.292 +}