src/test/SDL_test_harness.c
changeset 6756 398073b195bb
parent 6727 1b5280cd5885
child 6757 9935f71c8c81
     1.1 --- a/src/test/SDL_test_harness.c	Sat Dec 15 00:30:17 2012 +0000
     1.2 +++ b/src/test/SDL_test_harness.c	Fri Dec 14 23:05:34 2012 -0800
     1.3 @@ -28,11 +28,14 @@
     1.4  #include <string.h>
     1.5  #include <time.h>
     1.6  
     1.7 -/* Assert check message format */
     1.8 -const char *SDLTest_TestCheckFmt = "Test '%s': %s";
     1.9 +/* Invalid test name/description message format */
    1.10 +const char *SDLTest_InvalidNameFormat = "(Invalid)";
    1.11  
    1.12 -/* Invalid test name/description message format */
    1.13 -const char *SDLTest_InvalidNameFmt = "(Invalid)";
    1.14 +/* Log summary message format */
    1.15 +const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d";
    1.16 +
    1.17 +/* Final result message format */
    1.18 +const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n";
    1.19  
    1.20  /*! \brief Timeout for single test case execution */
    1.21  static Uint32 SDLTest_TestCaseTimeout = 3600;
    1.22 @@ -228,7 +231,7 @@
    1.23  
    1.24  	if (!testCase->enabled)
    1.25  	{
    1.26 -		SDLTest_Log((char *)SDLTest_TestCheckFmt, testCase->name, "Skipped");
    1.27 +		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped");
    1.28  		return TEST_RESULT_SKIPPED;
    1.29  	}
    1.30  
    1.31 @@ -245,7 +248,7 @@
    1.32  	if (testSuite->testSetUp) {
    1.33  		testSuite->testSetUp(0x0);
    1.34  		if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
    1.35 -			SDLTest_LogError((char *)SDLTest_TestCheckFmt, testSuite->name, "Failed");
    1.36 +			SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed");
    1.37  			return TEST_RESULT_SETUP_FAILURE;
    1.38  		}
    1.39  	}
    1.40 @@ -268,16 +271,6 @@
    1.41  	SDLTest_Log("Fuzzer invocations: %d", SDLTest_GetFuzzerInvocationCount());
    1.42  	SDLTest_LogAssertSummary();
    1.43  
    1.44 -	// Analyze assert count to determine final test case result
    1.45 -	switch (testResult) {
    1.46 -		case TEST_RESULT_PASSED:
    1.47 -			SDLTest_LogError((char *)SDLTest_TestCheckFmt, testCase->name, "Failed");
    1.48 -		case TEST_RESULT_FAILED:
    1.49 -			SDLTest_Log((char *)SDLTest_TestCheckFmt, testCase->name, "Passed");
    1.50 -		case TEST_RESULT_NO_ASSERT:
    1.51 -			SDLTest_LogError((char *)SDLTest_TestCheckFmt, testCase->name, "No Asserts");
    1.52 -	}
    1.53 -
    1.54  	return testResult;
    1.55  }
    1.56  
    1.57 @@ -295,7 +288,7 @@
    1.58  		testSuite=&testSuites[suiteCounter];
    1.59  		suiteCounter++;
    1.60  		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
    1.61 -			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFmt);
    1.62 +			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
    1.63  
    1.64  		// Loop over all test cases
    1.65  		testCounter = 0;
    1.66 @@ -304,15 +297,21 @@
    1.67  			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
    1.68  			testCounter++;
    1.69  			SDLTest_Log("  Test Case %i - %s: %s", testCounter, 
    1.70 -				(testCase->name) ? testCase->name : SDLTest_InvalidNameFmt, 
    1.71 -				(testCase->description) ? testCase->description : SDLTest_InvalidNameFmt);
    1.72 +				(testCase->name) ? testCase->name : SDLTest_InvalidNameFormat, 
    1.73 +				(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
    1.74  		}
    1.75  	}
    1.76  }
    1.77  
    1.78 +/* Gets a timer value in seconds */
    1.79 +float GetClock()
    1.80 +{
    1.81 +	float currentClock = (float)clock();
    1.82 +	return currentClock / (float)CLOCKS_PER_SEC;
    1.83 +}
    1.84  
    1.85  /**
    1.86 - * \brief Execute a test using the given execution key.
    1.87 + * \brief Execute a test suite using the given run seend and execution key.
    1.88   *
    1.89   * \param testSuites Suites containing the test case.
    1.90   * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
    1.91 @@ -322,7 +321,7 @@
    1.92   * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
    1.93   */
    1.94  int
    1.95 -SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites, char *userRunSeed, Uint64 userExecKey, int testIterations)
    1.96 +SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, Uint64 userExecKey, int testIterations)
    1.97  {
    1.98  	int suiteCounter;
    1.99  	int testCounter;
   1.100 @@ -330,22 +329,25 @@
   1.101  	SDLTest_TestSuiteReference *testSuite;
   1.102  	SDLTest_TestCaseReference *testCase;
   1.103  	char *runSeed = NULL;
   1.104 +	char *currentSuiteName;
   1.105 +	char *currentTestName;
   1.106  	Uint64 execKey;
   1.107 -	Uint32 runStartTicks;
   1.108 -	time_t runStartTimestamp;
   1.109 -	Uint32 suiteStartTicks;
   1.110 -	time_t suiteStartTimestamp;
   1.111 -	Uint32 testStartTicks;
   1.112 -	time_t testStartTimestamp;
   1.113 -	Uint32 runEndTicks;
   1.114 -	time_t runEndTimestamp;
   1.115 -	Uint32 suiteEndTicks;
   1.116 -	time_t suiteEndTimestamp;
   1.117 -	Uint32 testEndTicks;
   1.118 -	time_t testEndTimestamp;
   1.119 -	int testResult;
   1.120 -	int totalTestFailedCount, totalTestPassedCount, totalTestSkippedCount;
   1.121 -	int testFailedCount, testPassedCount, testSkippedCount;
   1.122 +	float runStartSeconds;
   1.123 +	float suiteStartSeconds;
   1.124 +	float testStartSeconds;
   1.125 +	float runEndSeconds;
   1.126 +	float suiteEndSeconds;
   1.127 +	float testEndSeconds;
   1.128 +	int testResult = 0;
   1.129 +	int runResult = 0;
   1.130 +	Uint32 totalTestFailedCount = 0;
   1.131 +	Uint32 totalTestPassedCount = 0;
   1.132 +	Uint32 totalTestSkippedCount = 0;
   1.133 +	Uint32 testFailedCount = 0;
   1.134 +	Uint32 testPassedCount = 0;
   1.135 +	Uint32 testSkippedCount = 0;
   1.136 +	Uint32 countSum = 0;
   1.137 +	char *logFormat = (char *)SDLTest_LogSummaryFormat;
   1.138  
   1.139  	// Sanitize test iterations
   1.140  	if (testIterations < 1) {
   1.141 @@ -356,36 +358,41 @@
   1.142  	if (userRunSeed == NULL || strlen(userRunSeed) == 0) {
   1.143  		runSeed = SDLTest_GenerateRunSeed(16);
   1.144  		if (runSeed == NULL) {
   1.145 -			SDLTest_LogError("Generating a random run seed failed");
   1.146 +			SDLTest_LogError("Generating a random seed failed");
   1.147  			return 2;
   1.148  		}
   1.149  	}
   1.150  
   1.151  	// Reset per-run counters
   1.152 -	totalTestFailedCount = totalTestPassedCount = totalTestSkippedCount = 0;
   1.153 +	totalTestFailedCount = 0;
   1.154 +	totalTestPassedCount = 0;
   1.155 +	totalTestSkippedCount = 0;
   1.156  
   1.157  	// Take time - run start
   1.158 -	runStartTicks = SDL_GetTicks();
   1.159 -	runStartTimestamp = time(0);
   1.160 +	runStartSeconds = GetClock();
   1.161  
   1.162 -	// TODO log run started
   1.163 +	// Log run with fuzzer parameters
   1.164 +	SDLTest_Log("::::: Test Run '%s' started\n", runSeed);
   1.165  
   1.166  	// Loop over all suites
   1.167  	suiteCounter = 0;
   1.168 -	while(&testSuites[suiteCounter]) {
   1.169 -		testSuite=&testSuites[suiteCounter];
   1.170 +	while(testSuites[suiteCounter]) {
   1.171 +		testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   1.172  		suiteCounter++;
   1.173  
   1.174  		// Reset per-suite counters
   1.175 -		testFailedCount = testPassedCount = testSkippedCount = 0;
   1.176 +		testFailedCount = 0;
   1.177 +		testPassedCount = 0;
   1.178 +		testSkippedCount = 0;
   1.179  
   1.180  		// Take time - suite start
   1.181 -		suiteStartTicks = SDL_GetTicks();
   1.182 -		suiteStartTimestamp = time(0);
   1.183 +		suiteStartSeconds = GetClock();
   1.184  
   1.185 -		// TODO log suite started
   1.186 -		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   1.187 -			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFmt);
   1.188 +		// Log suite started
   1.189 +		currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   1.190 +		SDLTest_Log("===== Test Suite %i: %s started\n", 
   1.191 +			suiteCounter, 
   1.192 +			currentSuiteName);
   1.193  
   1.194  		// Loop over all test cases
   1.195  		testCounter = 0;
   1.196 @@ -395,13 +402,15 @@
   1.197  			testCounter++;
   1.198  			
   1.199  			// Take time - test start
   1.200 -			testStartTicks = SDL_GetTicks();
   1.201 -			testStartTimestamp = time(0);
   1.202 +			testStartSeconds = GetClock();
   1.203  
   1.204 -			// TODO log test started
   1.205 -			SDLTest_Log("Test Case %i - %s: %s", testCounter, 
   1.206 -				(testCase->name) ? testCase->name : SDLTest_InvalidNameFmt, 
   1.207 -				(testCase->description) ? testCase->description : SDLTest_InvalidNameFmt);
   1.208 +			// Log test started
   1.209 +			currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   1.210 +			SDLTest_Log("----- Test Case %i: %s started", 
   1.211 +				testCounter, 
   1.212 +				currentTestName);
   1.213 +			SDLTest_Log("Test Description: %s", 
   1.214 +				(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   1.215  
   1.216  			// Loop over all iterations
   1.217  			iterationCounter = 0;
   1.218 @@ -431,24 +440,67 @@
   1.219  			}
   1.220  
   1.221  			// Take time - test end
   1.222 -			testEndTicks = SDL_GetTicks();
   1.223 -			testEndTimestamp = time(0);
   1.224 +			testEndSeconds = GetClock();
   1.225 +
   1.226 +			SDLTest_Log("Test Case %s ended", currentTestName);
   1.227 +
   1.228 +			// Log test runtime
   1.229 +			SDLTest_Log("Test runtime: %.1f sec", testEndSeconds - testStartSeconds);
   1.230  
   1.231 -			// TODO log test ended
   1.232 +			// Log final test result
   1.233 +			switch (testResult) {
   1.234 +				case TEST_RESULT_PASSED:
   1.235 +					SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   1.236 +					break;
   1.237 +				case TEST_RESULT_FAILED:
   1.238 +					SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
   1.239 +					break;
   1.240 +				case TEST_RESULT_NO_ASSERT:
   1.241 +					SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
   1.242 +					break;
   1.243 +			}
   1.244  		}
   1.245  
   1.246  		// Take time - suite end
   1.247 -		suiteEndTicks = SDL_GetTicks();
   1.248 -		suiteEndTimestamp = time(0);
   1.249 +		suiteEndSeconds = GetClock();
   1.250 +
   1.251 +		// Log suite runtime
   1.252 +		SDLTest_Log("Suite runtime: %.1f sec", suiteEndSeconds - suiteStartSeconds);
   1.253  
   1.254 -		// TODO log suite ended
   1.255 +		// Log summary and final Suite result
   1.256 +	    countSum = testPassedCount + testFailedCount + testSkippedCount;
   1.257 +		if (testFailedCount == 0)
   1.258 +		{
   1.259 +			SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   1.260 +			SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
   1.261 +		} 
   1.262 +		else 
   1.263 +		{
   1.264 +			SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   1.265 +			SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
   1.266 +		}
   1.267  	}
   1.268  
   1.269  	// Take time - run end
   1.270 -	runEndTicks = SDL_GetTicks();
   1.271 -	runEndTimestamp = time(0);
   1.272 +	runEndSeconds = GetClock();
   1.273 +
   1.274 +	// Log total runtime
   1.275 +	SDLTest_Log("Total runtime: %.1f sec", runEndSeconds - runStartSeconds);
   1.276  
   1.277 -	// TODO log run ended
   1.278 +	// Log summary and final run result
   1.279 +	countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   1.280 +	if (testFailedCount == 0)
   1.281 +	{
   1.282 +		runResult = 0;
   1.283 +		SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   1.284 +		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run", runSeed, "Passed");
   1.285 +	} 
   1.286 +	else 
   1.287 +	{
   1.288 +		runResult = 1;
   1.289 +		SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   1.290 +		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run", runSeed, "Failed");
   1.291 +	}
   1.292  
   1.293 -	return (totalTestFailedCount ? 1 : 0);
   1.294 +	return runResult;
   1.295  }