src/test/SDL_test_harness.c
changeset 7189 414be1d64060
parent 6885 700f1b25f77f
child 7191 75360622e65f
     1.1 --- a/src/test/SDL_test_harness.c	Sat May 18 14:48:19 2013 +0200
     1.2 +++ b/src/test/SDL_test_harness.c	Sat May 18 09:35:09 2013 -0700
     1.3 @@ -57,20 +57,20 @@
     1.4  	SDLTest_RandomContext randomContext;
     1.5  	int counter;
     1.6  
     1.7 -	// Sanity check input
     1.8 +	/* Sanity check input */
     1.9  	if (length <= 0) {
    1.10  		SDLTest_LogError("The length of the harness seed must be >0.");
    1.11  		return NULL;
    1.12  	}
    1.13  
    1.14 -	// Allocate output buffer
    1.15 +	/* Allocate output buffer */
    1.16  	seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    1.17  	if (seed == NULL) {
    1.18  		SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    1.19  		return NULL;
    1.20  	}
    1.21  
    1.22 -	// Generate a random string of alphanumeric characters
    1.23 +	/* Generate a random string of alphanumeric characters */
    1.24  	SDLTest_RandomInitTime(&randomContext);
    1.25  	for (counter = 0; counter < length - 1; ++counter) {
    1.26  		unsigned int number = SDLTest_Random(&randomContext);
    1.27 @@ -129,11 +129,11 @@
    1.28  		return -1;
    1.29  	}
    1.30  
    1.31 -	// Convert iteration number into a string
    1.32 +	/* Convert iteration number into a string */
    1.33  	SDL_memset(iterationString, 0, sizeof(iterationString));
    1.34  	SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
    1.35  
    1.36 -	// Combine the parameters into single string
    1.37 +	/* Combine the parameters into single string */
    1.38  	runSeedLength = SDL_strlen(runSeed);
    1.39  	suiteNameLength = SDL_strlen(suiteName);
    1.40  	testNameLength = SDL_strlen(testName);
    1.41 @@ -146,7 +146,7 @@
    1.42  	}
    1.43  	SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
    1.44  
    1.45 -	// Hash string and use half of the digest as 64bit exec key
    1.46 +	/* Hash string and use half of the digest as 64bit exec key */
    1.47  	SDLTest_Md5Init(&md5Context);
    1.48  	SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
    1.49  	SDLTest_Md5Final(&md5Context);
    1.50 @@ -208,7 +208,7 @@
    1.51  	SDLTest_BailOut()
    1.52  {
    1.53  	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
    1.54 -	exit(TEST_ABORTED); // bail out from the test
    1.55 +	exit(TEST_ABORTED); /* bail out from the test */
    1.56  }
    1.57  
    1.58  /**
    1.59 @@ -224,6 +224,7 @@
    1.60  SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
    1.61  {
    1.62  	SDL_TimerID timer = 0;
    1.63 +	int testCaseResult = 0;
    1.64  	int testResult = 0;
    1.65  	int fuzzerCount;
    1.66  
    1.67 @@ -240,16 +241,16 @@
    1.68  	}
    1.69  
    1.70  
    1.71 -	// Initialize fuzzer
    1.72 +	/* Initialize fuzzer */
    1.73  	SDLTest_FuzzerInit(execKey);
    1.74  
    1.75 -	// Reset assert tracker
    1.76 +	/* Reset assert tracker */
    1.77  	SDLTest_ResetAssertSummary();
    1.78  
    1.79 -	// Set timeout timer
    1.80 +	/* Set timeout timer */
    1.81  	timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
    1.82  
    1.83 -	// Maybe run suite initalizer function
    1.84 +	/* Maybe run suite initalizer function */
    1.85  	if (testSuite->testSetUp) {
    1.86  		testSuite->testSetUp(0x0);
    1.87  		if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
    1.88 @@ -258,26 +259,53 @@
    1.89  		}
    1.90  	}
    1.91  
    1.92 -	// Run test case function
    1.93 -	testCase->testCase(0x0);
    1.94 -	testResult = SDLTest_AssertSummaryToTestResult();
    1.95 +	/* Run test case function */
    1.96 +	testCaseResult = testCase->testCase(0x0);
    1.97 +	
    1.98 +	/* Convert test execution result into harness result */
    1.99 +	if (testCaseResult == TEST_SKIPPED) {
   1.100 +		/* Test was programatically skipped */
   1.101 +		testResult = TEST_RESULT_SKIPPED;
   1.102 +	} else if (testCaseResult == TEST_STARTED) {
   1.103 +		/* Test did not return a TEST_COMPLETED value; assume it failed */
   1.104 +		testResult = TEST_RESULT_FAILED;
   1.105 +	} else if (testCaseResult == TEST_ABORTED) {
   1.106 +		/* Test was aborted early; assume it failed */
   1.107 +		testResult = TEST_RESULT_FAILED;
   1.108 +	} else {
   1.109 +		/* Perform failure analysis based on asserts */
   1.110 +		testResult = SDLTest_AssertSummaryToTestResult();
   1.111 +	}
   1.112  
   1.113 -	// Maybe run suite cleanup function (ignore failed asserts)
   1.114 +	/* Maybe run suite cleanup function (ignore failed asserts) */
   1.115  	if (testSuite->testTearDown) {
   1.116  		testSuite->testTearDown(0x0);
   1.117  	}
   1.118  
   1.119 -	// Cancel timeout timer
   1.120 +	/* Cancel timeout timer */
   1.121  	if (timer) {
   1.122  		SDL_RemoveTimer(timer);
   1.123  	}
   1.124  
   1.125 -	// Report on asserts and fuzzer usage
   1.126 +	/* Report on asserts and fuzzer usage */
   1.127  	fuzzerCount = SDLTest_GetFuzzerInvocationCount();
   1.128  	if (fuzzerCount > 0) {
   1.129  		SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
   1.130  	}
   1.131 -	SDLTest_LogAssertSummary();
   1.132 +
   1.133 +	/* Final log based on test execution result */
   1.134 +	if (testCaseResult == TEST_SKIPPED) {
   1.135 +		/* Test was programatically skipped */
   1.136 +		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)");
   1.137 +	} else if (testCaseResult == TEST_STARTED) {
   1.138 +		/* Test did not return a TEST_COMPLETED value; assume it failed */
   1.139 +		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
   1.140 +	} else if (testCaseResult == TEST_ABORTED) {
   1.141 +		/* Test was aborted early; assume it failed */
   1.142 +		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)");
   1.143 +	} else {
   1.144 +		SDLTest_LogAssertSummary();
   1.145 +	}
   1.146  
   1.147  	return testResult;
   1.148  }
   1.149 @@ -290,7 +318,7 @@
   1.150  	SDLTest_TestSuiteReference *testSuite;
   1.151  	SDLTest_TestCaseReference *testCase;
   1.152  
   1.153 -	// Loop over all suites
   1.154 +	/* Loop over all suites */
   1.155  	suiteCounter = 0;
   1.156  	while(&testSuites[suiteCounter]) {
   1.157  		testSuite=&testSuites[suiteCounter];
   1.158 @@ -298,7 +326,7 @@
   1.159  		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   1.160  			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   1.161  
   1.162 -		// Loop over all test cases
   1.163 +		/* Loop over all test cases */
   1.164  		testCounter = 0;
   1.165  		while(testSuite->testCases[testCounter])
   1.166  		{
   1.167 @@ -365,12 +393,12 @@
   1.168  	Uint32 countSum = 0;
   1.169  	char *logFormat = (char *)SDLTest_LogSummaryFormat;
   1.170  
   1.171 -	// Sanitize test iterations
   1.172 +	/* Sanitize test iterations */
   1.173  	if (testIterations < 1) {
   1.174  		testIterations = 1;
   1.175  	}
   1.176  
   1.177 -	// Generate run see if we don't have one already
   1.178 +	/* Generate run see if we don't have one already */
   1.179  	if (userRunSeed == NULL || SDL_strlen(userRunSeed) == 0) {
   1.180  		runSeed = SDLTest_GenerateRunSeed(16);
   1.181  		if (runSeed == NULL) {
   1.182 @@ -382,18 +410,18 @@
   1.183  	}
   1.184  
   1.185  
   1.186 -	// Reset per-run counters
   1.187 +	/* Reset per-run counters */
   1.188  	totalTestFailedCount = 0;
   1.189  	totalTestPassedCount = 0;
   1.190  	totalTestSkippedCount = 0;
   1.191  
   1.192 -	// Take time - run start
   1.193 +	/* Take time - run start */
   1.194  	runStartSeconds = GetClock();
   1.195  
   1.196 -	// Log run with fuzzer parameters
   1.197 +	/* Log run with fuzzer parameters */
   1.198  	SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   1.199  
   1.200 -	// Initialize filtering
   1.201 +	/* Initialize filtering */
   1.202  	if (filter != NULL && SDL_strlen(filter) > 0) {
   1.203  		/* Loop over all suites to check if we have a filter match */
   1.204  		suiteCounter = 0;
   1.205 @@ -433,36 +461,36 @@
   1.206  		}		
   1.207  	}
   1.208  
   1.209 -	// Loop over all suites
   1.210 +	/* Loop over all suites */
   1.211  	suiteCounter = 0;
   1.212  	while(testSuites[suiteCounter]) {
   1.213  		testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   1.214  		currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   1.215  		suiteCounter++;
   1.216  
   1.217 -		// Filter suite if flag set and we have a name
   1.218 +		/* Filter suite if flag set and we have a name */
   1.219  		if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
   1.220  			SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
   1.221 -				// Skip suite
   1.222 +				/* Skip suite */
   1.223  				SDLTest_Log("===== Test Suite %i: '%s' skipped\n", 
   1.224  					suiteCounter, 
   1.225  					currentSuiteName);
   1.226  		} else {
   1.227  
   1.228 -			// Reset per-suite counters
   1.229 +			/* Reset per-suite counters */
   1.230  			testFailedCount = 0;
   1.231  			testPassedCount = 0;
   1.232  			testSkippedCount = 0;
   1.233  
   1.234 -			// Take time - suite start
   1.235 +			/* Take time - suite start */
   1.236  			suiteStartSeconds = GetClock();
   1.237  
   1.238 -			// Log suite started
   1.239 +			/* Log suite started */
   1.240  			SDLTest_Log("===== Test Suite %i: '%s' started\n", 
   1.241  				suiteCounter, 
   1.242  				currentSuiteName);
   1.243  
   1.244 -			// Loop over all test cases
   1.245 +			/* Loop over all test cases */
   1.246  			testCounter = 0;
   1.247  			while(testSuite->testCases[testCounter])
   1.248  			{
   1.249 @@ -470,25 +498,25 @@
   1.250  				currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   1.251  				testCounter++;
   1.252  
   1.253 -				// Filter tests if flag set and we have a name
   1.254 +				/* Filter tests if flag set and we have a name */
   1.255  				if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
   1.256  					SDL_strcmp(testFilterName, testCase->name) != 0) {
   1.257 -						// Skip test
   1.258 +						/* Skip test */
   1.259  						SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n", 
   1.260  							suiteCounter,
   1.261  							testCounter,
   1.262  							currentTestName);
   1.263  				} else {
   1.264 -					// Override 'disabled' flag if we specified a test filter (i.e. force run for debugging)
   1.265 +					/* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
   1.266  					if (testFilter == 1 && !testCase->enabled) {
   1.267  						SDLTest_Log("Force run of disabled test since test filter was set");
   1.268  						testCase->enabled = 1;
   1.269  					}
   1.270  
   1.271 -					// Take time - test start
   1.272 +					/* Take time - test start */
   1.273  					testStartSeconds = GetClock();
   1.274  
   1.275 -					// Log test started
   1.276 +					/* Log test started */
   1.277  					SDLTest_Log("----- Test Case %i.%i: '%s' started",
   1.278  						suiteCounter,
   1.279  						testCounter, 
   1.280 @@ -498,7 +526,7 @@
   1.281  							(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   1.282  					}
   1.283  
   1.284 -					// Loop over all iterations
   1.285 +					/* Loop over all iterations */
   1.286  					iterationCounter = 0;
   1.287  					while(iterationCounter < testIterations)
   1.288  					{
   1.289 @@ -525,21 +553,21 @@
   1.290  						}
   1.291  					}
   1.292  
   1.293 -					// Take time - test end
   1.294 +					/* Take time - test end */
   1.295  					testEndSeconds = GetClock();
   1.296  					runtime = testEndSeconds - testStartSeconds;
   1.297  					if (runtime < 0.0f) runtime = 0.0f;
   1.298  
   1.299  					if (testIterations > 1) {
   1.300 -						// Log test runtime
   1.301 +						/* Log test runtime */
   1.302  						SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
   1.303  						SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
   1.304  					} else {
   1.305 -						// Log test runtime
   1.306 +						/* Log test runtime */
   1.307  						SDLTest_Log("Total Test runtime: %.1f sec", runtime);
   1.308  					}
   1.309  
   1.310 -					// Log final test result
   1.311 +					/* Log final test result */
   1.312  					switch (testResult) {
   1.313  					case TEST_RESULT_PASSED:
   1.314  						SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   1.315 @@ -555,15 +583,15 @@
   1.316  				}
   1.317  			}
   1.318  
   1.319 -			// Take time - suite end
   1.320 +			/* Take time - suite end */
   1.321  			suiteEndSeconds = GetClock();
   1.322  			runtime = suiteEndSeconds - suiteStartSeconds;
   1.323  			if (runtime < 0.0f) runtime = 0.0f;
   1.324  
   1.325 -			// Log suite runtime
   1.326 +			/* Log suite runtime */
   1.327  			SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
   1.328  
   1.329 -			// Log summary and final Suite result
   1.330 +			/* Log summary and final Suite result */
   1.331  			countSum = testPassedCount + testFailedCount + testSkippedCount;
   1.332  			if (testFailedCount == 0)
   1.333  			{
   1.334 @@ -579,15 +607,15 @@
   1.335  		}
   1.336  	}
   1.337  
   1.338 -	// Take time - run end
   1.339 +	/* Take time - run end */
   1.340  	runEndSeconds = GetClock();
   1.341  	runtime = runEndSeconds - runStartSeconds;
   1.342  	if (runtime < 0.0f) runtime = 0.0f;
   1.343  
   1.344 -	// Log total runtime
   1.345 +	/* Log total runtime */
   1.346  	SDLTest_Log("Total Run runtime: %.1f sec", runtime);
   1.347  
   1.348 -	// Log summary and final run result
   1.349 +	/* Log summary and final run result */
   1.350  	countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   1.351  	if (totalTestFailedCount == 0)
   1.352  	{