src/test/SDL_test_harness.c
author Andreas Schiffler
Sat, 15 Dec 2012 21:50:17 -0800
changeset 6757 9935f71c8c81
parent 6756 398073b195bb
child 6760 04dcce3081e6
permissions -rw-r--r--
Fixes in harness and fuzzer test lib components; improve harness driver; add rect test suite
     1 /*
     2   Simple DirectMedia Layer
     3   Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
     4 
     5   This software is provided 'as-is', without any express or implied
     6   warranty.  In no event will the authors be held liable for any damages
     7   arising from the use of this software.
     8 
     9   Permission is granted to anyone to use this software for any purpose,
    10   including commercial applications, and to alter it and redistribute it
    11   freely, subject to the following restrictions:
    12 
    13   1. The origin of this software must not be misrepresented; you must not
    14      claim that you wrote the original software. If you use this software
    15      in a product, an acknowledgment in the product documentation would be
    16      appreciated but is not required.
    17   2. Altered source versions must be plainly marked as such, and must not be
    18      misrepresented as being the original software.
    19   3. This notice may not be removed or altered from any source distribution.
    20 */
    21 
    22 #include "SDL_config.h"
    23 
    24 #include "SDL_test.h"
    25 
    26 #include <stdio.h>
    27 #include <stdlib.h>
    28 #include <string.h>
    29 #include <time.h>
    30 
    31 /* Invalid test name/description message format */
    32 const char *SDLTest_InvalidNameFormat = "(Invalid)";
    33 
    34 /* Log summary message format */
    35 const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d";
    36 
    37 /* Final result message format */
    38 const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n";
    39 
    40 /*! \brief Timeout for single test case execution */
    41 static Uint32 SDLTest_TestCaseTimeout = 3600;
    42 
    43 /**
    44  * Generates a random run seed string for the harness. The generated seed
    45  * will contain alphanumeric characters (0-9A-Z).
    46  *
    47  * Note: The returned string needs to be deallocated by the caller.
    48  *
    49  * \param length The length of the seed string to generate
    50  *
    51  * \returns The generated seed string
    52  */
    53 char *
    54 SDLTest_GenerateRunSeed(const int length)
    55 {
    56 	char *seed = NULL;
    57 	SDLTest_RandomContext randomContext;
    58 	int counter;
    59 
    60 	// Sanity check input
    61 	if (length <= 0) {
    62 		SDLTest_LogError("The length of the harness seed must be >0.");
    63 		return NULL;
    64 	}
    65 
    66 	// Allocate output buffer
    67 	seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    68 	if (seed == NULL) {
    69 		SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    70 		return NULL;
    71 	}
    72 
    73 	// Generate a random string of alphanumeric characters
    74 	SDLTest_RandomInitTime(&randomContext);
    75 	for (counter = 0; counter < length - 1; ++counter) {
    76 		unsigned int number = SDLTest_Random(&randomContext);
    77 		char ch = (char) (number % (91 - 48)) + 48;
    78 		if (ch >= 58 && ch <= 64) {
    79 			ch = 65;
    80 		}
    81 		seed[counter] = ch;
    82 	}
    83 	seed[counter] = '\0';
    84 
    85 	return seed;
    86 }
    87 
    88 /**
    89  * Generates an execution key for the fuzzer.
    90  *
    91  * \param runSeed		The run seed to use
    92  * \param suiteName		The name of the test suite
    93  * \param testName		The name of the test
    94  * \param iteration		The iteration count
    95  *
    96  * \returns The generated execution key to initialize the fuzzer with.
    97  *
    98  */
    99 Uint64
   100 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
   101 {
   102 	SDLTest_Md5Context md5Context;
   103 	Uint64 *keys;
   104 	char iterationString[16];
   105 	Uint32 runSeedLength;
   106 	Uint32 suiteNameLength;
   107 	Uint32 testNameLength;
   108 	Uint32 iterationStringLength;
   109 	Uint32 entireStringLength;
   110 	char *buffer;
   111 
   112 	if (runSeed == NULL || strlen(runSeed)==0) {
   113 		SDLTest_LogError("Invalid runSeed string.");
   114 		return -1;
   115 	}
   116 
   117 	if (suiteName == NULL || strlen(suiteName)==0) {
   118 		SDLTest_LogError("Invalid suiteName string.");
   119 		return -1;
   120 	}
   121 
   122 	if (testName == NULL || strlen(testName)==0) {
   123 		SDLTest_LogError("Invalid testName string.");
   124 		return -1;
   125 	}
   126 
   127 	if (iteration <= 0) {
   128 		SDLTest_LogError("Invalid iteration count.");
   129 		return -1;
   130 	}
   131 
   132 	// Convert iteration number into a string
   133 	memset(iterationString, 0, sizeof(iterationString));
   134 	SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
   135 
   136 	// Combine the parameters into single string
   137 	runSeedLength = strlen(runSeed);
   138 	suiteNameLength = strlen(suiteName);
   139 	testNameLength = strlen(testName);
   140 	iterationStringLength = strlen(iterationString);
   141 	entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
   142 	buffer = (char *)SDL_malloc(entireStringLength);
   143 	if (buffer == NULL) {
   144 		SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
   145 		return 0;
   146 	}
   147 	SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
   148 
   149 	// Hash string and use half of the digest as 64bit exec key
   150 	SDLTest_Md5Init(&md5Context);
   151 	SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
   152 	SDLTest_Md5Final(&md5Context);
   153 	SDL_free(buffer);
   154 	keys = (Uint64 *)md5Context.digest;
   155 
   156 	return keys[0];
   157 }
   158 
   159 /**
   160  * \brief Set timeout handler for test.
   161  *
   162  * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   163  *
   164  * \param timeout Timeout interval in seconds.
   165  * \param callback Function that will be called after timeout has elapsed.
   166  * 
   167  * \return Timer id or -1 on failure.
   168  */
   169 SDL_TimerID
   170 SDLTest_SetTestTimeout(int timeout, void (*callback)())
   171 {
   172 	Uint32 timeoutInMilliseconds;
   173 	SDL_TimerID timerID;
   174 
   175 	if (callback == NULL) {
   176 		SDLTest_LogError("Timeout callback can't be NULL");
   177 		return -1;
   178 	}
   179 
   180 	if (timeout < 0) {
   181 		SDLTest_LogError("Timeout value must be bigger than zero.");
   182 		return -1;
   183 	}
   184 
   185 	/* Init SDL timer if not initialized before */
   186 	if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
   187 		if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
   188 			SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
   189 			return -1;
   190 		}
   191 	}
   192 
   193 	/* Set timer */
   194 	timeoutInMilliseconds = timeout * 1000;
   195 	timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
   196 	if (timerID == 0) {
   197 		SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
   198 		return -1;
   199 	}
   200 
   201 	return timerID;
   202 }
   203 
   204 void
   205 SDLTest_BailOut()
   206 {
   207 	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   208 	exit(TEST_ABORTED); // bail out from the test
   209 }
   210 
   211 /**
   212  * \brief Execute a test using the given execution key.
   213  *
   214  * \param testSuite Suite containing the test case.
   215  * \param testCase Case to execute.
   216  * \param execKey Execution key for the fuzzer.
   217  *
   218  * \returns Test case result.
   219  */
   220 int
   221 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
   222 {
   223 	SDL_TimerID timer = 0;
   224 	int testResult = 0;
   225 	int fuzzerCount;
   226 
   227 	if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
   228 	{
   229 		SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
   230 		return TEST_RESULT_SETUP_FAILURE;
   231 	}
   232 
   233 	if (!testCase->enabled)
   234 	{
   235 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped");
   236 		return TEST_RESULT_SKIPPED;
   237 	}
   238 
   239         // Initialize fuzzer
   240 	SDLTest_FuzzerInit(execKey);
   241 
   242 	// Reset assert tracker
   243 	SDLTest_ResetAssertSummary();
   244 
   245 	// Set timeout timer
   246 	timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
   247 
   248 	// Maybe run suite initalizer function
   249 	if (testSuite->testSetUp) {
   250 		testSuite->testSetUp(0x0);
   251 		if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
   252 			SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed");
   253 			return TEST_RESULT_SETUP_FAILURE;
   254 		}
   255 	}
   256 
   257 	// Run test case function
   258 	testCase->testCase(0x0);
   259 	testResult = SDLTest_AssertSummaryToTestResult();
   260 
   261 	// Maybe run suite cleanup function (ignore failed asserts)
   262 	if (testSuite->testTearDown) {
   263 		testSuite->testTearDown(0x0);
   264 	}
   265 
   266 	// Cancel timeout timer
   267 	if (timer) {
   268 		SDL_RemoveTimer(timer);
   269 	}
   270 
   271 	// Report on asserts and fuzzer usage
   272 	fuzzerCount = SDLTest_GetFuzzerInvocationCount();
   273 	if (fuzzerCount > 0) {
   274 		SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
   275 	}
   276 	SDLTest_LogAssertSummary();
   277 
   278 	return testResult;
   279 }
   280 
   281 /* Prints summary of all suites/tests contained in the given reference */
   282 void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
   283 {
   284 	int suiteCounter;
   285 	int testCounter;
   286 	SDLTest_TestSuiteReference *testSuite;
   287 	SDLTest_TestCaseReference *testCase;
   288 
   289 	// Loop over all suites
   290 	suiteCounter = 0;
   291 	while(&testSuites[suiteCounter]) {
   292 		testSuite=&testSuites[suiteCounter];
   293 		suiteCounter++;
   294 		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   295 			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   296 
   297 		// Loop over all test cases
   298 		testCounter = 0;
   299 		while(testSuite->testCases[testCounter])
   300 		{
   301 			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   302 			testCounter++;
   303 			SDLTest_Log("  Test Case %i - %s: %s", testCounter, 
   304 				(testCase->name) ? testCase->name : SDLTest_InvalidNameFormat, 
   305 				(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   306 		}
   307 	}
   308 }
   309 
   310 /* Gets a timer value in seconds */
   311 float GetClock()
   312 {
   313 	float currentClock = (float)clock();
   314 	return currentClock / (float)CLOCKS_PER_SEC;
   315 }
   316 
   317 /**
   318  * \brief Execute a test suite using the given run seend and execution key.
   319  *
   320  * \param testSuites Suites containing the test case.
   321  * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   322  * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   323  * \param testIterations Number of iterations to run each test case.
   324  *
   325  * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   326  */
   327 int
   328 SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, Uint64 userExecKey, int testIterations)
   329 {
   330 	int suiteCounter;
   331 	int testCounter;
   332 	int iterationCounter;
   333 	SDLTest_TestSuiteReference *testSuite;
   334 	SDLTest_TestCaseReference *testCase;
   335 	char *runSeed = NULL;
   336 	char *currentSuiteName;
   337 	char *currentTestName;
   338 	Uint64 execKey;
   339 	float runStartSeconds;
   340 	float suiteStartSeconds;
   341 	float testStartSeconds;
   342 	float runEndSeconds;
   343 	float suiteEndSeconds;
   344 	float testEndSeconds;
   345 	int testResult = 0;
   346 	int runResult = 0;
   347 	Uint32 totalTestFailedCount = 0;
   348 	Uint32 totalTestPassedCount = 0;
   349 	Uint32 totalTestSkippedCount = 0;
   350 	Uint32 testFailedCount = 0;
   351 	Uint32 testPassedCount = 0;
   352 	Uint32 testSkippedCount = 0;
   353 	Uint32 countSum = 0;
   354 	char *logFormat = (char *)SDLTest_LogSummaryFormat;
   355 
   356 	// Sanitize test iterations
   357 	if (testIterations < 1) {
   358 		testIterations = 1;
   359 	}
   360 
   361 	// Generate run see if we don't have one already
   362 	if (userRunSeed == NULL || strlen(userRunSeed) == 0) {
   363 		runSeed = SDLTest_GenerateRunSeed(16);
   364 		if (runSeed == NULL) {
   365 			SDLTest_LogError("Generating a random seed failed");
   366 			return 2;
   367 		}
   368 	} else {
   369 		runSeed = userRunSeed;
   370 	}
   371 
   372 	// Reset per-run counters
   373 	totalTestFailedCount = 0;
   374 	totalTestPassedCount = 0;
   375 	totalTestSkippedCount = 0;
   376 
   377 	// Take time - run start
   378 	runStartSeconds = GetClock();
   379 
   380 	// Log run with fuzzer parameters
   381 	SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   382 
   383 	// Loop over all suites
   384 	suiteCounter = 0;
   385 	while(testSuites[suiteCounter]) {
   386 		testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   387 		suiteCounter++;
   388 
   389 		// Reset per-suite counters
   390 		testFailedCount = 0;
   391 		testPassedCount = 0;
   392 		testSkippedCount = 0;
   393 
   394 		// Take time - suite start
   395 		suiteStartSeconds = GetClock();
   396 
   397 		// Log suite started
   398 		currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   399 		SDLTest_Log("===== Test Suite %i: '%s' started\n", 
   400 			suiteCounter, 
   401 			currentSuiteName);
   402 
   403 		// Loop over all test cases
   404 		testCounter = 0;
   405 		while(testSuite->testCases[testCounter])
   406 		{
   407 			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   408 			testCounter++;
   409 			
   410 			// Take time - test start
   411 			testStartSeconds = GetClock();
   412 
   413 			// Log test started
   414 			currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   415 			SDLTest_Log("----- Test Case %i.%i: '%s' started",
   416 			        suiteCounter,
   417 				testCounter, 
   418 				currentTestName);
   419 			if (testCase->description != NULL && strlen(testCase->description)>0) {
   420 				SDLTest_Log("Test Description: '%s'", 
   421 					(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   422 			}
   423 			
   424 			// Loop over all iterations
   425 			iterationCounter = 0;
   426 			while(iterationCounter < testIterations)
   427 			{
   428 				iterationCounter++;
   429 
   430 				if (userExecKey != 0) {
   431 					execKey = userExecKey;
   432 				} else {
   433 					execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
   434 				}
   435 
   436 				SDLTest_Log("Test Iteration %i: execKey %llu", iterationCounter, execKey);
   437 				testResult = SDLTest_RunTest(testSuite, testCase, execKey);
   438 
   439 				if (testResult == TEST_RESULT_PASSED) {
   440 					testPassedCount++;
   441 					totalTestPassedCount++;
   442 				} else if (testResult == TEST_RESULT_SKIPPED) {
   443 					testSkippedCount++;
   444 					totalTestSkippedCount++;
   445 				} else {
   446 					testFailedCount++;
   447 					totalTestFailedCount++;
   448 				}
   449 			}
   450 
   451 			// Take time - test end
   452 			testEndSeconds = GetClock();
   453 
   454 			if (testIterations > 1) {
   455         			// Log test runtime
   456 	        		SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, testEndSeconds - testStartSeconds);
   457 	        		SDLTest_Log("Test runtime: %.5f sec", (testEndSeconds - testStartSeconds) / (float)testIterations);
   458                         } else {
   459         			// Log test runtime
   460 	        		SDLTest_Log("Test runtime: %.1f sec", testEndSeconds - testStartSeconds);
   461                         }
   462 
   463 			// Log final test result
   464 			switch (testResult) {
   465 				case TEST_RESULT_PASSED:
   466 					SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   467 					break;
   468 				case TEST_RESULT_FAILED:
   469 					SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
   470 					break;
   471 				case TEST_RESULT_NO_ASSERT:
   472 					SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
   473 					break;
   474 			}
   475 		}
   476 
   477 		// Take time - suite end
   478 		suiteEndSeconds = GetClock();
   479 
   480 		// Log suite runtime
   481 		SDLTest_Log("Suite runtime: %.1f sec", suiteEndSeconds - suiteStartSeconds);
   482 
   483 		// Log summary and final Suite result
   484 		countSum = testPassedCount + testFailedCount + testSkippedCount;
   485 		if (testFailedCount == 0)
   486 		{
   487 			SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   488 			SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
   489 		} 
   490 		else 
   491 		{
   492 			SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   493 			SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
   494 		}
   495 	}
   496 
   497 	// Take time - run end
   498 	runEndSeconds = GetClock();
   499 
   500 	// Log total runtime
   501 	SDLTest_Log("Total runtime: %.1f sec", runEndSeconds - runStartSeconds);
   502 
   503 	// Log summary and final run result
   504 	countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   505 	if (testFailedCount == 0)
   506 	{
   507 		runResult = 0;
   508 		SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   509 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
   510 	} 
   511 	else 
   512 	{
   513 		runResult = 1;
   514 		SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   515 		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
   516 	}
   517 
   518 	SDLTest_Log("Exit code: %d", runResult);	
   519 	return runResult;
   520 }