src/test/SDL_test_harness.c
author Andreas Schiffler
Sun, 16 Dec 2012 21:59:29 -0800
changeset 6760 04dcce3081e6
parent 6757 9935f71c8c81
child 6763 9cbd31a3450b
permissions -rw-r--r--
Port clipboard and rwops test suites from GSOC code; minor updates to harness and fuzzer in test lib
     1 /*
     2   Simple DirectMedia Layer
     3   Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
     4 
     5   This software is provided 'as-is', without any express or implied
     6   warranty.  In no event will the authors be held liable for any damages
     7   arising from the use of this software.
     8 
     9   Permission is granted to anyone to use this software for any purpose,
    10   including commercial applications, and to alter it and redistribute it
    11   freely, subject to the following restrictions:
    12 
    13   1. The origin of this software must not be misrepresented; you must not
    14      claim that you wrote the original software. If you use this software
    15      in a product, an acknowledgment in the product documentation would be
    16      appreciated but is not required.
    17   2. Altered source versions must be plainly marked as such, and must not be
    18      misrepresented as being the original software.
    19   3. This notice may not be removed or altered from any source distribution.
    20 */
    21 
    22 #include "SDL_config.h"
    23 
    24 #include "SDL_test.h"
    25 
    26 #include <stdio.h>
    27 #include <stdlib.h>
    28 #include <string.h>
    29 #include <time.h>
    30 
    31 /* Invalid test name/description message format */
    32 const char *SDLTest_InvalidNameFormat = "(Invalid)";
    33 
    34 /* Log summary message format */
    35 const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d";
    36 
    37 /* Final result message format */
    38 const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n";
    39 
    40 /*! \brief Timeout for single test case execution */
    41 static Uint32 SDLTest_TestCaseTimeout = 3600;
    42 
    43 /**
    44  * Generates a random run seed string for the harness. The generated seed
    45  * will contain alphanumeric characters (0-9A-Z).
    46  *
    47  * Note: The returned string needs to be deallocated by the caller.
    48  *
    49  * \param length The length of the seed string to generate
    50  *
    51  * \returns The generated seed string
    52  */
    53 char *
    54 SDLTest_GenerateRunSeed(const int length)
    55 {
    56 	char *seed = NULL;
    57 	SDLTest_RandomContext randomContext;
    58 	int counter;
    59 
    60 	// Sanity check input
    61 	if (length <= 0) {
    62 		SDLTest_LogError("The length of the harness seed must be >0.");
    63 		return NULL;
    64 	}
    65 
    66 	// Allocate output buffer
    67 	seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    68 	if (seed == NULL) {
    69 		SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    70 		return NULL;
    71 	}
    72 
    73 	// Generate a random string of alphanumeric characters
    74 	SDLTest_RandomInitTime(&randomContext);
    75 	for (counter = 0; counter < length - 1; ++counter) {
    76 		unsigned int number = SDLTest_Random(&randomContext);
    77 		char ch = (char) (number % (91 - 48)) + 48;
    78 		if (ch >= 58 && ch <= 64) {
    79 			ch = 65;
    80 		}
    81 		seed[counter] = ch;
    82 	}
    83 	seed[counter] = '\0';
    84 
    85 	return seed;
    86 }
    87 
    88 /**
    89  * Generates an execution key for the fuzzer.
    90  *
    91  * \param runSeed		The run seed to use
    92  * \param suiteName		The name of the test suite
    93  * \param testName		The name of the test
    94  * \param iteration		The iteration count
    95  *
    96  * \returns The generated execution key to initialize the fuzzer with.
    97  *
    98  */
    99 Uint64
   100 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
   101 {
   102 	SDLTest_Md5Context md5Context;
   103 	Uint64 *keys;
   104 	char iterationString[16];
   105 	Uint32 runSeedLength;
   106 	Uint32 suiteNameLength;
   107 	Uint32 testNameLength;
   108 	Uint32 iterationStringLength;
   109 	Uint32 entireStringLength;
   110 	char *buffer;
   111 
   112 	if (runSeed == NULL || strlen(runSeed)==0) {
   113 		SDLTest_LogError("Invalid runSeed string.");
   114 		return -1;
   115 	}
   116 
   117 	if (suiteName == NULL || strlen(suiteName)==0) {
   118 		SDLTest_LogError("Invalid suiteName string.");
   119 		return -1;
   120 	}
   121 
   122 	if (testName == NULL || strlen(testName)==0) {
   123 		SDLTest_LogError("Invalid testName string.");
   124 		return -1;
   125 	}
   126 
   127 	if (iteration <= 0) {
   128 		SDLTest_LogError("Invalid iteration count.");
   129 		return -1;
   130 	}
   131 
   132 	// Convert iteration number into a string
   133 	memset(iterationString, 0, sizeof(iterationString));
   134 	SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
   135 
   136 	// Combine the parameters into single string
   137 	runSeedLength = strlen(runSeed);
   138 	suiteNameLength = strlen(suiteName);
   139 	testNameLength = strlen(testName);
   140 	iterationStringLength = strlen(iterationString);
   141 	entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
   142 	buffer = (char *)SDL_malloc(entireStringLength);
   143 	if (buffer == NULL) {
   144 		SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
   145 		return 0;
   146 	}
   147 	SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
   148 
   149 	// Hash string and use half of the digest as 64bit exec key
   150 	SDLTest_Md5Init(&md5Context);
   151 	SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
   152 	SDLTest_Md5Final(&md5Context);
   153 	SDL_free(buffer);
   154 	keys = (Uint64 *)md5Context.digest;
   155 
   156 	return keys[0];
   157 }
   158 
   159 /**
   160  * \brief Set timeout handler for test.
   161  *
   162  * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   163  *
   164  * \param timeout Timeout interval in seconds.
   165  * \param callback Function that will be called after timeout has elapsed.
   166  * 
   167  * \return Timer id or -1 on failure.
   168  */
   169 SDL_TimerID
   170 SDLTest_SetTestTimeout(int timeout, void (*callback)())
   171 {
   172 	Uint32 timeoutInMilliseconds;
   173 	SDL_TimerID timerID;
   174 
   175 	if (callback == NULL) {
   176 		SDLTest_LogError("Timeout callback can't be NULL");
   177 		return -1;
   178 	}
   179 
   180 	if (timeout < 0) {
   181 		SDLTest_LogError("Timeout value must be bigger than zero.");
   182 		return -1;
   183 	}
   184 
   185 	/* Init SDL timer if not initialized before */
   186 	if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
   187 		if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
   188 			SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
   189 			return -1;
   190 		}
   191 	}
   192 
   193 	/* Set timer */
   194 	timeoutInMilliseconds = timeout * 1000;
   195 	timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
   196 	if (timerID == 0) {
   197 		SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
   198 		return -1;
   199 	}
   200 
   201 	return timerID;
   202 }
   203 
   204 void
   205 SDLTest_BailOut()
   206 {
   207 	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   208 	exit(TEST_ABORTED); // bail out from the test
   209 }
   210 
   211 /**
   212  * \brief Execute a test using the given execution key.
   213  *
   214  * \param testSuite Suite containing the test case.
   215  * \param testCase Case to execute.
   216  * \param execKey Execution key for the fuzzer.
   217  *
   218  * \returns Test case result.
   219  */
   220 int
   221 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
   222 {
   223 	SDL_TimerID timer = 0;
   224 	int testResult = 0;
   225 	int fuzzerCount;
   226 
   227 	if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
   228 	{
   229 		SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
   230 		return TEST_RESULT_SETUP_FAILURE;
   231 	}
   232 
   233 	if (!testCase->enabled)
   234 	{
   235 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped");
   236 		return TEST_RESULT_SKIPPED;
   237 	}
   238 
   239         // Initialize fuzzer
   240 	SDLTest_FuzzerInit(execKey);
   241 
   242 	// Reset assert tracker
   243 	SDLTest_ResetAssertSummary();
   244 
   245 	// Set timeout timer
   246 	timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
   247 
   248 	// Maybe run suite initalizer function
   249 	if (testSuite->testSetUp) {
   250 		testSuite->testSetUp(0x0);
   251 		if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
   252 			SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed");
   253 			return TEST_RESULT_SETUP_FAILURE;
   254 		}
   255 	}
   256 
   257 	// Run test case function
   258 	testCase->testCase(0x0);
   259 	testResult = SDLTest_AssertSummaryToTestResult();
   260 
   261 	// Maybe run suite cleanup function (ignore failed asserts)
   262 	if (testSuite->testTearDown) {
   263 		testSuite->testTearDown(0x0);
   264 	}
   265 
   266 	// Cancel timeout timer
   267 	if (timer) {
   268 		SDL_RemoveTimer(timer);
   269 	}
   270 
   271 	// Report on asserts and fuzzer usage
   272 	fuzzerCount = SDLTest_GetFuzzerInvocationCount();
   273 	if (fuzzerCount > 0) {
   274 		SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
   275 	}
   276 	SDLTest_LogAssertSummary();
   277 
   278 	return testResult;
   279 }
   280 
   281 /* Prints summary of all suites/tests contained in the given reference */
   282 void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
   283 {
   284 	int suiteCounter;
   285 	int testCounter;
   286 	SDLTest_TestSuiteReference *testSuite;
   287 	SDLTest_TestCaseReference *testCase;
   288 
   289 	// Loop over all suites
   290 	suiteCounter = 0;
   291 	while(&testSuites[suiteCounter]) {
   292 		testSuite=&testSuites[suiteCounter];
   293 		suiteCounter++;
   294 		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   295 			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   296 
   297 		// Loop over all test cases
   298 		testCounter = 0;
   299 		while(testSuite->testCases[testCounter])
   300 		{
   301 			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   302 			testCounter++;
   303 			SDLTest_Log("  Test Case %i - %s: %s", testCounter, 
   304 				(testCase->name) ? testCase->name : SDLTest_InvalidNameFormat, 
   305 				(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   306 		}
   307 	}
   308 }
   309 
   310 /* Gets a timer value in seconds */
   311 float GetClock()
   312 {
   313 	float currentClock = (float)clock();
   314 	return currentClock / (float)CLOCKS_PER_SEC;
   315 }
   316 
   317 /**
   318  * \brief Execute a test suite using the given run seend and execution key.
   319  *
   320  * \param testSuites Suites containing the test case.
   321  * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   322  * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   323  * \param testIterations Number of iterations to run each test case.
   324  *
   325  * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   326  */
   327 int
   328 SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, Uint64 userExecKey, int testIterations)
   329 {
   330 	int suiteCounter;
   331 	int testCounter;
   332 	int iterationCounter;
   333 	SDLTest_TestSuiteReference *testSuite;
   334 	SDLTest_TestCaseReference *testCase;
   335 	char *runSeed = NULL;
   336 	char *currentSuiteName;
   337 	char *currentTestName;
   338 	Uint64 execKey;
   339 	float runStartSeconds;
   340 	float suiteStartSeconds;
   341 	float testStartSeconds;
   342 	float runEndSeconds;
   343 	float suiteEndSeconds;
   344 	float testEndSeconds;
   345 	float runtime;
   346 	int testResult = 0;
   347 	int runResult = 0;
   348 	Uint32 totalTestFailedCount = 0;
   349 	Uint32 totalTestPassedCount = 0;
   350 	Uint32 totalTestSkippedCount = 0;
   351 	Uint32 testFailedCount = 0;
   352 	Uint32 testPassedCount = 0;
   353 	Uint32 testSkippedCount = 0;
   354 	Uint32 countSum = 0;
   355 	char *logFormat = (char *)SDLTest_LogSummaryFormat;
   356 
   357 	// Sanitize test iterations
   358 	if (testIterations < 1) {
   359 		testIterations = 1;
   360 	}
   361 
   362 	// Generate run see if we don't have one already
   363 	if (userRunSeed == NULL || strlen(userRunSeed) == 0) {
   364 		runSeed = SDLTest_GenerateRunSeed(16);
   365 		if (runSeed == NULL) {
   366 			SDLTest_LogError("Generating a random seed failed");
   367 			return 2;
   368 		}
   369 	} else {
   370 		runSeed = userRunSeed;
   371 	}
   372 
   373 	// Reset per-run counters
   374 	totalTestFailedCount = 0;
   375 	totalTestPassedCount = 0;
   376 	totalTestSkippedCount = 0;
   377 
   378 	// Take time - run start
   379 	runStartSeconds = GetClock();
   380 
   381 	// Log run with fuzzer parameters
   382 	SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
   383 
   384 	// Loop over all suites
   385 	suiteCounter = 0;
   386 	while(testSuites[suiteCounter]) {
   387 		testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   388 		suiteCounter++;
   389 
   390 		// Reset per-suite counters
   391 		testFailedCount = 0;
   392 		testPassedCount = 0;
   393 		testSkippedCount = 0;
   394 
   395 		// Take time - suite start
   396 		suiteStartSeconds = GetClock();
   397 
   398 		// Log suite started
   399 		currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   400 		SDLTest_Log("===== Test Suite %i: '%s' started\n", 
   401 			suiteCounter, 
   402 			currentSuiteName);
   403 
   404 		// Loop over all test cases
   405 		testCounter = 0;
   406 		while(testSuite->testCases[testCounter])
   407 		{
   408 			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   409 			testCounter++;
   410 			
   411 			// Take time - test start
   412 			testStartSeconds = GetClock();
   413 
   414 			// Log test started
   415 			currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   416 			SDLTest_Log("----- Test Case %i.%i: '%s' started",
   417 			        suiteCounter,
   418 				testCounter, 
   419 				currentTestName);
   420 			if (testCase->description != NULL && strlen(testCase->description)>0) {
   421 				SDLTest_Log("Test Description: '%s'", 
   422 					(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   423 			}
   424 			
   425 			// Loop over all iterations
   426 			iterationCounter = 0;
   427 			while(iterationCounter < testIterations)
   428 			{
   429 				iterationCounter++;
   430 
   431 				if (userExecKey != 0) {
   432 					execKey = userExecKey;
   433 				} else {
   434 					execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
   435 				}
   436 
   437 				SDLTest_Log("Test Iteration %i: execKey %llu", iterationCounter, execKey);
   438 				testResult = SDLTest_RunTest(testSuite, testCase, execKey);
   439 
   440 				if (testResult == TEST_RESULT_PASSED) {
   441 					testPassedCount++;
   442 					totalTestPassedCount++;
   443 				} else if (testResult == TEST_RESULT_SKIPPED) {
   444 					testSkippedCount++;
   445 					totalTestSkippedCount++;
   446 				} else {
   447 					testFailedCount++;
   448 					totalTestFailedCount++;
   449 				}
   450 			}
   451 
   452 			// Take time - test end
   453 			testEndSeconds = GetClock();
   454 			runtime = testEndSeconds - testStartSeconds;
   455 			if (runtime < 0.0f) runtime = 0.0f;
   456 
   457 			if (testIterations > 1) {
   458         			// Log test runtime
   459 	        		SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
   460 	        		SDLTest_Log("Test runtime: %.5f sec", runtime / (float)testIterations);
   461                         } else {
   462         			// Log test runtime
   463 	        		SDLTest_Log("Test runtime: %.1f sec", runtime);
   464                         }
   465 
   466 			// Log final test result
   467 			switch (testResult) {
   468 				case TEST_RESULT_PASSED:
   469 					SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   470 					break;
   471 				case TEST_RESULT_FAILED:
   472 					SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
   473 					break;
   474 				case TEST_RESULT_NO_ASSERT:
   475 					SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
   476 					break;
   477 			}
   478 		}
   479 
   480 		// Take time - suite end
   481 		suiteEndSeconds = GetClock();
   482 		runtime = suiteEndSeconds - suiteStartSeconds;
   483 		if (runtime < 0.0f) runtime = 0.0f;
   484 
   485 		// Log suite runtime
   486 		SDLTest_Log("Suite runtime: %.1f sec", runtime);
   487 
   488 		// Log summary and final Suite result
   489 		countSum = testPassedCount + testFailedCount + testSkippedCount;
   490 		if (testFailedCount == 0)
   491 		{
   492 			SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   493 			SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
   494 		} 
   495 		else 
   496 		{
   497 			SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   498 			SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
   499 		}
   500 	}
   501 
   502 	// Take time - run end
   503 	runEndSeconds = GetClock();
   504 	runtime = runEndSeconds - runStartSeconds;
   505 	if (runtime < 0.0f) runtime = 0.0f;
   506 
   507 	// Log total runtime
   508 	SDLTest_Log("Total runtime: %.1f sec", runtime);
   509 
   510 	// Log summary and final run result
   511 	countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   512 	if (testFailedCount == 0)
   513 	{
   514 		runResult = 0;
   515 		SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   516 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
   517 	} 
   518 	else 
   519 	{
   520 		runResult = 1;
   521 		SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   522 		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
   523 	}
   524 
   525 	SDLTest_Log("Exit code: %d", runResult);	
   526 	return runResult;
   527 }