src/test/SDL_test_harness.c
author Andreas Schiffler
Fri, 14 Dec 2012 23:05:34 -0800
changeset 6756 398073b195bb
parent 6727 1b5280cd5885
child 6757 9935f71c8c81
permissions -rw-r--r--
Refactor/fix test lib harness, assert and log component; add harness driver; port platform suite from GSOC code
     1 /*
     2   Simple DirectMedia Layer
     3   Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
     4 
     5   This software is provided 'as-is', without any express or implied
     6   warranty.  In no event will the authors be held liable for any damages
     7   arising from the use of this software.
     8 
     9   Permission is granted to anyone to use this software for any purpose,
    10   including commercial applications, and to alter it and redistribute it
    11   freely, subject to the following restrictions:
    12 
    13   1. The origin of this software must not be misrepresented; you must not
    14      claim that you wrote the original software. If you use this software
    15      in a product, an acknowledgment in the product documentation would be
    16      appreciated but is not required.
    17   2. Altered source versions must be plainly marked as such, and must not be
    18      misrepresented as being the original software.
    19   3. This notice may not be removed or altered from any source distribution.
    20 */
    21 
    22 #include "SDL_config.h"
    23 
    24 #include "SDL_test.h"
    25 
    26 #include <stdio.h>
    27 #include <stdlib.h>
    28 #include <string.h>
    29 #include <time.h>
    30 
    31 /* Invalid test name/description message format */
    32 const char *SDLTest_InvalidNameFormat = "(Invalid)";
    33 
    34 /* Log summary message format */
    35 const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d";
    36 
    37 /* Final result message format */
    38 const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n";
    39 
    40 /*! \brief Timeout for single test case execution */
    41 static Uint32 SDLTest_TestCaseTimeout = 3600;
    42 
    43 /**
    44  * Generates a random run seed string for the harness. The generated seed
    45  * will contain alphanumeric characters (0-9A-Z).
    46  *
    47  * Note: The returned string needs to be deallocated by the caller.
    48  *
    49  * \param length The length of the seed string to generate
    50  *
    51  * \returns The generated seed string
    52  */
    53 char *
    54 SDLTest_GenerateRunSeed(const int length)
    55 {
    56 	char *seed = NULL;
    57 	SDLTest_RandomContext randomContext;
    58 	int counter;
    59 
    60 	// Sanity check input
    61 	if (length <= 0) {
    62 		SDLTest_LogError("The length of the harness seed must be >0.");
    63 		return NULL;
    64 	}
    65 
    66 	// Allocate output buffer
    67 	seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    68 	if (seed == NULL) {
    69 		SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    70 		return NULL;
    71 	}
    72 
    73 	// Generate a random string of alphanumeric characters
    74 	SDLTest_RandomInitTime(&randomContext);
    75 	for (counter = 0; counter < length - 1; ++counter) {
    76 		unsigned int number = SDLTest_Random(&randomContext);
    77 		char ch = (char) (number % (91 - 48)) + 48;
    78 		if (ch >= 58 && ch <= 64) {
    79 			ch = 65;
    80 		}
    81 		seed[counter] = ch;
    82 	}
    83 	seed[counter] = '\0';
    84 
    85 	return seed;
    86 }
    87 
    88 /**
    89  * Generates an execution key for the fuzzer.
    90  *
    91  * \param runSeed		The run seed to use
    92  * \param suiteName		The name of the test suite
    93  * \param testName		The name of the test
    94  * \param iteration		The iteration count
    95  *
    96  * \returns The generated execution key to initialize the fuzzer with.
    97  *
    98  */
    99 Uint64
   100 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
   101 {
   102 	SDLTest_Md5Context md5Context;
   103 	Uint64 *keys;
   104 	char iterationString[16];
   105 	Uint32 runSeedLength;
   106 	Uint32 suiteNameLength;
   107 	Uint32 testNameLength;
   108 	Uint32 iterationStringLength;
   109 	Uint32 entireStringLength;
   110 	char *buffer;
   111 
   112 	if (runSeed == NULL || strlen(runSeed)==0) {
   113 		SDLTest_LogError("Invalid runSeed string.");
   114 		return -1;
   115 	}
   116 
   117 	if (suiteName == NULL || strlen(suiteName)==0) {
   118 		SDLTest_LogError("Invalid suiteName string.");
   119 		return -1;
   120 	}
   121 
   122 	if (testName == NULL || strlen(testName)==0) {
   123 		SDLTest_LogError("Invalid testName string.");
   124 		return -1;
   125 	}
   126 
   127 	if (iteration <= 0) {
   128 		SDLTest_LogError("Invalid iteration count.");
   129 		return -1;
   130 	}
   131 
   132 	// Convert iteration number into a string
   133 	memset(iterationString, 0, sizeof(iterationString));
   134 	SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
   135 
   136 	// Combine the parameters into single string
   137 	runSeedLength = strlen(runSeed);
   138 	suiteNameLength = strlen(suiteName);
   139 	testNameLength = strlen(testName);
   140 	iterationStringLength = strlen(iterationString);
   141 	entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
   142 	buffer = (char *)SDL_malloc(entireStringLength);
   143 	if (buffer == NULL) {
   144 		SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
   145 		return 0;
   146 	}
   147 	SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
   148 
   149 	// Hash string and use half of the digest as 64bit exec key
   150 	SDLTest_Md5Init(&md5Context);
   151 	SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
   152 	SDLTest_Md5Final(&md5Context);
   153 	SDL_free(buffer);
   154 	keys = (Uint64 *)md5Context.digest;
   155 
   156 	return keys[0];
   157 }
   158 
   159 /**
   160  * \brief Set timeout handler for test.
   161  *
   162  * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   163  *
   164  * \param timeout Timeout interval in seconds.
   165  * \param callback Function that will be called after timeout has elapsed.
   166  * 
   167  * \return Timer id or -1 on failure.
   168  */
   169 SDL_TimerID
   170 SDLTest_SetTestTimeout(int timeout, void (*callback)())
   171 {
   172 	Uint32 timeoutInMilliseconds;
   173 	SDL_TimerID timerID;
   174 
   175 	if (callback == NULL) {
   176 		SDLTest_LogError("Timeout callback can't be NULL");
   177 		return -1;
   178 	}
   179 
   180 	if (timeout < 0) {
   181 		SDLTest_LogError("Timeout value must be bigger than zero.");
   182 		return -1;
   183 	}
   184 
   185 	/* Init SDL timer if not initialized before */
   186 	if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
   187 		if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
   188 			SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
   189 			return -1;
   190 		}
   191 	}
   192 
   193 	/* Set timer */
   194 	timeoutInMilliseconds = timeout * 1000;
   195 	timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
   196 	if (timerID == 0) {
   197 		SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
   198 		return -1;
   199 	}
   200 
   201 	return timerID;
   202 }
   203 
   204 void
   205 SDLTest_BailOut()
   206 {
   207 	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   208 	exit(TEST_ABORTED); // bail out from the test
   209 }
   210 
   211 /**
   212  * \brief Execute a test using the given execution key.
   213  *
   214  * \param testSuite Suite containing the test case.
   215  * \param testCase Case to execute.
   216  * \param execKey Execution key for the fuzzer.
   217  *
   218  * \returns Test case result.
   219  */
   220 int
   221 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
   222 {
   223 	SDL_TimerID timer = 0;
   224 	int testResult = 0;
   225 
   226 	if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
   227 	{
   228 		SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
   229 		return TEST_RESULT_SETUP_FAILURE;
   230 	}
   231 
   232 	if (!testCase->enabled)
   233 	{
   234 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped");
   235 		return TEST_RESULT_SKIPPED;
   236 	}
   237 
   238     // Initialize fuzzer
   239 	SDLTest_FuzzerInit(execKey);
   240 
   241 	// Reset assert tracker
   242 	SDLTest_ResetAssertSummary();
   243 
   244 	// Set timeout timer
   245 	timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
   246 
   247 	// Maybe run suite initalizer function
   248 	if (testSuite->testSetUp) {
   249 		testSuite->testSetUp(0x0);
   250 		if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
   251 			SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed");
   252 			return TEST_RESULT_SETUP_FAILURE;
   253 		}
   254 	}
   255 
   256 	// Run test case function
   257 	testCase->testCase(0x0);
   258 	testResult = SDLTest_AssertSummaryToTestResult();
   259 
   260 	// Maybe run suite cleanup function (ignore failed asserts)
   261 	if (testSuite->testTearDown) {
   262 		testSuite->testTearDown(0x0);
   263 	}
   264 
   265 	// Cancel timeout timer
   266 	if (timer) {
   267 		SDL_RemoveTimer(timer);
   268 	}
   269 
   270 	// Report on asserts and fuzzer usage
   271 	SDLTest_Log("Fuzzer invocations: %d", SDLTest_GetFuzzerInvocationCount());
   272 	SDLTest_LogAssertSummary();
   273 
   274 	return testResult;
   275 }
   276 
   277 /* Prints summary of all suites/tests contained in the given reference */
   278 void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
   279 {
   280 	int suiteCounter;
   281 	int testCounter;
   282 	SDLTest_TestSuiteReference *testSuite;
   283 	SDLTest_TestCaseReference *testCase;
   284 
   285 	// Loop over all suites
   286 	suiteCounter = 0;
   287 	while(&testSuites[suiteCounter]) {
   288 		testSuite=&testSuites[suiteCounter];
   289 		suiteCounter++;
   290 		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   291 			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   292 
   293 		// Loop over all test cases
   294 		testCounter = 0;
   295 		while(testSuite->testCases[testCounter])
   296 		{
   297 			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   298 			testCounter++;
   299 			SDLTest_Log("  Test Case %i - %s: %s", testCounter, 
   300 				(testCase->name) ? testCase->name : SDLTest_InvalidNameFormat, 
   301 				(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   302 		}
   303 	}
   304 }
   305 
   306 /* Gets a timer value in seconds */
   307 float GetClock()
   308 {
   309 	float currentClock = (float)clock();
   310 	return currentClock / (float)CLOCKS_PER_SEC;
   311 }
   312 
   313 /**
   314  * \brief Execute a test suite using the given run seend and execution key.
   315  *
   316  * \param testSuites Suites containing the test case.
   317  * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   318  * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   319  * \param testIterations Number of iterations to run each test case.
   320  *
   321  * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   322  */
   323 int
   324 SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, Uint64 userExecKey, int testIterations)
   325 {
   326 	int suiteCounter;
   327 	int testCounter;
   328 	int iterationCounter;
   329 	SDLTest_TestSuiteReference *testSuite;
   330 	SDLTest_TestCaseReference *testCase;
   331 	char *runSeed = NULL;
   332 	char *currentSuiteName;
   333 	char *currentTestName;
   334 	Uint64 execKey;
   335 	float runStartSeconds;
   336 	float suiteStartSeconds;
   337 	float testStartSeconds;
   338 	float runEndSeconds;
   339 	float suiteEndSeconds;
   340 	float testEndSeconds;
   341 	int testResult = 0;
   342 	int runResult = 0;
   343 	Uint32 totalTestFailedCount = 0;
   344 	Uint32 totalTestPassedCount = 0;
   345 	Uint32 totalTestSkippedCount = 0;
   346 	Uint32 testFailedCount = 0;
   347 	Uint32 testPassedCount = 0;
   348 	Uint32 testSkippedCount = 0;
   349 	Uint32 countSum = 0;
   350 	char *logFormat = (char *)SDLTest_LogSummaryFormat;
   351 
   352 	// Sanitize test iterations
   353 	if (testIterations < 1) {
   354 		testIterations = 1;
   355 	}
   356 
   357 	// Generate run see if we don't have one already
   358 	if (userRunSeed == NULL || strlen(userRunSeed) == 0) {
   359 		runSeed = SDLTest_GenerateRunSeed(16);
   360 		if (runSeed == NULL) {
   361 			SDLTest_LogError("Generating a random seed failed");
   362 			return 2;
   363 		}
   364 	}
   365 
   366 	// Reset per-run counters
   367 	totalTestFailedCount = 0;
   368 	totalTestPassedCount = 0;
   369 	totalTestSkippedCount = 0;
   370 
   371 	// Take time - run start
   372 	runStartSeconds = GetClock();
   373 
   374 	// Log run with fuzzer parameters
   375 	SDLTest_Log("::::: Test Run '%s' started\n", runSeed);
   376 
   377 	// Loop over all suites
   378 	suiteCounter = 0;
   379 	while(testSuites[suiteCounter]) {
   380 		testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
   381 		suiteCounter++;
   382 
   383 		// Reset per-suite counters
   384 		testFailedCount = 0;
   385 		testPassedCount = 0;
   386 		testSkippedCount = 0;
   387 
   388 		// Take time - suite start
   389 		suiteStartSeconds = GetClock();
   390 
   391 		// Log suite started
   392 		currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
   393 		SDLTest_Log("===== Test Suite %i: %s started\n", 
   394 			suiteCounter, 
   395 			currentSuiteName);
   396 
   397 		// Loop over all test cases
   398 		testCounter = 0;
   399 		while(testSuite->testCases[testCounter])
   400 		{
   401 			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   402 			testCounter++;
   403 			
   404 			// Take time - test start
   405 			testStartSeconds = GetClock();
   406 
   407 			// Log test started
   408 			currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
   409 			SDLTest_Log("----- Test Case %i: %s started", 
   410 				testCounter, 
   411 				currentTestName);
   412 			SDLTest_Log("Test Description: %s", 
   413 				(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
   414 
   415 			// Loop over all iterations
   416 			iterationCounter = 0;
   417 			while(iterationCounter < testIterations)
   418 			{
   419 				iterationCounter++;
   420 
   421 				if(userExecKey != 0) {
   422 					execKey = userExecKey;
   423 				} else {
   424 					execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
   425 				}
   426 
   427 				SDLTest_Log("Test Iteration %i: execKey %d", iterationCounter, execKey);
   428 				testResult = SDLTest_RunTest(testSuite, testCase, execKey);
   429 
   430 				if (testResult == TEST_RESULT_PASSED) {
   431 					testPassedCount++;
   432 					totalTestPassedCount++;
   433 				} else if (testResult == TEST_RESULT_SKIPPED) {
   434 					testSkippedCount++;
   435 					totalTestSkippedCount++;
   436 				} else {
   437 					testFailedCount++;
   438 					totalTestFailedCount++;
   439 				}
   440 			}
   441 
   442 			// Take time - test end
   443 			testEndSeconds = GetClock();
   444 
   445 			SDLTest_Log("Test Case %s ended", currentTestName);
   446 
   447 			// Log test runtime
   448 			SDLTest_Log("Test runtime: %.1f sec", testEndSeconds - testStartSeconds);
   449 
   450 			// Log final test result
   451 			switch (testResult) {
   452 				case TEST_RESULT_PASSED:
   453 					SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
   454 					break;
   455 				case TEST_RESULT_FAILED:
   456 					SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed");
   457 					break;
   458 				case TEST_RESULT_NO_ASSERT:
   459 					SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts");
   460 					break;
   461 			}
   462 		}
   463 
   464 		// Take time - suite end
   465 		suiteEndSeconds = GetClock();
   466 
   467 		// Log suite runtime
   468 		SDLTest_Log("Suite runtime: %.1f sec", suiteEndSeconds - suiteStartSeconds);
   469 
   470 		// Log summary and final Suite result
   471 	    countSum = testPassedCount + testFailedCount + testSkippedCount;
   472 		if (testFailedCount == 0)
   473 		{
   474 			SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   475 			SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed");
   476 		} 
   477 		else 
   478 		{
   479 			SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
   480 			SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed");
   481 		}
   482 	}
   483 
   484 	// Take time - run end
   485 	runEndSeconds = GetClock();
   486 
   487 	// Log total runtime
   488 	SDLTest_Log("Total runtime: %.1f sec", runEndSeconds - runStartSeconds);
   489 
   490 	// Log summary and final run result
   491 	countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
   492 	if (testFailedCount == 0)
   493 	{
   494 		runResult = 0;
   495 		SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   496 		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run", runSeed, "Passed");
   497 	} 
   498 	else 
   499 	{
   500 		runResult = 1;
   501 		SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
   502 		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run", runSeed, "Failed");
   503 	}
   504 
   505 	return runResult;
   506 }