src/test/SDL_test_harness.c
author Andreas Schiffler
Sun, 09 Dec 2012 17:56:19 -0800
changeset 6727 1b5280cd5885
parent 6721 53b71f45a53a
child 6756 398073b195bb
permissions -rw-r--r--
Added existing common.c/.h functions to test lib; minor assert refactoring
     1 /*
     2   Simple DirectMedia Layer
     3   Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
     4 
     5   This software is provided 'as-is', without any express or implied
     6   warranty.  In no event will the authors be held liable for any damages
     7   arising from the use of this software.
     8 
     9   Permission is granted to anyone to use this software for any purpose,
    10   including commercial applications, and to alter it and redistribute it
    11   freely, subject to the following restrictions:
    12 
    13   1. The origin of this software must not be misrepresented; you must not
    14      claim that you wrote the original software. If you use this software
    15      in a product, an acknowledgment in the product documentation would be
    16      appreciated but is not required.
    17   2. Altered source versions must be plainly marked as such, and must not be
    18      misrepresented as being the original software.
    19   3. This notice may not be removed or altered from any source distribution.
    20 */
    21 
    22 #include "SDL_config.h"
    23 
    24 #include "SDL_test.h"
    25 
    26 #include <stdio.h>
    27 #include <stdlib.h>
    28 #include <string.h>
    29 #include <time.h>
    30 
    31 /* Assert check message format */
    32 const char *SDLTest_TestCheckFmt = "Test '%s': %s";
    33 
    34 /* Invalid test name/description message format */
    35 const char *SDLTest_InvalidNameFmt = "(Invalid)";
    36 
    37 /*! \brief Timeout for single test case execution */
    38 static Uint32 SDLTest_TestCaseTimeout = 3600;
    39 
    40 /**
    41  * Generates a random run seed string for the harness. The generated seed
    42  * will contain alphanumeric characters (0-9A-Z).
    43  *
    44  * Note: The returned string needs to be deallocated by the caller.
    45  *
    46  * \param length The length of the seed string to generate
    47  *
    48  * \returns The generated seed string
    49  */
    50 char *
    51 SDLTest_GenerateRunSeed(const int length)
    52 {
    53 	char *seed = NULL;
    54 	SDLTest_RandomContext randomContext;
    55 	int counter;
    56 
    57 	// Sanity check input
    58 	if (length <= 0) {
    59 		SDLTest_LogError("The length of the harness seed must be >0.");
    60 		return NULL;
    61 	}
    62 
    63 	// Allocate output buffer
    64 	seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    65 	if (seed == NULL) {
    66 		SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
    67 		return NULL;
    68 	}
    69 
    70 	// Generate a random string of alphanumeric characters
    71 	SDLTest_RandomInitTime(&randomContext);
    72 	for (counter = 0; counter < length - 1; ++counter) {
    73 		unsigned int number = SDLTest_Random(&randomContext);
    74 		char ch = (char) (number % (91 - 48)) + 48;
    75 		if (ch >= 58 && ch <= 64) {
    76 			ch = 65;
    77 		}
    78 		seed[counter] = ch;
    79 	}
    80 	seed[counter] = '\0';
    81 
    82 	return seed;
    83 }
    84 
    85 /**
    86  * Generates an execution key for the fuzzer.
    87  *
    88  * \param runSeed		The run seed to use
    89  * \param suiteName		The name of the test suite
    90  * \param testName		The name of the test
    91  * \param iteration		The iteration count
    92  *
    93  * \returns The generated execution key to initialize the fuzzer with.
    94  *
    95  */
    96 Uint64
    97 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
    98 {
    99 	SDLTest_Md5Context md5Context;
   100 	Uint64 *keys;
   101 	char iterationString[16];
   102 	Uint32 runSeedLength;
   103 	Uint32 suiteNameLength;
   104 	Uint32 testNameLength;
   105 	Uint32 iterationStringLength;
   106 	Uint32 entireStringLength;
   107 	char *buffer;
   108 
   109 	if (runSeed == NULL || strlen(runSeed)==0) {
   110 		SDLTest_LogError("Invalid runSeed string.");
   111 		return -1;
   112 	}
   113 
   114 	if (suiteName == NULL || strlen(suiteName)==0) {
   115 		SDLTest_LogError("Invalid suiteName string.");
   116 		return -1;
   117 	}
   118 
   119 	if (testName == NULL || strlen(testName)==0) {
   120 		SDLTest_LogError("Invalid testName string.");
   121 		return -1;
   122 	}
   123 
   124 	if (iteration <= 0) {
   125 		SDLTest_LogError("Invalid iteration count.");
   126 		return -1;
   127 	}
   128 
   129 	// Convert iteration number into a string
   130 	memset(iterationString, 0, sizeof(iterationString));
   131 	SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
   132 
   133 	// Combine the parameters into single string
   134 	runSeedLength = strlen(runSeed);
   135 	suiteNameLength = strlen(suiteName);
   136 	testNameLength = strlen(testName);
   137 	iterationStringLength = strlen(iterationString);
   138 	entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
   139 	buffer = (char *)SDL_malloc(entireStringLength);
   140 	if (buffer == NULL) {
   141 		SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
   142 		return 0;
   143 	}
   144 	SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
   145 
   146 	// Hash string and use half of the digest as 64bit exec key
   147 	SDLTest_Md5Init(&md5Context);
   148 	SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
   149 	SDLTest_Md5Final(&md5Context);
   150 	SDL_free(buffer);
   151 	keys = (Uint64 *)md5Context.digest;
   152 
   153 	return keys[0];
   154 }
   155 
   156 /**
   157  * \brief Set timeout handler for test.
   158  *
   159  * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
   160  *
   161  * \param timeout Timeout interval in seconds.
   162  * \param callback Function that will be called after timeout has elapsed.
   163  * 
   164  * \return Timer id or -1 on failure.
   165  */
   166 SDL_TimerID
   167 SDLTest_SetTestTimeout(int timeout, void (*callback)())
   168 {
   169 	Uint32 timeoutInMilliseconds;
   170 	SDL_TimerID timerID;
   171 
   172 	if (callback == NULL) {
   173 		SDLTest_LogError("Timeout callback can't be NULL");
   174 		return -1;
   175 	}
   176 
   177 	if (timeout < 0) {
   178 		SDLTest_LogError("Timeout value must be bigger than zero.");
   179 		return -1;
   180 	}
   181 
   182 	/* Init SDL timer if not initialized before */
   183 	if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
   184 		if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
   185 			SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
   186 			return -1;
   187 		}
   188 	}
   189 
   190 	/* Set timer */
   191 	timeoutInMilliseconds = timeout * 1000;
   192 	timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
   193 	if (timerID == 0) {
   194 		SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
   195 		return -1;
   196 	}
   197 
   198 	return timerID;
   199 }
   200 
   201 void
   202 SDLTest_BailOut()
   203 {
   204 	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
   205 	exit(TEST_ABORTED); // bail out from the test
   206 }
   207 
   208 /**
   209  * \brief Execute a test using the given execution key.
   210  *
   211  * \param testSuite Suite containing the test case.
   212  * \param testCase Case to execute.
   213  * \param execKey Execution key for the fuzzer.
   214  *
   215  * \returns Test case result.
   216  */
   217 int
   218 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
   219 {
   220 	SDL_TimerID timer = 0;
   221 	int testResult = 0;
   222 
   223 	if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
   224 	{
   225 		SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
   226 		return TEST_RESULT_SETUP_FAILURE;
   227 	}
   228 
   229 	if (!testCase->enabled)
   230 	{
   231 		SDLTest_Log((char *)SDLTest_TestCheckFmt, testCase->name, "Skipped");
   232 		return TEST_RESULT_SKIPPED;
   233 	}
   234 
   235     // Initialize fuzzer
   236 	SDLTest_FuzzerInit(execKey);
   237 
   238 	// Reset assert tracker
   239 	SDLTest_ResetAssertSummary();
   240 
   241 	// Set timeout timer
   242 	timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
   243 
   244 	// Maybe run suite initalizer function
   245 	if (testSuite->testSetUp) {
   246 		testSuite->testSetUp(0x0);
   247 		if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
   248 			SDLTest_LogError((char *)SDLTest_TestCheckFmt, testSuite->name, "Failed");
   249 			return TEST_RESULT_SETUP_FAILURE;
   250 		}
   251 	}
   252 
   253 	// Run test case function
   254 	testCase->testCase(0x0);
   255 	testResult = SDLTest_AssertSummaryToTestResult();
   256 
   257 	// Maybe run suite cleanup function (ignore failed asserts)
   258 	if (testSuite->testTearDown) {
   259 		testSuite->testTearDown(0x0);
   260 	}
   261 
   262 	// Cancel timeout timer
   263 	if (timer) {
   264 		SDL_RemoveTimer(timer);
   265 	}
   266 
   267 	// Report on asserts and fuzzer usage
   268 	SDLTest_Log("Fuzzer invocations: %d", SDLTest_GetFuzzerInvocationCount());
   269 	SDLTest_LogAssertSummary();
   270 
   271 	// Analyze assert count to determine final test case result
   272 	switch (testResult) {
   273 		case TEST_RESULT_PASSED:
   274 			SDLTest_LogError((char *)SDLTest_TestCheckFmt, testCase->name, "Failed");
   275 		case TEST_RESULT_FAILED:
   276 			SDLTest_Log((char *)SDLTest_TestCheckFmt, testCase->name, "Passed");
   277 		case TEST_RESULT_NO_ASSERT:
   278 			SDLTest_LogError((char *)SDLTest_TestCheckFmt, testCase->name, "No Asserts");
   279 	}
   280 
   281 	return testResult;
   282 }
   283 
   284 /* Prints summary of all suites/tests contained in the given reference */
   285 void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
   286 {
   287 	int suiteCounter;
   288 	int testCounter;
   289 	SDLTest_TestSuiteReference *testSuite;
   290 	SDLTest_TestCaseReference *testCase;
   291 
   292 	// Loop over all suites
   293 	suiteCounter = 0;
   294 	while(&testSuites[suiteCounter]) {
   295 		testSuite=&testSuites[suiteCounter];
   296 		suiteCounter++;
   297 		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   298 			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFmt);
   299 
   300 		// Loop over all test cases
   301 		testCounter = 0;
   302 		while(testSuite->testCases[testCounter])
   303 		{
   304 			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   305 			testCounter++;
   306 			SDLTest_Log("  Test Case %i - %s: %s", testCounter, 
   307 				(testCase->name) ? testCase->name : SDLTest_InvalidNameFmt, 
   308 				(testCase->description) ? testCase->description : SDLTest_InvalidNameFmt);
   309 		}
   310 	}
   311 }
   312 
   313 
   314 /**
   315  * \brief Execute a test using the given execution key.
   316  *
   317  * \param testSuites Suites containing the test case.
   318  * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
   319  * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
   320  * \param testIterations Number of iterations to run each test case.
   321  *
   322  * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
   323  */
   324 int
   325 SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites, char *userRunSeed, Uint64 userExecKey, int testIterations)
   326 {
   327 	int suiteCounter;
   328 	int testCounter;
   329 	int iterationCounter;
   330 	SDLTest_TestSuiteReference *testSuite;
   331 	SDLTest_TestCaseReference *testCase;
   332 	char *runSeed = NULL;
   333 	Uint64 execKey;
   334 	Uint32 runStartTicks;
   335 	time_t runStartTimestamp;
   336 	Uint32 suiteStartTicks;
   337 	time_t suiteStartTimestamp;
   338 	Uint32 testStartTicks;
   339 	time_t testStartTimestamp;
   340 	Uint32 runEndTicks;
   341 	time_t runEndTimestamp;
   342 	Uint32 suiteEndTicks;
   343 	time_t suiteEndTimestamp;
   344 	Uint32 testEndTicks;
   345 	time_t testEndTimestamp;
   346 	int testResult;
   347 	int totalTestFailedCount, totalTestPassedCount, totalTestSkippedCount;
   348 	int testFailedCount, testPassedCount, testSkippedCount;
   349 
   350 	// Sanitize test iterations
   351 	if (testIterations < 1) {
   352 		testIterations = 1;
   353 	}
   354 
   355 	// Generate run see if we don't have one already
   356 	if (userRunSeed == NULL || strlen(userRunSeed) == 0) {
   357 		runSeed = SDLTest_GenerateRunSeed(16);
   358 		if (runSeed == NULL) {
   359 			SDLTest_LogError("Generating a random run seed failed");
   360 			return 2;
   361 		}
   362 	}
   363 
   364 	// Reset per-run counters
   365 	totalTestFailedCount = totalTestPassedCount = totalTestSkippedCount = 0;
   366 
   367 	// Take time - run start
   368 	runStartTicks = SDL_GetTicks();
   369 	runStartTimestamp = time(0);
   370 
   371 	// TODO log run started
   372 
   373 	// Loop over all suites
   374 	suiteCounter = 0;
   375 	while(&testSuites[suiteCounter]) {
   376 		testSuite=&testSuites[suiteCounter];
   377 		suiteCounter++;
   378 
   379 		// Reset per-suite counters
   380 		testFailedCount = testPassedCount = testSkippedCount = 0;
   381 
   382 		// Take time - suite start
   383 		suiteStartTicks = SDL_GetTicks();
   384 		suiteStartTimestamp = time(0);
   385 
   386 		// TODO log suite started
   387 		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
   388 			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFmt);
   389 
   390 		// Loop over all test cases
   391 		testCounter = 0;
   392 		while(testSuite->testCases[testCounter])
   393 		{
   394 			testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
   395 			testCounter++;
   396 			
   397 			// Take time - test start
   398 			testStartTicks = SDL_GetTicks();
   399 			testStartTimestamp = time(0);
   400 
   401 			// TODO log test started
   402 			SDLTest_Log("Test Case %i - %s: %s", testCounter, 
   403 				(testCase->name) ? testCase->name : SDLTest_InvalidNameFmt, 
   404 				(testCase->description) ? testCase->description : SDLTest_InvalidNameFmt);
   405 
   406 			// Loop over all iterations
   407 			iterationCounter = 0;
   408 			while(iterationCounter < testIterations)
   409 			{
   410 				iterationCounter++;
   411 
   412 				if(userExecKey != 0) {
   413 					execKey = userExecKey;
   414 				} else {
   415 					execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
   416 				}
   417 
   418 				SDLTest_Log("Test Iteration %i: execKey %d", iterationCounter, execKey);
   419 				testResult = SDLTest_RunTest(testSuite, testCase, execKey);
   420 
   421 				if (testResult == TEST_RESULT_PASSED) {
   422 					testPassedCount++;
   423 					totalTestPassedCount++;
   424 				} else if (testResult == TEST_RESULT_SKIPPED) {
   425 					testSkippedCount++;
   426 					totalTestSkippedCount++;
   427 				} else {
   428 					testFailedCount++;
   429 					totalTestFailedCount++;
   430 				}
   431 			}
   432 
   433 			// Take time - test end
   434 			testEndTicks = SDL_GetTicks();
   435 			testEndTimestamp = time(0);
   436 
   437 			// TODO log test ended
   438 		}
   439 
   440 		// Take time - suite end
   441 		suiteEndTicks = SDL_GetTicks();
   442 		suiteEndTimestamp = time(0);
   443 
   444 		// TODO log suite ended
   445 	}
   446 
   447 	// Take time - run end
   448 	runEndTicks = SDL_GetTicks();
   449 	runEndTimestamp = time(0);
   450 
   451 	// TODO log run ended
   452 
   453 	return (totalTestFailedCount ? 1 : 0);
   454 }