Adding elementary support for fuzzing.

New options: --seed [VALUE] --exec-key [EXEC-KEY] --iterations VALUE
libsdl-org · Jul 24, 2011 · 3153d49 · 3153d49
1 parent 8c7f144
commit 3153d49
Show file tree

Hide file tree

Showing 11 changed files with 206 additions and 51 deletions.
diff --git a/test/test-automation/Makefile.am b/test/test-automation/Makefile.am
@@ -1,16 +1,20 @@
 ACLOCAL_AMFLAGS = -I acinclude -I build-scripts
 
 SUBDIRS = testdummy testrect testplatform testaudio testsurface
+runnerdir = .
 
 bin_PROGRAMS = runner
-runner_SOURCES = runner.c support.c 
+runner_SOURCES = runner.c support.c
+##nobase_runner_HEADERS = fuzzer.h logger.h plain_logger.h  xml_logger.h xml.h
 runner_CLAGS = -W -Wall -Wextra -g `sdl-config --cflags` -DSDL_NO_COMPAT
 runner_LDADD = libtest.la
-runner_LDFLAGS = `sdl-config --libs` 
+runner_LDFLAGS = `sdl-config --libs`
+## -I .libs/libtest.so
 
 lib_LTLIBRARIES = libtest.la
 libtest_la_SOURCES = SDL_test.c logger_helpers.c plain_logger.c xml_logger.c xml.c \
-	common/common.c common/img_blit.c common/img_blitblend.c common/img_face.c common/img_primitives.c common/img_primitivesblend.c
+	common/common.c common/img_blit.c common/img_blitblend.c common/img_face.c common/img_primitives.c common/img_primitivesblend.c \
+	fuzzer/utl_crc32.c fuzzer/utl_md5.c fuzzer/utl_random.c fuzzer/fuzzer.c
 libtest_la_CLAGS = -fPIC -g
 libtest_la_LDFLAGS = `sdl-config --libs`
 

diff --git a/test/test-automation/SDL_test.c b/test/test-automation/SDL_test.c
@@ -23,6 +23,7 @@
 #include <time.h>
 
 #include "logger.h"
+#include "fuzzer/fuzzer.h"
 
 #include "SDL_test.h"
 
@@ -36,8 +37,13 @@ int _testAssertsFailed;
 int _testAssertsPassed;
 
 void
-_InitTestEnvironment()
+_InitTestEnvironment(const int execKey)
 {
+	// The execKey gets corrupted while passing arguments
+	// hence the global variable to circumvent the problem
+	InitFuzzer(globalExecKey);
+
+
 	_testReturnValue = TEST_RESULT_PASS;
 	_testAssertsFailed = 0;
 	_testAssertsPassed = 0;
@@ -53,6 +59,8 @@ _QuitTestEnvironment()
 		_testReturnValue = TEST_RESULT_NO_ASSERT;
 	}
 
+	DeinitFuzzer();
+
 	return _testReturnValue;
 }
 
@@ -61,6 +69,7 @@ _CountFailedAsserts() {
 	return _testAssertsFailed;
 }
 
+
 void
 AssertEquals(int expected, int actual, char *message, ...)
 {

diff --git a/test/test-automation/SDL_test.h b/test/test-automation/SDL_test.h
@@ -26,6 +26,7 @@
 #include "common/common.h"
 #include "common/images.h"
 
+#include "fuzzer/fuzzer.h"
 
 extern int _testReturnValue;
 extern int _testAssertsFailed;
@@ -68,7 +69,7 @@ typedef struct TestCaseReference {
  *  Initialized the test environment such as asserts. Must be called at
  *  the beginning of every test case, before doing anything else.
  */
-void _InitTestEnvironment();
+void _InitTestEnvironment(const int execKey);
 
 /*!
  *  Deinitializes the test environment and

diff --git a/test/test-automation/logger.h b/test/test-automation/logger.h
@@ -37,7 +37,7 @@ typedef	void (*SuiteEndedFp)(int testsPassed, int testsFailed, int testsSkipped,
 		                time_t endTime, double totalRuntime);
 
 typedef	void (*TestStartedFp)(const char *testName, const char *suiteName,
-                              const char *testDescription, time_t startTime);
+                              const char *testDescription, int execKey, time_t startTime);
 typedef	void (*TestEndedFp)(const char *testName, const char *suiteName, int testResult,
                             time_t endTime, double totalRuntime);
 
@@ -67,4 +67,9 @@ extern AssertWithValuesFp AssertWithValues;
 extern AssertSummaryFp AssertSummary;
 extern LogFp Log;
 
+extern int globalExecKey;
+//! Run seed for harness
+extern const char *runSeed;
+
+
 #endif
diff --git a/test/test-automation/plain_logger.c b/test/test-automation/plain_logger.c
@@ -80,9 +80,10 @@ PlainSuiteEnded(int testsPassed, int testsFailed, int testsSkipped,
 }
 
 void
-PlainTestStarted(const char *testName, const char *suiteName, const char *testDescription, time_t startTime)
+PlainTestStarted(const char *testName, const char *suiteName,
+				const char *testDescription, int execKey, time_t startTime)
 {
-	Output(indentLevel++, "Executing test: %s (in %s)", testName, suiteName);
+	Output(indentLevel++, "Executing test: %s (in %s). Execution key: %d", testName, suiteName, execKey);
 }
 
 void

diff --git a/test/test-automation/plain_logger.h b/test/test-automation/plain_logger.h
@@ -55,10 +55,11 @@ void PlainSuiteEnded(int testsPassed, int testsFailed, int testsSkipped,
  * \param testName Name of the test that'll be executed
  * \param suiteName Name of the suite of the test
  * \param testDescription Description of the test
+ * \param execKey Execution key for fuzzing
  * \param startTime When the test started to execute
  */
 void PlainTestStarted(const char *testName, const char *suiteName,
-                      const char *testDescription, time_t startTime);
+                      const char *testDescription, int execKey, time_t startTime);
 
 /*!
  * Prints information about the test test that was just executed

diff --git a/test/test-automation/runner.c b/test/test-automation/runner.c
@@ -28,6 +28,9 @@
 
 #include <sys/types.h>
 
+#include "fuzzer/fuzzer.h"
+
+
 #include "config.h"
 
 #include "SDL_test.h"
@@ -82,12 +85,24 @@ char testcase_name_substring[NAME_BUFFER_SIZE];
 
 //! Name for user-supplied XSL style sheet name
 char xsl_stylesheet_name[NAME_BUFFER_SIZE];
-
+//! User-suppled timeout value for tests
 int universal_timeout = -1;
 
 //! Default directory of the test suites
 #define DEFAULT_TEST_DIRECTORY "tests/"
 
+int globalExecKey = -1;
+const char *runSeed = "seed";
+
+int userExecKey = 0;
+
+//! How man time a test will be invocated
+int testInvocationCount = 1;
+
+// \todo move this upper!! (and add comments)
+int totalTestFailureCount = 0, totalTestPassCount = 0, totalTestSkipCount = 0;
+int testFailureCount = 0, testPassCount = 0, testSkipCount = 0;
+
 
 /*!
  * Holds information about test suite such as it's name
@@ -674,8 +689,12 @@ CheckTestRequirements(TestCase *testCase)
  * \param test result
  */
 int
-RunTest(TestCase *testCase)
+RunTest(TestCase *testCase, const int execKey)
 {
+	if(!testCase) {
+		return -1;
+	}
+
 	int runnable = CheckTestRequirements(testCase);
 	if(runnable != 1) {
 		return TEST_RESULT_SKIPPED;
@@ -719,15 +738,15 @@ RunTest(TestCase *testCase)
  * \return The return value of the test. Zero means success, non-zero failure.
  */
 int
-ExecuteTest(TestCase *testItem) {
+ExecuteTest(TestCase *testItem, const int execKey) {
 	int retVal = -1;
 
 	if(execute_inproc) {
-		retVal = RunTest(testItem);
+		retVal = RunTest(testItem, execKey);
 	} else {
 		int childpid = fork();
 		if(childpid == 0) {
-			exit(RunTest(testItem));
+			exit(RunTest(testItem, execKey));
 		} else {
 			int stat_lock = -1;
 			int child = wait(&stat_lock);
@@ -736,6 +755,20 @@ ExecuteTest(TestCase *testItem) {
 		}
 	}
 
+	if(retVal == TEST_RESULT_SKIPPED) {
+		testSkipCount++;
+		totalTestSkipCount++;
+	}
+	else if(retVal) {
+		totalTestFailureCount++;
+		testFailureCount++;
+	}
+	else {
+		totalTestPassCount++;
+		testPassCount++;
+	}
+
+	// return the value for logger
 	return retVal;
 }
 
@@ -890,6 +923,40 @@ ParseOptions(int argc, char *argv[])
 
     	  universal_timeout = atoi(timeoutString);
       }
+      else if(SDL_strcmp(arg, "--seed") == 0) {
+    	  if( (i + 1) < argc)  {
+    		  runSeed = argv[++i];
+    	  }  else {
+    		  printf("runner: seed value is missing\n");
+    		  PrintUsage();
+    		  exit(1);
+    	  }
+    	  //!Ê\todo should the seed be copied to a buffer?
+      }
+      else if(SDL_strcmp(arg, "--iterations") == 0) {
+    	  char *iterationsString = NULL;
+    	  if( (i + 1) < argc)  {
+    		  iterationsString = argv[++i];
+    	  }  else {
+    		  printf("runner: iterations value is missing\n");
+    		  PrintUsage();
+    		  exit(1);
+    	  }
+
+    	  testInvocationCount = atoi(iterationsString);
+      }
+      else if(SDL_strcmp(arg, "--exec-key") == 0) {
+    	  char *execKeyString = NULL;
+    	  if( (i + 1) < argc)  {
+    		  execKeyString = argv[++i];
+    	  }  else {
+    		  printf("runner: execkey value is missing\n");
+    		  PrintUsage();
+    		  exit(1);
+    	  }
+
+    	  userExecKey = atoi(execKeyString);
+      }
       else if(SDL_strcmp(arg, "--test") == 0 || SDL_strcmp(arg, "-t") == 0) {
     	  only_selected_test = 1;
     	  char *testName = NULL;
@@ -976,10 +1043,12 @@ main(int argc, char *argv[])
 {
 	ParseOptions(argc, argv);
 
+	CRC32_CTX crcContext;
+	utl_crc32Init(&crcContext);
+
 	// print: Testing against SDL version fuu (rev: bar) if verbose == true
 
-	int totalTestFailureCount = 0, totalTestPassCount = 0, totalTestSkipCount = 0;
-	int testFailureCount = 0, testPassCount = 0, testSkipCount = 0;
+
 	char *testSuiteName = NULL;
 	int suiteCounter = 0;
 
@@ -1021,54 +1090,56 @@ main(int argc, char *argv[])
 	char *currentSuiteName = NULL;
 	int suiteStartTime = SDL_GetTicks();
 
+	int notFirstSuite = 0;
+	int startNewSuite = 1;
 	TestCase *testItem = NULL;
 	for(testItem = testCases; testItem; testItem = testItem->next) {
-		if(currentSuiteName == NULL) {
-			currentSuiteName = testItem->suiteName;
-			SuiteStarted(currentSuiteName, time(0));
-
-			testFailureCount = testPassCount = testSkipCount = 0;
-
-			suiteCounter++;
+		if(currentSuiteName && strncmp(currentSuiteName, testItem->suiteName, NAME_BUFFER_SIZE) != 0) {
+			startNewSuite = 1;
 		}
-		else if(strncmp(currentSuiteName, testItem->suiteName, NAME_BUFFER_SIZE) != 0) {
-			const double suiteRuntime = (SDL_GetTicks() - suiteStartTime) / 1000.0f;
 
-			SuiteEnded(testPassCount, testFailureCount, testSkipCount, time(0),
-						suiteRuntime);
+		if(startNewSuite) {
+			if(notFirstSuite) {
+				const double suiteRuntime = (SDL_GetTicks() - suiteStartTime) / 1000.0f;
+
+				SuiteEnded(testPassCount, testFailureCount, testSkipCount, time(0),
+							suiteRuntime);
+			}
 
 			suiteStartTime = SDL_GetTicks();
 
 			currentSuiteName = testItem->suiteName;
 			SuiteStarted(currentSuiteName, time(0));
-
 			testFailureCount = testPassCount = testSkipCount = 0;
-
 			suiteCounter++;
+
+			startNewSuite = 0;
+			notFirstSuite = 1;
 		}
 
-		TestStarted(testItem->testName, testItem->suiteName,
-                    testItem->description, time(0));
+		int currentIteration = testInvocationCount;
+		while(currentIteration > 0) {
+			if(userExecKey != 0) {
+				globalExecKey = userExecKey;
+			} else {
+				const int execKey = GenerateExecKey(crcContext, runSeed, testItem->suiteName,
+											  testItem->testName, currentIteration);
+				globalExecKey = execKey;
+			}
 
-		const Uint32 testTimeStart = SDL_GetTicks();
+			TestStarted(testItem->testName, testItem->suiteName,
+						testItem->description, globalExecKey, time(0));
 
-		int retVal = ExecuteTest(testItem);
-		if(retVal == 3) {
-			testSkipCount++;
-			totalTestSkipCount++;
-		}
-		else if(retVal) {
-			totalTestFailureCount++;
-			testFailureCount++;
-		}
-		else {
-			totalTestPassCount++;
-			testPassCount++;
-		}
+			const Uint32 testTimeStart = SDL_GetTicks();
+
+			int retVal = ExecuteTest(testItem, globalExecKey);
 
-		const double testTotalRuntime = (SDL_GetTicks() - testTimeStart) / 1000.0f;
+			const double testTotalRuntime = (SDL_GetTicks() - testTimeStart) / 1000.0f;
 
-		TestEnded(testItem->testName, testItem->suiteName, retVal, time(0), testTotalRuntime);
+			TestEnded(testItem->testName, testItem->suiteName, retVal, time(0), testTotalRuntime);
+
+			currentIteration--;
+		}
 	}
 
 	if(currentSuiteName) {
@@ -1082,11 +1153,13 @@ main(int argc, char *argv[])
 	const Uint32 endTicks = SDL_GetTicks();
 	const double totalRunTime = (endTicks - startTicks) / 1000.0f;
 
-	RunEnded(totalTestPassCount + totalTestFailureCount, suiteCounter,
+	RunEnded(totalTestPassCount + totalTestFailureCount + totalTestSkipCount, suiteCounter,
 			 totalTestPassCount, totalTestFailureCount, totalTestSkipCount, time(0), totalRunTime);
 
 	// Some SDL subsystem might be init'ed so shut them down
 	SDL_Quit();
 
+	utl_crc32Done(&crcContext);
+
 	return (totalTestFailureCount ? 1 : 0);
 }