From 3153d49828b4410a7ea688d2eed9231d6c70c976 Mon Sep 17 00:00:00 2001 From: Markus Kauppila Date: Sun, 24 Jul 2011 18:21:53 +0300 Subject: [PATCH] Adding elementary support for fuzzing. New options: --seed [VALUE] --exec-key [EXEC-KEY] --iterations VALUE --- test/test-automation/Makefile.am | 10 +- test/test-automation/SDL_test.c | 11 +- test/test-automation/SDL_test.h | 3 +- test/test-automation/logger.h | 7 +- test/test-automation/plain_logger.c | 5 +- test/test-automation/plain_logger.h | 3 +- test/test-automation/runner.c | 151 +++++++++++++++------ test/test-automation/testdummy/testdummy.c | 13 ++ test/test-automation/testrect/testrect.c | 36 ++++- test/test-automation/xml_logger.c | 14 +- test/test-automation/xml_logger.h | 4 +- 11 files changed, 206 insertions(+), 51 deletions(-) diff --git a/test/test-automation/Makefile.am b/test/test-automation/Makefile.am index fe88cf2f3..30b610916 100644 --- a/test/test-automation/Makefile.am +++ b/test/test-automation/Makefile.am @@ -1,16 +1,20 @@ ACLOCAL_AMFLAGS = -I acinclude -I build-scripts SUBDIRS = testdummy testrect testplatform testaudio testsurface +runnerdir = . bin_PROGRAMS = runner -runner_SOURCES = runner.c support.c +runner_SOURCES = runner.c support.c +##nobase_runner_HEADERS = fuzzer.h logger.h plain_logger.h xml_logger.h xml.h runner_CLAGS = -W -Wall -Wextra -g `sdl-config --cflags` -DSDL_NO_COMPAT runner_LDADD = libtest.la -runner_LDFLAGS = `sdl-config --libs` +runner_LDFLAGS = `sdl-config --libs` +## -I .libs/libtest.so lib_LTLIBRARIES = libtest.la libtest_la_SOURCES = SDL_test.c logger_helpers.c plain_logger.c xml_logger.c xml.c \ - common/common.c common/img_blit.c common/img_blitblend.c common/img_face.c common/img_primitives.c common/img_primitivesblend.c + common/common.c common/img_blit.c common/img_blitblend.c common/img_face.c common/img_primitives.c common/img_primitivesblend.c \ + fuzzer/utl_crc32.c fuzzer/utl_md5.c fuzzer/utl_random.c fuzzer/fuzzer.c libtest_la_CLAGS = -fPIC -g libtest_la_LDFLAGS = `sdl-config --libs` diff --git a/test/test-automation/SDL_test.c b/test/test-automation/SDL_test.c index ac959648d..89c4a486e 100644 --- a/test/test-automation/SDL_test.c +++ b/test/test-automation/SDL_test.c @@ -23,6 +23,7 @@ #include #include "logger.h" +#include "fuzzer/fuzzer.h" #include "SDL_test.h" @@ -36,8 +37,13 @@ int _testAssertsFailed; int _testAssertsPassed; void -_InitTestEnvironment() +_InitTestEnvironment(const int execKey) { + // The execKey gets corrupted while passing arguments + // hence the global variable to circumvent the problem + InitFuzzer(globalExecKey); + + _testReturnValue = TEST_RESULT_PASS; _testAssertsFailed = 0; _testAssertsPassed = 0; @@ -53,6 +59,8 @@ _QuitTestEnvironment() _testReturnValue = TEST_RESULT_NO_ASSERT; } + DeinitFuzzer(); + return _testReturnValue; } @@ -61,6 +69,7 @@ _CountFailedAsserts() { return _testAssertsFailed; } + void AssertEquals(int expected, int actual, char *message, ...) { diff --git a/test/test-automation/SDL_test.h b/test/test-automation/SDL_test.h index 428d2dee4..db42dddea 100644 --- a/test/test-automation/SDL_test.h +++ b/test/test-automation/SDL_test.h @@ -26,6 +26,7 @@ #include "common/common.h" #include "common/images.h" +#include "fuzzer/fuzzer.h" extern int _testReturnValue; extern int _testAssertsFailed; @@ -68,7 +69,7 @@ typedef struct TestCaseReference { * Initialized the test environment such as asserts. Must be called at * the beginning of every test case, before doing anything else. */ -void _InitTestEnvironment(); +void _InitTestEnvironment(const int execKey); /*! * Deinitializes the test environment and diff --git a/test/test-automation/logger.h b/test/test-automation/logger.h index 67e91e207..3a3fd6588 100644 --- a/test/test-automation/logger.h +++ b/test/test-automation/logger.h @@ -37,7 +37,7 @@ typedef void (*SuiteEndedFp)(int testsPassed, int testsFailed, int testsSkipped, time_t endTime, double totalRuntime); typedef void (*TestStartedFp)(const char *testName, const char *suiteName, - const char *testDescription, time_t startTime); + const char *testDescription, int execKey, time_t startTime); typedef void (*TestEndedFp)(const char *testName, const char *suiteName, int testResult, time_t endTime, double totalRuntime); @@ -67,4 +67,9 @@ extern AssertWithValuesFp AssertWithValues; extern AssertSummaryFp AssertSummary; extern LogFp Log; +extern int globalExecKey; +//! Run seed for harness +extern const char *runSeed; + + #endif diff --git a/test/test-automation/plain_logger.c b/test/test-automation/plain_logger.c index 8c1f14858..4001db437 100644 --- a/test/test-automation/plain_logger.c +++ b/test/test-automation/plain_logger.c @@ -80,9 +80,10 @@ PlainSuiteEnded(int testsPassed, int testsFailed, int testsSkipped, } void -PlainTestStarted(const char *testName, const char *suiteName, const char *testDescription, time_t startTime) +PlainTestStarted(const char *testName, const char *suiteName, + const char *testDescription, int execKey, time_t startTime) { - Output(indentLevel++, "Executing test: %s (in %s)", testName, suiteName); + Output(indentLevel++, "Executing test: %s (in %s). Execution key: %d", testName, suiteName, execKey); } void diff --git a/test/test-automation/plain_logger.h b/test/test-automation/plain_logger.h index 8c915a183..0a4d2409d 100644 --- a/test/test-automation/plain_logger.h +++ b/test/test-automation/plain_logger.h @@ -55,10 +55,11 @@ void PlainSuiteEnded(int testsPassed, int testsFailed, int testsSkipped, * \param testName Name of the test that'll be executed * \param suiteName Name of the suite of the test * \param testDescription Description of the test + * \param execKey Execution key for fuzzing * \param startTime When the test started to execute */ void PlainTestStarted(const char *testName, const char *suiteName, - const char *testDescription, time_t startTime); + const char *testDescription, int execKey, time_t startTime); /*! * Prints information about the test test that was just executed diff --git a/test/test-automation/runner.c b/test/test-automation/runner.c index 09eaf6f4c..4637568f0 100644 --- a/test/test-automation/runner.c +++ b/test/test-automation/runner.c @@ -28,6 +28,9 @@ #include +#include "fuzzer/fuzzer.h" + + #include "config.h" #include "SDL_test.h" @@ -82,12 +85,24 @@ char testcase_name_substring[NAME_BUFFER_SIZE]; //! Name for user-supplied XSL style sheet name char xsl_stylesheet_name[NAME_BUFFER_SIZE]; - +//! User-suppled timeout value for tests int universal_timeout = -1; //! Default directory of the test suites #define DEFAULT_TEST_DIRECTORY "tests/" +int globalExecKey = -1; +const char *runSeed = "seed"; + +int userExecKey = 0; + +//! How man time a test will be invocated +int testInvocationCount = 1; + +// \todo move this upper!! (and add comments) +int totalTestFailureCount = 0, totalTestPassCount = 0, totalTestSkipCount = 0; +int testFailureCount = 0, testPassCount = 0, testSkipCount = 0; + /*! * Holds information about test suite such as it's name @@ -674,8 +689,12 @@ CheckTestRequirements(TestCase *testCase) * \param test result */ int -RunTest(TestCase *testCase) +RunTest(TestCase *testCase, const int execKey) { + if(!testCase) { + return -1; + } + int runnable = CheckTestRequirements(testCase); if(runnable != 1) { return TEST_RESULT_SKIPPED; @@ -719,15 +738,15 @@ RunTest(TestCase *testCase) * \return The return value of the test. Zero means success, non-zero failure. */ int -ExecuteTest(TestCase *testItem) { +ExecuteTest(TestCase *testItem, const int execKey) { int retVal = -1; if(execute_inproc) { - retVal = RunTest(testItem); + retVal = RunTest(testItem, execKey); } else { int childpid = fork(); if(childpid == 0) { - exit(RunTest(testItem)); + exit(RunTest(testItem, execKey)); } else { int stat_lock = -1; int child = wait(&stat_lock); @@ -736,6 +755,20 @@ ExecuteTest(TestCase *testItem) { } } + if(retVal == TEST_RESULT_SKIPPED) { + testSkipCount++; + totalTestSkipCount++; + } + else if(retVal) { + totalTestFailureCount++; + testFailureCount++; + } + else { + totalTestPassCount++; + testPassCount++; + } + + // return the value for logger return retVal; } @@ -890,6 +923,40 @@ ParseOptions(int argc, char *argv[]) universal_timeout = atoi(timeoutString); } + else if(SDL_strcmp(arg, "--seed") == 0) { + if( (i + 1) < argc) { + runSeed = argv[++i]; + } else { + printf("runner: seed value is missing\n"); + PrintUsage(); + exit(1); + } + //!Ê\todo should the seed be copied to a buffer? + } + else if(SDL_strcmp(arg, "--iterations") == 0) { + char *iterationsString = NULL; + if( (i + 1) < argc) { + iterationsString = argv[++i]; + } else { + printf("runner: iterations value is missing\n"); + PrintUsage(); + exit(1); + } + + testInvocationCount = atoi(iterationsString); + } + else if(SDL_strcmp(arg, "--exec-key") == 0) { + char *execKeyString = NULL; + if( (i + 1) < argc) { + execKeyString = argv[++i]; + } else { + printf("runner: execkey value is missing\n"); + PrintUsage(); + exit(1); + } + + userExecKey = atoi(execKeyString); + } else if(SDL_strcmp(arg, "--test") == 0 || SDL_strcmp(arg, "-t") == 0) { only_selected_test = 1; char *testName = NULL; @@ -976,10 +1043,12 @@ main(int argc, char *argv[]) { ParseOptions(argc, argv); + CRC32_CTX crcContext; + utl_crc32Init(&crcContext); + // print: Testing against SDL version fuu (rev: bar) if verbose == true - int totalTestFailureCount = 0, totalTestPassCount = 0, totalTestSkipCount = 0; - int testFailureCount = 0, testPassCount = 0, testSkipCount = 0; + char *testSuiteName = NULL; int suiteCounter = 0; @@ -1021,54 +1090,56 @@ main(int argc, char *argv[]) char *currentSuiteName = NULL; int suiteStartTime = SDL_GetTicks(); + int notFirstSuite = 0; + int startNewSuite = 1; TestCase *testItem = NULL; for(testItem = testCases; testItem; testItem = testItem->next) { - if(currentSuiteName == NULL) { - currentSuiteName = testItem->suiteName; - SuiteStarted(currentSuiteName, time(0)); - - testFailureCount = testPassCount = testSkipCount = 0; - - suiteCounter++; + if(currentSuiteName && strncmp(currentSuiteName, testItem->suiteName, NAME_BUFFER_SIZE) != 0) { + startNewSuite = 1; } - else if(strncmp(currentSuiteName, testItem->suiteName, NAME_BUFFER_SIZE) != 0) { - const double suiteRuntime = (SDL_GetTicks() - suiteStartTime) / 1000.0f; - SuiteEnded(testPassCount, testFailureCount, testSkipCount, time(0), - suiteRuntime); + if(startNewSuite) { + if(notFirstSuite) { + const double suiteRuntime = (SDL_GetTicks() - suiteStartTime) / 1000.0f; + + SuiteEnded(testPassCount, testFailureCount, testSkipCount, time(0), + suiteRuntime); + } suiteStartTime = SDL_GetTicks(); currentSuiteName = testItem->suiteName; SuiteStarted(currentSuiteName, time(0)); - testFailureCount = testPassCount = testSkipCount = 0; - suiteCounter++; + + startNewSuite = 0; + notFirstSuite = 1; } - TestStarted(testItem->testName, testItem->suiteName, - testItem->description, time(0)); + int currentIteration = testInvocationCount; + while(currentIteration > 0) { + if(userExecKey != 0) { + globalExecKey = userExecKey; + } else { + const int execKey = GenerateExecKey(crcContext, runSeed, testItem->suiteName, + testItem->testName, currentIteration); + globalExecKey = execKey; + } - const Uint32 testTimeStart = SDL_GetTicks(); + TestStarted(testItem->testName, testItem->suiteName, + testItem->description, globalExecKey, time(0)); - int retVal = ExecuteTest(testItem); - if(retVal == 3) { - testSkipCount++; - totalTestSkipCount++; - } - else if(retVal) { - totalTestFailureCount++; - testFailureCount++; - } - else { - totalTestPassCount++; - testPassCount++; - } + const Uint32 testTimeStart = SDL_GetTicks(); + + int retVal = ExecuteTest(testItem, globalExecKey); - const double testTotalRuntime = (SDL_GetTicks() - testTimeStart) / 1000.0f; + const double testTotalRuntime = (SDL_GetTicks() - testTimeStart) / 1000.0f; - TestEnded(testItem->testName, testItem->suiteName, retVal, time(0), testTotalRuntime); + TestEnded(testItem->testName, testItem->suiteName, retVal, time(0), testTotalRuntime); + + currentIteration--; + } } if(currentSuiteName) { @@ -1082,11 +1153,13 @@ main(int argc, char *argv[]) const Uint32 endTicks = SDL_GetTicks(); const double totalRunTime = (endTicks - startTicks) / 1000.0f; - RunEnded(totalTestPassCount + totalTestFailureCount, suiteCounter, + RunEnded(totalTestPassCount + totalTestFailureCount + totalTestSkipCount, suiteCounter, totalTestPassCount, totalTestFailureCount, totalTestSkipCount, time(0), totalRunTime); // Some SDL subsystem might be init'ed so shut them down SDL_Quit(); + utl_crc32Done(&crcContext); + return (totalTestFailureCount ? 1 : 0); } diff --git a/test/test-automation/testdummy/testdummy.c b/test/test-automation/testdummy/testdummy.c index ec2dcbfdf..2bc63b2f3 100644 --- a/test/test-automation/testdummy/testdummy.c +++ b/test/test-automation/testdummy/testdummy.c @@ -29,6 +29,7 @@ #include #include "../SDL_test.h" +//#include "fuzzer/fuzzer.h" /* Test case references */ static const TestCaseReference test1 = @@ -89,6 +90,18 @@ void dummycase1(void *arg) { AssertEquals(5, 5, "Assert message"); + + for(; 0 ;) { + int min = 50; + int max = 69; + int random = RandomPositiveIntegerInRange(min, max); + if(random < min || random > max ) { + AssertFail("Generated incorrect integer"); + } + Log(0, "%d", random); + } + + //Log(0, "Random: %s", RandomAsciiString()); } void diff --git a/test/test-automation/testrect/testrect.c b/test/test-automation/testrect/testrect.c index dcd48b1ca..014d6c8bd 100644 --- a/test/test-automation/testrect/testrect.c +++ b/test/test-automation/testrect/testrect.c @@ -12,9 +12,14 @@ static const TestCaseReference test1 = (TestCaseReference){ "rect_testIntersectRectAndLine", "description", TEST_ENABLED, 0, 0 }; +static const TestCaseReference test2 = + (TestCaseReference){ "rect_testIntersectRectAndLineFuzzed", "Tests rect to line intersection with fuzzed values", TEST_ENABLED, 0, 0 }; + + + /* Test suite */ extern const TestCaseReference *testSuite[] = { - &test1, NULL + &test1, &test2, NULL }; TestCaseReference **QueryTestSuite() { @@ -23,6 +28,9 @@ TestCaseReference **QueryTestSuite() { /*! * \brief Tests SDL_IntersectRectAndLine() + * + * \sa + * http://wiki.libsdl.org/moin.cgi/SDL_IntersectRectAndLine */ int rect_testIntersectRectAndLine (void *arg) { @@ -131,3 +139,29 @@ int rect_testIntersectRectAndLine (void *arg) "diagonal line to upper right was incorrectly clipped: %d,%d - %d,%d", x1, y1, x2, y2); } + +/*! + * \brief Tests SDL_IntersectRectAndLine() + * + * \sa + * http://wiki.libsdl.org/moin.cgi/SDL_IntersectRectAndLine + */ +int rect_testIntersectRectAndLineFuzzed(void *arg) +{ + SDL_Rect rect = { 0, 0, RandomInteger(), RandomInteger() }; + int x1, y1; + int x2, y2; + SDL_bool clipped; + + x1 = -RandomInteger(); + y1 = RandomInteger(); + x2 = -RandomInteger(); + y2 = RandomInteger(); + clipped = SDL_IntersectRectAndLine(&rect, &x1, &y1, &x2, &y2); + + AssertTrue( !clipped, + /*&& x1 == -10 && y1 == 0 && x2 == -10 && y2 == 31, */ + "line outside to the left was incorrectly clipped: %d,%d - %d,%d", + x1, y1, x2, y2); +} + diff --git a/test/test-automation/xml_logger.c b/test/test-automation/xml_logger.c index 07bf46a88..2d4b6c9df 100644 --- a/test/test-automation/xml_logger.c +++ b/test/test-automation/xml_logger.c @@ -35,6 +35,7 @@ const char *documentRoot = "testlog"; const char *parametersElementName = "parameters"; const char *parameterElementName = "parameter"; const char *startTimeElementName = "startTime"; +const char *execKeyElementName = "executionKey"; const char *numSuitesElementName = "numSuites"; const char *numTestElementName = "numTests"; const char *numPassedTestsElementName = "numPassedTests"; @@ -310,7 +311,7 @@ XMLSuiteEnded(int testsPassed, int testsFailed, int testsSkipped, void XMLTestStarted(const char *testName, const char *suiteName, - const char *testDescription, time_t startTime) + const char *testDescription, int execKey, time_t startTime) { char * output = XMLOpenElement(testElementName); XMLOutputter(indentLevel++, YES, output); @@ -335,6 +336,17 @@ XMLTestStarted(const char *testName, const char *suiteName, output = XMLCloseElement(descriptionElementName); XMLOutputter(--indentLevel, YES, output); + // log exec key + output = XMLOpenElement(execKeyElementName); + XMLOutputter(indentLevel++, NO, output); + + output = XMLAddContent(IntToString(execKey)); + XMLOutputter(indentLevel, NO, output); + + output = XMLCloseElement(execKeyElementName); + XMLOutputter(--indentLevel, YES, output); + + // log start time output = XMLOpenElement(startTimeElementName); XMLOutputter(indentLevel++, NO, output); diff --git a/test/test-automation/xml_logger.h b/test/test-automation/xml_logger.h index 159e9b4e1..bdbe313ab 100644 --- a/test/test-automation/xml_logger.h +++ b/test/test-automation/xml_logger.h @@ -53,9 +53,11 @@ void XMLSuiteEnded(int testsPassed, int testsFailed, int testsSkipped, * \param testName Name of the test that'll be executed * \param suiteName Name of the suite of the test * \param testDescription Description of the test + * \param execKey Execution key for fuzzing * \param startTime When the test started to execute */ -void XMLTestStarted(const char *testName, const char *suiteName, const char *testDescription, time_t startTime); +void XMLTestStarted(const char *testName, const char *suiteName, + const char *testDescription, int execKey, time_t startTime); /*! * Prints information about the test test that was just executed in XML