From d975a7a37ba65f1f027eb7140159cfe6fca305e5 Mon Sep 17 00:00:00 2001 From: Andreas Schiffler Date: Tue, 4 Dec 2012 19:21:10 -0800 Subject: [PATCH] Added to harness in test lib (work in progress) --- include/SDL_test_assert.h | 6 + include/SDL_test_harness.h | 17 +-- src/test/SDL_test_assert.c | 12 +- src/test/SDL_test_harness.c | 270 +++++++++++++++++++++++++++++++++++- 4 files changed, 282 insertions(+), 23 deletions(-) diff --git a/include/SDL_test_assert.h b/include/SDL_test_assert.h index f0e1e6d22..7159ccd41 100644 --- a/include/SDL_test_assert.h +++ b/include/SDL_test_assert.h @@ -54,6 +54,12 @@ extern "C" { */ #define ASSERT_PASS 1 +/*! \brief counts the failed asserts */ +static Uint32 SDLTest_AssertsFailed = 0; + +/*! \brief counts the passed asserts */ +static Uint32 SDLTest_AssertsPassed = 0; + /** * \brief Assert that logs and break execution flow on failures. * diff --git a/include/SDL_test_harness.h b/include/SDL_test_harness.h index cad58b2c0..cf30600e2 100644 --- a/include/SDL_test_harness.h +++ b/include/SDL_test_harness.h @@ -49,22 +49,17 @@ extern "C" { #define TEST_ENABLED 1 #define TEST_DISABLED 0 -//! Definitions of assert results -#define ASSERT_PASS 1 -#define ASSERT_FAIL 0 - //! Definition of all the possible test return values of the test case method #define TEST_ABORTED -1 #define TEST_COMPLETED 0 #define TEST_SKIPPED 1 //! Definition of all the possible test results for the harness -#define TEST_RESULT_PASSED 0 -#define TEST_RESULT_FAILED 1 +#define TEST_RESULT_PASSED 0 +#define TEST_RESULT_FAILED 1 #define TEST_RESULT_NO_ASSERT 2 -#define TEST_RESULT_SKIPPED 3 -#define TEST_RESULT_KILLED 4 -#define TEST_RESULT_SETUP_FAILURE 5 +#define TEST_RESULT_SKIPPED 3 +#define TEST_RESULT_SETUP_FAILURE 4 //!< Function pointer to a test case setup function (run before every test) typedef void (*SDLTest_TestCaseSetUpFp)(void *arg); @@ -92,7 +87,7 @@ typedef struct SDLTest_TestCaseReference { /** * Holds information about a test suite (multiple test cases). */ -typedef struct TestSuiteReference { +typedef struct SDLTest_TestSuiteReference { /*!< "PlatformSuite" */ char *name; /*!< The function that is run before each test. NULL skips. */ @@ -101,7 +96,7 @@ typedef struct TestSuiteReference { const SDLTest_TestCaseReference **testCases; /*!< The function that is run after each test. NULL skips. */ SDLTest_TestCaseTearDownFp testTearDown; -} TestSuiteReference; +} SDLTest_TestSuiteReference; /* Ends C function definitions when using C++ */ #ifdef __cplusplus diff --git a/src/test/SDL_test_assert.c b/src/test/SDL_test_assert.c index 1123ccded..8298db171 100644 --- a/src/test/SDL_test_assert.c +++ b/src/test/SDL_test_assert.c @@ -29,14 +29,8 @@ #include "SDL_test.h" -/*! \brief counts the failed asserts */ -static Uint32 SDLTest_AssertsFailed = 0; - -/*! \brief counts the passed asserts */ -static Uint32 SDLTest_AssertsPassed = 0; - /* Assert check message format */ -const char *SDLTest_AssertCheckFmt = "Assert %s: %s"; +const char *SDLTest_AssertCheckFmt = "Assert '%s': %s"; /* Assert summary message format */ const char *SDLTest_AssertSummaryFmt = "Assert Summary: Total=%d Passed=%d Failed=%d"; @@ -58,12 +52,12 @@ int SDLTest_AssertCheck(int assertCondition, char *assertDescription) if (assertCondition == ASSERT_FAIL) { SDLTest_AssertsFailed++; - SDLTest_LogError(fmt, "Failed", assertDescription); + SDLTest_LogError(fmt, assertDescription, "Failed"); } else { SDLTest_AssertsPassed++; - SDLTest_Log(fmt, "Passed", assertDescription); + SDLTest_Log(fmt, assertDescription, "Passed"); } return assertCondition; diff --git a/src/test/SDL_test_harness.c b/src/test/SDL_test_harness.c index 8140aae43..ccb8455f4 100644 --- a/src/test/SDL_test_harness.c +++ b/src/test/SDL_test_harness.c @@ -23,10 +23,19 @@ #include "SDL_test.h" -#include +#include +#include #include +#include -// TODO: port over remaining harness +/* Assert check message format */ +const char *SDLTest_TestCheckFmt = "Test '%s': %s"; + +/* Invalid test name/description message format */ +const char *SDLTest_InvalidNameFmt = "(Invalid)"; + +/*! \brief Timeout for single test case execution */ +static Uint32 SDLTest_TestCaseTimeout = 3600; /** * Generates a random run seed string for the harness. The generated seed @@ -155,7 +164,7 @@ SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iter * \return Timer id or -1 on failure. */ SDL_TimerID -SetTestTimeout(int timeout, void (*callback)()) +SDLTest_SetTestTimeout(int timeout, void (*callback)()) { Uint32 timeoutInMilliseconds; SDL_TimerID timerID; @@ -188,3 +197,258 @@ SetTestTimeout(int timeout, void (*callback)()) return timerID; } + +void +SDLTest_BailOut() +{ + SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run."); + exit(TEST_ABORTED); // bail out from the test +} + +/** + * \brief Execute a test using the given execution key. + * + * \param testSuite Suite containing the test case. + * \param testCase Case to execute. + * \param execKey Execution key for the fuzzer. + * + * \returns Test case result. + */ +int +SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey) +{ + SDL_TimerID timer = 0; + + if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL) + { + SDLTest_LogError("Setup failure: testSuite or testCase references NULL"); + return TEST_RESULT_SETUP_FAILURE; + } + + if (!testCase->enabled) + { + SDLTest_Log((char *)SDLTest_TestCheckFmt, testCase->name, "Skipped"); + return TEST_RESULT_SKIPPED; + } + + // Initialize fuzzer + SDLTest_FuzzerInit(execKey); + + // Reset assert tracker + SDLTest_ResetAssertSummary(); + + // Set timeout timer + timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut); + + // Maybe run suite initalizer function + if (testSuite->testSetUp) { + testSuite->testSetUp(0x0); + if (SDLTest_AssertsFailed > 0) { + SDLTest_LogError((char *)SDLTest_TestCheckFmt, testSuite->name, "Failed"); + return TEST_RESULT_SETUP_FAILURE; + } + } + + // Run test case function + testCase->testCase(0x0); + + // Maybe run suite cleanup function + if (testSuite->testTearDown) { + testSuite->testTearDown(0x0); + } + + // Cancel timeout timer + if (timer) { + SDL_RemoveTimer(timer); + } + + // Report on asserts and fuzzer usage + SDLTest_Log("Fuzzer invocations: %d", SDLTest_GetFuzzerInvocationCount()); + SDLTest_LogAssertSummary(); + + // Analyze assert count to determine test case result + if (SDLTest_AssertsFailed > 0) { + SDLTest_LogError((char *)SDLTest_TestCheckFmt, testCase->name, "Failed"); + return TEST_RESULT_FAILED; + } else { + if (SDLTest_AssertsPassed > 0) { + SDLTest_Log((char *)SDLTest_TestCheckFmt, testCase->name, "Passed"); + return TEST_RESULT_PASSED; + } else { + SDLTest_LogError((char *)SDLTest_TestCheckFmt, testCase->name, "No Asserts"); + return TEST_RESULT_NO_ASSERT; + } + } +} + +/* Prints summary of all suites/tests contained in the given reference */ +void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites) +{ + int suiteCounter; + int testCounter; + SDLTest_TestSuiteReference *testSuite; + SDLTest_TestCaseReference *testCase; + + // Loop over all suites + suiteCounter = 0; + while(&testSuites[suiteCounter]) { + testSuite=&testSuites[suiteCounter]; + suiteCounter++; + SDLTest_Log("Test Suite %i - %s\n", suiteCounter, + (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFmt); + + // Loop over all test cases + testCounter = 0; + while(testSuite->testCases[testCounter]) + { + testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter]; + testCounter++; + SDLTest_Log(" Test Case %i - %s: %s", testCounter, + (testCase->name) ? testCase->name : SDLTest_InvalidNameFmt, + (testCase->description) ? testCase->description : SDLTest_InvalidNameFmt); + } + } +} + + +/** + * \brief Execute a test using the given execution key. + * + * \param testSuites Suites containing the test case. + * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one. + * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one. + * \param testIterations Number of iterations to run each test case. + * + * \returns Test run result; 0 when all tests passed, 1 if any tests failed. + */ +int +SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites, char *userRunSeed, Uint64 userExecKey, int testIterations) +{ + int suiteCounter; + int testCounter; + int iterationCounter; + SDLTest_TestSuiteReference *testSuite; + SDLTest_TestCaseReference *testCase; + char *runSeed; + Uint64 execKey; + Uint32 runStartTicks; + time_t runStartTimestamp; + Uint32 suiteStartTicks; + time_t suiteStartTimestamp; + Uint32 testStartTicks; + time_t testStartTimestamp; + Uint32 runEndTicks; + time_t runEndTimestamp; + Uint32 suiteEndTicks; + time_t suiteEndTimestamp; + Uint32 testEndTicks; + time_t testEndTimestamp; + int testResult; + int totalTestFailedCount, totalTestPassedCount, totalTestSkippedCount; + int testFailedCount, testPassedCount, testSkippedCount; + + // Sanitize test iterations + if (testIterations < 1) { + testIterations = 1; + } + + // Generate run see if we don't have one already + if (userRunSeed == NULL || strlen(userRunSeed) == 0) { + runSeed = SDLTest_GenerateRunSeed(16); + if (runSeed == NULL) { + SDLTest_LogError("Generating a random run seed failed"); + return 2; + } + } + + // Reset per-run counters + totalTestFailedCount = totalTestPassedCount = totalTestSkippedCount = 0; + + // Take time - run start + runStartTicks = SDL_GetTicks(); + runStartTimestamp = time(0); + + // TODO log run started + + // Loop over all suites + suiteCounter = 0; + while(&testSuites[suiteCounter]) { + testSuite=&testSuites[suiteCounter]; + suiteCounter++; + + // Reset per-suite counters + testFailedCount = testPassedCount = testSkippedCount = 0; + + // Take time - suite start + suiteStartTicks = SDL_GetTicks(); + suiteStartTimestamp = time(0); + + // TODO log suite started + SDLTest_Log("Test Suite %i - %s\n", suiteCounter, + (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFmt); + + // Loop over all test cases + testCounter = 0; + while(testSuite->testCases[testCounter]) + { + testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter]; + testCounter++; + + // Take time - test start + testStartTicks = SDL_GetTicks(); + testStartTimestamp = time(0); + + // TODO log test started + SDLTest_Log("Test Case %i - %s: %s", testCounter, + (testCase->name) ? testCase->name : SDLTest_InvalidNameFmt, + (testCase->description) ? testCase->description : SDLTest_InvalidNameFmt); + + // Loop over all iterations + iterationCounter = 0; + while(iterationCounter < testIterations) + { + iterationCounter++; + + if(userExecKey != 0) { + execKey = userExecKey; + } else { + execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter); + } + + SDLTest_Log("Test Iteration %i: execKey %d", iterationCounter, execKey); + testResult = SDLTest_RunTest(testSuite, testCase, execKey); + + if (testResult == TEST_RESULT_PASSED) { + testPassedCount++; + totalTestPassedCount++; + } else if (testResult == TEST_RESULT_SKIPPED) { + testSkippedCount++; + totalTestSkippedCount++; + } else { + testFailedCount++; + totalTestFailedCount++; + } + } + + // Take time - test end + testEndTicks = SDL_GetTicks(); + testEndTimestamp = time(0); + + // TODO log test ended + } + + // Take time - suite end + suiteEndTicks = SDL_GetTicks(); + suiteEndTimestamp = time(0); + + // TODO log suite ended + } + + // Take time - run end + runEndTicks = SDL_GetTicks(); + runEndTimestamp = time(0); + + // TODO log run ended + + return (totalTestFailedCount ? 1 : 0); +}