From 06944f7d29a4f86a3266542e86bd236064510737 Mon Sep 17 00:00:00 2001 From: Andreas Schiffler Date: Sat, 18 May 2013 09:35:09 -0700 Subject: [PATCH] Update test harness to handle test return codes; fix comment format in harness; update Main test suite to handle globally disabled features --- include/SDL_test_harness.h | 7 +- src/test/SDL_test_harness.c | 126 ++++++++++++++++++++++-------------- test/testautomation_main.c | 26 ++++++-- 3 files changed, 102 insertions(+), 57 deletions(-) diff --git a/include/SDL_test_harness.h b/include/SDL_test_harness.h index e5fe673af..f2c553cb2 100644 --- a/include/SDL_test_harness.h +++ b/include/SDL_test_harness.h @@ -51,8 +51,9 @@ extern "C" { //! Definition of all the possible test return values of the test case method #define TEST_ABORTED -1 -#define TEST_COMPLETED 0 -#define TEST_SKIPPED 1 +#define TEST_STARTED 0 +#define TEST_COMPLETED 1 +#define TEST_SKIPPED 2 //! Definition of all the possible test results for the harness #define TEST_RESULT_PASSED 0 @@ -65,7 +66,7 @@ extern "C" { typedef void (*SDLTest_TestCaseSetUpFp)(void *arg); //!< Function pointer to a test case function -typedef void (*SDLTest_TestCaseFp)(void *arg); +typedef int (*SDLTest_TestCaseFp)(void *arg); //!< Function pointer to a test case teardown function (run after every test) typedef void (*SDLTest_TestCaseTearDownFp)(void *arg); diff --git a/src/test/SDL_test_harness.c b/src/test/SDL_test_harness.c index fcd2e4f5c..d3c0266cf 100644 --- a/src/test/SDL_test_harness.c +++ b/src/test/SDL_test_harness.c @@ -57,20 +57,20 @@ SDLTest_GenerateRunSeed(const int length) SDLTest_RandomContext randomContext; int counter; - // Sanity check input + /* Sanity check input */ if (length <= 0) { SDLTest_LogError("The length of the harness seed must be >0."); return NULL; } - // Allocate output buffer + /* Allocate output buffer */ seed = (char *)SDL_malloc((length + 1) * sizeof(char)); if (seed == NULL) { SDLTest_LogError("SDL_malloc for run seed output buffer failed."); return NULL; } - // Generate a random string of alphanumeric characters + /* Generate a random string of alphanumeric characters */ SDLTest_RandomInitTime(&randomContext); for (counter = 0; counter < length - 1; ++counter) { unsigned int number = SDLTest_Random(&randomContext); @@ -129,11 +129,11 @@ SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iter return -1; } - // Convert iteration number into a string + /* Convert iteration number into a string */ SDL_memset(iterationString, 0, sizeof(iterationString)); SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration); - // Combine the parameters into single string + /* Combine the parameters into single string */ runSeedLength = SDL_strlen(runSeed); suiteNameLength = SDL_strlen(suiteName); testNameLength = SDL_strlen(testName); @@ -146,7 +146,7 @@ SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iter } SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration); - // Hash string and use half of the digest as 64bit exec key + /* Hash string and use half of the digest as 64bit exec key */ SDLTest_Md5Init(&md5Context); SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength); SDLTest_Md5Final(&md5Context); @@ -208,7 +208,7 @@ void SDLTest_BailOut() { SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run."); - exit(TEST_ABORTED); // bail out from the test + exit(TEST_ABORTED); /* bail out from the test */ } /** @@ -224,6 +224,7 @@ int SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey) { SDL_TimerID timer = 0; + int testCaseResult = 0; int testResult = 0; int fuzzerCount; @@ -240,16 +241,16 @@ SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference } - // Initialize fuzzer + /* Initialize fuzzer */ SDLTest_FuzzerInit(execKey); - // Reset assert tracker + /* Reset assert tracker */ SDLTest_ResetAssertSummary(); - // Set timeout timer + /* Set timeout timer */ timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut); - // Maybe run suite initalizer function + /* Maybe run suite initalizer function */ if (testSuite->testSetUp) { testSuite->testSetUp(0x0); if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) { @@ -258,26 +259,53 @@ SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference } } - // Run test case function - testCase->testCase(0x0); - testResult = SDLTest_AssertSummaryToTestResult(); + /* Run test case function */ + testCaseResult = testCase->testCase(0x0); + + /* Convert test execution result into harness result */ + if (testCaseResult == TEST_SKIPPED) { + /* Test was programatically skipped */ + testResult = TEST_RESULT_SKIPPED; + } else if (testCaseResult == TEST_STARTED) { + /* Test did not return a TEST_COMPLETED value; assume it failed */ + testResult = TEST_RESULT_FAILED; + } else if (testCaseResult == TEST_ABORTED) { + /* Test was aborted early; assume it failed */ + testResult = TEST_RESULT_FAILED; + } else { + /* Perform failure analysis based on asserts */ + testResult = SDLTest_AssertSummaryToTestResult(); + } - // Maybe run suite cleanup function (ignore failed asserts) + /* Maybe run suite cleanup function (ignore failed asserts) */ if (testSuite->testTearDown) { testSuite->testTearDown(0x0); } - // Cancel timeout timer + /* Cancel timeout timer */ if (timer) { SDL_RemoveTimer(timer); } - // Report on asserts and fuzzer usage + /* Report on asserts and fuzzer usage */ fuzzerCount = SDLTest_GetFuzzerInvocationCount(); if (fuzzerCount > 0) { SDLTest_Log("Fuzzer invocations: %d", fuzzerCount); } - SDLTest_LogAssertSummary(); + + /* Final log based on test execution result */ + if (testCaseResult == TEST_SKIPPED) { + /* Test was programatically skipped */ + SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)"); + } else if (testCaseResult == TEST_STARTED) { + /* Test did not return a TEST_COMPLETED value; assume it failed */ + SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)"); + } else if (testCaseResult == TEST_ABORTED) { + /* Test was aborted early; assume it failed */ + SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)"); + } else { + SDLTest_LogAssertSummary(); + } return testResult; } @@ -290,7 +318,7 @@ void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites) SDLTest_TestSuiteReference *testSuite; SDLTest_TestCaseReference *testCase; - // Loop over all suites + /* Loop over all suites */ suiteCounter = 0; while(&testSuites[suiteCounter]) { testSuite=&testSuites[suiteCounter]; @@ -298,7 +326,7 @@ void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites) SDLTest_Log("Test Suite %i - %s\n", suiteCounter, (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat); - // Loop over all test cases + /* Loop over all test cases */ testCounter = 0; while(testSuite->testCases[testCounter]) { @@ -365,12 +393,12 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user Uint32 countSum = 0; char *logFormat = (char *)SDLTest_LogSummaryFormat; - // Sanitize test iterations + /* Sanitize test iterations */ if (testIterations < 1) { testIterations = 1; } - // Generate run see if we don't have one already + /* Generate run see if we don't have one already */ if (userRunSeed == NULL || SDL_strlen(userRunSeed) == 0) { runSeed = SDLTest_GenerateRunSeed(16); if (runSeed == NULL) { @@ -382,18 +410,18 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user } - // Reset per-run counters + /* Reset per-run counters */ totalTestFailedCount = 0; totalTestPassedCount = 0; totalTestSkippedCount = 0; - // Take time - run start + /* Take time - run start */ runStartSeconds = GetClock(); - // Log run with fuzzer parameters + /* Log run with fuzzer parameters */ SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed); - // Initialize filtering + /* Initialize filtering */ if (filter != NULL && SDL_strlen(filter) > 0) { /* Loop over all suites to check if we have a filter match */ suiteCounter = 0; @@ -433,36 +461,36 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user } } - // Loop over all suites + /* Loop over all suites */ suiteCounter = 0; while(testSuites[suiteCounter]) { testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter]; currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat); suiteCounter++; - // Filter suite if flag set and we have a name + /* Filter suite if flag set and we have a name */ if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL && SDL_strcmp(suiteFilterName, testSuite->name) != 0) { - // Skip suite + /* Skip suite */ SDLTest_Log("===== Test Suite %i: '%s' skipped\n", suiteCounter, currentSuiteName); } else { - // Reset per-suite counters + /* Reset per-suite counters */ testFailedCount = 0; testPassedCount = 0; testSkippedCount = 0; - // Take time - suite start + /* Take time - suite start */ suiteStartSeconds = GetClock(); - // Log suite started + /* Log suite started */ SDLTest_Log("===== Test Suite %i: '%s' started\n", suiteCounter, currentSuiteName); - // Loop over all test cases + /* Loop over all test cases */ testCounter = 0; while(testSuite->testCases[testCounter]) { @@ -470,25 +498,25 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat); testCounter++; - // Filter tests if flag set and we have a name + /* Filter tests if flag set and we have a name */ if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL && SDL_strcmp(testFilterName, testCase->name) != 0) { - // Skip test + /* Skip test */ SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n", suiteCounter, testCounter, currentTestName); } else { - // Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) + /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */ if (testFilter == 1 && !testCase->enabled) { SDLTest_Log("Force run of disabled test since test filter was set"); testCase->enabled = 1; } - // Take time - test start + /* Take time - test start */ testStartSeconds = GetClock(); - // Log test started + /* Log test started */ SDLTest_Log("----- Test Case %i.%i: '%s' started", suiteCounter, testCounter, @@ -498,7 +526,7 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat); } - // Loop over all iterations + /* Loop over all iterations */ iterationCounter = 0; while(iterationCounter < testIterations) { @@ -525,21 +553,21 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user } } - // Take time - test end + /* Take time - test end */ testEndSeconds = GetClock(); runtime = testEndSeconds - testStartSeconds; if (runtime < 0.0f) runtime = 0.0f; if (testIterations > 1) { - // Log test runtime + /* Log test runtime */ SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime); SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations); } else { - // Log test runtime + /* Log test runtime */ SDLTest_Log("Total Test runtime: %.1f sec", runtime); } - // Log final test result + /* Log final test result */ switch (testResult) { case TEST_RESULT_PASSED: SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed"); @@ -555,15 +583,15 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user } } - // Take time - suite end + /* Take time - suite end */ suiteEndSeconds = GetClock(); runtime = suiteEndSeconds - suiteStartSeconds; if (runtime < 0.0f) runtime = 0.0f; - // Log suite runtime + /* Log suite runtime */ SDLTest_Log("Total Suite runtime: %.1f sec", runtime); - // Log summary and final Suite result + /* Log summary and final Suite result */ countSum = testPassedCount + testFailedCount + testSkippedCount; if (testFailedCount == 0) { @@ -579,15 +607,15 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user } } - // Take time - run end + /* Take time - run end */ runEndSeconds = GetClock(); runtime = runEndSeconds - runStartSeconds; if (runtime < 0.0f) runtime = 0.0f; - // Log total runtime + /* Log total runtime */ SDLTest_Log("Total Run runtime: %.1f sec", runtime); - // Log summary and final run result + /* Log summary and final run result */ countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount; if (totalTestFailedCount == 0) { diff --git a/test/testautomation_main.c b/test/testautomation_main.c index 9873c7e41..a39bf3ca1 100644 --- a/test/testautomation_main.c +++ b/test/testautomation_main.c @@ -11,13 +11,16 @@ /*! - * \brief Tests SDL_Init() and SDL_Quit() + * \brief Tests SDL_Init() and SDL_Quit() of Joystick and Haptic subsystems * \sa * http://wiki.libsdl.org/moin.cgi/SDL_Init * http://wiki.libsdl.org/moin.cgi/SDL_Quit */ -static int main_testInitQuit (void *arg) +static int main_testInitQuitJoystickHaptic (void *arg) { +#if defined SDL_JOYSTICK_DISABLED || defined SDL_HAPTIC_DISABLED + return TEST_SKIPPED; +#else int enabled_subsystems; int initialized_subsystems = SDL_INIT_JOYSTICK | SDL_INIT_HAPTIC; @@ -32,6 +35,7 @@ static int main_testInitQuit (void *arg) SDLTest_AssertCheck( enabled_subsystems == 0, "SDL_Quit should shut down everything (%i)", enabled_subsystems ); return TEST_COMPLETED; +#endif } /*! @@ -42,6 +46,9 @@ static int main_testInitQuit (void *arg) */ static int main_testInitQuitSubSystem (void *arg) { +#if defined SDL_JOYSTICK_DISABLED || defined SDL_HAPTIC_DISABLED || defined SDL_GAMECONTROLLER_DISABLED + return TEST_SKIPPED; +#else int i; int subsystems[] = { SDL_INIT_JOYSTICK, SDL_INIT_HAPTIC, SDL_INIT_GAMECONTROLLER }; @@ -61,11 +68,15 @@ static int main_testInitQuitSubSystem (void *arg) } return TEST_COMPLETED; +#endif } const int joy_and_controller = SDL_INIT_JOYSTICK | SDL_INIT_GAMECONTROLLER; static int main_testImpliedJoystickInit (void *arg) { +#if defined SDL_JOYSTICK_DISABLED || defined SDL_GAMECONTROLLER_DISABLED + return TEST_SKIPPED; +#else int initialized_system; // First initialize the controller @@ -82,11 +93,15 @@ static int main_testImpliedJoystickInit (void *arg) initialized_system = SDL_WasInit(joy_and_controller); SDLTest_AssertCheck( (initialized_system & joy_and_controller) == 0, "SDL_WasInit() should be false for joystick & controller (%x)", initialized_system ); - return TEST_COMPLETED; + return TEST_COMPLETED; +#endif } static int main_testImpliedJoystickQuit (void *arg) { +#if defined SDL_JOYSTICK_DISABLED || defined SDL_GAMECONTROLLER_DISABLED + return TEST_SKIPPED; +#else int initialized_system; // First initialize the controller and the joystick (explicitly) @@ -106,11 +121,12 @@ static int main_testImpliedJoystickQuit (void *arg) SDL_QuitSubSystem(SDL_INIT_JOYSTICK); - return TEST_COMPLETED; + return TEST_COMPLETED; +#endif } static const SDLTest_TestCaseReference mainTest1 = - { (SDLTest_TestCaseFp)main_testInitQuit, "main_testInitQuit", "Tests SDL_Init/Quit", TEST_ENABLED}; + { (SDLTest_TestCaseFp)main_testInitQuitJoystickHaptic, "main_testInitQuitJoystickHaptic", "Tests SDL_Init/Quit of Joystick and Haptic subsystem", TEST_ENABLED}; static const SDLTest_TestCaseReference mainTest2 = { (SDLTest_TestCaseFp)main_testInitQuitSubSystem, "main_testInitQuitSubSystem", "Tests SDL_InitSubSystem/QuitSubSystem", TEST_ENABLED};