From dd3d8891b23c1ff2de880a4605f89a77d480753f Mon Sep 17 00:00:00 2001 From: Markus Kauppila Date: Mon, 11 Jul 2011 21:09:28 +0300 Subject: [PATCH] If any assert in SetUp function fails that test will be skipped. --- test/test-automation/SDL_test.c | 9 +- test/test-automation/SDL_test.h | 22 +- test/test-automation/logger.h | 2 +- test/test-automation/plain_logger.c | 10 +- test/test-automation/plain_logger.h | 2 +- test/test-automation/runner.c | 530 ++++++++++-------- test/test-automation/style.xsl | 5 + test/test-automation/testdummy/testdummy.c | 3 + .../test-automation/testsurface/testsurface.c | 67 +-- test/test-automation/xml_logger.c | 18 +- test/test-automation/xml_logger.h | 2 +- 11 files changed, 377 insertions(+), 293 deletions(-) diff --git a/test/test-automation/SDL_test.c b/test/test-automation/SDL_test.c index 0462230f8..e9811199f 100644 --- a/test/test-automation/SDL_test.c +++ b/test/test-automation/SDL_test.c @@ -36,7 +36,7 @@ int _testAssertsFailed; int _testAssertsPassed; void -_InitTestEnvironment() // InitTestEnvironment +_InitTestEnvironment() { _testReturnValue = 0; _testAssertsFailed = 0; @@ -56,8 +56,13 @@ _QuitTestEnvironment() return _testReturnValue; } +int +_CountFailedAsserts() { + return _testAssertsFailed; +} + void -AssertEquals(const int expected, const int actual, char *message, ...) +AssertEquals(int expected, int actual, char *message, ...) { va_list args; char buf[256]; diff --git a/test/test-automation/SDL_test.h b/test/test-automation/SDL_test.h index efb651f8a..041670f79 100644 --- a/test/test-automation/SDL_test.h +++ b/test/test-automation/SDL_test.h @@ -69,13 +69,19 @@ void _InitTestEnvironment(); */ int _QuitTestEnvironment(); +/*! + * Can be used to query the number of failed asserts + * \return Returns the failed assert count. + */ +int _CountFailedAsserts(); + /*! * Assert function. Tests if the expected value equals the actual value, then * the test assert succeeds, otherwise it fails and warns about it. * * \param expected Value user expects to have * \param actual The actual value of tested variable - * \param message Message that will be printed if assert fails + * \param message Message that will be printed */ void AssertEquals(const int expected, const int actual, char *message, ...); @@ -85,18 +91,22 @@ void AssertEquals(const int expected, const int actual, char *message, ...); * assert passes, otherwise it fails. * * \param condition Condition which will be evaluated - * \param message Message that will be printed if assert fails + * \param message Message that will be printed */ void AssertTrue(int condition, char *message, ...); /*! -\todo add markup -*/ + * Assert function which will always fail + * + * \param message Message that will be printed + */ void AssertFail(char *message, ...); /*! -\todo add markup -*/ + * Assert function which will always pass + * + * \param message Message that will be printed + */ void AssertPass(char *message, ...); #endif diff --git a/test/test-automation/logger.h b/test/test-automation/logger.h index 4b137db38..116b99a81 100644 --- a/test/test-automation/logger.h +++ b/test/test-automation/logger.h @@ -30,7 +30,7 @@ */ typedef void (*RunStartedFp)(int parameterCount, char *runnerParameters[], time_t eventTime, void *data); typedef void (*RunEndedFp)(int testCount, int suiteCount, int testPassCount, int testFailCount, - time_t endTime, double totalRuntime); + int testSkippedCount, time_t endTime, double totalRuntime); typedef void (*SuiteStartedFp)(const char *suiteName, time_t eventTime); typedef void (*SuiteEndedFp)(int testsPassed, int testsFailed, int testsSkipped, diff --git a/test/test-automation/plain_logger.c b/test/test-automation/plain_logger.c index 0e3ff7839..1b186ea6b 100644 --- a/test/test-automation/plain_logger.c +++ b/test/test-automation/plain_logger.c @@ -54,13 +54,14 @@ PlainRunStarted(int parameterCount, char *runnerParameters[], time_t eventTime, void PlainRunEnded(int testCount, int suiteCount, int testPassCount, int testFailCount, - time_t endTime, double totalRuntime) + int testSkippedCount, time_t endTime, double totalRuntime) { Output(indentLevel, "Ran %d tests in %0.5f seconds from %d suites.", testCount, totalRuntime, suiteCount); Output(indentLevel, "%d tests passed", testPassCount); Output(indentLevel, "%d tests failed", testFailCount); + Output(indentLevel, "%d tests skipped", testSkippedCount); } void @@ -91,6 +92,9 @@ PlainTestEnded(const char *testName, const char *suiteName, if(testResult) { if(testResult == 2) { Output(--indentLevel, "%s: failed -> no assert", testName); + } + else if(testResult == 3) { + Output(--indentLevel, "%s: skipped", testName); } else { Output(--indentLevel, "%s: failed", testName); } @@ -104,7 +108,7 @@ PlainAssert(const char *assertName, int assertResult, const char *assertMessage, time_t eventTime) { const char *result = (assertResult) ? "passed" : "failed"; - Output(indentLevel, "%s: %s; %s", assertName, result, assertMessage); + Output(indentLevel, "%s: %s - %s", assertName, result, assertMessage); } void @@ -112,7 +116,7 @@ PlainAssertWithValues(const char *assertName, int assertResult, const char *asse int actualValue, int expected, time_t eventTime) { const char *result = (assertResult) ? "passed" : "failed"; - Output(indentLevel, "%s %s (expected %d, actualValue &d): %s", + Output(indentLevel, "%s: %s (expected %d, actualValue &d) - %s", assertName, result, expected, actualValue, assertMessage); } diff --git a/test/test-automation/plain_logger.h b/test/test-automation/plain_logger.h index d699f2552..a37b047de 100644 --- a/test/test-automation/plain_logger.h +++ b/test/test-automation/plain_logger.h @@ -26,7 +26,7 @@ void PlainRunStarted(int parameterCount, char *runnerParameters[], time_t eventT * \param totalRuntime How long the execution took */ void PlainRunEnded(int testCount, int suiteCount, int testPassCount, int testFailCount, - time_t endTime, double totalRuntime); + int testSkippedCount, time_t endTime, double totalRuntime); /*! * Prints the data about the test suite that'll be executed next diff --git a/test/test-automation/runner.c b/test/test-automation/runner.c index 9c52d1f89..a989e01c7 100644 --- a/test/test-automation/runner.c +++ b/test/test-automation/runner.c @@ -46,6 +46,8 @@ typedef int (*QuitTestInvironmentFp)(void); typedef void (*TestCaseSetUpFp)(void *arg); //!< Function pointer to a test case tear down function typedef void (*TestCaseTearDownFp)(void *arg); +//!< Function pointer to a function which returns the failed assert count +typedef int (*CountFailedAssertsFp)(void); //!< Flag for executing tests in-process @@ -115,6 +117,8 @@ typedef struct TestCaseItem { TestCaseTearDownFp testTearDown; QuitTestInvironmentFp quitTestEnvironment; + CountFailedAssertsFp countFailedAsserts; + struct TestCaseItem *next; } TestCase; @@ -126,6 +130,7 @@ QuitTestInvironmentFp LoadQuitTestInvironmentFunction(void *suite); TestCaseReference **QueryTestCaseReferences(void *library); TestCaseSetUpFp LoadTestSetUpFunction(void *suite); TestCaseTearDownFp LoadTestTearDownFunction(void *suite); +CountFailedAssertsFp LoadCountFailedAssertsFunction(void *suite); /*! Pointers to selected logger implementation */ @@ -141,143 +146,6 @@ AssertSummaryFp AssertSummary = NULL; LogFp Log = NULL; -/*! - * Goes through the previously loaded test suites and - * loads test cases from them. Test cases are filtered - * during the process. Function will only return the - * test cases which aren't filtered out. - * - * \param suites previously loaded test suites - * - * \return Test cases that survived filtering process. - */ -TestCase * -LoadTestCases(TestSuiteReference *suites) -{ - TestCase *testCases = NULL; - - TestSuiteReference *suiteReference = NULL; - for(suiteReference = suites; suiteReference; suiteReference = suiteReference->next) { - TestCaseReference **tests = QueryTestCaseReferences(suiteReference->library); - - TestCaseReference *testReference = NULL; - int counter = 0; - for(testReference = tests[counter]; testReference; testReference = tests[++counter]) { - - void *suite = suiteReference->library; - - // Load test case functions - InitTestInvironmentFp initTestEnvironment = LoadInitTestInvironmentFunction(suiteReference->library); - QuitTestInvironmentFp quitTestEnvironment = LoadQuitTestInvironmentFunction(suiteReference->library); - - TestCaseSetUpFp testSetUp = LoadTestSetUpFunction(suiteReference->library); - TestCaseTearDownFp testTearDown = LoadTestTearDownFunction(suiteReference->library); - - TestCaseFp testCase = LoadTestCaseFunction(suiteReference->library, testReference->name); - - // Do the filtering - if(FilterTestCase(testReference)) { - TestCase *item = SDL_malloc(sizeof(TestCase)); - memset(item, 0, sizeof(TestCase)); - - item->initTestEnvironment = initTestEnvironment; - item->quitTestEnvironment = quitTestEnvironment; - - item->testSetUp = testSetUp; - item->testTearDown = testTearDown; - - item->testCase = testCase; - - // copy suite name - int length = SDL_strlen(suiteReference->name) + 1; - item->suiteName = SDL_malloc(length); - strncpy(item->suiteName, suiteReference->name, length); - - // copy test name - length = SDL_strlen(testReference->name) + 1; - item->testName = SDL_malloc(length); - strncpy(item->testName, testReference->name, length); - - // copy test description - length = SDL_strlen(testReference->description) + 1; - item->description = SDL_malloc(length); - strncpy(item->description, testReference->description, length); - - item->requirements = testReference->requirements; - item->timeout = testReference->timeout; - - // prepend the list - item->next = testCases; - testCases = item; - - //printf("Added test: %s\n", testReference->name); - } - } - } - - return testCases; -} - - -/*! - * Unloads the given TestCases. Frees all the resources - * allocated for test cases. - * - * \param testCases Test cases to be deallocated - */ -void -UnloadTestCases(TestCase *testCases) -{ - TestCase *ref = testCases; - while(ref) { - SDL_free(ref->testName); - SDL_free(ref->suiteName); - SDL_free(ref->description); - - TestCase *temp = ref->next; - SDL_free(ref); - ref = temp; - } - - testCases = NULL; -} - - -/*! - * Filters a test case based on its properties in TestCaseReference and user - * preference. - * - * \return Non-zero means test will be added to execution list, zero means opposite - */ -int -FilterTestCase(TestCaseReference *testReference) -{ - int retVal = 1; - - if(testReference->enabled == TEST_DISABLED) { - retVal = 0; - } - - if(only_selected_test) { - if(SDL_strncmp(testReference->name, selected_test_name, NAME_BUFFER_SIZE) == 0) { - retVal = 1; - } else { - retVal = 0; - } - } - - if(only_tests_with_string) { - if(strstr(testReference->name, testcase_name_substring) != NULL) { - retVal = 1; - } else { - retVal = 0; - } - } - - return retVal; -} - - /*! * Scans the tests/ directory and returns the names * of the dynamic libraries implementing the test suites. @@ -298,10 +166,9 @@ ScanForTestSuites(char *directoryName, char *extension) { typedef struct dirent Entry; DIR *directory = opendir(directoryName); - TestSuiteReference *suites = NULL; - Entry *entry = NULL; + if(!directory) { fprintf(stderr, "Failed to open test suite directory: %s\n", directoryName); perror("Error message"); @@ -323,8 +190,11 @@ ScanForTestSuites(char *directoryName, char *extension) if(ok && SDL_strcmp(ext, extension) == 0) { // create test suite reference TestSuiteReference *reference = (TestSuiteReference *) SDL_malloc(sizeof(TestSuiteReference)); - memset(reference, 0, sizeof(TestSuiteReference)); + if(reference == NULL) { + fprintf(stderr, "Allocating TestSuiteReference failed\n"); + } + memset(reference, 0, sizeof(TestSuiteReference)); const int dirSize = SDL_strlen(directoryName); const int extSize = SDL_strlen(ext); @@ -427,6 +297,147 @@ UnloadTestSuites(TestSuiteReference *suites) } +/*! + * Goes through the previously loaded test suites and + * loads test cases from them. Test cases are filtered + * during the process. Function will only return the + * test cases which aren't filtered out. + * + * \param suites previously loaded test suites + * + * \return Test cases that survived filtering process. + */ +TestCase * +LoadTestCases(TestSuiteReference *suites) +{ + TestCase *testCases = NULL; + + TestSuiteReference *suiteReference = NULL; + for(suiteReference = suites; suiteReference; suiteReference = suiteReference->next) { + TestCaseReference **tests = QueryTestCaseReferences(suiteReference->library); + + TestCaseReference *testReference = NULL; + int counter = 0; + for(testReference = tests[counter]; testReference; testReference = tests[++counter]) { + + void *suite = suiteReference->library; + + // Load test case functions + InitTestInvironmentFp initTestEnvironment = LoadInitTestInvironmentFunction(suiteReference->library); + QuitTestInvironmentFp quitTestEnvironment = LoadQuitTestInvironmentFunction(suiteReference->library); + + TestCaseSetUpFp testSetUp = LoadTestSetUpFunction(suiteReference->library); + TestCaseTearDownFp testTearDown = LoadTestTearDownFunction(suiteReference->library); + + TestCaseFp testCase = LoadTestCaseFunction(suiteReference->library, testReference->name); + + CountFailedAssertsFp countFailedAsserts = LoadCountFailedAssertsFunction(suiteReference->library); + + // Do the filtering + if(FilterTestCase(testReference)) { + TestCase *item = SDL_malloc(sizeof(TestCase)); + memset(item, 0, sizeof(TestCase)); + + item->initTestEnvironment = initTestEnvironment; + item->quitTestEnvironment = quitTestEnvironment; + + item->testSetUp = testSetUp; + item->testTearDown = testTearDown; + + item->testCase = testCase; + + item->countFailedAsserts = countFailedAsserts; + + // copy suite name + int length = SDL_strlen(suiteReference->name) + 1; + item->suiteName = SDL_malloc(length); + strncpy(item->suiteName, suiteReference->name, length); + + // copy test name + length = SDL_strlen(testReference->name) + 1; + item->testName = SDL_malloc(length); + strncpy(item->testName, testReference->name, length); + + // copy test description + length = SDL_strlen(testReference->description) + 1; + item->description = SDL_malloc(length); + strncpy(item->description, testReference->description, length); + + item->requirements = testReference->requirements; + item->timeout = testReference->timeout; + + // prepend the list + item->next = testCases; + testCases = item; + + //printf("Added test: %s\n", testReference->name); + } + } + } + + return testCases; +} + + +/*! + * Unloads the given TestCases. Frees all the resources + * allocated for test cases. + * + * \param testCases Test cases to be deallocated + */ +void +UnloadTestCases(TestCase *testCases) +{ + TestCase *ref = testCases; + while(ref) { + SDL_free(ref->testName); + SDL_free(ref->suiteName); + SDL_free(ref->description); + + TestCase *temp = ref->next; + SDL_free(ref); + ref = temp; + } + + testCases = NULL; +} + + +/*! + * Filters a test case based on its properties in TestCaseReference and user + * preference. + * + * \return Non-zero means test will be added to execution list, zero means opposite + */ +int +FilterTestCase(TestCaseReference *testReference) +{ + int retVal = 1; + + if(testReference->enabled == TEST_DISABLED) { + retVal = 0; + } + + if(only_selected_test) { + if(SDL_strncmp(testReference->name, selected_test_name, NAME_BUFFER_SIZE) == 0) { + retVal = 1; + } else { + retVal = 0; + } + } + + if(only_tests_with_string) { + if(strstr(testReference->name, testcase_name_substring) != NULL) { + retVal = 1; + } else { + retVal = 0; + } + } + + return retVal; +} + + /*! * Loads the test case references from the given test suite. @@ -436,21 +447,21 @@ UnloadTestSuites(TestSuiteReference *suites) TestCaseReference ** QueryTestCaseReferences(void *library) { - TestCaseReference **(*suite)(void); + TestCaseReference **(*suite)(void); - suite = (TestCaseReference **(*)(void)) SDL_LoadFunction(library, "QueryTestSuite"); - if(suite == NULL) { - fprintf(stderr, "Loading QueryTestCaseReferences() failed.\n"); - fprintf(stderr, "%s\n", SDL_GetError()); - } + suite = (TestCaseReference **(*)(void)) SDL_LoadFunction(library, "QueryTestSuite"); + if(suite == NULL) { + fprintf(stderr, "Loading QueryTestCaseReferences() failed.\n"); + fprintf(stderr, "%s\n", SDL_GetError()); + } - TestCaseReference **tests = suite(); - if(tests == NULL) { - fprintf(stderr, "Failed to load test references.\n"); - fprintf(stderr, "%s\n", SDL_GetError()); - } + TestCaseReference **tests = suite(); + if(tests == NULL) { + fprintf(stderr, "Failed to load test references.\n"); + fprintf(stderr, "%s\n", SDL_GetError()); + } - return tests; + return tests; } @@ -554,6 +565,81 @@ LoadQuitTestInvironmentFunction(void *suite) { return testEnvQuit; } +/*! + * Loads function that returns failed assert count in the current + * test environment + * + * \param suite Used test suite + * + * \return Function pointer to _CountFailedAsserts function + */ +CountFailedAssertsFp +LoadCountFailedAssertsFunction(void *suite) { + CountFailedAssertsFp countFailedAssert = (CountFailedAssertsFp) SDL_LoadFunction(suite, "_CountFailedAsserts"); + if(countFailedAssert == NULL) { + fprintf(stderr, "Loading _CountFailedAsserts function failed, countFailedAssert == NULL\n"); + fprintf(stderr, "%s\n", SDL_GetError()); + } + + return countFailedAssert; +} + + +/* + * Execute the test + * + * \param testItem Test to be executed + */ +int +RunTest(TestCase *testItem) { + testItem->initTestEnvironment(); + + if(testItem->testSetUp) { + testItem->testSetUp(0x0); + } + + int cntFailedAsserts = testItem->countFailedAsserts(); + if(cntFailedAsserts != 0) { + return 3; + } + + testItem->testCase(0x0); + + if(testItem->testTearDown) { + testItem->testTearDown(0x0); + } + + return testItem->quitTestEnvironment(); +} + +/*! + * Executes a test case. Loads the test, executes it and + * returns the tests return value to the caller. + * + * \param testItem The test case that will be executed + * \return The return value of the test. Zero means success, non-zero failure. + */ +int +ExecuteTest(TestCase *testItem) { + int retVal = 1; + + if(execute_inproc) { + retVal = RunTest(testItem); + } else { + int childpid = fork(); + if(childpid == 0) { + exit(RunTest(testItem)); + } else { + int stat_lock = -1; + int child = wait(&stat_lock); + + retVal = HandleChildProcessReturnValue(stat_lock); + } + } + + return retVal; +} + /*! * If using out-of-proc execution of tests. This function @@ -584,56 +670,58 @@ HandleChildProcessReturnValue(int stat_lock) /*! - * Executes a test case. Loads the test, executes it and - * returns the tests return value to the caller. + * Sets up the logger. * - * \param testItem The test case that will be executed - * \return The return value of the test. Zero means success, non-zero failure. + * \return Some special data that will be passed to StartRun() logger call */ -int -ExecuteTest(TestCase *testItem) { - int retVal = 1; - if(execute_inproc) { - testItem->initTestEnvironment(); +void * +SetUpLogger() +{ + void *loggerData = NULL; + if(xml_enabled) { + RunStarted = XMLRunStarted; + RunEnded = XMLRunEnded; - if(testItem->testSetUp) { - testItem->testSetUp(0x0); - } + SuiteStarted = XMLSuiteStarted; + SuiteEnded = XMLSuiteEnded; - testItem->testCase(0x0); + TestStarted = XMLTestStarted; + TestEnded = XMLTestEnded; - if(testItem->testTearDown) { - testItem->testTearDown(0x0); + Assert = XMLAssert; + AssertWithValues = XMLAssertWithValues; + AssertSummary = XMLAssertSummary; + + Log = XMLLog; + + char *sheet = NULL; + if(xsl_enabled) { + sheet = "style.xsl"; // default style sheet; } - retVal = testItem->quitTestEnvironment(); - } else { - int childpid = fork(); - if(childpid == 0) { - testItem->initTestEnvironment(); + if(custom_xsl_enabled) { + sheet = xsl_stylesheet_name; + } - if(testItem->testSetUp) { - testItem->testSetUp(0x0); - } + loggerData = sheet; + } else { + RunStarted = PlainRunStarted; + RunEnded = PlainRunEnded; - testItem->testCase(0x0); + SuiteStarted = PlainSuiteStarted; + SuiteEnded = PlainSuiteEnded; - // note: if test case is is aborted by some signal - // then TearDown function won't be called - if(testItem->testTearDown) { - testItem->testTearDown(0x0); - } + TestStarted = PlainTestStarted; + TestEnded = PlainTestEnded; - exit(testItem->quitTestEnvironment()); - } else { - int stat_lock = -1; - int child = wait(&stat_lock); + Assert = PlainAssert; + AssertWithValues = PlainAssertWithValues; + AssertSummary = PlainAssertSummary; - retVal = HandleChildProcessReturnValue(stat_lock); - } + Log = PlainLog; } - return retVal; + return loggerData; } @@ -771,7 +859,7 @@ main(int argc, char *argv[]) // print: Testing against SDL version fuu (rev: bar) if verbose == true - int totalTestfailureCount = 0, totalTestPassCount = 0; + int totalTestFailureCount = 0, totalTestPassCount = 0, totalTestSkipCount = 0; int testFailureCount = 0, testPassCount = 0, testSkipCount = 0; char *testSuiteName = NULL; int suiteCounter = 0; @@ -782,49 +870,7 @@ main(int argc, char *argv[]) char *extension = "dylib"; #endif - void *loggerData = NULL; - if(xml_enabled) { - RunStarted = XMLRunStarted; - RunEnded = XMLRunEnded; - - SuiteStarted = XMLSuiteStarted; - SuiteEnded = XMLSuiteEnded; - - TestStarted = XMLTestStarted; - TestEnded = XMLTestEnded; - - Assert = XMLAssert; - AssertWithValues = XMLAssertWithValues; - AssertSummary = XMLAssertSummary; - - Log = XMLLog; - - char *sheet = NULL; - if(xsl_enabled) { - sheet = "style.xsl"; // default style sheet; - } - - if(custom_xsl_enabled) { - sheet = xsl_stylesheet_name; - } - - loggerData = sheet; - } else { - RunStarted = PlainRunStarted; - RunEnded = PlainRunEnded; - - SuiteStarted = PlainSuiteStarted; - SuiteEnded = PlainSuiteEnded; - - TestStarted = PlainTestStarted; - TestEnded = PlainTestEnded; - - Assert = PlainAssert; - AssertWithValues = PlainAssertWithValues; - AssertSummary = PlainAssertSummary; - - Log = PlainLog; - } + void *loggerData = SetUpLogger(); const Uint32 startTicks = SDL_GetTicks(); @@ -845,9 +891,7 @@ main(int argc, char *argv[]) RunStarted(argc, argv, time(0), loggerData); - char *currentSuiteName = NULL; - int suiteStartTime = SDL_GetTicks(); TestCase *testItem = NULL; @@ -856,7 +900,7 @@ main(int argc, char *argv[]) currentSuiteName = testItem->suiteName; SuiteStarted(currentSuiteName, time(0)); - testFailureCount = testPassCount = 0; + testFailureCount = testPassCount = testSkipCount = 0; suiteCounter++; } @@ -871,7 +915,7 @@ main(int argc, char *argv[]) currentSuiteName = testItem->suiteName; SuiteStarted(currentSuiteName, time(0)); - testFailureCount = testPassCount = 0; + testFailureCount = testPassCount = testSkipCount = 0; suiteCounter++; } @@ -882,8 +926,12 @@ main(int argc, char *argv[]) const Uint32 testTimeStart = SDL_GetTicks(); int retVal = ExecuteTest(testItem); - if(retVal) { - totalTestfailureCount++; + if(retVal == 3) { + testSkipCount++; + totalTestSkipCount++; + } + else if(retVal) { + totalTestFailureCount++; testFailureCount++; } else { totalTestPassCount++; @@ -906,8 +954,8 @@ main(int argc, char *argv[]) const Uint32 endTicks = SDL_GetTicks(); const double totalRunTime = (endTicks - startTicks) / 1000.0f; - RunEnded(totalTestPassCount + totalTestfailureCount, suiteCounter, - totalTestPassCount, totalTestfailureCount, time(0), totalRunTime); + RunEnded(totalTestPassCount + totalTestFailureCount, suiteCounter, + totalTestPassCount, totalTestFailureCount, totalTestSkipCount, time(0), totalRunTime); - return (totalTestfailureCount ? 1 : 0); + return (totalTestFailureCount ? 1 : 0); } diff --git a/test/test-automation/style.xsl b/test/test-automation/style.xsl index 9b5cdde97..367fa4d7f 100644 --- a/test/test-automation/style.xsl +++ b/test/test-automation/style.xsl @@ -104,6 +104,7 @@ $(document).ready(function() { /* Color the tests based on the result */ $("span.testResult[result='passed']").addClass('passed'); $("span.testResult[result='failed']").addClass('failed'); + $("span.testResult[result='skipped']").addClass('skipped'); /* Color the asserts based on the result */ $("span.assertResult[result='pass']").addClass('passed'); @@ -157,6 +158,10 @@ div, h1 { color: red; } +.skipped { + color: gray; +} + diff --git a/test/test-automation/testdummy/testdummy.c b/test/test-automation/testdummy/testdummy.c index c42792832..acaa2965d 100644 --- a/test/test-automation/testdummy/testdummy.c +++ b/test/test-automation/testdummy/testdummy.c @@ -56,6 +56,9 @@ TestCaseReference **QueryTestSuite() { * SetUp function can be used to create a test fixture for test cases. * The function will be called right before executing the test case. * + * Note: If any assert in the function fails then the test will be skipped. + * In practice, the entire suite will be skipped if assert failure happens. + * * Note: this function is optional. * * \param arg parameters given to test. Usually NULL diff --git a/test/test-automation/testsurface/testsurface.c b/test/test-automation/testsurface/testsurface.c index 71bef4e6a..bd52580a9 100644 --- a/test/test-automation/testsurface/testsurface.c +++ b/test/test-automation/testsurface/testsurface.c @@ -32,11 +32,39 @@ TestCaseReference **QueryTestSuite() { return (TestCaseReference **)testSuite; } +/* Function prototypes */ +SDL_Surface *_CreateTestSurface(); + + +/* Create test fixture */ + +static SDL_Surface *testsur = NULL; + + +void +SetUp(void *arg) +{ + int ret = SDL_Init(SDL_INIT_VIDEO); + AssertTrue(ret == 0, "SDL_Init(SDL_INIT_VIDEO)"); + + testsur = _CreateTestSurface(); + AssertTrue(testsur != NULL, "SDL_Init(SDL_INIT_VIDEO)"); +} + +void +TearDown(void *arg) +{ + SDL_FreeSurface( testsur ); + + SDL_Quit(); +} + /* Helper functions for the test cases */ #define TEST_SURFACE_WIDTH 80 #define TEST_SURFACE_HEIGHT 60 + /*! * Creates test surface */ @@ -66,7 +94,7 @@ _CreateTestSurface() /** * @brief Tests a blend mode. */ -int _testBlitBlendMode(SDL_Surface *testsur, SDL_Surface *face, int mode) +void _testBlitBlendMode(SDL_Surface *testsur, SDL_Surface *face, int mode) { int ret; int i, j, ni, nj; @@ -102,8 +130,6 @@ int _testBlitBlendMode(SDL_Surface *testsur, SDL_Surface *face, int mode) ret = SDL_BlitSurface( face, NULL, testsur, &rect ); AssertTrue(ret != 0, "SDL_BlitSurface"); } } - - return 0; } /* Test case functions */ @@ -115,13 +141,8 @@ void surface_testLoad(void *arg) int ret; SDL_Surface *face, *rface; - ret = SDL_Init(SDL_INIT_VIDEO); - AssertTrue(ret == 0, "SDL_Init(SDL_INIT_VIDEO)"); - - SDL_Surface *testsur = _CreateTestSurface(); - - /* Clear surface. */ - ret = SDL_FillRect( testsur, NULL, + /* Clear surface. */ + ret = SDL_FillRect( testsur, NULL, SDL_MapRGB( testsur->format, 0, 0, 0 ) ); AssertTrue(ret == 0, "SDL_FillRect"); @@ -151,10 +172,6 @@ void surface_testLoad(void *arg) /* Clean up. */ SDL_FreeSurface( rface ); SDL_FreeSurface( face ); - - SDL_FreeSurface( testsur ); - - SDL_Quit(); } @@ -163,14 +180,8 @@ void surface_testLoad(void *arg) */ void surface_testLoadFailure(void *arg) { - int ret = SDL_Init(SDL_INIT_VIDEO); - AssertTrue(ret == 0, "SDL_Init(SDL_INIT_VIDEO)"); - SDL_Surface *face = SDL_LoadBMP("nonexistant.bmp"); - AssertTrue(face == NULL, "SDL_CreateLoadBmp"); - - SDL_Quit(); } @@ -184,11 +195,6 @@ void surface_testBlit(void *arg) SDL_Surface *face; int i, j, ni, nj; - ret = SDL_Init(SDL_INIT_VIDEO); - AssertTrue(ret == 0, "SDL_Init(SDL_INIT_VIDEO)"); - - SDL_Surface *testsur = _CreateTestSurface(); - /* Clear surface. */ ret = SDL_FillRect( testsur, NULL, SDL_MapRGB( testsur->format, 0, 0, 0 ) ); @@ -292,9 +298,6 @@ void surface_testBlit(void *arg) /* Clean up. */ SDL_FreeSurface( face ); - SDL_FreeSurface( testsur ); - - SDL_Quit(); } /** @@ -308,11 +311,6 @@ void surface_testBlitBlend(void *arg) int i, j, ni, nj; int mode; - ret = SDL_Init(SDL_INIT_VIDEO); - AssertTrue(ret == 0, "SDL_Init(SDL_INIT_VIDEO)"); - - SDL_Surface *testsur = _CreateTestSurface(); - /* Clear surface. */ ret = SDL_FillRect( testsur, NULL, SDL_MapRGB( testsur->format, 0, 0, 0 ) ); @@ -415,7 +413,4 @@ void surface_testBlitBlend(void *arg) /* Clean up. */ SDL_FreeSurface( face ); - SDL_FreeSurface( testsur ); - - SDL_Quit(); } diff --git a/test/test-automation/xml_logger.c b/test/test-automation/xml_logger.c index 507c7a5f2..3481b588c 100644 --- a/test/test-automation/xml_logger.c +++ b/test/test-automation/xml_logger.c @@ -38,6 +38,7 @@ const char *numSuitesElementName = "numSuites"; const char *numTestElementName = "numTests"; const char *numPassedTestsElementName = "numPassedTests"; const char *numFailedTestsElementName = "numFailedTests"; +const char *numSkippedTestsElementName = "numSkippedTests"; const char *endTimeElementName = "endTime"; const char *totalRuntimeElementName = "totalRuntime"; const char *suiteElementName = "suite"; @@ -145,7 +146,7 @@ XMLRunStarted(int parameterCount, char *runnerParameters[], time_t eventTime, void XMLRunEnded(int testCount, int suiteCount, int testPassCount, int testFailCount, - time_t endTime, double totalRuntime) + int testSkippedCount, time_t endTime, double totalRuntime) { // log suite count char *output = XMLOpenElement(numSuitesElementName); @@ -187,7 +188,17 @@ XMLRunEnded(int testCount, int suiteCount, int testPassCount, int testFailCount, output = XMLCloseElement(numFailedTestsElementName); XMLOutputter(--indentLevel, YES, output); - // log end timte + // log skipped test count + output = XMLOpenElement(numSkippedTestsElementName); + XMLOutputter(indentLevel++, NO, output); + + output = XMLAddContent(IntToString(testSkippedCount)); + XMLOutputter(indentLevel, NO, output); + + output = XMLCloseElement(numSkippedTestsElementName); + XMLOutputter(--indentLevel, YES, output); + + // log end tite output = XMLOpenElement(endTimeElementName); XMLOutputter(indentLevel++, NO, output); @@ -342,6 +353,9 @@ XMLTestEnded(const char *testName, const char *suiteName, if(testResult) { if(testResult == 2) { output = XMLAddContent("failed. No assert"); + } + else if(testResult == 3) { + output = XMLAddContent("skipped"); } else { output = XMLAddContent("failed"); } diff --git a/test/test-automation/xml_logger.h b/test/test-automation/xml_logger.h index 518faa1da..6ca129e2e 100644 --- a/test/test-automation/xml_logger.h +++ b/test/test-automation/xml_logger.h @@ -24,7 +24,7 @@ void XMLRunStarted(int parameterCount, char *runnerParameters[], time_t eventTim * \param totalRuntime How long the execution took */ void XMLRunEnded(int testCount, int suiteCount, int testPassCount, int testFailCount, - time_t endTime, double totalRuntime); + int testSkippedCount, time_t endTime, double totalRuntime); /*! * Prints the data about the test suite that'll be executed next in XML