From bbe8e9cbe43e5d0846229241b37f4f948449606d Mon Sep 17 00:00:00 2001 From: Markus Kauppila Date: Wed, 13 Jul 2011 19:51:25 +0300 Subject: [PATCH] Harness kills hung tests (won't work with --in-proc option). Added result description to test logs (tells why test failed, such as exceeding its timeout). --- test/test-automation/SDL_test.c | 10 ++-- test/test-automation/SDL_test.h | 7 +++ test/test-automation/plain_logger.c | 27 ++++++---- test/test-automation/runner.c | 26 +++++++++- test/test-automation/style.xsl | 5 ++ test/test-automation/testdummy/testdummy.c | 3 +- test/test-automation/xml_logger.c | 58 ++++++++++++++++++---- 7 files changed, 108 insertions(+), 28 deletions(-) diff --git a/test/test-automation/SDL_test.c b/test/test-automation/SDL_test.c index e9811199f..0564d3017 100644 --- a/test/test-automation/SDL_test.c +++ b/test/test-automation/SDL_test.c @@ -38,7 +38,7 @@ int _testAssertsPassed; void _InitTestEnvironment() { - _testReturnValue = 0; + _testReturnValue = TEST_RESULT_PASS; _testAssertsFailed = 0; _testAssertsPassed = 0; } @@ -50,7 +50,7 @@ _QuitTestEnvironment() _testAssertsFailed, _testAssertsPassed, time(0)); if(_testAssertsFailed == 0 && _testAssertsPassed == 0) { - _testReturnValue = 2; + _testReturnValue = TEST_RESULT_NO_ASSERT; } return _testReturnValue; @@ -75,7 +75,7 @@ AssertEquals(int expected, int actual, char *message, ...) if(expected != expected) { AssertWithValues("AssertEquals", 0, buf, actual, expected, time(0)); - _testReturnValue = 1; + _testReturnValue = TEST_RESULT_FAILURE; _testAssertsFailed++; } else { AssertWithValues("AssertEquals", 1, buf, @@ -97,7 +97,7 @@ AssertTrue(int condition, char *message, ...) if (!condition) { Assert("AssertTrue", 0, buf, time(0)); - _testReturnValue = 1; + _testReturnValue = TEST_RESULT_FAILURE; _testAssertsFailed++; } else { Assert("AssertTrue", 1, buf, time(0)); @@ -133,7 +133,7 @@ AssertFail(char *message, ...) Assert("AssertFail", 0, buf, time(0)); - _testReturnValue = 1; + _testReturnValue = TEST_RESULT_FAILURE; _testAssertsFailed++; } diff --git a/test/test-automation/SDL_test.h b/test/test-automation/SDL_test.h index 041670f79..e56c2a8d9 100644 --- a/test/test-automation/SDL_test.h +++ b/test/test-automation/SDL_test.h @@ -39,6 +39,13 @@ extern AssertFp testAssert; #define TEST_ENABLED 1 #define TEST_DISABLED 0 +#define TEST_RESULT_PASS 0 +#define TEST_RESULT_FAILURE 1 +#define TEST_RESULT_NO_ASSERT 2 +#define TEST_RESULT_SKIPPED 3 +#define TEST_RESULT_KILLED 4 +#define TEST_RESULT_SETUP_FAILURE 5 + /*! * Holds information about a test case */ diff --git a/test/test-automation/plain_logger.c b/test/test-automation/plain_logger.c index 5cf4d2667..44997141b 100644 --- a/test/test-automation/plain_logger.c +++ b/test/test-automation/plain_logger.c @@ -8,6 +8,7 @@ #include "logger_helpers.h" #include "plain_logger.h" +#include "SDL_test.h" static int indentLevel; @@ -88,17 +89,25 @@ void PlainTestEnded(const char *testName, const char *suiteName, int testResult, time_t endTime, double totalRuntime) { - if(testResult) { - if(testResult == 2) { + switch(testResult) { + case TEST_RESULT_PASS: + Output(--indentLevel, "%s: ok", testName); + break; + case TEST_RESULT_FAILURE: + Output(--indentLevel, "%s: failed", testName); + break; + case TEST_RESULT_NO_ASSERT: Output(--indentLevel, "%s: failed -> no assert", testName); - } - else if(testResult == 3) { + break; + case TEST_RESULT_SKIPPED: Output(--indentLevel, "%s: skipped", testName); - } else { - Output(--indentLevel, "%s: failed", testName); - } - } else { - Output(--indentLevel, "%s: ok", testName); + break; + case TEST_RESULT_KILLED: + Output(--indentLevel, "%s: killed, exceeded timeout", testName); + break; + case TEST_RESULT_SETUP_FAILURE: + Output(--indentLevel, "%s: killed, setup failure", testName); + break; } } diff --git a/test/test-automation/runner.c b/test/test-automation/runner.c index a989e01c7..72d98504b 100644 --- a/test/test-automation/runner.c +++ b/test/test-automation/runner.c @@ -600,7 +600,7 @@ RunTest(TestCase *testItem) { int cntFailedAsserts = testItem->countFailedAsserts(); if(cntFailedAsserts != 0) { - return 3; + return TEST_RESULT_SETUP_FAILURE; } testItem->testCase(0x0); @@ -612,6 +612,22 @@ RunTest(TestCase *testItem) { return testItem->quitTestEnvironment(); } +/*! + * Kills test that hungs. Test hungs when its execution + * takes longer than timeout specified for it. + * + * When test will be killed SIG_ALRM will be triggered and + * it'll call this function which kills the test process. + * + * Note: if runner is executed with --in-proc then hung tests + * can't be killed + * + * \param signum + */ +void KillHungTest(int signum) { + exit(TEST_RESULT_KILLED); +} + /*! * Executes a test case. Loads the test, executes it and * returns the tests return value to the caller. @@ -621,13 +637,18 @@ RunTest(TestCase *testItem) { */ int ExecuteTest(TestCase *testItem) { - int retVal = 1; + int retVal = -1; if(execute_inproc) { retVal = RunTest(testItem); } else { int childpid = fork(); if(childpid == 0) { + if(testItem->timeout > 0) { + signal(SIGALRM, KillHungTest); + alarm((unsigned int) testItem->timeout); + } + exit(RunTest(testItem)); } else { int stat_lock = -1; @@ -641,6 +662,7 @@ ExecuteTest(TestCase *testItem) { } + /*! * If using out-of-proc execution of tests. This function * will handle the return value of the child process diff --git a/test/test-automation/style.xsl b/test/test-automation/style.xsl index 367fa4d7f..69515f4f7 100644 --- a/test/test-automation/style.xsl +++ b/test/test-automation/style.xsl @@ -209,6 +209,11 @@ div, h1 { + + + () + + (Total runtime: seconds)
Description:
[Show Assert Summary]
diff --git a/test/test-automation/testdummy/testdummy.c b/test/test-automation/testdummy/testdummy.c index 26779e4d1..dd8e864a6 100644 --- a/test/test-automation/testdummy/testdummy.c +++ b/test/test-automation/testdummy/testdummy.c @@ -32,7 +32,7 @@ /* Test case references */ static const TestCaseReference test1 = - (TestCaseReference){ "dummycase1", "description", TEST_ENABLED, 0, 0}; + (TestCaseReference){ "dummycase1", "description", TEST_ENABLED, 0, 4}; static const TestCaseReference test2 = (TestCaseReference){ "dummycase2", "description", TEST_ENABLED, 0, 0}; @@ -89,6 +89,7 @@ void dummycase1(void *arg) { //AssertEquals(5, 5, "Assert message"); + while(1); } void diff --git a/test/test-automation/xml_logger.c b/test/test-automation/xml_logger.c index 3481b588c..03aae4a11 100644 --- a/test/test-automation/xml_logger.c +++ b/test/test-automation/xml_logger.c @@ -26,6 +26,7 @@ #include "xml.h" #include "logger_helpers.h" +#include "SDL_test.h" #include "xml_logger.h" @@ -49,6 +50,7 @@ const char *testElementName = "test"; const char *nameElementName = "name"; const char *descriptionElementName = "description"; const char *resultElementName = "result"; +const char *resultDescriptionElementName = "resultDescription"; const char *assertElementName = "assert"; const char *messageElementName = "message"; const char *timeElementName = "time"; @@ -347,27 +349,61 @@ void XMLTestEnded(const char *testName, const char *suiteName, int testResult, time_t endTime, double totalRuntime) { + // Log test result char *output = XMLOpenElement(resultElementName); XMLOutputter(indentLevel++, NO, output); - if(testResult) { - if(testResult == 2) { - output = XMLAddContent("failed. No assert"); - } - else if(testResult == 3) { + switch(testResult) { + case TEST_RESULT_PASS: + output = XMLAddContent("passed"); + break; + case TEST_RESULT_FAILURE: + output = XMLAddContent("failed"); + break; + case TEST_RESULT_NO_ASSERT: + output = XMLAddContent("failed"); + break; + case TEST_RESULT_SKIPPED: output = XMLAddContent("skipped"); - } else { + break; + case TEST_RESULT_KILLED: output = XMLAddContent("failed"); - } - XMLOutputter(indentLevel, NO, output); - } else { - output = XMLAddContent("passed"); - XMLOutputter(indentLevel, NO, output); + break; + case TEST_RESULT_SETUP_FAILURE: + output = XMLAddContent("failed"); + break; } + XMLOutputter(indentLevel, NO, output); output = XMLCloseElement(resultElementName); XMLOutputter(--indentLevel, YES, output); + // Log description of test result. Why the test failed, + // if there's some specific reason + output = XMLOpenElement(resultDescriptionElementName); + XMLOutputter(indentLevel++, NO, output); + + switch(testResult) { + case TEST_RESULT_PASS: + case TEST_RESULT_FAILURE: + case TEST_RESULT_SKIPPED: + output = XMLAddContent(""); + break; + case TEST_RESULT_NO_ASSERT: + output = XMLAddContent("No assert"); + break; + case TEST_RESULT_KILLED: + output = XMLAddContent("Timeout exceeded"); + break; + case TEST_RESULT_SETUP_FAILURE: + output = XMLAddContent("Setup failure, couldn't be executed"); + break; + } + XMLOutputter(indentLevel, NO, output); + + output = XMLCloseElement(resultDescriptionElementName); + XMLOutputter(--indentLevel, YES, output); + // log total runtime output = XMLOpenElement(endTimeElementName); XMLOutputter(indentLevel++, NO, output);