Harness kills hung tests (won't work with --in-proc option).
Added result description to test logs (tells why test failed,
such as exceeding its timeout).
1.1 --- a/test/test-automation/SDL_test.c Tue Jul 12 23:53:57 2011 +0300
1.2 +++ b/test/test-automation/SDL_test.c Wed Jul 13 19:51:25 2011 +0300
1.3 @@ -38,7 +38,7 @@
1.4 void
1.5 _InitTestEnvironment()
1.6 {
1.7 - _testReturnValue = 0;
1.8 + _testReturnValue = TEST_RESULT_PASS;
1.9 _testAssertsFailed = 0;
1.10 _testAssertsPassed = 0;
1.11 }
1.12 @@ -50,7 +50,7 @@
1.13 _testAssertsFailed, _testAssertsPassed, time(0));
1.14
1.15 if(_testAssertsFailed == 0 && _testAssertsPassed == 0) {
1.16 - _testReturnValue = 2;
1.17 + _testReturnValue = TEST_RESULT_NO_ASSERT;
1.18 }
1.19
1.20 return _testReturnValue;
1.21 @@ -75,7 +75,7 @@
1.22 if(expected != expected) {
1.23 AssertWithValues("AssertEquals", 0, buf, actual, expected, time(0));
1.24
1.25 - _testReturnValue = 1;
1.26 + _testReturnValue = TEST_RESULT_FAILURE;
1.27 _testAssertsFailed++;
1.28 } else {
1.29 AssertWithValues("AssertEquals", 1, buf,
1.30 @@ -97,7 +97,7 @@
1.31 if (!condition) {
1.32 Assert("AssertTrue", 0, buf, time(0));
1.33
1.34 - _testReturnValue = 1;
1.35 + _testReturnValue = TEST_RESULT_FAILURE;
1.36 _testAssertsFailed++;
1.37 } else {
1.38 Assert("AssertTrue", 1, buf, time(0));
1.39 @@ -133,7 +133,7 @@
1.40
1.41 Assert("AssertFail", 0, buf, time(0));
1.42
1.43 - _testReturnValue = 1;
1.44 + _testReturnValue = TEST_RESULT_FAILURE;
1.45 _testAssertsFailed++;
1.46 }
1.47
2.1 --- a/test/test-automation/SDL_test.h Tue Jul 12 23:53:57 2011 +0300
2.2 +++ b/test/test-automation/SDL_test.h Wed Jul 13 19:51:25 2011 +0300
2.3 @@ -39,6 +39,13 @@
2.4 #define TEST_ENABLED 1
2.5 #define TEST_DISABLED 0
2.6
2.7 +#define TEST_RESULT_PASS 0
2.8 +#define TEST_RESULT_FAILURE 1
2.9 +#define TEST_RESULT_NO_ASSERT 2
2.10 +#define TEST_RESULT_SKIPPED 3
2.11 +#define TEST_RESULT_KILLED 4
2.12 +#define TEST_RESULT_SETUP_FAILURE 5
2.13 +
2.14 /*!
2.15 * Holds information about a test case
2.16 */
3.1 --- a/test/test-automation/plain_logger.c Tue Jul 12 23:53:57 2011 +0300
3.2 +++ b/test/test-automation/plain_logger.c Wed Jul 13 19:51:25 2011 +0300
3.3 @@ -8,6 +8,7 @@
3.4
3.5 #include "logger_helpers.h"
3.6 #include "plain_logger.h"
3.7 +#include "SDL_test.h"
3.8
3.9 static int indentLevel;
3.10
3.11 @@ -88,17 +89,25 @@
3.12 PlainTestEnded(const char *testName, const char *suiteName,
3.13 int testResult, time_t endTime, double totalRuntime)
3.14 {
3.15 - if(testResult) {
3.16 - if(testResult == 2) {
3.17 + switch(testResult) {
3.18 + case TEST_RESULT_PASS:
3.19 + Output(--indentLevel, "%s: ok", testName);
3.20 + break;
3.21 + case TEST_RESULT_FAILURE:
3.22 + Output(--indentLevel, "%s: failed", testName);
3.23 + break;
3.24 + case TEST_RESULT_NO_ASSERT:
3.25 Output(--indentLevel, "%s: failed -> no assert", testName);
3.26 - }
3.27 - else if(testResult == 3) {
3.28 + break;
3.29 + case TEST_RESULT_SKIPPED:
3.30 Output(--indentLevel, "%s: skipped", testName);
3.31 - } else {
3.32 - Output(--indentLevel, "%s: failed", testName);
3.33 - }
3.34 - } else {
3.35 - Output(--indentLevel, "%s: ok", testName);
3.36 + break;
3.37 + case TEST_RESULT_KILLED:
3.38 + Output(--indentLevel, "%s: killed, exceeded timeout", testName);
3.39 + break;
3.40 + case TEST_RESULT_SETUP_FAILURE:
3.41 + Output(--indentLevel, "%s: killed, setup failure", testName);
3.42 + break;
3.43 }
3.44 }
3.45
4.1 --- a/test/test-automation/runner.c Tue Jul 12 23:53:57 2011 +0300
4.2 +++ b/test/test-automation/runner.c Wed Jul 13 19:51:25 2011 +0300
4.3 @@ -600,7 +600,7 @@
4.4
4.5 int cntFailedAsserts = testItem->countFailedAsserts();
4.6 if(cntFailedAsserts != 0) {
4.7 - return 3;
4.8 + return TEST_RESULT_SETUP_FAILURE;
4.9 }
4.10
4.11 testItem->testCase(0x0);
4.12 @@ -613,6 +613,22 @@
4.13 }
4.14
4.15 /*!
4.16 + * Kills test that hungs. Test hungs when its execution
4.17 + * takes longer than timeout specified for it.
4.18 + *
4.19 + * When test will be killed SIG_ALRM will be triggered and
4.20 + * it'll call this function which kills the test process.
4.21 + *
4.22 + * Note: if runner is executed with --in-proc then hung tests
4.23 + * can't be killed
4.24 + *
4.25 + * \param signum
4.26 + */
4.27 +void KillHungTest(int signum) {
4.28 + exit(TEST_RESULT_KILLED);
4.29 +}
4.30 +
4.31 +/*!
4.32 * Executes a test case. Loads the test, executes it and
4.33 * returns the tests return value to the caller.
4.34 *
4.35 @@ -621,13 +637,18 @@
4.36 */
4.37 int
4.38 ExecuteTest(TestCase *testItem) {
4.39 - int retVal = 1;
4.40 + int retVal = -1;
4.41
4.42 if(execute_inproc) {
4.43 retVal = RunTest(testItem);
4.44 } else {
4.45 int childpid = fork();
4.46 if(childpid == 0) {
4.47 + if(testItem->timeout > 0) {
4.48 + signal(SIGALRM, KillHungTest);
4.49 + alarm((unsigned int) testItem->timeout);
4.50 + }
4.51 +
4.52 exit(RunTest(testItem));
4.53 } else {
4.54 int stat_lock = -1;
4.55 @@ -641,6 +662,7 @@
4.56 }
4.57
4.58
4.59 +
4.60 /*!
4.61 * If using out-of-proc execution of tests. This function
4.62 * will handle the return value of the child process
5.1 --- a/test/test-automation/style.xsl Tue Jul 12 23:53:57 2011 +0300
5.2 +++ b/test/test-automation/style.xsl Wed Jul 13 19:51:25 2011 +0300
5.3 @@ -209,6 +209,11 @@
5.4 <xsl:value-of select="result"/>
5.5 </xsl:attribute><xsl:value-of select="result"/>
5.6 </span>
5.7 + <xsl:if test="resultDescription != ''">
5.8 + <span xml:space="preserve">
5.9 + (<xsl:value-of select="resultDescription"/>)
5.10 + </span>
5.11 + </xsl:if>
5.12 (Total runtime: <xsl:value-of select="totalRuntime"/> seconds)<br/>
5.13 Description: <span class="description"> <xsl:value-of select="description"/> </span><br/>
5.14 <span class="switch show-asserts" uid="{generate-id(assertSummary)}">[Show Assert Summary]</span><br/>
6.1 --- a/test/test-automation/testdummy/testdummy.c Tue Jul 12 23:53:57 2011 +0300
6.2 +++ b/test/test-automation/testdummy/testdummy.c Wed Jul 13 19:51:25 2011 +0300
6.3 @@ -32,7 +32,7 @@
6.4
6.5 /* Test case references */
6.6 static const TestCaseReference test1 =
6.7 - (TestCaseReference){ "dummycase1", "description", TEST_ENABLED, 0, 0};
6.8 + (TestCaseReference){ "dummycase1", "description", TEST_ENABLED, 0, 4};
6.9
6.10 static const TestCaseReference test2 =
6.11 (TestCaseReference){ "dummycase2", "description", TEST_ENABLED, 0, 0};
6.12 @@ -89,6 +89,7 @@
6.13 dummycase1(void *arg)
6.14 {
6.15 //AssertEquals(5, 5, "Assert message");
6.16 + while(1);
6.17 }
6.18
6.19 void
7.1 --- a/test/test-automation/xml_logger.c Tue Jul 12 23:53:57 2011 +0300
7.2 +++ b/test/test-automation/xml_logger.c Wed Jul 13 19:51:25 2011 +0300
7.3 @@ -26,6 +26,7 @@
7.4
7.5 #include "xml.h"
7.6 #include "logger_helpers.h"
7.7 +#include "SDL_test.h"
7.8
7.9 #include "xml_logger.h"
7.10
7.11 @@ -49,6 +50,7 @@
7.12 const char *nameElementName = "name";
7.13 const char *descriptionElementName = "description";
7.14 const char *resultElementName = "result";
7.15 +const char *resultDescriptionElementName = "resultDescription";
7.16 const char *assertElementName = "assert";
7.17 const char *messageElementName = "message";
7.18 const char *timeElementName = "time";
7.19 @@ -347,27 +349,61 @@
7.20 XMLTestEnded(const char *testName, const char *suiteName,
7.21 int testResult, time_t endTime, double totalRuntime)
7.22 {
7.23 + // Log test result
7.24 char *output = XMLOpenElement(resultElementName);
7.25 XMLOutputter(indentLevel++, NO, output);
7.26
7.27 - if(testResult) {
7.28 - if(testResult == 2) {
7.29 - output = XMLAddContent("failed. No assert");
7.30 - }
7.31 - else if(testResult == 3) {
7.32 + switch(testResult) {
7.33 + case TEST_RESULT_PASS:
7.34 + output = XMLAddContent("passed");
7.35 + break;
7.36 + case TEST_RESULT_FAILURE:
7.37 + output = XMLAddContent("failed");
7.38 + break;
7.39 + case TEST_RESULT_NO_ASSERT:
7.40 + output = XMLAddContent("failed");
7.41 + break;
7.42 + case TEST_RESULT_SKIPPED:
7.43 output = XMLAddContent("skipped");
7.44 - } else {
7.45 + break;
7.46 + case TEST_RESULT_KILLED:
7.47 output = XMLAddContent("failed");
7.48 - }
7.49 - XMLOutputter(indentLevel, NO, output);
7.50 - } else {
7.51 - output = XMLAddContent("passed");
7.52 - XMLOutputter(indentLevel, NO, output);
7.53 + break;
7.54 + case TEST_RESULT_SETUP_FAILURE:
7.55 + output = XMLAddContent("failed");
7.56 + break;
7.57 }
7.58 + XMLOutputter(indentLevel, NO, output);
7.59
7.60 output = XMLCloseElement(resultElementName);
7.61 XMLOutputter(--indentLevel, YES, output);
7.62
7.63 + // Log description of test result. Why the test failed,
7.64 + // if there's some specific reason
7.65 + output = XMLOpenElement(resultDescriptionElementName);
7.66 + XMLOutputter(indentLevel++, NO, output);
7.67 +
7.68 + switch(testResult) {
7.69 + case TEST_RESULT_PASS:
7.70 + case TEST_RESULT_FAILURE:
7.71 + case TEST_RESULT_SKIPPED:
7.72 + output = XMLAddContent("");
7.73 + break;
7.74 + case TEST_RESULT_NO_ASSERT:
7.75 + output = XMLAddContent("No assert");
7.76 + break;
7.77 + case TEST_RESULT_KILLED:
7.78 + output = XMLAddContent("Timeout exceeded");
7.79 + break;
7.80 + case TEST_RESULT_SETUP_FAILURE:
7.81 + output = XMLAddContent("Setup failure, couldn't be executed");
7.82 + break;
7.83 + }
7.84 + XMLOutputter(indentLevel, NO, output);
7.85 +
7.86 + output = XMLCloseElement(resultDescriptionElementName);
7.87 + XMLOutputter(--indentLevel, YES, output);
7.88 +
7.89 // log total runtime
7.90 output = XMLOpenElement(endTimeElementName);
7.91 XMLOutputter(indentLevel++, NO, output);