Harness kills hung tests (won't work with --in-proc option).
authorMarkus Kauppila
Wed, 13 Jul 2011 19:51:25 +0300
changeset 5758e497802a71b4
parent 5757 a97102f40230
child 5759 59615ddece72
Harness kills hung tests (won't work with --in-proc option).
Added result description to test logs (tells why test failed,
such as exceeding its timeout).
test/test-automation/SDL_test.c
test/test-automation/SDL_test.h
test/test-automation/plain_logger.c
test/test-automation/runner.c
test/test-automation/style.xsl
test/test-automation/testdummy/testdummy.c
test/test-automation/xml_logger.c
     1.1 --- a/test/test-automation/SDL_test.c	Tue Jul 12 23:53:57 2011 +0300
     1.2 +++ b/test/test-automation/SDL_test.c	Wed Jul 13 19:51:25 2011 +0300
     1.3 @@ -38,7 +38,7 @@
     1.4  void
     1.5  _InitTestEnvironment()
     1.6  {
     1.7 -	_testReturnValue = 0;
     1.8 +	_testReturnValue = TEST_RESULT_PASS;
     1.9  	_testAssertsFailed = 0;
    1.10  	_testAssertsPassed = 0;
    1.11  }
    1.12 @@ -50,7 +50,7 @@
    1.13                    _testAssertsFailed, _testAssertsPassed, time(0));
    1.14  
    1.15  	if(_testAssertsFailed == 0 && _testAssertsPassed == 0) {
    1.16 -		_testReturnValue = 2;
    1.17 +		_testReturnValue = TEST_RESULT_NO_ASSERT;
    1.18  	}
    1.19  
    1.20  	return _testReturnValue;
    1.21 @@ -75,7 +75,7 @@
    1.22     if(expected != expected) {
    1.23        AssertWithValues("AssertEquals", 0, buf, actual, expected, time(0));
    1.24  
    1.25 -      _testReturnValue = 1;
    1.26 +      _testReturnValue = TEST_RESULT_FAILURE;
    1.27        _testAssertsFailed++;
    1.28     } else {
    1.29  	   AssertWithValues("AssertEquals", 1, buf,
    1.30 @@ -97,7 +97,7 @@
    1.31     if (!condition) {
    1.32        Assert("AssertTrue", 0, buf, time(0));
    1.33  
    1.34 -      _testReturnValue = 1;
    1.35 +      _testReturnValue = TEST_RESULT_FAILURE;
    1.36        _testAssertsFailed++;
    1.37     } else {
    1.38  		Assert("AssertTrue", 1, buf, time(0));
    1.39 @@ -133,7 +133,7 @@
    1.40  
    1.41     Assert("AssertFail", 0, buf, time(0));
    1.42  
    1.43 -   _testReturnValue = 1;
    1.44 +   _testReturnValue = TEST_RESULT_FAILURE;
    1.45     _testAssertsFailed++;
    1.46  }
    1.47  
     2.1 --- a/test/test-automation/SDL_test.h	Tue Jul 12 23:53:57 2011 +0300
     2.2 +++ b/test/test-automation/SDL_test.h	Wed Jul 13 19:51:25 2011 +0300
     2.3 @@ -39,6 +39,13 @@
     2.4  #define TEST_ENABLED  1
     2.5  #define TEST_DISABLED 0
     2.6  
     2.7 +#define TEST_RESULT_PASS			0
     2.8 +#define TEST_RESULT_FAILURE			1
     2.9 +#define TEST_RESULT_NO_ASSERT		2
    2.10 +#define TEST_RESULT_SKIPPED			3
    2.11 +#define TEST_RESULT_KILLED			4
    2.12 +#define TEST_RESULT_SETUP_FAILURE	5
    2.13 +
    2.14  /*!
    2.15   * Holds information about a test case
    2.16   */
     3.1 --- a/test/test-automation/plain_logger.c	Tue Jul 12 23:53:57 2011 +0300
     3.2 +++ b/test/test-automation/plain_logger.c	Wed Jul 13 19:51:25 2011 +0300
     3.3 @@ -8,6 +8,7 @@
     3.4  
     3.5  #include "logger_helpers.h"
     3.6  #include "plain_logger.h"
     3.7 +#include "SDL_test.h"
     3.8  
     3.9  static int indentLevel;
    3.10  
    3.11 @@ -88,17 +89,25 @@
    3.12  PlainTestEnded(const char *testName, const char *suiteName,
    3.13            int testResult, time_t endTime, double totalRuntime)
    3.14  {
    3.15 -	if(testResult) {
    3.16 -		if(testResult == 2) {
    3.17 +	switch(testResult) {
    3.18 +		case TEST_RESULT_PASS:
    3.19 +			Output(--indentLevel, "%s: ok", testName);
    3.20 +			break;
    3.21 +		case TEST_RESULT_FAILURE:
    3.22 +			Output(--indentLevel, "%s: failed", testName);
    3.23 +			break;
    3.24 +		case TEST_RESULT_NO_ASSERT:
    3.25  			Output(--indentLevel, "%s: failed -> no assert", testName);
    3.26 -		}
    3.27 -		else if(testResult == 3) {
    3.28 +			break;
    3.29 +		case TEST_RESULT_SKIPPED:
    3.30  			Output(--indentLevel, "%s: skipped", testName);
    3.31 -		} else {
    3.32 -			Output(--indentLevel, "%s: failed", testName);
    3.33 -		}
    3.34 -	} else {
    3.35 -		Output(--indentLevel, "%s: ok", testName);
    3.36 +			break;
    3.37 +		case TEST_RESULT_KILLED:
    3.38 +			Output(--indentLevel, "%s: killed, exceeded timeout", testName);
    3.39 +			break;
    3.40 +		case TEST_RESULT_SETUP_FAILURE:
    3.41 +			Output(--indentLevel, "%s: killed, setup failure", testName);
    3.42 +			break;
    3.43  	}
    3.44  }
    3.45  
     4.1 --- a/test/test-automation/runner.c	Tue Jul 12 23:53:57 2011 +0300
     4.2 +++ b/test/test-automation/runner.c	Wed Jul 13 19:51:25 2011 +0300
     4.3 @@ -600,7 +600,7 @@
     4.4  
     4.5  	int cntFailedAsserts = testItem->countFailedAsserts();
     4.6  	if(cntFailedAsserts != 0) {
     4.7 -		return 3;
     4.8 +		return TEST_RESULT_SETUP_FAILURE;
     4.9  	}
    4.10  
    4.11  	testItem->testCase(0x0);
    4.12 @@ -613,6 +613,22 @@
    4.13  }
    4.14  
    4.15  /*!
    4.16 + * Kills test that hungs. Test hungs when its execution
    4.17 + * takes longer than timeout specified for it.
    4.18 + *
    4.19 + * When test will be killed SIG_ALRM will be triggered and
    4.20 + * it'll call this function which kills the test process.
    4.21 + *
    4.22 + * Note: if runner is executed with --in-proc then hung tests
    4.23 + * can't be killed
    4.24 + *
    4.25 + * \param signum
    4.26 + */
    4.27 +void KillHungTest(int signum) {
    4.28 +	exit(TEST_RESULT_KILLED);
    4.29 +}
    4.30 +
    4.31 +/*!
    4.32   * Executes a test case. Loads the test, executes it and
    4.33   * returns the tests return value to the caller.
    4.34   *
    4.35 @@ -621,13 +637,18 @@
    4.36   */
    4.37  int
    4.38  ExecuteTest(TestCase *testItem) {
    4.39 -	int retVal = 1;
    4.40 +	int retVal = -1;
    4.41  
    4.42  	if(execute_inproc) {
    4.43  		retVal = RunTest(testItem);
    4.44  	} else {
    4.45  		int childpid = fork();
    4.46  		if(childpid == 0) {
    4.47 +			if(testItem->timeout > 0) {
    4.48 +				signal(SIGALRM, KillHungTest);
    4.49 +				alarm((unsigned int) testItem->timeout);
    4.50 +			}
    4.51 +
    4.52  			exit(RunTest(testItem));
    4.53  		} else {
    4.54  			int stat_lock = -1;
    4.55 @@ -641,6 +662,7 @@
    4.56  }
    4.57  
    4.58  
    4.59 +
    4.60  /*!
    4.61   * If using out-of-proc execution of tests. This function
    4.62   * will handle the return value of the child process
     5.1 --- a/test/test-automation/style.xsl	Tue Jul 12 23:53:57 2011 +0300
     5.2 +++ b/test/test-automation/style.xsl	Wed Jul 13 19:51:25 2011 +0300
     5.3 @@ -209,6 +209,11 @@
     5.4  			    <xsl:value-of select="result"/> 
     5.5  	           </xsl:attribute><xsl:value-of select="result"/> 
     5.6  	        </span> 
     5.7 +			<xsl:if test="resultDescription != ''">
     5.8 +			  <span xml:space="preserve">
     5.9 +			  	 (<xsl:value-of select="resultDescription"/>)  
    5.10 +			  </span>
    5.11 +			</xsl:if>
    5.12  			(Total runtime: <xsl:value-of select="totalRuntime"/> seconds)<br/>
    5.13  			Description: <span class="description"> <xsl:value-of select="description"/> </span><br/>
    5.14  		      <span class="switch show-asserts" uid="{generate-id(assertSummary)}">[Show Assert Summary]</span><br/>
     6.1 --- a/test/test-automation/testdummy/testdummy.c	Tue Jul 12 23:53:57 2011 +0300
     6.2 +++ b/test/test-automation/testdummy/testdummy.c	Wed Jul 13 19:51:25 2011 +0300
     6.3 @@ -32,7 +32,7 @@
     6.4  
     6.5  /* Test case references */
     6.6  static const TestCaseReference test1 =
     6.7 -		(TestCaseReference){ "dummycase1", "description", TEST_ENABLED, 0, 0};
     6.8 +		(TestCaseReference){ "dummycase1", "description", TEST_ENABLED, 0, 4};
     6.9  
    6.10  static const TestCaseReference test2 =
    6.11  		(TestCaseReference){ "dummycase2", "description", TEST_ENABLED, 0, 0};
    6.12 @@ -89,6 +89,7 @@
    6.13  dummycase1(void *arg)
    6.14  {
    6.15  	//AssertEquals(5, 5, "Assert message");
    6.16 +	while(1);
    6.17  }
    6.18  
    6.19  void
     7.1 --- a/test/test-automation/xml_logger.c	Tue Jul 12 23:53:57 2011 +0300
     7.2 +++ b/test/test-automation/xml_logger.c	Wed Jul 13 19:51:25 2011 +0300
     7.3 @@ -26,6 +26,7 @@
     7.4  
     7.5  #include "xml.h"
     7.6  #include "logger_helpers.h"
     7.7 +#include "SDL_test.h"
     7.8  
     7.9  #include "xml_logger.h"
    7.10  
    7.11 @@ -49,6 +50,7 @@
    7.12  const char *nameElementName = "name";
    7.13  const char *descriptionElementName = "description";
    7.14  const char *resultElementName = "result";
    7.15 +const char *resultDescriptionElementName = "resultDescription";
    7.16  const char *assertElementName = "assert";
    7.17  const char *messageElementName = "message";
    7.18  const char *timeElementName = "time";
    7.19 @@ -347,27 +349,61 @@
    7.20  XMLTestEnded(const char *testName, const char *suiteName,
    7.21            int testResult, time_t endTime, double totalRuntime)
    7.22  {
    7.23 +	// Log test result
    7.24  	char *output = XMLOpenElement(resultElementName);
    7.25  	XMLOutputter(indentLevel++, NO, output);
    7.26  
    7.27 -	if(testResult) {
    7.28 -		if(testResult == 2) {
    7.29 -			output = XMLAddContent("failed. No assert");
    7.30 -		}
    7.31 -		else if(testResult == 3) {
    7.32 +	switch(testResult) {
    7.33 +		case TEST_RESULT_PASS:
    7.34 +			output = XMLAddContent("passed");
    7.35 +			break;
    7.36 +		case TEST_RESULT_FAILURE:
    7.37 +			output = XMLAddContent("failed");
    7.38 +			break;
    7.39 +		case TEST_RESULT_NO_ASSERT:
    7.40 +			output = XMLAddContent("failed");
    7.41 +			break;
    7.42 +		case TEST_RESULT_SKIPPED:
    7.43  			output = XMLAddContent("skipped");
    7.44 -		} else {
    7.45 +			break;
    7.46 +		case TEST_RESULT_KILLED:
    7.47  			output = XMLAddContent("failed");
    7.48 -		}
    7.49 -		XMLOutputter(indentLevel, NO, output);
    7.50 -	} else {
    7.51 -		output = XMLAddContent("passed");
    7.52 -		XMLOutputter(indentLevel, NO, output);
    7.53 +			break;
    7.54 +		case TEST_RESULT_SETUP_FAILURE:
    7.55 +			output = XMLAddContent("failed");
    7.56 +			break;
    7.57  	}
    7.58 +	XMLOutputter(indentLevel, NO, output);
    7.59  
    7.60  	output = XMLCloseElement(resultElementName);
    7.61  	XMLOutputter(--indentLevel, YES, output);
    7.62  
    7.63 +	// Log description of test result. Why the test failed,
    7.64 +	// if there's some specific reason
    7.65 +	output = XMLOpenElement(resultDescriptionElementName);
    7.66 +	XMLOutputter(indentLevel++, NO, output);
    7.67 +
    7.68 +	switch(testResult) {
    7.69 +		case TEST_RESULT_PASS:
    7.70 +		case TEST_RESULT_FAILURE:
    7.71 +		case TEST_RESULT_SKIPPED:
    7.72 +			output = XMLAddContent("");
    7.73 +			break;
    7.74 +		case TEST_RESULT_NO_ASSERT:
    7.75 +			output = XMLAddContent("No assert");
    7.76 +			break;
    7.77 +		case TEST_RESULT_KILLED:
    7.78 +			output = XMLAddContent("Timeout exceeded");
    7.79 +			break;
    7.80 +		case TEST_RESULT_SETUP_FAILURE:
    7.81 +			output = XMLAddContent("Setup failure, couldn't be executed");
    7.82 +			break;
    7.83 +	}
    7.84 +	XMLOutputter(indentLevel, NO, output);
    7.85 +
    7.86 +	output = XMLCloseElement(resultDescriptionElementName);
    7.87 +	XMLOutputter(--indentLevel, YES, output);
    7.88 +
    7.89  	// log total runtime
    7.90  	output = XMLOpenElement(endTimeElementName);
    7.91  	XMLOutputter(indentLevel++, NO, output);