Harness kills hung tests (won't work with --in-proc option).

Added result description to test logs (tells why test failed, such as exceeding its timeout).
libsdl-org · Jul 13, 2011 · bbe8e9c · bbe8e9c
1 parent f7f5009
commit bbe8e9c
Show file tree

Hide file tree

Showing 7 changed files with 108 additions and 28 deletions.
diff --git a/test/test-automation/SDL_test.c b/test/test-automation/SDL_test.c
@@ -38,7 +38,7 @@ int _testAssertsPassed;
 void
 _InitTestEnvironment()
 {
-	_testReturnValue = 0;
+	_testReturnValue = TEST_RESULT_PASS;
 	_testAssertsFailed = 0;
 	_testAssertsPassed = 0;
 }
@@ -50,7 +50,7 @@ _QuitTestEnvironment()
                   _testAssertsFailed, _testAssertsPassed, time(0));
 
 	if(_testAssertsFailed == 0 && _testAssertsPassed == 0) {
-		_testReturnValue = 2;
+		_testReturnValue = TEST_RESULT_NO_ASSERT;
 	}
 
 	return _testReturnValue;
@@ -75,7 +75,7 @@ AssertEquals(int expected, int actual, char *message, ...)
    if(expected != expected) {
       AssertWithValues("AssertEquals", 0, buf, actual, expected, time(0));
 
-      _testReturnValue = 1;
+      _testReturnValue = TEST_RESULT_FAILURE;
       _testAssertsFailed++;
    } else {
 	   AssertWithValues("AssertEquals", 1, buf,
@@ -97,7 +97,7 @@ AssertTrue(int condition, char *message, ...)
    if (!condition) {
       Assert("AssertTrue", 0, buf, time(0));
 
-      _testReturnValue = 1;
+      _testReturnValue = TEST_RESULT_FAILURE;
       _testAssertsFailed++;
    } else {
 		Assert("AssertTrue", 1, buf, time(0));
@@ -133,7 +133,7 @@ AssertFail(char *message, ...)
 
    Assert("AssertFail", 0, buf, time(0));
 
-   _testReturnValue = 1;
+   _testReturnValue = TEST_RESULT_FAILURE;
    _testAssertsFailed++;
 }
 
diff --git a/test/test-automation/SDL_test.h b/test/test-automation/SDL_test.h
@@ -39,6 +39,13 @@ extern AssertFp testAssert;
 #define TEST_ENABLED  1
 #define TEST_DISABLED 0
 
+#define TEST_RESULT_PASS			0
+#define TEST_RESULT_FAILURE			1
+#define TEST_RESULT_NO_ASSERT		2
+#define TEST_RESULT_SKIPPED			3
+#define TEST_RESULT_KILLED			4
+#define TEST_RESULT_SETUP_FAILURE	5
+
 /*!
  * Holds information about a test case
  */

diff --git a/test/test-automation/plain_logger.c b/test/test-automation/plain_logger.c
@@ -8,6 +8,7 @@
 
 #include "logger_helpers.h"
 #include "plain_logger.h"
+#include "SDL_test.h"
 
 static int indentLevel;
 
@@ -88,17 +89,25 @@ void
 PlainTestEnded(const char *testName, const char *suiteName,
           int testResult, time_t endTime, double totalRuntime)
 {
-	if(testResult) {
-		if(testResult == 2) {
+	switch(testResult) {
+		case TEST_RESULT_PASS:
+			Output(--indentLevel, "%s: ok", testName);
+			break;
+		case TEST_RESULT_FAILURE:
+			Output(--indentLevel, "%s: failed", testName);
+			break;
+		case TEST_RESULT_NO_ASSERT:
 			Output(--indentLevel, "%s: failed -> no assert", testName);
-		}
-		else if(testResult == 3) {
+			break;
+		case TEST_RESULT_SKIPPED:
 			Output(--indentLevel, "%s: skipped", testName);
-		} else {
-			Output(--indentLevel, "%s: failed", testName);
-		}
-	} else {
-		Output(--indentLevel, "%s: ok", testName);
+			break;
+		case TEST_RESULT_KILLED:
+			Output(--indentLevel, "%s: killed, exceeded timeout", testName);
+			break;
+		case TEST_RESULT_SETUP_FAILURE:
+			Output(--indentLevel, "%s: killed, setup failure", testName);
+			break;
 	}
 }
 

diff --git a/test/test-automation/runner.c b/test/test-automation/runner.c
@@ -600,7 +600,7 @@ RunTest(TestCase *testItem) {
 
 	int cntFailedAsserts = testItem->countFailedAsserts();
 	if(cntFailedAsserts != 0) {
-		return 3;
+		return TEST_RESULT_SETUP_FAILURE;
 	}
 
 	testItem->testCase(0x0);
@@ -612,6 +612,22 @@ RunTest(TestCase *testItem) {
 	return testItem->quitTestEnvironment();
 }
 
+/*!
+ * Kills test that hungs. Test hungs when its execution
+ * takes longer than timeout specified for it.
+ *
+ * When test will be killed SIG_ALRM will be triggered and
+ * it'll call this function which kills the test process.
+ *
+ * Note: if runner is executed with --in-proc then hung tests
+ * can't be killed
+ *
+ * \param signum
+ */
+void KillHungTest(int signum) {
+	exit(TEST_RESULT_KILLED);
+}
+
 /*!
  * Executes a test case. Loads the test, executes it and
  * returns the tests return value to the caller.
@@ -621,13 +637,18 @@ RunTest(TestCase *testItem) {
  */
 int
 ExecuteTest(TestCase *testItem) {
-	int retVal = 1;
+	int retVal = -1;
 
 	if(execute_inproc) {
 		retVal = RunTest(testItem);
 	} else {
 		int childpid = fork();
 		if(childpid == 0) {
+			if(testItem->timeout > 0) {
+				signal(SIGALRM, KillHungTest);
+				alarm((unsigned int) testItem->timeout);
+			}
+
 			exit(RunTest(testItem));
 		} else {
 			int stat_lock = -1;
@@ -641,6 +662,7 @@ ExecuteTest(TestCase *testItem) {
 }
 
 
+
 /*!
  * If using out-of-proc execution of tests. This function
  * will handle the return value of the child process

diff --git a/test/test-automation/style.xsl b/test/test-automation/style.xsl
@@ -209,6 +209,11 @@ div, h1 {
 			    <xsl:value-of select="result"/> 
 	           </xsl:attribute><xsl:value-of select="result"/> 
 	        </span> 
+			<xsl:if test="resultDescription != ''">
+			  <span xml:space="preserve">
+			  	 (<xsl:value-of select="resultDescription"/>)  
+			  </span>
+			</xsl:if>
 			(Total runtime: <xsl:value-of select="totalRuntime"/> seconds)<br/>
 			Description: <span class="description"> <xsl:value-of select="description"/> </span><br/>
 		      <span class="switch show-asserts" uid="{generate-id(assertSummary)}">[Show Assert Summary]</span><br/>

diff --git a/test/test-automation/testdummy/testdummy.c b/test/test-automation/testdummy/testdummy.c
@@ -32,7 +32,7 @@
 
 /* Test case references */
 static const TestCaseReference test1 =
-		(TestCaseReference){ "dummycase1", "description", TEST_ENABLED, 0, 0};
+		(TestCaseReference){ "dummycase1", "description", TEST_ENABLED, 0, 4};
 
 static const TestCaseReference test2 =
 		(TestCaseReference){ "dummycase2", "description", TEST_ENABLED, 0, 0};
@@ -89,6 +89,7 @@ void
 dummycase1(void *arg)
 {
 	//AssertEquals(5, 5, "Assert message");
+	while(1);
 }
 
 void

diff --git a/test/test-automation/xml_logger.c b/test/test-automation/xml_logger.c
@@ -26,6 +26,7 @@
 
 #include "xml.h"
 #include "logger_helpers.h"
+#include "SDL_test.h"
 
 #include "xml_logger.h"
 
@@ -49,6 +50,7 @@ const char *testElementName = "test";
 const char *nameElementName = "name";
 const char *descriptionElementName = "description";
 const char *resultElementName = "result";
+const char *resultDescriptionElementName = "resultDescription";
 const char *assertElementName = "assert";
 const char *messageElementName = "message";
 const char *timeElementName = "time";
@@ -347,27 +349,61 @@ void
 XMLTestEnded(const char *testName, const char *suiteName,
           int testResult, time_t endTime, double totalRuntime)
 {
+	// Log test result
 	char *output = XMLOpenElement(resultElementName);
 	XMLOutputter(indentLevel++, NO, output);
 
-	if(testResult) {
-		if(testResult == 2) {
-			output = XMLAddContent("failed. No assert");
-		}
-		else if(testResult == 3) {
+	switch(testResult) {
+		case TEST_RESULT_PASS:
+			output = XMLAddContent("passed");
+			break;
+		case TEST_RESULT_FAILURE:
+			output = XMLAddContent("failed");
+			break;
+		case TEST_RESULT_NO_ASSERT:
+			output = XMLAddContent("failed");
+			break;
+		case TEST_RESULT_SKIPPED:
 			output = XMLAddContent("skipped");
-		} else {
+			break;
+		case TEST_RESULT_KILLED:
 			output = XMLAddContent("failed");
-		}
-		XMLOutputter(indentLevel, NO, output);
-	} else {
-		output = XMLAddContent("passed");
-		XMLOutputter(indentLevel, NO, output);
+			break;
+		case TEST_RESULT_SETUP_FAILURE:
+			output = XMLAddContent("failed");
+			break;
 	}
+	XMLOutputter(indentLevel, NO, output);
 
 	output = XMLCloseElement(resultElementName);
 	XMLOutputter(--indentLevel, YES, output);
 
+	// Log description of test result. Why the test failed,
+	// if there's some specific reason
+	output = XMLOpenElement(resultDescriptionElementName);
+	XMLOutputter(indentLevel++, NO, output);
+
+	switch(testResult) {
+		case TEST_RESULT_PASS:
+		case TEST_RESULT_FAILURE:
+		case TEST_RESULT_SKIPPED:
+			output = XMLAddContent("");
+			break;
+		case TEST_RESULT_NO_ASSERT:
+			output = XMLAddContent("No assert");
+			break;
+		case TEST_RESULT_KILLED:
+			output = XMLAddContent("Timeout exceeded");
+			break;
+		case TEST_RESULT_SETUP_FAILURE:
+			output = XMLAddContent("Setup failure, couldn't be executed");
+			break;
+	}
+	XMLOutputter(indentLevel, NO, output);
+
+	output = XMLCloseElement(resultDescriptionElementName);
+	XMLOutputter(--indentLevel, YES, output);
+
 	// log total runtime
 	output = XMLOpenElement(endTimeElementName);
 	XMLOutputter(indentLevel++, NO, output);