test/test-automation/runner.c
author Markus Kauppila
Sat, 04 Jun 2011 17:58:46 +0300
changeset 5679 10a5199c4169
parent 5678 50401a6c3c95
child 5680 e61bdd43a2c3
permissions -rw-r--r--
Test case fails if it doesn't contain any asserts.
     1 /*
     2   Copyright (C) 2011 Markus Kauppila <markus.kauppila@gmail.com>
     3 
     4   This software is provided 'as-is', without any express or implied
     5   warranty.  In no event will the authors be held liable for any damages
     6   arising from the use of this software.
     7 
     8   Permission is granted to anyone to use this software for any purpose,
     9   including commercial applications, and to alter it and redistribute it
    10   freely, subject to the following restrictions:
    11 
    12   1. The origin of this software must not be misrepresented; you must not
    13      claim that you wrote the original software. If you use this software
    14      in a product, an acknowledgment in the product documentation would be
    15      appreciated but is not required.
    16   2. Altered source versions must be plainly marked as such, and must not be
    17      misrepresented as being the original software.
    18   3. This notice may not be removed or altered from any source distribution.
    19 */
    20 
    21 #include "SDL/SDL.h"
    22 
    23 #include <stdio.h>
    24 #include <stdlib.h>
    25 #include <unistd.h>
    26 #include <sys/types.h>
    27 
    28 #include "SDL_test.h"
    29 
    30 //!< Function pointer to a test case function
    31 typedef void (*TestCase)(void *arg);
    32 //!< Function pointer to a test case init function
    33 typedef void (*TestCaseInit)(void);
    34 //!< Function pointer to a test case quit function
    35 typedef int  (*TestCaseQuit)(void);
    36 
    37 //!< Flag for executing tests in-process
    38 static int execute_inproc = 0;
    39 
    40 /*!
    41  * Returns the name for the dynamic library
    42  * which implements the test suite.
    43  *
    44  * (in the future: scans the test/ directory and
    45  * returns the names of the dynamic libraries
    46  * implementing the test suites)
    47  *
    48  * \return Name of the dummy test suite
    49  */
    50 char *
    51 ScanForTestSuites() {
    52 #if defined(linux) || defined( __linux)
    53 	char *libName = "tests/libtest.so";
    54 #else
    55 	char *libName = "tests/libtest.dylib";
    56 #endif
    57 	return libName;
    58 }
    59 
    60 
    61 /*!
    62  * Loads test suite which is implemented as dynamic library.
    63  *
    64  * \param test0,330
    65  *
    66  * \return Pointer to loaded test suite, or NULL if library could not be loaded
    67  */
    68 void *
    69 LoadTestSuite(char *testSuiteName)
    70 {
    71 	void *library = SDL_LoadObject(testSuiteName);
    72 	if(library == NULL) {
    73 		fprintf(stderr, "Loading %s failed\n", testSuiteName);
    74 		fprintf(stderr, "%s\n", SDL_GetError());
    75 	}
    76 
    77 	return library;
    78 }
    79 
    80 /*!
    81  * Loads the test case references from the given test suite.
    82 
    83  * \param library Previously loaded dynamic library AKA test suite
    84  * \return Pointer to array of TestCaseReferences or NULL if function failed
    85  */
    86 TestCaseReference **
    87 QueryTestCases(void *library)
    88 {
    89 	TestCaseReference **(*suite)(void);
    90 
    91 	suite = (TestCaseReference **(*)(void)) SDL_LoadFunction(library, "QueryTestSuite");
    92 	if(suite == NULL) {
    93 		fprintf(stderr, "Loading QueryTestCaseReferences() failed.\n");
    94 		fprintf(stderr, "%s\n", SDL_GetError());
    95 	}
    96 
    97 	TestCaseReference **tests = suite();
    98 	if(tests == NULL) {
    99 		fprintf(stderr, "Failed to load test references.\n");
   100 		fprintf(stderr, "%s\n", SDL_GetError());
   101 	}
   102 
   103 	return tests;
   104 }
   105 
   106 
   107 /*!
   108  * Loads test case from a test suite
   109  *
   110  * \param suite a test suite
   111  * \param testName Name of the test that is going to be loaded
   112  *
   113  * \return Function Pointer (TestCase) to loaded test case, NULL if function failed
   114  */
   115 TestCase
   116 LoadTestCase(void *suite, char *testName)
   117 {
   118 	TestCase test = (TestCase) SDL_LoadFunction(suite, testName);
   119 	if(test == NULL) {
   120 		fprintf(stderr, "Loading test failed, tests == NULL\n");
   121 		fprintf(stderr, "%s\n", SDL_GetError());
   122 	}
   123 
   124 	return test;
   125 }
   126 
   127 /*!
   128  * Loads function that initialises the test case from the
   129  * given test suite.
   130  *
   131  * \param suite Used test suite
   132  *
   133  * \return Function pointer (TestCaseInit) which points to loaded init function. NULL if function fails.
   134  */
   135 TestCaseInit
   136 LoadTestCaseInit(void *suite) {
   137 	TestCaseInit testCaseInit = (TestCaseInit) SDL_LoadFunction(suite, "_TestCaseInit");
   138 	if(testCaseInit == NULL) {
   139 		fprintf(stderr, "Loading TestCaseInit function failed, testCaseInit == NULL\n");
   140 		fprintf(stderr, "%s\n", SDL_GetError());
   141 	}
   142 
   143 	return testCaseInit;
   144 }
   145 
   146 /*!
   147  * Loads function that deinitialises the executed test case from the
   148  * given test suite.
   149  *
   150  * \param suite Used test suite
   151  *
   152  * \return Function pointer (TestCaseInit) which points to loaded init function. NULL if function fails.
   153  */
   154 TestCaseQuit
   155 LoadTestCaseQuit(void *suite) {
   156 	TestCaseQuit testCaseQuit = (TestCaseQuit) SDL_LoadFunction(suite, "_TestCaseQuit");
   157 	if(testCaseQuit == NULL) {
   158 		fprintf(stderr, "Loading TestCaseQuit function failed, testCaseQuit == NULL\n");
   159 		fprintf(stderr, "%s\n", SDL_GetError());
   160 	}
   161 
   162 	return testCaseQuit;
   163 }
   164 
   165 /*!
   166  * If using out-of-proc execution of tests. This function
   167  * will handle the return value of the child process
   168  * and interprets it to the runner. Also prints warnings
   169  * if child was aborted by a signela.
   170  *
   171  * \param stat_lock information about the exited child process
   172  *
   173  * \return 0 if test case succeeded, 1 otherwise
   174  */
   175 int
   176 HandleTestReturnValue(int stat_lock)
   177 {
   178 	//! \todo rename to: HandleChildProcessReturnValue?
   179 	int returnValue = -1;
   180 
   181 	if(WIFEXITED(stat_lock)) {
   182 		returnValue = WEXITSTATUS(stat_lock);
   183 	} else if(WIFSIGNALED(stat_lock)) {
   184 		int signal = WTERMSIG(stat_lock);
   185 		fprintf(stderr, "FAILURE: test was aborted due to signal no %d\n", signal);
   186 		returnValue = 1;
   187 	}
   188 
   189 	return returnValue;
   190 }
   191 
   192 /*!
   193  * Prints usage information
   194  */
   195 void printUsage() {
   196 	  printf("Usage: ./runner [--in-proc] [--help]\n");
   197 	  printf("Options:\n");
   198 	  printf(" --in-proc        Executes tests in-process\n");
   199 	  printf(" --help           Print this help\n");
   200 }
   201 
   202 /*!
   203  * Parse command line arguments
   204  *
   205  * \param argc Count of command line arguments
   206  * \param argv Array of commond lines arguments
   207  */
   208 void
   209 ParseOptions(int argc, char *argv[])
   210 {
   211    int i;
   212 
   213    for (i = 1; i < argc; ++i) {
   214       const char *arg = argv[i];
   215       if(SDL_strcmp(arg, "--in-proc") == 0) {
   216          execute_inproc = 1;
   217       }
   218       else if(SDL_strcmp(arg, "--help") == 0 || SDL_strcmp(arg, "-h") == 0) {
   219     	  printUsage();
   220     	  exit(0);
   221       } else {
   222     	  printf("runner: unknown command '%s'\n", arg);
   223     	  printUsage();
   224     	  exit(0);
   225       }
   226    }
   227 }
   228 
   229 /*!
   230  * Entry point for test runner
   231  *
   232  * \param argc Count of command line arguments
   233  * \param argv Array of commond lines arguments
   234  */
   235 int
   236 main(int argc, char *argv[])
   237 {
   238 	ParseOptions(argc, argv);
   239 
   240 	// print: Testing against SDL version fuu (rev: bar) if verbose == true
   241 
   242 	int failureCount = 0, passCount = 0;
   243 
   244 	const Uint32 startTicks = SDL_GetTicks();
   245 
   246 	char *testSuiteName = ScanForTestSuites();
   247 	void *suite = LoadTestSuite(testSuiteName);
   248 	TestCaseReference **tests = QueryTestCases(suite);
   249 
   250 	TestCaseReference *reference = NULL;
   251 	int counter = 0;
   252 
   253 	for(reference = tests[counter]; reference; reference = tests[++counter]) {
   254 		if(reference->enabled == TEST_DISABLED) {
   255 			printf("Test %s (in %s) disabled. Omitting...\n", reference->name, testSuiteName);
   256 		} else {
   257 			char *testname = reference->name;
   258 
   259 			printf("Running %s (in %s):\n", testname, testSuiteName);
   260 
   261 			TestCaseInit testCaseInit = LoadTestCaseInit(suite);
   262 			TestCaseQuit testCaseQuit = LoadTestCaseQuit(suite);
   263 			TestCase test = (TestCase) LoadTestCase(suite, testname);
   264 
   265 			int retVal = 1;
   266 			if(execute_inproc) {
   267 				testCaseInit();
   268 
   269 				test(0x0);
   270 
   271 				retVal = testCaseQuit();
   272 			} else {
   273 				int childpid = fork();
   274 				if(childpid == 0) {
   275 					testCaseInit();
   276 
   277 					test(0x0);
   278 
   279 					return testCaseQuit();
   280 				} else {
   281 					int stat_lock = -1;
   282 					int child = wait(&stat_lock);
   283 
   284 					retVal = HandleTestReturnValue(stat_lock);
   285 				}
   286 			}
   287 
   288 			if(retVal) {
   289 				failureCount++;
   290 				if(retVal == 2) {
   291 					printf("%s (in %s): FAILED -> No asserts\n", testname, testSuiteName);
   292 				} else {
   293 					printf("%s (in %s): FAILED\n", testname, testSuiteName);
   294 				}
   295 			} else {
   296 				passCount++;
   297 				printf("%s (in %s): ok\n", testname, testSuiteName);
   298 			}
   299 		}
   300 
   301 		printf("\n");
   302 	}
   303 	
   304 	SDL_UnloadObject(suite);
   305 
   306 	const Uint32 endTicks = SDL_GetTicks();
   307 
   308 	printf("Ran %d tests in %0.5f seconds.\n", (passCount + failureCount), (endTicks-startTicks)/1000.0f);
   309 
   310 	printf("%d tests passed\n", passCount);
   311 	printf("%d tests failed\n", failureCount);
   312 
   313 	return 0;
   314 }