test/test-automation/runner.c
author Markus Kauppila
Sat, 04 Jun 2011 18:08:54 +0300
changeset 5680 e61bdd43a2c3
parent 5679 10a5199c4169
child 5681 7abfcc71aa7d
permissions -rw-r--r--
Fixed old comments and added new ones.
markus@5659
     1
/*
markus@5659
     2
  Copyright (C) 2011 Markus Kauppila <markus.kauppila@gmail.com>
markus@5659
     3
markus@5659
     4
  This software is provided 'as-is', without any express or implied
markus@5659
     5
  warranty.  In no event will the authors be held liable for any damages
markus@5659
     6
  arising from the use of this software.
markus@5659
     7
markus@5659
     8
  Permission is granted to anyone to use this software for any purpose,
markus@5659
     9
  including commercial applications, and to alter it and redistribute it
markus@5659
    10
  freely, subject to the following restrictions:
markus@5659
    11
markus@5659
    12
  1. The origin of this software must not be misrepresented; you must not
markus@5659
    13
     claim that you wrote the original software. If you use this software
markus@5659
    14
     in a product, an acknowledgment in the product documentation would be
markus@5659
    15
     appreciated but is not required.
markus@5659
    16
  2. Altered source versions must be plainly marked as such, and must not be
markus@5659
    17
     misrepresented as being the original software.
markus@5659
    18
  3. This notice may not be removed or altered from any source distribution.
markus@5659
    19
*/
markus@5659
    20
markus@5662
    21
#include "SDL/SDL.h"
markus@5659
    22
markus@5659
    23
#include <stdio.h>
markus@5659
    24
#include <stdlib.h>
markus@5659
    25
#include <unistd.h>
markus@5659
    26
#include <sys/types.h>
markus@5659
    27
markus@5674
    28
#include "SDL_test.h"
markus@5662
    29
markus@5672
    30
//!< Function pointer to a test case function
markus@5678
    31
typedef void (*TestCase)(void *arg);
markus@5678
    32
//!< Function pointer to a test case init function
markus@5678
    33
typedef void (*TestCaseInit)(void);
markus@5678
    34
//!< Function pointer to a test case quit function
markus@5678
    35
typedef int  (*TestCaseQuit)(void);
markus@5672
    36
markus@5676
    37
//!< Flag for executing tests in-process
markus@5676
    38
static int execute_inproc = 0;
markus@5676
    39
markus@5671
    40
/*!
markus@5676
    41
 * Returns the name for the dynamic library
markus@5676
    42
 * which implements the test suite.
markus@5671
    43
 *
markus@5676
    44
 * (in the future: scans the test/ directory and
markus@5676
    45
 * returns the names of the dynamic libraries
markus@5676
    46
 * implementing the test suites)
markus@5676
    47
 *
markus@5676
    48
 * \return Name of the dummy test suite
markus@5671
    49
 */
markus@5676
    50
char *
markus@5676
    51
ScanForTestSuites() {
markus@5666
    52
#if defined(linux) || defined( __linux)
markus@5666
    53
	char *libName = "tests/libtest.so";
markus@5667
    54
#else
markus@5679
    55
	char *libName = "tests/libtest.dylib";
markus@5666
    56
#endif
markus@5676
    57
	return libName;
markus@5676
    58
}
markus@5666
    59
markus@5676
    60
markus@5676
    61
/*!
markus@5676
    62
 * Loads test suite which is implemented as dynamic library.
markus@5676
    63
 *
markus@5680
    64
 * \param testSuiteName Name of the test suite which will be loaded
markus@5676
    65
 *
markus@5676
    66
 * \return Pointer to loaded test suite, or NULL if library could not be loaded
markus@5676
    67
 */
markus@5676
    68
void *
markus@5676
    69
LoadTestSuite(char *testSuiteName)
markus@5676
    70
{
markus@5676
    71
	void *library = SDL_LoadObject(testSuiteName);
markus@5659
    72
	if(library == NULL) {
markus@5676
    73
		fprintf(stderr, "Loading %s failed\n", testSuiteName);
markus@5675
    74
		fprintf(stderr, "%s\n", SDL_GetError());
markus@5659
    75
	}
markus@5659
    76
markus@5667
    77
	return library;
markus@5667
    78
}
markus@5667
    79
markus@5671
    80
/*!
markus@5671
    81
 * Loads the test case references from the given test suite.
markus@5671
    82
markus@5671
    83
 * \param library Previously loaded dynamic library AKA test suite
markus@5675
    84
 * \return Pointer to array of TestCaseReferences or NULL if function failed
markus@5671
    85
 */
markus@5673
    86
TestCaseReference **
markus@5673
    87
QueryTestCases(void *library)
markus@5673
    88
{
markus@5668
    89
	TestCaseReference **(*suite)(void);
markus@5667
    90
markus@5671
    91
	suite = (TestCaseReference **(*)(void)) SDL_LoadFunction(library, "QueryTestSuite");
markus@5659
    92
	if(suite == NULL) {
markus@5675
    93
		fprintf(stderr, "Loading QueryTestCaseReferences() failed.\n");
markus@5675
    94
		fprintf(stderr, "%s\n", SDL_GetError());
markus@5667
    95
	}
markus@5659
    96
markus@5668
    97
	TestCaseReference **tests = suite();
markus@5667
    98
	if(tests == NULL) {
markus@5675
    99
		fprintf(stderr, "Failed to load test references.\n");
markus@5675
   100
		fprintf(stderr, "%s\n", SDL_GetError());
markus@5667
   101
	}
markus@5667
   102
markus@5667
   103
	return tests;
markus@5667
   104
}
markus@5662
   105
markus@5678
   106
markus@5672
   107
/*!
markus@5672
   108
 * Loads test case from a test suite
markus@5672
   109
 *
markus@5673
   110
 * \param suite a test suite
markus@5673
   111
 * \param testName Name of the test that is going to be loaded
markus@5671
   112
 *
markus@5675
   113
 * \return Function Pointer (TestCase) to loaded test case, NULL if function failed
markus@5671
   114
 */
markus@5673
   115
TestCase
markus@5673
   116
LoadTestCase(void *suite, char *testName)
markus@5673
   117
{
markus@5678
   118
	TestCase test = (TestCase) SDL_LoadFunction(suite, testName);
markus@5672
   119
	if(test == NULL) {
markus@5675
   120
		fprintf(stderr, "Loading test failed, tests == NULL\n");
markus@5675
   121
		fprintf(stderr, "%s\n", SDL_GetError());
markus@5672
   122
	}
markus@5672
   123
markus@5672
   124
	return test;
markus@5672
   125
}
markus@5672
   126
markus@5678
   127
/*!
markus@5678
   128
 * Loads function that initialises the test case from the
markus@5678
   129
 * given test suite.
markus@5678
   130
 *
markus@5678
   131
 * \param suite Used test suite
markus@5678
   132
 *
markus@5678
   133
 * \return Function pointer (TestCaseInit) which points to loaded init function. NULL if function fails.
markus@5678
   134
 */
markus@5678
   135
TestCaseInit
markus@5678
   136
LoadTestCaseInit(void *suite) {
markus@5678
   137
	TestCaseInit testCaseInit = (TestCaseInit) SDL_LoadFunction(suite, "_TestCaseInit");
markus@5678
   138
	if(testCaseInit == NULL) {
markus@5678
   139
		fprintf(stderr, "Loading TestCaseInit function failed, testCaseInit == NULL\n");
markus@5678
   140
		fprintf(stderr, "%s\n", SDL_GetError());
markus@5678
   141
	}
markus@5678
   142
markus@5678
   143
	return testCaseInit;
markus@5678
   144
}
markus@5678
   145
markus@5678
   146
/*!
markus@5678
   147
 * Loads function that deinitialises the executed test case from the
markus@5678
   148
 * given test suite.
markus@5678
   149
 *
markus@5678
   150
 * \param suite Used test suite
markus@5678
   151
 *
markus@5678
   152
 * \return Function pointer (TestCaseInit) which points to loaded init function. NULL if function fails.
markus@5678
   153
 */
markus@5678
   154
TestCaseQuit
markus@5678
   155
LoadTestCaseQuit(void *suite) {
markus@5678
   156
	TestCaseQuit testCaseQuit = (TestCaseQuit) SDL_LoadFunction(suite, "_TestCaseQuit");
markus@5678
   157
	if(testCaseQuit == NULL) {
markus@5678
   158
		fprintf(stderr, "Loading TestCaseQuit function failed, testCaseQuit == NULL\n");
markus@5678
   159
		fprintf(stderr, "%s\n", SDL_GetError());
markus@5678
   160
	}
markus@5678
   161
markus@5678
   162
	return testCaseQuit;
markus@5678
   163
}
markus@5672
   164
markus@5671
   165
/*!
markus@5672
   166
 * If using out-of-proc execution of tests. This function
markus@5672
   167
 * will handle the return value of the child process
markus@5672
   168
 * and interprets it to the runner. Also prints warnings
markus@5672
   169
 * if child was aborted by a signela.
markus@5671
   170
 *
markus@5672
   171
 * \param stat_lock information about the exited child process
markus@5671
   172
 *
markus@5672
   173
 * \return 0 if test case succeeded, 1 otherwise
markus@5671
   174
 */
markus@5673
   175
int
markus@5673
   176
HandleTestReturnValue(int stat_lock)
markus@5673
   177
{
markus@5673
   178
	//! \todo rename to: HandleChildProcessReturnValue?
markus@5672
   179
	int returnValue = -1;
markus@5659
   180
markus@5672
   181
	if(WIFEXITED(stat_lock)) {
markus@5672
   182
		returnValue = WEXITSTATUS(stat_lock);
markus@5667
   183
	} else if(WIFSIGNALED(stat_lock)) {
markus@5667
   184
		int signal = WTERMSIG(stat_lock);
markus@5675
   185
		fprintf(stderr, "FAILURE: test was aborted due to signal no %d\n", signal);
markus@5672
   186
		returnValue = 1;
markus@5667
   187
	}
markus@5667
   188
markus@5672
   189
	return returnValue;
markus@5667
   190
}
markus@5667
   191
markus@5672
   192
/*!
markus@5678
   193
 * Prints usage information
markus@5678
   194
 */
markus@5678
   195
void printUsage() {
markus@5678
   196
	  printf("Usage: ./runner [--in-proc] [--help]\n");
markus@5678
   197
	  printf("Options:\n");
markus@5678
   198
	  printf(" --in-proc        Executes tests in-process\n");
markus@5678
   199
	  printf(" --help           Print this help\n");
markus@5678
   200
}
markus@5678
   201
markus@5678
   202
/*!
markus@5672
   203
 * Parse command line arguments
markus@5676
   204
 *
markus@5676
   205
 * \param argc Count of command line arguments
markus@5676
   206
 * \param argv Array of commond lines arguments
markus@5672
   207
 */
markus@5672
   208
void
markus@5672
   209
ParseOptions(int argc, char *argv[])
markus@5672
   210
{
markus@5672
   211
   int i;
markus@5667
   212
markus@5672
   213
   for (i = 1; i < argc; ++i) {
markus@5672
   214
      const char *arg = argv[i];
markus@5676
   215
      if(SDL_strcmp(arg, "--in-proc") == 0) {
markus@5672
   216
         execute_inproc = 1;
markus@5672
   217
      }
markus@5676
   218
      else if(SDL_strcmp(arg, "--help") == 0 || SDL_strcmp(arg, "-h") == 0) {
markus@5678
   219
    	  printUsage();
markus@5678
   220
    	  exit(0);
markus@5678
   221
      } else {
markus@5678
   222
    	  printf("runner: unknown command '%s'\n", arg);
markus@5678
   223
    	  printUsage();
markus@5676
   224
    	  exit(0);
markus@5676
   225
      }
markus@5672
   226
   }
markus@5672
   227
}
markus@5667
   228
markus@5675
   229
/*!
markus@5675
   230
 * Entry point for test runner
markus@5675
   231
 *
markus@5675
   232
 * \param argc Count of command line arguments
markus@5675
   233
 * \param argv Array of commond lines arguments
markus@5675
   234
 */
markus@5672
   235
int
markus@5672
   236
main(int argc, char *argv[])
markus@5672
   237
{
markus@5672
   238
	ParseOptions(argc, argv);
markus@5672
   239
markus@5673
   240
	// print: Testing against SDL version fuu (rev: bar) if verbose == true
markus@5667
   241
markus@5667
   242
	int failureCount = 0, passCount = 0;
markus@5659
   243
markus@5667
   244
	const Uint32 startTicks = SDL_GetTicks();
markus@5667
   245
markus@5676
   246
	char *testSuiteName = ScanForTestSuites();
markus@5676
   247
	void *suite = LoadTestSuite(testSuiteName);
markus@5671
   248
	TestCaseReference **tests = QueryTestCases(suite);
markus@5669
   249
markus@5669
   250
	TestCaseReference *reference = NULL;
markus@5669
   251
	int counter = 0;
markus@5669
   252
markus@5671
   253
	for(reference = tests[counter]; reference; reference = tests[++counter]) {
markus@5671
   254
		if(reference->enabled == TEST_DISABLED) {
markus@5676
   255
			printf("Test %s (in %s) disabled. Omitting...\n", reference->name, testSuiteName);
markus@5671
   256
		} else {
markus@5671
   257
			char *testname = reference->name;
markus@5668
   258
markus@5676
   259
			printf("Running %s (in %s):\n", testname, testSuiteName);
markus@5662
   260
markus@5678
   261
			TestCaseInit testCaseInit = LoadTestCaseInit(suite);
markus@5678
   262
			TestCaseQuit testCaseQuit = LoadTestCaseQuit(suite);
markus@5678
   263
			TestCase test = (TestCase) LoadTestCase(suite, testname);
markus@5678
   264
markus@5672
   265
			int retVal = 1;
markus@5672
   266
			if(execute_inproc) {
markus@5678
   267
				testCaseInit();
markus@5678
   268
markus@5678
   269
				test(0x0);
markus@5678
   270
markus@5678
   271
				retVal = testCaseQuit();
markus@5667
   272
			} else {
markus@5672
   273
				int childpid = fork();
markus@5672
   274
				if(childpid == 0) {
markus@5678
   275
					testCaseInit();
markus@5678
   276
markus@5678
   277
					test(0x0);
markus@5678
   278
markus@5678
   279
					return testCaseQuit();
markus@5672
   280
				} else {
markus@5672
   281
					int stat_lock = -1;
markus@5672
   282
					int child = wait(&stat_lock);
markus@5667
   283
markus@5672
   284
					retVal = HandleTestReturnValue(stat_lock);
markus@5671
   285
				}
markus@5659
   286
			}
markus@5672
   287
markus@5672
   288
			if(retVal) {
markus@5672
   289
				failureCount++;
markus@5679
   290
				if(retVal == 2) {
markus@5679
   291
					printf("%s (in %s): FAILED -> No asserts\n", testname, testSuiteName);
markus@5679
   292
				} else {
markus@5679
   293
					printf("%s (in %s): FAILED\n", testname, testSuiteName);
markus@5679
   294
				}
markus@5672
   295
			} else {
markus@5672
   296
				passCount++;
markus@5676
   297
				printf("%s (in %s): ok\n", testname, testSuiteName);
markus@5672
   298
			}
markus@5659
   299
		}
markus@5667
   300
markus@5667
   301
		printf("\n");
markus@5659
   302
	}
markus@5659
   303
	
markus@5671
   304
	SDL_UnloadObject(suite);
markus@5662
   305
markus@5662
   306
	const Uint32 endTicks = SDL_GetTicks();
markus@5659
   307
markus@5672
   308
	printf("Ran %d tests in %0.5f seconds.\n", (passCount + failureCount), (endTicks-startTicks)/1000.0f);
markus@5662
   309
markus@5662
   310
	printf("%d tests passed\n", passCount);
markus@5662
   311
	printf("%d tests failed\n", failureCount);
markus@5662
   312
markus@5659
   313
	return 0;
markus@5659
   314
}