This repository has been archived by the owner on Feb 11, 2021. It is now read-only.
/
runner.c
763 lines (614 loc) · 20.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
/*
Copyright (C) 2011 Markus Kauppila <markus.kauppila@gmail.com>
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
*/
21
#include "SDL/SDL.h"
22
23
24
25
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
26
#include <string.h>
27
#include <dirent.h>
28
29
30
#include <sys/types.h>
31
#include "SDL_test.h"
32
#include "logger.h"
33
34
//!< Function pointer to a test case function
35
typedef void (*TestCaseFp)(void *arg);
36
//!< Function pointer to a test case init function
37
typedef void (*TestCaseInitFp)(const int);
38
//!< Function pointer to a test case quit function
39
40
typedef int (*TestCaseQuitFp)(void);
41
42
43
//!< Flag for executing tests in-process
static int execute_inproc = 0;
44
45
//!< Flag for only printing out the test names
static int only_print_tests = 0;
46
47
48
49
//!< Flag for executing only test with selected name
static int only_selected_test = 0;
//!< Flag for executing only the selected test suite
static int only_selected_suite = 0;
50
51
//!< Flag for executing only tests that contain certain string in their name
static int only_tests_with_string = 0;
52
53
//!< Flag for enabling XML logging
static int xml_enabled = 0;
54
55
//! Flag for enabling user-supplied style sheet for XML test report
static int custom_xsl_enabled = 0;
56
57
58
//!< Size of the test and suite name buffers
59
#define NAME_BUFFER_SIZE 1024
60
61
62
63
//!< Name of the selected test
char selected_test_name[NAME_BUFFER_SIZE];
//!< Name of the selected suite
char selected_suite_name[NAME_BUFFER_SIZE];
64
65
66
67
//!< substring of test case name
char testcase_name_substring[NAME_BUFFER_SIZE];
68
69
70
//! Name for user-supplied XSL style sheet name
char xsl_stylesheet_name[NAME_BUFFER_SIZE];
71
72
73
//! Default directory of the test suites
#define DEFAULT_TEST_DIRECTORY "tests/"
74
75
/*!
76
77
* Holds information about test suite such as it's name
* and pointer to dynamic library. Implemented as linked list.
78
79
*/
typedef struct TestSuiteReference {
80
char *name; //!< test suite name
81
void *library; //!< pointer to shared/dynamic library implementing the suite
82
83
84
85
struct TestSuiteReference *next; //!< Pointer to next item in the list
} TestSuiteReference;
86
87
88
89
90
91
/*!
* Holds information about the tests that will be executed.
*
* Implemented as linked list.
*/
92
93
94
95
typedef struct TestCaseItem {
char *testName;
char *suiteName;
96
97
98
99
char *description;
long requirements;
long timeout;
100
101
102
TestCaseInitFp testCaseInit;
TestCaseFp testCase;
TestCaseQuitFp testCaseQuit;
103
104
struct TestCaseItem *next;
105
106
107
108
109
110
111
112
113
114
} TestCase;
/*! Some function prototypes. Add the rest of functions and move to runner.h */
TestCaseFp LoadTestCaseFunction(void *suite, char *testName);
TestCaseInitFp LoadTestCaseInitFunction(void *suite);
TestCaseQuitFp LoadTestCaseQuitFunction(void *suite);
TestCaseReference **QueryTestCaseReferences(void *library);
115
116
/*!
117
118
* Scans the tests/ directory and returns the names
* of the dynamic libraries implementing the test suites.
119
*
120
121
* Note: currently function assumes that test suites names
* are in following format: libtestsuite.dylib or libtestsuite.so.
122
*
123
124
125
126
* Note: if only_selected_suite flags is non-zero, only the selected
* test will be loaded.
*
* \param directoryName Name of the directory which will be scanned
127
* \param extension What file extension is used with dynamic objects
128
*
129
* \return Pointer to TestSuiteReference which holds all the info about suites
130
*/
131
TestSuiteReference *
132
133
ScanForTestSuites(char *directoryName, char *extension)
{
134
typedef struct dirent Entry;
135
DIR *directory = opendir(directoryName);
136
137
138
139
TestSuiteReference *suites = NULL;
Entry *entry = NULL;
140
if(!directory) {
141
perror("Couldn't open test suite directory!");
142
}
143
144
while(entry = readdir(directory)) {
145
if(strlen(entry->d_name) > 2) { // discards . and ..
146
147
148
const char *delimiters = ".";
char *name = strtok(entry->d_name, delimiters);
char *ext = strtok(NULL, delimiters);
149
150
151
152
153
154
// filter out all other suites but the selected test suite
int ok = 1;
if(only_selected_suite) {
ok = SDL_strncmp(selected_suite_name, name, NAME_BUFFER_SIZE) == 0;
}
155
156
157
158
if(ok && SDL_strcmp(ext, extension) == 0) {
char buffer[NAME_BUFFER_SIZE];
memset(buffer, 0, NAME_BUFFER_SIZE);
159
160
161
162
163
strcat(buffer, directoryName);
strcat(buffer, name);
strcat(buffer, ".");
strcat(buffer, ext);
164
165
166
167
// create test suite reference
TestSuiteReference *reference = (TestSuiteReference *) SDL_malloc(sizeof(TestSuiteReference));
memset(reference, 0, sizeof(TestSuiteReference));
168
169
170
int length = strlen(buffer) + 1; // + 1 for '\0'?
reference->name = SDL_malloc(length * sizeof(char));
171
172
strcpy(reference->name, buffer);
173
174
175
reference->next = suites;
suites = reference;
176
177
//printf("Reference added to: %s\n", buffer);
178
179
180
181
}
}
}
182
183
closedir(directory);
184
return suites;
185
186
}
187
188
189
190
191
192
193
/*!
* Goes through the previously loaded test suites and
* loads test cases from them. Test cases are filtered
* during the process. Function will only return the
* test cases which aren't filtered out.
*
194
* \param suites previously loaded test suites
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
*
* \return Test cases that survived filtering process.
*/
TestCase *
LoadTestCases(TestSuiteReference *suites)
{
TestCase *testCases = NULL;
TestSuiteReference *suiteReference = NULL;
for(suiteReference = suites; suiteReference; suiteReference = suiteReference->next) {
TestCaseReference **tests = QueryTestCaseReferences(suiteReference->library);
TestCaseReference *testReference = NULL;
int counter = 0;
for(testReference = tests[counter]; testReference; testReference = tests[++counter]) {
void *suite = suiteReference->library;
// Load test case functions
TestCaseInitFp testCaseInit = LoadTestCaseInitFunction(suiteReference->library);
TestCaseQuitFp testCaseQuit = LoadTestCaseQuitFunction(suiteReference->library);
TestCaseFp testCase = (TestCaseFp) LoadTestCaseFunction(suiteReference->library, testReference->name);
// Do the filtering
if(FilterTestCase(testReference)) {
TestCase *item = SDL_malloc(sizeof(TestCase));
memset(item, 0, sizeof(TestCase));
item->testCaseInit = testCaseInit;
item->testCase = testCase;
item->testCaseQuit = testCaseQuit;
227
// copy suite name
228
int length = SDL_strlen(suiteReference->name) + 1;
229
item->suiteName = SDL_malloc(length);
230
strncpy(item->suiteName, suiteReference->name, length);
231
232
// copy test name
233
length = SDL_strlen(testReference->name) + 1;
234
item->testName = SDL_malloc(length);
235
strncpy(item->testName, testReference->name, length);
236
237
// copy test description
238
length = SDL_strlen(testReference->description) + 1;
239
item->description = SDL_malloc(length);
240
strncpy(item->description, testReference->description, length);
241
242
243
244
item->requirements = testReference->requirements;
item->timeout = testReference->timeout;
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
// prepend the list
item->next = testCases;
testCases = item;
//printf("Added test: %s\n", testReference->name);
}
}
}
return testCases;
}
/*!
* Unloads the given TestCases. Frees all the resources
* allocated for test cases.
*
* \param testCases Test cases to be deallocated
*/
void
UnloadTestCases(TestCase *testCases)
{
TestCase *ref = testCases;
while(ref) {
SDL_free(ref->testName);
SDL_free(ref->suiteName);
271
SDL_free(ref->description);
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
TestCase *temp = ref->next;
SDL_free(ref);
ref = temp;
}
testCases = NULL;
}
/*!
* Filters a test case based on its properties in TestCaseReference and user
* preference.
*
* \return Non-zero means test will be added to execution list, zero means opposite
*/
int
FilterTestCase(TestCaseReference *testReference)
{
int retVal = 1;
if(testReference->enabled == TEST_DISABLED) {
retVal = 0;
}
297
298
299
300
301
302
303
304
if(only_selected_test) {
if(SDL_strncmp(testReference->name, selected_test_name, NAME_BUFFER_SIZE) == 0) {
retVal = 1;
} else {
retVal = 0;
}
}
305
306
307
308
309
310
311
312
313
314
315
316
if(only_tests_with_string) {
if(strstr(testReference->name, testcase_name_substring) != NULL) {
retVal = 1;
} else {
retVal = 0;
}
}
return retVal;
}
317
318
319
/*!
* Loads test suite which is implemented as dynamic library.
*
320
* \param testSuiteName Name of the test suite which will be loaded
321
322
323
324
325
326
327
*
* \return Pointer to loaded test suite, or NULL if library could not be loaded
*/
void *
LoadTestSuite(char *testSuiteName)
{
void *library = SDL_LoadObject(testSuiteName);
328
if(library == NULL) {
329
fprintf(stderr, "Loading %s failed\n", testSuiteName);
330
fprintf(stderr, "%s\n", SDL_GetError());
331
332
}
333
334
335
return library;
}
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
/*!
* Goes through all the given TestSuiteReferences
* and loads the dynamic libraries. Updates the suites
* parameter on-the-fly and returns it.
*
* \param suites Suites that will be loaded
*
* \return Updated TestSuiteReferences with pointer to loaded libraries
*/
TestSuiteReference *
LoadTestSuites(TestSuiteReference *suites)
{
TestSuiteReference *reference = NULL;
for(reference = suites; reference; reference = reference->next) {
reference->library = LoadTestSuite(reference->name);
}
return suites;
}
/*!
* Unloads the given TestSuiteReferences. Frees all
* the allocated resources including the dynamic libraries.
*
* \param suites TestSuiteReferences for deallocation process
*/
void
UnloadTestSuites(TestSuiteReference *suites)
{
TestSuiteReference *ref = suites;
while(ref) {
SDL_free(ref->name);
SDL_UnloadObject(ref->library);
TestSuiteReference *temp = ref->next;
SDL_free(ref);
ref = temp;
}
suites = NULL;
}
381
382
383
384
/*!
* Loads the test case references from the given test suite.
* \param library Previously loaded dynamic library AKA test suite
385
* \return Pointer to array of TestCaseReferences or NULL if function failed
386
*/
387
TestCaseReference **
388
QueryTestCaseReferences(void *library)
389
{
390
TestCaseReference **(*suite)(void);
391
392
393
394
395
396
suite = (TestCaseReference **(*)(void)) SDL_LoadFunction(library, "QueryTestSuite");
if(suite == NULL) {
fprintf(stderr, "Loading QueryTestCaseReferences() failed.\n");
fprintf(stderr, "%s\n", SDL_GetError());
}
397
398
399
400
401
402
TestCaseReference **tests = suite();
if(tests == NULL) {
fprintf(stderr, "Failed to load test references.\n");
fprintf(stderr, "%s\n", SDL_GetError());
}
403
404
return tests;
405
406
}
407
408
409
410
/*!
* Loads test case from a test suite
*
411
412
* \param suite a test suite
* \param testName Name of the test that is going to be loaded
413
*
414
* \return Function Pointer (TestCase) to loaded test case, NULL if function failed
415
*/
416
417
TestCaseFp
LoadTestCaseFunction(void *suite, char *testName)
418
{
419
TestCaseFp test = (TestCaseFp) SDL_LoadFunction(suite, testName);
420
if(test == NULL) {
421
422
fprintf(stderr, "Loading test failed, tests == NULL\n");
fprintf(stderr, "%s\n", SDL_GetError());
423
424
425
426
427
}
return test;
}
428
429
430
431
432
433
434
435
436
/*!
* Loads function that initialises the test case from the
* given test suite.
*
* \param suite Used test suite
*
* \return Function pointer (TestCaseInit) which points to loaded init function. NULL if function fails.
*/
437
438
439
TestCaseInitFp
LoadTestCaseInitFunction(void *suite) {
TestCaseInitFp testCaseInit = (TestCaseInitFp) SDL_LoadFunction(suite, "_TestCaseInit");
440
441
442
443
444
445
446
447
if(testCaseInit == NULL) {
fprintf(stderr, "Loading TestCaseInit function failed, testCaseInit == NULL\n");
fprintf(stderr, "%s\n", SDL_GetError());
}
return testCaseInit;
}
448
449
450
451
452
453
454
455
456
/*!
* Loads function that deinitialises the executed test case from the
* given test suite.
*
* \param suite Used test suite
*
* \return Function pointer (TestCaseInit) which points to loaded init function. NULL if function fails.
*/
457
458
459
TestCaseQuitFp
LoadTestCaseQuitFunction(void *suite) {
TestCaseQuitFp testCaseQuit = (TestCaseQuitFp) SDL_LoadFunction(suite, "_TestCaseQuit");
460
461
462
463
464
465
466
if(testCaseQuit == NULL) {
fprintf(stderr, "Loading TestCaseQuit function failed, testCaseQuit == NULL\n");
fprintf(stderr, "%s\n", SDL_GetError());
}
return testCaseQuit;
}
467
468
469
/*!
470
471
472
473
* If using out-of-proc execution of tests. This function
* will handle the return value of the child process
* and interprets it to the runner. Also prints warnings
* if child was aborted by a signela.
474
*
475
* \param stat_lock information about the exited child process
476
*
477
* \return 0 if test case succeeded, 1 otherwise
478
*/
479
int
480
HandleChildProcessReturnValue(int stat_lock)
481
{
482
int returnValue = -1;
483
484
485
if(WIFEXITED(stat_lock)) {
returnValue = WEXITSTATUS(stat_lock);
486
487
} else if(WIFSIGNALED(stat_lock)) {
int signal = WTERMSIG(stat_lock);
488
fprintf(stderr, "FAILURE: test was aborted due to signal no %d\n", signal);
489
returnValue = 1;
490
491
}
492
return returnValue;
493
494
}
495
496
497
498
499
/*!
* Executes a test case. Loads the test, executes it and
* returns the tests return value to the caller.
*
500
* \param testItem The test case that will be executed
501
502
503
* \return The return value of the test. Zero means success, non-zero failure.
*/
int
504
ExecuteTest(TestCase *testItem) {
505
506
int retVal = 1;
if(execute_inproc) {
507
testItem->testCaseInit(xml_enabled);
508
509
testItem->testCase(0x0);
510
511
retVal = testItem->testCaseQuit();
512
513
514
} else {
int childpid = fork();
if(childpid == 0) {
515
testItem->testCaseInit(xml_enabled);
516
517
testItem->testCase(0x0);
518
519
exit(testItem->testCaseQuit());
520
521
522
523
} else {
int stat_lock = -1;
int child = wait(&stat_lock);
524
retVal = HandleChildProcessReturnValue(stat_lock);
525
526
527
528
529
530
531
}
}
return retVal;
}
532
533
534
/*!
* Prints usage information
*/
535
536
void
printUsage() {
537
538
printf("Usage: ./runner [--in-proc] [--suite SUITE] [--test TEST]\n");
printf(" [--name-contains SUBSTR] [--help]\n");
539
printf("Options:\n");
540
printf(" --in-proc Executes tests in-process\n");
541
printf(" --show-tests Prints out all the executable tests\n");
542
printf(" --xml Enables XML logger\n");
543
printf(" --xsl FILENAME Use the given file as XSL style sheet for XML\n"); // \todo add to wiki
544
545
546
547
printf(" -t --test TEST Executes only tests with given name\n");
printf(" -ts --name-contains SUBSTR Executes only tests that have given\n");
printf(" substring in test name\n");
printf(" -s --suite SUITE Executes only the given test suite\n");
548
549
printf(" -h --help Print this help\n");
550
551
}
552
553
554
/*!
* Parse command line arguments
555
556
557
*
* \param argc Count of command line arguments
* \param argv Array of commond lines arguments
558
559
560
561
562
563
564
565
*/
void
ParseOptions(int argc, char *argv[])
{
int i;
for (i = 1; i < argc; ++i) {
const char *arg = argv[i];
566
if(SDL_strcmp(arg, "--in-proc") == 0) {
567
568
execute_inproc = 1;
}
569
570
else if(SDL_strcmp(arg, "--show-tests") == 0) {
only_print_tests = 1;
571
}
572
573
574
else if(SDL_strcmp(arg, "--xml") == 0) {
xml_enabled = 1;
}
575
576
else if(SDL_strcmp(arg, "--test") == 0 || SDL_strcmp(arg, "-t") == 0) {
only_selected_test = 1;
577
578
579
580
581
582
583
584
585
char *testName = NULL;
if( (i + 1) < argc) {
testName = argv[++i];
} else {
printf("runner: test name is missing\n");
printUsage();
exit(1);
}
586
587
memset(selected_test_name, 0, NAME_BUFFER_SIZE);
588
589
strcpy(selected_test_name, testName);
}
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
else if(SDL_strcmp(arg, "--xsl") == 0) {
custom_xsl_enabled = 1;
char *stylesheet = NULL;
if( (i + 1) < argc) {
stylesheet = argv[++i];
} else {
printf("runner: filename of XSL stylesheet is missing\n");
printUsage();
exit(1);
}
memset(xsl_stylesheet_name, 0, NAME_BUFFER_SIZE);
strncpy(xsl_stylesheet_name, stylesheet, NAME_BUFFER_SIZE);
}
605
else if(SDL_strcmp(arg, "--name-contains") == 0 || SDL_strcmp(arg, "-ts") == 0) {
606
607
608
609
610
611
612
613
614
615
616
617
618
619
only_tests_with_string = 1;
char *substring = NULL;
if( (i + 1) < argc) {
substring = argv[++i];
} else {
printf("runner: substring of test name is missing\n");
printUsage();
exit(1);
}
memset(testcase_name_substring, 0, NAME_BUFFER_SIZE);
strcpy(testcase_name_substring, substring);
}
620
621
622
else if(SDL_strcmp(arg, "--suite") == 0 || SDL_strcmp(arg, "-s") == 0) {
only_selected_suite = 1;
623
624
625
626
627
628
629
630
631
632
char *suiteName = NULL;
if( (i + 1) < argc) {
suiteName = argv[++i];
} else {
printf("runner: suite name is missing\n");
printUsage();
exit(1);
}
memset(selected_suite_name, 0, NAME_BUFFER_SIZE);
633
634
strcpy(selected_suite_name, suiteName);
}
635
636
637
638
else if(SDL_strcmp(arg, "--help") == 0 || SDL_strcmp(arg, "-h") == 0) {
printUsage();
exit(0);
}
639
else {
640
641
printf("runner: unknown command '%s'\n", arg);
printUsage();
642
643
exit(0);
}
644
645
}
}
646
647
648
649
650
651
652
653
/*!
* Entry point for test runner
*
* \param argc Count of command line arguments
* \param argv Array of commond lines arguments
*/
654
655
656
657
int
main(int argc, char *argv[])
{
ParseOptions(argc, argv);
658
659
// print: Testing against SDL version fuu (rev: bar) if verbose == true
660
661
662
int totalTestfailureCount = 0, totalTestPassCount = 0;
int testFailureCount = 0, testPassCount = 0, testSkipCount = 0;
663
664
char *testSuiteName = NULL;
int suiteCounter = 0;
665
666
667
668
669
670
#if defined(linux) || defined( __linux)
char *extension = "so";
#else
char *extension = "dylib";
#endif
671
672
if(xml_enabled) {
SetupXMLLogger();
673
674
RunStarted(argc, argv, time(0), xsl_stylesheet_name);
675
676
} else {
SetupPlainLogger();
677
678
RunStarted(argc, argv, time(0), NULL);
679
}
680
681
const Uint32 startTicks = SDL_GetTicks();
682
683
TestSuiteReference *suites = ScanForTestSuites(DEFAULT_TEST_DIRECTORY, extension);
684
suites = LoadTestSuites(suites);
685
686
TestCase *testCases = LoadTestCases(suites);
687
688
689
690
691
// if --show-tests option is given, only print tests and exit
if(only_print_tests) {
TestCase *testItem = NULL;
for(testItem = testCases; testItem; testItem = testItem->next) {
692
//! \todo This should be handled by the logging system?
693
694
695
696
697
698
printf("%s (in %s)\n", testItem->testName, testItem->suiteName);
}
return 0;
}
699
700
char *currentSuiteName = NULL;
701
702
int suiteStartTime = SDL_GetTicks();
703
TestCase *testItem = NULL;
704
for(testItem = testCases; testItem; testItem = testItem->next) {
705
706
if(currentSuiteName == NULL) {
currentSuiteName = testItem->suiteName;
707
SuiteStarted(currentSuiteName, time(0));
708
709
710
711
712
713
testFailureCount = testPassCount = 0;
suiteCounter++;
}
else if(strncmp(currentSuiteName, testItem->suiteName, NAME_BUFFER_SIZE) != 0) {
714
715
716
717
const double suiteRuntime = (SDL_GetTicks() - suiteStartTime) / 1000.0f;
SuiteEnded(testPassCount, testFailureCount, testSkipCount, time(0),
suiteRuntime);
718
719
720
suiteStartTime = SDL_GetTicks();
721
currentSuiteName = testItem->suiteName;
722
SuiteStarted(currentSuiteName, time(0));
723
724
725
726
testFailureCount = testPassCount = 0;
suiteCounter++;
727
728
729
}
TestStarted(testItem->testName, testItem->suiteName,
730
731
732
testItem->description, time(0));
const Uint32 testTimeStart = SDL_GetTicks();
733
734
735
int retVal = ExecuteTest(testItem);
if(retVal) {
736
737
totalTestfailureCount++;
testFailureCount++;
738
} else {
739
740
totalTestPassCount++;
testPassCount++;
741
}
742
743
744
745
const double testTotalRuntime = (SDL_GetTicks() - testTimeStart) / 1000.0f;
TestEnded(testItem->testName, testItem->suiteName, retVal, time(0), testTotalRuntime);
746
}
747
748
if(currentSuiteName) {
749
750
SuiteEnded(testPassCount, testFailureCount, testSkipCount, time(0),
(SDL_GetTicks() - suiteStartTime) / 1000.0f);
751
}
752
753
754
755
UnloadTestCases(testCases);
UnloadTestSuites(suites);
756
const Uint32 endTicks = SDL_GetTicks();
757
const double totalRunTime = (endTicks - startTicks) / 1000.0f;
758
759
RunEnded(totalTestPassCount + totalTestfailureCount, suiteCounter,
760
totalTestPassCount, totalTestfailureCount, time(0), totalRunTime);
761
762
return (totalTestfailureCount ? 1 : 0);
763
}