Creating pipeline for test case selection. Work in progress.
authorMarkus Kauppila
Thu, 09 Jun 2011 16:37:51 +0300
changeset 5692249e54c47b90
parent 5691 c0b4bcd901a0
child 5693 cd0110de8ad5
Creating pipeline for test case selection. Work in progress.
test/test-automation/runner.c
test/test-automation/testdummy/testdummy.c
     1.1 --- a/test/test-automation/runner.c	Wed Jun 08 19:41:12 2011 +0300
     1.2 +++ b/test/test-automation/runner.c	Thu Jun 09 16:37:51 2011 +0300
     1.3 @@ -61,10 +61,22 @@
     1.4   * linked list. \todo write better doc
     1.5   */
     1.6  typedef struct TestSuiteReference {
     1.7 -	char *name; //<! test suite name
     1.8 +	char *name; //!< test suite name
     1.9 +	void *library; //!< pointer to shared/dynamic library implemeting the suite
    1.10 +
    1.11  	struct TestSuiteReference *next; //!< Pointer to next item in the list
    1.12  } TestSuiteReference;
    1.13  
    1.14 +typedef struct TestCaseItem {
    1.15 +	char *testName;
    1.16 +	char *suiteName;
    1.17 +
    1.18 +	TestCaseInit testCaseInit;
    1.19 +	TestCase testCase;
    1.20 +	TestCaseQuit testCaseQuit;
    1.21 +
    1.22 +	struct TestCaseItem *next;
    1.23 +} TestCaseItem; //!< \todo rename
    1.24  
    1.25  /*!
    1.26   * Scans the tests/ directory and returns the names
    1.27 @@ -88,49 +100,49 @@
    1.28  	TestSuiteReference *suites = NULL;
    1.29  
    1.30  	Entry *entry = NULL;
    1.31 -	if(directory) {
    1.32 -		while(entry = readdir(directory)) {
    1.33 -			if(entry->d_namlen > 2) { // discards . and ..
    1.34 +	if(!directory) {
    1.35 +		perror("Couldn't open directory: tests/");
    1.36 +	}
    1.37 +
    1.38 +	while(entry = readdir(directory)) {
    1.39 +		if(entry->d_namlen > 2) { // discards . and ..
    1.40 +			const char *delimiters = ".";
    1.41 +			char *name = strtok(entry->d_name, delimiters);
    1.42 +			char *ext = strtok(NULL, delimiters);
    1.43 +
    1.44 +			// filter out all other suites but the selected test suite
    1.45 +			int ok = 1;
    1.46 +			if(only_selected_suite) {
    1.47 +				ok = SDL_strncmp(selected_suite_name, name, NAME_BUFFER_SIZE) == 0;
    1.48 +			}
    1.49 +
    1.50 +			if(ok && SDL_strcmp(ext, extension)  == 0) {
    1.51  				char buffer[NAME_BUFFER_SIZE];
    1.52  				memset(buffer, 0, NAME_BUFFER_SIZE);
    1.53  
    1.54 -				char *name = strtok(entry->d_name, ".");
    1.55 -				char *ext = strtok(NULL, ".");
    1.56 +				strcat(buffer, directoryName);
    1.57 +				strcat(buffer, name);
    1.58 +				strcat(buffer, ".");
    1.59 +				strcat(buffer, ext);
    1.60  
    1.61 -				// filter out all other suites but the selected test suite
    1.62 -				int ok = 1;
    1.63 -				if(only_selected_suite) {
    1.64 -					ok = SDL_strncmp(selected_suite_name, name, NAME_BUFFER_SIZE) == 0;
    1.65 -				}
    1.66 -
    1.67 -				if(ok && SDL_strcmp(ext, extension)  == 0) {
    1.68 -					strcat(buffer, directoryName);
    1.69 -					strcat(buffer, name);
    1.70 -					strcat(buffer, ".");
    1.71 -					strcat(buffer, ext);
    1.72 +				// create test suite reference
    1.73 +				TestSuiteReference *reference = (TestSuiteReference *) SDL_malloc(sizeof(TestSuiteReference));
    1.74 +				memset(reference, 0, sizeof(TestSuiteReference));
    1.75  
    1.76 -					// create tes suite reference
    1.77 -					TestSuiteReference *reference = (TestSuiteReference *) SDL_malloc(sizeof(TestSuiteReference));
    1.78 -					memset(reference, 0, sizeof(TestSuiteReference));
    1.79 +				int length = strlen(buffer) + 1; // + 1 for '\0'?
    1.80 +				reference->name = SDL_malloc(length * sizeof(char));
    1.81  
    1.82 -					int length = strlen(buffer) + 1; // + 1 for '\0'?
    1.83 -					reference->name = SDL_malloc(length * sizeof(char));
    1.84 +				strcpy(reference->name, buffer);
    1.85  
    1.86 -					strcpy(reference->name, buffer);
    1.87 -
    1.88 -					reference->next = suites;
    1.89 +				reference->next = suites;
    1.90 +				suites = reference;
    1.91  
    1.92 -					suites = reference;
    1.93 -
    1.94 -					printf("Reference added to: %s\n", buffer);
    1.95 -				}
    1.96 +				printf("Reference added to: %s\n", buffer);
    1.97  			}
    1.98  		}
    1.99 +	}
   1.100  
   1.101 -		closedir(directory);
   1.102 -	} else {
   1.103 -		perror("Couldn't open directory: tests/");
   1.104 -	}
   1.105 +	closedir(directory);
   1.106  
   1.107  	return suites;
   1.108  }
   1.109 @@ -281,26 +293,22 @@
   1.110   * \return The return value of the test. Zero means success, non-zero failure.
   1.111   */
   1.112  int
   1.113 -ExecuteTest(void *suite, TestCaseReference *testReference) {
   1.114 -	TestCaseInit testCaseInit = LoadTestCaseInit(suite);
   1.115 -	TestCaseQuit testCaseQuit = LoadTestCaseQuit(suite);
   1.116 -	TestCase test = (TestCase) LoadTestCase(suite, testReference->name);
   1.117 -
   1.118 +ExecuteTest(TestCaseItem *testItem) {
   1.119  	int retVal = 1;
   1.120  	if(execute_inproc) {
   1.121 -		testCaseInit();
   1.122 +		testItem->testCaseInit();
   1.123  
   1.124 -		test(0x0);
   1.125 +		testItem->testCase(0x0);
   1.126  
   1.127 -		retVal = testCaseQuit();
   1.128 +		retVal = testItem->testCaseQuit();
   1.129  	} else {
   1.130  		int childpid = fork();
   1.131  		if(childpid == 0) {
   1.132 -			testCaseInit();
   1.133 +			testItem->testCaseInit();
   1.134  
   1.135 -			test(0x0);
   1.136 +			testItem->testCase(0x0);
   1.137  
   1.138 -			exit(testCaseQuit());
   1.139 +			exit(testItem->testCaseQuit());
   1.140  		} else {
   1.141  			int stat_lock = -1;
   1.142  			int child = wait(&stat_lock);
   1.143 @@ -388,6 +396,118 @@
   1.144  
   1.145  
   1.146  /*!
   1.147 + * \todo add comment
   1.148 + */
   1.149 +TestCaseItem *
   1.150 +LoadTestCases(TestSuiteReference *suites) {
   1.151 +	TestCaseItem *testCases = NULL;
   1.152 +
   1.153 +	TestSuiteReference *suiteReference = NULL;
   1.154 +	for(suiteReference = suites; suiteReference; suiteReference = suiteReference->next) {
   1.155 +		TestCaseReference **tests = QueryTestCases(suiteReference->library);
   1.156 +
   1.157 +		TestCaseReference *testReference = NULL;
   1.158 +		int counter = 0;
   1.159 +		for(testReference = tests[counter]; testReference; testReference = tests[++counter]) {
   1.160 +
   1.161 +			void *suite = suiteReference->library;
   1.162 +			// Load test case functions
   1.163 +			TestCaseInit testCaseInit = LoadTestCaseInit(suiteReference->library);
   1.164 +			TestCaseQuit testCaseQuit = LoadTestCaseQuit(suiteReference->library);
   1.165 +			TestCase testCase = (TestCase) LoadTestCase(suiteReference->library, testReference->name);
   1.166 +
   1.167 +			// Do the filtering
   1.168 +			if(FilterTestCase(testReference)) {
   1.169 +				//!< \todo deallocate these
   1.170 +				TestCaseItem *item = SDL_malloc(sizeof(TestCaseItem));
   1.171 +				memset(item, 0, sizeof(TestCaseItem));
   1.172 +
   1.173 +				item->testCaseInit = testCaseInit;
   1.174 +				item->testCase = testCase;
   1.175 +				item->testCaseQuit = testCaseQuit;
   1.176 +
   1.177 +				// prepend the list
   1.178 +				item->next = testCases;
   1.179 +				testCases = item;
   1.180 +
   1.181 +				printf("Added test: %s\n", testReference->name);
   1.182 +			}
   1.183 +		}
   1.184 +	}
   1.185 +
   1.186 +	return testCases;
   1.187 +}
   1.188 +
   1.189 +/*!
   1.190 + * \todo add comment
   1.191 + */
   1.192 +void
   1.193 +UnloadTestCases(TestCaseItem *item) {
   1.194 +	TestCaseItem *ref = item;
   1.195 +	while(ref) {
   1.196 +		TestCaseItem *temp = ref->next;
   1.197 +		SDL_free(ref);
   1.198 +		ref = temp;
   1.199 +	}
   1.200 +
   1.201 +	item = NULL;
   1.202 +}
   1.203 +
   1.204 +/*!
   1.205 + * \todo add comment
   1.206 + */
   1.207 +int
   1.208 +FilterTestCase(TestCaseReference *testReference) {
   1.209 +	//int retVal = 1;
   1.210 +
   1.211 +	if(testReference->enabled == TEST_DISABLED) {
   1.212 +		//retVal = 0;
   1.213 +		return 0;
   1.214 +	}
   1.215 +
   1.216 +	if(1 && strstr(testReference->name, "rect") != NULL) {
   1.217 +		//retVal = 1;
   1.218 +		return 1;
   1.219 +	} else {
   1.220 +		return 0;
   1.221 +	}
   1.222 +
   1.223 +	return 1;
   1.224 +}
   1.225 +
   1.226 +/*!
   1.227 + * \todo add comment
   1.228 + */
   1.229 +TestSuiteReference *
   1.230 +LoadTestSuites(TestSuiteReference *suites) {
   1.231 +	TestSuiteReference *reference = NULL;
   1.232 +	for(reference = suites; reference; reference = reference->next) {
   1.233 +		reference->library = LoadTestSuite(reference->name);
   1.234 +	}
   1.235 +
   1.236 +	return suites;
   1.237 +}
   1.238 +
   1.239 +
   1.240 +/*!
   1.241 + * \todo add comment
   1.242 + */
   1.243 +void UnloadTestSuites(TestSuiteReference *suites) {
   1.244 +	TestSuiteReference *ref = suites;
   1.245 +	while(ref) {
   1.246 +		SDL_free(ref->name);
   1.247 +		SDL_UnloadObject(ref->library);
   1.248 +
   1.249 +		TestSuiteReference *temp = ref->next;
   1.250 +		SDL_free(ref);
   1.251 +		ref = temp;
   1.252 +	}
   1.253 +
   1.254 +	suites = NULL;
   1.255 +}
   1.256 +
   1.257 +
   1.258 +/*!
   1.259   * Entry point for test runner
   1.260   *
   1.261   * \param argc Count of command line arguments
   1.262 @@ -411,53 +531,37 @@
   1.263  #endif
   1.264  
   1.265  	const Uint32 startTicks = SDL_GetTicks();
   1.266 +
   1.267  	TestSuiteReference *suites = ScanForTestSuites(DEFAULT_TEST_DIRECTORY, extension);
   1.268 -
   1.269 -	// load the suites
   1.270 -	// load tests and filter them
   1.271 -	// end result: list of tests to run
   1.272 -
   1.273 -	TestSuiteReference *suiteReference = NULL;
   1.274 -	for(suiteReference = suites; suiteReference; suiteReference = suiteReference->next) {
   1.275 -		char *testSuiteName = suiteReference->name;
   1.276 +	suites = LoadTestSuites(suites);
   1.277  
   1.278 -		// if the current suite isn't selected, go to next suite
   1.279 -		void *suite = LoadTestSuite(testSuiteName);
   1.280 -		TestCaseReference **tests = QueryTestCases(suite);
   1.281 +	// load tests and filter them
   1.282 +	TestCaseItem *testCases = LoadTestCases(suites);
   1.283  
   1.284 -		TestCaseReference *reference = NULL;
   1.285 -		int counter = 0;
   1.286 -		for(reference = tests[counter]; reference; reference = tests[++counter]) {
   1.287 -			if(only_selected_test && SDL_strncmp(selected_test_name, reference->name, NAME_BUFFER_SIZE) != 0) {
   1.288 -				continue;
   1.289 -			}
   1.290 +	// end result: list of tests to run
   1.291 +	TestCaseItem *testItem = NULL;
   1.292 +	for(testItem = testCases; testItem; testItem = testItem->next) {
   1.293 +		int retVal = ExecuteTest(testItem);
   1.294  
   1.295 -			if(reference->enabled == TEST_DISABLED) {
   1.296 -				printf("Test %s (in %s) disabled. Omitting...\n", reference->name, testSuiteName);
   1.297 +		if(retVal) {
   1.298 +			failureCount++;
   1.299 +			if(retVal == 2) {
   1.300 +				//printf("%s (in %s): FAILED -> No asserts\n", reference->name, testSuiteName);
   1.301 +				printf("%s (in %s): FAILED -> No asserts\n", "<test name>", "<suite name>");
   1.302  			} else {
   1.303 -				printf("Executing %s (in %s):\n", reference->name, testSuiteName);
   1.304 -
   1.305 -				int retVal = ExecuteTest(suite, reference);
   1.306 -
   1.307 -				if(retVal) {
   1.308 -					failureCount++;
   1.309 -					if(retVal == 2) {
   1.310 -						printf("%s (in %s): FAILED -> No asserts\n", reference->name, testSuiteName);
   1.311 -					} else {
   1.312 -						printf("%s (in %s): FAILED\n", reference->name, testSuiteName);
   1.313 -					}
   1.314 -				} else {
   1.315 -					passCount++;
   1.316 -					printf("%s (in %s): ok\n", reference->name, testSuiteName);
   1.317 -				}
   1.318 +				printf("%s (in %s): FAILED\n", "<test name>", "<suite name>");
   1.319  			}
   1.320 -
   1.321 -			printf("\n");
   1.322 +		} else {
   1.323 +			passCount++;
   1.324 +			printf("%s (in %s): ok\n", "<test name>", "<suite name>");
   1.325  		}
   1.326  
   1.327 -		SDL_UnloadObject(suite);
   1.328 +		printf("\n");
   1.329  	}
   1.330  
   1.331 +	UnloadTestCases(testCases);
   1.332 +	UnloadTestSuites(suites);
   1.333 +
   1.334  	const Uint32 endTicks = SDL_GetTicks();
   1.335  
   1.336  	printf("Ran %d tests in %0.5f seconds.\n", (passCount + failureCount), (endTicks-startTicks)/1000.0f);
   1.337 @@ -465,15 +569,6 @@
   1.338  	printf("%d tests passed\n", passCount);
   1.339  	printf("%d tests failed\n", failureCount);
   1.340  
   1.341 -	// Deallocate the memory used by test suites
   1.342 -	TestSuiteReference *ref = suites;
   1.343 -	while(ref) {
   1.344 -		SDL_free(ref->name);
   1.345 -
   1.346 -		TestSuiteReference *temp = ref->next;
   1.347 -		SDL_free(ref);
   1.348 -		ref = temp;
   1.349 -	}
   1.350  
   1.351  	return 0;
   1.352  }
     2.1 --- a/test/test-automation/testdummy/testdummy.c	Wed Jun 08 19:41:12 2011 +0300
     2.2 +++ b/test/test-automation/testdummy/testdummy.c	Thu Jun 09 16:37:51 2011 +0300
     2.3 @@ -60,6 +60,7 @@
     2.4  {
     2.5  	const char *revision = SDL_GetRevision();
     2.6  
     2.7 +	printf("Dummycase 1\n");
     2.8  	printf("Revision is %s\n", revision);
     2.9  
    2.10  	AssertEquals(3, 5, "fails");
    2.11 @@ -69,11 +70,13 @@
    2.12  {
    2.13  	char *msg = "eello";
    2.14  	//msg[0] = 'H';
    2.15 +	printf("Dummycase 2\n");
    2.16  	AssertTrue(0, "fails");
    2.17  }
    2.18  
    2.19  void dummycase3(void *arg)
    2.20  {
    2.21 +	printf("Dummycase 3\n");
    2.22  	AssertTrue(1, "passes");
    2.23  }
    2.24