Skip to content
This repository has been archived by the owner on Feb 11, 2021. It is now read-only.

Commit

Permalink
Fixes in harness and fuzzer test lib components; improve harness driv…
Browse files Browse the repository at this point in the history 
…er; add rect test suite
  • Loading branch information
@ferzkopp
ferzkopp committed Dec 16, 2012
1 parent 2ac7334 commit 5c0dd47
Show file tree
Hide file tree
Showing 5 changed files with 1,743 additions and 38 deletions.
6 changes: 4 additions & 2 deletions src/test/SDL_test_fuzzer.c
View file
Expand Up @@ -38,7 +38,7 @@
/**
*Counter for fuzzer invocations
*/
static int fuzzerInvocationCounter;
static int fuzzerInvocationCounter = 0;

/**
* Context for shared random number generator
Expand All @@ -52,9 +52,11 @@ static SDLTest_RandomContext rndContext;
void
SDLTest_FuzzerInit(Uint64 execKey)
{
Uint32 a = (execKey >> 32) & 0x00000000FFFFFFFF;
Uint32 a = (execKey >> 32) & 0x00000000FFFFFFFF;
Uint32 b = execKey & 0x00000000FFFFFFFF;
SDL_memset((void *)&rndContext, 0, sizeof(SDLTest_RandomContext));
SDLTest_RandomInit(&rndContext, a, b);
fuzzerInvocationCounter = 0;
}

int
Expand Down
48 changes: 31 additions & 17 deletions src/test/SDL_test_harness.c
View file
Expand Up @@ -222,6 +222,7 @@ SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference
{
SDL_TimerID timer = 0;
int testResult = 0;
int fuzzerCount;

if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
{
Expand All @@ -235,7 +236,7 @@ SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference
return TEST_RESULT_SKIPPED;
}

// Initialize fuzzer
// Initialize fuzzer
SDLTest_FuzzerInit(execKey);

// Reset assert tracker
Expand Down Expand Up @@ -268,7 +269,10 @@ SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference
}

// Report on asserts and fuzzer usage
SDLTest_Log("Fuzzer invocations: %d", SDLTest_GetFuzzerInvocationCount());
fuzzerCount = SDLTest_GetFuzzerInvocationCount();
if (fuzzerCount > 0) {
SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
}
SDLTest_LogAssertSummary();

return testResult;
Expand Down Expand Up @@ -361,6 +365,8 @@ SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, U
SDLTest_LogError("Generating a random seed failed");
return 2;
}
} else {
runSeed = userRunSeed;
}

// Reset per-run counters
Expand All @@ -372,7 +378,7 @@ SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, U
runStartSeconds = GetClock();

// Log run with fuzzer parameters
SDLTest_Log("::::: Test Run '%s' started\n", runSeed);
SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);

// Loop over all suites
suiteCounter = 0;
Expand All @@ -390,7 +396,7 @@ SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, U

// Log suite started
currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
SDLTest_Log("===== Test Suite %i: %s started\n",
SDLTest_Log("===== Test Suite %i: '%s' started\n",
suiteCounter,
currentSuiteName);

Expand All @@ -406,25 +412,28 @@ SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, U

// Log test started
currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
SDLTest_Log("----- Test Case %i: %s started",
SDLTest_Log("----- Test Case %i.%i: '%s' started",
suiteCounter,
testCounter,
currentTestName);
SDLTest_Log("Test Description: %s",
(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);

if (testCase->description != NULL && strlen(testCase->description)>0) {
SDLTest_Log("Test Description: '%s'",
(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
}

// Loop over all iterations
iterationCounter = 0;
while(iterationCounter < testIterations)
{
iterationCounter++;

if(userExecKey != 0) {
if (userExecKey != 0) {
execKey = userExecKey;
} else {
execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
}

SDLTest_Log("Test Iteration %i: execKey %d", iterationCounter, execKey);
SDLTest_Log("Test Iteration %i: execKey %llu", iterationCounter, execKey);
testResult = SDLTest_RunTest(testSuite, testCase, execKey);

if (testResult == TEST_RESULT_PASSED) {
Expand All @@ -442,10 +451,14 @@ SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, U
// Take time - test end
testEndSeconds = GetClock();

SDLTest_Log("Test Case %s ended", currentTestName);

// Log test runtime
SDLTest_Log("Test runtime: %.1f sec", testEndSeconds - testStartSeconds);
if (testIterations > 1) {
// Log test runtime
SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, testEndSeconds - testStartSeconds);
SDLTest_Log("Test runtime: %.5f sec", (testEndSeconds - testStartSeconds) / (float)testIterations);
} else {
// Log test runtime
SDLTest_Log("Test runtime: %.1f sec", testEndSeconds - testStartSeconds);
}

// Log final test result
switch (testResult) {
Expand All @@ -468,7 +481,7 @@ SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, U
SDLTest_Log("Suite runtime: %.1f sec", suiteEndSeconds - suiteStartSeconds);

// Log summary and final Suite result
countSum = testPassedCount + testFailedCount + testSkippedCount;
countSum = testPassedCount + testFailedCount + testSkippedCount;
if (testFailedCount == 0)
{
SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
Expand All @@ -493,14 +506,15 @@ SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], char *userRunSeed, U
{
runResult = 0;
SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run", runSeed, "Passed");
SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed");
}
else
{
runResult = 1;
SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run", runSeed, "Failed");
SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
}

SDLTest_Log("Exit code: %d", runResult);
return runResult;
}
46 changes: 29 additions & 17 deletions test/testautomation.c
View file
Expand Up @@ -34,6 +34,9 @@ int
main(int argc, char *argv[])
{
int result;
int testIterations = 1;
Uint64 userExecKey = 0;
char *userRunSeed = NULL;
int i;

/* Initialize test framework */
Expand All @@ -52,27 +55,33 @@ main(int argc, char *argv[])
consumed = SDLTest_CommonArg(state, i);
if (consumed == 0) {
consumed = -1;
/* Parse additional parameters
if (SDL_strcasecmp(argv[i], "--BLAH") == 0) {
if (SDL_strcasecmp(argv[i], "--iterations") == 0) {
if (argv[i + 1]) {
if (SDL_strcasecmp(argv[i + 1], "BLUB") == 0) {
blah = blub;
consumed = 2;
}
testIterations = SDL_atoi(argv[i + 1]);
if (testIterations < 1) testIterations = 1;
consumed = 2;
}
} else if (SDL_strcasecmp(argv[i], "--BINGO") == 0) {
bingo = SDL_TRUE;
consumed = 1;
}
*/
}
else if (SDL_strcasecmp(argv[i], "--execKey") == 0) {
if (argv[i + 1]) {
SDL_sscanf(argv[i + 1], "%llu", &userExecKey);
consumed = 2;
}
}
else if (SDL_strcasecmp(argv[i], "--seed") == 0) {
if (argv[i + 1]) {
userRunSeed = SDL_strdup(argv[i + 1]);
consumed = 2;
}
}
}
if (consumed < 0) {
fprintf(stderr,
"Usage: %s %s [--BLAH BLUB --BINGO]\n",
"Usage: %s %s [--iterations #] [--execKey #] [--seed string]\n",
argv[0], SDLTest_CommonUsage(state));
quit(1);
}

i += consumed;
}

Expand All @@ -89,10 +98,13 @@ main(int argc, char *argv[])
}

/* Call Harness */
// TODO: pass custom parameters
result = SDLTest_RunSuites(testSuites, NULL, 0, 1);
//int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites, char *userRunSeed, Uint64 userExecKey, int testIterations);

result = SDLTest_RunSuites(testSuites, userRunSeed, userExecKey, testIterations);

/* Clean up */
if (userRunSeed != NULL) {
SDL_free(userRunSeed);
}

/* Shutdown everything */
quit(result);
return(result);
Expand Down

0 comments on commit 5c0dd47

Please sign in to comment.