Skip to content

Commit

Permalink
Fix bug/add test coverage for SDLTest_GenerateRunSeed helper; improve…
Browse files Browse the repository at this point in the history 
… test harness adding output of repro steps for failures; improve negative test for SDL_GetError/SDL_SetError
  • Loading branch information
@ferzkopp
ferzkopp committed Mar 11, 2014
1 parent b677d1d commit 7a36070
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 11 deletions.
48 changes: 44 additions & 4 deletions src/test/SDL_test_harness.c
View file
Expand Up @@ -67,20 +67,21 @@ SDLTest_GenerateRunSeed(const int length)
seed = (char *)SDL_malloc((length + 1) * sizeof(char));
if (seed == NULL) {
SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
SDL_Error(SDL_ENOMEM);
return NULL;
}

/* Generate a random string of alphanumeric characters */
SDLTest_RandomInitTime(&randomContext);
for (counter = 0; counter < length - 1; ++counter) {
for (counter = 0; counter < length; counter++) {
unsigned int number = SDLTest_Random(&randomContext);
char ch = (char) (number % (91 - 48)) + 48;
if (ch >= 58 && ch <= 64) {
ch = 65;
}
seed[counter] = ch;
}
seed[counter] = '\0';
seed[length] = '\0';

return seed;
}
Expand Down Expand Up @@ -141,7 +142,8 @@ SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iter
entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
buffer = (char *)SDL_malloc(entireStringLength);
if (buffer == NULL) {
SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
SDLTest_LogError("Failed to allocate buffer for execKey generation.");
SDL_Error(SDL_ENOMEM);
return 0;
}
SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
Expand Down Expand Up @@ -347,7 +349,7 @@ float GetClock()
}

/**
* \brief Execute a test suite using the given run seend and execution key.
* \brief Execute a test suite using the given run seed and execution key.
*
* The filter string is matched to the suite name (full comparison) to select a single suite,
* or if no suite matches, it is matched to the test names (full comparison) to select a single test.
Expand All @@ -362,6 +364,8 @@ float GetClock()
*/
int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
{
int totalNumberOfTests = 0;
int failedNumberOfTests = 0;
int suiteCounter;
int testCounter;
int iterationCounter;
Expand Down Expand Up @@ -392,6 +396,7 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
Uint32 testSkippedCount = 0;
Uint32 countSum = 0;
char *logFormat = (char *)SDLTest_LogSummaryFormat;
SDLTest_TestCaseReference **failedTests;

/* Sanitize test iterations */
if (testIterations < 1) {
Expand Down Expand Up @@ -421,6 +426,27 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
/* Log run with fuzzer parameters */
SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);

/* Count the total number of tests */
suiteCounter = 0;
while (testSuites[suiteCounter]) {
testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
suiteCounter++;
testCounter = 0;
while (testSuite->testCases[testCounter])
{
testCounter++;
totalNumberOfTests++;
}
}

/* Pre-allocate an array for tracking failed tests (potentially all test cases) */
failedTests = (SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
if (failedTests == NULL) {
SDLTest_LogError("Unable to allocate cache for failed tests");
SDL_Error(SDL_ENOMEM);
return -1;
}

/* Initialize filtering */
if (filter != NULL && filter[0] != '\0') {
/* Loop over all suites to check if we have a filter match */
Expand Down Expand Up @@ -580,6 +606,11 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
break;
}

/* Collect failed test case references for repro-step display */
if (testResult == TEST_RESULT_FAILED) {
failedTests[failedNumberOfTests] = testCase;
failedNumberOfTests++;
}
}
}

Expand Down Expand Up @@ -630,6 +661,15 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
}

/* Print repro steps for failed tests */
if (failedNumberOfTests > 0) {
SDLTest_Log("Harness input to repro failures:");
for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
}
}
SDL_free(failedTests);

SDLTest_Log("Exit code: %d", runResult);
return runResult;
}
44 changes: 38 additions & 6 deletions test/testautomation_platform.c
View file
Expand Up @@ -283,6 +283,7 @@ int platform_testDefaultInit(void *arg)
*/
int platform_testGetSetClearError(void *arg)
{
int result;
const char *testError = "Testing";
char *lastError;
int len;
Expand All @@ -301,8 +302,9 @@ int platform_testGetSetClearError(void *arg)
"SDL_GetError(): no message expected, len: %i", len);
}

SDL_SetError("%s", testError);
result = SDL_SetError("%s", testError);
SDLTest_AssertPass("SDL_SetError()");
SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
lastError = (char *)SDL_GetError();
SDLTest_AssertCheck(lastError != NULL,
"SDL_GetError() != NULL");
Expand Down Expand Up @@ -333,12 +335,14 @@ int platform_testGetSetClearError(void *arg)
*/
int platform_testSetErrorEmptyInput(void *arg)
{
int result;
const char *testError = "";
char *lastError;
int len;

SDL_SetError("%s", testError);
result = SDL_SetError("%s", testError);
SDLTest_AssertPass("SDL_SetError()");
SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
lastError = (char *)SDL_GetError();
SDLTest_AssertCheck(lastError != NULL,
"SDL_GetError() != NULL");
Expand Down Expand Up @@ -369,7 +373,8 @@ int platform_testSetErrorEmptyInput(void *arg)
*/
int platform_testSetErrorInvalidInput(void *arg)
{
const char *testError = NULL;
int result;
const char *invalidError = NULL;
const char *probeError = "Testing";
char *lastError;
int len;
Expand All @@ -379,8 +384,9 @@ int platform_testSetErrorInvalidInput(void *arg)
SDLTest_AssertPass("SDL_ClearError()");

/* Check for no-op */
SDL_SetError(testError);
result = SDL_SetError(invalidError);
SDLTest_AssertPass("SDL_SetError()");
SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
lastError = (char *)SDL_GetError();
SDLTest_AssertCheck(lastError != NULL,
"SDL_GetError() != NULL");
Expand All @@ -397,12 +403,14 @@ int platform_testSetErrorInvalidInput(void *arg)
}

/* Set */
SDL_SetError(probeError);
result = SDL_SetError(probeError);
SDLTest_AssertPass("SDL_SetError()");
SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);

/* Check for no-op */
SDL_SetError(testError);
result = SDL_SetError(invalidError);
SDLTest_AssertPass("SDL_SetError()");
SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
lastError = (char *)SDL_GetError();
SDLTest_AssertCheck(lastError != NULL,
"SDL_GetError() != NULL");
Expand All @@ -419,6 +427,30 @@ int platform_testSetErrorInvalidInput(void *arg)
lastError);
}

/* Reset */
SDL_ClearError();
SDLTest_AssertPass("SDL_ClearError()");

/* Set and check */
result = SDL_SetError(probeError);
SDLTest_AssertPass("SDL_SetError()");
SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
lastError = (char *)SDL_GetError();
SDLTest_AssertCheck(lastError != NULL,
"SDL_GetError() != NULL");
if (lastError != NULL)
{
len = SDL_strlen(lastError);
SDLTest_AssertCheck(len == SDL_strlen(probeError),
"SDL_GetError(): expected message len %i, was len: %i",
SDL_strlen(probeError),
len);
SDLTest_AssertCheck(SDL_strcmp(lastError, probeError) == 0,
"SDL_GetError(): expected message '%s', was message: '%s'",
probeError,
lastError);
}

/* Clean up */
SDL_ClearError();
SDLTest_AssertPass("SDL_ClearError()");
Expand Down
38 changes: 37 additions & 1 deletion test/testautomation_sdltest.c
View file
Expand Up @@ -21,6 +21,39 @@

/* Test case functions */

/* Forward declarations for internal harness functions */
extern char *SDLTest_GenerateRunSeed(const int length);

/**
* @brief Calls to SDLTest_GenerateRunSeed()
*/
int
sdltest_generateRunSeed(void *arg)
{
char* result;
int i, l;

for (i = 1; i <= 10; i += 3) {
result = SDLTest_GenerateRunSeed((const int)i);
SDLTest_AssertPass("Call to SDLTest_GenerateRunSeed()");
SDLTest_AssertCheck(result != NULL, "Verify returned value is not NULL");
if (result != NULL) {
l = SDL_strlen(result);
SDLTest_AssertCheck(l == i, "Verify length of returned value is %d, got: %d", i, l);
SDL_free(result);
}
}

/* Negative cases */
for (i = -2; i <= 0; i++) {
result = SDLTest_GenerateRunSeed((const int)i);
SDLTest_AssertPass("Call to SDLTest_GenerateRunSeed()");
SDLTest_AssertCheck(result == NULL, "Verify returned value is not NULL");
}

return TEST_COMPLETED;
}

/**
* @brief Calls to SDLTest_GetFuzzerInvocationCount()
*/
Expand Down Expand Up @@ -1263,11 +1296,14 @@ static const SDLTest_TestCaseReference sdltestTest13 =
static const SDLTest_TestCaseReference sdltestTest14 =
{ (SDLTest_TestCaseFp)sdltest_randomAsciiStringOfSize, "sdltest_randomAsciiStringOfSize", "Calls to fixed size ASCII string generator", TEST_ENABLED };

static const SDLTest_TestCaseReference sdltestTest15 =
{ (SDLTest_TestCaseFp)sdltest_generateRunSeed, "sdltest_generateRunSeed", "Checks internal harness function SDLTest_GenerateRunSeed", TEST_ENABLED };

/* Sequence of SDL_test test cases */
static const SDLTest_TestCaseReference *sdltestTests[] = {
&sdltestTest1, &sdltestTest2, &sdltestTest3, &sdltestTest4, &sdltestTest5, &sdltestTest6,
&sdltestTest7, &sdltestTest8, &sdltestTest9, &sdltestTest10, &sdltestTest11, &sdltestTest12,
&sdltestTest13, &sdltestTest14, NULL
&sdltestTest13, &sdltestTest14, &sdltestTest15, NULL
};

/* SDL_test test suite (global) */
Expand Down

0 comments on commit 7a36070

Please sign in to comment.