Fix bug/add test coverage for SDLTest_GenerateRunSeed helper; improve test harness adding output of repro steps for failures; improve negative test for SDL_GetError/SDL_SetError
authorAndreas Schiffler <aschiffler@ferzkopp.net>
Tue, 11 Mar 2014 07:17:56 -0700
changeset 860557faccca4fab
parent 8604 459bd7365e9d
child 8606 29ffd0d3926b
Fix bug/add test coverage for SDLTest_GenerateRunSeed helper; improve test harness adding output of repro steps for failures; improve negative test for SDL_GetError/SDL_SetError
src/test/SDL_test_harness.c
test/testautomation_platform.c
test/testautomation_sdltest.c
     1.1 --- a/src/test/SDL_test_harness.c	Mon Mar 10 19:59:06 2014 -0700
     1.2 +++ b/src/test/SDL_test_harness.c	Tue Mar 11 07:17:56 2014 -0700
     1.3 @@ -67,12 +67,13 @@
     1.4      seed = (char *)SDL_malloc((length + 1) * sizeof(char));
     1.5      if (seed == NULL) {
     1.6          SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
     1.7 +        SDL_Error(SDL_ENOMEM);
     1.8          return NULL;
     1.9      }
    1.10  
    1.11      /* Generate a random string of alphanumeric characters */
    1.12      SDLTest_RandomInitTime(&randomContext);
    1.13 -    for (counter = 0; counter < length - 1; ++counter) {
    1.14 +    for (counter = 0; counter < length; counter++) {
    1.15          unsigned int number = SDLTest_Random(&randomContext);
    1.16          char ch = (char) (number % (91 - 48)) + 48;
    1.17          if (ch >= 58 && ch <= 64) {
    1.18 @@ -80,7 +81,7 @@
    1.19          }
    1.20          seed[counter] = ch;
    1.21      }
    1.22 -    seed[counter] = '\0';
    1.23 +    seed[length] = '\0';
    1.24  
    1.25      return seed;
    1.26  }
    1.27 @@ -141,7 +142,8 @@
    1.28      entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
    1.29      buffer = (char *)SDL_malloc(entireStringLength);
    1.30      if (buffer == NULL) {
    1.31 -        SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
    1.32 +        SDLTest_LogError("Failed to allocate buffer for execKey generation.");
    1.33 +        SDL_Error(SDL_ENOMEM);
    1.34          return 0;
    1.35      }
    1.36      SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
    1.37 @@ -347,7 +349,7 @@
    1.38  }
    1.39  
    1.40  /**
    1.41 -* \brief Execute a test suite using the given run seend and execution key.
    1.42 +* \brief Execute a test suite using the given run seed and execution key.
    1.43  *
    1.44  * The filter string is matched to the suite name (full comparison) to select a single suite,
    1.45  * or if no suite matches, it is matched to the test names (full comparison) to select a single test.
    1.46 @@ -362,6 +364,8 @@
    1.47  */
    1.48  int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
    1.49  {
    1.50 +    int totalNumberOfTests = 0;
    1.51 +    int failedNumberOfTests = 0;
    1.52      int suiteCounter;
    1.53      int testCounter;
    1.54      int iterationCounter;
    1.55 @@ -392,6 +396,7 @@
    1.56      Uint32 testSkippedCount = 0;
    1.57      Uint32 countSum = 0;
    1.58      char *logFormat = (char *)SDLTest_LogSummaryFormat;
    1.59 +    SDLTest_TestCaseReference **failedTests;
    1.60  
    1.61      /* Sanitize test iterations */
    1.62      if (testIterations < 1) {
    1.63 @@ -421,6 +426,27 @@
    1.64      /* Log run with fuzzer parameters */
    1.65      SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
    1.66  
    1.67 +	/* Count the total number of tests */
    1.68 +    suiteCounter = 0;
    1.69 +    while (testSuites[suiteCounter]) {
    1.70 +        testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
    1.71 +        suiteCounter++;
    1.72 +        testCounter = 0;
    1.73 +        while (testSuite->testCases[testCounter])
    1.74 +        {
    1.75 +            testCounter++;
    1.76 +			totalNumberOfTests++;
    1.77 +		}
    1.78 +	}
    1.79 +
    1.80 +	/* Pre-allocate an array for tracking failed tests (potentially all test cases) */
    1.81 +	failedTests = (SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
    1.82 +	if (failedTests == NULL) {	
    1.83 +	   SDLTest_LogError("Unable to allocate cache for failed tests");
    1.84 +           SDL_Error(SDL_ENOMEM);	   
    1.85 +           return -1;
    1.86 +	}
    1.87 +
    1.88      /* Initialize filtering */
    1.89      if (filter != NULL && filter[0] != '\0') {
    1.90          /* Loop over all suites to check if we have a filter match */
    1.91 @@ -580,6 +606,11 @@
    1.92                          break;
    1.93                      }
    1.94  
    1.95 +                    /* Collect failed test case references for repro-step display */
    1.96 +                    if (testResult == TEST_RESULT_FAILED) {
    1.97 +                        failedTests[failedNumberOfTests] = testCase;
    1.98 +                        failedNumberOfTests++;
    1.99 +                    }
   1.100                  }
   1.101              }
   1.102  
   1.103 @@ -630,6 +661,15 @@
   1.104          SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
   1.105      }
   1.106  
   1.107 +    /* Print repro steps for failed tests */
   1.108 +    if (failedNumberOfTests > 0) {
   1.109 +        SDLTest_Log("Harness input to repro failures:");
   1.110 +        for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
   1.111 +          SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
   1.112 +        }
   1.113 +    }
   1.114 +    SDL_free(failedTests);
   1.115 +
   1.116      SDLTest_Log("Exit code: %d", runResult);
   1.117      return runResult;
   1.118  }
     2.1 --- a/test/testautomation_platform.c	Mon Mar 10 19:59:06 2014 -0700
     2.2 +++ b/test/testautomation_platform.c	Tue Mar 11 07:17:56 2014 -0700
     2.3 @@ -283,6 +283,7 @@
     2.4   */
     2.5  int platform_testGetSetClearError(void *arg)
     2.6  {
     2.7 +   int result;
     2.8     const char *testError = "Testing";
     2.9     char *lastError;
    2.10     int len;
    2.11 @@ -301,8 +302,9 @@
    2.12               "SDL_GetError(): no message expected, len: %i", len);
    2.13     }
    2.14  
    2.15 -   SDL_SetError("%s", testError);
    2.16 +   result = SDL_SetError("%s", testError);
    2.17     SDLTest_AssertPass("SDL_SetError()");
    2.18 +   SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
    2.19     lastError = (char *)SDL_GetError();
    2.20     SDLTest_AssertCheck(lastError != NULL,
    2.21               "SDL_GetError() != NULL");
    2.22 @@ -333,12 +335,14 @@
    2.23   */
    2.24  int platform_testSetErrorEmptyInput(void *arg)
    2.25  {
    2.26 +   int result;
    2.27     const char *testError = "";
    2.28     char *lastError;
    2.29     int len;
    2.30  
    2.31 -   SDL_SetError("%s", testError);
    2.32 +   result = SDL_SetError("%s", testError);
    2.33     SDLTest_AssertPass("SDL_SetError()");
    2.34 +   SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
    2.35     lastError = (char *)SDL_GetError();
    2.36     SDLTest_AssertCheck(lastError != NULL,
    2.37               "SDL_GetError() != NULL");
    2.38 @@ -369,7 +373,8 @@
    2.39   */
    2.40  int platform_testSetErrorInvalidInput(void *arg)
    2.41  {
    2.42 -   const char *testError = NULL;
    2.43 +   int result;
    2.44 +   const char *invalidError = NULL;
    2.45     const char *probeError = "Testing";
    2.46     char *lastError;
    2.47     int len;
    2.48 @@ -379,8 +384,9 @@
    2.49     SDLTest_AssertPass("SDL_ClearError()");
    2.50  
    2.51     /* Check for no-op */
    2.52 -   SDL_SetError(testError);
    2.53 +   result = SDL_SetError(invalidError);
    2.54     SDLTest_AssertPass("SDL_SetError()");
    2.55 +   SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
    2.56     lastError = (char *)SDL_GetError();
    2.57     SDLTest_AssertCheck(lastError != NULL,
    2.58               "SDL_GetError() != NULL");
    2.59 @@ -397,12 +403,14 @@
    2.60     }
    2.61  
    2.62     /* Set */
    2.63 -   SDL_SetError(probeError);
    2.64 +   result = SDL_SetError(probeError);
    2.65     SDLTest_AssertPass("SDL_SetError()");
    2.66 +   SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
    2.67  
    2.68     /* Check for no-op */
    2.69 -   SDL_SetError(testError);
    2.70 +   result = SDL_SetError(invalidError);
    2.71     SDLTest_AssertPass("SDL_SetError()");
    2.72 +   SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
    2.73     lastError = (char *)SDL_GetError();
    2.74     SDLTest_AssertCheck(lastError != NULL,
    2.75               "SDL_GetError() != NULL");
    2.76 @@ -419,6 +427,30 @@
    2.77               lastError);
    2.78     }
    2.79  
    2.80 +   /* Reset */
    2.81 +   SDL_ClearError();
    2.82 +   SDLTest_AssertPass("SDL_ClearError()");
    2.83 +
    2.84 +   /* Set and check */
    2.85 +   result = SDL_SetError(probeError);
    2.86 +   SDLTest_AssertPass("SDL_SetError()");
    2.87 +   SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
    2.88 +   lastError = (char *)SDL_GetError();
    2.89 +   SDLTest_AssertCheck(lastError != NULL,
    2.90 +             "SDL_GetError() != NULL");
    2.91 +   if (lastError != NULL)
    2.92 +   {
    2.93 +     len = SDL_strlen(lastError);
    2.94 +     SDLTest_AssertCheck(len == SDL_strlen(probeError),
    2.95 +             "SDL_GetError(): expected message len %i, was len: %i",
    2.96 +             SDL_strlen(probeError),
    2.97 +             len);
    2.98 +     SDLTest_AssertCheck(SDL_strcmp(lastError, probeError) == 0,
    2.99 +             "SDL_GetError(): expected message '%s', was message: '%s'",
   2.100 +             probeError,
   2.101 +             lastError);
   2.102 +   }
   2.103 +   
   2.104     /* Clean up */
   2.105     SDL_ClearError();
   2.106     SDLTest_AssertPass("SDL_ClearError()");
     3.1 --- a/test/testautomation_sdltest.c	Mon Mar 10 19:59:06 2014 -0700
     3.2 +++ b/test/testautomation_sdltest.c	Tue Mar 11 07:17:56 2014 -0700
     3.3 @@ -21,6 +21,39 @@
     3.4  
     3.5  /* Test case functions */
     3.6  
     3.7 +/* Forward declarations for internal harness functions */
     3.8 +extern char *SDLTest_GenerateRunSeed(const int length);
     3.9 +
    3.10 +/**
    3.11 + * @brief Calls to SDLTest_GenerateRunSeed()
    3.12 + */
    3.13 +int
    3.14 +sdltest_generateRunSeed(void *arg)
    3.15 +{
    3.16 +  char* result;
    3.17 +  int i, l;
    3.18 +  
    3.19 +  for (i = 1; i <= 10; i += 3) {   
    3.20 +     result = SDLTest_GenerateRunSeed((const int)i);
    3.21 +     SDLTest_AssertPass("Call to SDLTest_GenerateRunSeed()");
    3.22 +     SDLTest_AssertCheck(result != NULL, "Verify returned value is not NULL");
    3.23 +     if (result != NULL) {
    3.24 +       l = SDL_strlen(result);
    3.25 +       SDLTest_AssertCheck(l == i, "Verify length of returned value is %d, got: %d", i, l);
    3.26 +       SDL_free(result);
    3.27 +     }
    3.28 +  }
    3.29 +
    3.30 +  /* Negative cases */
    3.31 +  for (i = -2; i <= 0; i++) {   
    3.32 +     result = SDLTest_GenerateRunSeed((const int)i);
    3.33 +     SDLTest_AssertPass("Call to SDLTest_GenerateRunSeed()");
    3.34 +     SDLTest_AssertCheck(result == NULL, "Verify returned value is not NULL");
    3.35 +  }
    3.36 +  
    3.37 +  return TEST_COMPLETED;
    3.38 +}
    3.39 +
    3.40  /**
    3.41   * @brief Calls to SDLTest_GetFuzzerInvocationCount()
    3.42   */
    3.43 @@ -1263,11 +1296,14 @@
    3.44  static const SDLTest_TestCaseReference sdltestTest14 =
    3.45          { (SDLTest_TestCaseFp)sdltest_randomAsciiStringOfSize, "sdltest_randomAsciiStringOfSize", "Calls to fixed size ASCII string generator", TEST_ENABLED };
    3.46  
    3.47 +static const SDLTest_TestCaseReference sdltestTest15 =
    3.48 +        { (SDLTest_TestCaseFp)sdltest_generateRunSeed, "sdltest_generateRunSeed", "Checks internal harness function SDLTest_GenerateRunSeed", TEST_ENABLED };
    3.49 +
    3.50  /* Sequence of SDL_test test cases */
    3.51  static const SDLTest_TestCaseReference *sdltestTests[] =  {
    3.52      &sdltestTest1, &sdltestTest2, &sdltestTest3, &sdltestTest4, &sdltestTest5, &sdltestTest6,
    3.53      &sdltestTest7, &sdltestTest8, &sdltestTest9, &sdltestTest10, &sdltestTest11, &sdltestTest12,
    3.54 -    &sdltestTest13, &sdltestTest14, NULL
    3.55 +    &sdltestTest13, &sdltestTest14, &sdltestTest15, NULL
    3.56  };
    3.57  
    3.58  /* SDL_test test suite (global) */