From 8dc030e34a950a0bda60c07d29148bd4e9570603 Mon Sep 17 00:00:00 2001 From: Andreas Schiffler Date: Fri, 30 Nov 2012 23:25:34 -0800 Subject: [PATCH] Add log, assert and harness (partial) to test lib --- include/SDL_log.h | 6 +- include/SDL_test.h | 3 + include/SDL_test_assert.h | 83 +++++++++++++++++++++ include/SDL_test_harness.h | 116 +++++++++++++++++++++++++++++ include/SDL_test_log.h | 71 ++++++++++++++++++ src/SDL_log.c | 7 +- src/test/SDL_test_assert.c | 96 ++++++++++++++++++++++++ src/test/SDL_test_harness.c | 142 ++++++++++++++++++++++++++++++++++++ src/test/SDL_test_log.c | 100 +++++++++++++++++++++++++ 9 files changed, 621 insertions(+), 3 deletions(-) create mode 100644 include/SDL_test_assert.h create mode 100644 include/SDL_test_harness.h create mode 100644 include/SDL_test_log.h create mode 100644 src/test/SDL_test_assert.c create mode 100644 src/test/SDL_test_harness.c create mode 100644 src/test/SDL_test_log.c diff --git a/include/SDL_log.h b/include/SDL_log.h index c87e9dd07..e61c44ce7 100644 --- a/include/SDL_log.h +++ b/include/SDL_log.h @@ -59,8 +59,9 @@ extern "C" { * \brief The predefined log categories * * By default the application category is enabled at the INFO level, - * the assert category is enabled at the WARN level, and all other - * categories are enabled at the CRITICAL level. + * the assert category is enabled at the WARN level, test is enabled + * at the VERBOSE level and all other categories are enabled at the + * CRITICAL level. */ enum { @@ -72,6 +73,7 @@ enum SDL_LOG_CATEGORY_VIDEO, SDL_LOG_CATEGORY_RENDER, SDL_LOG_CATEGORY_INPUT, + SDL_LOG_CATEGORY_TEST, /* Reserved for future SDL library use */ SDL_LOG_CATEGORY_RESERVED1, diff --git a/include/SDL_test.h b/include/SDL_test.h index 292ac6dc8..ff29cc4b8 100644 --- a/include/SDL_test.h +++ b/include/SDL_test.h @@ -36,6 +36,9 @@ #include "SDL_test_fuzzer.h" #include "SDL_test_crc32.h" #include "SDL_test_md5.h" +#include "SDL_test_log.h" +#include "SDL_test_assert.h" +#include "SDL_test_harness.h" #include "begin_code.h" /* Set up for C function definitions, even when using C++ */ diff --git a/include/SDL_test_assert.h b/include/SDL_test_assert.h new file mode 100644 index 000000000..d1533916f --- /dev/null +++ b/include/SDL_test_assert.h @@ -0,0 +1,83 @@ +/* + Simple DirectMedia Layer + Copyright (C) 1997-2012 Sam Lantinga + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. +*/ + +/** + * \file SDL_test_assert.h + * + * Include file for SDL test framework. + * + * This code is a part of the SDL2_test library, not the main SDL library. + */ + +/* + * + * Assert API for test code and test cases + * + */ + +#ifndef _SDL_test_assert_h +#define _SDL_test_assert_h + +#include "begin_code.h" +/* Set up for C function definitions, even when using C++ */ +#ifdef __cplusplus +/* *INDENT-OFF* */ +extern "C" { +/* *INDENT-ON* */ +#endif + +/** + * \brief Assert that logs and break execution flow on failures. + * + * \param assertCondition Evaluated condition or variable to assert; fail (==0) or pass (!=0). + * \param assertDescription Message to log with the assert describing it. + */ +void SDLTest_Assert(int assertCondition, char *assertDescription); + +/** + * \brief Assert for test cases that logs but does not break execution flow on failures. + * + * \param assertCondition Evaluated condition or variable to assert; fail (==0) or pass (!=0). + * \param assertDescription Message to log with the assert describing it. + */ +void SDLTest_AssertCheck(int assertCondition, char *assertDescription); + +/** + * \brief Resets the assert summary counters to zero. + */ +void SDLTest_ResetAssertSummary(); + +/** + * \brief Logs summary of all assertions (total, pass, fail) since last reset as INFO or ERROR. + * + */ +void SDLTest_LogAssertSummary(); + +#ifdef __cplusplus +/* *INDENT-OFF* */ +} +/* *INDENT-ON* */ +#endif +#include "close_code.h" + +#endif /* _SDL_test_assert_h */ + +/* vi: set ts=4 sw=4 expandtab: */ diff --git a/include/SDL_test_harness.h b/include/SDL_test_harness.h new file mode 100644 index 000000000..cad58b2c0 --- /dev/null +++ b/include/SDL_test_harness.h @@ -0,0 +1,116 @@ +/* + Simple DirectMedia Layer + Copyright (C) 1997-2012 Sam Lantinga + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. +*/ + +/** + * \file SDL_test_harness.h + * + * Include file for SDL test framework. + * + * This code is a part of the SDL2_test library, not the main SDL library. + */ + +/* + Defines types for test case definitions and the test execution harness API. + + Based on original GSOC code by Markus Kauppila +*/ + +#ifndef _SDL_test_harness_h +#define _SDL_test_harness_h + +#include "begin_code.h" +/* Set up for C function definitions, even when using C++ */ +#ifdef __cplusplus +/* *INDENT-OFF* */ +extern "C" { +/* *INDENT-ON* */ +#endif + + +//! Definitions for test case structures +#define TEST_ENABLED 1 +#define TEST_DISABLED 0 + +//! Definitions of assert results +#define ASSERT_PASS 1 +#define ASSERT_FAIL 0 + +//! Definition of all the possible test return values of the test case method +#define TEST_ABORTED -1 +#define TEST_COMPLETED 0 +#define TEST_SKIPPED 1 + +//! Definition of all the possible test results for the harness +#define TEST_RESULT_PASSED 0 +#define TEST_RESULT_FAILED 1 +#define TEST_RESULT_NO_ASSERT 2 +#define TEST_RESULT_SKIPPED 3 +#define TEST_RESULT_KILLED 4 +#define TEST_RESULT_SETUP_FAILURE 5 + +//!< Function pointer to a test case setup function (run before every test) +typedef void (*SDLTest_TestCaseSetUpFp)(void *arg); + +//!< Function pointer to a test case function +typedef void (*SDLTest_TestCaseFp)(void *arg); + +//!< Function pointer to a test case teardown function (run after every test) +typedef void (*SDLTest_TestCaseTearDownFp)(void *arg); + +/** + * Holds information about a single test case. + */ +typedef struct SDLTest_TestCaseReference { + /*!< Func2Stress */ + SDLTest_TestCaseFp testCase; + /*!< Short name (or function name) "Func2Stress" */ + char *name; + /*!< Long name or full description "This test pushes func2() to the limit." */ + char *description; + /*!< Set to TEST_ENABLED or TEST_DISABLED (test won't be run) */ + int enabled; +} SDLTest_TestCaseReference; + +/** + * Holds information about a test suite (multiple test cases). + */ +typedef struct TestSuiteReference { + /*!< "PlatformSuite" */ + char *name; + /*!< The function that is run before each test. NULL skips. */ + SDLTest_TestCaseSetUpFp testSetUp; + /*!< The test cases that are run as part of the suite. Last item should be NULL. */ + const SDLTest_TestCaseReference **testCases; + /*!< The function that is run after each test. NULL skips. */ + SDLTest_TestCaseTearDownFp testTearDown; +} TestSuiteReference; + +/* Ends C function definitions when using C++ */ +#ifdef __cplusplus +/* *INDENT-OFF* */ +} +/* *INDENT-ON* */ +#endif +#include "close_code.h" + +#endif /* _SDL_test_harness_h */ + +/* vi: set ts=4 sw=4 expandtab: */ diff --git a/include/SDL_test_log.h b/include/SDL_test_log.h new file mode 100644 index 000000000..e6b8d2b5c --- /dev/null +++ b/include/SDL_test_log.h @@ -0,0 +1,71 @@ +/* + Simple DirectMedia Layer + Copyright (C) 1997-2012 Sam Lantinga + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. +*/ + +/** + * \file SDL_test_log.h + * + * Include file for SDL test framework. + * + * This code is a part of the SDL2_test library, not the main SDL library. + */ + +/* + * + * Wrapper to log in the TEST category + * + */ + +#ifndef _SDL_test_log_h +#define _SDL_test_log_h + +#include "begin_code.h" +/* Set up for C function definitions, even when using C++ */ +#ifdef __cplusplus +/* *INDENT-OFF* */ +extern "C" { +/* *INDENT-ON* */ +#endif + +/** + * \brief Prints given message with a timestamp in the TEST category and INFO priority. + * + * \param fmt Message to be logged + */ +void SDLTest_Log(char *fmt, ...); + +/** + * \brief Prints given message with a timestamp in the TEST category and the ERROR priority. + * + * \param fmt Message to be logged + */ +void SDLTest_LogError(char *fmt, ...); + +/* Ends C function definitions when using C++ */ +#ifdef __cplusplus +/* *INDENT-OFF* */ +} +/* *INDENT-ON* */ +#endif +#include "close_code.h" + +#endif /* _SDL_test_log_h */ + +/* vi: set ts=4 sw=4 expandtab: */ diff --git a/src/SDL_log.c b/src/SDL_log.c index 109e5a848..b4f524326 100644 --- a/src/SDL_log.c +++ b/src/SDL_log.c @@ -37,6 +37,7 @@ #define DEFAULT_PRIORITY SDL_LOG_PRIORITY_CRITICAL #define DEFAULT_ASSERT_PRIORITY SDL_LOG_PRIORITY_WARN #define DEFAULT_APPLICATION_PRIORITY SDL_LOG_PRIORITY_INFO +#define DEFAULT_TEST_PRIORITY SDL_LOG_PRIORITY_VERBOSE typedef struct SDL_LogLevel { @@ -54,6 +55,7 @@ static SDL_LogLevel *SDL_loglevels; static SDL_LogPriority SDL_default_priority = DEFAULT_PRIORITY; static SDL_LogPriority SDL_assert_priority = DEFAULT_ASSERT_PRIORITY; static SDL_LogPriority SDL_application_priority = DEFAULT_APPLICATION_PRIORITY; +static SDL_LogPriority SDL_test_priority = DEFAULT_TEST_PRIORITY; static SDL_LogOutputFunction SDL_log_function = SDL_LogOutput; static void *SDL_log_userdata = NULL; @@ -135,7 +137,9 @@ SDL_LogGetPriority(int category) } } - if (category == SDL_LOG_CATEGORY_APPLICATION) { + if (category == SDL_LOG_CATEGORY_TEST) { + return SDL_test_priority; + } else if (category == SDL_LOG_CATEGORY_APPLICATION) { return SDL_application_priority; } else if (category == SDL_LOG_CATEGORY_ASSERT) { return SDL_assert_priority; @@ -158,6 +162,7 @@ SDL_LogResetPriorities(void) SDL_default_priority = DEFAULT_PRIORITY; SDL_assert_priority = DEFAULT_ASSERT_PRIORITY; SDL_application_priority = DEFAULT_APPLICATION_PRIORITY; + SDL_test_priority = DEFAULT_TEST_PRIORITY; } void diff --git a/src/test/SDL_test_assert.c b/src/test/SDL_test_assert.c new file mode 100644 index 000000000..698c9115a --- /dev/null +++ b/src/test/SDL_test_assert.c @@ -0,0 +1,96 @@ +/* + Simple DirectMedia Layer + Copyright (C) 1997-2012 Sam Lantinga + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. +*/ + +/* + + Used by the test framework and test cases. + +*/ + +#include "SDL_config.h" + +#include "SDL_test.h" + +/*! \brief counts the failed asserts */ +static Uint32 SDLTest_testAssertsFailed = 0; + +/*! \brief counts the passed asserts */ +static Uint32 SDLTest_testAssertsPassed = 0; + +/* Assert check message format */ +const char *SDLTest_AssertCheckFmt = "Assert %s: %s"; + +/* Assert summary message format */ +const char *SDLTest_AssertSummaryFmt = "Assert Summary: Total=%d Passed=%d Failed=%d"; + +/* + * Assert that logs and break execution flow on failures (i.e. for harness errors). + */ +void SDLTest_Assert(int assertCondition, char *assertDescription) +{ + SDLTest_AssertCheck(assertCondition, assertDescription); + SDL_assert((assertCondition)); +} + +/* + * Assert that logs but does not break execution flow on failures (i.e. for test cases). + */ +void SDLTest_AssertCheck(int assertCondition, char *assertDescription) +{ + char *fmt = (char *)SDLTest_AssertCheckFmt; + if (assertCondition) + { + SDLTest_testAssertsPassed++; + SDLTest_Log(fmt, "Passed", assertDescription); + } + else + { + SDLTest_testAssertsFailed++; + SDLTest_LogError(fmt, "Failed", assertDescription); + } +} + +/* + * Resets the assert summary counters to zero. + */ +void SDLTest_ResetAssertSummary() +{ + SDLTest_testAssertsPassed = 0; + SDLTest_testAssertsFailed = 0; +} + +/* + * Logs summary of all assertions (total, pass, fail) since last reset + * as INFO (failed==0) or ERROR (failed > 0). + */ +void SDLTest_LogAssertSummary() +{ + char *fmt = (char *)SDLTest_AssertSummaryFmt; + Uint32 totalAsserts = SDLTest_testAssertsPassed + SDLTest_testAssertsFailed; + if (SDLTest_testAssertsFailed == 0) + { + SDLTest_Log(fmt, totalAsserts, SDLTest_testAssertsPassed, SDLTest_testAssertsFailed); + } + else + { + SDLTest_LogError(fmt, totalAsserts, SDLTest_testAssertsPassed, SDLTest_testAssertsFailed); + } +} diff --git a/src/test/SDL_test_harness.c b/src/test/SDL_test_harness.c new file mode 100644 index 000000000..c52e4cc69 --- /dev/null +++ b/src/test/SDL_test_harness.c @@ -0,0 +1,142 @@ +/* + Simple DirectMedia Layer + Copyright (C) 1997-2012 Sam Lantinga + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. +*/ + +#include "SDL_config.h" + +#include "SDL_test.h" + +// TODO: port over harness + +/** + * Generates a random run seed string for the harness. The generated seed + * will contain alphanumeric characters (0-9A-Z). + * + * Note: The returned string needs to be deallocated by the caller. + * + * \param length The length of the seed string to generate + * + * \returns The generated seed string + */ +char * +SDLTest_GenerateRunSeed(const int length) +{ + char *seed = NULL; + SDLTest_RandomContext randomContext; + int counter; + + // Sanity check input + if (length <= 0) { + SDLTest_LogError("The length of the harness seed must be >0."); + return NULL; + } + + // Allocate output buffer + seed = (char *)SDL_malloc((length + 1) * sizeof(char)); + if (seed == NULL) { + SDLTest_LogError("SDL_malloc for run seed output buffer failed."); + return NULL; + } + + // Generate a random string of alphanumeric characters + SDLTest_RandomInitTime(&randomContext); + for (counter = 0; counter < length - 1; ++counter) { + unsigned int number = SDLTest_Random(&randomContext); + char ch = (char) (number % (91 - 48)) + 48; + if (ch >= 58 && ch <= 64) { + ch = 65; + } + seed[counter] = ch; + } + seed[counter] = '\0'; + + return seed; +} + +/** + * Generates an execution key for the fuzzer. + * + * \param runSeed The run seed to use + * \param suiteName The name of the test suite + * \param testName The name of the test + * \param iteration The iteration count + * + * \returns The generated execution key to initialize the fuzzer with. + * + */ +Uint64 +SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration) +{ + SDLTest_Md5Context md5Context; + Uint64 *keys; + char iterationString[16]; + Uint32 runSeedLength; + Uint32 suiteNameLength; + Uint32 testNameLength; + Uint32 iterationStringLength; + Uint32 entireStringLength; + char *buffer; + + if (runSeed == NULL || strlen(runSeed)==0) { + SDLTest_LogError("Invalid runSeed string."); + return -1; + } + + if (suiteName == NULL || strlen(suiteName)==0) { + SDLTest_LogError("Invalid suiteName string."); + return -1; + } + + if (testName == NULL || strlen(testName)==0) { + SDLTest_LogError("Invalid testName string."); + return -1; + } + + if (iteration <= 0) { + SDLTest_LogError("Invalid iteration count."); + return -1; + } + + // Convert iteration number into a string + memset(iterationString, 0, sizeof(iterationString)); + SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration); + + // Combine the parameters into single string + runSeedLength = strlen(runSeed); + suiteNameLength = strlen(suiteName); + testNameLength = strlen(testName); + iterationStringLength = strlen(iterationString); + entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1; + buffer = (char *)SDL_malloc(entireStringLength); + if (buffer == NULL) { + SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation."); + return 0; + } + SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration); + + // Hash string and use half of the digest as 64bit exec key + SDLTest_Md5Init(&md5Context); + SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength); + SDLTest_Md5Final(&md5Context); + SDL_free(buffer); + keys = (Uint64 *)md5Context.digest; + + return keys[0]; +} diff --git a/src/test/SDL_test_log.c b/src/test/SDL_test_log.c new file mode 100644 index 000000000..ca387d577 --- /dev/null +++ b/src/test/SDL_test_log.c @@ -0,0 +1,100 @@ +/* + Simple DirectMedia Layer + Copyright (C) 1997-2012 Sam Lantinga + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. +*/ + +/* + + Used by the test framework and test cases. + +*/ + +#include "SDL_config.h" + +#include /* va_list */ +#include + +#include "SDL_test.h" + +/* + * Note: Maximum size of SDLTest log message is less than SDLs limit + * to ensure we can fit additional information such as the timestamp. + */ +#define SDLTEST_MAX_LOGMESSAGE_LENGTH 3584 + +/*! + * Converts unix timestamp to its ascii representation in localtime + * + * Note: Uses a static buffer internally, so the return value + * isn't valid after the next call of this function. If you + * want to retain the return value, make a copy of it. + * + * \param timestamp A Timestamp, i.e. time(0) + * + * \return Ascii representation of the timestamp in localtime + */ +char *SDLTest_TimestampToString(const time_t timestamp) +{ + time_t copy; + static char buffer[256]; + struct tm *local; + + memset(buffer, 0, sizeof(buffer));\ + copy = timestamp; + local = localtime(©); + strftime(buffer, sizeof(buffer), "%a %Y-%m-%d %H:%M:%S %Z", local); + + return buffer; +} + +/* + * Prints given message with a timestamp in the TEST category and INFO priority. + */ +void SDLTest_Log(char *fmt, ...) +{ + va_list list; + char logMessage[SDLTEST_MAX_LOGMESSAGE_LENGTH]; + + // Print log message into a buffer + memset(logMessage, 0, SDLTEST_MAX_LOGMESSAGE_LENGTH); + va_start(list, fmt); + SDL_vsnprintf(logMessage, SDLTEST_MAX_LOGMESSAGE_LENGTH - 1, fmt, list); + va_end(list); + + // Log with timestamp and newline + SDL_LogMessage(SDL_LOG_CATEGORY_TEST, SDL_LOG_PRIORITY_INFO, "%s: %s\n", SDLTest_TimestampToString(time(0)), logMessage); +} + +/* + * Prints given message with a timestamp in the TEST category and the ERROR priority. + */ +void SDLTest_LogError(char *fmt, ...) +{ + va_list list; + char logMessage[SDLTEST_MAX_LOGMESSAGE_LENGTH]; + + // Print log message into a buffer + memset(logMessage, 0, SDLTEST_MAX_LOGMESSAGE_LENGTH); + va_start(list, fmt); + SDL_vsnprintf(logMessage, SDLTEST_MAX_LOGMESSAGE_LENGTH - 1, fmt, list); + va_end(list); + + // Log with timestamp and newline + SDL_LogMessage(SDL_LOG_CATEGORY_TEST, SDL_LOG_PRIORITY_ERROR, "%s: %s\n", SDLTest_TimestampToString(time(0)), logMessage); +}