visualtest/src/testharness.c
author Sam Lantinga
Tue, 10 Jan 2017 08:54:33 -0800
changeset 10806 36f40b8cc979
parent 7924 fcb86d323770
permissions -rw-r--r--
Fixed bugs 2570, 3145, improved OpenGL ES context support on Windows and X11

Mark Callow

The attached patch does the following for the X11 and Windows platforms, the only ones where SDL attempts to use context_create_es_profile:

- Adds SDL_HINT_OPENGL_ES_DRIVER by which the application can
say to use the OpenGL ES driver & EGL rather than the Open GL
driver. (For bug #2570)
- Adds code to {WIN,X11}_GL_InitExtensions to determine the maximum
OpenGL ES version supported by the OpenGL driver (for bug #3145)
- Modifies the test that determines whether to use the OpenGL
driver or the real OpenGL ES driver to take into account the
hint, the requested and supported ES version and whether ES 1.X
is being requested. (For bug #2570 & bug #3145)
- Enables the testgles2 test for __WINDOWS__ and __LINUX__ and adds
the test to the VisualC projects.

With the fix in place I have run testdraw2, testgl and testgles2 without any issues and have run my own apps that use OpenGL, OpenGL ES 3 and OpenGL ES 1.1.
icculus@7924
     1
/* See COPYING.txt for the full license governing this code. */
icculus@7924
     2
/**
icculus@7924
     3
 *  \file testharness.c 
icculus@7924
     4
 *
icculus@7924
     5
 *  Source file for the test harness.
icculus@7924
     6
 */
icculus@7924
     7
icculus@7924
     8
#include <stdlib.h>
icculus@7924
     9
#include <SDL_test.h>
icculus@7924
    10
#include <SDL.h>
icculus@7924
    11
#include <SDL_assert.h>
icculus@7924
    12
#include "SDL_visualtest_harness_argparser.h"
icculus@7924
    13
#include "SDL_visualtest_process.h"
icculus@7924
    14
#include "SDL_visualtest_variators.h"
icculus@7924
    15
#include "SDL_visualtest_screenshot.h"
icculus@7924
    16
#include "SDL_visualtest_mischelper.h"
icculus@7924
    17
icculus@7924
    18
#if defined(__WIN32__) && !defined(__CYGWIN__)
icculus@7924
    19
#include <direct.h>
icculus@7924
    20
#elif defined(__WIN32__) && defined(__CYGWIN__)
icculus@7924
    21
#include <signal.h>
icculus@7924
    22
#elif defined(__LINUX__)
icculus@7924
    23
#include <sys/stat.h>
icculus@7924
    24
#include <sys/types.h>
icculus@7924
    25
#include <signal.h>
icculus@7924
    26
#else
icculus@7924
    27
#error "Unsupported platform"
icculus@7924
    28
#endif
icculus@7924
    29
icculus@7924
    30
/** Code for the user event triggered when a new action is to be executed */
icculus@7924
    31
#define ACTION_TIMER_EVENT 0
icculus@7924
    32
/** Code for the user event triggered when the maximum timeout is reached */
icculus@7924
    33
#define KILL_TIMER_EVENT 1
icculus@7924
    34
/** FPS value used for delays in the action loop */
icculus@7924
    35
#define ACTION_LOOP_FPS 10
icculus@7924
    36
icculus@7924
    37
/** Value returned by RunSUTAndTest() when the test has passed */
icculus@7924
    38
#define TEST_PASSED 1
icculus@7924
    39
/** Value returned by RunSUTAndTest() when the test has failed */
icculus@7924
    40
#define TEST_FAILED 0
icculus@7924
    41
/** Value returned by RunSUTAndTest() on a fatal error */
icculus@7924
    42
#define TEST_ERROR -1
icculus@7924
    43
icculus@7924
    44
static SDL_ProcessInfo pinfo;
icculus@7924
    45
static SDL_ProcessExitStatus sut_exitstatus;
icculus@7924
    46
static SDLVisualTest_HarnessState state;
icculus@7924
    47
static SDLVisualTest_Variator variator;
icculus@7924
    48
static SDLVisualTest_ActionNode* current; /* the current action being performed */
icculus@7924
    49
static SDL_TimerID action_timer, kill_timer;
icculus@7924
    50
icculus@7924
    51
/* returns a char* to be passed as the format argument of a printf-style function. */
icculus@7924
    52
static char*
icculus@7924
    53
usage()
icculus@7924
    54
{
icculus@7924
    55
    return "Usage: \n%s --sutapp xyz"
icculus@7924
    56
           " [--sutargs abc | --parameter-config xyz.parameters"
icculus@7924
    57
           " [--variator exhaustive|random]" 
icculus@7924
    58
           " [--num-variations N] [--no-launch]] [--timeout hh:mm:ss]"
icculus@7924
    59
           " [--action-config xyz.actions]"
icculus@7924
    60
           " [--output-dir /path/to/output]"
icculus@7924
    61
           " [--verify-dir /path/to/verify]"
icculus@7924
    62
           " or --config app.config";
icculus@7924
    63
}
icculus@7924
    64
icculus@7924
    65
/* register Ctrl+C handlers */
icculus@7924
    66
#if defined(__LINUX__) || defined(__CYGWIN__)
icculus@7924
    67
static void
icculus@7924
    68
CtrlCHandlerCallback(int signum)
icculus@7924
    69
{
icculus@7924
    70
    SDL_Event event;
icculus@7924
    71
    SDLTest_Log("Ctrl+C received");
icculus@7924
    72
    event.type = SDL_QUIT;
icculus@7924
    73
    SDL_PushEvent(&event);
icculus@7924
    74
}
icculus@7924
    75
#endif
icculus@7924
    76
icculus@7924
    77
static Uint32
icculus@7924
    78
ActionTimerCallback(Uint32 interval, void* param)
icculus@7924
    79
{
icculus@7924
    80
    SDL_Event event;
icculus@7924
    81
    SDL_UserEvent userevent;
icculus@7924
    82
    Uint32 next_action_time;
icculus@7924
    83
icculus@7924
    84
    /* push an event to handle the action */
icculus@7924
    85
    userevent.type = SDL_USEREVENT;
icculus@7924
    86
    userevent.code = ACTION_TIMER_EVENT;
icculus@7924
    87
    userevent.data1 = &current->action;
icculus@7924
    88
    userevent.data2 = NULL;
icculus@7924
    89
icculus@7924
    90
    event.type = SDL_USEREVENT;
icculus@7924
    91
    event.user = userevent;
icculus@7924
    92
    SDL_PushEvent(&event);
icculus@7924
    93
icculus@7924
    94
    /* calculate the new interval and return it */
icculus@7924
    95
    if(current->next)
icculus@7924
    96
        next_action_time = current->next->action.time - current->action.time;
icculus@7924
    97
    else
icculus@7924
    98
    {
icculus@7924
    99
        next_action_time = 0;
icculus@7924
   100
        action_timer = 0;
icculus@7924
   101
    }
icculus@7924
   102
icculus@7924
   103
    current = current->next;
icculus@7924
   104
    return next_action_time;
icculus@7924
   105
}
icculus@7924
   106
icculus@7924
   107
static Uint32
icculus@7924
   108
KillTimerCallback(Uint32 interval, void* param)
icculus@7924
   109
{
icculus@7924
   110
    SDL_Event event;
icculus@7924
   111
    SDL_UserEvent userevent;
icculus@7924
   112
icculus@7924
   113
    userevent.type = SDL_USEREVENT;
icculus@7924
   114
    userevent.code = KILL_TIMER_EVENT;
icculus@7924
   115
    userevent.data1 = NULL;
icculus@7924
   116
    userevent.data2 = NULL;
icculus@7924
   117
icculus@7924
   118
    event.type = SDL_USEREVENT;
icculus@7924
   119
    event.user = userevent;
icculus@7924
   120
    SDL_PushEvent(&event);
icculus@7924
   121
icculus@7924
   122
    kill_timer = 0;
icculus@7924
   123
    return 0;
icculus@7924
   124
}
icculus@7924
   125
icculus@7924
   126
static int
icculus@7924
   127
ProcessAction(SDLVisualTest_Action* action, int* sut_running, char* args)
icculus@7924
   128
{
icculus@7924
   129
    if(!action || !sut_running)
icculus@7924
   130
        return TEST_ERROR;
icculus@7924
   131
icculus@7924
   132
    switch(action->type)
icculus@7924
   133
    {
icculus@7924
   134
        case SDL_ACTION_KILL:
icculus@7924
   135
            SDLTest_Log("Action: Kill SUT");
icculus@7924
   136
            if(SDL_IsProcessRunning(&pinfo) == 1 &&
icculus@7924
   137
               !SDL_KillProcess(&pinfo, &sut_exitstatus))
icculus@7924
   138
            {
icculus@7924
   139
                SDLTest_LogError("SDL_KillProcess() failed");
icculus@7924
   140
                return TEST_ERROR;
icculus@7924
   141
            }
icculus@7924
   142
            *sut_running = 0;
icculus@7924
   143
        break;
icculus@7924
   144
icculus@7924
   145
        case SDL_ACTION_QUIT:
icculus@7924
   146
            SDLTest_Log("Action: Quit SUT");
icculus@7924
   147
            if(SDL_IsProcessRunning(&pinfo) == 1 &&
icculus@7924
   148
               !SDL_QuitProcess(&pinfo, &sut_exitstatus))
icculus@7924
   149
            {
icculus@7924
   150
                SDLTest_LogError("SDL_QuitProcess() failed");
icculus@7924
   151
                return TEST_FAILED;
icculus@7924
   152
            }
icculus@7924
   153
            *sut_running = 0;
icculus@7924
   154
        break;
icculus@7924
   155
icculus@7924
   156
        case SDL_ACTION_LAUNCH:
icculus@7924
   157
        {
icculus@7924
   158
            char* path;
icculus@7924
   159
            char* args;
icculus@7924
   160
            SDL_ProcessInfo action_process;
icculus@7924
   161
            SDL_ProcessExitStatus ps;
icculus@7924
   162
icculus@7924
   163
            path = action->extra.process.path;
icculus@7924
   164
            args = action->extra.process.args;
icculus@7924
   165
            if(args)
icculus@7924
   166
            {
icculus@7924
   167
                SDLTest_Log("Action: Launch process: %s with arguments: %s",
icculus@7924
   168
                            path, args);
icculus@7924
   169
            }
icculus@7924
   170
            else
icculus@7924
   171
                SDLTest_Log("Action: Launch process: %s", path);
icculus@7924
   172
            if(!SDL_LaunchProcess(path, args, &action_process))
icculus@7924
   173
            {
icculus@7924
   174
                SDLTest_LogError("SDL_LaunchProcess() failed");
icculus@7924
   175
                return TEST_ERROR;
icculus@7924
   176
            }
icculus@7924
   177
icculus@7924
   178
            /* small delay so that the process can do its job */
icculus@7924
   179
            SDL_Delay(1000);
icculus@7924
   180
icculus@7924
   181
            if(SDL_IsProcessRunning(&action_process) > 0)
icculus@7924
   182
            {
icculus@7924
   183
                SDLTest_LogError("Process %s took too long too complete."
icculus@7924
   184
                                    " Force killing...", action->extra);
icculus@7924
   185
                if(!SDL_KillProcess(&action_process, &ps))
icculus@7924
   186
                {
icculus@7924
   187
                    SDLTest_LogError("SDL_KillProcess() failed");
icculus@7924
   188
                    return TEST_ERROR;
icculus@7924
   189
                }
icculus@7924
   190
            }
icculus@7924
   191
        }
icculus@7924
   192
        break;
icculus@7924
   193
icculus@7924
   194
        case SDL_ACTION_SCREENSHOT:
icculus@7924
   195
        {
icculus@7924
   196
            char path[MAX_PATH_LEN], hash[33];
icculus@7924
   197
icculus@7924
   198
            SDLTest_Log("Action: Take screenshot");
icculus@7924
   199
            /* can't take a screenshot if the SUT isn't running */
icculus@7924
   200
            if(SDL_IsProcessRunning(&pinfo) != 1)
icculus@7924
   201
            {
icculus@7924
   202
                SDLTest_LogError("SUT has quit.");
icculus@7924
   203
                *sut_running = 0;
icculus@7924
   204
                return TEST_FAILED;
icculus@7924
   205
            }
icculus@7924
   206
icculus@7924
   207
            /* file name for the screenshot image */
icculus@7924
   208
            SDLVisualTest_HashString(args, hash);
icculus@7924
   209
            SDL_snprintf(path, MAX_PATH_LEN, "%s/%s", state.output_dir, hash);
icculus@7924
   210
            if(!SDLVisualTest_ScreenshotProcess(&pinfo, path))
icculus@7924
   211
            {
icculus@7924
   212
                SDLTest_LogError("SDLVisualTest_ScreenshotProcess() failed");
icculus@7924
   213
                return TEST_ERROR;
icculus@7924
   214
            }
icculus@7924
   215
        }
icculus@7924
   216
        break;
icculus@7924
   217
icculus@7924
   218
        case SDL_ACTION_VERIFY:
icculus@7924
   219
        {
icculus@7924
   220
            int ret;
icculus@7924
   221
icculus@7924
   222
            SDLTest_Log("Action: Verify screenshot");
icculus@7924
   223
            ret = SDLVisualTest_VerifyScreenshots(args, state.output_dir,
icculus@7924
   224
                                                  state.verify_dir);
icculus@7924
   225
icculus@7924
   226
            if(ret == -1)
icculus@7924
   227
            {
icculus@7924
   228
                SDLTest_LogError("SDLVisualTest_VerifyScreenshots() failed");
icculus@7924
   229
                return TEST_ERROR;
icculus@7924
   230
            }
icculus@7924
   231
            else if(ret == 0)
icculus@7924
   232
            {
icculus@7924
   233
                SDLTest_Log("Verification failed: Images were not equal.");
icculus@7924
   234
                return TEST_FAILED;
icculus@7924
   235
            }
icculus@7924
   236
            else if(ret == 1)
icculus@7924
   237
                SDLTest_Log("Verification successful.");
icculus@7924
   238
            else
icculus@7924
   239
            {
icculus@7924
   240
                SDLTest_Log("Verfication skipped.");
icculus@7924
   241
                return TEST_FAILED;
icculus@7924
   242
            }
icculus@7924
   243
        }
icculus@7924
   244
        break;
icculus@7924
   245
icculus@7924
   246
        default:
icculus@7924
   247
            SDLTest_LogError("Invalid action type");
icculus@7924
   248
            return TEST_ERROR;
icculus@7924
   249
        break;
icculus@7924
   250
    }
icculus@7924
   251
icculus@7924
   252
    return TEST_PASSED;
icculus@7924
   253
}
icculus@7924
   254
icculus@7924
   255
static int
icculus@7924
   256
RunSUTAndTest(char* sutargs, int variation_num)
icculus@7924
   257
{
icculus@7924
   258
    int success, sut_running, return_code;
icculus@7924
   259
    char hash[33];
icculus@7924
   260
    SDL_Event event;
icculus@7924
   261
icculus@7924
   262
    return_code = TEST_PASSED;
icculus@7924
   263
icculus@7924
   264
    if(!sutargs)
icculus@7924
   265
    {
icculus@7924
   266
        SDLTest_LogError("sutargs argument cannot be NULL");
icculus@7924
   267
        return_code = TEST_ERROR;
icculus@7924
   268
        goto runsutandtest_cleanup_generic;
icculus@7924
   269
    }
icculus@7924
   270
icculus@7924
   271
    SDLVisualTest_HashString(sutargs, hash);
icculus@7924
   272
    SDLTest_Log("Hash: %s", hash);
icculus@7924
   273
icculus@7924
   274
    success = SDL_LaunchProcess(state.sutapp, sutargs, &pinfo);
icculus@7924
   275
    if(!success)
icculus@7924
   276
    {
icculus@7924
   277
        SDLTest_Log("Could not launch SUT.");
icculus@7924
   278
        return_code = TEST_ERROR;
icculus@7924
   279
        goto runsutandtest_cleanup_generic;
icculus@7924
   280
    }
icculus@7924
   281
    SDLTest_Log("SUT launch successful.");
icculus@7924
   282
    SDLTest_Log("Process will be killed in %d milliseconds", state.timeout);
icculus@7924
   283
    sut_running = 1;
icculus@7924
   284
icculus@7924
   285
    /* launch the timers */
icculus@7924
   286
    SDLTest_Log("Performing actions..");
icculus@7924
   287
    current = state.action_queue.front;
icculus@7924
   288
    action_timer = 0;
icculus@7924
   289
    kill_timer = 0;
icculus@7924
   290
    if(current)
icculus@7924
   291
    {
icculus@7924
   292
        action_timer = SDL_AddTimer(current->action.time, ActionTimerCallback, NULL);
icculus@7924
   293
        if(!action_timer)
icculus@7924
   294
        {
icculus@7924
   295
            SDLTest_LogError("SDL_AddTimer() failed");
icculus@7924
   296
            return_code = TEST_ERROR;
icculus@7924
   297
            goto runsutandtest_cleanup_timer;
icculus@7924
   298
        }
icculus@7924
   299
    }
icculus@7924
   300
    kill_timer = SDL_AddTimer(state.timeout, KillTimerCallback, NULL);
icculus@7924
   301
    if(!kill_timer)
icculus@7924
   302
    {
icculus@7924
   303
        SDLTest_LogError("SDL_AddTimer() failed");
icculus@7924
   304
        return_code = TEST_ERROR;
icculus@7924
   305
        goto runsutandtest_cleanup_timer;
icculus@7924
   306
    }
icculus@7924
   307
icculus@7924
   308
    /* the timer stops running if the actions queue is empty, and the
icculus@7924
   309
       SUT stops running if it crashes or if we encounter a KILL/QUIT action */
icculus@7924
   310
    while(sut_running)
icculus@7924
   311
    {
icculus@7924
   312
        /* process the actions by using an event queue */
icculus@7924
   313
        while(SDL_PollEvent(&event))
icculus@7924
   314
        {
icculus@7924
   315
            if(event.type == SDL_USEREVENT)
icculus@7924
   316
            {
icculus@7924
   317
                if(event.user.code == ACTION_TIMER_EVENT)
icculus@7924
   318
                {
icculus@7924
   319
                    SDLVisualTest_Action* action;
icculus@7924
   320
icculus@7924
   321
                    action = (SDLVisualTest_Action*)event.user.data1;
icculus@7924
   322
icculus@7924
   323
                    switch(ProcessAction(action, &sut_running, sutargs))
icculus@7924
   324
                    {
icculus@7924
   325
                        case TEST_PASSED:
icculus@7924
   326
                        break;
icculus@7924
   327
icculus@7924
   328
                        case TEST_FAILED:
icculus@7924
   329
                            return_code = TEST_FAILED;
icculus@7924
   330
                            goto runsutandtest_cleanup_timer;
icculus@7924
   331
                        break;
icculus@7924
   332
icculus@7924
   333
                        default:
icculus@7924
   334
                            SDLTest_LogError("ProcessAction() failed");
icculus@7924
   335
                            return_code = TEST_ERROR;
icculus@7924
   336
                            goto runsutandtest_cleanup_timer;
icculus@7924
   337
                    }
icculus@7924
   338
                }
icculus@7924
   339
                else if(event.user.code == KILL_TIMER_EVENT)
icculus@7924
   340
                {
icculus@7924
   341
                    SDLTest_LogError("Maximum timeout reached. Force killing..");
icculus@7924
   342
                    return_code = TEST_FAILED;
icculus@7924
   343
                    goto runsutandtest_cleanup_timer;
icculus@7924
   344
                }
icculus@7924
   345
            }
icculus@7924
   346
            else if(event.type == SDL_QUIT)
icculus@7924
   347
            {
icculus@7924
   348
                SDLTest_LogError("Received QUIT event. Testharness is quitting..");
icculus@7924
   349
                return_code = TEST_ERROR;
icculus@7924
   350
                goto runsutandtest_cleanup_timer;
icculus@7924
   351
            }
icculus@7924
   352
        }
icculus@7924
   353
        SDL_Delay(1000/ACTION_LOOP_FPS);
icculus@7924
   354
    }
icculus@7924
   355
icculus@7924
   356
    SDLTest_Log("SUT exit code was: %d", sut_exitstatus.exit_status);
icculus@7924
   357
    if(sut_exitstatus.exit_status == 0)
icculus@7924
   358
    {
icculus@7924
   359
        return_code = TEST_PASSED;
icculus@7924
   360
        goto runsutandtest_cleanup_timer;
icculus@7924
   361
    }
icculus@7924
   362
    else
icculus@7924
   363
    {
icculus@7924
   364
        return_code = TEST_FAILED;
icculus@7924
   365
        goto runsutandtest_cleanup_timer;
icculus@7924
   366
    }
icculus@7924
   367
icculus@7924
   368
    return_code = TEST_ERROR;
icculus@7924
   369
    goto runsutandtest_cleanup_generic;
icculus@7924
   370
icculus@7924
   371
runsutandtest_cleanup_timer:
icculus@7924
   372
    if(action_timer && !SDL_RemoveTimer(action_timer))
icculus@7924
   373
    {
icculus@7924
   374
        SDLTest_Log("SDL_RemoveTimer() failed");
icculus@7924
   375
        return_code = TEST_ERROR;
icculus@7924
   376
    }
icculus@7924
   377
icculus@7924
   378
    if(kill_timer && !SDL_RemoveTimer(kill_timer))
icculus@7924
   379
    {
icculus@7924
   380
        SDLTest_Log("SDL_RemoveTimer() failed");
icculus@7924
   381
        return_code = TEST_ERROR;
icculus@7924
   382
    }
icculus@7924
   383
/* runsutandtest_cleanup_process: */
icculus@7924
   384
    if(SDL_IsProcessRunning(&pinfo) && !SDL_KillProcess(&pinfo, &sut_exitstatus))
icculus@7924
   385
    {
icculus@7924
   386
        SDLTest_Log("SDL_KillProcess() failed");
icculus@7924
   387
        return_code = TEST_ERROR;
icculus@7924
   388
    }
icculus@7924
   389
runsutandtest_cleanup_generic:
icculus@7924
   390
    return return_code;
icculus@7924
   391
}
icculus@7924
   392
icculus@7924
   393
/** Entry point for testharness */
icculus@7924
   394
int
icculus@7924
   395
main(int argc, char* argv[])
icculus@7924
   396
{
icculus@7924
   397
    int i, passed, return_code, failed;
icculus@7924
   398
icculus@7924
   399
    /* freeing resources, linux style! */
icculus@7924
   400
    return_code = 0;
icculus@7924
   401
icculus@7924
   402
    if(argc < 2)
icculus@7924
   403
    {
icculus@7924
   404
        SDLTest_Log(usage(), argv[0]);
icculus@7924
   405
        goto cleanup_generic;
icculus@7924
   406
    }
icculus@7924
   407
icculus@7924
   408
#if defined(__LINUX__) || defined(__CYGWIN__)
icculus@7924
   409
    signal(SIGINT, CtrlCHandlerCallback);
icculus@7924
   410
#endif
icculus@7924
   411
icculus@7924
   412
    /* parse arguments */
icculus@7924
   413
    if(!SDLVisualTest_ParseHarnessArgs(argv + 1, &state))
icculus@7924
   414
    {
icculus@7924
   415
        SDLTest_Log(usage(), argv[0]);
icculus@7924
   416
        return_code = 1;
icculus@7924
   417
        goto cleanup_generic;
icculus@7924
   418
    }
icculus@7924
   419
    SDLTest_Log("Parsed harness arguments successfully.");
icculus@7924
   420
icculus@7924
   421
    /* initialize SDL */
icculus@7924
   422
    if(SDL_Init(SDL_INIT_TIMER) == -1)
icculus@7924
   423
    {
icculus@7924
   424
        SDLTest_LogError("SDL_Init() failed.");
icculus@7924
   425
        SDLVisualTest_FreeHarnessState(&state);
icculus@7924
   426
        return_code = 1;
icculus@7924
   427
        goto cleanup_harness_state;
icculus@7924
   428
    }
icculus@7924
   429
icculus@7924
   430
    /* create an output directory if none exists */
icculus@7924
   431
#if defined(__LINUX__) || defined(__CYGWIN__)
icculus@7924
   432
    mkdir(state.output_dir, 0777);
icculus@7924
   433
#elif defined(__WIN32__)
icculus@7924
   434
    _mkdir(state.output_dir);
icculus@7924
   435
#else
icculus@7924
   436
#error "Unsupported platform"
icculus@7924
   437
#endif
icculus@7924
   438
icculus@7924
   439
    /* test with sutargs */
icculus@7924
   440
    if(SDL_strlen(state.sutargs))
icculus@7924
   441
    {
icculus@7924
   442
        SDLTest_Log("Running: %s %s", state.sutapp, state.sutargs);
icculus@7924
   443
        if(!state.no_launch)
icculus@7924
   444
        {
icculus@7924
   445
            switch(RunSUTAndTest(state.sutargs, 0))
icculus@7924
   446
            {
icculus@7924
   447
                case TEST_PASSED:
icculus@7924
   448
                    SDLTest_Log("Status: PASSED");
icculus@7924
   449
                break;
icculus@7924
   450
icculus@7924
   451
                case TEST_FAILED:
icculus@7924
   452
                    SDLTest_Log("Status: FAILED");
icculus@7924
   453
                break;
icculus@7924
   454
icculus@7924
   455
                case TEST_ERROR:
icculus@7924
   456
                    SDLTest_LogError("Some error occurred while testing.");
icculus@7924
   457
                    return_code = 1;
icculus@7924
   458
                    goto cleanup_sdl;
icculus@7924
   459
                break;
icculus@7924
   460
            }
icculus@7924
   461
        }
icculus@7924
   462
    }
icculus@7924
   463
icculus@7924
   464
    if(state.sut_config.num_options > 0)
icculus@7924
   465
    {
icculus@7924
   466
        char* variator_name = state.variator_type == SDL_VARIATOR_RANDOM ?
icculus@7924
   467
                              "RANDOM" : "EXHAUSTIVE";
icculus@7924
   468
        if(state.num_variations > 0)
icculus@7924
   469
            SDLTest_Log("Testing SUT with variator: %s for %d variations",
icculus@7924
   470
                        variator_name, state.num_variations);
icculus@7924
   471
        else
icculus@7924
   472
            SDLTest_Log("Testing SUT with variator: %s and ALL variations",
icculus@7924
   473
                        variator_name);
icculus@7924
   474
        /* initialize the variator */
icculus@7924
   475
        if(!SDLVisualTest_InitVariator(&variator, &state.sut_config,
icculus@7924
   476
                                       state.variator_type, 0))
icculus@7924
   477
        {
icculus@7924
   478
            SDLTest_LogError("Could not initialize variator");
icculus@7924
   479
            return_code = 1;
icculus@7924
   480
            goto cleanup_sdl;
icculus@7924
   481
        }
icculus@7924
   482
icculus@7924
   483
        /* iterate through all the variations */
icculus@7924
   484
        passed = 0;
icculus@7924
   485
        failed = 0;
icculus@7924
   486
        for(i = 0; state.num_variations > 0 ? (i < state.num_variations) : 1; i++)
icculus@7924
   487
        {
icculus@7924
   488
            char* args = SDLVisualTest_GetNextVariation(&variator);
icculus@7924
   489
            if(!args)
icculus@7924
   490
                break;
icculus@7924
   491
            SDLTest_Log("\nVariation number: %d\nArguments: %s", i + 1, args);
icculus@7924
   492
icculus@7924
   493
            if(!state.no_launch)
icculus@7924
   494
            {
icculus@7924
   495
                switch(RunSUTAndTest(args, i + 1))
icculus@7924
   496
                {
icculus@7924
   497
                    case TEST_PASSED:
icculus@7924
   498
                        SDLTest_Log("Status: PASSED");
icculus@7924
   499
                        passed++;
icculus@7924
   500
                    break;
icculus@7924
   501
icculus@7924
   502
                    case TEST_FAILED:
icculus@7924
   503
                        SDLTest_Log("Status: FAILED");
icculus@7924
   504
                        failed++;
icculus@7924
   505
                    break;
icculus@7924
   506
icculus@7924
   507
                    case TEST_ERROR:
icculus@7924
   508
                        SDLTest_LogError("Some error occurred while testing.");
icculus@7924
   509
                        goto cleanup_variator;
icculus@7924
   510
                    break;
icculus@7924
   511
                }
icculus@7924
   512
            }
icculus@7924
   513
        }
icculus@7924
   514
        if(!state.no_launch)
icculus@7924
   515
        {
icculus@7924
   516
            /* report stats */
icculus@7924
   517
            SDLTest_Log("Testing complete.");
icculus@7924
   518
            SDLTest_Log("%d/%d tests passed.", passed, passed + failed);
icculus@7924
   519
        }
icculus@7924
   520
        goto cleanup_variator;
icculus@7924
   521
    }
icculus@7924
   522
 
icculus@7924
   523
    goto cleanup_sdl;
icculus@7924
   524
icculus@7924
   525
cleanup_variator:
icculus@7924
   526
    SDLVisualTest_FreeVariator(&variator);
icculus@7924
   527
cleanup_sdl:
icculus@7924
   528
    SDL_Quit();
icculus@7924
   529
cleanup_harness_state:
icculus@7924
   530
    SDLVisualTest_FreeHarnessState(&state);
icculus@7924
   531
cleanup_generic:
icculus@7924
   532
    return return_code;
icculus@7924
   533
}