visualtest/src/testharness.c
author Sam Lantinga <slouken@libsdl.org>
Wed, 23 May 2018 17:15:35 -0700
changeset 11999 f3fc4b3d54f6
parent 7924 fcb86d323770
permissions -rw-r--r--
Better fix for axis sorting with some Android controllers
icculus@7924
     1
/* See COPYING.txt for the full license governing this code. */
icculus@7924
     2
/**
icculus@7924
     3
 *  \file testharness.c 
icculus@7924
     4
 *
icculus@7924
     5
 *  Source file for the test harness.
icculus@7924
     6
 */
icculus@7924
     7
icculus@7924
     8
#include <stdlib.h>
icculus@7924
     9
#include <SDL_test.h>
icculus@7924
    10
#include <SDL.h>
icculus@7924
    11
#include <SDL_assert.h>
icculus@7924
    12
#include "SDL_visualtest_harness_argparser.h"
icculus@7924
    13
#include "SDL_visualtest_process.h"
icculus@7924
    14
#include "SDL_visualtest_variators.h"
icculus@7924
    15
#include "SDL_visualtest_screenshot.h"
icculus@7924
    16
#include "SDL_visualtest_mischelper.h"
icculus@7924
    17
icculus@7924
    18
#if defined(__WIN32__) && !defined(__CYGWIN__)
icculus@7924
    19
#include <direct.h>
icculus@7924
    20
#elif defined(__WIN32__) && defined(__CYGWIN__)
icculus@7924
    21
#include <signal.h>
icculus@7924
    22
#elif defined(__LINUX__)
icculus@7924
    23
#include <sys/stat.h>
icculus@7924
    24
#include <sys/types.h>
icculus@7924
    25
#include <signal.h>
icculus@7924
    26
#else
icculus@7924
    27
#error "Unsupported platform"
icculus@7924
    28
#endif
icculus@7924
    29
icculus@7924
    30
/** Code for the user event triggered when a new action is to be executed */
icculus@7924
    31
#define ACTION_TIMER_EVENT 0
icculus@7924
    32
/** Code for the user event triggered when the maximum timeout is reached */
icculus@7924
    33
#define KILL_TIMER_EVENT 1
icculus@7924
    34
/** FPS value used for delays in the action loop */
icculus@7924
    35
#define ACTION_LOOP_FPS 10
icculus@7924
    36
icculus@7924
    37
/** Value returned by RunSUTAndTest() when the test has passed */
icculus@7924
    38
#define TEST_PASSED 1
icculus@7924
    39
/** Value returned by RunSUTAndTest() when the test has failed */
icculus@7924
    40
#define TEST_FAILED 0
icculus@7924
    41
/** Value returned by RunSUTAndTest() on a fatal error */
icculus@7924
    42
#define TEST_ERROR -1
icculus@7924
    43
icculus@7924
    44
static SDL_ProcessInfo pinfo;
icculus@7924
    45
static SDL_ProcessExitStatus sut_exitstatus;
icculus@7924
    46
static SDLVisualTest_HarnessState state;
icculus@7924
    47
static SDLVisualTest_Variator variator;
icculus@7924
    48
static SDLVisualTest_ActionNode* current; /* the current action being performed */
icculus@7924
    49
static SDL_TimerID action_timer, kill_timer;
icculus@7924
    50
icculus@7924
    51
/* returns a char* to be passed as the format argument of a printf-style function. */
icculus@7924
    52
static char*
icculus@7924
    53
usage()
icculus@7924
    54
{
icculus@7924
    55
    return "Usage: \n%s --sutapp xyz"
icculus@7924
    56
           " [--sutargs abc | --parameter-config xyz.parameters"
icculus@7924
    57
           " [--variator exhaustive|random]" 
icculus@7924
    58
           " [--num-variations N] [--no-launch]] [--timeout hh:mm:ss]"
icculus@7924
    59
           " [--action-config xyz.actions]"
icculus@7924
    60
           " [--output-dir /path/to/output]"
icculus@7924
    61
           " [--verify-dir /path/to/verify]"
icculus@7924
    62
           " or --config app.config";
icculus@7924
    63
}
icculus@7924
    64
icculus@7924
    65
/* register Ctrl+C handlers */
icculus@7924
    66
#if defined(__LINUX__) || defined(__CYGWIN__)
icculus@7924
    67
static void
icculus@7924
    68
CtrlCHandlerCallback(int signum)
icculus@7924
    69
{
icculus@7924
    70
    SDL_Event event;
icculus@7924
    71
    SDLTest_Log("Ctrl+C received");
icculus@7924
    72
    event.type = SDL_QUIT;
icculus@7924
    73
    SDL_PushEvent(&event);
icculus@7924
    74
}
icculus@7924
    75
#endif
icculus@7924
    76
icculus@7924
    77
static Uint32
icculus@7924
    78
ActionTimerCallback(Uint32 interval, void* param)
icculus@7924
    79
{
icculus@7924
    80
    SDL_Event event;
icculus@7924
    81
    SDL_UserEvent userevent;
icculus@7924
    82
    Uint32 next_action_time;
icculus@7924
    83
icculus@7924
    84
    /* push an event to handle the action */
icculus@7924
    85
    userevent.type = SDL_USEREVENT;
icculus@7924
    86
    userevent.code = ACTION_TIMER_EVENT;
icculus@7924
    87
    userevent.data1 = &current->action;
icculus@7924
    88
    userevent.data2 = NULL;
icculus@7924
    89
icculus@7924
    90
    event.type = SDL_USEREVENT;
icculus@7924
    91
    event.user = userevent;
icculus@7924
    92
    SDL_PushEvent(&event);
icculus@7924
    93
icculus@7924
    94
    /* calculate the new interval and return it */
icculus@7924
    95
    if(current->next)
icculus@7924
    96
        next_action_time = current->next->action.time - current->action.time;
icculus@7924
    97
    else
icculus@7924
    98
    {
icculus@7924
    99
        next_action_time = 0;
icculus@7924
   100
        action_timer = 0;
icculus@7924
   101
    }
icculus@7924
   102
icculus@7924
   103
    current = current->next;
icculus@7924
   104
    return next_action_time;
icculus@7924
   105
}
icculus@7924
   106
icculus@7924
   107
static Uint32
icculus@7924
   108
KillTimerCallback(Uint32 interval, void* param)
icculus@7924
   109
{
icculus@7924
   110
    SDL_Event event;
icculus@7924
   111
    SDL_UserEvent userevent;
icculus@7924
   112
icculus@7924
   113
    userevent.type = SDL_USEREVENT;
icculus@7924
   114
    userevent.code = KILL_TIMER_EVENT;
icculus@7924
   115
    userevent.data1 = NULL;
icculus@7924
   116
    userevent.data2 = NULL;
icculus@7924
   117
icculus@7924
   118
    event.type = SDL_USEREVENT;
icculus@7924
   119
    event.user = userevent;
icculus@7924
   120
    SDL_PushEvent(&event);
icculus@7924
   121
icculus@7924
   122
    kill_timer = 0;
icculus@7924
   123
    return 0;
icculus@7924
   124
}
icculus@7924
   125
icculus@7924
   126
static int
icculus@7924
   127
ProcessAction(SDLVisualTest_Action* action, int* sut_running, char* args)
icculus@7924
   128
{
icculus@7924
   129
    if(!action || !sut_running)
icculus@7924
   130
        return TEST_ERROR;
icculus@7924
   131
icculus@7924
   132
    switch(action->type)
icculus@7924
   133
    {
icculus@7924
   134
        case SDL_ACTION_KILL:
icculus@7924
   135
            SDLTest_Log("Action: Kill SUT");
icculus@7924
   136
            if(SDL_IsProcessRunning(&pinfo) == 1 &&
icculus@7924
   137
               !SDL_KillProcess(&pinfo, &sut_exitstatus))
icculus@7924
   138
            {
icculus@7924
   139
                SDLTest_LogError("SDL_KillProcess() failed");
icculus@7924
   140
                return TEST_ERROR;
icculus@7924
   141
            }
icculus@7924
   142
            *sut_running = 0;
icculus@7924
   143
        break;
icculus@7924
   144
icculus@7924
   145
        case SDL_ACTION_QUIT:
icculus@7924
   146
            SDLTest_Log("Action: Quit SUT");
icculus@7924
   147
            if(SDL_IsProcessRunning(&pinfo) == 1 &&
icculus@7924
   148
               !SDL_QuitProcess(&pinfo, &sut_exitstatus))
icculus@7924
   149
            {
icculus@7924
   150
                SDLTest_LogError("SDL_QuitProcess() failed");
icculus@7924
   151
                return TEST_FAILED;
icculus@7924
   152
            }
icculus@7924
   153
            *sut_running = 0;
icculus@7924
   154
        break;
icculus@7924
   155
icculus@7924
   156
        case SDL_ACTION_LAUNCH:
icculus@7924
   157
        {
icculus@7924
   158
            char* path;
icculus@7924
   159
            char* args;
icculus@7924
   160
            SDL_ProcessInfo action_process;
icculus@7924
   161
            SDL_ProcessExitStatus ps;
icculus@7924
   162
icculus@7924
   163
            path = action->extra.process.path;
icculus@7924
   164
            args = action->extra.process.args;
icculus@7924
   165
            if(args)
icculus@7924
   166
            {
icculus@7924
   167
                SDLTest_Log("Action: Launch process: %s with arguments: %s",
icculus@7924
   168
                            path, args);
icculus@7924
   169
            }
icculus@7924
   170
            else
icculus@7924
   171
                SDLTest_Log("Action: Launch process: %s", path);
icculus@7924
   172
            if(!SDL_LaunchProcess(path, args, &action_process))
icculus@7924
   173
            {
icculus@7924
   174
                SDLTest_LogError("SDL_LaunchProcess() failed");
icculus@7924
   175
                return TEST_ERROR;
icculus@7924
   176
            }
icculus@7924
   177
icculus@7924
   178
            /* small delay so that the process can do its job */
icculus@7924
   179
            SDL_Delay(1000);
icculus@7924
   180
icculus@7924
   181
            if(SDL_IsProcessRunning(&action_process) > 0)
icculus@7924
   182
            {
icculus@7924
   183
                SDLTest_LogError("Process %s took too long too complete."
icculus@7924
   184
                                    " Force killing...", action->extra);
icculus@7924
   185
                if(!SDL_KillProcess(&action_process, &ps))
icculus@7924
   186
                {
icculus@7924
   187
                    SDLTest_LogError("SDL_KillProcess() failed");
icculus@7924
   188
                    return TEST_ERROR;
icculus@7924
   189
                }
icculus@7924
   190
            }
icculus@7924
   191
        }
icculus@7924
   192
        break;
icculus@7924
   193
icculus@7924
   194
        case SDL_ACTION_SCREENSHOT:
icculus@7924
   195
        {
icculus@7924
   196
            char path[MAX_PATH_LEN], hash[33];
icculus@7924
   197
icculus@7924
   198
            SDLTest_Log("Action: Take screenshot");
icculus@7924
   199
            /* can't take a screenshot if the SUT isn't running */
icculus@7924
   200
            if(SDL_IsProcessRunning(&pinfo) != 1)
icculus@7924
   201
            {
icculus@7924
   202
                SDLTest_LogError("SUT has quit.");
icculus@7924
   203
                *sut_running = 0;
icculus@7924
   204
                return TEST_FAILED;
icculus@7924
   205
            }
icculus@7924
   206
icculus@7924
   207
            /* file name for the screenshot image */
icculus@7924
   208
            SDLVisualTest_HashString(args, hash);
icculus@7924
   209
            SDL_snprintf(path, MAX_PATH_LEN, "%s/%s", state.output_dir, hash);
icculus@7924
   210
            if(!SDLVisualTest_ScreenshotProcess(&pinfo, path))
icculus@7924
   211
            {
icculus@7924
   212
                SDLTest_LogError("SDLVisualTest_ScreenshotProcess() failed");
icculus@7924
   213
                return TEST_ERROR;
icculus@7924
   214
            }
icculus@7924
   215
        }
icculus@7924
   216
        break;
icculus@7924
   217
icculus@7924
   218
        case SDL_ACTION_VERIFY:
icculus@7924
   219
        {
icculus@7924
   220
            int ret;
icculus@7924
   221
icculus@7924
   222
            SDLTest_Log("Action: Verify screenshot");
icculus@7924
   223
            ret = SDLVisualTest_VerifyScreenshots(args, state.output_dir,
icculus@7924
   224
                                                  state.verify_dir);
icculus@7924
   225
icculus@7924
   226
            if(ret == -1)
icculus@7924
   227
            {
icculus@7924
   228
                SDLTest_LogError("SDLVisualTest_VerifyScreenshots() failed");
icculus@7924
   229
                return TEST_ERROR;
icculus@7924
   230
            }
icculus@7924
   231
            else if(ret == 0)
icculus@7924
   232
            {
icculus@7924
   233
                SDLTest_Log("Verification failed: Images were not equal.");
icculus@7924
   234
                return TEST_FAILED;
icculus@7924
   235
            }
icculus@7924
   236
            else if(ret == 1)
icculus@7924
   237
                SDLTest_Log("Verification successful.");
icculus@7924
   238
            else
icculus@7924
   239
            {
icculus@7924
   240
                SDLTest_Log("Verfication skipped.");
icculus@7924
   241
                return TEST_FAILED;
icculus@7924
   242
            }
icculus@7924
   243
        }
icculus@7924
   244
        break;
icculus@7924
   245
icculus@7924
   246
        default:
icculus@7924
   247
            SDLTest_LogError("Invalid action type");
icculus@7924
   248
            return TEST_ERROR;
icculus@7924
   249
        break;
icculus@7924
   250
    }
icculus@7924
   251
icculus@7924
   252
    return TEST_PASSED;
icculus@7924
   253
}
icculus@7924
   254
icculus@7924
   255
static int
icculus@7924
   256
RunSUTAndTest(char* sutargs, int variation_num)
icculus@7924
   257
{
icculus@7924
   258
    int success, sut_running, return_code;
icculus@7924
   259
    char hash[33];
icculus@7924
   260
    SDL_Event event;
icculus@7924
   261
icculus@7924
   262
    return_code = TEST_PASSED;
icculus@7924
   263
icculus@7924
   264
    if(!sutargs)
icculus@7924
   265
    {
icculus@7924
   266
        SDLTest_LogError("sutargs argument cannot be NULL");
icculus@7924
   267
        return_code = TEST_ERROR;
icculus@7924
   268
        goto runsutandtest_cleanup_generic;
icculus@7924
   269
    }
icculus@7924
   270
icculus@7924
   271
    SDLVisualTest_HashString(sutargs, hash);
icculus@7924
   272
    SDLTest_Log("Hash: %s", hash);
icculus@7924
   273
icculus@7924
   274
    success = SDL_LaunchProcess(state.sutapp, sutargs, &pinfo);
icculus@7924
   275
    if(!success)
icculus@7924
   276
    {
icculus@7924
   277
        SDLTest_Log("Could not launch SUT.");
icculus@7924
   278
        return_code = TEST_ERROR;
icculus@7924
   279
        goto runsutandtest_cleanup_generic;
icculus@7924
   280
    }
icculus@7924
   281
    SDLTest_Log("SUT launch successful.");
icculus@7924
   282
    SDLTest_Log("Process will be killed in %d milliseconds", state.timeout);
icculus@7924
   283
    sut_running = 1;
icculus@7924
   284
icculus@7924
   285
    /* launch the timers */
icculus@7924
   286
    SDLTest_Log("Performing actions..");
icculus@7924
   287
    current = state.action_queue.front;
icculus@7924
   288
    action_timer = 0;
icculus@7924
   289
    kill_timer = 0;
icculus@7924
   290
    if(current)
icculus@7924
   291
    {
icculus@7924
   292
        action_timer = SDL_AddTimer(current->action.time, ActionTimerCallback, NULL);
icculus@7924
   293
        if(!action_timer)
icculus@7924
   294
        {
icculus@7924
   295
            SDLTest_LogError("SDL_AddTimer() failed");
icculus@7924
   296
            return_code = TEST_ERROR;
icculus@7924
   297
            goto runsutandtest_cleanup_timer;
icculus@7924
   298
        }
icculus@7924
   299
    }
icculus@7924
   300
    kill_timer = SDL_AddTimer(state.timeout, KillTimerCallback, NULL);
icculus@7924
   301
    if(!kill_timer)
icculus@7924
   302
    {
icculus@7924
   303
        SDLTest_LogError("SDL_AddTimer() failed");
icculus@7924
   304
        return_code = TEST_ERROR;
icculus@7924
   305
        goto runsutandtest_cleanup_timer;
icculus@7924
   306
    }
icculus@7924
   307
icculus@7924
   308
    /* the timer stops running if the actions queue is empty, and the
icculus@7924
   309
       SUT stops running if it crashes or if we encounter a KILL/QUIT action */
icculus@7924
   310
    while(sut_running)
icculus@7924
   311
    {
icculus@7924
   312
        /* process the actions by using an event queue */
icculus@7924
   313
        while(SDL_PollEvent(&event))
icculus@7924
   314
        {
icculus@7924
   315
            if(event.type == SDL_USEREVENT)
icculus@7924
   316
            {
icculus@7924
   317
                if(event.user.code == ACTION_TIMER_EVENT)
icculus@7924
   318
                {
icculus@7924
   319
                    SDLVisualTest_Action* action;
icculus@7924
   320
icculus@7924
   321
                    action = (SDLVisualTest_Action*)event.user.data1;
icculus@7924
   322
icculus@7924
   323
                    switch(ProcessAction(action, &sut_running, sutargs))
icculus@7924
   324
                    {
icculus@7924
   325
                        case TEST_PASSED:
icculus@7924
   326
                        break;
icculus@7924
   327
icculus@7924
   328
                        case TEST_FAILED:
icculus@7924
   329
                            return_code = TEST_FAILED;
icculus@7924
   330
                            goto runsutandtest_cleanup_timer;
icculus@7924
   331
                        break;
icculus@7924
   332
icculus@7924
   333
                        default:
icculus@7924
   334
                            SDLTest_LogError("ProcessAction() failed");
icculus@7924
   335
                            return_code = TEST_ERROR;
icculus@7924
   336
                            goto runsutandtest_cleanup_timer;
icculus@7924
   337
                    }
icculus@7924
   338
                }
icculus@7924
   339
                else if(event.user.code == KILL_TIMER_EVENT)
icculus@7924
   340
                {
icculus@7924
   341
                    SDLTest_LogError("Maximum timeout reached. Force killing..");
icculus@7924
   342
                    return_code = TEST_FAILED;
icculus@7924
   343
                    goto runsutandtest_cleanup_timer;
icculus@7924
   344
                }
icculus@7924
   345
            }
icculus@7924
   346
            else if(event.type == SDL_QUIT)
icculus@7924
   347
            {
icculus@7924
   348
                SDLTest_LogError("Received QUIT event. Testharness is quitting..");
icculus@7924
   349
                return_code = TEST_ERROR;
icculus@7924
   350
                goto runsutandtest_cleanup_timer;
icculus@7924
   351
            }
icculus@7924
   352
        }
icculus@7924
   353
        SDL_Delay(1000/ACTION_LOOP_FPS);
icculus@7924
   354
    }
icculus@7924
   355
icculus@7924
   356
    SDLTest_Log("SUT exit code was: %d", sut_exitstatus.exit_status);
icculus@7924
   357
    if(sut_exitstatus.exit_status == 0)
icculus@7924
   358
    {
icculus@7924
   359
        return_code = TEST_PASSED;
icculus@7924
   360
        goto runsutandtest_cleanup_timer;
icculus@7924
   361
    }
icculus@7924
   362
    else
icculus@7924
   363
    {
icculus@7924
   364
        return_code = TEST_FAILED;
icculus@7924
   365
        goto runsutandtest_cleanup_timer;
icculus@7924
   366
    }
icculus@7924
   367
icculus@7924
   368
    return_code = TEST_ERROR;
icculus@7924
   369
    goto runsutandtest_cleanup_generic;
icculus@7924
   370
icculus@7924
   371
runsutandtest_cleanup_timer:
icculus@7924
   372
    if(action_timer && !SDL_RemoveTimer(action_timer))
icculus@7924
   373
    {
icculus@7924
   374
        SDLTest_Log("SDL_RemoveTimer() failed");
icculus@7924
   375
        return_code = TEST_ERROR;
icculus@7924
   376
    }
icculus@7924
   377
icculus@7924
   378
    if(kill_timer && !SDL_RemoveTimer(kill_timer))
icculus@7924
   379
    {
icculus@7924
   380
        SDLTest_Log("SDL_RemoveTimer() failed");
icculus@7924
   381
        return_code = TEST_ERROR;
icculus@7924
   382
    }
icculus@7924
   383
/* runsutandtest_cleanup_process: */
icculus@7924
   384
    if(SDL_IsProcessRunning(&pinfo) && !SDL_KillProcess(&pinfo, &sut_exitstatus))
icculus@7924
   385
    {
icculus@7924
   386
        SDLTest_Log("SDL_KillProcess() failed");
icculus@7924
   387
        return_code = TEST_ERROR;
icculus@7924
   388
    }
icculus@7924
   389
runsutandtest_cleanup_generic:
icculus@7924
   390
    return return_code;
icculus@7924
   391
}
icculus@7924
   392
icculus@7924
   393
/** Entry point for testharness */
icculus@7924
   394
int
icculus@7924
   395
main(int argc, char* argv[])
icculus@7924
   396
{
icculus@7924
   397
    int i, passed, return_code, failed;
icculus@7924
   398
icculus@7924
   399
    /* freeing resources, linux style! */
icculus@7924
   400
    return_code = 0;
icculus@7924
   401
icculus@7924
   402
    if(argc < 2)
icculus@7924
   403
    {
icculus@7924
   404
        SDLTest_Log(usage(), argv[0]);
icculus@7924
   405
        goto cleanup_generic;
icculus@7924
   406
    }
icculus@7924
   407
icculus@7924
   408
#if defined(__LINUX__) || defined(__CYGWIN__)
icculus@7924
   409
    signal(SIGINT, CtrlCHandlerCallback);
icculus@7924
   410
#endif
icculus@7924
   411
icculus@7924
   412
    /* parse arguments */
icculus@7924
   413
    if(!SDLVisualTest_ParseHarnessArgs(argv + 1, &state))
icculus@7924
   414
    {
icculus@7924
   415
        SDLTest_Log(usage(), argv[0]);
icculus@7924
   416
        return_code = 1;
icculus@7924
   417
        goto cleanup_generic;
icculus@7924
   418
    }
icculus@7924
   419
    SDLTest_Log("Parsed harness arguments successfully.");
icculus@7924
   420
icculus@7924
   421
    /* initialize SDL */
icculus@7924
   422
    if(SDL_Init(SDL_INIT_TIMER) == -1)
icculus@7924
   423
    {
icculus@7924
   424
        SDLTest_LogError("SDL_Init() failed.");
icculus@7924
   425
        SDLVisualTest_FreeHarnessState(&state);
icculus@7924
   426
        return_code = 1;
icculus@7924
   427
        goto cleanup_harness_state;
icculus@7924
   428
    }
icculus@7924
   429
icculus@7924
   430
    /* create an output directory if none exists */
icculus@7924
   431
#if defined(__LINUX__) || defined(__CYGWIN__)
icculus@7924
   432
    mkdir(state.output_dir, 0777);
icculus@7924
   433
#elif defined(__WIN32__)
icculus@7924
   434
    _mkdir(state.output_dir);
icculus@7924
   435
#else
icculus@7924
   436
#error "Unsupported platform"
icculus@7924
   437
#endif
icculus@7924
   438
icculus@7924
   439
    /* test with sutargs */
icculus@7924
   440
    if(SDL_strlen(state.sutargs))
icculus@7924
   441
    {
icculus@7924
   442
        SDLTest_Log("Running: %s %s", state.sutapp, state.sutargs);
icculus@7924
   443
        if(!state.no_launch)
icculus@7924
   444
        {
icculus@7924
   445
            switch(RunSUTAndTest(state.sutargs, 0))
icculus@7924
   446
            {
icculus@7924
   447
                case TEST_PASSED:
icculus@7924
   448
                    SDLTest_Log("Status: PASSED");
icculus@7924
   449
                break;
icculus@7924
   450
icculus@7924
   451
                case TEST_FAILED:
icculus@7924
   452
                    SDLTest_Log("Status: FAILED");
icculus@7924
   453
                break;
icculus@7924
   454
icculus@7924
   455
                case TEST_ERROR:
icculus@7924
   456
                    SDLTest_LogError("Some error occurred while testing.");
icculus@7924
   457
                    return_code = 1;
icculus@7924
   458
                    goto cleanup_sdl;
icculus@7924
   459
                break;
icculus@7924
   460
            }
icculus@7924
   461
        }
icculus@7924
   462
    }
icculus@7924
   463
icculus@7924
   464
    if(state.sut_config.num_options > 0)
icculus@7924
   465
    {
icculus@7924
   466
        char* variator_name = state.variator_type == SDL_VARIATOR_RANDOM ?
icculus@7924
   467
                              "RANDOM" : "EXHAUSTIVE";
icculus@7924
   468
        if(state.num_variations > 0)
icculus@7924
   469
            SDLTest_Log("Testing SUT with variator: %s for %d variations",
icculus@7924
   470
                        variator_name, state.num_variations);
icculus@7924
   471
        else
icculus@7924
   472
            SDLTest_Log("Testing SUT with variator: %s and ALL variations",
icculus@7924
   473
                        variator_name);
icculus@7924
   474
        /* initialize the variator */
icculus@7924
   475
        if(!SDLVisualTest_InitVariator(&variator, &state.sut_config,
icculus@7924
   476
                                       state.variator_type, 0))
icculus@7924
   477
        {
icculus@7924
   478
            SDLTest_LogError("Could not initialize variator");
icculus@7924
   479
            return_code = 1;
icculus@7924
   480
            goto cleanup_sdl;
icculus@7924
   481
        }
icculus@7924
   482
icculus@7924
   483
        /* iterate through all the variations */
icculus@7924
   484
        passed = 0;
icculus@7924
   485
        failed = 0;
icculus@7924
   486
        for(i = 0; state.num_variations > 0 ? (i < state.num_variations) : 1; i++)
icculus@7924
   487
        {
icculus@7924
   488
            char* args = SDLVisualTest_GetNextVariation(&variator);
icculus@7924
   489
            if(!args)
icculus@7924
   490
                break;
icculus@7924
   491
            SDLTest_Log("\nVariation number: %d\nArguments: %s", i + 1, args);
icculus@7924
   492
icculus@7924
   493
            if(!state.no_launch)
icculus@7924
   494
            {
icculus@7924
   495
                switch(RunSUTAndTest(args, i + 1))
icculus@7924
   496
                {
icculus@7924
   497
                    case TEST_PASSED:
icculus@7924
   498
                        SDLTest_Log("Status: PASSED");
icculus@7924
   499
                        passed++;
icculus@7924
   500
                    break;
icculus@7924
   501
icculus@7924
   502
                    case TEST_FAILED:
icculus@7924
   503
                        SDLTest_Log("Status: FAILED");
icculus@7924
   504
                        failed++;
icculus@7924
   505
                    break;
icculus@7924
   506
icculus@7924
   507
                    case TEST_ERROR:
icculus@7924
   508
                        SDLTest_LogError("Some error occurred while testing.");
icculus@7924
   509
                        goto cleanup_variator;
icculus@7924
   510
                    break;
icculus@7924
   511
                }
icculus@7924
   512
            }
icculus@7924
   513
        }
icculus@7924
   514
        if(!state.no_launch)
icculus@7924
   515
        {
icculus@7924
   516
            /* report stats */
icculus@7924
   517
            SDLTest_Log("Testing complete.");
icculus@7924
   518
            SDLTest_Log("%d/%d tests passed.", passed, passed + failed);
icculus@7924
   519
        }
icculus@7924
   520
        goto cleanup_variator;
icculus@7924
   521
    }
icculus@7924
   522
 
icculus@7924
   523
    goto cleanup_sdl;
icculus@7924
   524
icculus@7924
   525
cleanup_variator:
icculus@7924
   526
    SDLVisualTest_FreeVariator(&variator);
icculus@7924
   527
cleanup_sdl:
icculus@7924
   528
    SDL_Quit();
icculus@7924
   529
cleanup_harness_state:
icculus@7924
   530
    SDLVisualTest_FreeHarnessState(&state);
icculus@7924
   531
cleanup_generic:
icculus@7924
   532
    return return_code;
icculus@7924
   533
}