visualtest/src/testharness.c
author Sam Lantinga <slouken@libsdl.org>
Thu, 07 Jun 2018 17:07:05 -0700
changeset 12013 270e17c85d11
parent 7924 fcb86d323770
permissions -rw-r--r--
Don't crash on exit from SDLActivity if we don't have a singleton for some reason. (Thanks Rachel!)
     1 /* See COPYING.txt for the full license governing this code. */
     2 /**
     3  *  \file testharness.c 
     4  *
     5  *  Source file for the test harness.
     6  */
     7 
     8 #include <stdlib.h>
     9 #include <SDL_test.h>
    10 #include <SDL.h>
    11 #include <SDL_assert.h>
    12 #include "SDL_visualtest_harness_argparser.h"
    13 #include "SDL_visualtest_process.h"
    14 #include "SDL_visualtest_variators.h"
    15 #include "SDL_visualtest_screenshot.h"
    16 #include "SDL_visualtest_mischelper.h"
    17 
    18 #if defined(__WIN32__) && !defined(__CYGWIN__)
    19 #include <direct.h>
    20 #elif defined(__WIN32__) && defined(__CYGWIN__)
    21 #include <signal.h>
    22 #elif defined(__LINUX__)
    23 #include <sys/stat.h>
    24 #include <sys/types.h>
    25 #include <signal.h>
    26 #else
    27 #error "Unsupported platform"
    28 #endif
    29 
    30 /** Code for the user event triggered when a new action is to be executed */
    31 #define ACTION_TIMER_EVENT 0
    32 /** Code for the user event triggered when the maximum timeout is reached */
    33 #define KILL_TIMER_EVENT 1
    34 /** FPS value used for delays in the action loop */
    35 #define ACTION_LOOP_FPS 10
    36 
    37 /** Value returned by RunSUTAndTest() when the test has passed */
    38 #define TEST_PASSED 1
    39 /** Value returned by RunSUTAndTest() when the test has failed */
    40 #define TEST_FAILED 0
    41 /** Value returned by RunSUTAndTest() on a fatal error */
    42 #define TEST_ERROR -1
    43 
    44 static SDL_ProcessInfo pinfo;
    45 static SDL_ProcessExitStatus sut_exitstatus;
    46 static SDLVisualTest_HarnessState state;
    47 static SDLVisualTest_Variator variator;
    48 static SDLVisualTest_ActionNode* current; /* the current action being performed */
    49 static SDL_TimerID action_timer, kill_timer;
    50 
    51 /* returns a char* to be passed as the format argument of a printf-style function. */
    52 static char*
    53 usage()
    54 {
    55     return "Usage: \n%s --sutapp xyz"
    56            " [--sutargs abc | --parameter-config xyz.parameters"
    57            " [--variator exhaustive|random]" 
    58            " [--num-variations N] [--no-launch]] [--timeout hh:mm:ss]"
    59            " [--action-config xyz.actions]"
    60            " [--output-dir /path/to/output]"
    61            " [--verify-dir /path/to/verify]"
    62            " or --config app.config";
    63 }
    64 
    65 /* register Ctrl+C handlers */
    66 #if defined(__LINUX__) || defined(__CYGWIN__)
    67 static void
    68 CtrlCHandlerCallback(int signum)
    69 {
    70     SDL_Event event;
    71     SDLTest_Log("Ctrl+C received");
    72     event.type = SDL_QUIT;
    73     SDL_PushEvent(&event);
    74 }
    75 #endif
    76 
    77 static Uint32
    78 ActionTimerCallback(Uint32 interval, void* param)
    79 {
    80     SDL_Event event;
    81     SDL_UserEvent userevent;
    82     Uint32 next_action_time;
    83 
    84     /* push an event to handle the action */
    85     userevent.type = SDL_USEREVENT;
    86     userevent.code = ACTION_TIMER_EVENT;
    87     userevent.data1 = &current->action;
    88     userevent.data2 = NULL;
    89 
    90     event.type = SDL_USEREVENT;
    91     event.user = userevent;
    92     SDL_PushEvent(&event);
    93 
    94     /* calculate the new interval and return it */
    95     if(current->next)
    96         next_action_time = current->next->action.time - current->action.time;
    97     else
    98     {
    99         next_action_time = 0;
   100         action_timer = 0;
   101     }
   102 
   103     current = current->next;
   104     return next_action_time;
   105 }
   106 
   107 static Uint32
   108 KillTimerCallback(Uint32 interval, void* param)
   109 {
   110     SDL_Event event;
   111     SDL_UserEvent userevent;
   112 
   113     userevent.type = SDL_USEREVENT;
   114     userevent.code = KILL_TIMER_EVENT;
   115     userevent.data1 = NULL;
   116     userevent.data2 = NULL;
   117 
   118     event.type = SDL_USEREVENT;
   119     event.user = userevent;
   120     SDL_PushEvent(&event);
   121 
   122     kill_timer = 0;
   123     return 0;
   124 }
   125 
   126 static int
   127 ProcessAction(SDLVisualTest_Action* action, int* sut_running, char* args)
   128 {
   129     if(!action || !sut_running)
   130         return TEST_ERROR;
   131 
   132     switch(action->type)
   133     {
   134         case SDL_ACTION_KILL:
   135             SDLTest_Log("Action: Kill SUT");
   136             if(SDL_IsProcessRunning(&pinfo) == 1 &&
   137                !SDL_KillProcess(&pinfo, &sut_exitstatus))
   138             {
   139                 SDLTest_LogError("SDL_KillProcess() failed");
   140                 return TEST_ERROR;
   141             }
   142             *sut_running = 0;
   143         break;
   144 
   145         case SDL_ACTION_QUIT:
   146             SDLTest_Log("Action: Quit SUT");
   147             if(SDL_IsProcessRunning(&pinfo) == 1 &&
   148                !SDL_QuitProcess(&pinfo, &sut_exitstatus))
   149             {
   150                 SDLTest_LogError("SDL_QuitProcess() failed");
   151                 return TEST_FAILED;
   152             }
   153             *sut_running = 0;
   154         break;
   155 
   156         case SDL_ACTION_LAUNCH:
   157         {
   158             char* path;
   159             char* args;
   160             SDL_ProcessInfo action_process;
   161             SDL_ProcessExitStatus ps;
   162 
   163             path = action->extra.process.path;
   164             args = action->extra.process.args;
   165             if(args)
   166             {
   167                 SDLTest_Log("Action: Launch process: %s with arguments: %s",
   168                             path, args);
   169             }
   170             else
   171                 SDLTest_Log("Action: Launch process: %s", path);
   172             if(!SDL_LaunchProcess(path, args, &action_process))
   173             {
   174                 SDLTest_LogError("SDL_LaunchProcess() failed");
   175                 return TEST_ERROR;
   176             }
   177 
   178             /* small delay so that the process can do its job */
   179             SDL_Delay(1000);
   180 
   181             if(SDL_IsProcessRunning(&action_process) > 0)
   182             {
   183                 SDLTest_LogError("Process %s took too long too complete."
   184                                     " Force killing...", action->extra);
   185                 if(!SDL_KillProcess(&action_process, &ps))
   186                 {
   187                     SDLTest_LogError("SDL_KillProcess() failed");
   188                     return TEST_ERROR;
   189                 }
   190             }
   191         }
   192         break;
   193 
   194         case SDL_ACTION_SCREENSHOT:
   195         {
   196             char path[MAX_PATH_LEN], hash[33];
   197 
   198             SDLTest_Log("Action: Take screenshot");
   199             /* can't take a screenshot if the SUT isn't running */
   200             if(SDL_IsProcessRunning(&pinfo) != 1)
   201             {
   202                 SDLTest_LogError("SUT has quit.");
   203                 *sut_running = 0;
   204                 return TEST_FAILED;
   205             }
   206 
   207             /* file name for the screenshot image */
   208             SDLVisualTest_HashString(args, hash);
   209             SDL_snprintf(path, MAX_PATH_LEN, "%s/%s", state.output_dir, hash);
   210             if(!SDLVisualTest_ScreenshotProcess(&pinfo, path))
   211             {
   212                 SDLTest_LogError("SDLVisualTest_ScreenshotProcess() failed");
   213                 return TEST_ERROR;
   214             }
   215         }
   216         break;
   217 
   218         case SDL_ACTION_VERIFY:
   219         {
   220             int ret;
   221 
   222             SDLTest_Log("Action: Verify screenshot");
   223             ret = SDLVisualTest_VerifyScreenshots(args, state.output_dir,
   224                                                   state.verify_dir);
   225 
   226             if(ret == -1)
   227             {
   228                 SDLTest_LogError("SDLVisualTest_VerifyScreenshots() failed");
   229                 return TEST_ERROR;
   230             }
   231             else if(ret == 0)
   232             {
   233                 SDLTest_Log("Verification failed: Images were not equal.");
   234                 return TEST_FAILED;
   235             }
   236             else if(ret == 1)
   237                 SDLTest_Log("Verification successful.");
   238             else
   239             {
   240                 SDLTest_Log("Verfication skipped.");
   241                 return TEST_FAILED;
   242             }
   243         }
   244         break;
   245 
   246         default:
   247             SDLTest_LogError("Invalid action type");
   248             return TEST_ERROR;
   249         break;
   250     }
   251 
   252     return TEST_PASSED;
   253 }
   254 
   255 static int
   256 RunSUTAndTest(char* sutargs, int variation_num)
   257 {
   258     int success, sut_running, return_code;
   259     char hash[33];
   260     SDL_Event event;
   261 
   262     return_code = TEST_PASSED;
   263 
   264     if(!sutargs)
   265     {
   266         SDLTest_LogError("sutargs argument cannot be NULL");
   267         return_code = TEST_ERROR;
   268         goto runsutandtest_cleanup_generic;
   269     }
   270 
   271     SDLVisualTest_HashString(sutargs, hash);
   272     SDLTest_Log("Hash: %s", hash);
   273 
   274     success = SDL_LaunchProcess(state.sutapp, sutargs, &pinfo);
   275     if(!success)
   276     {
   277         SDLTest_Log("Could not launch SUT.");
   278         return_code = TEST_ERROR;
   279         goto runsutandtest_cleanup_generic;
   280     }
   281     SDLTest_Log("SUT launch successful.");
   282     SDLTest_Log("Process will be killed in %d milliseconds", state.timeout);
   283     sut_running = 1;
   284 
   285     /* launch the timers */
   286     SDLTest_Log("Performing actions..");
   287     current = state.action_queue.front;
   288     action_timer = 0;
   289     kill_timer = 0;
   290     if(current)
   291     {
   292         action_timer = SDL_AddTimer(current->action.time, ActionTimerCallback, NULL);
   293         if(!action_timer)
   294         {
   295             SDLTest_LogError("SDL_AddTimer() failed");
   296             return_code = TEST_ERROR;
   297             goto runsutandtest_cleanup_timer;
   298         }
   299     }
   300     kill_timer = SDL_AddTimer(state.timeout, KillTimerCallback, NULL);
   301     if(!kill_timer)
   302     {
   303         SDLTest_LogError("SDL_AddTimer() failed");
   304         return_code = TEST_ERROR;
   305         goto runsutandtest_cleanup_timer;
   306     }
   307 
   308     /* the timer stops running if the actions queue is empty, and the
   309        SUT stops running if it crashes or if we encounter a KILL/QUIT action */
   310     while(sut_running)
   311     {
   312         /* process the actions by using an event queue */
   313         while(SDL_PollEvent(&event))
   314         {
   315             if(event.type == SDL_USEREVENT)
   316             {
   317                 if(event.user.code == ACTION_TIMER_EVENT)
   318                 {
   319                     SDLVisualTest_Action* action;
   320 
   321                     action = (SDLVisualTest_Action*)event.user.data1;
   322 
   323                     switch(ProcessAction(action, &sut_running, sutargs))
   324                     {
   325                         case TEST_PASSED:
   326                         break;
   327 
   328                         case TEST_FAILED:
   329                             return_code = TEST_FAILED;
   330                             goto runsutandtest_cleanup_timer;
   331                         break;
   332 
   333                         default:
   334                             SDLTest_LogError("ProcessAction() failed");
   335                             return_code = TEST_ERROR;
   336                             goto runsutandtest_cleanup_timer;
   337                     }
   338                 }
   339                 else if(event.user.code == KILL_TIMER_EVENT)
   340                 {
   341                     SDLTest_LogError("Maximum timeout reached. Force killing..");
   342                     return_code = TEST_FAILED;
   343                     goto runsutandtest_cleanup_timer;
   344                 }
   345             }
   346             else if(event.type == SDL_QUIT)
   347             {
   348                 SDLTest_LogError("Received QUIT event. Testharness is quitting..");
   349                 return_code = TEST_ERROR;
   350                 goto runsutandtest_cleanup_timer;
   351             }
   352         }
   353         SDL_Delay(1000/ACTION_LOOP_FPS);
   354     }
   355 
   356     SDLTest_Log("SUT exit code was: %d", sut_exitstatus.exit_status);
   357     if(sut_exitstatus.exit_status == 0)
   358     {
   359         return_code = TEST_PASSED;
   360         goto runsutandtest_cleanup_timer;
   361     }
   362     else
   363     {
   364         return_code = TEST_FAILED;
   365         goto runsutandtest_cleanup_timer;
   366     }
   367 
   368     return_code = TEST_ERROR;
   369     goto runsutandtest_cleanup_generic;
   370 
   371 runsutandtest_cleanup_timer:
   372     if(action_timer && !SDL_RemoveTimer(action_timer))
   373     {
   374         SDLTest_Log("SDL_RemoveTimer() failed");
   375         return_code = TEST_ERROR;
   376     }
   377 
   378     if(kill_timer && !SDL_RemoveTimer(kill_timer))
   379     {
   380         SDLTest_Log("SDL_RemoveTimer() failed");
   381         return_code = TEST_ERROR;
   382     }
   383 /* runsutandtest_cleanup_process: */
   384     if(SDL_IsProcessRunning(&pinfo) && !SDL_KillProcess(&pinfo, &sut_exitstatus))
   385     {
   386         SDLTest_Log("SDL_KillProcess() failed");
   387         return_code = TEST_ERROR;
   388     }
   389 runsutandtest_cleanup_generic:
   390     return return_code;
   391 }
   392 
   393 /** Entry point for testharness */
   394 int
   395 main(int argc, char* argv[])
   396 {
   397     int i, passed, return_code, failed;
   398 
   399     /* freeing resources, linux style! */
   400     return_code = 0;
   401 
   402     if(argc < 2)
   403     {
   404         SDLTest_Log(usage(), argv[0]);
   405         goto cleanup_generic;
   406     }
   407 
   408 #if defined(__LINUX__) || defined(__CYGWIN__)
   409     signal(SIGINT, CtrlCHandlerCallback);
   410 #endif
   411 
   412     /* parse arguments */
   413     if(!SDLVisualTest_ParseHarnessArgs(argv + 1, &state))
   414     {
   415         SDLTest_Log(usage(), argv[0]);
   416         return_code = 1;
   417         goto cleanup_generic;
   418     }
   419     SDLTest_Log("Parsed harness arguments successfully.");
   420 
   421     /* initialize SDL */
   422     if(SDL_Init(SDL_INIT_TIMER) == -1)
   423     {
   424         SDLTest_LogError("SDL_Init() failed.");
   425         SDLVisualTest_FreeHarnessState(&state);
   426         return_code = 1;
   427         goto cleanup_harness_state;
   428     }
   429 
   430     /* create an output directory if none exists */
   431 #if defined(__LINUX__) || defined(__CYGWIN__)
   432     mkdir(state.output_dir, 0777);
   433 #elif defined(__WIN32__)
   434     _mkdir(state.output_dir);
   435 #else
   436 #error "Unsupported platform"
   437 #endif
   438 
   439     /* test with sutargs */
   440     if(SDL_strlen(state.sutargs))
   441     {
   442         SDLTest_Log("Running: %s %s", state.sutapp, state.sutargs);
   443         if(!state.no_launch)
   444         {
   445             switch(RunSUTAndTest(state.sutargs, 0))
   446             {
   447                 case TEST_PASSED:
   448                     SDLTest_Log("Status: PASSED");
   449                 break;
   450 
   451                 case TEST_FAILED:
   452                     SDLTest_Log("Status: FAILED");
   453                 break;
   454 
   455                 case TEST_ERROR:
   456                     SDLTest_LogError("Some error occurred while testing.");
   457                     return_code = 1;
   458                     goto cleanup_sdl;
   459                 break;
   460             }
   461         }
   462     }
   463 
   464     if(state.sut_config.num_options > 0)
   465     {
   466         char* variator_name = state.variator_type == SDL_VARIATOR_RANDOM ?
   467                               "RANDOM" : "EXHAUSTIVE";
   468         if(state.num_variations > 0)
   469             SDLTest_Log("Testing SUT with variator: %s for %d variations",
   470                         variator_name, state.num_variations);
   471         else
   472             SDLTest_Log("Testing SUT with variator: %s and ALL variations",
   473                         variator_name);
   474         /* initialize the variator */
   475         if(!SDLVisualTest_InitVariator(&variator, &state.sut_config,
   476                                        state.variator_type, 0))
   477         {
   478             SDLTest_LogError("Could not initialize variator");
   479             return_code = 1;
   480             goto cleanup_sdl;
   481         }
   482 
   483         /* iterate through all the variations */
   484         passed = 0;
   485         failed = 0;
   486         for(i = 0; state.num_variations > 0 ? (i < state.num_variations) : 1; i++)
   487         {
   488             char* args = SDLVisualTest_GetNextVariation(&variator);
   489             if(!args)
   490                 break;
   491             SDLTest_Log("\nVariation number: %d\nArguments: %s", i + 1, args);
   492 
   493             if(!state.no_launch)
   494             {
   495                 switch(RunSUTAndTest(args, i + 1))
   496                 {
   497                     case TEST_PASSED:
   498                         SDLTest_Log("Status: PASSED");
   499                         passed++;
   500                     break;
   501 
   502                     case TEST_FAILED:
   503                         SDLTest_Log("Status: FAILED");
   504                         failed++;
   505                     break;
   506 
   507                     case TEST_ERROR:
   508                         SDLTest_LogError("Some error occurred while testing.");
   509                         goto cleanup_variator;
   510                     break;
   511                 }
   512             }
   513         }
   514         if(!state.no_launch)
   515         {
   516             /* report stats */
   517             SDLTest_Log("Testing complete.");
   518             SDLTest_Log("%d/%d tests passed.", passed, passed + failed);
   519         }
   520         goto cleanup_variator;
   521     }
   522  
   523     goto cleanup_sdl;
   524 
   525 cleanup_variator:
   526     SDLVisualTest_FreeVariator(&variator);
   527 cleanup_sdl:
   528     SDL_Quit();
   529 cleanup_harness_state:
   530     SDLVisualTest_FreeHarnessState(&state);
   531 cleanup_generic:
   532     return return_code;
   533 }