visualtest/src/testharness.c
author Sam Lantinga
Tue, 10 Jan 2017 08:54:33 -0800
changeset 10806 36f40b8cc979
parent 7924 fcb86d323770
permissions -rw-r--r--
Fixed bugs 2570, 3145, improved OpenGL ES context support on Windows and X11

Mark Callow

The attached patch does the following for the X11 and Windows platforms, the only ones where SDL attempts to use context_create_es_profile:

- Adds SDL_HINT_OPENGL_ES_DRIVER by which the application can
say to use the OpenGL ES driver & EGL rather than the Open GL
driver. (For bug #2570)
- Adds code to {WIN,X11}_GL_InitExtensions to determine the maximum
OpenGL ES version supported by the OpenGL driver (for bug #3145)
- Modifies the test that determines whether to use the OpenGL
driver or the real OpenGL ES driver to take into account the
hint, the requested and supported ES version and whether ES 1.X
is being requested. (For bug #2570 & bug #3145)
- Enables the testgles2 test for __WINDOWS__ and __LINUX__ and adds
the test to the VisualC projects.

With the fix in place I have run testdraw2, testgl and testgles2 without any issues and have run my own apps that use OpenGL, OpenGL ES 3 and OpenGL ES 1.1.
     1 /* See COPYING.txt for the full license governing this code. */
     2 /**
     3  *  \file testharness.c 
     4  *
     5  *  Source file for the test harness.
     6  */
     7 
     8 #include <stdlib.h>
     9 #include <SDL_test.h>
    10 #include <SDL.h>
    11 #include <SDL_assert.h>
    12 #include "SDL_visualtest_harness_argparser.h"
    13 #include "SDL_visualtest_process.h"
    14 #include "SDL_visualtest_variators.h"
    15 #include "SDL_visualtest_screenshot.h"
    16 #include "SDL_visualtest_mischelper.h"
    17 
    18 #if defined(__WIN32__) && !defined(__CYGWIN__)
    19 #include <direct.h>
    20 #elif defined(__WIN32__) && defined(__CYGWIN__)
    21 #include <signal.h>
    22 #elif defined(__LINUX__)
    23 #include <sys/stat.h>
    24 #include <sys/types.h>
    25 #include <signal.h>
    26 #else
    27 #error "Unsupported platform"
    28 #endif
    29 
    30 /** Code for the user event triggered when a new action is to be executed */
    31 #define ACTION_TIMER_EVENT 0
    32 /** Code for the user event triggered when the maximum timeout is reached */
    33 #define KILL_TIMER_EVENT 1
    34 /** FPS value used for delays in the action loop */
    35 #define ACTION_LOOP_FPS 10
    36 
    37 /** Value returned by RunSUTAndTest() when the test has passed */
    38 #define TEST_PASSED 1
    39 /** Value returned by RunSUTAndTest() when the test has failed */
    40 #define TEST_FAILED 0
    41 /** Value returned by RunSUTAndTest() on a fatal error */
    42 #define TEST_ERROR -1
    43 
    44 static SDL_ProcessInfo pinfo;
    45 static SDL_ProcessExitStatus sut_exitstatus;
    46 static SDLVisualTest_HarnessState state;
    47 static SDLVisualTest_Variator variator;
    48 static SDLVisualTest_ActionNode* current; /* the current action being performed */
    49 static SDL_TimerID action_timer, kill_timer;
    50 
    51 /* returns a char* to be passed as the format argument of a printf-style function. */
    52 static char*
    53 usage()
    54 {
    55     return "Usage: \n%s --sutapp xyz"
    56            " [--sutargs abc | --parameter-config xyz.parameters"
    57            " [--variator exhaustive|random]" 
    58            " [--num-variations N] [--no-launch]] [--timeout hh:mm:ss]"
    59            " [--action-config xyz.actions]"
    60            " [--output-dir /path/to/output]"
    61            " [--verify-dir /path/to/verify]"
    62            " or --config app.config";
    63 }
    64 
    65 /* register Ctrl+C handlers */
    66 #if defined(__LINUX__) || defined(__CYGWIN__)
    67 static void
    68 CtrlCHandlerCallback(int signum)
    69 {
    70     SDL_Event event;
    71     SDLTest_Log("Ctrl+C received");
    72     event.type = SDL_QUIT;
    73     SDL_PushEvent(&event);
    74 }
    75 #endif
    76 
    77 static Uint32
    78 ActionTimerCallback(Uint32 interval, void* param)
    79 {
    80     SDL_Event event;
    81     SDL_UserEvent userevent;
    82     Uint32 next_action_time;
    83 
    84     /* push an event to handle the action */
    85     userevent.type = SDL_USEREVENT;
    86     userevent.code = ACTION_TIMER_EVENT;
    87     userevent.data1 = &current->action;
    88     userevent.data2 = NULL;
    89 
    90     event.type = SDL_USEREVENT;
    91     event.user = userevent;
    92     SDL_PushEvent(&event);
    93 
    94     /* calculate the new interval and return it */
    95     if(current->next)
    96         next_action_time = current->next->action.time - current->action.time;
    97     else
    98     {
    99         next_action_time = 0;
   100         action_timer = 0;
   101     }
   102 
   103     current = current->next;
   104     return next_action_time;
   105 }
   106 
   107 static Uint32
   108 KillTimerCallback(Uint32 interval, void* param)
   109 {
   110     SDL_Event event;
   111     SDL_UserEvent userevent;
   112 
   113     userevent.type = SDL_USEREVENT;
   114     userevent.code = KILL_TIMER_EVENT;
   115     userevent.data1 = NULL;
   116     userevent.data2 = NULL;
   117 
   118     event.type = SDL_USEREVENT;
   119     event.user = userevent;
   120     SDL_PushEvent(&event);
   121 
   122     kill_timer = 0;
   123     return 0;
   124 }
   125 
   126 static int
   127 ProcessAction(SDLVisualTest_Action* action, int* sut_running, char* args)
   128 {
   129     if(!action || !sut_running)
   130         return TEST_ERROR;
   131 
   132     switch(action->type)
   133     {
   134         case SDL_ACTION_KILL:
   135             SDLTest_Log("Action: Kill SUT");
   136             if(SDL_IsProcessRunning(&pinfo) == 1 &&
   137                !SDL_KillProcess(&pinfo, &sut_exitstatus))
   138             {
   139                 SDLTest_LogError("SDL_KillProcess() failed");
   140                 return TEST_ERROR;
   141             }
   142             *sut_running = 0;
   143         break;
   144 
   145         case SDL_ACTION_QUIT:
   146             SDLTest_Log("Action: Quit SUT");
   147             if(SDL_IsProcessRunning(&pinfo) == 1 &&
   148                !SDL_QuitProcess(&pinfo, &sut_exitstatus))
   149             {
   150                 SDLTest_LogError("SDL_QuitProcess() failed");
   151                 return TEST_FAILED;
   152             }
   153             *sut_running = 0;
   154         break;
   155 
   156         case SDL_ACTION_LAUNCH:
   157         {
   158             char* path;
   159             char* args;
   160             SDL_ProcessInfo action_process;
   161             SDL_ProcessExitStatus ps;
   162 
   163             path = action->extra.process.path;
   164             args = action->extra.process.args;
   165             if(args)
   166             {
   167                 SDLTest_Log("Action: Launch process: %s with arguments: %s",
   168                             path, args);
   169             }
   170             else
   171                 SDLTest_Log("Action: Launch process: %s", path);
   172             if(!SDL_LaunchProcess(path, args, &action_process))
   173             {
   174                 SDLTest_LogError("SDL_LaunchProcess() failed");
   175                 return TEST_ERROR;
   176             }
   177 
   178             /* small delay so that the process can do its job */
   179             SDL_Delay(1000);
   180 
   181             if(SDL_IsProcessRunning(&action_process) > 0)
   182             {
   183                 SDLTest_LogError("Process %s took too long too complete."
   184                                     " Force killing...", action->extra);
   185                 if(!SDL_KillProcess(&action_process, &ps))
   186                 {
   187                     SDLTest_LogError("SDL_KillProcess() failed");
   188                     return TEST_ERROR;
   189                 }
   190             }
   191         }
   192         break;
   193 
   194         case SDL_ACTION_SCREENSHOT:
   195         {
   196             char path[MAX_PATH_LEN], hash[33];
   197 
   198             SDLTest_Log("Action: Take screenshot");
   199             /* can't take a screenshot if the SUT isn't running */
   200             if(SDL_IsProcessRunning(&pinfo) != 1)
   201             {
   202                 SDLTest_LogError("SUT has quit.");
   203                 *sut_running = 0;
   204                 return TEST_FAILED;
   205             }
   206 
   207             /* file name for the screenshot image */
   208             SDLVisualTest_HashString(args, hash);
   209             SDL_snprintf(path, MAX_PATH_LEN, "%s/%s", state.output_dir, hash);
   210             if(!SDLVisualTest_ScreenshotProcess(&pinfo, path))
   211             {
   212                 SDLTest_LogError("SDLVisualTest_ScreenshotProcess() failed");
   213                 return TEST_ERROR;
   214             }
   215         }
   216         break;
   217 
   218         case SDL_ACTION_VERIFY:
   219         {
   220             int ret;
   221 
   222             SDLTest_Log("Action: Verify screenshot");
   223             ret = SDLVisualTest_VerifyScreenshots(args, state.output_dir,
   224                                                   state.verify_dir);
   225 
   226             if(ret == -1)
   227             {
   228                 SDLTest_LogError("SDLVisualTest_VerifyScreenshots() failed");
   229                 return TEST_ERROR;
   230             }
   231             else if(ret == 0)
   232             {
   233                 SDLTest_Log("Verification failed: Images were not equal.");
   234                 return TEST_FAILED;
   235             }
   236             else if(ret == 1)
   237                 SDLTest_Log("Verification successful.");
   238             else
   239             {
   240                 SDLTest_Log("Verfication skipped.");
   241                 return TEST_FAILED;
   242             }
   243         }
   244         break;
   245 
   246         default:
   247             SDLTest_LogError("Invalid action type");
   248             return TEST_ERROR;
   249         break;
   250     }
   251 
   252     return TEST_PASSED;
   253 }
   254 
   255 static int
   256 RunSUTAndTest(char* sutargs, int variation_num)
   257 {
   258     int success, sut_running, return_code;
   259     char hash[33];
   260     SDL_Event event;
   261 
   262     return_code = TEST_PASSED;
   263 
   264     if(!sutargs)
   265     {
   266         SDLTest_LogError("sutargs argument cannot be NULL");
   267         return_code = TEST_ERROR;
   268         goto runsutandtest_cleanup_generic;
   269     }
   270 
   271     SDLVisualTest_HashString(sutargs, hash);
   272     SDLTest_Log("Hash: %s", hash);
   273 
   274     success = SDL_LaunchProcess(state.sutapp, sutargs, &pinfo);
   275     if(!success)
   276     {
   277         SDLTest_Log("Could not launch SUT.");
   278         return_code = TEST_ERROR;
   279         goto runsutandtest_cleanup_generic;
   280     }
   281     SDLTest_Log("SUT launch successful.");
   282     SDLTest_Log("Process will be killed in %d milliseconds", state.timeout);
   283     sut_running = 1;
   284 
   285     /* launch the timers */
   286     SDLTest_Log("Performing actions..");
   287     current = state.action_queue.front;
   288     action_timer = 0;
   289     kill_timer = 0;
   290     if(current)
   291     {
   292         action_timer = SDL_AddTimer(current->action.time, ActionTimerCallback, NULL);
   293         if(!action_timer)
   294         {
   295             SDLTest_LogError("SDL_AddTimer() failed");
   296             return_code = TEST_ERROR;
   297             goto runsutandtest_cleanup_timer;
   298         }
   299     }
   300     kill_timer = SDL_AddTimer(state.timeout, KillTimerCallback, NULL);
   301     if(!kill_timer)
   302     {
   303         SDLTest_LogError("SDL_AddTimer() failed");
   304         return_code = TEST_ERROR;
   305         goto runsutandtest_cleanup_timer;
   306     }
   307 
   308     /* the timer stops running if the actions queue is empty, and the
   309        SUT stops running if it crashes or if we encounter a KILL/QUIT action */
   310     while(sut_running)
   311     {
   312         /* process the actions by using an event queue */
   313         while(SDL_PollEvent(&event))
   314         {
   315             if(event.type == SDL_USEREVENT)
   316             {
   317                 if(event.user.code == ACTION_TIMER_EVENT)
   318                 {
   319                     SDLVisualTest_Action* action;
   320 
   321                     action = (SDLVisualTest_Action*)event.user.data1;
   322 
   323                     switch(ProcessAction(action, &sut_running, sutargs))
   324                     {
   325                         case TEST_PASSED:
   326                         break;
   327 
   328                         case TEST_FAILED:
   329                             return_code = TEST_FAILED;
   330                             goto runsutandtest_cleanup_timer;
   331                         break;
   332 
   333                         default:
   334                             SDLTest_LogError("ProcessAction() failed");
   335                             return_code = TEST_ERROR;
   336                             goto runsutandtest_cleanup_timer;
   337                     }
   338                 }
   339                 else if(event.user.code == KILL_TIMER_EVENT)
   340                 {
   341                     SDLTest_LogError("Maximum timeout reached. Force killing..");
   342                     return_code = TEST_FAILED;
   343                     goto runsutandtest_cleanup_timer;
   344                 }
   345             }
   346             else if(event.type == SDL_QUIT)
   347             {
   348                 SDLTest_LogError("Received QUIT event. Testharness is quitting..");
   349                 return_code = TEST_ERROR;
   350                 goto runsutandtest_cleanup_timer;
   351             }
   352         }
   353         SDL_Delay(1000/ACTION_LOOP_FPS);
   354     }
   355 
   356     SDLTest_Log("SUT exit code was: %d", sut_exitstatus.exit_status);
   357     if(sut_exitstatus.exit_status == 0)
   358     {
   359         return_code = TEST_PASSED;
   360         goto runsutandtest_cleanup_timer;
   361     }
   362     else
   363     {
   364         return_code = TEST_FAILED;
   365         goto runsutandtest_cleanup_timer;
   366     }
   367 
   368     return_code = TEST_ERROR;
   369     goto runsutandtest_cleanup_generic;
   370 
   371 runsutandtest_cleanup_timer:
   372     if(action_timer && !SDL_RemoveTimer(action_timer))
   373     {
   374         SDLTest_Log("SDL_RemoveTimer() failed");
   375         return_code = TEST_ERROR;
   376     }
   377 
   378     if(kill_timer && !SDL_RemoveTimer(kill_timer))
   379     {
   380         SDLTest_Log("SDL_RemoveTimer() failed");
   381         return_code = TEST_ERROR;
   382     }
   383 /* runsutandtest_cleanup_process: */
   384     if(SDL_IsProcessRunning(&pinfo) && !SDL_KillProcess(&pinfo, &sut_exitstatus))
   385     {
   386         SDLTest_Log("SDL_KillProcess() failed");
   387         return_code = TEST_ERROR;
   388     }
   389 runsutandtest_cleanup_generic:
   390     return return_code;
   391 }
   392 
   393 /** Entry point for testharness */
   394 int
   395 main(int argc, char* argv[])
   396 {
   397     int i, passed, return_code, failed;
   398 
   399     /* freeing resources, linux style! */
   400     return_code = 0;
   401 
   402     if(argc < 2)
   403     {
   404         SDLTest_Log(usage(), argv[0]);
   405         goto cleanup_generic;
   406     }
   407 
   408 #if defined(__LINUX__) || defined(__CYGWIN__)
   409     signal(SIGINT, CtrlCHandlerCallback);
   410 #endif
   411 
   412     /* parse arguments */
   413     if(!SDLVisualTest_ParseHarnessArgs(argv + 1, &state))
   414     {
   415         SDLTest_Log(usage(), argv[0]);
   416         return_code = 1;
   417         goto cleanup_generic;
   418     }
   419     SDLTest_Log("Parsed harness arguments successfully.");
   420 
   421     /* initialize SDL */
   422     if(SDL_Init(SDL_INIT_TIMER) == -1)
   423     {
   424         SDLTest_LogError("SDL_Init() failed.");
   425         SDLVisualTest_FreeHarnessState(&state);
   426         return_code = 1;
   427         goto cleanup_harness_state;
   428     }
   429 
   430     /* create an output directory if none exists */
   431 #if defined(__LINUX__) || defined(__CYGWIN__)
   432     mkdir(state.output_dir, 0777);
   433 #elif defined(__WIN32__)
   434     _mkdir(state.output_dir);
   435 #else
   436 #error "Unsupported platform"
   437 #endif
   438 
   439     /* test with sutargs */
   440     if(SDL_strlen(state.sutargs))
   441     {
   442         SDLTest_Log("Running: %s %s", state.sutapp, state.sutargs);
   443         if(!state.no_launch)
   444         {
   445             switch(RunSUTAndTest(state.sutargs, 0))
   446             {
   447                 case TEST_PASSED:
   448                     SDLTest_Log("Status: PASSED");
   449                 break;
   450 
   451                 case TEST_FAILED:
   452                     SDLTest_Log("Status: FAILED");
   453                 break;
   454 
   455                 case TEST_ERROR:
   456                     SDLTest_LogError("Some error occurred while testing.");
   457                     return_code = 1;
   458                     goto cleanup_sdl;
   459                 break;
   460             }
   461         }
   462     }
   463 
   464     if(state.sut_config.num_options > 0)
   465     {
   466         char* variator_name = state.variator_type == SDL_VARIATOR_RANDOM ?
   467                               "RANDOM" : "EXHAUSTIVE";
   468         if(state.num_variations > 0)
   469             SDLTest_Log("Testing SUT with variator: %s for %d variations",
   470                         variator_name, state.num_variations);
   471         else
   472             SDLTest_Log("Testing SUT with variator: %s and ALL variations",
   473                         variator_name);
   474         /* initialize the variator */
   475         if(!SDLVisualTest_InitVariator(&variator, &state.sut_config,
   476                                        state.variator_type, 0))
   477         {
   478             SDLTest_LogError("Could not initialize variator");
   479             return_code = 1;
   480             goto cleanup_sdl;
   481         }
   482 
   483         /* iterate through all the variations */
   484         passed = 0;
   485         failed = 0;
   486         for(i = 0; state.num_variations > 0 ? (i < state.num_variations) : 1; i++)
   487         {
   488             char* args = SDLVisualTest_GetNextVariation(&variator);
   489             if(!args)
   490                 break;
   491             SDLTest_Log("\nVariation number: %d\nArguments: %s", i + 1, args);
   492 
   493             if(!state.no_launch)
   494             {
   495                 switch(RunSUTAndTest(args, i + 1))
   496                 {
   497                     case TEST_PASSED:
   498                         SDLTest_Log("Status: PASSED");
   499                         passed++;
   500                     break;
   501 
   502                     case TEST_FAILED:
   503                         SDLTest_Log("Status: FAILED");
   504                         failed++;
   505                     break;
   506 
   507                     case TEST_ERROR:
   508                         SDLTest_LogError("Some error occurred while testing.");
   509                         goto cleanup_variator;
   510                     break;
   511                 }
   512             }
   513         }
   514         if(!state.no_launch)
   515         {
   516             /* report stats */
   517             SDLTest_Log("Testing complete.");
   518             SDLTest_Log("%d/%d tests passed.", passed, passed + failed);
   519         }
   520         goto cleanup_variator;
   521     }
   522  
   523     goto cleanup_sdl;
   524 
   525 cleanup_variator:
   526     SDLVisualTest_FreeVariator(&variator);
   527 cleanup_sdl:
   528     SDL_Quit();
   529 cleanup_harness_state:
   530     SDLVisualTest_FreeHarnessState(&state);
   531 cleanup_generic:
   532     return return_code;
   533 }