/* See COPYING.txt for the full license governing this code. */

 /**

  *  \file testharness.c 

  *

  *  Source file for the test harness.

  */

 #include <stdlib.h>

 #include <SDL_test.h>

 #include <SDL.h>

 #include <SDL_assert.h>

 #include "SDL_visualtest_harness_argparser.h"

 #include "SDL_visualtest_process.h"

 #include "SDL_visualtest_variators.h"

 #include "SDL_visualtest_screenshot.h"

 #include "SDL_visualtest_mischelper.h"

 #if defined(__WIN32__) && !defined(__CYGWIN__)

 #include <direct.h>

 #elif defined(__WIN32__) && defined(__CYGWIN__)

 #include <signal.h>

 #elif defined(__LINUX__)

 #include <sys/stat.h>

 #include <sys/types.h>

 #include <signal.h>

 #else

 #error "Unsupported platform"

 #endif

 /** Code for the user event triggered when a new action is to be executed */

 #define ACTION_TIMER_EVENT 0

 /** Code for the user event triggered when the maximum timeout is reached */

 #define KILL_TIMER_EVENT 1

 /** FPS value used for delays in the action loop */

 #define ACTION_LOOP_FPS 10

 /** Value returned by RunSUTAndTest() when the test has passed */

 #define TEST_PASSED 1

 /** Value returned by RunSUTAndTest() when the test has failed */

 #define TEST_FAILED 0

 /** Value returned by RunSUTAndTest() on a fatal error */

 #define TEST_ERROR -1

 static SDL_ProcessInfo pinfo;

 static SDL_ProcessExitStatus sut_exitstatus;

 static SDLVisualTest_HarnessState state;

 static SDLVisualTest_Variator variator;

 static SDLVisualTest_ActionNode* current; /* the current action being performed */

 static SDL_TimerID action_timer, kill_timer;

 /* returns a char* to be passed as the format argument of a printf-style function. */

 static char*

 usage()

 {

     return "Usage: \n%s --sutapp xyz"

            " [--sutargs abc | --parameter-config xyz.parameters"

            " [--variator exhaustive|random]" 

            " [--num-variations N] [--no-launch]] [--timeout hh:mm:ss]"

            " [--action-config xyz.actions]"

            " [--output-dir /path/to/output]"

            " [--verify-dir /path/to/verify]"

            " or --config app.config";

 }

 /* register Ctrl+C handlers */

 #if defined(__LINUX__) || defined(__CYGWIN__)

 static void

 CtrlCHandlerCallback(int signum)

 {

     SDL_Event event;

     SDLTest_Log("Ctrl+C received");

     event.type = SDL_QUIT;

     SDL_PushEvent(&event);

 }

 #endif

 static Uint32

 ActionTimerCallback(Uint32 interval, void* param)

 {

     SDL_Event event;

     SDL_UserEvent userevent;

     Uint32 next_action_time;

     /* push an event to handle the action */

     userevent.type = SDL_USEREVENT;

     userevent.code = ACTION_TIMER_EVENT;

     userevent.data1 = &current->action;

     userevent.data2 = NULL;

     event.type = SDL_USEREVENT;

     event.user = userevent;

     SDL_PushEvent(&event);

     /* calculate the new interval and return it */

     if(current->next)

         next_action_time = current->next->action.time - current->action.time;

     else

     {

         next_action_time = 0;

         action_timer = 0;

     }

     current = current->next;

     return next_action_time;

 }

 static Uint32

 KillTimerCallback(Uint32 interval, void* param)

 {

     SDL_Event event;

     SDL_UserEvent userevent;

     userevent.type = SDL_USEREVENT;

     userevent.code = KILL_TIMER_EVENT;

     userevent.data1 = NULL;

     userevent.data2 = NULL;

     event.type = SDL_USEREVENT;

     event.user = userevent;

     SDL_PushEvent(&event);

     kill_timer = 0;

     return 0;

 }

 static int

 ProcessAction(SDLVisualTest_Action* action, int* sut_running, char* args)

 {

     if(!action || !sut_running)

         return TEST_ERROR;

     switch(action->type)

     {

         case SDL_ACTION_KILL:

             SDLTest_Log("Action: Kill SUT");

             if(SDL_IsProcessRunning(&pinfo) == 1 &&

                !SDL_KillProcess(&pinfo, &sut_exitstatus))

             {

                 SDLTest_LogError("SDL_KillProcess() failed");

                 return TEST_ERROR;

             }

             *sut_running = 0;

         break;

         case SDL_ACTION_QUIT:

             SDLTest_Log("Action: Quit SUT");

             if(SDL_IsProcessRunning(&pinfo) == 1 &&

                !SDL_QuitProcess(&pinfo, &sut_exitstatus))

             {

                 SDLTest_LogError("SDL_QuitProcess() failed");

                 return TEST_FAILED;

             }

             *sut_running = 0;

         break;

         case SDL_ACTION_LAUNCH:

         {

             char* path;

             char* args;

             SDL_ProcessInfo action_process;

             SDL_ProcessExitStatus ps;

             path = action->extra.process.path;

             args = action->extra.process.args;

             if(args)

             {

                 SDLTest_Log("Action: Launch process: %s with arguments: %s",

                             path, args);

             }

             else

                 SDLTest_Log("Action: Launch process: %s", path);

             if(!SDL_LaunchProcess(path, args, &action_process))

             {

                 SDLTest_LogError("SDL_LaunchProcess() failed");

                 return TEST_ERROR;

             }

             /* small delay so that the process can do its job */

             SDL_Delay(1000);

             if(SDL_IsProcessRunning(&action_process) > 0)

             {

                 SDLTest_LogError("Process %s took too long too complete."

                                     " Force killing...", action->extra);

                 if(!SDL_KillProcess(&action_process, &ps))

                 {

                     SDLTest_LogError("SDL_KillProcess() failed");

                     return TEST_ERROR;

                 }

             }

         }

         break;

         case SDL_ACTION_SCREENSHOT:

         {

             char path[MAX_PATH_LEN], hash[33];

             SDLTest_Log("Action: Take screenshot");

             /* can't take a screenshot if the SUT isn't running */

             if(SDL_IsProcessRunning(&pinfo) != 1)

             {

                 SDLTest_LogError("SUT has quit.");

                 *sut_running = 0;

                 return TEST_FAILED;

             }

             /* file name for the screenshot image */

             SDLVisualTest_HashString(args, hash);

             SDL_snprintf(path, MAX_PATH_LEN, "%s/%s", state.output_dir, hash);

             if(!SDLVisualTest_ScreenshotProcess(&pinfo, path))

             {

                 SDLTest_LogError("SDLVisualTest_ScreenshotProcess() failed");

                 return TEST_ERROR;

             }

         }

         break;

         case SDL_ACTION_VERIFY:

         {

             int ret;

             SDLTest_Log("Action: Verify screenshot");

             ret = SDLVisualTest_VerifyScreenshots(args, state.output_dir,

                                                   state.verify_dir);

             if(ret == -1)

             {

                 SDLTest_LogError("SDLVisualTest_VerifyScreenshots() failed");

                 return TEST_ERROR;

             }

             else if(ret == 0)

             {

                 SDLTest_Log("Verification failed: Images were not equal.");

                 return TEST_FAILED;

             }

             else if(ret == 1)

                 SDLTest_Log("Verification successful.");

             else

             {

                 SDLTest_Log("Verfication skipped.");

                 return TEST_FAILED;

             }

         }

         break;

         default:

             SDLTest_LogError("Invalid action type");

             return TEST_ERROR;

         break;

     }

     return TEST_PASSED;

 }

 static int

 RunSUTAndTest(char* sutargs, int variation_num)

 {

     int success, sut_running, return_code;

     char hash[33];

     SDL_Event event;

     return_code = TEST_PASSED;

     if(!sutargs)

     {

         SDLTest_LogError("sutargs argument cannot be NULL");

         return_code = TEST_ERROR;

         goto runsutandtest_cleanup_generic;

     }

     SDLVisualTest_HashString(sutargs, hash);

     SDLTest_Log("Hash: %s", hash);

     success = SDL_LaunchProcess(state.sutapp, sutargs, &pinfo);

     if(!success)

     {

         SDLTest_Log("Could not launch SUT.");

         return_code = TEST_ERROR;

         goto runsutandtest_cleanup_generic;

     }

     SDLTest_Log("SUT launch successful.");

     SDLTest_Log("Process will be killed in %d milliseconds", state.timeout);

     sut_running = 1;

     /* launch the timers */

     SDLTest_Log("Performing actions..");

     current = state.action_queue.front;

     action_timer = 0;

     kill_timer = 0;

     if(current)

     {

         action_timer = SDL_AddTimer(current->action.time, ActionTimerCallback, NULL);

         if(!action_timer)

         {

             SDLTest_LogError("SDL_AddTimer() failed");

             return_code = TEST_ERROR;

             goto runsutandtest_cleanup_timer;

         }

     }

     kill_timer = SDL_AddTimer(state.timeout, KillTimerCallback, NULL);

     if(!kill_timer)

     {

         SDLTest_LogError("SDL_AddTimer() failed");

         return_code = TEST_ERROR;

         goto runsutandtest_cleanup_timer;

     }

     /* the timer stops running if the actions queue is empty, and the

        SUT stops running if it crashes or if we encounter a KILL/QUIT action */

     while(sut_running)

     {

         /* process the actions by using an event queue */

         while(SDL_PollEvent(&event))

         {

             if(event.type == SDL_USEREVENT)

             {

                 if(event.user.code == ACTION_TIMER_EVENT)

                 {

                     SDLVisualTest_Action* action;

                     action = (SDLVisualTest_Action*)event.user.data1;

                     switch(ProcessAction(action, &sut_running, sutargs))

                     {

                         case TEST_PASSED:

                         break;

                         case TEST_FAILED:

                             return_code = TEST_FAILED;

                             goto runsutandtest_cleanup_timer;

                         break;

                         default:

                             SDLTest_LogError("ProcessAction() failed");

                             return_code = TEST_ERROR;

                             goto runsutandtest_cleanup_timer;

                     }

                 }

                 else if(event.user.code == KILL_TIMER_EVENT)

                 {

                     SDLTest_LogError("Maximum timeout reached. Force killing..");

                     return_code = TEST_FAILED;

                     goto runsutandtest_cleanup_timer;

                 }

             }

             else if(event.type == SDL_QUIT)

             {

                 SDLTest_LogError("Received QUIT event. Testharness is quitting..");

                 return_code = TEST_ERROR;

                 goto runsutandtest_cleanup_timer;

             }

         }

         SDL_Delay(1000/ACTION_LOOP_FPS);

     }

     SDLTest_Log("SUT exit code was: %d", sut_exitstatus.exit_status);

     if(sut_exitstatus.exit_status == 0)

     {

         return_code = TEST_PASSED;

         goto runsutandtest_cleanup_timer;

     }

     else

     {

         return_code = TEST_FAILED;

         goto runsutandtest_cleanup_timer;

     }

     return_code = TEST_ERROR;

     goto runsutandtest_cleanup_generic;

 runsutandtest_cleanup_timer:

     if(action_timer && !SDL_RemoveTimer(action_timer))

     {

         SDLTest_Log("SDL_RemoveTimer() failed");

         return_code = TEST_ERROR;

     }

     if(kill_timer && !SDL_RemoveTimer(kill_timer))

     {

         SDLTest_Log("SDL_RemoveTimer() failed");

         return_code = TEST_ERROR;

     }

 /* runsutandtest_cleanup_process: */

     if(SDL_IsProcessRunning(&pinfo) && !SDL_KillProcess(&pinfo, &sut_exitstatus))

     {

         SDLTest_Log("SDL_KillProcess() failed");

         return_code = TEST_ERROR;

     }

 runsutandtest_cleanup_generic:

     return return_code;

 }

 /** Entry point for testharness */

 int

 main(int argc, char* argv[])

 {

     int i, passed, return_code, failed;

     /* freeing resources, linux style! */

     return_code = 0;

     if(argc < 2)

     {

         SDLTest_Log(usage(), argv[0]);

         goto cleanup_generic;

     }

 #if defined(__LINUX__) || defined(__CYGWIN__)

     signal(SIGINT, CtrlCHandlerCallback);

 #endif

     /* parse arguments */

     if(!SDLVisualTest_ParseHarnessArgs(argv + 1, &state))

     {

         SDLTest_Log(usage(), argv[0]);

         return_code = 1;

         goto cleanup_generic;

     }

     SDLTest_Log("Parsed harness arguments successfully.");

     /* initialize SDL */

     if(SDL_Init(SDL_INIT_TIMER) == -1)

     {

         SDLTest_LogError("SDL_Init() failed.");

         SDLVisualTest_FreeHarnessState(&state);

         return_code = 1;

         goto cleanup_harness_state;

     }

     /* create an output directory if none exists */

 #if defined(__LINUX__) || defined(__CYGWIN__)

     mkdir(state.output_dir, 0777);

 #elif defined(__WIN32__)

     _mkdir(state.output_dir);

 #else

 #error "Unsupported platform"

 #endif

     /* test with sutargs */

     if(SDL_strlen(state.sutargs))

     {

         SDLTest_Log("Running: %s %s", state.sutapp, state.sutargs);

         if(!state.no_launch)

         {

             switch(RunSUTAndTest(state.sutargs, 0))

             {

                 case TEST_PASSED:

                     SDLTest_Log("Status: PASSED");

                 break;

                 case TEST_FAILED:

                     SDLTest_Log("Status: FAILED");

                 break;

                 case TEST_ERROR:

                     SDLTest_LogError("Some error occurred while testing.");

                     return_code = 1;

                     goto cleanup_sdl;

                 break;

             }

         }

     }

     if(state.sut_config.num_options > 0)

     {

         char* variator_name = state.variator_type == SDL_VARIATOR_RANDOM ?

                               "RANDOM" : "EXHAUSTIVE";

         if(state.num_variations > 0)

             SDLTest_Log("Testing SUT with variator: %s for %d variations",

                         variator_name, state.num_variations);

         else

             SDLTest_Log("Testing SUT with variator: %s and ALL variations",

                         variator_name);

         /* initialize the variator */

         if(!SDLVisualTest_InitVariator(&variator, &state.sut_config,

                                        state.variator_type, 0))

         {

             SDLTest_LogError("Could not initialize variator");

             return_code = 1;

             goto cleanup_sdl;

         }

         /* iterate through all the variations */

         passed = 0;

         failed = 0;

         for(i = 0; state.num_variations > 0 ? (i < state.num_variations) : 1; i++)

         {

             char* args = SDLVisualTest_GetNextVariation(&variator);

             if(!args)

                 break;

             SDLTest_Log("\nVariation number: %d\nArguments: %s", i + 1, args);

             if(!state.no_launch)

             {

                 switch(RunSUTAndTest(args, i + 1))

                 {

                     case TEST_PASSED:

                         SDLTest_Log("Status: PASSED");

                         passed++;

                     break;

                     case TEST_FAILED:

                         SDLTest_Log("Status: FAILED");

                         failed++;

                     break;

                     case TEST_ERROR:

                         SDLTest_LogError("Some error occurred while testing.");

                         goto cleanup_variator;

                     break;

                 }

             }

         }

         if(!state.no_launch)

         {

             /* report stats */

             SDLTest_Log("Testing complete.");

             SDLTest_Log("%d/%d tests passed.", passed, passed + failed);

         }

         goto cleanup_variator;

     }

     goto cleanup_sdl;

 cleanup_variator:

     SDLVisualTest_FreeVariator(&variator);

 cleanup_sdl:

     SDL_Quit();

 cleanup_harness_state:

     SDLVisualTest_FreeHarnessState(&state);

 cleanup_generic:

     return return_code;

 }