diff --git a/include/SDL3/SDL_test_harness.h b/include/SDL3/SDL_test_harness.h index c8ad19bc2..65630d848 100644 --- a/include/SDL3/SDL_test_harness.h +++ b/include/SDL3/SDL_test_harness.h @@ -112,18 +112,37 @@ typedef struct SDLTest_TestSuiteReference { char *SDLTest_GenerateRunSeed(char *buffer, int length); /* - * Execute a test suite using the given run seed and execution key. + * Holds information about the execution of test suites. + * */ +typedef struct SDLTest_TestSuiteRunner SDLTest_TestSuiteRunner; + +/* + * Create a new test suite runner, that will execute the given test suites. + * It will register the harness cli arguments to the common SDL state. * - * \param testSuites Suites containing the test case. - * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one. - * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one. - * \param filter Filter specification. NULL disables. Case sensitive. - * \param testIterations Number of iterations to run each test case. - * \param randomOrder allow to run suites and tests in random order when there is no filter + * \param state Common SDL state on which to register CLI arguments. + * \param testSuites NULL-terminated test suites containing test cases. * * \returns the test run result: 0 when all tests passed, 1 if any tests failed. */ -int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations, SDL_bool randomOrder); +SDLTest_TestSuiteRunner * SDLTest_CreateTestSuiteRunner(SDLTest_CommonState *state, SDLTest_TestSuiteReference *testSuites[]); + +/* + * Destroy a test suite runner. + * It will unregister the harness cli arguments to the common SDL state. + * + * \param runner The runner that should be destroyed. + */ +void SDLTest_DestroyTestSuiteRunner(SDLTest_TestSuiteRunner *runner); + +/* + * Execute a test suite, using the configured run seed, execution key, filter, etc. + * + * \param runner The runner that should be executed. + * + * \returns the test run result: 0 when all tests passed, 1 if any tests failed. + */ +int SDLTest_ExecuteTestSuiteRunner(SDLTest_TestSuiteRunner *runner); /* Ends C function definitions when using C++ */ diff --git a/src/test/SDL_test_harness.c b/src/test/SDL_test_harness.c index 6f3161d28..6136e0299 100644 --- a/src/test/SDL_test_harness.c +++ b/src/test/SDL_test_harness.c @@ -47,8 +47,31 @@ /* Final result message format */ #define SDLTEST_FINAL_RESULT_FORMAT COLOR_YELLOW ">>> %s '%s':" COLOR_END " %s\n" +typedef struct SDLTest_TestSuiteRunner { + struct + { + SDLTest_TestSuiteReference **testSuites; + char *runSeed; + Uint64 execKey; + char *filter; + int testIterations; + SDL_bool randomOrder; + } user; + + SDLTest_ArgumentParser argparser; +} SDLTest_TestSuiteRunner; + /* ! Timeout for single test case execution */ -static Uint32 SDLTest_TestCaseTimeout = 3600; +static Uint32 SDLTest_TestCaseTimeout = 3600;; + +static const char *common_harness_usage[] = { + "[--iterations #]", + "[--execKey #]", + "[--seed string]", + "[--filter suite_name|test_name]", + "[--random-order]", + NULL +}; char *SDLTest_GenerateRunSeed(char *buffer, int length) { @@ -348,16 +371,11 @@ static float GetClock(void) * The filter string is matched to the suite name (full comparison) to select a single suite, * or if no suite matches, it is matched to the test names (full comparison) to select a single test. * - * \param testSuites Suites containing the test case. - * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one. - * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one. - * \param filter Filter specification. NULL disables. Case sensitive. - * \param testIterations Number of iterations to run each test case. - * \param randomOrder allow to run suites and tests in random order when there is no filter + * \param runner The runner to execute. * * \returns Test run result; 0 when all tests passed, 1 if any tests failed. */ -int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations, SDL_bool randomOrder) +int SDLTest_ExecuteTestSuiteRunner(SDLTest_TestSuiteRunner *runner) { int totalNumberOfTests = 0; int failedNumberOfTests = 0; @@ -398,19 +416,19 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user int *arraySuites = NULL; /* Sanitize test iterations */ - if (testIterations < 1) { - testIterations = 1; + if (runner->user.testIterations < 1) { + runner->user.testIterations = 1; } /* Generate run see if we don't have one already */ - if (!userRunSeed || userRunSeed[0] == '\0') { + if (!runner->user.runSeed || runner->user.runSeed[0] == '\0') { runSeed = SDLTest_GenerateRunSeed(generatedSeed, 16); if (!runSeed) { SDLTest_LogError("Generating a random seed failed"); return 2; } } else { - runSeed = userRunSeed; + runSeed = runner->user.runSeed; } /* Reset per-run counters */ @@ -426,8 +444,8 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user /* Count the total number of tests */ suiteCounter = 0; - while (testSuites[suiteCounter]) { - testSuite = testSuites[suiteCounter]; + while (runner->user.testSuites[suiteCounter]) { + testSuite = runner->user.testSuites[suiteCounter]; suiteCounter++; testCounter = 0; while (testSuite->testCases[testCounter]) { @@ -449,13 +467,13 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user } /* Initialize filtering */ - if (filter && filter[0] != '\0') { + if (runner->user.filter && runner->user.filter[0] != '\0') { /* Loop over all suites to check if we have a filter match */ suiteCounter = 0; - while (testSuites[suiteCounter] && suiteFilter == 0) { - testSuite = testSuites[suiteCounter]; + while (runner->user.testSuites[suiteCounter] && suiteFilter == 0) { + testSuite = runner->user.testSuites[suiteCounter]; suiteCounter++; - if (testSuite->name && SDL_strcasecmp(filter, testSuite->name) == 0) { + if (testSuite->name && SDL_strcasecmp(runner->user.filter, testSuite->name) == 0) { /* Matched a suite name */ suiteFilter = 1; suiteFilterName = testSuite->name; @@ -468,7 +486,7 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user while (testSuite->testCases[testCounter] && testFilter == 0) { testCase = testSuite->testCases[testCounter]; testCounter++; - if (testCase->name && SDL_strcasecmp(filter, testCase->name) == 0) { + if (testCase->name && SDL_strcasecmp(runner->user.filter, testCase->name) == 0) { /* Matched a test name */ suiteFilter = 1; suiteFilterName = testSuite->name; @@ -481,9 +499,9 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user } if (suiteFilter == 0 && testFilter == 0) { - SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter); - for (suiteCounter = 0; testSuites[suiteCounter]; ++suiteCounter) { - testSuite = testSuites[suiteCounter]; + SDLTest_LogError("Filter '%s' did not match any test suite/case.", runner->user.filter); + for (suiteCounter = 0; runner->user.testSuites[suiteCounter]; ++suiteCounter) { + testSuite = runner->user.testSuites[suiteCounter]; if (testSuite->name) { SDLTest_Log("Test suite: %s", testSuite->name); } @@ -499,11 +517,11 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user return 2; } - randomOrder = SDL_FALSE; + runner->user.randomOrder = SDL_FALSE; } /* Number of test suites */ - while (testSuites[nbSuites]) { + while (runner->user.testSuites[nbSuites]) { nbSuites++; } @@ -520,8 +538,8 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user /* Exclude last test "subsystemsTestSuite" which is said to interfer with other tests */ nbSuites--; - if (userExecKey != 0) { - execKey = userExecKey; + if (runner->user.execKey != 0) { + execKey = runner->user.execKey; } else { /* dummy values to have random numbers working */ execKey = SDLTest_GenerateExecKey(runSeed, "random testSuites", "initialisation", 1); @@ -544,7 +562,7 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user * If some random value were used at initialization before the tests start, the --seed wouldn't do the same with or without randomOrder. */ /* Swap */ - if (randomOrder) { + if (runner->user.randomOrder) { tmp = arraySuites[b]; arraySuites[b] = arraySuites[a]; arraySuites[a] = tmp; @@ -558,7 +576,7 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user /* Loop over all suites */ for (i = 0; i < nbSuites; i++) { suiteCounter = arraySuites[i]; - testSuite = testSuites[suiteCounter]; + testSuite = runner->user.testSuites[suiteCounter]; currentSuiteName = (testSuite->name ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT); suiteCounter++; @@ -595,7 +613,7 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user b = SDLTest_RandomIntegerInRange(0, nbTestCases - 1); /* Swap */ /* See previous note */ - if (randomOrder) { + if (runner->user.randomOrder) { tmp = arrayTestCases[b]; arrayTestCases[b] = arrayTestCases[a]; arrayTestCases[a] = tmp; @@ -652,11 +670,11 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user /* Loop over all iterations */ iterationCounter = 0; - while (iterationCounter < testIterations) { + while (iterationCounter < runner->user.testIterations) { iterationCounter++; - if (userExecKey != 0) { - execKey = userExecKey; + if (runner->user.execKey != 0) { + execKey = runner->user.execKey; } else { execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter); } @@ -683,10 +701,10 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user runtime = 0.0f; } - if (testIterations > 1) { + if (runner->user.testIterations > 1) { /* Log test runtime */ - SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime); - SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations); + SDLTest_Log("Runtime of %i iterations: %.1f sec", runner->user.testIterations, runtime); + SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)runner->user.testIterations); } else { /* Log test runtime */ SDLTest_Log("Total Test runtime: %.1f sec", runtime); @@ -773,3 +791,83 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user SDLTest_Log("Exit code: %d", runResult); return runResult; } + +static int SDLTest_TestSuiteCommonArg(void *data, char **argv, int index) +{ + SDLTest_TestSuiteRunner *runner = data; + + if (SDL_strcasecmp(argv[index], "--iterations") == 0) { + if (argv[index + 1]) { + runner->user.testIterations = SDL_atoi(argv[index + 1]); + if (runner->user.testIterations < 1) { + runner->user.testIterations = 1; + } + return 2; + } + } + else if (SDL_strcasecmp(argv[index], "--execKey") == 0) { + if (argv[index + 1]) { + (void)SDL_sscanf(argv[index + 1], "%" SDL_PRIu64, &runner->user.execKey); + return 2; + } + } + else if (SDL_strcasecmp(argv[index], "--seed") == 0) { + if (argv[index + 1]) { + runner->user.runSeed = SDL_strdup(argv[index + 1]); + return 2; + } + } + else if (SDL_strcasecmp(argv[index], "--filter") == 0) { + if (argv[index + 1]) { + runner->user.filter = SDL_strdup(argv[index + 1]); + return 2; + } + } + else if (SDL_strcasecmp(argv[index], "--random-order") == 0) { + runner->user.randomOrder = SDL_TRUE; + return 1; + } + return 0; +} + +SDLTest_TestSuiteRunner *SDLTest_CreateTestSuiteRunner(SDLTest_CommonState *state, SDLTest_TestSuiteReference *testSuites[]) +{ + SDLTest_TestSuiteRunner *runner; + SDLTest_ArgumentParser *argparser; + + if (!state) { + SDLTest_LogError("SDL Test Suites require a common state"); + return NULL; + } + + runner = SDL_calloc(1, sizeof(SDLTest_TestSuiteRunner)); + if (!runner) { + SDLTest_LogError("Failed to allocate memory for test suite runner"); + return NULL; + } + runner->user.testSuites = testSuites; + + runner->argparser.parse_arguments = SDLTest_TestSuiteCommonArg; + runner->argparser.usage = common_harness_usage; + runner->argparser.data = runner; + + /* Find last argument description and append our description */ + argparser = state->argparser; + for (;;) { + if (argparser->next == NULL) { + argparser->next = &runner->argparser; + break; + } + argparser = argparser->next; + + } + + return runner; +} + +void SDLTest_DestroyTestSuiteRunner(SDLTest_TestSuiteRunner *runner) { + + SDL_free(runner->user.filter); + SDL_free(runner->user.runSeed); + SDL_free(runner); +} diff --git a/test/testautomation.c b/test/testautomation.c index 6480feb70..a813b3153 100644 --- a/test/testautomation.c +++ b/test/testautomation.c @@ -19,6 +19,7 @@ #include "testautomation_suites.h" static SDLTest_CommonState *state; +static SDLTest_TestSuiteRunner *runner; /* All test suites */ static SDLTest_TestSuiteReference *testSuites[] = { @@ -55,6 +56,7 @@ static SDLTest_TestSuiteReference *testSuites[] = { static void quit(int rc) { + SDLTest_DestroyTestSuiteRunner(runner); SDLTest_CommonQuit(state); /* Let 'main()' return normally */ if (rc != 0) { @@ -65,14 +67,9 @@ quit(int rc) int main(int argc, char *argv[]) { int result; - int testIterations = 1; - Uint64 userExecKey = 0; - char *userRunSeed = NULL; - char *filter = NULL; int i, done; SDL_Event event; int list = 0; - SDL_bool randomOrder = SDL_FALSE; /* Initialize test framework */ state = SDLTest_CommonCreateState(argv, SDL_INIT_VIDEO | SDL_INIT_AUDIO); @@ -83,6 +80,8 @@ int main(int argc, char *argv[]) /* No need of windows (or update testautomation_mouse.c:mouse_getMouseFocus() */ state->num_windows = 0; + runner = SDLTest_CreateTestSuiteRunner(state, testSuites); + /* Parse commandline */ for (i = 1; i < argc;) { int consumed; @@ -90,46 +89,15 @@ int main(int argc, char *argv[]) consumed = SDLTest_CommonArg(state, i); if (consumed == 0) { consumed = -1; - if (SDL_strcasecmp(argv[i], "--iterations") == 0) { - if (argv[i + 1]) { - testIterations = SDL_atoi(argv[i + 1]); - if (testIterations < 1) { - testIterations = 1; - } - consumed = 2; - } - } else if (SDL_strcasecmp(argv[i], "--execKey") == 0) { - if (argv[i + 1]) { - (void)SDL_sscanf(argv[i + 1], "%" SDL_PRIu64, &userExecKey); - consumed = 2; - } - } else if (SDL_strcasecmp(argv[i], "--seed") == 0) { - if (argv[i + 1]) { - userRunSeed = SDL_strdup(argv[i + 1]); - consumed = 2; - } - } else if (SDL_strcasecmp(argv[i], "--filter") == 0) { - if (argv[i + 1]) { - filter = SDL_strdup(argv[i + 1]); - consumed = 2; - } - } else if (SDL_strcasecmp(argv[i], "--list") == 0) { + + if (SDL_strcasecmp(argv[i], "--list") == 0) { consumed = 1; list = 1; - } else if (SDL_strcasecmp(argv[i], "--random-order") == 0) { - consumed = 1; - randomOrder = SDL_TRUE; } - } if (consumed < 0) { static const char *options[] = { - "[--iterations #]", - "[--execKey #]", - "[--seed string]", - "[--filter suite_name|test_name]", "[--list]", - "[--random-order]", NULL }; SDLTest_CommonLogUsage(state, argv[0], options); quit(1); @@ -166,7 +134,7 @@ int main(int argc, char *argv[]) } /* Call Harness */ - result = SDLTest_RunSuites(testSuites, userRunSeed, userExecKey, filter, testIterations, randomOrder); + result = SDLTest_ExecuteTestSuiteRunner(runner); /* Empty event queue */ done = 0; @@ -177,10 +145,6 @@ int main(int argc, char *argv[]) SDL_Delay(10); } - /* Clean up */ - SDL_free(userRunSeed); - SDL_free(filter); - /* Shutdown everything */ quit(0); return result;