From babb3b3bc2a94811ccd721abf0e9199d737f4a1d Mon Sep 17 00:00:00 2001 From: Pekka Paalanen Date: Fri, 1 Nov 2019 14:02:15 +0200 Subject: [PATCH] tests: thread-based client harness This replaces the old test harness with a new one. The old harness relied on fork()'ing each test which makes tests independent, but makes debugging them harder. The new harness runs client code in a thread instead of a new process. A side-effect of not fork()'ing anymore is that any failure will stop running a test series short. Fortunately we do not have any tests that are expected to crash or fail. The old harness executed 'weston' from Meson, with lots of setup as both command line options and environment variables. The new harness executes wet_main() instead: the test program itself calls the compositor main function to execute the compositor in-process. Command line arguments are configured in the test program itself, not in meson.build. Environment variables aside, you are able to run a test by simply executing the test program, even if it is a plugin test. The new harness adds a new type of iteration: fixtures. For now, fixtures are used to set up the compositor for tests that need a compositor. If necessary, a fixture setup may include a data array of arbitrary type for executing the test series for each element in the array. This will be most useful for running screenshooting tests with both Pixman- and GL-renderers. The new harness outputs TAP formatted results into stdout. Meson is not switched to consume TAP yet though, because it would require a Meson version requirement bump and would not have any benefits at this time. OTOH outputting TAP is trivial and sets up a clear precedent of random test chatter belonging to stderr. This commit migrates only few tests to actually make use of the new features: roles is a basic client test, subsurface-shot is a client test that demonstrates the fixture array, and plugin-registry is a plugin test. The rest of the tests will be migrated later. Once all tests are migrated, we can remove the test-specific setup from meson.build, leaving only the actual build instructions in there. The not migrated tests and stand-alone tests suffer only a minor change: they no longer fork() for each TEST(), otherwise they keep running as before. Signed-off-by: Pekka Paalanen --- tests/meson.build | 55 ++- tests/plugin-registry-test.c | 37 +- tests/roles-test.c | 13 + tests/subsurface-shot-test.c | 27 +- tests/weston-test-fixture-compositor.c | 277 ++++++++++++ tests/weston-test-fixture-compositor.h | 127 ++++++ tests/weston-test-runner.c | 603 ++++++++++++++++++++----- tests/weston-test-runner.h | 180 +++++++- tests/weston-test.c | 204 ++++++++- tests/weston-testsuite-data.h | 88 ++++ 10 files changed, 1450 insertions(+), 161 deletions(-) create mode 100644 tests/weston-test-fixture-compositor.c create mode 100644 tests/weston-test-fixture-compositor.h create mode 100644 tests/weston-testsuite-data.h diff --git a/tests/meson.build b/tests/meson.build index a0735ad4..49135b43 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -11,7 +11,10 @@ env_modmap += 'weston-test-desktop-shell.so=@0@;'.format(plugin_test_shell_deskt lib_test_runner = static_library( 'test-runner', 'weston-test-runner.c', - dependencies: dep_wayland_client, + dependencies: [ + dep_libweston_private_h_deps, + dep_wayland_client, + ], include_directories: common_inc, install: false, ) @@ -22,13 +25,17 @@ dep_test_runner = declare_dependency( lib_test_client = static_library( 'test-client', - 'weston-test-client-helper.c', - weston_test_client_protocol_h, - weston_test_protocol_c, + [ + 'weston-test-client-helper.c', + 'weston-test-fixture-compositor.c', + weston_test_client_protocol_h, + weston_test_protocol_c, + ], include_directories: common_inc, dependencies: [ dep_libshared, dep_wayland_client, + dep_libexec_weston, dep_pixman, dependency('cairo'), ], @@ -49,10 +56,15 @@ exe_plugin_test = shared_library( weston_test_server_protocol_h, weston_test_protocol_c, include_directories: common_inc, - dependencies: [ dep_libexec_weston, dep_libweston_private ], + dependencies: [ + dep_libexec_weston, + dep_libweston_private, + dep_threads + ], name_prefix: '', install: false, ) +config_h.set_quoted('TESTSUITE_PLUGIN_PATH', exe_plugin_test.full_path()) deps_zuc = [ dep_libshared ] if get_option('test-junit-xml') @@ -99,6 +111,12 @@ dep_zucmain = declare_dependency( dependencies: dep_zuc ) +tests = [ + { 'name': 'plugin-registry', }, + { 'name': 'roles', }, + { 'name': 'subsurface-shot', }, +] + tests_standalone = [ ['config-parser', [], [ dep_zucmain ]], ['matrix', [], [ dep_libm, dep_matrix_c ]], @@ -149,9 +167,7 @@ tests_weston = [ input_timestamps_unstable_v1_protocol_c, ] ], - ['roles'], ['subsurface'], - ['subsurface-shot'], [ 'text', [ @@ -185,7 +201,6 @@ if get_option('xwayland') endif tests_weston_plugin = [ - ['plugin-registry'], ['surface'], ['surface-global'], ['surface-screenshot', 'surface-screenshot-test.c', dep_libshared], @@ -235,6 +250,30 @@ env_test_weston = [ 'WESTON_DATA_DIR=' + join_paths(meson.current_source_dir(), '..', 'data'), ] +foreach t : tests + t_name = 'test-' + t.get('name') + t_sources = t.get('sources', [t.get('name') + '-test.c']) + t_sources += weston_test_client_protocol_h + + t_deps = [ dep_test_client, dep_libweston_private_h ] + t_deps += t.get('dep_objs', []) + + t_exe = executable( + t_name, + t_sources, + c_args: [ + '-DUNIT_TEST', + '-DTHIS_TEST_NAME="' + t_name + '"', + ], + build_by_default: true, + include_directories: common_inc, + dependencies: t_deps, + install: false, + ) + + test(t.get('name'), t_exe, depends: t.get('test_deps', []), env: env_test_weston) +endforeach + # FIXME: the multiple loops is lame. rethink this. foreach t : tests_standalone if t[0] != 'zuc' diff --git a/tests/plugin-registry-test.c b/tests/plugin-registry-test.c index 4ffc69e5..85de5f8a 100644 --- a/tests/plugin-registry-test.c +++ b/tests/plugin-registry-test.c @@ -31,6 +31,20 @@ #include "compositor/weston.h" #include +#include "weston-test-runner.h" +#include "weston-test-fixture-compositor.h" + +static enum test_result_code +fixture_setup(struct weston_test_harness *harness) +{ + struct compositor_setup setup; + + compositor_setup_defaults(&setup); + + return weston_test_harness_execute_as_plugin(harness, &setup); +} +DECLARE_FIXTURE_SETUP(fixture_setup); + static void dummy_func(void) { @@ -67,13 +81,14 @@ init_tests(struct weston_compositor *compositor) sizeof(my_test_api)) == 0); } -static void -runtime_tests(void *data) +PLUGIN_TEST(plugin_registry_test) { - struct weston_compositor *compositor = data; + /* struct weston_compositor *compositor; */ const struct my_api *api; size_t sz = sizeof(struct my_api); + init_tests(compositor); + assert(weston_plugin_api_get(compositor, MY_API_NAME, sz) == &my_test_api); @@ -84,20 +99,4 @@ runtime_tests(void *data) api = weston_plugin_api_get(compositor, MY_API_NAME, sz); assert(api && api->func2 == dummy_func); - - weston_compositor_exit(compositor); -} - -WL_EXPORT int -wet_module_init(struct weston_compositor *compositor, - int *argc, char *argv[]) -{ - struct wl_event_loop *loop; - - init_tests(compositor); - - loop = wl_display_get_event_loop(compositor->wl_display); - wl_event_loop_add_idle(loop, runtime_tests, compositor); - - return 0; } diff --git a/tests/roles-test.c b/tests/roles-test.c index e0c4377d..fbce0539 100644 --- a/tests/roles-test.c +++ b/tests/roles-test.c @@ -30,6 +30,19 @@ #include #include "weston-test-client-helper.h" +#include "weston-test-fixture-compositor.h" + +static enum test_result_code +fixture_setup(struct weston_test_harness *harness) +{ + struct compositor_setup setup; + + compositor_setup_defaults(&setup); + setup.logging_scopes = "log,proto,test-harness-plugin"; + + return weston_test_harness_execute_as_client(harness, &setup); +} +DECLARE_FIXTURE_SETUP(fixture_setup); static struct wl_subcompositor * get_subcompositor(struct client *client) diff --git a/tests/subsurface-shot-test.c b/tests/subsurface-shot-test.c index d3dc05ba..dcbce0c3 100644 --- a/tests/subsurface-shot-test.c +++ b/tests/subsurface-shot-test.c @@ -31,9 +31,32 @@ #include #include "weston-test-client-helper.h" +#include "weston-test-fixture-compositor.h" -char *server_parameters = "--use-pixman --width=320 --height=240" - " --shell=weston-test-desktop-shell.so"; +static const enum renderer_type renderers[] = { + RENDERER_PIXMAN, + RENDERER_GL, +}; + +static enum test_result_code +fixture_setup(struct weston_test_harness *harness, const enum renderer_type *arg) +{ + struct compositor_setup setup; + + compositor_setup_defaults(&setup); + setup.renderer = *arg; + setup.width = 320; + setup.height = 240; + setup.shell = SHELL_TEST_DESKTOP; + setup.logging_scopes = "log,test-harness-plugin"; + + /* This test fails due to color rounding on GL */ + if (setup.renderer == RENDERER_GL) + return RESULT_SKIP; + + return weston_test_harness_execute_as_client(harness, &setup); +} +DECLARE_FIXTURE_SETUP_WITH_ARG(fixture_setup, renderers); static struct wl_subcompositor * get_subcompositor(struct client *client) diff --git a/tests/weston-test-fixture-compositor.c b/tests/weston-test-fixture-compositor.c new file mode 100644 index 00000000..04c06409 --- /dev/null +++ b/tests/weston-test-fixture-compositor.c @@ -0,0 +1,277 @@ +/* + * Copyright 2019 Collabora, Ltd. + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "config.h" + +#include +#include + +#include "shared/helpers.h" +#include "weston-test-fixture-compositor.h" +#include "weston.h" + +struct prog_args { + int argc; + char **argv; + char **saved; + int alloc; +}; + +static void +prog_args_init(struct prog_args *p) +{ + memset(p, 0, sizeof(*p)); +} + +static void +prog_args_take(struct prog_args *p, char *arg) +{ + assert(arg); + + if (p->argc == p->alloc) { + p->alloc += 10; + p->argv = realloc(p->argv, sizeof(char *) * p->alloc); + assert(p->argv); + } + + p->argv[p->argc++] = arg; +} + +/* + * The program to be executed will trample on argv, hence we need a copy to + * be able to free all our args. + */ +static void +prog_args_save(struct prog_args *p) +{ + assert(p->saved == NULL); + + p->saved = calloc(p->argc, sizeof(char *)); + assert(p->saved); + + memcpy(p->saved, p->argv, sizeof(char *) * p->argc); +} + +static void +prog_args_fini(struct prog_args *p) +{ + int i; + + assert(p->saved); + + for (i = 0; i < p->argc; i++) + free(p->saved[i]); + free(p->saved); + free(p->argv); + prog_args_init(p); +} + +/** Initialize part of compositor setup + * + * \param setup The variable to initialize. + * \param testset_name Value for testset_name member. + * + * \ingroup testharness_private + */ +void +compositor_setup_defaults_(struct compositor_setup *setup, + const char *testset_name) +{ + *setup = (struct compositor_setup) { + .backend = WESTON_BACKEND_HEADLESS, + .renderer = RENDERER_NOOP, + .shell = SHELL_DESKTOP, + .xwayland = false, + .width = 320, + .height = 240, + .config_file = NULL, + .extra_module = NULL, + .logging_scopes = NULL, + .testset_name = testset_name, + }; +} + +static const char * +backend_to_str(enum weston_compositor_backend b) +{ + static const char * const names[] = { + [WESTON_BACKEND_DRM] = "drm-backend.so", + [WESTON_BACKEND_FBDEV] = "fbdev-backend.so", + [WESTON_BACKEND_HEADLESS] = "headless-backend.so", + [WESTON_BACKEND_RDP] = "rdp-backend.so", + [WESTON_BACKEND_WAYLAND] = "wayland-backend.so", + [WESTON_BACKEND_X11] = "X11-backend.so", + }; + assert(b >= 0 && b < ARRAY_LENGTH(names)); + return names[b]; +} + +static const char * +renderer_to_arg(enum weston_compositor_backend b, enum renderer_type r) +{ + static const char * const headless_names[] = { + [RENDERER_NOOP] = NULL, + [RENDERER_PIXMAN] = "--use-pixman", + [RENDERER_GL] = "--use-gl", + }; + + assert(r >= 0 && r < ARRAY_LENGTH(headless_names)); + + switch (b) { + case WESTON_BACKEND_HEADLESS: + return headless_names[r]; + default: + assert(0 && "renderer_to_str() does not know the backend"); + } + + return NULL; +} + +static const char * +shell_to_str(enum shell_type t) +{ + static const char * const names[] = { + [SHELL_TEST_DESKTOP] = "weston-test-desktop-shell.so", + [SHELL_DESKTOP] = "desktop-shell.so", + [SHELL_FULLSCREEN] = "fullscreen-shell.so", + [SHELL_IVI] = "ivi-shell.so", + }; + assert(t >= 0 && t < ARRAY_LENGTH(names)); + return names[t]; +} + +/** Execute compositor + * + * Manufactures the compositor command line and calls wet_main(). + * + * Returns RESULT_SKIP if the given setup contains features that were disabled + * in the build, e.g. GL-renderer or DRM-backend. + * + * \ingroup testharness_private + */ +int +execute_compositor(const struct compositor_setup *setup, + struct wet_testsuite_data *data) +{ + struct prog_args args; + char *tmp; + const char *ctmp; + int ret; + +#ifndef BUILD_DRM_COMPOSITOR + if (setup->backend == WESTON_BACKEND_DRM) { + fprintf(stderr, "DRM-backend required but not built, skipping.\n"); + return RESULT_SKIP; + } +#endif + +#ifndef BUILD_FBDEV_COMPOSITOR + if (setup->backend == WESTON_BACKEND_FBDEV) { + fprintf(stderr, "fbdev-backend required but not built, skipping.\n"); + return RESULT_SKIP; + } +#endif + +#ifndef BUILD_RDP_COMPOSITOR + if (setup->backend == WESTON_BACKEND_RDP) { + fprintf(stderr, "RDP-backend required but not built, skipping.\n"); + return RESULT_SKIP; + } +#endif + +#ifndef BUILD_WAYLAND_COMPOSITOR + if (setup->backend == WESTON_BACKEND_WAYLAND) { + fprintf(stderr, "wayland-backend required but not built, skipping.\n"); + return RESULT_SKIP; + } +#endif + +#ifndef BUILD_X11_COMPOSITOR + if (setup->backend == WESTON_BACKEND_X11) { + fprintf(stderr, "X11-backend required but not built, skipping.\n"); + return RESULT_SKIP; + } +#endif + +#ifndef ENABLE_EGL + if (setup->renderer == RENDERER_GL) { + fprintf(stderr, "GL-renderer required but not built, skipping.\n"); + return RESULT_SKIP; + } +#endif + + prog_args_init(&args); + + /* argv[0] */ + asprintf(&tmp, "weston-%s", setup->testset_name); + prog_args_take(&args, tmp); + + asprintf(&tmp, "--backend=%s", backend_to_str(setup->backend)); + prog_args_take(&args, tmp); + + asprintf(&tmp, "--socket=%s", setup->testset_name); + prog_args_take(&args, tmp); + + asprintf(&tmp, "--modules=%s%s%s", TESTSUITE_PLUGIN_PATH, + setup->extra_module ? "," : "", + setup->extra_module ? setup->extra_module : ""); + prog_args_take(&args, tmp); + + asprintf(&tmp, "--width=%d", setup->width); + prog_args_take(&args, tmp); + + asprintf(&tmp, "--height=%d", setup->height); + prog_args_take(&args, tmp); + + if (setup->config_file) { + asprintf(&tmp, "--config=%s", setup->config_file); + prog_args_take(&args, tmp); + } else { + prog_args_take(&args, strdup("--no-config")); + } + + ctmp = renderer_to_arg(setup->backend, setup->renderer); + if (ctmp) + prog_args_take(&args, strdup(ctmp)); + + asprintf(&tmp, "--shell=%s", shell_to_str(setup->shell)); + prog_args_take(&args, tmp); + + if (setup->logging_scopes) { + asprintf(&tmp, "--logger-scopes=%s", setup->logging_scopes); + prog_args_take(&args, tmp); + } + + if (setup->xwayland) + prog_args_take(&args, strdup("--xwayland")); + + wet_testsuite_data_set(data); + prog_args_save(&args); + ret = wet_main(args.argc, args.argv); + + prog_args_fini(&args); + + return ret; +} diff --git a/tests/weston-test-fixture-compositor.h b/tests/weston-test-fixture-compositor.h new file mode 100644 index 00000000..6e8d680e --- /dev/null +++ b/tests/weston-test-fixture-compositor.h @@ -0,0 +1,127 @@ +/* + * Copyright 2019 Collabora, Ltd. + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef WESTON_TEST_FIXTURE_COMPOSITOR_H +#define WESTON_TEST_FIXTURE_COMPOSITOR_H + +#include + +#include "weston-testsuite-data.h" + +/** Weston renderer type + * + * \sa compositor_setup + * \ingroup testharness + */ +enum renderer_type { + /** Dummy renderer that does nothing. */ + RENDERER_NOOP = 0, + /** Pixman-renderer */ + RENDERER_PIXMAN, + /** GL-renderer */ + RENDERER_GL +}; + +/** Weston shell plugin + * + * \sa compositor_setup + * \ingroup testharness + */ +enum shell_type { + /** Desktop test-shell with predictable window placement and + * no helper clients */ + SHELL_TEST_DESKTOP = 0, + /** The full desktop shell. */ + SHELL_DESKTOP, + /** The ivi-shell. */ + SHELL_IVI, + /** The fullscreen-shell. */ + SHELL_FULLSCREEN +}; + +/** Weston compositor configuration + * + * This structure determines the Weston compositor command line arguments. + * You should always use compositor_setup_defaults() to initialize this, then + * override any members you need with assignments. + * + * \ingroup testharness + */ +struct compositor_setup { + /** The backend to use. */ + enum weston_compositor_backend backend; + /** The renderer to use. */ + enum renderer_type renderer; + /** The shell plugin to use. */ + enum shell_type shell; + /** Whether to enable xwayland support. */ + bool xwayland; + /** Default output width. */ + unsigned width; + /** Default output height. */ + unsigned height; + /** The absolute path to \c weston.ini to use, + * or NULL for \c --no-config . */ + const char *config_file; + /** Full path to an extra plugin to load, or NULL for none. */ + const char *extra_module; + /** Debug scopes for the compositor log, + * or NULL for compositor defaults. */ + const char *logging_scopes; + /** The name of this test program, used as a unique identifier. */ + const char *testset_name; +}; + +void +compositor_setup_defaults_(struct compositor_setup *setup, + const char *testset_name); + +/** Initialize compositor setup to defaults + * + * \param s The variable to initialize. + * + * The defaults are: + * - backend: headless + * - renderer: noop + * - shell: desktop shell + * - xwayland: no + * - width: 320 + * - height: 240 + * - config_file: none + * - extra_module: none + * - logging_scopes: compositor defaults + * - testset_name: the test name from meson.build + * + * \ingroup testharness + */ +#define compositor_setup_defaults(s) do {\ + compositor_setup_defaults_(s, THIS_TEST_NAME); \ +} while (0) + +int +execute_compositor(const struct compositor_setup *setup, + struct wet_testsuite_data *data); + +#endif /* WESTON_TEST_FIXTURE_COMPOSITOR_H */ diff --git a/tests/weston-test-runner.c b/tests/weston-test-runner.c index ee17ba4d..8f945f94 100644 --- a/tests/weston-test-runner.c +++ b/tests/weston-test-runner.c @@ -33,23 +33,47 @@ #include #include #include +#include #include "weston-test-runner.h" +#include "weston-testsuite-data.h" +#include "shared/string-helpers.h" -#define SKIP 77 - -char __attribute__((weak)) *server_parameters=""; +/** + * \defgroup testharness Test harness + * \defgroup testharness_private Test harness private + */ extern const struct weston_test_entry __start_test_section, __stop_test_section; static const char *test_name_; +/** Get the test name string with counter + * + * \return The test name with fixture number \c f%%d- prefixed. For an array + * driven test, e.g. defined with TEST_P(), the name has a \c -e%%d suffix to + * indicate the array element number. + * + * This is only usable from code paths inside TEST(), TEST_P(), PLUGIN_TEST() + * etc. defined functions. + * + * \ingroup testharness + */ const char * get_test_name(void) { return test_name_; } +/** Print into test log + * + * This is exactly like printf() except the output goes to the test log, + * which is at stderr. + * + * \param fmt printf format string + * + * \ingroup testharness + */ void testlog(const char *fmt, ...) { @@ -72,157 +96,502 @@ find_test(const char *name) return NULL; } -static void -run_test(const struct weston_test_entry *t, void *data, int iteration) +static enum test_result_code +run_test(int fixture_nr, const struct weston_test_entry *t, void *data, + int iteration) { char str[512]; if (data) { - snprintf(str, sizeof(str), "%s[%d]", t->name, iteration); - test_name_ = str; + snprintf(str, sizeof(str), "f%d-%s-e%d", + fixture_nr, t->name, iteration); } else { - test_name_ = t->name; + snprintf(str, sizeof(str), "f%d-%s", fixture_nr, t->name); } + test_name_ = str; t->run(data); + test_name_ = NULL; + + /* + * XXX: We should return t->run(data); but that requires changing + * the function signature and stop using assert() in tests. + * https://gitlab.freedesktop.org/wayland/weston/issues/311 + */ + return RESULT_OK; } static void list_tests(void) { + const struct fixture_setup_array *fsa; const struct weston_test_entry *t; - fprintf(stderr, "Available test names:\n"); - for (t = &__start_test_section; t < &__stop_test_section; t++) - fprintf(stderr, " %s\n", t->name); + fsa = fixture_setup_array_get_(); + + printf("Fixture setups: %d\n", fsa->n_elements); + + for (t = &__start_test_section; t < &__stop_test_section; t++) { + printf(" %s\n", t->name); + if (t->n_elements > 1) + printf(" with array of %d cases\n", t->n_elements); + } } -/* iteration is valid only if test_data is not NULL */ -static int -exec_and_report_test(const struct weston_test_entry *t, - void *test_data, int iteration) +struct weston_test_harness { + int32_t fixt_ind; + char *chosen_testname; + int32_t case_ind; + + struct wet_testsuite_data data; +}; + +typedef void (*weston_test_cb)(struct wet_testsuite_data *suite_data, + const struct weston_test_entry *t, + const void *test_data, + int iteration); + +static void +for_each_test_case(struct wet_testsuite_data *data, weston_test_cb cb) { - int success = 0; - int skip = 0; - int hardfail = 0; - siginfo_t info; + unsigned i; - pid_t pid = fork(); - assert(pid >= 0); + for (i = 0; i < data->tests_count; i++) { + const struct weston_test_entry *t = &data->tests[i]; + const void *current_test_data = t->table_data; + int elem; + int elem_end; - if (pid == 0) { - run_test(t, test_data, iteration); - exit(EXIT_SUCCESS); - } + if (data->case_index == -1) { + elem = 0; + elem_end = t->n_elements; + } else { + elem = data->case_index; + elem_end = elem + 1; + } - if (waitid(P_ALL, 0, &info, WEXITED)) { - fprintf(stderr, "waitid failed: %s\n", strerror(errno)); - abort(); - } - - if (test_data) - fprintf(stderr, "test \"%s/%i\":\t", t->name, iteration); - else - fprintf(stderr, "test \"%s\":\t", t->name); - - switch (info.si_code) { - case CLD_EXITED: - fprintf(stderr, "exit status %d", info.si_status); - if (info.si_status == EXIT_SUCCESS) - success = 1; - else if (info.si_status == SKIP) - skip = 1; - break; - case CLD_KILLED: - case CLD_DUMPED: - fprintf(stderr, "signal %d", info.si_status); - if (info.si_status != SIGABRT) - hardfail = 1; - break; - } - - if (success && !hardfail) { - fprintf(stderr, ", pass.\n"); - return 1; - } else if (skip) { - fprintf(stderr, ", skip.\n"); - return SKIP; - } else { - fprintf(stderr, ", fail.\n"); - return 0; + for (; elem < elem_end; elem++) { + current_test_data = (char *)t->table_data + + elem * t->element_size; + cb(data, t, current_test_data, elem); + } } } -/* Returns number of tests and number of pass / fail in param args. - * Even non-iterated tests go through here, they simply have n_elements = 1 and - * table_data = NULL. +static const char * +result_to_str(enum test_result_code ret) +{ + static const char *names[] = { + [RESULT_FAIL] = "fail", + [RESULT_HARD_ERROR] = "hard error", + [RESULT_OK] = "ok", + [RESULT_SKIP] = "skip", + }; + + assert(ret >= 0 && ret < ARRAY_LENGTH(names)); + return names[ret]; +} + +static void +run_case(struct wet_testsuite_data *suite_data, + const struct weston_test_entry *t, + const void *test_data, + int iteration) +{ + enum test_result_code ret; + const char *fail = ""; + const char *skip = ""; + int fixture_nr = suite_data->fixture_iteration + 1; + int iteration_nr = iteration + 1; + + testlog("*** Run fixture %d, %s/%d\n", + fixture_nr, t->name, iteration_nr); + + if (suite_data->type == TEST_TYPE_PLUGIN) { + ret = run_test(fixture_nr, t, suite_data->compositor, + iteration); + } else { + ret = run_test(fixture_nr, t, (void *)test_data, iteration); + } + + switch (ret) { + case RESULT_OK: + suite_data->passed++; + break; + case RESULT_FAIL: + case RESULT_HARD_ERROR: + suite_data->failed++; + fail = "not "; + break; + case RESULT_SKIP: + suite_data->skipped++; + skip = " # SKIP"; + break; + } + + testlog("*** Result fixture %d, %s/%d: %s\n", + fixture_nr, t->name, iteration_nr, result_to_str(ret)); + + suite_data->counter++; + printf("%sok %d fixture %d %s/%d%s\n", fail, suite_data->counter, + fixture_nr, t->name, iteration_nr, skip); +} + +/* This function might run in a new thread */ +static void +testsuite_run(struct wet_testsuite_data *data) +{ + for_each_test_case(data, run_case); +} + +static void +count_case(struct wet_testsuite_data *suite_data, + const struct weston_test_entry *t, + const void *test_data, + int iteration) +{ + suite_data->total++; +} + +static void +tap_plan(struct wet_testsuite_data *data, int count_fixtures) +{ + data->total = 0; + for_each_test_case(data, count_case); + + printf("1..%d\n", data->total * count_fixtures); +} + +static void +skip_case(struct wet_testsuite_data *suite_data, + const struct weston_test_entry *t, + const void *test_data, + int iteration) +{ + int fixture_nr = suite_data->fixture_iteration + 1; + int iteration_nr = iteration + 1; + + suite_data->counter++; + printf("ok %d fixture %d %s/%d # SKIP fixture\n", suite_data->counter, + fixture_nr, t->name, iteration_nr); +} + +static void +tap_skip_fixture(struct wet_testsuite_data *data) +{ + for_each_test_case(data, skip_case); +} + +static void +help(const char *exe) +{ + printf( + "Usage: %s [options] [testname [index]]\n" + "\n" + "This is a Weston test suite executable that runs some tests.\n" + "Options:\n" + " -f, --fixture N Run only fixture index N. Indices start from 1.\n" + " -h, --help Print this help and exit with success.\n" + " -l, --list List all tests in this executable and exit with success.\n" + "testname: Optional; name of the test to execute instead of all tests.\n" + "index: Optional; for a multi-case test, run the given case only.\n", + exe); +} + +static void +parse_command_line(struct weston_test_harness *harness, int argc, char **argv) +{ + int c; + static const struct option opts[] = { + { "fixture", required_argument, NULL, 'f' }, + { "help", no_argument, NULL, 'h' }, + { "list", no_argument, NULL, 'l' }, + { 0, 0, NULL, 0 } + }; + + while ((c = getopt_long(argc, argv, "f:hl", opts, NULL)) != -1) { + switch (c) { + case 'f': + if (!safe_strtoint(optarg, &harness->fixt_ind)) { + fprintf(stderr, + "Error: '%s' does not look like a number (command line).\n", + optarg); + exit(RESULT_HARD_ERROR); + } + harness->fixt_ind--; /* convert base-1 to base 0 */ + break; + case 'h': + help(argv[0]); + exit(RESULT_OK); + case 'l': + list_tests(); + exit(RESULT_OK); + case 0: + break; + default: + exit(RESULT_HARD_ERROR); + } + } + + if (optind < argc) + harness->chosen_testname = argv[optind++]; + + if (optind < argc) { + if (!safe_strtoint(argv[optind], &harness->case_ind)) { + fprintf(stderr, + "Error: '%s' does not look like a number (command line).\n", + argv[optind]); + exit(RESULT_HARD_ERROR); + } + harness->case_ind--; /* convert base-1 to base 0 */ + optind++; + } + + if (optind < argc) { + fprintf(stderr, "Unexpected extra arguments given (command line).\n\n"); + help(argv[0]); + exit(RESULT_HARD_ERROR); + } +} + +static struct weston_test_harness * +weston_test_harness_create(int argc, char **argv) +{ + const struct fixture_setup_array *fsa; + struct weston_test_harness *harness; + + harness = zalloc(sizeof(*harness)); + assert(harness); + + harness->fixt_ind = -1; + harness->case_ind = -1; + parse_command_line(harness, argc, argv); + + fsa = fixture_setup_array_get_(); + if (harness->fixt_ind < -1 || harness->fixt_ind >= fsa->n_elements) { + fprintf(stderr, + "Error: fixture index %d (command line) is invalid for this program.\n", + harness->fixt_ind + 1); + exit(RESULT_HARD_ERROR); + } + + if (harness->chosen_testname) { + const struct weston_test_entry *t; + + t = find_test(harness->chosen_testname); + if (!t) { + fprintf(stderr, + "Error: test '%s' not found (command line).\n", + harness->chosen_testname); + exit(RESULT_HARD_ERROR); + } + + if (harness->case_ind < -1 || + harness->case_ind >= t->n_elements) { + fprintf(stderr, + "Error: case index %d (command line) is invalid for this test.\n", + harness->case_ind + 1); + exit(RESULT_HARD_ERROR); + } + + harness->data.tests = t; + harness->data.tests_count = 1; + harness->data.case_index = harness->case_ind; + } else { + harness->data.tests = &__start_test_section; + harness->data.tests_count = + &__stop_test_section - &__start_test_section; + harness->data.case_index = -1; + } + + harness->data.run = testsuite_run; + + return harness; +} + +static void +weston_test_harness_destroy(struct weston_test_harness *harness) +{ + free(harness); +} + +static enum test_result_code +counts_to_result(const struct wet_testsuite_data *data) +{ + /* RESULT_SKIP is reserved for fixture setup itself skipping everything */ + if (data->total == data->passed + data->skipped) + return RESULT_OK; + return RESULT_FAIL; +} + +/** Execute all tests as client tests + * + * \param harness The test harness context. + * \param setup The compositor configuration. + * + * Initializes the compositor with the given setup and executes the compositor. + * The compositor creates a new thread where all tests in the test program are + * serially executed. Once the thread finishes, the compositor returns from its + * event loop and cleans up. + * + * Returns RESULT_SKIP if the requested compositor features, e.g. GL-renderer, + * are not built. + * + * \sa DECLARE_FIXTURE_SETUP(), DECLARE_FIXTURE_SETUP_WITH_ARG() + * \ingroup testharness */ -static int -iterate_test(const struct weston_test_entry *t, int *passed, int *skipped) +enum test_result_code +weston_test_harness_execute_as_client(struct weston_test_harness *harness, + const struct compositor_setup *setup) { - int ret, i; - void *current_test_data = (void *) t->table_data; - for (i = 0; i < t->n_elements; ++i, current_test_data += t->element_size) - { - ret = exec_and_report_test(t, current_test_data, i); - if (ret == SKIP) - ++(*skipped); - else if (ret) - ++(*passed); - } + struct wet_testsuite_data *data = &harness->data; - return t->n_elements; + data->type = TEST_TYPE_CLIENT; + return execute_compositor(setup, data); } -int main(int argc, char *argv[]) +/** Execute all tests as plugin tests + * + * \param harness The test harness context. + * \param setup The compositor configuration. + * + * Initializes the compositor with the given setup and executes the compositor. + * The compositor executes all tests in the test program serially from an idle + * handler, then returns from its event loop and cleans up. + * + * Returns RESULT_SKIP if the requested compositor features, e.g. GL-renderer, + * are not built. + * + * \sa DECLARE_FIXTURE_SETUP(), DECLARE_FIXTURE_SETUP_WITH_ARG() + * \ingroup testharness + */ +enum test_result_code +weston_test_harness_execute_as_plugin(struct weston_test_harness *harness, + const struct compositor_setup *setup) { - const struct weston_test_entry *t; - int total = 0; - int pass = 0; - int skip = 0; + struct wet_testsuite_data *data = &harness->data; - if (argc == 2) { - const char *testname = argv[1]; - if (strcmp(testname, "--help") == 0 || - strcmp(testname, "-h") == 0) { - fprintf(stderr, "Usage: %s [test-name]\n", program_invocation_short_name); - list_tests(); - exit(EXIT_SUCCESS); - } + data->type = TEST_TYPE_PLUGIN; + return execute_compositor(setup, data); +} - if (strcmp(testname, "--params") == 0 || - strcmp(testname, "-p") == 0) { - printf("%s", server_parameters); - exit(EXIT_SUCCESS); - } +/** Execute all tests as standalone tests + * + * \param harness The test harness context. + * + * Executes all tests in the test program serially without any further setup, + * particularly without any compositor instance created. + * + * \sa DECLARE_FIXTURE_SETUP(), DECLARE_FIXTURE_SETUP_WITH_ARG() + * \ingroup testharness + */ +enum test_result_code +weston_test_harness_execute_standalone(struct weston_test_harness *harness) +{ + struct wet_testsuite_data *data = &harness->data; - t = find_test(argv[1]); - if (t == NULL) { - fprintf(stderr, "unknown test: \"%s\"\n", argv[1]); - list_tests(); - exit(EXIT_FAILURE); - } + data->type = TEST_TYPE_STANDALONE; + data->run(data); - int number_passed_in_test = 0, number_skipped_in_test = 0; - total += iterate_test(t, &number_passed_in_test, &number_skipped_in_test); - pass += number_passed_in_test; - skip += number_skipped_in_test; + return RESULT_OK; +} + +/** Fixture data array getter method + * + * DECLARE_FIXTURE_SETUP_WITH_ARG() overrides this in test programs. + * The default implementation has no data and makes the tests run once. + * + * \ingroup testharness + */ +__attribute__((weak)) const struct fixture_setup_array * +fixture_setup_array_get_(void) +{ + /* A dummy fixture without a data array. */ + static const struct fixture_setup_array default_fsa = { + .array = NULL, + .element_size = 0, + .n_elements = 1, + }; + + return &default_fsa; +} + +/** Fixture setup function + * + * DECLARE_FIXTURE_SETUP() and DECLARE_FIXTURE_SETUP_WITH_ARG() override + * this in test programs. + * The default implementation just calls + * weston_test_harness_execute_standalone(). + * + * \ingroup testharness + */ +__attribute__((weak)) enum test_result_code +fixture_setup_run_(struct weston_test_harness *harness, const void *arg_) +{ + return weston_test_harness_execute_standalone(harness); +} + +static void +fixture_report(const struct wet_testsuite_data *d, enum test_result_code ret) +{ + int fixture_nr = d->fixture_iteration + 1; + + testlog("--- Fixture %d %s: passed %d, skipped %d, failed %d, total %d\n", + fixture_nr, result_to_str(ret), + d->passed, d->skipped, d->failed, d->total); +} + +int +main(int argc, char *argv[]) +{ + struct weston_test_harness *harness; + enum test_result_code ret; + enum test_result_code result = RESULT_OK; + const struct fixture_setup_array *fsa; + const char *array_data; + int fi; + int fi_end; + + harness = weston_test_harness_create(argc, argv); + + fsa = fixture_setup_array_get_(); + array_data = fsa->array; + + if (harness->fixt_ind == -1) { + fi = 0; + fi_end = fsa->n_elements; } else { - for (t = &__start_test_section; t < &__stop_test_section; t++) { - int number_passed_in_test = 0, number_skipped_in_test = 0; - total += iterate_test(t, &number_passed_in_test, &number_skipped_in_test); - pass += number_passed_in_test; - skip += number_skipped_in_test; - } + fi = harness->fixt_ind; + fi_end = fi + 1; } - fprintf(stderr, "%d tests, %d pass, %d skip, %d fail\n", - total, pass, skip, total - pass - skip); + tap_plan(&harness->data, fi_end - fi); + testlog("Iterating through %d fixtures.\n", fi_end - fi); - if (skip == total) - return SKIP; - else if (pass + skip == total) - return EXIT_SUCCESS; + for (; fi < fi_end; fi++) { + const void *arg = array_data + fi * fsa->element_size; - return EXIT_FAILURE; + testlog("--- Fixture %d...\n", fi + 1); + harness->data.fixture_iteration = fi; + harness->data.passed = 0; + harness->data.skipped = 0; + harness->data.failed = 0; + + ret = fixture_setup_run_(harness, arg); + fixture_report(&harness->data, ret); + + if (ret == RESULT_SKIP) { + tap_skip_fixture(&harness->data); + continue; + } + + if (ret != RESULT_OK && result != RESULT_HARD_ERROR) + result = ret; + else if (counts_to_result(&harness->data) != RESULT_OK) + result = RESULT_FAIL; + } + + weston_test_harness_destroy(harness); + + return result; } diff --git a/tests/weston-test-runner.h b/tests/weston-test-runner.h index df9a0b80..c47deba9 100644 --- a/tests/weston-test-runner.h +++ b/tests/weston-test-runner.h @@ -33,11 +33,22 @@ #include #include "shared/helpers.h" +#include "weston-test-fixture-compositor.h" +#include "weston-testsuite-data.h" #ifdef NDEBUG #error "Tests must not be built with NDEBUG defined, they rely on assert()." #endif +/** Test program entry + * + * Each invocation of TEST(), TEST_P(), or PLUGIN_TEST() will create one + * more weston_test_entry in a custom named section in the final binary. + * Iterating through the section then allows to iterate through all + * the defined tests. + * + * \ingroup testharness_private + */ struct weston_test_entry { const char *name; void (*run)(void *); @@ -75,22 +86,171 @@ struct weston_test_entry { ARRAY_LENGTH(test_data)) \ TEST_BEGIN(name, void *data) \ +/** Add a test with no parameters + * + * This defines one test as a new function. Use this macro in place of the + * function signature and put the function body after this. + * + * \param name Name for the test, must be a valid function name. + * + * \ingroup testharness + */ #define TEST(name) NO_ARG_TEST(name) -#define TEST_P(name, data) ARG_TEST(name, data) + +/** Add an array driven test with a parameter + * + * This defines an array of tests as a new function. Use this macro in place + * of the function signature and put the function body after this. The function + * will be executed once for each element in \c data_array, passing the + * element as the argument void *data to the function. + * + * This macro is not usable if fixture setup is using + * weston_test_harness_execute_as_plugin(). + * + * \param name Name for the test, must be a valid function name. + * \param data_array A static const array of any type. The length will be + * recorded automatically. + * + * \ingroup testharness + */ +#define TEST_P(name, data_array) ARG_TEST(name, data_array) + +/** Add a test with weston_compositor argument + * + * This defines one test as a new function. Use this macro in place of the + * function signature and put the function body after this. The function + * will have one argument struct weston_compositor *compositor. + * + * This macro is only usable if fixture setup is using + * weston_test_harness_execute_as_plugin(). + * + * \param name Name for the test, must be a valid function name. + * + * \ingroup testharness + */ +#define PLUGIN_TEST(name) \ + TEST_COMMON(wrap##name, name, NULL, 0, 1) \ + static void name(struct weston_compositor *); \ + static void wrap##name(void *compositor) \ + { \ + name(compositor); \ + } \ + TEST_BEGIN(name, struct weston_compositor *compositor) void testlog(const char *fmt, ...) WL_PRINTF(1, 2); -/** - * Get the test name string with counter - * - * \return The test name. For an iterated test, e.g. defined with TEST_P(), - * the name has a '[%d]' suffix to indicate the iteration. - * - * This is only usable from code paths inside TEST(), TEST_P(), etc. - * defined functions. - */ const char * get_test_name(void); +/** Fixture setup array record + * + * Helper to store the attributes of the data array passed in to + * DECLARE_FIXTURE_SETUP_WITH_ARG(). + * + * \ingroup testharness_private + */ +struct fixture_setup_array { + const void *array; + size_t element_size; + int n_elements; +}; + +const struct fixture_setup_array * +fixture_setup_array_get_(void); + +/** Test harness context + * + * \ingroup testharness + */ +struct weston_test_harness; + +enum test_result_code +fixture_setup_run_(struct weston_test_harness *harness, const void *arg_); + +/** Register a fixture setup function + * + * This registers the given (preferably static) function to be used for setting + * up any fixtures you might need. The function must have the signature: + * + * \code + * enum test_result_code func_(struct weston_test_harness *harness) + * \endcode + * + * The function must call one of weston_test_harness_execute_standalone(), + * weston_test_harness_execute_as_plugin() or + * weston_test_harness_execute_as_client() passing in the \c harness argument, + * and return the return value from that call. The function can also return a + * test_result_code on its own if it does not want to run the tests, + * e.g. RESULT_SKIP or RESULT_HARD_ERROR. + * + * The function will be called once to run all tests. + * + * \param func_ The function to be used as fixture setup. + * + * \ingroup testharness + */ +#define DECLARE_FIXTURE_SETUP(func_) \ + enum test_result_code \ + fixture_setup_run_(struct weston_test_harness *harness, \ + const void *arg_) \ + { \ + return func_(harness); \ + } + +/** Register a fixture setup function with a data array + * + * This registers the given (preferably static) function to be used for setting + * up any fixtures you might need. The function must have the signature: + * + * \code + * enum test_result_code func_(struct weston_test_harness *harness, typeof(array_[0]) *arg) + * \endcode + * + * The function must call one of weston_test_harness_execute_standalone(), + * weston_test_harness_execute_as_plugin() or + * weston_test_harness_execute_as_client() passing in the \c harness argument, + * and return the return value from that call. The function can also return a + * test_result_code on its own if it does not want to run the tests, + * e.g. RESULT_SKIP or RESULT_HARD_ERROR. + * + * The function will be called once with each element of the array pointed to + * by \c arg, so that all tests would be repeated for each element in turn. + * + * \param func_ The function to be used as fixture setup. + * \param array_ A static const array of arbitrary type. + * + * \ingroup testharness + */ +#define DECLARE_FIXTURE_SETUP_WITH_ARG(func_, array_) \ + const struct fixture_setup_array * \ + fixture_setup_array_get_(void) \ + { \ + static const struct fixture_setup_array arr = { \ + .array = array_, \ + .element_size = sizeof(array_[0]), \ + .n_elements = ARRAY_LENGTH(array_) \ + }; \ + return &arr; \ + } \ + \ + enum test_result_code \ + fixture_setup_run_(struct weston_test_harness *harness, \ + const void *arg_) \ + { \ + typeof(array_[0]) *arg = arg_; \ + return func_(harness, arg); \ + } + +enum test_result_code +weston_test_harness_execute_as_client(struct weston_test_harness *harness, + const struct compositor_setup *setup); + +enum test_result_code +weston_test_harness_execute_as_plugin(struct weston_test_harness *harness, + const struct compositor_setup *setup); + +enum test_result_code +weston_test_harness_execute_standalone(struct weston_test_harness *harness); + #endif diff --git a/tests/weston-test.c b/tests/weston-test.c index c404f281..f019b0a4 100644 --- a/tests/weston-test.c +++ b/tests/weston-test.c @@ -32,12 +32,17 @@ #include #include #include +#include +#include #include +#include #include "backend.h" #include "libweston-internal.h" #include "compositor/weston.h" #include "weston-test-server-protocol.h" +#include "weston.h" +#include "weston-testsuite-data.h" #include "shared/helpers.h" #include "shared/timespec-util.h" @@ -46,15 +51,19 @@ struct weston_test { struct weston_compositor *compositor; - /* XXX: missing compositor destroy listener - * https://gitlab.freedesktop.org/wayland/weston/issues/300 - */ + struct wl_listener destroy_listener; + + struct weston_log_scope *log; + struct weston_layer layer; struct weston_process process; struct weston_seat seat; struct weston_touch_device *touch_device[MAX_TOUCH_DEVICES]; int nr_touch_devices; bool is_seat_initialized; + + pthread_t client_thread; + struct wl_event_source *client_source; }; struct weston_test_surface { @@ -667,6 +676,174 @@ idle_launch_client(void *data) weston_watch_process(&test->process); } +static void +client_thread_cleanup(void *data_) +{ + struct wet_testsuite_data *data = data_; + + close(data->thread_event_pipe); + data->thread_event_pipe = -1; +} + +static void * +client_thread_routine(void *data_) +{ + struct wet_testsuite_data *data = data_; + + pthread_setname_np(pthread_self(), "client"); + pthread_cleanup_push(client_thread_cleanup, data); + data->run(data); + pthread_cleanup_pop(true); + + return NULL; +} + +static void +client_thread_join(struct weston_test *test) +{ + assert(test->client_source); + + pthread_join(test->client_thread, NULL); + wl_event_source_remove(test->client_source); + test->client_source = NULL; + + weston_log_scope_printf(test->log, "Test thread reaped.\n"); +} + +static int +handle_client_thread_event(int fd, uint32_t mask, void *data_) +{ + struct weston_test *test = data_; + + weston_log_scope_printf(test->log, + "Received thread event mask 0x%x\n", mask); + + if (mask != WL_EVENT_HANGUP) + weston_log("%s: unexpected event %u\n", __func__, mask); + + client_thread_join(test); + weston_compositor_exit(test->compositor); + + return 0; +} + +static int +create_client_thread(struct weston_test *test, struct wet_testsuite_data *data) +{ + struct wl_event_loop *loop; + int pipefd[2] = { -1, -1 }; + sigset_t saved; + sigset_t blocked; + int ret; + + weston_log_scope_printf(test->log, "Creating a thread for running tests...\n"); + + if (pipe2(pipefd, O_CLOEXEC | O_NONBLOCK) < 0) { + weston_log("Creating pipe for a client thread failed: %s\n", + strerror(errno)); + return -1; + } + + loop = wl_display_get_event_loop(test->compositor->wl_display); + test->client_source = wl_event_loop_add_fd(loop, pipefd[0], + WL_EVENT_READABLE, + handle_client_thread_event, + test); + close(pipefd[0]); + + if (!test->client_source) { + weston_log("Adding client thread fd to event loop failed.\n"); + goto out_pipe; + } + + data->thread_event_pipe = pipefd[1]; + + /* Ensure we don't accidentally get signals to the thread. */ + sigfillset(&blocked); + sigdelset(&blocked, SIGSEGV); + sigdelset(&blocked, SIGFPE); + sigdelset(&blocked, SIGILL); + sigdelset(&blocked, SIGCONT); + sigdelset(&blocked, SIGSYS); + if (pthread_sigmask(SIG_BLOCK, &blocked, &saved) != 0) + goto out_source; + + ret = pthread_create(&test->client_thread, NULL, + client_thread_routine, data); + + pthread_sigmask(SIG_SETMASK, &saved, NULL); + + if (ret != 0) { + weston_log("Creating client thread failed: %s (%d)\n", + strerror(ret), ret); + goto out_source; + } + + return 0; + +out_source: + data->thread_event_pipe = -1; + wl_event_source_remove(test->client_source); + test->client_source = NULL; + +out_pipe: + close(pipefd[1]); + + return -1; +} + +static void +idle_launch_testsuite(void *test_) +{ + struct weston_test *test = test_; + struct wet_testsuite_data *data = wet_testsuite_data_get(); + + if (!data) + return; + + switch (data->type) { + case TEST_TYPE_CLIENT: + if (create_client_thread(test, data) < 0) { + weston_log("Error: creating client thread for test suite failed.\n"); + weston_compositor_exit_with_code(test->compositor, + RESULT_HARD_ERROR); + } + break; + + case TEST_TYPE_PLUGIN: + data->compositor = test->compositor; + weston_log_scope_printf(test->log, + "Running tests from idle handler...\n"); + data->run(data); + weston_compositor_exit(test->compositor); + break; + + case TEST_TYPE_STANDALONE: + weston_log("Error: unknown test internal type %d.\n", + data->type); + weston_compositor_exit_with_code(test->compositor, + RESULT_HARD_ERROR); + } +} + +static void +handle_compositor_destroy(struct wl_listener *listener, + void *weston_compositor) +{ + struct weston_test *test; + + test = wl_container_of(listener, test, destroy_listener); + + if (test->client_source) { + weston_log_scope_printf(test->log, "Cancelling client thread...\n"); + pthread_cancel(test->client_thread); + client_thread_join(test); + } + + weston_log_scope_destroy(test->log); + test->log = NULL; +} + WL_EXPORT int wet_module_init(struct weston_compositor *ec, int *argc, char *argv[]) @@ -678,19 +855,36 @@ wet_module_init(struct weston_compositor *ec, if (test == NULL) return -1; + if (!weston_compositor_add_destroy_listener_once(ec, + &test->destroy_listener, + handle_compositor_destroy)) { + free(test); + return 0; + } + test->compositor = ec; weston_layer_init(&test->layer, ec); weston_layer_set_position(&test->layer, WESTON_LAYER_POSITION_CURSOR - 1); + test->log = weston_compositor_add_log_scope(ec, "test-harness-plugin", + "weston-test plugin's own actions", + NULL, NULL, NULL); + if (wl_global_create(ec->wl_display, &weston_test_interface, 1, test, bind_test) == NULL) - return -1; + goto out_free; if (test_seat_init(test) == -1) - return -1; + goto out_free; loop = wl_display_get_event_loop(ec->wl_display); wl_event_loop_add_idle(loop, idle_launch_client, test); + wl_event_loop_add_idle(loop, idle_launch_testsuite, test); return 0; + +out_free: + wl_list_remove(&test->destroy_listener.link); + free(test); + return -1; } diff --git a/tests/weston-testsuite-data.h b/tests/weston-testsuite-data.h new file mode 100644 index 00000000..06c35bd7 --- /dev/null +++ b/tests/weston-testsuite-data.h @@ -0,0 +1,88 @@ +/* + * Copyright 2019 Collabora, Ltd. + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef WESTON_TESTSUITE_DATA_H +#define WESTON_TESTSUITE_DATA_H + +/** Standard return codes + * + * Both Autotools and Meson use these codes as test program exit codes + * to denote the test result for the whole process. + * + * \ingroup testharness + */ +enum test_result_code { + RESULT_OK = 0, + RESULT_SKIP = 77, + RESULT_FAIL = 1, + RESULT_HARD_ERROR = 99, +}; + +struct weston_test; +struct weston_compositor; + +/** Weston test types + * + * \sa weston_test_harness_execute_standalone + * weston_test_harness_execute_as_plugin + * weston_test_harness_execute_as_client + * + * \ingroup testharness_private + */ +enum test_type { + TEST_TYPE_STANDALONE, + TEST_TYPE_PLUGIN, + TEST_TYPE_CLIENT, +}; + +/** Test harness specific data for running tests + * + * \ingroup testharness_private + */ +struct wet_testsuite_data { + void (*run)(struct wet_testsuite_data *); + + /* test definitions */ + const struct weston_test_entry *tests; + unsigned tests_count; + int case_index; + enum test_type type; + struct weston_compositor *compositor; + + /* client thread control */ + int thread_event_pipe; + + /* informational run state */ + int fixture_iteration; + + /* test counts */ + unsigned counter; + unsigned passed; + unsigned skipped; + unsigned failed; + unsigned total; +}; + +#endif /* WESTON_TESTSUITE_DATA_H */