From 7ec1edde9eba46964cd2e8fca2c004b2021e79f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Fri, 12 Jun 2020 20:02:20 +0100 Subject: [PATCH 01/21] tests/docker: bump fedora to 32 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We should be keeping this up to date as Fedora goes out of support quite quickly. Signed-off-by: Alex Bennée Reviewed-by: Richard Henderson Message-Id: <20200612190237.30436-2-alex.bennee@linaro.org> --- tests/docker/dockerfiles/fedora.docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index 92b6e11c8a..798ddd2c3e 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -1,4 +1,4 @@ -FROM fedora:30 +FROM fedora:32 # Please keep this list sorted alphabetically ENV PACKAGES \ From 67953a379ea5497e7cffda14b9362c673845abbb Mon Sep 17 00:00:00 2001 From: Claudio Fontana Date: Fri, 12 Jun 2020 20:02:21 +0100 Subject: [PATCH 02/21] Makefile: dtc: update, build the libfdt target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dtc submodule update, now call the libfdt target from the new dtc Makefile, which has been changed to not require bison, flex, etc. This removes warnings during the build. scripts/ symlink and tests directory creation are not necessary, and neither is calling the clean rule explicitly. Signed-off-by: Claudio Fontana Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Message-Id: <20200518160319.18861-2-cfontana@suse.de> Message-Id: <20200612190237.30436-3-alex.bennee@linaro.org> --- Makefile | 10 +++++----- configure | 1 - dtc | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index ed0ed93b2d..895410fbf9 100644 --- a/Makefile +++ b/Makefile @@ -526,13 +526,14 @@ $(SOFTMMU_FUZZ_RULES): $(edk2-decompressed) $(TARGET_DIRS_RULES): $(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C $(dir $@) V="$(V)" TARGET_DIR="$(dir $@)" $(notdir $@),) -DTC_MAKE_ARGS=-I$(SRC_PATH)/dtc VPATH=$(SRC_PATH)/dtc -C dtc V="$(V)" LIBFDT_srcdir=$(SRC_PATH)/dtc/libfdt +# LIBFDT_lib="": avoid breaking existing trees with objects requiring -fPIC +DTC_MAKE_ARGS=-I$(SRC_PATH)/dtc VPATH=$(SRC_PATH)/dtc -C dtc V="$(V)" LIBFDT_lib="" DTC_CFLAGS=$(CFLAGS) $(QEMU_CFLAGS) -DTC_CPPFLAGS=-I$(BUILD_DIR)/dtc -I$(SRC_PATH)/dtc -I$(SRC_PATH)/dtc/libfdt +DTC_CPPFLAGS=-I$(SRC_PATH)/dtc/libfdt .PHONY: dtc/all -dtc/all: .git-submodule-status dtc/libfdt dtc/tests - $(call quiet-command,$(MAKE) $(DTC_MAKE_ARGS) CPPFLAGS="$(DTC_CPPFLAGS)" CFLAGS="$(DTC_CFLAGS)" LDFLAGS="$(QEMU_LDFLAGS)" ARFLAGS="$(ARFLAGS)" CC="$(CC)" AR="$(AR)" LD="$(LD)" $(SUBDIR_MAKEFLAGS) libfdt/libfdt.a,) +dtc/all: .git-submodule-status dtc/libfdt + $(call quiet-command,$(MAKE) $(DTC_MAKE_ARGS) CPPFLAGS="$(DTC_CPPFLAGS)" CFLAGS="$(DTC_CFLAGS)" LDFLAGS="$(QEMU_LDFLAGS)" ARFLAGS="$(ARFLAGS)" CC="$(CC)" AR="$(AR)" LD="$(LD)" $(SUBDIR_MAKEFLAGS) libfdt,) dtc/%: .git-submodule-status @mkdir -p $@ @@ -820,7 +821,6 @@ distclean: clean rm -rf $$d || exit 1 ; \ done rm -Rf .sdk - if test -f dtc/version_gen.h; then $(MAKE) $(DTC_MAKE_ARGS) clean; fi KEYMAPS=da en-gb et fr fr-ch is lt no pt-br sv \ ar de en-us fi fr-be hr it lv nl pl ru th \ diff --git a/configure b/configure index 89867a1720..8a9f544b0b 100755 --- a/configure +++ b/configure @@ -4312,7 +4312,6 @@ EOF mkdir -p dtc if [ "$pwd_is_source_path" != "y" ] ; then symlink "$source_path/dtc/Makefile" "dtc/Makefile" - symlink "$source_path/dtc/scripts" "dtc/scripts" fi fdt_cflags="-I\$(SRC_PATH)/dtc/libfdt" fdt_ldflags="-L\$(BUILD_DIR)/dtc/libfdt" diff --git a/dtc b/dtc index 88f18909db..85e5d83984 160000 --- a/dtc +++ b/dtc @@ -1 +1 @@ -Subproject commit 88f18909db731a627456f26d779445f84e449536 +Subproject commit 85e5d839847af54efab170f2b1331b2a6421e647 From 897e34f20f2196089ad4910d95a7c74ee14292c8 Mon Sep 17 00:00:00 2001 From: Claudio Fontana Date: Fri, 12 Jun 2020 20:02:22 +0100 Subject: [PATCH 03/21] Makefile: remove old compatibility gunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Claudio Fontana Reviewed-by: Markus Armbruster Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Message-Id: <20200518160319.18861-3-cfontana@suse.de> Message-Id: <20200612190237.30436-4-alex.bennee@linaro.org> --- Makefile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Makefile b/Makefile index 895410fbf9..48f23aa978 100644 --- a/Makefile +++ b/Makefile @@ -562,12 +562,6 @@ slirp/all: .git-submodule-status CC="$(CC)" AR="$(AR)" LD="$(LD)" RANLIB="$(RANLIB)" \ CFLAGS="$(QEMU_CFLAGS) $(CFLAGS)" LDFLAGS="$(QEMU_LDFLAGS)") -# Compatibility gunk to keep make working across the rename of targets -# for recursion, to be removed some time after 4.1. -subdir-dtc: dtc/all -subdir-capstone: capstone/all -subdir-slirp: slirp/all - $(filter %/all, $(TARGET_DIRS_RULES)): libqemuutil.a $(common-obj-y) \ $(qom-obj-y) From 0aebab04b9289bd37017593b413ce7a762b54c55 Mon Sep 17 00:00:00 2001 From: Lingfeng Yang Date: Fri, 12 Jun 2020 20:02:23 +0100 Subject: [PATCH 04/21] configure: add --enable-tsan flag + fiber annotations for coroutine-ucontext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We tried running QEMU under tsan in 2016, but tsan's lack of support for longjmp-based fibers was a blocker: https://groups.google.com/forum/#!topic/thread-sanitizer/se0YuzfWazw Fortunately, thread sanitizer gained fiber support in early 2019: https://reviews.llvm.org/D54889 This patch brings tsan support upstream by importing the patch that annotated QEMU's coroutines as tsan fibers in Android's QEMU fork: https://android-review.googlesource.com/c/platform/external/qemu/+/844675 Tested with '--enable-tsan --cc=clang-9 --cxx=clang++-9 --disable-werror' configure flags. Signed-off-by: Lingfeng Yang Signed-off-by: Emilio G. Cota [cota: minor modifications + configure changes] Signed-off-by: Robert Foley [RF: configure changes, coroutine fix + minor modifications] Reviewed-by: Alex Bennée Signed-off-by: Alex Bennée Message-Id: <20200609200738.445-2-robert.foley@linaro.org> Message-Id: <20200612190237.30436-5-alex.bennee@linaro.org> --- configure | 47 +++++++++++++++++++++++++++- util/coroutine-ucontext.c | 66 +++++++++++++++++++++++++++++++++------ 2 files changed, 103 insertions(+), 10 deletions(-) diff --git a/configure b/configure index 8a9f544b0b..b01b5e3bed 100755 --- a/configure +++ b/configure @@ -395,6 +395,7 @@ gprof="no" debug_tcg="no" debug="no" sanitizers="no" +tsan="no" fortify_source="" strip_opt="yes" tcg_interpreter="no" @@ -1152,6 +1153,10 @@ for opt do ;; --disable-sanitizers) sanitizers="no" ;; + --enable-tsan) tsan="yes" + ;; + --disable-tsan) tsan="no" + ;; --enable-sparse) sparse="yes" ;; --disable-sparse) sparse="no" @@ -1764,6 +1769,7 @@ Advanced options (experts only): --with-pkgversion=VERS use specified string as sub-version of the package --enable-debug enable common debug build options --enable-sanitizers enable default sanitizers + --enable-tsan enable thread sanitizer --disable-strip disable stripping binaries --disable-werror disable compilation abort on warning --disable-stack-protector disable compiler-provided stack protection @@ -6220,6 +6226,30 @@ if test "$fuzzing" = "yes" ; then fi fi +# Thread sanitizer is, for now, much noisier than the other sanitizers; +# keep it separate until that is not the case. +if test "$tsan" = "yes" && test "$sanitizers" = "yes"; then + error_exit "TSAN is not supported with other sanitiziers." +fi +have_tsan=no +have_tsan_iface_fiber=no +if test "$tsan" = "yes" ; then + write_c_skeleton + if compile_prog "$CPU_CFLAGS -Werror -fsanitize=thread" "" ; then + have_tsan=yes + fi + cat > $TMPC << EOF +#include +int main(void) { + __tsan_create_fiber(0); + return 0; +} +EOF + if compile_prog "$CPU_CFLAGS -Werror -fsanitize=thread" "" ; then + have_tsan_iface_fiber=yes + fi +fi + ########################################## # check for libpmem @@ -6377,6 +6407,16 @@ if test "$have_asan" = "yes"; then "Without code annotation, the report may be inferior." fi fi +if test "$have_tsan" = "yes" ; then + if test "$have_tsan_iface_fiber" = "yes" ; then + QEMU_CFLAGS="-fsanitize=thread $QEMU_CFLAGS" + QEMU_LDFLAGS="-fsanitize=thread $QEMU_LDFLAGS" + else + error_exit "Cannot enable TSAN due to missing fiber annotation interface." + fi +elif test "$tsan" = "yes" ; then + error_exit "Cannot enable TSAN due to missing sanitize thread interface." +fi if test "$have_ubsan" = "yes"; then QEMU_CFLAGS="-fsanitize=undefined $QEMU_CFLAGS" QEMU_LDFLAGS="-fsanitize=undefined $QEMU_LDFLAGS" @@ -6412,7 +6452,8 @@ if test "$werror" = "yes"; then QEMU_CFLAGS="-Werror $QEMU_CFLAGS" fi -if test "$solaris" = "no" ; then +# Exclude --warn-common with TSan to suppress warnings from the TSan libraries. +if test "$solaris" = "no" && test "$tsan" = "no"; then if $ld --version 2>/dev/null | grep "GNU ld" >/dev/null 2>/dev/null ; then QEMU_LDFLAGS="-Wl,--warn-common $QEMU_LDFLAGS" fi @@ -7476,6 +7517,10 @@ if test "$have_asan_iface_fiber" = "yes" ; then echo "CONFIG_ASAN_IFACE_FIBER=y" >> $config_host_mak fi +if test "$have_tsan" = "yes" && test "$have_tsan_iface_fiber" = "yes" ; then + echo "CONFIG_TSAN=y" >> $config_host_mak +fi + if test "$has_environ" = "yes" ; then echo "CONFIG_HAS_ENVIRON=y" >> $config_host_mak fi diff --git a/util/coroutine-ucontext.c b/util/coroutine-ucontext.c index bd593e61bc..613f4c118e 100644 --- a/util/coroutine-ucontext.c +++ b/util/coroutine-ucontext.c @@ -37,12 +37,19 @@ #endif #endif +#ifdef CONFIG_TSAN +#include +#endif + typedef struct { Coroutine base; void *stack; size_t stack_size; sigjmp_buf env; + void *tsan_co_fiber; + void *tsan_caller_fiber; + #ifdef CONFIG_VALGRIND_H unsigned int valgrind_stack_id; #endif @@ -65,7 +72,18 @@ union cc_arg { int i[2]; }; -static void finish_switch_fiber(void *fake_stack_save) +/* QEMU_ALWAYS_INLINE only does so if __OPTIMIZE__, so we cannot use it. */ +static inline __attribute__((always_inline)) +void on_new_fiber(CoroutineUContext *co) +{ +#ifdef CONFIG_TSAN + co->tsan_co_fiber = __tsan_create_fiber(0); /* flags: sync on switch */ + co->tsan_caller_fiber = __tsan_get_current_fiber(); +#endif +} + +static inline __attribute__((always_inline)) +void finish_switch_fiber(void *fake_stack_save) { #ifdef CONFIG_ASAN const void *bottom_old; @@ -78,13 +96,30 @@ static void finish_switch_fiber(void *fake_stack_save) leader.stack_size = size_old; } #endif +#ifdef CONFIG_TSAN + if (fake_stack_save) { + __tsan_release(fake_stack_save); + __tsan_switch_to_fiber(fake_stack_save, 0); /* 0=synchronize */ + } +#endif } -static void start_switch_fiber(void **fake_stack_save, - const void *bottom, size_t size) +static inline __attribute__((always_inline)) void start_switch_fiber( + CoroutineAction action, void **fake_stack_save, + const void *bottom, size_t size, void *new_fiber) { #ifdef CONFIG_ASAN - __sanitizer_start_switch_fiber(fake_stack_save, bottom, size); + __sanitizer_start_switch_fiber( + action == COROUTINE_TERMINATE ? NULL : fake_stack_save, + bottom, size); +#endif +#ifdef CONFIG_TSAN + void *curr_fiber = + __tsan_get_current_fiber(); + __tsan_acquire(curr_fiber); + + *fake_stack_save = curr_fiber; + __tsan_switch_to_fiber(new_fiber, 0); /* 0=synchronize */ #endif } @@ -104,8 +139,12 @@ static void coroutine_trampoline(int i0, int i1) /* Initialize longjmp environment and switch back the caller */ if (!sigsetjmp(self->env, 0)) { - start_switch_fiber(&fake_stack_save, - leader.stack, leader.stack_size); + start_switch_fiber( + COROUTINE_YIELD, + &fake_stack_save, + leader.stack, + leader.stack_size, + self->tsan_caller_fiber); siglongjmp(*(sigjmp_buf *)co->entry_arg, 1); } @@ -154,12 +193,16 @@ Coroutine *qemu_coroutine_new(void) arg.p = co; + on_new_fiber(co); makecontext(&uc, (void (*)(void))coroutine_trampoline, 2, arg.i[0], arg.i[1]); /* swapcontext() in, siglongjmp() back out */ if (!sigsetjmp(old_env, 0)) { - start_switch_fiber(&fake_stack_save, co->stack, co->stack_size); + start_switch_fiber( + COROUTINE_YIELD, + &fake_stack_save, + co->stack, co->stack_size, co->tsan_co_fiber); swapcontext(&old_uc, &uc); } @@ -216,8 +259,8 @@ qemu_coroutine_switch(Coroutine *from_, Coroutine *to_, ret = sigsetjmp(from->env, 0); if (ret == 0) { - start_switch_fiber(action == COROUTINE_TERMINATE ? - NULL : &fake_stack_save, to->stack, to->stack_size); + start_switch_fiber(action, &fake_stack_save, + to->stack, to->stack_size, to->tsan_co_fiber); siglongjmp(to->env, action); } @@ -231,6 +274,11 @@ Coroutine *qemu_coroutine_self(void) if (!current) { current = &leader.base; } +#ifdef CONFIG_TSAN + if (!leader.tsan_co_fiber) { + leader.tsan_co_fiber = __tsan_get_current_fiber(); + } +#endif return current; } From 0c0fcc2052915c629daa2469bfae620da18b2394 Mon Sep 17 00:00:00 2001 From: "Emilio G. Cota" Date: Fri, 12 Jun 2020 20:02:24 +0100 Subject: [PATCH 05/21] cpu: convert queued work to a QSIMPLEQ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We convert queued work to a QSIMPLEQ, instead of open-coding it. While at it, make sure that all accesses to the list are performed while holding the list's lock. Reviewed-by: Richard Henderson Reviewed-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Emilio G. Cota Signed-off-by: Robert Foley Signed-off-by: Alex Bennée Message-Id: <20200609200738.445-3-robert.foley@linaro.org> Message-Id: <20200612190237.30436-6-alex.bennee@linaro.org> --- cpus-common.c | 25 ++++++++----------------- cpus.c | 14 ++++++++++++-- hw/core/cpu.c | 1 + include/hw/core/cpu.h | 6 +++--- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/cpus-common.c b/cpus-common.c index 70a9d12981..8f5512b3d7 100644 --- a/cpus-common.c +++ b/cpus-common.c @@ -97,7 +97,7 @@ void cpu_list_remove(CPUState *cpu) } struct qemu_work_item { - struct qemu_work_item *next; + QSIMPLEQ_ENTRY(qemu_work_item) node; run_on_cpu_func func; run_on_cpu_data data; bool free, exclusive, done; @@ -106,13 +106,7 @@ struct qemu_work_item { static void queue_work_on_cpu(CPUState *cpu, struct qemu_work_item *wi) { qemu_mutex_lock(&cpu->work_mutex); - if (cpu->queued_work_first == NULL) { - cpu->queued_work_first = wi; - } else { - cpu->queued_work_last->next = wi; - } - cpu->queued_work_last = wi; - wi->next = NULL; + QSIMPLEQ_INSERT_TAIL(&cpu->work_list, wi, node); wi->done = false; qemu_mutex_unlock(&cpu->work_mutex); @@ -306,17 +300,14 @@ void process_queued_cpu_work(CPUState *cpu) { struct qemu_work_item *wi; - if (cpu->queued_work_first == NULL) { + qemu_mutex_lock(&cpu->work_mutex); + if (QSIMPLEQ_EMPTY(&cpu->work_list)) { + qemu_mutex_unlock(&cpu->work_mutex); return; } - - qemu_mutex_lock(&cpu->work_mutex); - while (cpu->queued_work_first != NULL) { - wi = cpu->queued_work_first; - cpu->queued_work_first = wi->next; - if (!cpu->queued_work_first) { - cpu->queued_work_last = NULL; - } + while (!QSIMPLEQ_EMPTY(&cpu->work_list)) { + wi = QSIMPLEQ_FIRST(&cpu->work_list); + QSIMPLEQ_REMOVE_HEAD(&cpu->work_list, node); qemu_mutex_unlock(&cpu->work_mutex); if (wi->exclusive) { /* Running work items outside the BQL avoids the following deadlock: diff --git a/cpus.c b/cpus.c index 34fc203808..7317ae06b9 100644 --- a/cpus.c +++ b/cpus.c @@ -97,9 +97,19 @@ bool cpu_is_stopped(CPUState *cpu) return cpu->stopped || !runstate_is_running(); } +static inline bool cpu_work_list_empty(CPUState *cpu) +{ + bool ret; + + qemu_mutex_lock(&cpu->work_mutex); + ret = QSIMPLEQ_EMPTY(&cpu->work_list); + qemu_mutex_unlock(&cpu->work_mutex); + return ret; +} + static bool cpu_thread_is_idle(CPUState *cpu) { - if (cpu->stop || cpu->queued_work_first) { + if (cpu->stop || !cpu_work_list_empty(cpu)) { return false; } if (cpu_is_stopped(cpu)) { @@ -1518,7 +1528,7 @@ static void *qemu_tcg_rr_cpu_thread_fn(void *arg) cpu = first_cpu; } - while (cpu && !cpu->queued_work_first && !cpu->exit_request) { + while (cpu && cpu_work_list_empty(cpu) && !cpu->exit_request) { atomic_mb_set(&tcg_current_rr_cpu, cpu); current_cpu = cpu; diff --git a/hw/core/cpu.c b/hw/core/cpu.c index f31ec48ee6..80d51c24dd 100644 --- a/hw/core/cpu.c +++ b/hw/core/cpu.c @@ -370,6 +370,7 @@ static void cpu_common_initfn(Object *obj) cpu->nr_threads = 1; qemu_mutex_init(&cpu->work_mutex); + QSIMPLEQ_INIT(&cpu->work_list); QTAILQ_INIT(&cpu->breakpoints); QTAILQ_INIT(&cpu->watchpoints); diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h index 497600c49e..b3f4b79318 100644 --- a/include/hw/core/cpu.h +++ b/include/hw/core/cpu.h @@ -331,8 +331,8 @@ struct qemu_work_item; * @opaque: User data. * @mem_io_pc: Host Program Counter at which the memory was accessed. * @kvm_fd: vCPU file descriptor for KVM. - * @work_mutex: Lock to prevent multiple access to queued_work_*. - * @queued_work_first: First asynchronous work pending. + * @work_mutex: Lock to prevent multiple access to @work_list. + * @work_list: List of pending asynchronous work. * @trace_dstate_delayed: Delayed changes to trace_dstate (includes all changes * to @trace_dstate). * @trace_dstate: Dynamic tracing state of events for this vCPU (bitmask). @@ -376,7 +376,7 @@ struct CPUState { sigjmp_buf jmp_env; QemuMutex work_mutex; - struct qemu_work_item *queued_work_first, *queued_work_last; + QSIMPLEQ_HEAD(, qemu_work_item) work_list; CPUAddressSpace *cpu_ases; int num_ases; From 4384a70d0188a356d85a80c62ae88ec5aef8b3ee Mon Sep 17 00:00:00 2001 From: "Emilio G. Cota" Date: Fri, 12 Jun 2020 20:02:25 +0100 Subject: [PATCH 06/21] thread: add qemu_spin_destroy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It will be used for TSAN annotations. Signed-off-by: Emilio G. Cota Signed-off-by: Robert Foley Reviewed-by: Alex Bennée Signed-off-by: Alex Bennée Message-Id: <20200609200738.445-4-robert.foley@linaro.org> Message-Id: <20200612190237.30436-7-alex.bennee@linaro.org> --- include/qemu/thread.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/qemu/thread.h b/include/qemu/thread.h index 06c058fb58..9479facdcc 100644 --- a/include/qemu/thread.h +++ b/include/qemu/thread.h @@ -215,6 +215,9 @@ static inline void qemu_spin_init(QemuSpin *spin) __sync_lock_release(&spin->value); } +static inline void qemu_spin_destroy(QemuSpin *spin) +{ } + static inline void qemu_spin_lock(QemuSpin *spin) { while (unlikely(__sync_lock_test_and_set(&spin->value, true))) { From 816d9be5ea31d848ee090eca4a2bf185bd609066 Mon Sep 17 00:00:00 2001 From: "Emilio G. Cota" Date: Fri, 12 Jun 2020 20:02:26 +0100 Subject: [PATCH 07/21] cputlb: destroy CPUTLB with tlb_destroy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I was after adding qemu_spin_destroy calls, but while at it I noticed that we are leaking some memory. Signed-off-by: Emilio G. Cota Signed-off-by: Robert Foley Reviewed-by: Alex Bennée Signed-off-by: Alex Bennée Message-Id: <20200609200738.445-5-robert.foley@linaro.org> Message-Id: <20200612190237.30436-8-alex.bennee@linaro.org> --- accel/tcg/cputlb.c | 15 +++++++++++++++ exec.c | 1 + include/exec/exec-all.h | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index eb2cf9de5e..1e815357c7 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -270,6 +270,21 @@ void tlb_init(CPUState *cpu) } } +void tlb_destroy(CPUState *cpu) +{ + CPUArchState *env = cpu->env_ptr; + int i; + + qemu_spin_destroy(&env_tlb(env)->c.lock); + for (i = 0; i < NB_MMU_MODES; i++) { + CPUTLBDesc *desc = &env_tlb(env)->d[i]; + CPUTLBDescFast *fast = &env_tlb(env)->f[i]; + + g_free(fast->table); + g_free(desc->iotlb); + } +} + /* flush_all_helper: run fn across all cpus * * If the wait flag is set then the src cpu's helper will be queued as diff --git a/exec.c b/exec.c index 9c8f558590..d6712fba7e 100644 --- a/exec.c +++ b/exec.c @@ -892,6 +892,7 @@ void cpu_exec_unrealizefn(CPUState *cpu) { CPUClass *cc = CPU_GET_CLASS(cpu); + tlb_destroy(cpu); cpu_list_remove(cpu); if (cc->vmsd != NULL) { diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 8792bea07a..3cf88272df 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -124,6 +124,11 @@ void cpu_address_space_init(CPUState *cpu, int asidx, * @cpu: CPU whose TLB should be initialized */ void tlb_init(CPUState *cpu); +/** + * tlb_destroy - destroy a CPU's TLB + * @cpu: CPU whose TLB should be destroyed + */ +void tlb_destroy(CPUState *cpu); /** * tlb_flush_page: * @cpu: CPU whose TLB should be flushed @@ -284,6 +289,9 @@ void tlb_set_page(CPUState *cpu, target_ulong vaddr, static inline void tlb_init(CPUState *cpu) { } +static inline void tlb_destroy(CPUState *cpu) +{ +} static inline void tlb_flush_page(CPUState *cpu, target_ulong addr) { } From 5107a47bb2bf3a7492db5f86cf02302f170a0bf0 Mon Sep 17 00:00:00 2001 From: "Emilio G. Cota" Date: Fri, 12 Jun 2020 20:02:27 +0100 Subject: [PATCH 08/21] qht: call qemu_spin_destroy for head buckets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Emilio G. Cota Signed-off-by: Robert Foley Reviewed-by: Alex Bennée [AJB: add implied cota s-o-b c.f. github.com/cota/qemu/tree/tsan @ 1bd1209] Signed-off-by: Alex Bennée Message-Id: <20200609200738.445-6-robert.foley@linaro.org> Message-Id: <20200612190237.30436-9-alex.bennee@linaro.org> --- util/qht.c | 1 + 1 file changed, 1 insertion(+) diff --git a/util/qht.c b/util/qht.c index aa51be3c52..67e5d5b916 100644 --- a/util/qht.c +++ b/util/qht.c @@ -348,6 +348,7 @@ static inline void qht_chain_destroy(const struct qht_bucket *head) struct qht_bucket *curr = head->next; struct qht_bucket *prev; + qemu_spin_destroy(&head->lock); while (curr) { prev = curr; curr = curr->next; From 938e897a6615b63093881e8b58206f6f69212abe Mon Sep 17 00:00:00 2001 From: "Emilio G. Cota" Date: Fri, 12 Jun 2020 20:02:28 +0100 Subject: [PATCH 09/21] tcg: call qemu_spin_destroy for tb->jmp_lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Emilio G. Cota Signed-off-by: Robert Foley Signed-off-by: Alex Bennée [RF: minor changes + remove tb_destroy_func] Message-Id: <20200609200738.445-7-robert.foley@linaro.org> Message-Id: <20200612190237.30436-10-alex.bennee@linaro.org> --- accel/tcg/translate-all.c | 8 ++++++++ include/tcg/tcg.h | 1 + tcg/tcg.c | 9 +++++++++ 3 files changed, 18 insertions(+) diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index 42ce1dfcff..c937210e21 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -384,6 +384,11 @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, return 0; } +void tb_destroy(TranslationBlock *tb) +{ + qemu_spin_destroy(&tb->jmp_lock); +} + bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) { TranslationBlock *tb; @@ -413,6 +418,7 @@ bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) /* one-shot translation, invalidate it immediately */ tb_phys_invalidate(tb, -1); tcg_tb_remove(tb); + tb_destroy(tb); } r = true; } @@ -1886,6 +1892,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, orig_aligned -= ROUND_UP(sizeof(*tb), qemu_icache_linesize); atomic_set(&tcg_ctx->code_gen_ptr, (void *)orig_aligned); + tb_destroy(tb); return existing_tb; } tcg_tb_insert(tb); @@ -2235,6 +2242,7 @@ void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) tb_phys_invalidate(tb->orig_tb, -1); } tcg_tb_remove(tb); + tb_destroy(tb); } /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index 380014ed80..e63450a893 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -819,6 +819,7 @@ void tcg_pool_reset(TCGContext *s); TranslationBlock *tcg_tb_alloc(TCGContext *s); void tcg_region_init(void); +void tb_destroy(TranslationBlock *tb); void tcg_region_reset_all(void); size_t tcg_code_size(void); diff --git a/tcg/tcg.c b/tcg/tcg.c index 1aa6cb47f2..1362bc6101 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -502,6 +502,14 @@ size_t tcg_nb_tbs(void) return nb_tbs; } +static gboolean tcg_region_tree_traverse(gpointer k, gpointer v, gpointer data) +{ + TranslationBlock *tb = v; + + tb_destroy(tb); + return FALSE; +} + static void tcg_region_tree_reset_all(void) { size_t i; @@ -510,6 +518,7 @@ static void tcg_region_tree_reset_all(void) for (i = 0; i < region.n; i++) { struct tcg_region_tree *rt = region_trees + i * tree_size; + g_tree_foreach(rt->tree, tcg_region_tree_traverse, NULL); /* Increment the refcount first so that destroy acts as a reset */ g_tree_ref(rt->tree); g_tree_destroy(rt->tree); From 3f640eb8812f92dfcc4947fc1e7cd9004456d58a Mon Sep 17 00:00:00 2001 From: "Emilio G. Cota" Date: Fri, 12 Jun 2020 20:02:29 +0100 Subject: [PATCH 10/21] translate-all: call qemu_spin_destroy for PageDesc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The radix tree is append-only, but we can fail to insert a PageDesc if the insertion races with another thread. Signed-off-by: Emilio G. Cota Signed-off-by: Robert Foley Reviewed-by: Alex Bennée Signed-off-by: Alex Bennée Message-Id: <20200609200738.445-8-robert.foley@linaro.org> Message-Id: <20200612190237.30436-11-alex.bennee@linaro.org> --- accel/tcg/translate-all.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index c937210e21..c3d37058a1 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -547,6 +547,15 @@ static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) #endif existing = atomic_cmpxchg(lp, NULL, pd); if (unlikely(existing)) { +#ifndef CONFIG_USER_ONLY + { + int i; + + for (i = 0; i < V_L2_SIZE; i++) { + qemu_spin_destroy(&pd[i].lock); + } + } +#endif g_free(pd); pd = existing; } From 45a9595a1b16666cdac1969469937b72cfb28129 Mon Sep 17 00:00:00 2001 From: "Emilio G. Cota" Date: Fri, 12 Jun 2020 20:02:30 +0100 Subject: [PATCH 11/21] thread: add tsan annotations to QemuSpin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Emilio G. Cota Signed-off-by: Robert Foley Reviewed-by: Alex Bennée Signed-off-by: Alex Bennée Message-Id: <20200609200738.445-9-robert.foley@linaro.org> Message-Id: <20200612190237.30436-12-alex.bennee@linaro.org> --- include/qemu/thread.h | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/include/qemu/thread.h b/include/qemu/thread.h index 9479facdcc..4baf4d1715 100644 --- a/include/qemu/thread.h +++ b/include/qemu/thread.h @@ -206,6 +206,10 @@ void qemu_thread_atexit_add(struct Notifier *notifier); */ void qemu_thread_atexit_remove(struct Notifier *notifier); +#ifdef CONFIG_TSAN +#include +#endif + struct QemuSpin { int value; }; @@ -213,23 +217,46 @@ struct QemuSpin { static inline void qemu_spin_init(QemuSpin *spin) { __sync_lock_release(&spin->value); +#ifdef CONFIG_TSAN + __tsan_mutex_create(spin, __tsan_mutex_not_static); +#endif } -static inline void qemu_spin_destroy(QemuSpin *spin) -{ } +/* const parameter because the only purpose here is the TSAN annotation */ +static inline void qemu_spin_destroy(const QemuSpin *spin) +{ +#ifdef CONFIG_TSAN + __tsan_mutex_destroy((void *)spin, __tsan_mutex_not_static); +#endif +} static inline void qemu_spin_lock(QemuSpin *spin) { +#ifdef CONFIG_TSAN + __tsan_mutex_pre_lock(spin, 0); +#endif while (unlikely(__sync_lock_test_and_set(&spin->value, true))) { while (atomic_read(&spin->value)) { cpu_relax(); } } +#ifdef CONFIG_TSAN + __tsan_mutex_post_lock(spin, 0, 0); +#endif } static inline bool qemu_spin_trylock(QemuSpin *spin) { - return __sync_lock_test_and_set(&spin->value, true); +#ifdef CONFIG_TSAN + __tsan_mutex_pre_lock(spin, __tsan_mutex_try_lock); +#endif + bool busy = __sync_lock_test_and_set(&spin->value, true); +#ifdef CONFIG_TSAN + unsigned flags = __tsan_mutex_try_lock; + flags |= busy ? __tsan_mutex_try_lock_failed : 0; + __tsan_mutex_post_lock(spin, flags, 0); +#endif + return busy; } static inline bool qemu_spin_locked(QemuSpin *spin) @@ -239,7 +266,13 @@ static inline bool qemu_spin_locked(QemuSpin *spin) static inline void qemu_spin_unlock(QemuSpin *spin) { +#ifdef CONFIG_TSAN + __tsan_mutex_pre_unlock(spin, 0); +#endif __sync_lock_release(&spin->value); +#ifdef CONFIG_TSAN + __tsan_mutex_post_unlock(spin, 0); +#endif } struct QemuLockCnt { From df79fd5667fb99c6fdfa0d295cb020e9e3eef1e7 Mon Sep 17 00:00:00 2001 From: Robert Foley Date: Fri, 12 Jun 2020 20:02:31 +0100 Subject: [PATCH 12/21] tests/docker: Added docker build support for TSan. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added a new docker for ubuntu 20.04. This docker has support for Thread Sanitizer including one patch we need in one of the header files. https://github.com/llvm/llvm-project/commit/a72dc86cd This command will build with tsan enabled: make docker-test-tsan-ubuntu2004 V=1 Also added the TSAN suppresion file to disable certain cases of TSAN warnings. Cc: Fam Zheng Cc: Philippe Mathieu-Daudé Signed-off-by: Robert Foley Reviewed-by: Alex Bennée Signed-off-by: Alex Bennée Message-Id: <20200609200738.445-10-robert.foley@linaro.org> Message-Id: <20200612190237.30436-13-alex.bennee@linaro.org> --- tests/docker/dockerfiles/ubuntu2004.docker | 65 ++++++++++++++++++++++ tests/docker/test-tsan | 44 +++++++++++++++ tests/tsan/blacklist.tsan | 10 ++++ tests/tsan/suppressions.tsan | 14 +++++ 4 files changed, 133 insertions(+) create mode 100644 tests/docker/dockerfiles/ubuntu2004.docker create mode 100755 tests/docker/test-tsan create mode 100644 tests/tsan/blacklist.tsan create mode 100644 tests/tsan/suppressions.tsan diff --git a/tests/docker/dockerfiles/ubuntu2004.docker b/tests/docker/dockerfiles/ubuntu2004.docker new file mode 100644 index 0000000000..6050ce7e8a --- /dev/null +++ b/tests/docker/dockerfiles/ubuntu2004.docker @@ -0,0 +1,65 @@ +FROM ubuntu:20.04 +ENV PACKAGES flex bison \ + ccache \ + clang-10\ + gcc \ + gettext \ + git \ + glusterfs-common \ + libaio-dev \ + libattr1-dev \ + libbrlapi-dev \ + libbz2-dev \ + libcacard-dev \ + libcap-ng-dev \ + libcurl4-gnutls-dev \ + libdrm-dev \ + libepoxy-dev \ + libfdt-dev \ + libgbm-dev \ + libgtk-3-dev \ + libibverbs-dev \ + libiscsi-dev \ + libjemalloc-dev \ + libjpeg-turbo8-dev \ + liblzo2-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libnfs-dev \ + libnss3-dev \ + libnuma-dev \ + libpixman-1-dev \ + librados-dev \ + librbd-dev \ + librdmacm-dev \ + libsasl2-dev \ + libsdl2-dev \ + libseccomp-dev \ + libsnappy-dev \ + libspice-protocol-dev \ + libspice-server-dev \ + libssh-dev \ + libusb-1.0-0-dev \ + libusbredirhost-dev \ + libvdeplug-dev \ + libvte-2.91-dev \ + libxen-dev \ + libzstd-dev \ + make \ + python3-yaml \ + python3-sphinx \ + sparse \ + texinfo \ + xfslibs-dev\ + vim +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install $PACKAGES +RUN dpkg -l $PACKAGES | sort > /packages.txt +ENV FEATURES clang tsan pyyaml sdl2 + +# https://bugs.launchpad.net/qemu/+bug/1838763 +ENV QEMU_CONFIGURE_OPTS --disable-libssh + +# Apply patch https://reviews.llvm.org/D75820 +# This is required for TSan in clang-10 to compile with QEMU. +RUN sed -i 's/^const/static const/g' /usr/lib/llvm-10/lib/clang/10.0.0/include/sanitizer/tsan_interface.h diff --git a/tests/docker/test-tsan b/tests/docker/test-tsan new file mode 100755 index 0000000000..eb40ac45b7 --- /dev/null +++ b/tests/docker/test-tsan @@ -0,0 +1,44 @@ +#!/bin/bash -e +# +# This test will use TSan as part of a build and a make check. +# +# Copyright (c) 2020 Linaro +# Copyright (c) 2016 Red Hat Inc. +# +# Authors: +# Robert Foley +# Originally based on test-quick from Fam Zheng +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +. common.rc + +setup_tsan() +{ + requires clang tsan + tsan_log_dir="/tmp/qemu-test/build/tsan" + mkdir -p $tsan_log_dir > /dev/null || true + EXTRA_CONFIGURE_OPTS="${EXTRA_CONFIGURE_OPTS} --enable-tsan \ + --cc=clang-10 --cxx=clang++-10 \ + --disable-werror --extra-cflags=-O0" + # detect deadlocks is false currently simply because + # TSan crashes immediately with deadlock detector enabled. + # We have maxed out the history size to get the best chance of finding + # warnings during testing. + # Note, to get TSan to fail on warning, use exitcode=66 below. + tsan_opts="suppressions=/tmp/qemu-test/src/tests/tsan/suppressions.tsan\ + detect_deadlocks=false history_size=7\ + halt_on_error=0 exitcode=0 verbose=5\ + log_path=$tsan_log_dir/tsan_warning" + export TSAN_OPTIONS="$tsan_opts" +} + +cd "$BUILD_DIR" + +TARGET_LIST=${TARGET_LIST:-$DEF_TARGET_LIST} \ +setup_tsan +build_qemu +check_qemu +install_qemu diff --git a/tests/tsan/blacklist.tsan b/tests/tsan/blacklist.tsan new file mode 100644 index 0000000000..75e444f5dc --- /dev/null +++ b/tests/tsan/blacklist.tsan @@ -0,0 +1,10 @@ +# This is an example blacklist. +# To enable use of the blacklist add this to configure: +# "--extra-cflags=-fsanitize-blacklist=/tests/tsan/blacklist.tsan" +# The eventual goal would be to fix these warnings. + +# TSan is not happy about setting/getting of dirty bits, +# for example, cpu_physical_memory_set_dirty_range, +# and cpu_physical_memory_get_dirty. +src:bitops.c +src:bitmap.c diff --git a/tests/tsan/suppressions.tsan b/tests/tsan/suppressions.tsan new file mode 100644 index 0000000000..73414b9ebd --- /dev/null +++ b/tests/tsan/suppressions.tsan @@ -0,0 +1,14 @@ +# This is the set of runtime suppressions of TSan warnings. +# The goal would be to have here only items we do not +# plan to fix, and to explain why for each item. + +# TSan reports a double lock on RECURSIVE mutexes. +# Since the recursive lock is intentional, we choose to ignore it. +mutex:aio_context_acquire +mutex:pthread_mutex_lock + +# TSan reports a race betwen pthread_mutex_init() and +# pthread_mutex_lock(). Since this is outside of QEMU, +# we choose to ignore it. +race:pthread_mutex_init +race:pthread_mutex_lock From e51345eea9057c2a2098322df9e9797cd4bc7f8d Mon Sep 17 00:00:00 2001 From: Robert Foley Date: Fri, 12 Jun 2020 20:02:32 +0100 Subject: [PATCH 13/21] include/qemu: Added tsan.h for annotations. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These annotations will allow us to give tsan additional hints. For example, we can inform tsan about reads/writes to ignore to silence certain classes of warnings. We can also annotate threads so that the proper thread naming shows up in tsan warning results. Signed-off-by: Robert Foley Reviewed-by: Emilio G. Cota Reviewed-by: Alex Bennée Signed-off-by: Alex Bennée Message-Id: <20200609200738.445-11-robert.foley@linaro.org> Message-Id: <20200612190237.30436-14-alex.bennee@linaro.org> --- include/qemu/tsan.h | 71 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 include/qemu/tsan.h diff --git a/include/qemu/tsan.h b/include/qemu/tsan.h new file mode 100644 index 0000000000..09cc665f91 --- /dev/null +++ b/include/qemu/tsan.h @@ -0,0 +1,71 @@ +#ifndef QEMU_TSAN_H +#define QEMU_TSAN_H +/* + * tsan.h + * + * This file defines macros used to give ThreadSanitizer + * additional information to help suppress warnings. + * This is necessary since TSan does not provide a header file + * for these annotations. The standard way to include these + * is via the below macros. + * + * Annotation examples can be found here: + * https://github.com/llvm/llvm-project/tree/master/compiler-rt/test/tsan + * annotate_happens_before.cpp or ignore_race.cpp are good places to start. + * + * The full set of annotations can be found here in tsan_interface_ann.cpp. + * https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/tsan/rtl/ + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifdef CONFIG_TSAN +/* + * Informs TSan of a happens before/after relationship. + */ +#define QEMU_TSAN_ANNOTATE_HAPPENS_BEFORE(addr) \ + AnnotateHappensBefore(__FILE__, __LINE__, (void *)(addr)) +#define QEMU_TSAN_ANNOTATE_HAPPENS_AFTER(addr) \ + AnnotateHappensAfter(__FILE__, __LINE__, (void *)(addr)) +/* + * Gives TSan more information about thread names it can report the + * name of the thread in the warning report. + */ +#define QEMU_TSAN_ANNOTATE_THREAD_NAME(name) \ + AnnotateThreadName(__FILE__, __LINE__, (void *)(name)) +/* + * Allows defining a region of code on which TSan will not record memory READS. + * This has the effect of disabling race detection for this section of code. + */ +#define QEMU_TSAN_ANNOTATE_IGNORE_READS_BEGIN() \ + AnnotateIgnoreReadsBegin(__FILE__, __LINE__) +#define QEMU_TSAN_ANNOTATE_IGNORE_READS_END() \ + AnnotateIgnoreReadsEnd(__FILE__, __LINE__) +/* + * Allows defining a region of code on which TSan will not record memory + * WRITES. This has the effect of disabling race detection for this + * section of code. + */ +#define QEMU_TSAN_ANNOTATE_IGNORE_WRITES_BEGIN() \ + AnnotateIgnoreWritesBegin(__FILE__, __LINE__) +#define QEMU_TSAN_ANNOTATE_IGNORE_WRITES_END() \ + AnnotateIgnoreWritesEnd(__FILE__, __LINE__) +#else +#define QEMU_TSAN_ANNOTATE_HAPPENS_BEFORE(addr) +#define QEMU_TSAN_ANNOTATE_HAPPENS_AFTER(addr) +#define QEMU_TSAN_ANNOTATE_THREAD_NAME(name) +#define QEMU_TSAN_ANNOTATE_IGNORE_READS_BEGIN() +#define QEMU_TSAN_ANNOTATE_IGNORE_READS_END() +#define QEMU_TSAN_ANNOTATE_IGNORE_WRITES_BEGIN() +#define QEMU_TSAN_ANNOTATE_IGNORE_WRITES_END() +#endif + +void AnnotateHappensBefore(const char *f, int l, void *addr); +void AnnotateHappensAfter(const char *f, int l, void *addr); +void AnnotateThreadName(const char *f, int l, char *name); +void AnnotateIgnoreReadsBegin(const char *f, int l); +void AnnotateIgnoreReadsEnd(const char *f, int l); +void AnnotateIgnoreWritesBegin(const char *f, int l); +void AnnotateIgnoreWritesEnd(const char *f, int l); +#endif From ce9f0e5b26eb0f37329d27f5f2e3e453ac0acabc Mon Sep 17 00:00:00 2001 From: Robert Foley Date: Fri, 12 Jun 2020 20:02:33 +0100 Subject: [PATCH 14/21] util: Added tsan annotate for thread name. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows us to see the name of the thread in tsan warning reports such as this: Thread T7 'CPU 1/TCG' (tid=24317, running) created by main thread at: Signed-off-by: Robert Foley Reviewed-by: Emilio G. Cota Signed-off-by: Alex Bennée Message-Id: <20200609200738.445-12-robert.foley@linaro.org> Message-Id: <20200612190237.30436-15-alex.bennee@linaro.org> --- util/qemu-thread-posix.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/qemu-thread-posix.c b/util/qemu-thread-posix.c index 838980aaa5..b4c2359272 100644 --- a/util/qemu-thread-posix.c +++ b/util/qemu-thread-posix.c @@ -15,6 +15,7 @@ #include "qemu/atomic.h" #include "qemu/notify.h" #include "qemu-thread-common.h" +#include "qemu/tsan.h" static bool name_threads; @@ -513,6 +514,7 @@ static void *qemu_thread_start(void *args) # endif } #endif + QEMU_TSAN_ANNOTATE_THREAD_NAME(qemu_thread_args->name); g_free(qemu_thread_args->name); g_free(qemu_thread_args); pthread_cleanup_push(qemu_thread_atexit_notify, NULL); From 3b6882bd96c7913725831bef116fb5a91b327e90 Mon Sep 17 00:00:00 2001 From: Robert Foley Date: Fri, 12 Jun 2020 20:02:34 +0100 Subject: [PATCH 15/21] docs: Added details on TSan to testing.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds TSan details to testing.rst. This includes background and reference details on TSan, and details on how to build and test with TSan both with and without docker. Signed-off-by: Robert Foley Reviewed-by: Emilio G. Cota Reviewed-by: Alex Bennée Signed-off-by: Alex Bennée Message-Id: <20200609200738.445-13-robert.foley@linaro.org> Message-Id: <20200612190237.30436-16-alex.bennee@linaro.org> --- docs/devel/testing.rst | 107 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/docs/devel/testing.rst b/docs/devel/testing.rst index 770a987ea4..c1ff24370b 100644 --- a/docs/devel/testing.rst +++ b/docs/devel/testing.rst @@ -397,6 +397,113 @@ list is in the ``make docker`` help text. The frequently used ones are: * ``DEBUG=1``: enables debug. See the previous "Debugging a Docker test failure" section. +Thread Sanitizer +================ + +Thread Sanitizer (TSan) is a tool which can detect data races. QEMU supports +building and testing with this tool. + +For more information on TSan: + +https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual + +Thread Sanitizer in Docker +--------------------------- +TSan is currently supported in the ubuntu2004 docker. + +The test-tsan test will build using TSan and then run make check. + +.. code:: + + make docker-test-tsan@ubuntu2004 + +TSan warnings under docker are placed in files located at build/tsan/. + +We recommend using DEBUG=1 to allow launching the test from inside the docker, +and to allow review of the warnings generated by TSan. + +Building and Testing with TSan +------------------------------ + +It is possible to build and test with TSan, with a few additional steps. +These steps are normally done automatically in the docker. + +There is a one time patch needed in clang-9 or clang-10 at this time: + +.. code:: + + sed -i 's/^const/static const/g' \ + /usr/lib/llvm-10/lib/clang/10.0.0/include/sanitizer/tsan_interface.h + +To configure the build for TSan: + +.. code:: + + ../configure --enable-tsan --cc=clang-10 --cxx=clang++-10 \ + --disable-werror --extra-cflags="-O0" + +The runtime behavior of TSAN is controlled by the TSAN_OPTIONS environment +variable. + +More information on the TSAN_OPTIONS can be found here: + +https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags + +For example: + +.. code:: + + export TSAN_OPTIONS=suppressions=/tests/tsan/suppressions.tsan \ + detect_deadlocks=false history_size=7 exitcode=0 \ + log_path=/tsan/tsan_warning + +The above exitcode=0 has TSan continue without error if any warnings are found. +This allows for running the test and then checking the warnings afterwards. +If you want TSan to stop and exit with error on warnings, use exitcode=66. + +TSan Suppressions +----------------- +Keep in mind that for any data race warning, although there might be a data race +detected by TSan, there might be no actual bug here. TSan provides several +different mechanisms for suppressing warnings. In general it is recommended +to fix the code if possible to eliminate the data race rather than suppress +the warning. + +A few important files for suppressing warnings are: + +tests/tsan/suppressions.tsan - Has TSan warnings we wish to suppress at runtime. +The comment on each supression will typically indicate why we are +suppressing it. More information on the file format can be found here: + +https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions + +tests/tsan/blacklist.tsan - Has TSan warnings we wish to disable +at compile time for test or debug. +Add flags to configure to enable: + +"--extra-cflags=-fsanitize-blacklist=/tests/tsan/blacklist.tsan" + +More information on the file format can be found here under "Blacklist Format": + +https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags + +TSan Annotations +---------------- +include/qemu/tsan.h defines annotations. See this file for more descriptions +of the annotations themselves. Annotations can be used to suppress +TSan warnings or give TSan more information so that it can detect proper +relationships between accesses of data. + +Annotation examples can be found here: + +https://github.com/llvm/llvm-project/tree/master/compiler-rt/test/tsan/ + +Good files to start with are: annotate_happens_before.cpp and ignore_race.cpp + +The full set of annotations can be found here: + +https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cpp + VM testing ========== From ff8f63da5c11d9c9884c1aafc6320e51d64666e4 Mon Sep 17 00:00:00 2001 From: Robert Foley Date: Fri, 12 Jun 2020 20:02:35 +0100 Subject: [PATCH 16/21] tests: Disable select tests under TSan, which hit TSan issue. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disable a few tests under CONFIG_TSAN, which run into a known TSan issue that results in a hang. https://github.com/google/sanitizers/issues/1116 The disabled tests under TSan include all the qtests as well as the test-char, test-qga, and test-qdev-global-props. Signed-off-by: Robert Foley Reviewed-by: Emilio G. Cota Signed-off-by: Alex Bennée Message-Id: <20200609200738.445-14-robert.foley@linaro.org> Message-Id: <20200612190237.30436-17-alex.bennee@linaro.org> --- tests/Makefile.include | 9 +++++++-- tests/qtest/Makefile.include | 7 +++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/Makefile.include b/tests/Makefile.include index 5607c7290d..3f4448a20b 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -55,7 +55,6 @@ SYSEMU_TARGET_LIST := $(subst -softmmu.mak,,$(notdir \ check-unit-y += tests/check-qdict$(EXESUF) check-unit-y += tests/check-block-qdict$(EXESUF) -check-unit-$(CONFIG_SOFTMMU) += tests/test-char$(EXESUF) check-unit-y += tests/check-qnum$(EXESUF) check-unit-y += tests/check-qstring$(EXESUF) check-unit-y += tests/check-qlist$(EXESUF) @@ -108,7 +107,6 @@ check-unit-y += tests/test-qht$(EXESUF) check-unit-y += tests/test-qht-par$(EXESUF) check-unit-y += tests/test-bitops$(EXESUF) check-unit-y += tests/test-bitcnt$(EXESUF) -check-unit-$(CONFIG_SOFTMMU) += tests/test-qdev-global-props$(EXESUF) check-unit-y += tests/check-qom-interface$(EXESUF) check-unit-y += tests/check-qom-proplist$(EXESUF) check-unit-y += tests/test-qemu-opts$(EXESUF) @@ -123,9 +121,16 @@ check-speed-$(CONFIG_BLOCK) += tests/benchmark-crypto-cipher$(EXESUF) check-unit-$(CONFIG_BLOCK) += tests/test-crypto-secret$(EXESUF) check-unit-$(call land,$(CONFIG_BLOCK),$(CONFIG_GNUTLS)) += tests/test-crypto-tlscredsx509$(EXESUF) check-unit-$(call land,$(CONFIG_BLOCK),$(CONFIG_GNUTLS)) += tests/test-crypto-tlssession$(EXESUF) +ifndef CONFIG_TSAN +# Some tests: test-char, test-qdev-global-props, and test-qga, +# are not runnable under TSan due to a known issue. +# https://github.com/google/sanitizers/issues/1116 +check-unit-$(CONFIG_SOFTMMU) += tests/test-char$(EXESUF) +check-unit-$(CONFIG_SOFTMMU) += tests/test-qdev-global-props$(EXESUF) ifneq (,$(findstring qemu-ga,$(TOOLS))) check-unit-$(call land,$(CONFIG_LINUX),$(CONFIG_VIRTIO_SERIAL)) += tests/test-qga$(EXESUF) endif +endif check-unit-$(CONFIG_SOFTMMU) += tests/test-timed-average$(EXESUF) check-unit-$(call land,$(CONFIG_SOFTMMU),$(CONFIG_INOTIFY1)) += tests/test-util-filemonitor$(EXESUF) check-unit-$(CONFIG_SOFTMMU) += tests/test-util-sockets$(EXESUF) diff --git a/tests/qtest/Makefile.include b/tests/qtest/Makefile.include index 5023fa413d..98af2c2d93 100644 --- a/tests/qtest/Makefile.include +++ b/tests/qtest/Makefile.include @@ -314,12 +314,15 @@ tests/qtest/tpm-tis-device-test$(EXESUF): tests/qtest/tpm-tis-device-test.o test # QTest rules TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGET_DIRS))) +QTEST_TARGETS = +# The qtests are not runnable (yet) under TSan due to a known issue. +# https://github.com/google/sanitizers/issues/1116 +ifndef CONFIG_TSAN ifeq ($(CONFIG_POSIX),y) QTEST_TARGETS = $(TARGETS) check-qtest-y=$(foreach TARGET,$(TARGETS), $(check-qtest-$(TARGET)-y:%=tests/qtest/%$(EXESUF))) check-qtest-y += $(check-qtest-generic-y:%=tests/qtest/%$(EXESUF)) -else -QTEST_TARGETS = +endif endif qtest-obj-y = tests/qtest/libqtest.o $(test-util-obj-y) From 8cb14dbb8df0970143cf287661cb446abc697377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Fri, 12 Jun 2020 20:02:36 +0100 Subject: [PATCH 17/21] Revert ".shippable: temporaily disable some cross builds" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 12d43b5ae916809aad9ccf8aa2a0a06260527340. Signed-off-by: Alex Bennée Message-Id: <20200612190237.30436-18-alex.bennee@linaro.org> --- .shippable.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.shippable.yml b/.shippable.yml index 10cf219bff..2cce7b5689 100644 --- a/.shippable.yml +++ b/.shippable.yml @@ -5,8 +5,8 @@ env: global: - LC_ALL=C matrix: - # - IMAGE=debian-amd64 - # TARGET_LIST=x86_64-softmmu,x86_64-linux-user + - IMAGE=debian-amd64 + TARGET_LIST=x86_64-softmmu,x86_64-linux-user - IMAGE=debian-win32-cross TARGET_LIST=arm-softmmu,i386-softmmu,lm32-softmmu - IMAGE=debian-win64-cross @@ -19,10 +19,10 @@ env: TARGET_LIST=aarch64-softmmu,aarch64-linux-user - IMAGE=debian-s390x-cross TARGET_LIST=s390x-softmmu,s390x-linux-user - # - IMAGE=debian-mips-cross - # TARGET_LIST=mips-softmmu,mipsel-linux-user - # - IMAGE=debian-mips64el-cross - # TARGET_LIST=mips64el-softmmu,mips64el-linux-user + - IMAGE=debian-mips-cross + TARGET_LIST=mips-softmmu,mipsel-linux-user + - IMAGE=debian-mips64el-cross + TARGET_LIST=mips64el-softmmu,mips64el-linux-user - IMAGE=debian-ppc64el-cross TARGET_LIST=ppc64-softmmu,ppc64-linux-user,ppc64abi32-linux-user build: From 7e1236148c343f9189aa937eef97d42e2c8a00d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Fri, 12 Jun 2020 20:02:37 +0100 Subject: [PATCH 18/21] cirrus.yml: serialise make check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We do this on our other platforms to make it easier to see what has broken. Signed-off-by: Alex Bennée Reviewed-by: Li-Wen Hsu Message-Id: <20200612190237.30436-19-alex.bennee@linaro.org> --- .cirrus.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index ce7850a320..69342ae031 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -14,7 +14,7 @@ freebsd_12_task: - cd build - ../configure || { cat config.log; exit 1; } - gmake -j8 - - gmake -j8 V=1 check + - gmake V=1 check macos_task: osx_instance: @@ -26,7 +26,7 @@ macos_task: - cd build - ../configure --python=/usr/local/bin/python3 || { cat config.log; exit 1; } - gmake -j$(sysctl -n hw.ncpu) - - gmake check -j$(sysctl -n hw.ncpu) + - gmake check macos_xcode_task: osx_instance: @@ -39,4 +39,4 @@ macos_xcode_task: - cd build - ../configure --cc=clang || { cat config.log; exit 1; } - gmake -j$(sysctl -n hw.ncpu) - - gmake check -j$(sysctl -n hw.ncpu) + - gmake check From fefa0271544146e027cd682cf483223ee2f37e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 15 Jun 2020 15:19:21 +0100 Subject: [PATCH 19/21] tests/tcg: build plugin list from contents of src directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If you jump back and forth between branches while developing plugins you end up debugging failures caused by plugins left in the build directory. Fix this by basing plugins on the source tree instead. Signed-off-by: Alex Bennée Message-Id: <20200615141922.18829-2-alex.bennee@linaro.org> --- tests/tcg/Makefile.target | 11 ++++++----- tests/tcg/aarch64/Makefile.target | 2 +- tests/tcg/arm/Makefile.target | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/tcg/Makefile.target b/tests/tcg/Makefile.target index b3cff3cad1..2ae86776cd 100644 --- a/tests/tcg/Makefile.target +++ b/tests/tcg/Makefile.target @@ -126,9 +126,10 @@ RUN_TESTS=$(patsubst %,run-%, $(TESTS)) # If plugins exist also include those in the tests ifeq ($(CONFIG_PLUGIN),y) -PLUGIN_DIR=../../plugin -VPATH+=$(PLUGIN_DIR) -PLUGINS=$(notdir $(wildcard $(PLUGIN_DIR)/*.so)) +PLUGIN_SRC=$(SRC_PATH)/tests/plugin +PLUGIN_LIB=../../plugin +VPATH+=$(PLUGIN_LIB) +PLUGINS=$(patsubst %.c, lib%.so, $(notdir $(wildcard $(PLUGIN_SRC)/*.c))) # We need to ensure expand the run-plugin-TEST-with-PLUGIN # pre-requistes manually here as we can't use stems to handle it. We @@ -152,7 +153,7 @@ run-%: % run-plugin-%: $(call run-test, $@, $(QEMU) $(QEMU_OPTS) \ - -plugin $(PLUGIN_DIR)/$(call extract-plugin,$@) \ + -plugin $(PLUGIN_LIB)/$(call extract-plugin,$@) \ -d plugin -D $*.pout \ $(call strip-plugin,$<), \ "$* on $(TARGET_NAME)") @@ -168,7 +169,7 @@ run-plugin-%: $(call run-test, $@, \ $(QEMU) -monitor none -display none \ -chardev file$(COMMA)path=$@.out$(COMMA)id=output \ - -plugin $(PLUGIN_DIR)/$(call extract-plugin,$@) \ + -plugin $(PLUGIN_LIB)/$(call extract-plugin,$@) \ -d plugin -D $*.pout \ $(QEMU_OPTS) $(call strip-plugin,$<), \ "$* on $(TARGET_NAME)") diff --git a/tests/tcg/aarch64/Makefile.target b/tests/tcg/aarch64/Makefile.target index 312f36cde5..6d60a2f2ee 100644 --- a/tests/tcg/aarch64/Makefile.target +++ b/tests/tcg/aarch64/Makefile.target @@ -31,7 +31,7 @@ run-semihosting: semihosting run-plugin-semihosting-with-%: $(call run-test, $@, $(QEMU) $(QEMU_OPTS) \ - -plugin $(PLUGIN_DIR)/$(call extract-plugin,$@) \ + -plugin $(PLUGIN_LIB)/$(call extract-plugin,$@) \ $(call strip-plugin,$<) 2> $<.err, \ "$< on $(TARGET_NAME) with $*") diff --git a/tests/tcg/arm/Makefile.target b/tests/tcg/arm/Makefile.target index 3da09a38be..ec95156562 100644 --- a/tests/tcg/arm/Makefile.target +++ b/tests/tcg/arm/Makefile.target @@ -45,7 +45,7 @@ run-semihosting-arm: semihosting-arm run-plugin-semihosting-with-%: $(call run-test, $@, $(QEMU) $(QEMU_OPTS) \ - -plugin $(PLUGIN_DIR)/$(call extract-plugin,$@) \ + -plugin $(PLUGIN_LIB)/$(call extract-plugin,$@) \ $(call strip-plugin,$<) 2> $<.err, \ "$< on $(TARGET_NAME) with $*") From d16242e524a47d2c3e0281951c6d27979992ac3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 15 Jun 2020 15:19:22 +0100 Subject: [PATCH 20/21] tests/tcg: ensure -cpu max also used for plugin run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The check-tcg plugins build was failing because some special case tests that needed -cpu max failed because the plugin variant hadn't carried across the QEMU_OPTS tweak. Guests which globally set QEMU_OPTS=-cpu FOO where unaffected. Signed-off-by: Alex Bennée Message-Id: <20200615141922.18829-3-alex.bennee@linaro.org> --- tests/tcg/aarch64/Makefile.target | 3 ++- tests/tcg/i386/Makefile.target | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/tcg/aarch64/Makefile.target b/tests/tcg/aarch64/Makefile.target index 6d60a2f2ee..b617f2ac7e 100644 --- a/tests/tcg/aarch64/Makefile.target +++ b/tests/tcg/aarch64/Makefile.target @@ -20,8 +20,9 @@ run-fcvt: fcvt # Pauth Tests ifneq ($(DOCKER_IMAGE)$(CROSS_CC_HAS_ARMV8_3),) AARCH64_TESTS += pauth-1 pauth-2 pauth-4 -run-pauth-%: QEMU_OPTS += -cpu max pauth-%: CFLAGS += -march=armv8.3-a +run-pauth-%: QEMU_OPTS += -cpu max +run-plugin-pauth-%: QEMU_OPTS += -cpu max endif # Semihosting smoke test for linux-user diff --git a/tests/tcg/i386/Makefile.target b/tests/tcg/i386/Makefile.target index 53efec0668..1a6463a7dc 100644 --- a/tests/tcg/i386/Makefile.target +++ b/tests/tcg/i386/Makefile.target @@ -12,6 +12,7 @@ X86_64_TESTS:=$(filter test-i386-ssse3, $(ALL_X86_TESTS)) test-i386-pcmpistri: CFLAGS += -msse4.2 run-test-i386-pcmpistri: QEMU_OPTS += -cpu max +run-plugin-test-i386-pcmpistri-%: QEMU_OPTS += -cpu max # # hello-i386 is a barebones app From c81950a2f1923dec3f6b952ec6bb9b921be58a70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 10 Jun 2020 16:55:05 +0100 Subject: [PATCH 21/21] plugins: new lockstep plugin for debugging TCG changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When we make changes to the TCG we sometimes cause regressions that are deep into the execution cycle of the guest. Debugging this often requires comparing large volumes of trace information to figure out where behaviour has diverged. The lockstep plugin utilises a shared socket so two QEMU's running with the plugin will write their current execution position and wait to receive the position of their partner process. When execution diverges the plugins output where they were and the previous few blocks before unloading themselves and letting execution continue. Originally I planned for this to be most useful with -icount but it turns out you can get divergence pretty quickly due to asynchronous qemu_cpu_kick_rr_cpus() events causing one side to eventually run into a short block a few cycles before the other side. For this reason I've added a bit of tracking and I think the divergence reporting could be finessed to report only if we really start to diverge in execution. An example run would be: qemu-system-sparc -monitor none -parallel none -net none \ -M SS-20 -m 256 -kernel day11/zImage.elf \ -plugin ./tests/plugin/liblockstep.so,arg=lockstep-sparc.sock \ -d plugin,nochain with an identical command in another window in the same working directory. Signed-off-by: Alex Bennée Reviewed-by: Robert Foley Tested-by: Robert Foley Cc: Richard Henderson Cc: Mark Cave-Ayland Message-Id: <20200610155509.12850-3-alex.bennee@linaro.org> --- tests/plugin/Makefile | 1 + tests/plugin/lockstep.c | 340 ++++++++++++++++++++++++++++++++++++++ tests/tcg/Makefile.target | 3 +- 3 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 tests/plugin/lockstep.c diff --git a/tests/plugin/Makefile b/tests/plugin/Makefile index 75467b6db8..b3250e2504 100644 --- a/tests/plugin/Makefile +++ b/tests/plugin/Makefile @@ -13,6 +13,7 @@ NAMES += mem NAMES += hotblocks NAMES += howvec NAMES += hotpages +NAMES += lockstep SONAMES := $(addsuffix .so,$(addprefix lib,$(NAMES))) diff --git a/tests/plugin/lockstep.c b/tests/plugin/lockstep.c new file mode 100644 index 0000000000..a696673dff --- /dev/null +++ b/tests/plugin/lockstep.c @@ -0,0 +1,340 @@ +/* + * Lockstep Execution Plugin + * + * Allows you to execute two QEMU instances in lockstep and report + * when their execution diverges. This is mainly useful for developers + * who want to see where a change to TCG code generation has + * introduced a subtle and hard to find bug. + * + * Caveats: + * - single-threaded linux-user apps only with non-deterministic syscalls + * - no MTTCG enabled system emulation (icount may help) + * + * While icount makes things more deterministic it doesn't mean a + * particular run may execute the exact same sequence of blocks. An + * asynchronous event (for example X11 graphics update) may cause a + * block to end early and a new partial block to start. This means + * serial only test cases are a better bet. -d nochain may also help. + * + * This code is not thread safe! + * + * Copyright (c) 2020 Linaro Ltd + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +QEMU_PLUGIN_EXPORT int qemu_plugin_version = QEMU_PLUGIN_VERSION; + +/* saved so we can uninstall later */ +static qemu_plugin_id_t our_id; + +static unsigned long bb_count; +static unsigned long insn_count; + +/* Information about a translated block */ +typedef struct { + uint64_t pc; + uint64_t insns; +} BlockInfo; + +/* Information about an execution state in the log */ +typedef struct { + BlockInfo *block; + unsigned long insn_count; + unsigned long block_count; +} ExecInfo; + +/* The execution state we compare */ +typedef struct { + uint64_t pc; + unsigned long insn_count; +} ExecState; + +typedef struct { + GSList *log_pos; + int distance; +} DivergeState; + +/* list of translated block info */ +static GSList *blocks; + +/* execution log and points of divergence */ +static GSList *log, *divergence_log; + +static int socket_fd; +static char *path_to_unlink; + +static bool verbose; + +static void plugin_cleanup(qemu_plugin_id_t id) +{ + /* Free our block data */ + g_slist_free_full(blocks, &g_free); + g_slist_free_full(log, &g_free); + g_slist_free(divergence_log); + + close(socket_fd); + if (path_to_unlink) { + unlink(path_to_unlink); + } +} + +static void plugin_exit(qemu_plugin_id_t id, void *p) +{ + g_autoptr(GString) out = g_string_new("No divergence :-)\n"); + g_string_append_printf(out, "Executed %ld/%d blocks\n", + bb_count, g_slist_length(log)); + g_string_append_printf(out, "Executed ~%ld instructions\n", insn_count); + qemu_plugin_outs(out->str); + + plugin_cleanup(id); +} + +static void report_divergance(ExecState *us, ExecState *them) +{ + DivergeState divrec = { log, 0 }; + g_autoptr(GString) out = g_string_new(""); + bool diverged = false; + + /* + * If we have diverged before did we get back on track or are we + * totally loosing it? + */ + if (divergence_log) { + DivergeState *last = (DivergeState *) divergence_log->data; + GSList *entry; + + for (entry = log; g_slist_next(entry); entry = g_slist_next(entry)) { + if (entry == last->log_pos) { + break; + } + divrec.distance++; + } + + /* + * If the last two records are so close it is likely we will + * not recover synchronisation with the other end. + */ + if (divrec.distance == 1 && last->distance == 1) { + diverged = true; + } + } + divergence_log = g_slist_prepend(divergence_log, + g_memdup(&divrec, sizeof(divrec))); + + /* Output short log entry of going out of sync... */ + if (verbose || divrec.distance == 1 || diverged) { + g_string_printf(out, "@ %#016lx vs %#016lx (%d/%d since last)\n", + us->pc, them->pc, g_slist_length(divergence_log), + divrec.distance); + qemu_plugin_outs(out->str); + } + + if (diverged) { + int i; + GSList *entry; + + g_string_printf(out, "Δ insn_count @ %#016lx (%ld) vs %#016lx (%ld)\n", + us->pc, us->insn_count, them->pc, them->insn_count); + + for (entry = log, i = 0; + g_slist_next(entry) && i < 5; + entry = g_slist_next(entry), i++) { + ExecInfo *prev = (ExecInfo *) entry->data; + g_string_append_printf(out, + " previously @ %#016lx/%ld (%ld insns)\n", + prev->block->pc, prev->block->insns, + prev->insn_count); + } + qemu_plugin_outs(out->str); + qemu_plugin_outs("too much divergence... giving up."); + qemu_plugin_uninstall(our_id, plugin_cleanup); + } +} + +static void vcpu_tb_exec(unsigned int cpu_index, void *udata) +{ + BlockInfo *bi = (BlockInfo *) udata; + ExecState us, them; + ssize_t bytes; + ExecInfo *exec; + + us.pc = bi->pc; + us.insn_count = insn_count; + + /* + * Write our current position to the other end. If we fail the + * other end has probably died and we should shut down gracefully. + */ + bytes = write(socket_fd, &us, sizeof(ExecState)); + if (bytes < sizeof(ExecState)) { + qemu_plugin_outs(bytes < 0 ? + "problem writing to socket" : + "wrote less than expected to socket"); + qemu_plugin_uninstall(our_id, plugin_cleanup); + return; + } + + /* + * Now read where our peer has reached. Again a failure probably + * indicates the other end died and we should close down cleanly. + */ + bytes = read(socket_fd, &them, sizeof(ExecState)); + if (bytes < sizeof(ExecState)) { + qemu_plugin_outs(bytes < 0 ? + "problem reading from socket" : + "read less than expected"); + qemu_plugin_uninstall(our_id, plugin_cleanup); + return; + } + + /* + * Compare and report if we have diverged. + */ + if (us.pc != them.pc) { + report_divergance(&us, &them); + } + + /* + * Assume this block will execute fully and record it + * in the execution log. + */ + insn_count += bi->insns; + bb_count++; + exec = g_new0(ExecInfo, 1); + exec->block = bi; + exec->insn_count = insn_count; + exec->block_count = bb_count; + log = g_slist_prepend(log, exec); +} + +static void vcpu_tb_trans(qemu_plugin_id_t id, struct qemu_plugin_tb *tb) +{ + BlockInfo *bi = g_new0(BlockInfo, 1); + bi->pc = qemu_plugin_tb_vaddr(tb); + bi->insns = qemu_plugin_tb_n_insns(tb); + + /* save a reference so we can free later */ + blocks = g_slist_prepend(blocks, bi); + qemu_plugin_register_vcpu_tb_exec_cb(tb, vcpu_tb_exec, + QEMU_PLUGIN_CB_NO_REGS, (void *)bi); +} + + +/* + * Instead of encoding master/slave status into what is essentially + * two peers we shall just take the simple approach of checking for + * the existence of the pipe and assuming if it's not there we are the + * first process. + */ +static bool setup_socket(const char *path) +{ + struct sockaddr_un sockaddr; + int fd; + + fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + perror("create socket"); + return false; + } + + sockaddr.sun_family = AF_UNIX; + g_strlcpy(sockaddr.sun_path, path, sizeof(sockaddr.sun_path) - 1); + if (bind(fd, (struct sockaddr *)&sockaddr, sizeof(sockaddr)) < 0) { + perror("bind socket"); + close(fd); + return false; + } + + /* remember to clean-up */ + path_to_unlink = g_strdup(path); + + if (listen(fd, 1) < 0) { + perror("listen socket"); + close(fd); + return false; + } + + socket_fd = accept(fd, NULL, NULL); + if (socket_fd < 0 && errno != EINTR) { + perror("accept socket"); + return false; + } + + qemu_plugin_outs("setup_socket::ready\n"); + + return true; +} + +static bool connect_socket(const char *path) +{ + int fd; + struct sockaddr_un sockaddr; + + fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + perror("create socket"); + return false; + } + + sockaddr.sun_family = AF_UNIX; + g_strlcpy(sockaddr.sun_path, path, sizeof(sockaddr.sun_path) - 1); + + if (connect(fd, (struct sockaddr *)&sockaddr, sizeof(sockaddr)) < 0) { + perror("failed to connect"); + return false; + } + + qemu_plugin_outs("connect_socket::ready\n"); + + socket_fd = fd; + return true; +} + +static bool setup_unix_socket(const char *path) +{ + if (g_file_test(path, G_FILE_TEST_EXISTS)) { + return connect_socket(path); + } else { + return setup_socket(path); + } +} + + +QEMU_PLUGIN_EXPORT int qemu_plugin_install(qemu_plugin_id_t id, + const qemu_info_t *info, + int argc, char **argv) +{ + int i; + + if (!argc || !argv[0]) { + qemu_plugin_outs("Need a socket path to talk to other instance."); + return -1; + } + + for (i = 0; i < argc; i++) { + char *p = argv[i]; + if (strcmp(p, "verbose") == 0) { + verbose = true; + } else if (!setup_unix_socket(argv[0])) { + qemu_plugin_outs("Failed to setup socket for communications."); + return -1; + } + } + + our_id = id; + + qemu_plugin_register_vcpu_tb_trans_cb(id, vcpu_tb_trans); + qemu_plugin_register_atexit_cb(id, plugin_exit, NULL); + return 0; +} diff --git a/tests/tcg/Makefile.target b/tests/tcg/Makefile.target index 2ae86776cd..4b2b696fce 100644 --- a/tests/tcg/Makefile.target +++ b/tests/tcg/Makefile.target @@ -129,7 +129,8 @@ ifeq ($(CONFIG_PLUGIN),y) PLUGIN_SRC=$(SRC_PATH)/tests/plugin PLUGIN_LIB=../../plugin VPATH+=$(PLUGIN_LIB) -PLUGINS=$(patsubst %.c, lib%.so, $(notdir $(wildcard $(PLUGIN_SRC)/*.c))) +PLUGINS=$(filter-out liblockstep.so,\ + $(patsubst %.c, lib%.so, $(notdir $(wildcard $(PLUGIN_SRC)/*.c)))) # We need to ensure expand the run-plugin-TEST-with-PLUGIN # pre-requistes manually here as we can't use stems to handle it. We