From 22ea40f4ff072a113fdf96bff10bc81ee063da32 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Tue, 26 Jun 2012 17:19:10 +0200 Subject: [PATCH 01/13] savevm: Use a struct to pass all handlers This would make easier to add more operations in the next patches. Signed-off-by: Juan Quintela --- savevm.c | 54 +++++++++++++++++++++++++----------------------------- vmstate.h | 7 +++++++ 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/savevm.c b/savevm.c index a15c163b6e..73626d4ddd 100644 --- a/savevm.c +++ b/savevm.c @@ -1171,10 +1171,7 @@ typedef struct SaveStateEntry { int alias_id; int version_id; int section_id; - SaveSetParamsHandler *set_params; - SaveLiveStateHandler *save_live_state; - SaveStateHandler *save_state; - LoadStateHandler *load_state; + SaveVMHandlers *ops; const VMStateDescription *vmsd; void *opaque; CompatEntry *compat; @@ -1237,10 +1234,11 @@ int register_savevm_live(DeviceState *dev, se = g_malloc0(sizeof(SaveStateEntry)); se->version_id = version_id; se->section_id = global_section_id++; - se->set_params = set_params; - se->save_live_state = save_live_state; - se->save_state = save_state; - se->load_state = load_state; + se->ops = g_malloc0(sizeof(SaveVMHandlers)); + se->ops->set_params = set_params; + se->ops->save_live_state = save_live_state; + se->ops->save_state = save_state; + se->ops->load_state = load_state; se->opaque = opaque; se->vmsd = NULL; se->no_migrate = 0; @@ -1309,6 +1307,7 @@ void unregister_savevm(DeviceState *dev, const char *idstr, void *opaque) if (se->compat) { g_free(se->compat); } + g_free(se->ops); g_free(se); } } @@ -1327,9 +1326,6 @@ int vmstate_register_with_alias_id(DeviceState *dev, int instance_id, se = g_malloc0(sizeof(SaveStateEntry)); se->version_id = vmsd->version_id; se->section_id = global_section_id++; - se->save_live_state = NULL; - se->save_state = NULL; - se->load_state = NULL; se->opaque = opaque; se->vmsd = vmsd; se->alias_id = alias_id; @@ -1524,7 +1520,7 @@ void vmstate_save_state(QEMUFile *f, const VMStateDescription *vmsd, static int vmstate_load(QEMUFile *f, SaveStateEntry *se, int version_id) { if (!se->vmsd) { /* Old style */ - return se->load_state(f, se->opaque, version_id); + return se->ops->load_state(f, se->opaque, version_id); } return vmstate_load_state(f, se->vmsd, se->opaque, version_id); } @@ -1532,7 +1528,7 @@ static int vmstate_load(QEMUFile *f, SaveStateEntry *se, int version_id) static void vmstate_save(QEMUFile *f, SaveStateEntry *se) { if (!se->vmsd) { /* Old style */ - se->save_state(f, se->opaque); + se->ops->save_state(f, se->opaque); return; } vmstate_save_state(f,se->vmsd, se->opaque); @@ -1569,10 +1565,10 @@ int qemu_savevm_state_begin(QEMUFile *f, int ret; QTAILQ_FOREACH(se, &savevm_handlers, entry) { - if(se->set_params == NULL) { + if (!se->ops || !se->ops->set_params) { continue; } - se->set_params(params, se->opaque); + se->ops->set_params(params, se->opaque); } qemu_put_be32(f, QEMU_VM_FILE_MAGIC); @@ -1581,9 +1577,9 @@ int qemu_savevm_state_begin(QEMUFile *f, QTAILQ_FOREACH(se, &savevm_handlers, entry) { int len; - if (se->save_live_state == NULL) + if (!se->ops || !se->ops->save_live_state) { continue; - + } /* Section type */ qemu_put_byte(f, QEMU_VM_SECTION_START); qemu_put_be32(f, se->section_id); @@ -1596,7 +1592,7 @@ int qemu_savevm_state_begin(QEMUFile *f, qemu_put_be32(f, se->instance_id); qemu_put_be32(f, se->version_id); - ret = se->save_live_state(f, QEMU_VM_SECTION_START, se->opaque); + ret = se->ops->save_live_state(f, QEMU_VM_SECTION_START, se->opaque); if (ret < 0) { qemu_savevm_state_cancel(f); return ret; @@ -1623,9 +1619,9 @@ int qemu_savevm_state_iterate(QEMUFile *f) int ret = 1; QTAILQ_FOREACH(se, &savevm_handlers, entry) { - if (se->save_live_state == NULL) + if (!se->ops || !se->ops->save_live_state) { continue; - + } if (qemu_file_rate_limit(f)) { return 0; } @@ -1634,7 +1630,7 @@ int qemu_savevm_state_iterate(QEMUFile *f) qemu_put_byte(f, QEMU_VM_SECTION_PART); qemu_put_be32(f, se->section_id); - ret = se->save_live_state(f, QEMU_VM_SECTION_PART, se->opaque); + ret = se->ops->save_live_state(f, QEMU_VM_SECTION_PART, se->opaque); trace_savevm_section_end(se->section_id); if (ret <= 0) { @@ -1663,15 +1659,15 @@ int qemu_savevm_state_complete(QEMUFile *f) cpu_synchronize_all_states(); QTAILQ_FOREACH(se, &savevm_handlers, entry) { - if (se->save_live_state == NULL) + if (!se->ops || !se->ops->save_live_state) { continue; - + } trace_savevm_section_start(); /* Section type */ qemu_put_byte(f, QEMU_VM_SECTION_END); qemu_put_be32(f, se->section_id); - ret = se->save_live_state(f, QEMU_VM_SECTION_END, se->opaque); + ret = se->ops->save_live_state(f, QEMU_VM_SECTION_END, se->opaque); trace_savevm_section_end(se->section_id); if (ret < 0) { return ret; @@ -1681,9 +1677,9 @@ int qemu_savevm_state_complete(QEMUFile *f) QTAILQ_FOREACH(se, &savevm_handlers, entry) { int len; - if (se->save_state == NULL && se->vmsd == NULL) + if ((!se->ops || !se->ops->save_state) && !se->vmsd) { continue; - + } trace_savevm_section_start(); /* Section type */ qemu_put_byte(f, QEMU_VM_SECTION_FULL); @@ -1711,8 +1707,8 @@ void qemu_savevm_state_cancel(QEMUFile *f) SaveStateEntry *se; QTAILQ_FOREACH(se, &savevm_handlers, entry) { - if (se->save_live_state) { - se->save_live_state(f, -1, se->opaque); + if (se->ops && se->ops->save_live_state) { + se->ops->save_live_state(f, -1, se->opaque); } } } @@ -1765,7 +1761,7 @@ static int qemu_save_device_state(QEMUFile *f) if (se->is_ram) { continue; } - if (se->save_state == NULL && se->vmsd == NULL) { + if ((!se->ops || !se->ops->save_state) && !se->vmsd) { continue; } diff --git a/vmstate.h b/vmstate.h index 5af45e0c12..909af6934d 100644 --- a/vmstate.h +++ b/vmstate.h @@ -31,6 +31,13 @@ typedef void SaveStateHandler(QEMUFile *f, void *opaque); typedef int SaveLiveStateHandler(QEMUFile *f, int stage, void *opaque); typedef int LoadStateHandler(QEMUFile *f, void *opaque, int version_id); +typedef struct SaveVMHandlers { + SaveSetParamsHandler *set_params; + SaveStateHandler *save_state; + SaveLiveStateHandler *save_live_state; + LoadStateHandler *load_state; +} SaveVMHandlers; + int register_savevm(DeviceState *dev, const char *idstr, int instance_id, From 7908c78d3e1a117ae6b9545e601409eee3d33863 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Tue, 26 Jun 2012 18:46:10 +0200 Subject: [PATCH 02/13] savevm: Live migration handlers register the struct directly Notice that the live migration users never unregister, so no problem about freeing the ops structure. Signed-off-by: Juan Quintela --- arch_init.c | 9 +++++++-- block-migration.c | 10 ++++++++-- migration.h | 4 ++-- savevm.c | 18 +++++++----------- vl.c | 3 +-- vmstate.h | 5 +---- 6 files changed, 26 insertions(+), 23 deletions(-) diff --git a/arch_init.c b/arch_init.c index 5b0f5626a9..606b848474 100644 --- a/arch_init.c +++ b/arch_init.c @@ -298,7 +298,7 @@ static void migration_end(void) #define MAX_WAIT 50 /* ms, half buffered_file limit */ -int ram_save_live(QEMUFile *f, int stage, void *opaque) +static int ram_save_live(QEMUFile *f, int stage, void *opaque) { ram_addr_t addr; uint64_t bytes_transferred_last; @@ -437,7 +437,7 @@ static inline void *host_from_stream_offset(QEMUFile *f, return NULL; } -int ram_load(QEMUFile *f, void *opaque, int version_id) +static int ram_load(QEMUFile *f, void *opaque, int version_id) { ram_addr_t addr; int flags, ret = 0; @@ -534,6 +534,11 @@ done: return ret; } +SaveVMHandlers savevm_ram_handlers = { + .save_live_state = ram_save_live, + .load_state = ram_load, +}; + #ifdef HAS_AUDIO struct soundhw { const char *name; diff --git a/block-migration.c b/block-migration.c index b95b4e1389..00151a07d9 100644 --- a/block-migration.c +++ b/block-migration.c @@ -709,11 +709,17 @@ static void block_set_params(const MigrationParams *params, void *opaque) block_mig_state.blk_enable |= params->shared; } +SaveVMHandlers savevm_block_handlers = { + .set_params = block_set_params, + .save_live_state = block_save_live, + .load_state = block_load, +}; + void blk_mig_init(void) { QSIMPLEQ_INIT(&block_mig_state.bmds_list); QSIMPLEQ_INIT(&block_mig_state.blk_list); - register_savevm_live(NULL, "block", 0, 1, block_set_params, - block_save_live, NULL, block_load, &block_mig_state); + register_savevm_live(NULL, "block", 0, 1, &savevm_block_handlers, + &block_mig_state); } diff --git a/migration.h b/migration.h index de13004573..57572a61e9 100644 --- a/migration.h +++ b/migration.h @@ -18,6 +18,7 @@ #include "qemu-common.h" #include "notify.h" #include "error.h" +#include "vmstate.h" struct MigrationParams { bool blk; @@ -81,8 +82,7 @@ uint64_t ram_bytes_remaining(void); uint64_t ram_bytes_transferred(void); uint64_t ram_bytes_total(void); -int ram_save_live(QEMUFile *f, int stage, void *opaque); -int ram_load(QEMUFile *f, void *opaque, int version_id); +extern SaveVMHandlers savevm_ram_handlers; /** * @migrate_add_blocker - prevent migration from proceeding diff --git a/savevm.c b/savevm.c index 73626d4ddd..a451be22a6 100644 --- a/savevm.c +++ b/savevm.c @@ -1223,10 +1223,7 @@ int register_savevm_live(DeviceState *dev, const char *idstr, int instance_id, int version_id, - SaveSetParamsHandler *set_params, - SaveLiveStateHandler *save_live_state, - SaveStateHandler *save_state, - LoadStateHandler *load_state, + SaveVMHandlers *ops, void *opaque) { SaveStateEntry *se; @@ -1234,16 +1231,12 @@ int register_savevm_live(DeviceState *dev, se = g_malloc0(sizeof(SaveStateEntry)); se->version_id = version_id; se->section_id = global_section_id++; - se->ops = g_malloc0(sizeof(SaveVMHandlers)); - se->ops->set_params = set_params; - se->ops->save_live_state = save_live_state; - se->ops->save_state = save_state; - se->ops->load_state = load_state; + se->ops = ops; se->opaque = opaque; se->vmsd = NULL; se->no_migrate = 0; /* if this is a live_savem then set is_ram */ - if (save_live_state != NULL) { + if (ops->save_live_state != NULL) { se->is_ram = 1; } @@ -1282,8 +1275,11 @@ int register_savevm(DeviceState *dev, LoadStateHandler *load_state, void *opaque) { + SaveVMHandlers *ops = g_malloc0(sizeof(SaveVMHandlers)); + ops->save_state = save_state; + ops->load_state = load_state; return register_savevm_live(dev, idstr, instance_id, version_id, - NULL, NULL, save_state, load_state, opaque); + ops, opaque); } void unregister_savevm(DeviceState *dev, const char *idstr, void *opaque) diff --git a/vl.c b/vl.c index 8904db1a33..54e36ed10f 100644 --- a/vl.c +++ b/vl.c @@ -3437,8 +3437,7 @@ int main(int argc, char **argv, char **envp) default_drive(default_sdcard, snapshot, machine->use_scsi, IF_SD, 0, SD_OPTS); - register_savevm_live(NULL, "ram", 0, 4, NULL, ram_save_live, NULL, - ram_load, NULL); + register_savevm_live(NULL, "ram", 0, 4, &savevm_ram_handlers, NULL); if (nb_numa_nodes > 0) { int i; diff --git a/vmstate.h b/vmstate.h index 909af6934d..4bce53bc53 100644 --- a/vmstate.h +++ b/vmstate.h @@ -50,10 +50,7 @@ int register_savevm_live(DeviceState *dev, const char *idstr, int instance_id, int version_id, - SaveSetParamsHandler *set_params, - SaveLiveStateHandler *save_live_state, - SaveStateHandler *save_state, - LoadStateHandler *load_state, + SaveVMHandlers *ops, void *opaque); void unregister_savevm(DeviceState *dev, const char *idstr, void *opaque); From 8625368515411ce941ac268287ba9e962ad26460 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Tue, 26 Jun 2012 18:51:00 +0200 Subject: [PATCH 03/13] savevm: remove SaveSetParamsHandler It was used only once, just unfold. Signed-off-by: Juan Quintela --- vmstate.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vmstate.h b/vmstate.h index 4bce53bc53..5e1a7cc2bc 100644 --- a/vmstate.h +++ b/vmstate.h @@ -26,13 +26,12 @@ #ifndef QEMU_VMSTATE_H #define QEMU_VMSTATE_H 1 -typedef void SaveSetParamsHandler(const MigrationParams *params, void * opaque); typedef void SaveStateHandler(QEMUFile *f, void *opaque); typedef int SaveLiveStateHandler(QEMUFile *f, int stage, void *opaque); typedef int LoadStateHandler(QEMUFile *f, void *opaque, int version_id); typedef struct SaveVMHandlers { - SaveSetParamsHandler *set_params; + void (*set_params)(const MigrationParams *params, void * opaque); SaveStateHandler *save_state; SaveLiveStateHandler *save_live_state; LoadStateHandler *load_state; From 226aa6a9b19e5465418fa62c5188acac280f1c13 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Tue, 26 Jun 2012 18:53:29 +0200 Subject: [PATCH 04/13] savevm: remove SaveLiveStateHandler It was used only once, just unfold. Signed-off-by: Juan Quintela --- vmstate.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vmstate.h b/vmstate.h index 5e1a7cc2bc..0e248344fd 100644 --- a/vmstate.h +++ b/vmstate.h @@ -27,13 +27,12 @@ #define QEMU_VMSTATE_H 1 typedef void SaveStateHandler(QEMUFile *f, void *opaque); -typedef int SaveLiveStateHandler(QEMUFile *f, int stage, void *opaque); typedef int LoadStateHandler(QEMUFile *f, void *opaque, int version_id); typedef struct SaveVMHandlers { void (*set_params)(const MigrationParams *params, void * opaque); SaveStateHandler *save_state; - SaveLiveStateHandler *save_live_state; + int (*save_live_state)(QEMUFile *f, int stage, void *opaque); LoadStateHandler *load_state; } SaveVMHandlers; From 9b5bfab05f7c2a56d5908117e8cc62015e6b2c63 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Tue, 26 Jun 2012 19:26:41 +0200 Subject: [PATCH 05/13] savevm: Refactor cancel operation in its own operation Intead of abusing stage with value -1. Signed-off-by: Juan Quintela --- arch_init.c | 11 ++++++----- block-migration.c | 10 ++++++---- savevm.c | 4 ++-- vmstate.h | 1 + 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/arch_init.c b/arch_init.c index 606b848474..f6bc3fe69c 100644 --- a/arch_init.c +++ b/arch_init.c @@ -296,6 +296,11 @@ static void migration_end(void) memory_global_dirty_log_stop(); } +static void ram_migration_cancel(void *opaque) +{ + migration_end(); +} + #define MAX_WAIT 50 /* ms, half buffered_file limit */ static int ram_save_live(QEMUFile *f, int stage, void *opaque) @@ -306,11 +311,6 @@ static int ram_save_live(QEMUFile *f, int stage, void *opaque) int ret; int i; - if (stage < 0) { - migration_end(); - return 0; - } - memory_global_sync_dirty_bitmap(get_system_memory()); if (stage == 1) { @@ -537,6 +537,7 @@ done: SaveVMHandlers savevm_ram_handlers = { .save_live_state = ram_save_live, .load_state = ram_load, + .cancel = ram_migration_cancel, }; #ifdef HAS_AUDIO diff --git a/block-migration.c b/block-migration.c index 00151a07d9..cd8a8dd6cb 100644 --- a/block-migration.c +++ b/block-migration.c @@ -536,6 +536,11 @@ static void blk_mig_cleanup(void) } } +static void block_migration_cancel(void *opaque) +{ + blk_mig_cleanup(); +} + static int block_save_live(QEMUFile *f, int stage, void *opaque) { int ret; @@ -543,10 +548,6 @@ static int block_save_live(QEMUFile *f, int stage, void *opaque) DPRINTF("Enter save live stage %d submitted %d transferred %d\n", stage, block_mig_state.submitted, block_mig_state.transferred); - if (stage < 0) { - blk_mig_cleanup(); - return 0; - } if (block_mig_state.blk_enable != 1) { /* no need to migrate storage */ @@ -713,6 +714,7 @@ SaveVMHandlers savevm_block_handlers = { .set_params = block_set_params, .save_live_state = block_save_live, .load_state = block_load, + .cancel = block_migration_cancel, }; void blk_mig_init(void) diff --git a/savevm.c b/savevm.c index a451be22a6..888c5a2faf 100644 --- a/savevm.c +++ b/savevm.c @@ -1703,8 +1703,8 @@ void qemu_savevm_state_cancel(QEMUFile *f) SaveStateEntry *se; QTAILQ_FOREACH(se, &savevm_handlers, entry) { - if (se->ops && se->ops->save_live_state) { - se->ops->save_live_state(f, -1, se->opaque); + if (se->ops && se->ops->cancel) { + se->ops->cancel(se->opaque); } } } diff --git a/vmstate.h b/vmstate.h index 0e248344fd..1dd42f516a 100644 --- a/vmstate.h +++ b/vmstate.h @@ -33,6 +33,7 @@ typedef struct SaveVMHandlers { void (*set_params)(const MigrationParams *params, void * opaque); SaveStateHandler *save_state; int (*save_live_state)(QEMUFile *f, int stage, void *opaque); + void (*cancel)(void *opaque); LoadStateHandler *load_state; } SaveVMHandlers; From 6bd68781331590c4f2fbd1fdf98e80d7f6588b31 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Wed, 27 Jun 2012 10:59:15 +0200 Subject: [PATCH 06/13] savevm: introduce is_active method Enable the creation of a method to tell migration if that section is active and should be migrate. We use it for blk-migration, that is normally not active. We don't create the method for RAM, as setups without RAM are very strange O:-) Signed-off-by: Juan Quintela --- block-migration.c | 13 ++++++------- savevm.c | 15 +++++++++++++++ vmstate.h | 1 + 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/block-migration.c b/block-migration.c index cd8a8dd6cb..6d37dc157c 100644 --- a/block-migration.c +++ b/block-migration.c @@ -548,13 +548,6 @@ static int block_save_live(QEMUFile *f, int stage, void *opaque) DPRINTF("Enter save live stage %d submitted %d transferred %d\n", stage, block_mig_state.submitted, block_mig_state.transferred); - - if (block_mig_state.blk_enable != 1) { - /* no need to migrate storage */ - qemu_put_be64(f, BLK_MIG_FLAG_EOS); - return 1; - } - if (stage == 1) { init_blk_migration(f); @@ -710,11 +703,17 @@ static void block_set_params(const MigrationParams *params, void *opaque) block_mig_state.blk_enable |= params->shared; } +static bool block_is_active(void *opaque) +{ + return block_mig_state.blk_enable == 1; +} + SaveVMHandlers savevm_block_handlers = { .set_params = block_set_params, .save_live_state = block_save_live, .load_state = block_load, .cancel = block_migration_cancel, + .is_active = block_is_active, }; void blk_mig_init(void) diff --git a/savevm.c b/savevm.c index 888c5a2faf..afa0c9e3ff 100644 --- a/savevm.c +++ b/savevm.c @@ -1576,6 +1576,11 @@ int qemu_savevm_state_begin(QEMUFile *f, if (!se->ops || !se->ops->save_live_state) { continue; } + if (se->ops && se->ops->is_active) { + if (!se->ops->is_active(se->opaque)) { + continue; + } + } /* Section type */ qemu_put_byte(f, QEMU_VM_SECTION_START); qemu_put_be32(f, se->section_id); @@ -1618,6 +1623,11 @@ int qemu_savevm_state_iterate(QEMUFile *f) if (!se->ops || !se->ops->save_live_state) { continue; } + if (se->ops && se->ops->is_active) { + if (!se->ops->is_active(se->opaque)) { + continue; + } + } if (qemu_file_rate_limit(f)) { return 0; } @@ -1658,6 +1668,11 @@ int qemu_savevm_state_complete(QEMUFile *f) if (!se->ops || !se->ops->save_live_state) { continue; } + if (se->ops && se->ops->is_active) { + if (!se->ops->is_active(se->opaque)) { + continue; + } + } trace_savevm_section_start(); /* Section type */ qemu_put_byte(f, QEMU_VM_SECTION_END); diff --git a/vmstate.h b/vmstate.h index 1dd42f516a..96651a5b9c 100644 --- a/vmstate.h +++ b/vmstate.h @@ -35,6 +35,7 @@ typedef struct SaveVMHandlers { int (*save_live_state)(QEMUFile *f, int stage, void *opaque); void (*cancel)(void *opaque); LoadStateHandler *load_state; + bool (*is_active)(void *opaque); } SaveVMHandlers; int register_savevm(DeviceState *dev, From d1315aac6e4df1f472a6f87ef6e310b8c109f498 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Thu, 28 Jun 2012 15:11:57 +0200 Subject: [PATCH 07/13] savevm: split save_live_setup from save_live_state This patch splits stage 1 to its own function for both save_live users, ram and block. It is just a copy of the function, removing the parts of the other stages. Optimizations would came later. Signed-off-by: Juan Quintela --- arch_init.c | 93 ++++++++++++++++++++++++++++++++++------------- block-migration.c | 35 ++++++++++++++---- savevm.c | 4 +- vmstate.h | 1 + 4 files changed, 99 insertions(+), 34 deletions(-) diff --git a/arch_init.c b/arch_init.c index f6bc3fe69c..2a4903c4fa 100644 --- a/arch_init.c +++ b/arch_init.c @@ -303,44 +303,86 @@ static void ram_migration_cancel(void *opaque) #define MAX_WAIT 50 /* ms, half buffered_file limit */ -static int ram_save_live(QEMUFile *f, int stage, void *opaque) +static int ram_save_setup(QEMUFile *f, void *opaque) { ram_addr_t addr; - uint64_t bytes_transferred_last; + RAMBlock *block; double bwidth = 0; int ret; int i; memory_global_sync_dirty_bitmap(get_system_memory()); - if (stage == 1) { - RAMBlock *block; - bytes_transferred = 0; - last_block = NULL; - last_offset = 0; - sort_ram_list(); + bytes_transferred = 0; + last_block = NULL; + last_offset = 0; + sort_ram_list(); - /* Make sure all dirty bits are set */ - QLIST_FOREACH(block, &ram_list.blocks, next) { - for (addr = 0; addr < block->length; addr += TARGET_PAGE_SIZE) { - if (!memory_region_get_dirty(block->mr, addr, TARGET_PAGE_SIZE, - DIRTY_MEMORY_MIGRATION)) { - memory_region_set_dirty(block->mr, addr, TARGET_PAGE_SIZE); - } + /* Make sure all dirty bits are set */ + QLIST_FOREACH(block, &ram_list.blocks, next) { + for (addr = 0; addr < block->length; addr += TARGET_PAGE_SIZE) { + if (!memory_region_get_dirty(block->mr, addr, TARGET_PAGE_SIZE, + DIRTY_MEMORY_MIGRATION)) { + memory_region_set_dirty(block->mr, addr, TARGET_PAGE_SIZE); } } - - memory_global_dirty_log_start(); - - qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE); - - QLIST_FOREACH(block, &ram_list.blocks, next) { - qemu_put_byte(f, strlen(block->idstr)); - qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr)); - qemu_put_be64(f, block->length); - } } + memory_global_dirty_log_start(); + + qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE); + + QLIST_FOREACH(block, &ram_list.blocks, next) { + qemu_put_byte(f, strlen(block->idstr)); + qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr)); + qemu_put_be64(f, block->length); + } + + bwidth = qemu_get_clock_ns(rt_clock); + + i = 0; + while ((ret = qemu_file_rate_limit(f)) == 0) { + int bytes_sent; + + bytes_sent = ram_save_block(f); + bytes_transferred += bytes_sent; + if (bytes_sent == 0) { /* no more blocks */ + break; + } + /* we want to check in the 1st loop, just in case it was the 1st time + and we had to sync the dirty bitmap. + qemu_get_clock_ns() is a bit expensive, so we only check each some + iterations + */ + if ((i & 63) == 0) { + uint64_t t1 = (qemu_get_clock_ns(rt_clock) - bwidth) / 1000000; + if (t1 > MAX_WAIT) { + DPRINTF("big wait: " PRIu64 " milliseconds, %d iterations\n", + t1, i); + break; + } + } + i++; + } + + if (ret < 0) { + return ret; + } + + qemu_put_be64(f, RAM_SAVE_FLAG_EOS); + + return 0; +} + +static int ram_save_live(QEMUFile *f, int stage, void *opaque) +{ + uint64_t bytes_transferred_last; + double bwidth = 0; + int ret; + int i; + + memory_global_sync_dirty_bitmap(get_system_memory()); + bytes_transferred_last = bytes_transferred; bwidth = qemu_get_clock_ns(rt_clock); @@ -535,6 +577,7 @@ done: } SaveVMHandlers savevm_ram_handlers = { + .save_live_setup = ram_save_setup, .save_live_state = ram_save_live, .load_state = ram_load, .cancel = ram_migration_cancel, diff --git a/block-migration.c b/block-migration.c index 6d37dc157c..fc3d1f46c6 100644 --- a/block-migration.c +++ b/block-migration.c @@ -541,6 +541,33 @@ static void block_migration_cancel(void *opaque) blk_mig_cleanup(); } +static int block_save_setup(QEMUFile *f, void *opaque) +{ + int ret; + + DPRINTF("Enter save live setup submitted %d transferred %d\n", + block_mig_state.submitted, block_mig_state.transferred); + + init_blk_migration(f); + + /* start track dirty blocks */ + set_dirty_tracking(1); + + flush_blks(f); + + ret = qemu_file_get_error(f); + if (ret) { + blk_mig_cleanup(); + return ret; + } + + blk_mig_reset_dirty_cursor(); + + qemu_put_be64(f, BLK_MIG_FLAG_EOS); + + return 0; +} + static int block_save_live(QEMUFile *f, int stage, void *opaque) { int ret; @@ -548,13 +575,6 @@ static int block_save_live(QEMUFile *f, int stage, void *opaque) DPRINTF("Enter save live stage %d submitted %d transferred %d\n", stage, block_mig_state.submitted, block_mig_state.transferred); - if (stage == 1) { - init_blk_migration(f); - - /* start track dirty blocks */ - set_dirty_tracking(1); - } - flush_blks(f); ret = qemu_file_get_error(f); @@ -710,6 +730,7 @@ static bool block_is_active(void *opaque) SaveVMHandlers savevm_block_handlers = { .set_params = block_set_params, + .save_live_setup = block_save_setup, .save_live_state = block_save_live, .load_state = block_load, .cancel = block_migration_cancel, diff --git a/savevm.c b/savevm.c index afa0c9e3ff..0b80a94d98 100644 --- a/savevm.c +++ b/savevm.c @@ -1573,7 +1573,7 @@ int qemu_savevm_state_begin(QEMUFile *f, QTAILQ_FOREACH(se, &savevm_handlers, entry) { int len; - if (!se->ops || !se->ops->save_live_state) { + if (!se->ops || !se->ops->save_live_setup) { continue; } if (se->ops && se->ops->is_active) { @@ -1593,7 +1593,7 @@ int qemu_savevm_state_begin(QEMUFile *f, qemu_put_be32(f, se->instance_id); qemu_put_be32(f, se->version_id); - ret = se->ops->save_live_state(f, QEMU_VM_SECTION_START, se->opaque); + ret = se->ops->save_live_setup(f, se->opaque); if (ret < 0) { qemu_savevm_state_cancel(f); return ret; diff --git a/vmstate.h b/vmstate.h index 96651a5b9c..049f2b7a20 100644 --- a/vmstate.h +++ b/vmstate.h @@ -32,6 +32,7 @@ typedef int LoadStateHandler(QEMUFile *f, void *opaque, int version_id); typedef struct SaveVMHandlers { void (*set_params)(const MigrationParams *params, void * opaque); SaveStateHandler *save_state; + int (*save_live_setup)(QEMUFile *f, void *opaque); int (*save_live_state)(QEMUFile *f, int stage, void *opaque); void (*cancel)(void *opaque); LoadStateHandler *load_state; From 16310a3cca7320edb9341c976f7819de0a8c27e0 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Thu, 28 Jun 2012 15:31:37 +0200 Subject: [PATCH 08/13] savevm: split save_live into stage2 and stage3 We split it into 2 functions, foo_live_iterate, and foo_live_complete. At this point, we only remove the bits that are for the other stage, functionally this is equivalent to previous code. Signed-off-by: Juan Quintela --- arch_init.c | 74 +++++++++++++++++++++++-------- block-migration.c | 109 +++++++++++++++++++++++++++------------------- savevm.c | 10 ++--- vmstate.h | 3 +- 4 files changed, 128 insertions(+), 68 deletions(-) diff --git a/arch_init.c b/arch_init.c index 2a4903c4fa..7e04ea5e77 100644 --- a/arch_init.c +++ b/arch_init.c @@ -374,12 +374,13 @@ static int ram_save_setup(QEMUFile *f, void *opaque) return 0; } -static int ram_save_live(QEMUFile *f, int stage, void *opaque) +static int ram_save_iterate(QEMUFile *f, void *opaque) { uint64_t bytes_transferred_last; double bwidth = 0; int ret; int i; + uint64_t expected_time; memory_global_sync_dirty_bitmap(get_system_memory()); @@ -424,28 +425,64 @@ static int ram_save_live(QEMUFile *f, int stage, void *opaque) bwidth = 0.000001; } - /* try transferring iterative blocks of memory */ - if (stage == 3) { - int bytes_sent; + qemu_put_be64(f, RAM_SAVE_FLAG_EOS); - /* flush all remaining blocks regardless of rate limiting */ - while ((bytes_sent = ram_save_block(f)) != 0) { - bytes_transferred += bytes_sent; + expected_time = ram_save_remaining() * TARGET_PAGE_SIZE / bwidth; + + DPRINTF("ram_save_live: expected(" PRIu64 ") <= max(" PRIu64 ")?\n", + expected_time, migrate_max_downtime()); + + return expected_time <= migrate_max_downtime(); +} + +static int ram_save_complete(QEMUFile *f, void *opaque) +{ + double bwidth = 0; + int ret; + int i; + int bytes_sent; + + memory_global_sync_dirty_bitmap(get_system_memory()); + + bwidth = qemu_get_clock_ns(rt_clock); + + i = 0; + while ((ret = qemu_file_rate_limit(f)) == 0) { + bytes_sent = ram_save_block(f); + bytes_transferred += bytes_sent; + if (bytes_sent == 0) { /* no more blocks */ + break; } - memory_global_dirty_log_stop(); + /* we want to check in the 1st loop, just in case it was the 1st time + and we had to sync the dirty bitmap. + qemu_get_clock_ns() is a bit expensive, so we only check each some + iterations + */ + if ((i & 63) == 0) { + uint64_t t1 = (qemu_get_clock_ns(rt_clock) - bwidth) / 1000000; + if (t1 > MAX_WAIT) { + DPRINTF("big wait: " PRIu64 " milliseconds, %d iterations\n", + t1, i); + break; + } + } + i++; } + if (ret < 0) { + return ret; + } + + /* try transferring iterative blocks of memory */ + + /* flush all remaining blocks regardless of rate limiting */ + while ((bytes_sent = ram_save_block(f)) != 0) { + bytes_transferred += bytes_sent; + } + memory_global_dirty_log_stop(); + qemu_put_be64(f, RAM_SAVE_FLAG_EOS); - if (stage == 2) { - uint64_t expected_time; - expected_time = ram_save_remaining() * TARGET_PAGE_SIZE / bwidth; - - DPRINTF("ram_save_live: expected(" PRIu64 ") <= max(" PRIu64 ")?\n", - expected_time, migrate_max_downtime()); - - return expected_time <= migrate_max_downtime(); - } return 0; } @@ -578,7 +615,8 @@ done: SaveVMHandlers savevm_ram_handlers = { .save_live_setup = ram_save_setup, - .save_live_state = ram_save_live, + .save_live_iterate = ram_save_iterate, + .save_live_complete = ram_save_complete, .load_state = ram_load, .cancel = ram_migration_cancel, }; diff --git a/block-migration.c b/block-migration.c index fc3d1f46c6..7def8ab197 100644 --- a/block-migration.c +++ b/block-migration.c @@ -568,12 +568,12 @@ static int block_save_setup(QEMUFile *f, void *opaque) return 0; } -static int block_save_live(QEMUFile *f, int stage, void *opaque) +static int block_save_iterate(QEMUFile *f, void *opaque) { int ret; - DPRINTF("Enter save live stage %d submitted %d transferred %d\n", - stage, block_mig_state.submitted, block_mig_state.transferred); + DPRINTF("Enter save live iterate submitted %d transferred %d\n", + block_mig_state.submitted, block_mig_state.transferred); flush_blks(f); @@ -585,56 +585,76 @@ static int block_save_live(QEMUFile *f, int stage, void *opaque) blk_mig_reset_dirty_cursor(); - if (stage == 2) { - /* control the rate of transfer */ - while ((block_mig_state.submitted + - block_mig_state.read_done) * BLOCK_SIZE < - qemu_file_get_rate_limit(f)) { - if (block_mig_state.bulk_completed == 0) { - /* first finish the bulk phase */ - if (blk_mig_save_bulked_block(f) == 0) { - /* finished saving bulk on all devices */ - block_mig_state.bulk_completed = 1; - } - } else { - if (blk_mig_save_dirty_block(f, 1) == 0) { - /* no more dirty blocks */ - break; - } + /* control the rate of transfer */ + while ((block_mig_state.submitted + + block_mig_state.read_done) * BLOCK_SIZE < + qemu_file_get_rate_limit(f)) { + if (block_mig_state.bulk_completed == 0) { + /* first finish the bulk phase */ + if (blk_mig_save_bulked_block(f) == 0) { + /* finished saving bulk on all devices */ + block_mig_state.bulk_completed = 1; + } + } else { + if (blk_mig_save_dirty_block(f, 1) == 0) { + /* no more dirty blocks */ + break; } - } - - flush_blks(f); - - ret = qemu_file_get_error(f); - if (ret) { - blk_mig_cleanup(); - return ret; } } - if (stage == 3) { - /* we know for sure that save bulk is completed and - all async read completed */ - assert(block_mig_state.submitted == 0); + flush_blks(f); - while (blk_mig_save_dirty_block(f, 0) != 0); + ret = qemu_file_get_error(f); + if (ret) { blk_mig_cleanup(); - - /* report completion */ - qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS); - - ret = qemu_file_get_error(f); - if (ret) { - return ret; - } - - DPRINTF("Block migration completed\n"); + return ret; } qemu_put_be64(f, BLK_MIG_FLAG_EOS); - return ((stage == 2) && is_stage2_completed()); + return is_stage2_completed(); +} + +static int block_save_complete(QEMUFile *f, void *opaque) +{ + int ret; + + DPRINTF("Enter save live complete submitted %d transferred %d\n", + block_mig_state.submitted, block_mig_state.transferred); + + flush_blks(f); + + ret = qemu_file_get_error(f); + if (ret) { + blk_mig_cleanup(); + return ret; + } + + blk_mig_reset_dirty_cursor(); + + /* we know for sure that save bulk is completed and + all async read completed */ + assert(block_mig_state.submitted == 0); + + while (blk_mig_save_dirty_block(f, 0) != 0) { + /* Do nothing */ + } + blk_mig_cleanup(); + + /* report completion */ + qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS); + + ret = qemu_file_get_error(f); + if (ret) { + return ret; + } + + DPRINTF("Block migration completed\n"); + + qemu_put_be64(f, BLK_MIG_FLAG_EOS); + + return 0; } static int block_load(QEMUFile *f, void *opaque, int version_id) @@ -731,7 +751,8 @@ static bool block_is_active(void *opaque) SaveVMHandlers savevm_block_handlers = { .set_params = block_set_params, .save_live_setup = block_save_setup, - .save_live_state = block_save_live, + .save_live_iterate = block_save_iterate, + .save_live_complete = block_save_complete, .load_state = block_load, .cancel = block_migration_cancel, .is_active = block_is_active, diff --git a/savevm.c b/savevm.c index 0b80a94d98..6e82b2d3e3 100644 --- a/savevm.c +++ b/savevm.c @@ -1236,7 +1236,7 @@ int register_savevm_live(DeviceState *dev, se->vmsd = NULL; se->no_migrate = 0; /* if this is a live_savem then set is_ram */ - if (ops->save_live_state != NULL) { + if (ops->save_live_setup != NULL) { se->is_ram = 1; } @@ -1620,7 +1620,7 @@ int qemu_savevm_state_iterate(QEMUFile *f) int ret = 1; QTAILQ_FOREACH(se, &savevm_handlers, entry) { - if (!se->ops || !se->ops->save_live_state) { + if (!se->ops || !se->ops->save_live_iterate) { continue; } if (se->ops && se->ops->is_active) { @@ -1636,7 +1636,7 @@ int qemu_savevm_state_iterate(QEMUFile *f) qemu_put_byte(f, QEMU_VM_SECTION_PART); qemu_put_be32(f, se->section_id); - ret = se->ops->save_live_state(f, QEMU_VM_SECTION_PART, se->opaque); + ret = se->ops->save_live_iterate(f, se->opaque); trace_savevm_section_end(se->section_id); if (ret <= 0) { @@ -1665,7 +1665,7 @@ int qemu_savevm_state_complete(QEMUFile *f) cpu_synchronize_all_states(); QTAILQ_FOREACH(se, &savevm_handlers, entry) { - if (!se->ops || !se->ops->save_live_state) { + if (!se->ops || !se->ops->save_live_complete) { continue; } if (se->ops && se->ops->is_active) { @@ -1678,7 +1678,7 @@ int qemu_savevm_state_complete(QEMUFile *f) qemu_put_byte(f, QEMU_VM_SECTION_END); qemu_put_be32(f, se->section_id); - ret = se->ops->save_live_state(f, QEMU_VM_SECTION_END, se->opaque); + ret = se->ops->save_live_complete(f, se->opaque); trace_savevm_section_end(se->section_id); if (ret < 0) { return ret; diff --git a/vmstate.h b/vmstate.h index 049f2b7a20..5bd2b762ab 100644 --- a/vmstate.h +++ b/vmstate.h @@ -33,7 +33,8 @@ typedef struct SaveVMHandlers { void (*set_params)(const MigrationParams *params, void * opaque); SaveStateHandler *save_state; int (*save_live_setup)(QEMUFile *f, void *opaque); - int (*save_live_state)(QEMUFile *f, int stage, void *opaque); + int (*save_live_iterate)(QEMUFile *f, void *opaque); + int (*save_live_complete)(QEMUFile *f, void *opaque); void (*cancel)(void *opaque); LoadStateHandler *load_state; bool (*is_active)(void *opaque); From 5910d1e6cbb190809d5dd0d75bb5e8398dd3ffb7 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Thu, 28 Jun 2012 19:41:52 +0200 Subject: [PATCH 09/13] ram: save_live_setup() don't need to sent pages We should send pages on interate phase, not in setup one. This was a "bug". Just removing the loop does what we want. Tested that it works with current ram_load(). Signed-off-by: Juan Quintela --- arch_init.c | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/arch_init.c b/arch_init.c index 7e04ea5e77..2a6fc77a8e 100644 --- a/arch_init.c +++ b/arch_init.c @@ -307,9 +307,6 @@ static int ram_save_setup(QEMUFile *f, void *opaque) { ram_addr_t addr; RAMBlock *block; - double bwidth = 0; - int ret; - int i; memory_global_sync_dirty_bitmap(get_system_memory()); @@ -338,37 +335,6 @@ static int ram_save_setup(QEMUFile *f, void *opaque) qemu_put_be64(f, block->length); } - bwidth = qemu_get_clock_ns(rt_clock); - - i = 0; - while ((ret = qemu_file_rate_limit(f)) == 0) { - int bytes_sent; - - bytes_sent = ram_save_block(f); - bytes_transferred += bytes_sent; - if (bytes_sent == 0) { /* no more blocks */ - break; - } - /* we want to check in the 1st loop, just in case it was the 1st time - and we had to sync the dirty bitmap. - qemu_get_clock_ns() is a bit expensive, so we only check each some - iterations - */ - if ((i & 63) == 0) { - uint64_t t1 = (qemu_get_clock_ns(rt_clock) - bwidth) / 1000000; - if (t1 > MAX_WAIT) { - DPRINTF("big wait: " PRIu64 " milliseconds, %d iterations\n", - t1, i); - break; - } - } - i++; - } - - if (ret < 0) { - return ret; - } - qemu_put_be64(f, RAM_SAVE_FLAG_EOS); return 0; From b81e712d252cf41a0825f5abad0ce2f4746864f1 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Thu, 28 Jun 2012 19:48:09 +0200 Subject: [PATCH 10/13] ram: save_live_complete() only do one loop We were doing the same loop that stage2, and a new one for stage3. We only need the one for stage3. Signed-off-by: Juan Quintela --- arch_init.c | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/arch_init.c b/arch_init.c index 2a6fc77a8e..3c38a52168 100644 --- a/arch_init.c +++ b/arch_init.c @@ -403,42 +403,10 @@ static int ram_save_iterate(QEMUFile *f, void *opaque) static int ram_save_complete(QEMUFile *f, void *opaque) { - double bwidth = 0; - int ret; - int i; int bytes_sent; memory_global_sync_dirty_bitmap(get_system_memory()); - bwidth = qemu_get_clock_ns(rt_clock); - - i = 0; - while ((ret = qemu_file_rate_limit(f)) == 0) { - bytes_sent = ram_save_block(f); - bytes_transferred += bytes_sent; - if (bytes_sent == 0) { /* no more blocks */ - break; - } - /* we want to check in the 1st loop, just in case it was the 1st time - and we had to sync the dirty bitmap. - qemu_get_clock_ns() is a bit expensive, so we only check each some - iterations - */ - if ((i & 63) == 0) { - uint64_t t1 = (qemu_get_clock_ns(rt_clock) - bwidth) / 1000000; - if (t1 > MAX_WAIT) { - DPRINTF("big wait: " PRIu64 " milliseconds, %d iterations\n", - t1, i); - break; - } - } - i++; - } - - if (ret < 0) { - return ret; - } - /* try transferring iterative blocks of memory */ /* flush all remaining blocks regardless of rate limiting */ From 00d94f3f7648bc3d7126bc6b84d42e22194d9230 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Thu, 28 Jun 2012 19:51:17 +0200 Subject: [PATCH 11/13] ram: iterate phase We only need to synchronize the bitmap when the number of dirty pages is low. Not every time that we call the function. Signed-off-by: Juan Quintela --- arch_init.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/arch_init.c b/arch_init.c index 3c38a52168..b08c615dfd 100644 --- a/arch_init.c +++ b/arch_init.c @@ -348,8 +348,6 @@ static int ram_save_iterate(QEMUFile *f, void *opaque) int i; uint64_t expected_time; - memory_global_sync_dirty_bitmap(get_system_memory()); - bytes_transferred_last = bytes_transferred; bwidth = qemu_get_clock_ns(rt_clock); @@ -398,7 +396,13 @@ static int ram_save_iterate(QEMUFile *f, void *opaque) DPRINTF("ram_save_live: expected(" PRIu64 ") <= max(" PRIu64 ")?\n", expected_time, migrate_max_downtime()); - return expected_time <= migrate_max_downtime(); + if (expected_time <= migrate_max_downtime()) { + memory_global_sync_dirty_bitmap(get_system_memory()); + expected_time = ram_save_remaining() * TARGET_PAGE_SIZE / bwidth; + + return expected_time <= migrate_max_downtime(); + } + return 0; } static int ram_save_complete(QEMUFile *f, void *opaque) From 6d2fe810ed995dc79090c2ba179f4f0c694243b1 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Thu, 28 Jun 2012 19:58:47 +0200 Subject: [PATCH 12/13] ram: save_live_setup() we don't need to synchronize the dirty bitmap. 1st: we were synchonizing the dirty bitmap before calling memory_global_dirty_log_start(). 2nd: We are marking all pages as dirty anywhere, no reason to go through all the bitmap to "mark" dirty same pages twice. So, call removed. Signed-off-by: Juan Quintela --- arch_init.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch_init.c b/arch_init.c index b08c615dfd..8cbf8f4665 100644 --- a/arch_init.c +++ b/arch_init.c @@ -308,8 +308,6 @@ static int ram_save_setup(QEMUFile *f, void *opaque) ram_addr_t addr; RAMBlock *block; - memory_global_sync_dirty_bitmap(get_system_memory()); - bytes_transferred = 0; last_block = NULL; last_offset = 0; From 6c779f22a93cc6e4565b940ef616e3efc5b50ba5 Mon Sep 17 00:00:00 2001 From: Orit Wasserman Date: Tue, 10 Jul 2012 12:37:13 +0300 Subject: [PATCH 13/13] Change ram_save_block to return -1 if there are no more changes It will return 0 if the page is unmodifed. Signed-off-by: Orit Wasserman --- arch_init.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/arch_init.c b/arch_init.c index 8cbf8f4665..78cdf50c52 100644 --- a/arch_init.c +++ b/arch_init.c @@ -184,11 +184,19 @@ static void save_block_hdr(QEMUFile *f, RAMBlock *block, ram_addr_t offset, static RAMBlock *last_block; static ram_addr_t last_offset; +/* + * ram_save_block: Writes a page of memory to the stream f + * + * Returns: 0: if the page hasn't changed + * -1: if there are no more dirty pages + * n: the amount of bytes written in other case + */ + static int ram_save_block(QEMUFile *f) { RAMBlock *block = last_block; ram_addr_t offset = last_offset; - int bytes_sent = 0; + int bytes_sent = -1; MemoryRegion *mr; if (!block) @@ -354,10 +362,11 @@ static int ram_save_iterate(QEMUFile *f, void *opaque) int bytes_sent; bytes_sent = ram_save_block(f); - bytes_transferred += bytes_sent; - if (bytes_sent == 0) { /* no more blocks */ + /* no more blocks to sent */ + if (bytes_sent < 0) { break; } + bytes_transferred += bytes_sent; /* we want to check in the 1st loop, just in case it was the 1st time and we had to sync the dirty bitmap. qemu_get_clock_ns() is a bit expensive, so we only check each some @@ -405,14 +414,19 @@ static int ram_save_iterate(QEMUFile *f, void *opaque) static int ram_save_complete(QEMUFile *f, void *opaque) { - int bytes_sent; - memory_global_sync_dirty_bitmap(get_system_memory()); /* try transferring iterative blocks of memory */ /* flush all remaining blocks regardless of rate limiting */ - while ((bytes_sent = ram_save_block(f)) != 0) { + while (true) { + int bytes_sent; + + bytes_sent = ram_save_block(f); + /* no more blocks to sent */ + if (bytes_sent < 0) { + break; + } bytes_transferred += bytes_sent; } memory_global_dirty_log_stop();