ShuriZma
/
xemu
mirror of https://github.com/xemu-project/xemu.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Pull request 

Rebase notes:
 
 011/36:[0003] [FC] 'block/backup: upgrade copy_bitmap to BdrvDirtyBitmap'
 016/36:[----] [-C] 'iotests: Add virtio-scsi device helper'
 017/36:[0002] [FC] 'iotests: add test 257 for bitmap-mode backups'
 030/36:[0011] [FC] 'block/backup: teach TOP to never copy unallocated regions'
 032/36:[0018] [FC] 'iotests/257: test traditional sync modes'
 
 11: A new hbitmap call was added late in 4.1, changed to
     bdrv_dirty_bitmap_next_zero.
 16: Context-only (self.has_quit is new context in 040)
 17: Removed 'auto' to follow upstream trends in iotest fashion
 30: Handled explicitly on-list with R-B from Max.
 32: Fix capitalization in test, as mentioned on-list.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEE+ber27ys35W+dsvQfe+BBqr8OQ4FAl1XOE0ACgkQfe+BBqr8
 OQ5vZw//RyNt7SDRyB2X1eQnD1LGNunH7ZgucCNccFf0QlbfEYAQWCobylcLw5jk
 WH3GAbo41YgN5yhqTYKc17bXGTdmrQjE5INmpL1X1Eq85HqOGSxK94lQ3EBodbAS
 ktAriK2b9xW/DZ4OvP6pQ5U6PkQZk/GyzvcgteFjqbR6gGCpGGPr97nRjfn7aNzn
 7UuEl30CrdEZyVEkxIxTWnRWOSGXv3zPupkTzPb4PQ3nOEHX3iq+rmEkp9mE46dW
 VJucWC+SXGfI5+Lpsofa7+oJ2Cqa6spFRYk/3gLmahvm5bDT0CYG5XKCTSFkFedO
 lsibS/LkdecHDsTYlU5eIIhcD5l7hhPd+A4dlSQyNMqqHY90wPsaTgAy9czURjIO
 H37DJfAld1z8AE5ekXisdTIOypa7pmnS+4WyB73ZynJrCyxNRjI2heNavnrkPDMG
 IZQT64wGmaaleQZdOo30rAqp8Lsh7XTag0RRijlw8OdngYQyMzyp94uU0hm4grMu
 6UJD0w81jxacyRZ/rfUT4BLKwzly4vfS1LlVoNaQk5fgkuJeKbfYJnVgL7ZNfzvR
 +DTg1NHOudHkgUaW+LtV2F74D5k1xPYSpT+1WVDP6n8fzXEMyQPLb6RZcWhAazka
 8vAiRSLygoXdDzhQHqvSIqweuZPXOrpRjU1Iiva3r8wcxjKGj7M=
 =4WRP
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/jnsnow/tags/bitmaps-pull-request' into staging

Pull request

Rebase notes:

011/36:[0003] [FC] 'block/backup: upgrade copy_bitmap to BdrvDirtyBitmap'
016/36:[----] [-C] 'iotests: Add virtio-scsi device helper'
017/36:[0002] [FC] 'iotests: add test 257 for bitmap-mode backups'
030/36:[0011] [FC] 'block/backup: teach TOP to never copy unallocated regions'
032/36:[0018] [FC] 'iotests/257: test traditional sync modes'

11: A new hbitmap call was added late in 4.1, changed to
    bdrv_dirty_bitmap_next_zero.
16: Context-only (self.has_quit is new context in 040)
17: Removed 'auto' to follow upstream trends in iotest fashion
30: Handled explicitly on-list with R-B from Max.
32: Fix capitalization in test, as mentioned on-list.

# gpg: Signature made Sat 17 Aug 2019 00:12:13 BST
# gpg:                using RSA key F9B7ABDBBCACDF95BE76CBD07DEF8106AAFC390E
# gpg: Good signature from "John Snow (John Huston) <jsnow@redhat.com>" [full]
# Primary key fingerprint: FAEB 9711 A12C F475 812F  18F2 88A9 064D 1835 61EB
#      Subkey fingerprint: F9B7 ABDB BCAC DF95 BE76  CBD0 7DEF 8106 AAFC 390E

* remotes/jnsnow/tags/bitmaps-pull-request: (36 commits)
  tests/test-hbitmap: test next_zero and _next_dirty_area after truncate
  block/backup: refactor write_flags
  block/backup: deal with zero detection
  qapi: add dirty-bitmaps to query-named-block-nodes result
  iotests/257: test traditional sync modes
  block/backup: support bitmap sync modes for non-bitmap backups
  block/backup: teach TOP to never copy unallocated regions
  block/backup: add backup_is_cluster_allocated
  block/backup: centralize copy_bitmap initialization
  block/backup: improve sync=bitmap work estimates
  iotests/257: test API failures
  block/backup: hoist bitmap check into QMP interface
  iotests/257: Refactor backup helpers
  iotests/257: add EmulatedBitmap class
  iotests/257: add Pattern class
  iotests: test bitmap moving inside 254
  qapi: implement block-dirty-bitmap-remove transaction action
  blockdev: reduce aio_context locked sections in bitmap add/remove
  block/backup: loosen restriction on readonly bitmaps
  iotests: add test 257 for bitmap-mode backups
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell 2019-08-19 10:55:03 +01:00
parent afd7605393 a5f8a60b3e
commit 1f37316238
29 changed files with 6878 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
block.c
View File

@ -5346,7 +5346,7 @@ static void coroutine_fn bdrv_co_invalidate_cache(BlockDriverState *bs,
for (bm = bdrv_dirty_bitmap_next(bs, NULL); bm; for (bm = bdrv_dirty_bitmap_next(bs, NULL); bm;
bm = bdrv_dirty_bitmap_next(bs, bm)) bm = bdrv_dirty_bitmap_next(bs, bm))
{ {
bdrv_dirty_bitmap_set_migration(bm, false); bdrv_dirty_bitmap_skip_store(bm, false);
} }
ret = refresh_total_sectors(bs, bs->total_sectors); ret = refresh_total_sectors(bs, bs->total_sectors);

312
block/backup.c
View File

@ -38,24 +38,26 @@ typedef struct CowRequest {
typedef struct BackupBlockJob { typedef struct BackupBlockJob {
BlockJob common; BlockJob common;
BlockBackend *target; BlockBackend *target;
/* bitmap for sync=incremental */
BdrvDirtyBitmap *sync_bitmap; BdrvDirtyBitmap *sync_bitmap;
BdrvDirtyBitmap *copy_bitmap;
MirrorSyncMode sync_mode; MirrorSyncMode sync_mode;
BitmapSyncMode bitmap_mode;
BlockdevOnError on_source_error; BlockdevOnError on_source_error;
BlockdevOnError on_target_error; BlockdevOnError on_target_error;
CoRwlock flush_rwlock; CoRwlock flush_rwlock;
uint64_t len; uint64_t len;
uint64_t bytes_read; uint64_t bytes_read;
int64_t cluster_size; int64_t cluster_size;
bool compress;
NotifierWithReturn before_write; NotifierWithReturn before_write;
QLIST_HEAD(, CowRequest) inflight_reqs; QLIST_HEAD(, CowRequest) inflight_reqs;
HBitmap *copy_bitmap;
bool use_copy_range; bool use_copy_range;
int64_t copy_range_size; int64_t copy_range_size;
bool serialize_target_writes; BdrvRequestFlags write_flags;
bool initializing_bitmap;
} BackupBlockJob; } BackupBlockJob;
static const BlockJobDriver backup_job_driver; static const BlockJobDriver backup_job_driver;
@ -110,10 +112,9 @@ static int coroutine_fn backup_cow_with_bounce_buffer(BackupBlockJob *job,
BlockBackend *blk = job->common.blk; BlockBackend *blk = job->common.blk;
int nbytes; int nbytes;
int read_flags = is_write_notifier ? BDRV_REQ_NO_SERIALISING : 0; int read_flags = is_write_notifier ? BDRV_REQ_NO_SERIALISING : 0;
int write_flags = job->serialize_target_writes ? BDRV_REQ_SERIALISING : 0;
assert(QEMU_IS_ALIGNED(start, job->cluster_size)); assert(QEMU_IS_ALIGNED(start, job->cluster_size));
hbitmap_reset(job->copy_bitmap, start, job->cluster_size); bdrv_reset_dirty_bitmap(job->copy_bitmap, start, job->cluster_size);
nbytes = MIN(job->cluster_size, job->len - start); nbytes = MIN(job->cluster_size, job->len - start);
if (!*bounce_buffer) { if (!*bounce_buffer) {
*bounce_buffer = blk_blockalign(blk, job->cluster_size); *bounce_buffer = blk_blockalign(blk, job->cluster_size);
@ -128,14 +129,8 @@ static int coroutine_fn backup_cow_with_bounce_buffer(BackupBlockJob *job,
goto fail; goto fail;
} }
if (buffer_is_zero(*bounce_buffer, nbytes)) { ret = blk_co_pwrite(job->target, start, nbytes, *bounce_buffer,
ret = blk_co_pwrite_zeroes(job->target, start, job->write_flags);
nbytes, write_flags | BDRV_REQ_MAY_UNMAP);
} else {
ret = blk_co_pwrite(job->target, start,
nbytes, *bounce_buffer, write_flags |
(job->compress ? BDRV_REQ_WRITE_COMPRESSED : 0));
}
if (ret < 0) { if (ret < 0) {
trace_backup_do_cow_write_fail(job, start, ret); trace_backup_do_cow_write_fail(job, start, ret);
if (error_is_read) { if (error_is_read) {
@ -146,7 +141,7 @@ static int coroutine_fn backup_cow_with_bounce_buffer(BackupBlockJob *job,
return nbytes; return nbytes;
fail: fail:
hbitmap_set(job->copy_bitmap, start, job->cluster_size); bdrv_set_dirty_bitmap(job->copy_bitmap, start, job->cluster_size);
return ret; return ret;
} }
@ -163,24 +158,96 @@ static int coroutine_fn backup_cow_with_offload(BackupBlockJob *job,
BlockBackend *blk = job->common.blk; BlockBackend *blk = job->common.blk;
int nbytes; int nbytes;
int read_flags = is_write_notifier ? BDRV_REQ_NO_SERIALISING : 0; int read_flags = is_write_notifier ? BDRV_REQ_NO_SERIALISING : 0;
int write_flags = job->serialize_target_writes ? BDRV_REQ_SERIALISING : 0;
assert(QEMU_IS_ALIGNED(job->copy_range_size, job->cluster_size)); assert(QEMU_IS_ALIGNED(job->copy_range_size, job->cluster_size));
assert(QEMU_IS_ALIGNED(start, job->cluster_size)); assert(QEMU_IS_ALIGNED(start, job->cluster_size));
nbytes = MIN(job->copy_range_size, end - start); nbytes = MIN(job->copy_range_size, end - start);
nr_clusters = DIV_ROUND_UP(nbytes, job->cluster_size); nr_clusters = DIV_ROUND_UP(nbytes, job->cluster_size);
hbitmap_reset(job->copy_bitmap, start, job->cluster_size * nr_clusters); bdrv_reset_dirty_bitmap(job->copy_bitmap, start,
job->cluster_size * nr_clusters);
ret = blk_co_copy_range(blk, start, job->target, start, nbytes, ret = blk_co_copy_range(blk, start, job->target, start, nbytes,
read_flags, write_flags); read_flags, job->write_flags);
if (ret < 0) { if (ret < 0) {
trace_backup_do_cow_copy_range_fail(job, start, ret); trace_backup_do_cow_copy_range_fail(job, start, ret);
hbitmap_set(job->copy_bitmap, start, job->cluster_size * nr_clusters); bdrv_set_dirty_bitmap(job->copy_bitmap, start,
job->cluster_size * nr_clusters);
return ret; return ret;
} }
return nbytes; return nbytes;
} }
/*
* Check if the cluster starting at offset is allocated or not.
* return via pnum the number of contiguous clusters sharing this allocation.
*/
static int backup_is_cluster_allocated(BackupBlockJob *s, int64_t offset,
int64_t *pnum)
{
BlockDriverState *bs = blk_bs(s->common.blk);
int64_t count, total_count = 0;
int64_t bytes = s->len - offset;
int ret;
assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
while (true) {
ret = bdrv_is_allocated(bs, offset, bytes, &count);
if (ret < 0) {
return ret;
}
total_count += count;
if (ret || count == 0) {
/*
* ret: partial segment(s) are considered allocated.
* otherwise: unallocated tail is treated as an entire segment.
*/
*pnum = DIV_ROUND_UP(total_count, s->cluster_size);
return ret;
}
/* Unallocated segment(s) with uncertain following segment(s) */
if (total_count >= s->cluster_size) {
*pnum = total_count / s->cluster_size;
return 0;
}
offset += count;
bytes -= count;
}
}
/**
* Reset bits in copy_bitmap starting at offset if they represent unallocated
* data in the image. May reset subsequent contiguous bits.
* @return 0 when the cluster at @offset was unallocated,
* 1 otherwise, and -ret on error.
*/
static int64_t backup_bitmap_reset_unallocated(BackupBlockJob *s,
int64_t offset, int64_t *count)
{
int ret;
int64_t clusters, bytes, estimate;
ret = backup_is_cluster_allocated(s, offset, &clusters);
if (ret < 0) {
return ret;
}
bytes = clusters * s->cluster_size;
if (!ret) {
bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
estimate = bdrv_get_dirty_count(s->copy_bitmap);
job_progress_set_remaining(&s->common.job, estimate);
}
*count = bytes;
return ret;
}
static int coroutine_fn backup_do_cow(BackupBlockJob *job, static int coroutine_fn backup_do_cow(BackupBlockJob *job,
int64_t offset, uint64_t bytes, int64_t offset, uint64_t bytes,
bool *error_is_read, bool *error_is_read,
@ -190,6 +257,7 @@ static int coroutine_fn backup_do_cow(BackupBlockJob *job,
int ret = 0; int ret = 0;
int64_t start, end; /* bytes */ int64_t start, end; /* bytes */
void *bounce_buffer = NULL; void *bounce_buffer = NULL;
int64_t status_bytes;
qemu_co_rwlock_rdlock(&job->flush_rwlock); qemu_co_rwlock_rdlock(&job->flush_rwlock);
@ -204,17 +272,29 @@ static int coroutine_fn backup_do_cow(BackupBlockJob *job,
while (start < end) { while (start < end) {
int64_t dirty_end; int64_t dirty_end;
if (!hbitmap_get(job->copy_bitmap, start)) { if (!bdrv_dirty_bitmap_get(job->copy_bitmap, start)) {
trace_backup_do_cow_skip(job, start); trace_backup_do_cow_skip(job, start);
start += job->cluster_size; start += job->cluster_size;
continue; /* already copied */ continue; /* already copied */
} }
dirty_end = hbitmap_next_zero(job->copy_bitmap, start, (end - start)); dirty_end = bdrv_dirty_bitmap_next_zero(job->copy_bitmap, start,
(end - start));
if (dirty_end < 0) { if (dirty_end < 0) {
dirty_end = end; dirty_end = end;
} }
if (job->initializing_bitmap) {
ret = backup_bitmap_reset_unallocated(job, start, &status_bytes);
if (ret == 0) {
trace_backup_do_cow_skip_range(job, start, status_bytes);
start += status_bytes;
continue;
}
/* Clamp to known allocated region */
dirty_end = MIN(dirty_end, start + status_bytes);
}
trace_backup_do_cow_process(job, start); trace_backup_do_cow_process(job, start);
if (job->use_copy_range) { if (job->use_copy_range) {
@ -273,15 +353,29 @@ static void backup_cleanup_sync_bitmap(BackupBlockJob *job, int ret)
{ {
BdrvDirtyBitmap *bm; BdrvDirtyBitmap *bm;
BlockDriverState *bs = blk_bs(job->common.blk); BlockDriverState *bs = blk_bs(job->common.blk);
bool sync = (((ret == 0) || (job->bitmap_mode == BITMAP_SYNC_MODE_ALWAYS)) \
&& (job->bitmap_mode != BITMAP_SYNC_MODE_NEVER));
if (ret < 0) { if (sync) {
/* Merge the successor back into the parent, delete nothing. */ /*
bm = bdrv_reclaim_dirty_bitmap(bs, job->sync_bitmap, NULL); * We succeeded, or we always intended to sync the bitmap.
assert(bm); * Delete this bitmap and install the child.
} else { */
/* Everything is fine, delete this bitmap and install the backup. */
bm = bdrv_dirty_bitmap_abdicate(bs, job->sync_bitmap, NULL); bm = bdrv_dirty_bitmap_abdicate(bs, job->sync_bitmap, NULL);
assert(bm); } else {
/*
* We failed, or we never intended to sync the bitmap anyway.
* Merge the successor back into the parent, keeping all data.
*/
bm = bdrv_reclaim_dirty_bitmap(bs, job->sync_bitmap, NULL);
}
assert(bm);
if (ret < 0 && job->bitmap_mode == BITMAP_SYNC_MODE_ALWAYS) {
/* If we failed and synced, merge in the bits we didn't copy: */
bdrv_dirty_bitmap_merge_internal(bm, job->copy_bitmap,
NULL, true);
} }
} }
@ -304,14 +398,16 @@ static void backup_abort(Job *job)
static void backup_clean(Job *job) static void backup_clean(Job *job)
{ {
BackupBlockJob *s = container_of(job, BackupBlockJob, common.job); BackupBlockJob *s = container_of(job, BackupBlockJob, common.job);
BlockDriverState *bs = blk_bs(s->common.blk);
if (s->copy_bitmap) {
bdrv_release_dirty_bitmap(bs, s->copy_bitmap);
s->copy_bitmap = NULL;
}
assert(s->target); assert(s->target);
blk_unref(s->target); blk_unref(s->target);
s->target = NULL; s->target = NULL;
if (s->copy_bitmap) {
hbitmap_free(s->copy_bitmap);
s->copy_bitmap = NULL;
}
} }
void backup_do_checkpoint(BlockJob *job, Error **errp) void backup_do_checkpoint(BlockJob *job, Error **errp)
@ -326,7 +422,7 @@ void backup_do_checkpoint(BlockJob *job, Error **errp)
return; return;
} }
hbitmap_set(backup_job->copy_bitmap, 0, backup_job->len); bdrv_set_dirty_bitmap(backup_job->copy_bitmap, 0, backup_job->len);
} }
static void backup_drain(BlockJob *job) static void backup_drain(BlockJob *job)
@ -377,77 +473,57 @@ static bool coroutine_fn yield_and_check(BackupBlockJob *job)
return false; return false;
} }
static bool bdrv_is_unallocated_range(BlockDriverState *bs,
int64_t offset, int64_t bytes)
{
int64_t end = offset + bytes;
while (offset < end && !bdrv_is_allocated(bs, offset, bytes, &bytes)) {
if (bytes == 0) {
return true;
}
offset += bytes;
bytes = end - offset;
}
return offset >= end;
}
static int coroutine_fn backup_loop(BackupBlockJob *job) static int coroutine_fn backup_loop(BackupBlockJob *job)
{ {
int ret;
bool error_is_read; bool error_is_read;
int64_t offset; int64_t offset;
HBitmapIter hbi; BdrvDirtyBitmapIter *bdbi;
BlockDriverState *bs = blk_bs(job->common.blk); int ret = 0;
hbitmap_iter_init(&hbi, job->copy_bitmap, 0);
while ((offset = hbitmap_iter_next(&hbi)) != -1) {
if (job->sync_mode == MIRROR_SYNC_MODE_TOP &&
bdrv_is_unallocated_range(bs, offset, job->cluster_size))
{
hbitmap_reset(job->copy_bitmap, offset, job->cluster_size);
continue;
}
bdbi = bdrv_dirty_iter_new(job->copy_bitmap);
while ((offset = bdrv_dirty_iter_next(bdbi)) != -1) {
do { do {
if (yield_and_check(job)) { if (yield_and_check(job)) {
return 0; goto out;
} }
ret = backup_do_cow(job, offset, ret = backup_do_cow(job, offset,
job->cluster_size, &error_is_read, false); job->cluster_size, &error_is_read, false);
if (ret < 0 && backup_error_action(job, error_is_read, -ret) == if (ret < 0 && backup_error_action(job, error_is_read, -ret) ==
BLOCK_ERROR_ACTION_REPORT) BLOCK_ERROR_ACTION_REPORT)
{ {
return ret; goto out;
} }
} while (ret < 0); } while (ret < 0);
} }
return 0; out:
bdrv_dirty_iter_free(bdbi);
return ret;
} }
/* init copy_bitmap from sync_bitmap */ static void backup_init_copy_bitmap(BackupBlockJob *job)
static void backup_incremental_init_copy_bitmap(BackupBlockJob *job)
{ {
uint64_t offset = 0; bool ret;
uint64_t bytes = job->len; uint64_t estimate;
while (bdrv_dirty_bitmap_next_dirty_area(job->sync_bitmap, if (job->sync_mode == MIRROR_SYNC_MODE_BITMAP) {
&offset, &bytes)) ret = bdrv_dirty_bitmap_merge_internal(job->copy_bitmap,
{ job->sync_bitmap,
hbitmap_set(job->copy_bitmap, offset, bytes); NULL, true);
assert(ret);
offset += bytes; } else {
if (offset >= job->len) { if (job->sync_mode == MIRROR_SYNC_MODE_TOP) {
break; /*
* We can't hog the coroutine to initialize this thoroughly.
* Set a flag and resume work when we are able to yield safely.
*/
job->initializing_bitmap = true;
} }
bytes = job->len - offset; bdrv_set_dirty_bitmap(job->copy_bitmap, 0, job->len);
} }
/* TODO job_progress_set_remaining() would make more sense */ estimate = bdrv_get_dirty_count(job->copy_bitmap);
job_progress_update(&job->common.job, job_progress_set_remaining(&job->common.job, estimate);
job->len - hbitmap_count(job->copy_bitmap));
} }
static int coroutine_fn backup_run(Job *job, Error **errp) static int coroutine_fn backup_run(Job *job, Error **errp)
@ -459,17 +535,31 @@ static int coroutine_fn backup_run(Job *job, Error **errp)
QLIST_INIT(&s->inflight_reqs); QLIST_INIT(&s->inflight_reqs);
qemu_co_rwlock_init(&s->flush_rwlock); qemu_co_rwlock_init(&s->flush_rwlock);
job_progress_set_remaining(job, s->len); backup_init_copy_bitmap(s);
if (s->sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) {
backup_incremental_init_copy_bitmap(s);
} else {
hbitmap_set(s->copy_bitmap, 0, s->len);
}
s->before_write.notify = backup_before_write_notify; s->before_write.notify = backup_before_write_notify;
bdrv_add_before_write_notifier(bs, &s->before_write); bdrv_add_before_write_notifier(bs, &s->before_write);
if (s->sync_mode == MIRROR_SYNC_MODE_TOP) {
int64_t offset = 0;
int64_t count;
for (offset = 0; offset < s->len; ) {
if (yield_and_check(s)) {
ret = -ECANCELED;
goto out;
}
ret = backup_bitmap_reset_unallocated(s, offset, &count);
if (ret < 0) {
goto out;
}
offset += count;
}
s->initializing_bitmap = false;
}
if (s->sync_mode == MIRROR_SYNC_MODE_NONE) { if (s->sync_mode == MIRROR_SYNC_MODE_NONE) {
/* All bits are set in copy_bitmap to allow any cluster to be copied. /* All bits are set in copy_bitmap to allow any cluster to be copied.
* This does not actually require them to be copied. */ * This does not actually require them to be copied. */
@ -482,6 +572,7 @@ static int coroutine_fn backup_run(Job *job, Error **errp)
ret = backup_loop(s); ret = backup_loop(s);
} }
out:
notifier_with_return_remove(&s->before_write); notifier_with_return_remove(&s->before_write);
/* wait until pending backup_do_cow() calls have completed */ /* wait until pending backup_do_cow() calls have completed */
@ -545,6 +636,7 @@ static int64_t backup_calculate_cluster_size(BlockDriverState *target,
BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs, BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
BlockDriverState *target, int64_t speed, BlockDriverState *target, int64_t speed,
MirrorSyncMode sync_mode, BdrvDirtyBitmap *sync_bitmap, MirrorSyncMode sync_mode, BdrvDirtyBitmap *sync_bitmap,
BitmapSyncMode bitmap_mode,
bool compress, bool compress,
BlockdevOnError on_source_error, BlockdevOnError on_source_error,
BlockdevOnError on_target_error, BlockdevOnError on_target_error,
@ -556,11 +648,15 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
BackupBlockJob *job = NULL; BackupBlockJob *job = NULL;
int ret; int ret;
int64_t cluster_size; int64_t cluster_size;
HBitmap *copy_bitmap = NULL; BdrvDirtyBitmap *copy_bitmap = NULL;
assert(bs); assert(bs);
assert(target); assert(target);
/* QMP interface protects us from these cases */
assert(sync_mode != MIRROR_SYNC_MODE_INCREMENTAL);
assert(sync_bitmap || sync_mode != MIRROR_SYNC_MODE_BITMAP);
if (bs == target) { if (bs == target) {
error_setg(errp, "Source and target cannot be the same"); error_setg(errp, "Source and target cannot be the same");
return NULL; return NULL;
@ -592,10 +688,10 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
return NULL; return NULL;
} }
if (sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) { if (sync_bitmap) {
if (!sync_bitmap) { /* If we need to write to this bitmap, check that we can: */
error_setg(errp, "must provide a valid bitmap name for " if (bitmap_mode != BITMAP_SYNC_MODE_NEVER &&
"\"incremental\" sync mode"); bdrv_dirty_bitmap_check(sync_bitmap, BDRV_BITMAP_DEFAULT, errp)) {
return NULL; return NULL;
} }
@ -603,12 +699,6 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
if (bdrv_dirty_bitmap_create_successor(bs, sync_bitmap, errp) < 0) { if (bdrv_dirty_bitmap_create_successor(bs, sync_bitmap, errp) < 0) {
return NULL; return NULL;
} }
} else if (sync_bitmap) {
error_setg(errp,
"a sync_bitmap was provided to backup_run, "
"but received an incompatible sync_mode (%s)",
MirrorSyncMode_str(sync_mode));
return NULL;
} }
len = bdrv_getlength(bs); len = bdrv_getlength(bs);
@ -623,7 +713,11 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
goto error; goto error;
} }
copy_bitmap = hbitmap_alloc(len, ctz32(cluster_size)); copy_bitmap = bdrv_create_dirty_bitmap(bs, cluster_size, NULL, errp);
if (!copy_bitmap) {
goto error;
}
bdrv_disable_dirty_bitmap(copy_bitmap);
/* job->len is fixed, so we can't allow resize */ /* job->len is fixed, so we can't allow resize */
job = block_job_create(job_id, &backup_job_driver, txn, bs, job = block_job_create(job_id, &backup_job_driver, txn, bs,
@ -649,12 +743,18 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
job->on_source_error = on_source_error; job->on_source_error = on_source_error;
job->on_target_error = on_target_error; job->on_target_error = on_target_error;
job->sync_mode = sync_mode; job->sync_mode = sync_mode;
job->sync_bitmap = sync_mode == MIRROR_SYNC_MODE_INCREMENTAL ? job->sync_bitmap = sync_bitmap;
sync_bitmap : NULL; job->bitmap_mode = bitmap_mode;
job->compress = compress;
/*
* Set write flags:
* 1. Detect image-fleecing (and similar) schemes
* 2. Handle compression
*/
job->write_flags =
(bdrv_chain_contains(target, bs) ? BDRV_REQ_SERIALISING : 0) |
(compress ? BDRV_REQ_WRITE_COMPRESSED : 0);
/* Detect image-fleecing (and similar) schemes */
job->serialize_target_writes = bdrv_chain_contains(target, bs);
job->cluster_size = cluster_size; job->cluster_size = cluster_size;
job->copy_bitmap = copy_bitmap; job->copy_bitmap = copy_bitmap;
copy_bitmap = NULL; copy_bitmap = NULL;
@ -675,7 +775,7 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
error: error:
if (copy_bitmap) { if (copy_bitmap) {
assert(!job || !job->copy_bitmap); assert(!job || !job->copy_bitmap);
hbitmap_free(copy_bitmap); bdrv_release_dirty_bitmap(bs, copy_bitmap);
} }
if (sync_bitmap) { if (sync_bitmap) {
bdrv_reclaim_dirty_bitmap(bs, sync_bitmap, NULL); bdrv_reclaim_dirty_bitmap(bs, sync_bitmap, NULL);

92
block/dirty-bitmap.c
View File

@ -48,10 +48,9 @@ struct BdrvDirtyBitmap {
bool inconsistent; /* bitmap is persistent, but inconsistent. bool inconsistent; /* bitmap is persistent, but inconsistent.
It cannot be used at all in any way, except It cannot be used at all in any way, except
a QMP user can remove it. */ a QMP user can remove it. */
bool migration; /* Bitmap is selected for migration, it should bool skip_store; /* We are either migrating or deleting this
not be stored on the next inactivation * bitmap; it should not be stored on the next
(persistent flag doesn't matter until next * inactivation. */
invalidation).*/
QLIST_ENTRY(BdrvDirtyBitmap) list; QLIST_ENTRY(BdrvDirtyBitmap) list;
}; };
@ -509,14 +508,19 @@ BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs)
} }
/* Called within bdrv_dirty_bitmap_lock..unlock */ /* Called within bdrv_dirty_bitmap_lock..unlock */
bool bdrv_get_dirty_locked(BlockDriverState *bs, BdrvDirtyBitmap *bitmap, bool bdrv_dirty_bitmap_get_locked(BdrvDirtyBitmap *bitmap, int64_t offset)
int64_t offset)
{ {
if (bitmap) { return hbitmap_get(bitmap->bitmap, offset);
return hbitmap_get(bitmap->bitmap, offset); }
} else {
return false; bool bdrv_dirty_bitmap_get(BdrvDirtyBitmap *bitmap, int64_t offset)
} {
bool ret;
bdrv_dirty_bitmap_lock(bitmap);
ret = bdrv_dirty_bitmap_get_locked(bitmap, offset);
bdrv_dirty_bitmap_unlock(bitmap);
return ret;
} }
/** /**
@ -757,16 +761,16 @@ void bdrv_dirty_bitmap_set_inconsistent(BdrvDirtyBitmap *bitmap)
} }
/* Called with BQL taken. */ /* Called with BQL taken. */
void bdrv_dirty_bitmap_set_migration(BdrvDirtyBitmap *bitmap, bool migration) void bdrv_dirty_bitmap_skip_store(BdrvDirtyBitmap *bitmap, bool skip)
{ {
qemu_mutex_lock(bitmap->mutex); qemu_mutex_lock(bitmap->mutex);
bitmap->migration = migration; bitmap->skip_store = skip;
qemu_mutex_unlock(bitmap->mutex); qemu_mutex_unlock(bitmap->mutex);
} }
bool bdrv_dirty_bitmap_get_persistence(BdrvDirtyBitmap *bitmap) bool bdrv_dirty_bitmap_get_persistence(BdrvDirtyBitmap *bitmap)
{ {
return bitmap->persistent && !bitmap->migration; return bitmap->persistent && !bitmap->skip_store;
} }
bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap *bitmap) bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap *bitmap)
@ -778,7 +782,7 @@ bool bdrv_has_changed_persistent_bitmaps(BlockDriverState *bs)
{ {
BdrvDirtyBitmap *bm; BdrvDirtyBitmap *bm;
QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) { QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
if (bm->persistent && !bm->readonly && !bm->migration) { if (bm->persistent && !bm->readonly && !bm->skip_store) {
return true; return true;
} }
} }
@ -810,6 +814,12 @@ bool bdrv_dirty_bitmap_next_dirty_area(BdrvDirtyBitmap *bitmap,
return hbitmap_next_dirty_area(bitmap->bitmap, offset, bytes); return hbitmap_next_dirty_area(bitmap->bitmap, offset, bytes);
} }
/**
* bdrv_merge_dirty_bitmap: merge src into dest.
* Ensures permissions on bitmaps are reasonable; use for public API.
*
* @backup: If provided, make a copy of dest here prior to merge.
*/
void bdrv_merge_dirty_bitmap(BdrvDirtyBitmap *dest, const BdrvDirtyBitmap *src, void bdrv_merge_dirty_bitmap(BdrvDirtyBitmap *dest, const BdrvDirtyBitmap *src,
HBitmap **backup, Error **errp) HBitmap **backup, Error **errp)
{ {
@ -833,13 +843,7 @@ void bdrv_merge_dirty_bitmap(BdrvDirtyBitmap *dest, const BdrvDirtyBitmap *src,
goto out; goto out;
} }
if (backup) { ret = bdrv_dirty_bitmap_merge_internal(dest, src, backup, false);
*backup = dest->bitmap;
dest->bitmap = hbitmap_alloc(dest->size, hbitmap_granularity(*backup));
ret = hbitmap_merge(*backup, src->bitmap, dest->bitmap);
} else {
ret = hbitmap_merge(dest->bitmap, src->bitmap, dest->bitmap);
}
assert(ret); assert(ret);
out: out:
@ -848,3 +852,47 @@ out:
qemu_mutex_unlock(src->mutex); qemu_mutex_unlock(src->mutex);
} }
} }
/**
* bdrv_dirty_bitmap_merge_internal: merge src into dest.
* Does NOT check bitmap permissions; not suitable for use as public API.
*
* @backup: If provided, make a copy of dest here prior to merge.
* @lock: If true, lock and unlock bitmaps on the way in/out.
* returns true if the merge succeeded; false if unattempted.
*/
bool bdrv_dirty_bitmap_merge_internal(BdrvDirtyBitmap *dest,
const BdrvDirtyBitmap *src,
HBitmap **backup,
bool lock)
{
bool ret;
assert(!bdrv_dirty_bitmap_readonly(dest));
assert(!bdrv_dirty_bitmap_inconsistent(dest));
assert(!bdrv_dirty_bitmap_inconsistent(src));
if (lock) {
qemu_mutex_lock(dest->mutex);
if (src->mutex != dest->mutex) {
qemu_mutex_lock(src->mutex);
}
}
if (backup) {
*backup = dest->bitmap;
dest->bitmap = hbitmap_alloc(dest->size, hbitmap_granularity(*backup));
ret = hbitmap_merge(*backup, src->bitmap, dest->bitmap);
} else {
ret = hbitmap_merge(dest->bitmap, src->bitmap, dest->bitmap);
}
if (lock) {
qemu_mutex_unlock(dest->mutex);
if (src->mutex != dest->mutex) {
qemu_mutex_unlock(src->mutex);
}
}
return ret;
}

8
block/mirror.c
View File

@ -476,7 +476,7 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
int64_t next_offset = offset + nb_chunks * s->granularity; int64_t next_offset = offset + nb_chunks * s->granularity;
int64_t next_chunk = next_offset / s->granularity; int64_t next_chunk = next_offset / s->granularity;
if (next_offset >= s->bdev_length || if (next_offset >= s->bdev_length ||
!bdrv_get_dirty_locked(source, s->dirty_bitmap, next_offset)) { !bdrv_dirty_bitmap_get_locked(s->dirty_bitmap, next_offset)) {
break; break;
} }
if (test_bit(next_chunk, s->in_flight_bitmap)) { if (test_bit(next_chunk, s->in_flight_bitmap)) {
@ -1755,8 +1755,10 @@ void mirror_start(const char *job_id, BlockDriverState *bs,
bool is_none_mode; bool is_none_mode;
BlockDriverState *base; BlockDriverState *base;
if (mode == MIRROR_SYNC_MODE_INCREMENTAL) { if ((mode == MIRROR_SYNC_MODE_INCREMENTAL) ||
error_setg(errp, "Sync mode 'incremental' not supported"); (mode == MIRROR_SYNC_MODE_BITMAP)) {
error_setg(errp, "Sync mode '%s' not supported",
MirrorSyncMode_str(mode));
return; return;
} }
is_none_mode = mode == MIRROR_SYNC_MODE_NONE; is_none_mode = mode == MIRROR_SYNC_MODE_NONE;

5
block/qapi.c
View File

@ -79,6 +79,11 @@ BlockDeviceInfo *bdrv_block_device_info(BlockBackend *blk,
info->backing_file = g_strdup(bs->backing_file); info->backing_file = g_strdup(bs->backing_file);
} }
if (!QLIST_EMPTY(&bs->dirty_bitmaps)) {
info->has_dirty_bitmaps = true;
info->dirty_bitmaps = bdrv_query_dirty_bitmaps(bs);
}
info->detect_zeroes = bs->detect_zeroes; info->detect_zeroes = bs->detect_zeroes;
if (blk && blk_get_public(blk)->throttle_group_member.throttle_state) { if (blk && blk_get_public(blk)->throttle_group_member.throttle_state) {

2
block/replication.c
View File

@ -543,7 +543,7 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,
s->backup_job = backup_job_create( s->backup_job = backup_job_create(
NULL, s->secondary_disk->bs, s->hidden_disk->bs, NULL, s->secondary_disk->bs, s->hidden_disk->bs,
0, MIRROR_SYNC_MODE_NONE, NULL, false, 0, MIRROR_SYNC_MODE_NONE, NULL, 0, false,
BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT,
BLOCKDEV_ON_ERROR_REPORT, JOB_INTERNAL, BLOCKDEV_ON_ERROR_REPORT, JOB_INTERNAL,
backup_job_completed, bs, NULL, &local_err); backup_job_completed, bs, NULL, &local_err);

1
block/trace-events
View File

@ -41,6 +41,7 @@ mirror_yield_in_flight(void *s, int64_t offset, int in_flight) "s %p offset %" P
backup_do_cow_enter(void *job, int64_t start, int64_t offset, uint64_t bytes) "job %p start %" PRId64 " offset %" PRId64 " bytes %" PRIu64 backup_do_cow_enter(void *job, int64_t start, int64_t offset, uint64_t bytes) "job %p start %" PRId64 " offset %" PRId64 " bytes %" PRIu64
backup_do_cow_return(void *job, int64_t offset, uint64_t bytes, int ret) "job %p offset %" PRId64 " bytes %" PRIu64 " ret %d" backup_do_cow_return(void *job, int64_t offset, uint64_t bytes, int ret) "job %p offset %" PRId64 " bytes %" PRIu64 " ret %d"
backup_do_cow_skip(void *job, int64_t start) "job %p start %"PRId64 backup_do_cow_skip(void *job, int64_t start) "job %p start %"PRId64
backup_do_cow_skip_range(void *job, int64_t start, uint64_t bytes) "job %p start %"PRId64" bytes %"PRId64
backup_do_cow_process(void *job, int64_t start) "job %p start %"PRId64 backup_do_cow_process(void *job, int64_t start) "job %p start %"PRId64
backup_do_cow_read_fail(void *job, int64_t start, int ret) "job %p start %"PRId64" ret %d" backup_do_cow_read_fail(void *job, int64_t start, int ret) "job %p start %"PRId64" ret %d"
backup_do_cow_write_fail(void *job, int64_t start, int ret) "job %p start %"PRId64" ret %d" backup_do_cow_write_fail(void *job, int64_t start, int ret) "job %p start %"PRId64" ret %d"

369
blockdev.c
View File

@ -2136,6 +2136,51 @@ static void block_dirty_bitmap_merge_prepare(BlkActionState *common,
errp); errp);
} }
static BdrvDirtyBitmap *do_block_dirty_bitmap_remove(
const char *node, const char *name, bool release,
BlockDriverState **bitmap_bs, Error **errp);
static void block_dirty_bitmap_remove_prepare(BlkActionState *common,
Error **errp)
{
BlockDirtyBitmap *action;
BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
common, common);
if (action_check_completion_mode(common, errp) < 0) {
return;
}
action = common->action->u.block_dirty_bitmap_remove.data;
state->bitmap = do_block_dirty_bitmap_remove(action->node, action->name,
false, &state->bs, errp);
if (state->bitmap) {
bdrv_dirty_bitmap_skip_store(state->bitmap, true);
bdrv_dirty_bitmap_set_busy(state->bitmap, true);
}
}
static void block_dirty_bitmap_remove_abort(BlkActionState *common)
{
BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
common, common);
if (state->bitmap) {
bdrv_dirty_bitmap_skip_store(state->bitmap, false);
bdrv_dirty_bitmap_set_busy(state->bitmap, false);
}
}
static void block_dirty_bitmap_remove_commit(BlkActionState *common)
{
BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
common, common);
bdrv_dirty_bitmap_set_busy(state->bitmap, false);
bdrv_release_dirty_bitmap(state->bs, state->bitmap);
}
static void abort_prepare(BlkActionState *common, Error **errp) static void abort_prepare(BlkActionState *common, Error **errp)
{ {
error_setg(errp, "Transaction aborted using Abort action"); error_setg(errp, "Transaction aborted using Abort action");
@ -2213,6 +2258,12 @@ static const BlkActionOps actions[] = {
.commit = block_dirty_bitmap_free_backup, .commit = block_dirty_bitmap_free_backup,
.abort = block_dirty_bitmap_restore, .abort = block_dirty_bitmap_restore,
}, },
[TRANSACTION_ACTION_KIND_BLOCK_DIRTY_BITMAP_REMOVE] = {
.instance_size = sizeof(BlockDirtyBitmapState),
.prepare = block_dirty_bitmap_remove_prepare,
.commit = block_dirty_bitmap_remove_commit,
.abort = block_dirty_bitmap_remove_abort,
},
/* Where are transactions for MIRROR, COMMIT and STREAM? /* Where are transactions for MIRROR, COMMIT and STREAM?
* Although these blockjobs use transaction callbacks like the backup job, * Although these blockjobs use transaction callbacks like the backup job,
* these jobs do not necessarily adhere to transaction semantics. * these jobs do not necessarily adhere to transaction semantics.
@ -2813,7 +2864,6 @@ void qmp_block_dirty_bitmap_add(const char *node, const char *name,
{ {
BlockDriverState *bs; BlockDriverState *bs;
BdrvDirtyBitmap *bitmap; BdrvDirtyBitmap *bitmap;
AioContext *aio_context = NULL;
if (!name || name[0] == '\0') { if (!name || name[0] == '\0') {
error_setg(errp, "Bitmap name cannot be empty"); error_setg(errp, "Bitmap name cannot be empty");
@ -2849,16 +2899,20 @@ void qmp_block_dirty_bitmap_add(const char *node, const char *name,
} }
if (persistent) { if (persistent) {
aio_context = bdrv_get_aio_context(bs); AioContext *aio_context = bdrv_get_aio_context(bs);
bool ok;
aio_context_acquire(aio_context); aio_context_acquire(aio_context);
if (!bdrv_can_store_new_dirty_bitmap(bs, name, granularity, errp)) { ok = bdrv_can_store_new_dirty_bitmap(bs, name, granularity, errp);
goto out; aio_context_release(aio_context);
if (!ok) {
return;
} }
} }
bitmap = bdrv_create_dirty_bitmap(bs, granularity, name, errp); bitmap = bdrv_create_dirty_bitmap(bs, granularity, name, errp);
if (bitmap == NULL) { if (bitmap == NULL) {
goto out; return;
} }
if (disabled) { if (disabled) {
@ -2866,45 +2920,54 @@ void qmp_block_dirty_bitmap_add(const char *node, const char *name,
} }
bdrv_dirty_bitmap_set_persistence(bitmap, persistent); bdrv_dirty_bitmap_set_persistence(bitmap, persistent);
out: }
if (aio_context) {
aio_context_release(aio_context); static BdrvDirtyBitmap *do_block_dirty_bitmap_remove(
const char *node, const char *name, bool release,
BlockDriverState **bitmap_bs, Error **errp)
{
BlockDriverState *bs;
BdrvDirtyBitmap *bitmap;
bitmap = block_dirty_bitmap_lookup(node, name, &bs, errp);
if (!bitmap || !bs) {
return NULL;
} }
if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_BUSY | BDRV_BITMAP_RO,
errp)) {
return NULL;
}
if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
AioContext *aio_context = bdrv_get_aio_context(bs);
Error *local_err = NULL;
aio_context_acquire(aio_context);
bdrv_remove_persistent_dirty_bitmap(bs, name, &local_err);
aio_context_release(aio_context);
if (local_err != NULL) {
error_propagate(errp, local_err);
return NULL;
}
}
if (release) {
bdrv_release_dirty_bitmap(bs, bitmap);
}
if (bitmap_bs) {
*bitmap_bs = bs;
}
return release ? NULL : bitmap;
} }
void qmp_block_dirty_bitmap_remove(const char *node, const char *name, void qmp_block_dirty_bitmap_remove(const char *node, const char *name,
Error **errp) Error **errp)
{ {
BlockDriverState *bs; do_block_dirty_bitmap_remove(node, name, true, NULL, errp);
BdrvDirtyBitmap *bitmap;
Error *local_err = NULL;
AioContext *aio_context = NULL;
bitmap = block_dirty_bitmap_lookup(node, name, &bs, errp);
if (!bitmap || !bs) {
return;
}
if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_BUSY | BDRV_BITMAP_RO,
errp)) {
return;
}
if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
aio_context = bdrv_get_aio_context(bs);
aio_context_acquire(aio_context);
bdrv_remove_persistent_dirty_bitmap(bs, name, &local_err);
if (local_err != NULL) {
error_propagate(errp, local_err);
goto out;
}
}
bdrv_release_dirty_bitmap(bs, bitmap);
out:
if (aio_context) {
aio_context_release(aio_context);
}
} }
/** /**
@ -3427,20 +3490,16 @@ out:
aio_context_release(aio_context); aio_context_release(aio_context);
} }
static BlockJob *do_drive_backup(DriveBackup *backup, JobTxn *txn, /* Common QMP interface for drive-backup and blockdev-backup */
Error **errp) static BlockJob *do_backup_common(BackupCommon *backup,
BlockDriverState *bs,
BlockDriverState *target_bs,
AioContext *aio_context,
JobTxn *txn, Error **errp)
{ {
BlockDriverState *bs;
BlockDriverState *target_bs;
BlockDriverState *source = NULL;
BlockJob *job = NULL; BlockJob *job = NULL;
BdrvDirtyBitmap *bmap = NULL; BdrvDirtyBitmap *bmap = NULL;
AioContext *aio_context; int job_flags = JOB_DEFAULT;
QDict *options = NULL;
Error *local_err = NULL;
int flags, job_flags = JOB_DEFAULT;
int64_t size;
bool set_backing_hd = false;
int ret; int ret;
if (!backup->has_speed) { if (!backup->has_speed) {
@ -3452,9 +3511,6 @@ static BlockJob *do_drive_backup(DriveBackup *backup, JobTxn *txn,
if (!backup->has_on_target_error) { if (!backup->has_on_target_error) {
backup->on_target_error = BLOCKDEV_ON_ERROR_REPORT; backup->on_target_error = BLOCKDEV_ON_ERROR_REPORT;
} }
if (!backup->has_mode) {
backup->mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
}
if (!backup->has_job_id) { if (!backup->has_job_id) {
backup->job_id = NULL; backup->job_id = NULL;
} }
@ -3468,6 +3524,107 @@ static BlockJob *do_drive_backup(DriveBackup *backup, JobTxn *txn,
backup->compress = false; backup->compress = false;
} }
ret = bdrv_try_set_aio_context(target_bs, aio_context, errp);
if (ret < 0) {
return NULL;
}
if ((backup->sync == MIRROR_SYNC_MODE_BITMAP) ||
(backup->sync == MIRROR_SYNC_MODE_INCREMENTAL)) {
/* done before desugaring 'incremental' to print the right message */
if (!backup->has_bitmap) {
error_setg(errp, "must provide a valid bitmap name for "
"'%s' sync mode", MirrorSyncMode_str(backup->sync));
return NULL;
}
}
if (backup->sync == MIRROR_SYNC_MODE_INCREMENTAL) {
if (backup->has_bitmap_mode &&
backup->bitmap_mode != BITMAP_SYNC_MODE_ON_SUCCESS) {
error_setg(errp, "Bitmap sync mode must be '%s' "
"when using sync mode '%s'",
BitmapSyncMode_str(BITMAP_SYNC_MODE_ON_SUCCESS),
MirrorSyncMode_str(backup->sync));
return NULL;
}
backup->has_bitmap_mode = true;
backup->sync = MIRROR_SYNC_MODE_BITMAP;
backup->bitmap_mode = BITMAP_SYNC_MODE_ON_SUCCESS;
}
if (backup->has_bitmap) {
bmap = bdrv_find_dirty_bitmap(bs, backup->bitmap);
if (!bmap) {
error_setg(errp, "Bitmap '%s' could not be found", backup->bitmap);
return NULL;
}
if (!backup->has_bitmap_mode) {
error_setg(errp, "Bitmap sync mode must be given "
"when providing a bitmap");
return NULL;
}
if (bdrv_dirty_bitmap_check(bmap, BDRV_BITMAP_ALLOW_RO, errp)) {
return NULL;
}
/* This does not produce a useful bitmap artifact: */
if (backup->sync == MIRROR_SYNC_MODE_NONE) {
error_setg(errp, "sync mode '%s' does not produce meaningful bitmap"
" outputs", MirrorSyncMode_str(backup->sync));
return NULL;
}
/* If the bitmap isn't used for input or output, this is useless: */
if (backup->bitmap_mode == BITMAP_SYNC_MODE_NEVER &&
backup->sync != MIRROR_SYNC_MODE_BITMAP) {
error_setg(errp, "Bitmap sync mode '%s' has no meaningful effect"
" when combined with sync mode '%s'",
BitmapSyncMode_str(backup->bitmap_mode),
MirrorSyncMode_str(backup->sync));
return NULL;
}
}
if (!backup->has_bitmap && backup->has_bitmap_mode) {
error_setg(errp, "Cannot specify bitmap sync mode without a bitmap");
return NULL;
}
if (!backup->auto_finalize) {
job_flags |= JOB_MANUAL_FINALIZE;
}
if (!backup->auto_dismiss) {
job_flags |= JOB_MANUAL_DISMISS;
}
job = backup_job_create(backup->job_id, bs, target_bs, backup->speed,
backup->sync, bmap, backup->bitmap_mode,
backup->compress,
backup->on_source_error,
backup->on_target_error,
job_flags, NULL, NULL, txn, errp);
return job;
}
static BlockJob *do_drive_backup(DriveBackup *backup, JobTxn *txn,
Error **errp)
{
BlockDriverState *bs;
BlockDriverState *target_bs;
BlockDriverState *source = NULL;
BlockJob *job = NULL;
AioContext *aio_context;
QDict *options;
Error *local_err = NULL;
int flags;
int64_t size;
bool set_backing_hd = false;
if (!backup->has_mode) {
backup->mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
}
bs = bdrv_lookup_bs(backup->device, backup->device, errp); bs = bdrv_lookup_bs(backup->device, backup->device, errp);
if (!bs) { if (!bs) {
return NULL; return NULL;
@ -3531,10 +3688,10 @@ static BlockJob *do_drive_backup(DriveBackup *backup, JobTxn *txn,
goto out; goto out;
} }
options = qdict_new();
qdict_put_str(options, "discard", "unmap");
qdict_put_str(options, "detect-zeroes", "unmap");
if (backup->format) { if (backup->format) {
if (!options) {
options = qdict_new();
}
qdict_put_str(options, "driver", backup->format); qdict_put_str(options, "driver", backup->format);
} }
@ -3543,12 +3700,6 @@ static BlockJob *do_drive_backup(DriveBackup *backup, JobTxn *txn,
goto out; goto out;
} }
ret = bdrv_try_set_aio_context(target_bs, aio_context, errp);
if (ret < 0) {
bdrv_unref(target_bs);
goto out;
}
if (set_backing_hd) { if (set_backing_hd) {
bdrv_set_backing_hd(target_bs, source, &local_err); bdrv_set_backing_hd(target_bs, source, &local_err);
if (local_err) { if (local_err) {
@ -3556,31 +3707,8 @@ static BlockJob *do_drive_backup(DriveBackup *backup, JobTxn *txn,
} }
} }
if (backup->has_bitmap) { job = do_backup_common(qapi_DriveBackup_base(backup),
bmap = bdrv_find_dirty_bitmap(bs, backup->bitmap); bs, target_bs, aio_context, txn, errp);
if (!bmap) {
error_setg(errp, "Bitmap '%s' could not be found", backup->bitmap);
goto unref;
}
if (bdrv_dirty_bitmap_check(bmap, BDRV_BITMAP_DEFAULT, errp)) {
goto unref;
}
}
if (!backup->auto_finalize) {
job_flags |= JOB_MANUAL_FINALIZE;
}
if (!backup->auto_dismiss) {
job_flags |= JOB_MANUAL_DISMISS;
}
job = backup_job_create(backup->job_id, bs, target_bs, backup->speed,
backup->sync, bmap, backup->compress,
backup->on_source_error, backup->on_target_error,
job_flags, NULL, NULL, txn, &local_err);
if (local_err != NULL) {
error_propagate(errp, local_err);
goto unref;
}
unref: unref:
bdrv_unref(target_bs); bdrv_unref(target_bs);
@ -3614,78 +3742,25 @@ BlockJob *do_blockdev_backup(BlockdevBackup *backup, JobTxn *txn,
{ {
BlockDriverState *bs; BlockDriverState *bs;
BlockDriverState *target_bs; BlockDriverState *target_bs;
Error *local_err = NULL;
BdrvDirtyBitmap *bmap = NULL;
AioContext *aio_context; AioContext *aio_context;
BlockJob *job = NULL; BlockJob *job;
int job_flags = JOB_DEFAULT;
int ret;
if (!backup->has_speed) {
backup->speed = 0;
}
if (!backup->has_on_source_error) {
backup->on_source_error = BLOCKDEV_ON_ERROR_REPORT;
}
if (!backup->has_on_target_error) {
backup->on_target_error = BLOCKDEV_ON_ERROR_REPORT;
}
if (!backup->has_job_id) {
backup->job_id = NULL;
}
if (!backup->has_auto_finalize) {
backup->auto_finalize = true;
}
if (!backup->has_auto_dismiss) {
backup->auto_dismiss = true;
}
if (!backup->has_compress) {
backup->compress = false;
}
bs = bdrv_lookup_bs(backup->device, backup->device, errp); bs = bdrv_lookup_bs(backup->device, backup->device, errp);
if (!bs) { if (!bs) {
return NULL; return NULL;
} }
target_bs = bdrv_lookup_bs(backup->target, backup->target, errp);
if (!target_bs) {
return NULL;
}
aio_context = bdrv_get_aio_context(bs); aio_context = bdrv_get_aio_context(bs);
aio_context_acquire(aio_context); aio_context_acquire(aio_context);
target_bs = bdrv_lookup_bs(backup->target, backup->target, errp); job = do_backup_common(qapi_BlockdevBackup_base(backup),
if (!target_bs) { bs, target_bs, aio_context, txn, errp);
goto out;
}
ret = bdrv_try_set_aio_context(target_bs, aio_context, errp);
if (ret < 0) {
goto out;
}
if (backup->has_bitmap) {
bmap = bdrv_find_dirty_bitmap(bs, backup->bitmap);
if (!bmap) {
error_setg(errp, "Bitmap '%s' could not be found", backup->bitmap);
goto out;
}
if (bdrv_dirty_bitmap_check(bmap, BDRV_BITMAP_DEFAULT, errp)) {
goto out;
}
}
if (!backup->auto_finalize) {
job_flags |= JOB_MANUAL_FINALIZE;
}
if (!backup->auto_dismiss) {
job_flags |= JOB_MANUAL_DISMISS;
}
job = backup_job_create(backup->job_id, bs, target_bs, backup->speed,
backup->sync, bmap, backup->compress,
backup->on_source_error, backup->on_target_error,
job_flags, NULL, NULL, txn, &local_err);
if (local_err != NULL) {
error_propagate(errp, local_err);
}
out:
aio_context_release(aio_context); aio_context_release(aio_context);
return job; return job;
} }

7
include/block/block_int.h
View File

@ -1147,7 +1147,8 @@ void mirror_start(const char *job_id, BlockDriverState *bs,
* @target: Block device to write to. * @target: Block device to write to.
* @speed: The maximum speed, in bytes per second, or 0 for unlimited. * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
* @sync_mode: What parts of the disk image should be copied to the destination. * @sync_mode: What parts of the disk image should be copied to the destination.
* @sync_bitmap: The dirty bitmap if sync_mode is MIRROR_SYNC_MODE_INCREMENTAL. * @sync_bitmap: The dirty bitmap if sync_mode is 'bitmap' or 'incremental'
* @bitmap_mode: The bitmap synchronization policy to use.
* @on_source_error: The action to take upon error reading from the source. * @on_source_error: The action to take upon error reading from the source.
* @on_target_error: The action to take upon error writing to the target. * @on_target_error: The action to take upon error writing to the target.
* @creation_flags: Flags that control the behavior of the Job lifetime. * @creation_flags: Flags that control the behavior of the Job lifetime.
@ -1163,6 +1164,7 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
BlockDriverState *target, int64_t speed, BlockDriverState *target, int64_t speed,
MirrorSyncMode sync_mode, MirrorSyncMode sync_mode,
BdrvDirtyBitmap *sync_bitmap, BdrvDirtyBitmap *sync_bitmap,
BitmapSyncMode bitmap_mode,
bool compress, bool compress,
BlockdevOnError on_source_error, BlockdevOnError on_source_error,
BlockdevOnError on_target_error, BlockdevOnError on_target_error,
@ -1251,6 +1253,9 @@ void bdrv_set_dirty(BlockDriverState *bs, int64_t offset, int64_t bytes);
void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out); void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out);
void bdrv_restore_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *backup); void bdrv_restore_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *backup);
bool bdrv_dirty_bitmap_merge_internal(BdrvDirtyBitmap *dest,
const BdrvDirtyBitmap *src,
HBitmap **backup, bool lock);
void bdrv_inc_in_flight(BlockDriverState *bs); void bdrv_inc_in_flight(BlockDriverState *bs);
void bdrv_dec_in_flight(BlockDriverState *bs); void bdrv_dec_in_flight(BlockDriverState *bs);

6
include/block/dirty-bitmap.h
View File

@ -83,13 +83,13 @@ void bdrv_dirty_bitmap_set_inconsistent(BdrvDirtyBitmap *bitmap);
void bdrv_dirty_bitmap_set_busy(BdrvDirtyBitmap *bitmap, bool busy); void bdrv_dirty_bitmap_set_busy(BdrvDirtyBitmap *bitmap, bool busy);
void bdrv_merge_dirty_bitmap(BdrvDirtyBitmap *dest, const BdrvDirtyBitmap *src, void bdrv_merge_dirty_bitmap(BdrvDirtyBitmap *dest, const BdrvDirtyBitmap *src,
HBitmap **backup, Error **errp); HBitmap **backup, Error **errp);
void bdrv_dirty_bitmap_set_migration(BdrvDirtyBitmap *bitmap, bool migration); void bdrv_dirty_bitmap_skip_store(BdrvDirtyBitmap *bitmap, bool skip);
bool bdrv_dirty_bitmap_get(BdrvDirtyBitmap *bitmap, int64_t offset);
/* Functions that require manual locking. */ /* Functions that require manual locking. */
void bdrv_dirty_bitmap_lock(BdrvDirtyBitmap *bitmap); void bdrv_dirty_bitmap_lock(BdrvDirtyBitmap *bitmap);
void bdrv_dirty_bitmap_unlock(BdrvDirtyBitmap *bitmap); void bdrv_dirty_bitmap_unlock(BdrvDirtyBitmap *bitmap);
bool bdrv_get_dirty_locked(BlockDriverState *bs, BdrvDirtyBitmap *bitmap, bool bdrv_dirty_bitmap_get_locked(BdrvDirtyBitmap *bitmap, int64_t offset);
int64_t offset);
void bdrv_set_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap, void bdrv_set_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
int64_t offset, int64_t bytes); int64_t offset, int64_t bytes);
void bdrv_reset_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap, void bdrv_reset_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,

2
migration/block-dirty-bitmap.c
View File

@ -326,7 +326,7 @@ static int init_dirty_bitmap_migration(void)
/* unset migration flags here, to not roll back it */ /* unset migration flags here, to not roll back it */
QSIMPLEQ_FOREACH(dbms, &dirty_bitmap_mig_state.dbms_list, entry) { QSIMPLEQ_FOREACH(dbms, &dirty_bitmap_mig_state.dbms_list, entry) {
bdrv_dirty_bitmap_set_migration(dbms->bitmap, true); bdrv_dirty_bitmap_skip_store(dbms->bitmap, true);
} }
if (QSIMPLEQ_EMPTY(&dirty_bitmap_mig_state.dbms_list)) { if (QSIMPLEQ_EMPTY(&dirty_bitmap_mig_state.dbms_list)) {

5
migration/block.c
View File

@ -521,7 +521,6 @@ static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
int is_async) int is_async)
{ {
BlkMigBlock *blk; BlkMigBlock *blk;
BlockDriverState *bs = blk_bs(bmds->blk);
int64_t total_sectors = bmds->total_sectors; int64_t total_sectors = bmds->total_sectors;
int64_t sector; int64_t sector;
int nr_sectors; int nr_sectors;
@ -536,8 +535,8 @@ static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
blk_mig_unlock(); blk_mig_unlock();
} }
bdrv_dirty_bitmap_lock(bmds->dirty_bitmap); bdrv_dirty_bitmap_lock(bmds->dirty_bitmap);
if (bdrv_get_dirty_locked(bs, bmds->dirty_bitmap, if (bdrv_dirty_bitmap_get_locked(bmds->dirty_bitmap,
sector * BDRV_SECTOR_SIZE)) { sector * BDRV_SECTOR_SIZE)) {
if (total_sectors - sector < BDRV_SECTORS_PER_DIRTY_CHUNK) { if (total_sectors - sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
nr_sectors = total_sectors - sector; nr_sectors = total_sectors - sector;
} else { } else {

2
nbd/server.c
View File

@ -2004,7 +2004,7 @@ static unsigned int bitmap_to_extents(BdrvDirtyBitmap *bitmap, uint64_t offset,
bdrv_dirty_bitmap_lock(bitmap); bdrv_dirty_bitmap_lock(bitmap);
it = bdrv_dirty_iter_new(bitmap); it = bdrv_dirty_iter_new(bitmap);
dirty = bdrv_get_dirty_locked(NULL, bitmap, offset); dirty = bdrv_dirty_bitmap_get_locked(bitmap, offset);
assert(begin < overall_end && nb_extents); assert(begin < overall_end && nb_extents);
while (begin < overall_end && i < nb_extents) { while (begin < overall_end && i < nb_extents) {

196
qapi/block-core.json
View File

@ -360,6 +360,9 @@
# @write_threshold: configured write threshold for the device. # @write_threshold: configured write threshold for the device.
# 0 if disabled. (Since 2.3) # 0 if disabled. (Since 2.3)
# #
# @dirty-bitmaps: dirty bitmaps information (only present if node
# has one or more dirty bitmaps) (Since 4.2)
#
# Since: 0.14.0 # Since: 0.14.0
# #
## ##
@ -378,7 +381,7 @@
'*bps_wr_max_length': 'int', '*iops_max_length': 'int', '*bps_wr_max_length': 'int', '*iops_max_length': 'int',
'*iops_rd_max_length': 'int', '*iops_wr_max_length': 'int', '*iops_rd_max_length': 'int', '*iops_wr_max_length': 'int',
'*iops_size': 'int', '*group': 'str', 'cache': 'BlockdevCacheInfo', '*iops_size': 'int', '*group': 'str', 'cache': 'BlockdevCacheInfo',
'write_threshold': 'int' } } 'write_threshold': 'int', '*dirty-bitmaps': ['BlockDirtyInfo'] } }
## ##
# @BlockDeviceIoStatus: # @BlockDeviceIoStatus:
@ -656,6 +659,7 @@
# #
# @dirty-bitmaps: dirty bitmaps information (only present if the # @dirty-bitmaps: dirty bitmaps information (only present if the
# driver has one or more dirty bitmaps) (Since 2.0) # driver has one or more dirty bitmaps) (Since 2.0)
# Deprecated in 4.2; see BlockDeviceInfo instead.
# #
# @io-status: @BlockDeviceIoStatus. Only present if the device # @io-status: @BlockDeviceIoStatus. Only present if the device
# supports it and the VM is configured to stop on errors # supports it and the VM is configured to stop on errors
@ -1127,12 +1131,35 @@
# #
# @none: only copy data written from now on # @none: only copy data written from now on
# #
# @incremental: only copy data described by the dirty bitmap. Since: 2.4 # @incremental: only copy data described by the dirty bitmap. (since: 2.4)
#
# @bitmap: only copy data described by the dirty bitmap. (since: 4.2)
# Behavior on completion is determined by the BitmapSyncMode.
# #
# Since: 1.3 # Since: 1.3
## ##
{ 'enum': 'MirrorSyncMode', { 'enum': 'MirrorSyncMode',
'data': ['top', 'full', 'none', 'incremental'] } 'data': ['top', 'full', 'none', 'incremental', 'bitmap'] }
##
# @BitmapSyncMode:
#
# An enumeration of possible behaviors for the synchronization of a bitmap
# when used for data copy operations.
#
# @on-success: The bitmap is only synced when the operation is successful.
# This is the behavior always used for 'INCREMENTAL' backups.
#
# @never: The bitmap is never synchronized with the operation, and is
# treated solely as a read-only manifest of blocks to copy.
#
# @always: The bitmap is always synchronized with the operation,
# regardless of whether or not the operation was successful.
#
# Since: 4.2
##
{ 'enum': 'BitmapSyncMode',
'data': ['on-success', 'never', 'always'] }
## ##
# @MirrorCopyMode: # @MirrorCopyMode:
@ -1315,13 +1342,73 @@
'data': { 'node': 'str', 'overlay': 'str' } } 'data': { 'node': 'str', 'overlay': 'str' } }
## ##
# @DriveBackup: # @BackupCommon:
# #
# @job-id: identifier for the newly-created block job. If # @job-id: identifier for the newly-created block job. If
# omitted, the device name will be used. (Since 2.7) # omitted, the device name will be used. (Since 2.7)
# #
# @device: the device name or node-name of a root node which should be copied. # @device: the device name or node-name of a root node which should be copied.
# #
# @sync: what parts of the disk image should be copied to the destination
# (all the disk, only the sectors allocated in the topmost image, from a
# dirty bitmap, or only new I/O).
#
# @speed: the maximum speed, in bytes per second. The default is 0,
# for unlimited.
#
# @bitmap: The name of a dirty bitmap to use.
# Must be present if sync is "bitmap" or "incremental".
# Can be present if sync is "full" or "top".
# Must not be present otherwise.
# (Since 2.4 (drive-backup), 3.1 (blockdev-backup))
#
# @bitmap-mode: Specifies the type of data the bitmap should contain after
# the operation concludes.
# Must be present if a bitmap was provided,
# Must NOT be present otherwise. (Since 4.2)
#
# @compress: true to compress data, if the target format supports it.
# (default: false) (since 2.8)
#
# @on-source-error: the action to take on an error on the source,
# default 'report'. 'stop' and 'enospc' can only be used
# if the block device supports io-status (see BlockInfo).
#
# @on-target-error: the action to take on an error on the target,
# default 'report' (no limitations, since this applies to
# a different block device than @device).
#
# @auto-finalize: When false, this job will wait in a PENDING state after it has
# finished its work, waiting for @block-job-finalize before
# making any block graph changes.
# When true, this job will automatically
# perform its abort or commit actions.
# Defaults to true. (Since 2.12)
#
# @auto-dismiss: When false, this job will wait in a CONCLUDED state after it
# has completely ceased all work, and awaits @block-job-dismiss.
# When true, this job will automatically disappear from the query
# list without user intervention.
# Defaults to true. (Since 2.12)
#
# Note: @on-source-error and @on-target-error only affect background
# I/O. If an error occurs during a guest write request, the device's
# rerror/werror actions will be used.
#
# Since: 4.2
##
{ 'struct': 'BackupCommon',
'data': { '*job-id': 'str', 'device': 'str',
'sync': 'MirrorSyncMode', '*speed': 'int',
'*bitmap': 'str', '*bitmap-mode': 'BitmapSyncMode',
'*compress': 'bool',
'*on-source-error': 'BlockdevOnError',
'*on-target-error': 'BlockdevOnError',
'*auto-finalize': 'bool', '*auto-dismiss': 'bool' } }
##
# @DriveBackup:
#
# @target: the target of the new image. If the file exists, or if it # @target: the target of the new image. If the file exists, or if it
# is a device, the existing file/device will be used as the new # is a device, the existing file/device will be used as the new
# destination. If it does not exist, a new file will be created. # destination. If it does not exist, a new file will be created.
@ -1329,116 +1416,27 @@
# @format: the format of the new destination, default is to # @format: the format of the new destination, default is to
# probe if @mode is 'existing', else the format of the source # probe if @mode is 'existing', else the format of the source
# #
# @sync: what parts of the disk image should be copied to the destination
# (all the disk, only the sectors allocated in the topmost image, from a
# dirty bitmap, or only new I/O).
#
# @mode: whether and how QEMU should create a new image, default is # @mode: whether and how QEMU should create a new image, default is
# 'absolute-paths'. # 'absolute-paths'.
# #
# @speed: the maximum speed, in bytes per second
#
# @bitmap: the name of dirty bitmap if sync is "incremental".
# Must be present if sync is "incremental", must NOT be present
# otherwise. (Since 2.4)
#
# @compress: true to compress data, if the target format supports it.
# (default: false) (since 2.8)
#
# @on-source-error: the action to take on an error on the source,
# default 'report'. 'stop' and 'enospc' can only be used
# if the block device supports io-status (see BlockInfo).
#
# @on-target-error: the action to take on an error on the target,
# default 'report' (no limitations, since this applies to
# a different block device than @device).
#
# @auto-finalize: When false, this job will wait in a PENDING state after it has
# finished its work, waiting for @block-job-finalize before
# making any block graph changes.
# When true, this job will automatically
# perform its abort or commit actions.
# Defaults to true. (Since 2.12)
#
# @auto-dismiss: When false, this job will wait in a CONCLUDED state after it
# has completely ceased all work, and awaits @block-job-dismiss.
# When true, this job will automatically disappear from the query
# list without user intervention.
# Defaults to true. (Since 2.12)
#
# Note: @on-source-error and @on-target-error only affect background
# I/O. If an error occurs during a guest write request, the device's
# rerror/werror actions will be used.
#
# Since: 1.6 # Since: 1.6
## ##
{ 'struct': 'DriveBackup', { 'struct': 'DriveBackup',
'data': { '*job-id': 'str', 'device': 'str', 'target': 'str', 'base': 'BackupCommon',
'*format': 'str', 'sync': 'MirrorSyncMode', 'data': { 'target': 'str',
'*mode': 'NewImageMode', '*speed': 'int', '*format': 'str',
'*bitmap': 'str', '*compress': 'bool', '*mode': 'NewImageMode' } }
'*on-source-error': 'BlockdevOnError',
'*on-target-error': 'BlockdevOnError',
'*auto-finalize': 'bool', '*auto-dismiss': 'bool' } }
## ##
# @BlockdevBackup: # @BlockdevBackup:
# #
# @job-id: identifier for the newly-created block job. If
# omitted, the device name will be used. (Since 2.7)
#
# @device: the device name or node-name of a root node which should be copied.
#
# @target: the device name or node-name of the backup target node. # @target: the device name or node-name of the backup target node.
# #
# @sync: what parts of the disk image should be copied to the destination
# (all the disk, only the sectors allocated in the topmost image, or
# only new I/O).
#
# @speed: the maximum speed, in bytes per second. The default is 0,
# for unlimited.
#
# @bitmap: the name of dirty bitmap if sync is "incremental".
# Must be present if sync is "incremental", must NOT be present
# otherwise. (Since 3.1)
#
# @compress: true to compress data, if the target format supports it.
# (default: false) (since 2.8)
#
# @on-source-error: the action to take on an error on the source,
# default 'report'. 'stop' and 'enospc' can only be used
# if the block device supports io-status (see BlockInfo).
#
# @on-target-error: the action to take on an error on the target,
# default 'report' (no limitations, since this applies to
# a different block device than @device).
#
# @auto-finalize: When false, this job will wait in a PENDING state after it has
# finished its work, waiting for @block-job-finalize before
# making any block graph changes.
# When true, this job will automatically
# perform its abort or commit actions.
# Defaults to true. (Since 2.12)
#
# @auto-dismiss: When false, this job will wait in a CONCLUDED state after it
# has completely ceased all work, and awaits @block-job-dismiss.
# When true, this job will automatically disappear from the query
# list without user intervention.
# Defaults to true. (Since 2.12)
#
# Note: @on-source-error and @on-target-error only affect background
# I/O. If an error occurs during a guest write request, the device's
# rerror/werror actions will be used.
#
# Since: 2.3 # Since: 2.3
## ##
{ 'struct': 'BlockdevBackup', { 'struct': 'BlockdevBackup',
'data': { '*job-id': 'str', 'device': 'str', 'target': 'str', 'base': 'BackupCommon',
'sync': 'MirrorSyncMode', '*speed': 'int', 'data': { 'target': 'str' } }
'*bitmap': 'str', '*compress': 'bool',
'*on-source-error': 'BlockdevOnError',
'*on-target-error': 'BlockdevOnError',
'*auto-finalize': 'bool', '*auto-dismiss': 'bool' } }
## ##
# @blockdev-snapshot-sync: # @blockdev-snapshot-sync:

2
qapi/transaction.json
View File

@ -45,6 +45,7 @@
# #
# - @abort: since 1.6 # - @abort: since 1.6
# - @block-dirty-bitmap-add: since 2.5 # - @block-dirty-bitmap-add: since 2.5
# - @block-dirty-bitmap-remove: since 4.2
# - @block-dirty-bitmap-clear: since 2.5 # - @block-dirty-bitmap-clear: since 2.5
# - @block-dirty-bitmap-enable: since 4.0 # - @block-dirty-bitmap-enable: since 4.0
# - @block-dirty-bitmap-disable: since 4.0 # - @block-dirty-bitmap-disable: since 4.0
@ -61,6 +62,7 @@
'data': { 'data': {
'abort': 'Abort', 'abort': 'Abort',
'block-dirty-bitmap-add': 'BlockDirtyBitmapAdd', 'block-dirty-bitmap-add': 'BlockDirtyBitmapAdd',
'block-dirty-bitmap-remove': 'BlockDirtyBitmap',
'block-dirty-bitmap-clear': 'BlockDirtyBitmap', 'block-dirty-bitmap-clear': 'BlockDirtyBitmap',
'block-dirty-bitmap-enable': 'BlockDirtyBitmap', 'block-dirty-bitmap-enable': 'BlockDirtyBitmap',
'block-dirty-bitmap-disable': 'BlockDirtyBitmap', 'block-dirty-bitmap-disable': 'BlockDirtyBitmap',

12
qemu-deprecated.texi
View File

@ -154,6 +154,18 @@ The ``status'' field of the ``BlockDirtyInfo'' structure, returned by
the query-block command is deprecated. Two new boolean fields, the query-block command is deprecated. Two new boolean fields,
``recording'' and ``busy'' effectively replace it. ``recording'' and ``busy'' effectively replace it.
@subsection query-block result field dirty-bitmaps (Since 4.2)
The ``dirty-bitmaps`` field of the ``BlockInfo`` structure, returned by
the query-block command is itself now deprecated. The ``dirty-bitmaps``
field of the ``BlockDeviceInfo`` struct should be used instead, which is the
type of the ``inserted`` field in query-block replies, as well as the
type of array items in query-named-block-nodes.
Since the ``dirty-bitmaps`` field is optionally present in both the old and
new locations, clients must use introspection to learn where to anticipate
the field if/when it does appear in command output.
@subsection query-cpus (since 2.12.0) @subsection query-cpus (since 2.12.0)
The ``query-cpus'' command is replaced by the ``query-cpus-fast'' command. The ``query-cpus'' command is replaced by the ``query-cpus-fast'' command.

6
tests/qemu-iotests/040
View File

@ -85,11 +85,7 @@ class TestSingleDrive(ImageCommitTestCase):
qemu_io('-f', 'raw', '-c', 'write -P 0xab 0 524288', backing_img) qemu_io('-f', 'raw', '-c', 'write -P 0xab 0 524288', backing_img)
qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0xef 524288 524288', mid_img) qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0xef 524288 524288', mid_img)
self.vm = iotests.VM().add_drive(test_img, "node-name=top,backing.node-name=mid,backing.backing.node-name=base", interface="none") self.vm = iotests.VM().add_drive(test_img, "node-name=top,backing.node-name=mid,backing.backing.node-name=base", interface="none")
if iotests.qemu_default_machine == 's390-ccw-virtio': self.vm.add_device(iotests.get_virtio_scsi_device())
self.vm.add_device("virtio-scsi-ccw")
else:
self.vm.add_device("virtio-scsi-pci")
self.vm.add_device("scsi-hd,id=scsi0,drive=drive0") self.vm.add_device("scsi-hd,id=scsi0,drive=drive0")
self.vm.launch() self.vm.launch()
self.has_quit = False self.has_quit = False

6
tests/qemu-iotests/093
View File

@ -367,10 +367,8 @@ class ThrottleTestGroupNames(iotests.QMPTestCase):
class ThrottleTestRemovableMedia(iotests.QMPTestCase): class ThrottleTestRemovableMedia(iotests.QMPTestCase):
def setUp(self): def setUp(self):
self.vm = iotests.VM() self.vm = iotests.VM()
if iotests.qemu_default_machine == 's390-ccw-virtio': self.vm.add_device("{},id=virtio-scsi".format(
self.vm.add_device("virtio-scsi-ccw,id=virtio-scsi") iotests.get_virtio_scsi_device()))
else:
self.vm.add_device("virtio-scsi-pci,id=virtio-scsi")
self.vm.launch() self.vm.launch()
def tearDown(self): def tearDown(self):

7
tests/qemu-iotests/139
View File

@ -35,11 +35,8 @@ class TestBlockdevDel(iotests.QMPTestCase):
def setUp(self): def setUp(self):
iotests.qemu_img('create', '-f', iotests.imgfmt, base_img, '1M') iotests.qemu_img('create', '-f', iotests.imgfmt, base_img, '1M')
self.vm = iotests.VM() self.vm = iotests.VM()
if iotests.qemu_default_machine == 's390-ccw-virtio': self.vm.add_device("{},id=virtio-scsi".format(
self.vm.add_device("virtio-scsi-ccw,id=virtio-scsi") iotests.get_virtio_scsi_device()))
else:
self.vm.add_device("virtio-scsi-pci,id=virtio-scsi")
self.vm.launch() self.vm.launch()
def tearDown(self): def tearDown(self):

5
tests/qemu-iotests/238
View File

@ -23,10 +23,7 @@ import os
import iotests import iotests
from iotests import log from iotests import log
if iotests.qemu_default_machine == 's390-ccw-virtio': virtio_scsi_device = iotests.get_virtio_scsi_device()
virtio_scsi_device = 'virtio-scsi-ccw'
else:
virtio_scsi_device = 'virtio-scsi-pci'
vm = iotests.VM() vm = iotests.VM()
vm.launch() vm.launch()

30
tests/qemu-iotests/254
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
# #
# Test external snapshot with bitmap copying. # Test external snapshot with bitmap copying and moving.
# #
# Copyright (c) 2019 Virtuozzo International GmbH. All rights reserved. # Copyright (c) 2019 Virtuozzo International GmbH. All rights reserved.
# #
@ -32,6 +32,10 @@ vm = iotests.VM().add_drive(disk, opts='node-name=base')
vm.launch() vm.launch()
vm.qmp_log('block-dirty-bitmap-add', node='drive0', name='bitmap0') vm.qmp_log('block-dirty-bitmap-add', node='drive0', name='bitmap0')
vm.qmp_log('block-dirty-bitmap-add', node='drive0', name='bitmap1',
persistent=True)
vm.qmp_log('block-dirty-bitmap-add', node='drive0', name='bitmap2',
persistent=True)
vm.hmp_qemu_io('drive0', 'write 0 512K') vm.hmp_qemu_io('drive0', 'write 0 512K')
@ -39,16 +43,38 @@ vm.qmp_log('transaction', indent=2, actions=[
{'type': 'blockdev-snapshot-sync', {'type': 'blockdev-snapshot-sync',
'data': {'device': 'drive0', 'snapshot-file': top, 'data': {'device': 'drive0', 'snapshot-file': top,
'snapshot-node-name': 'snap'}}, 'snapshot-node-name': 'snap'}},
# copy non-persistent bitmap0
{'type': 'block-dirty-bitmap-add', {'type': 'block-dirty-bitmap-add',
'data': {'node': 'snap', 'name': 'bitmap0'}}, 'data': {'node': 'snap', 'name': 'bitmap0'}},
{'type': 'block-dirty-bitmap-merge', {'type': 'block-dirty-bitmap-merge',
'data': {'node': 'snap', 'target': 'bitmap0', 'data': {'node': 'snap', 'target': 'bitmap0',
'bitmaps': [{'node': 'base', 'name': 'bitmap0'}]}} 'bitmaps': [{'node': 'base', 'name': 'bitmap0'}]}},
# copy persistent bitmap1, original will be saved to base image
{'type': 'block-dirty-bitmap-add',
'data': {'node': 'snap', 'name': 'bitmap1', 'persistent': True}},
{'type': 'block-dirty-bitmap-merge',
'data': {'node': 'snap', 'target': 'bitmap1',
'bitmaps': [{'node': 'base', 'name': 'bitmap1'}]}},
# move persistent bitmap2, original will be removed and not saved
# to base image
{'type': 'block-dirty-bitmap-add',
'data': {'node': 'snap', 'name': 'bitmap2', 'persistent': True}},
{'type': 'block-dirty-bitmap-merge',
'data': {'node': 'snap', 'target': 'bitmap2',
'bitmaps': [{'node': 'base', 'name': 'bitmap2'}]}},
{'type': 'block-dirty-bitmap-remove',
'data': {'node': 'base', 'name': 'bitmap2'}}
], filters=[iotests.filter_qmp_testfiles]) ], filters=[iotests.filter_qmp_testfiles])
result = vm.qmp('query-block')['return'][0] result = vm.qmp('query-block')['return'][0]
log("query-block: device = {}, node-name = {}, dirty-bitmaps:".format( log("query-block: device = {}, node-name = {}, dirty-bitmaps:".format(
result['device'], result['inserted']['node-name'])) result['device'], result['inserted']['node-name']))
log(result['dirty-bitmaps'], indent=2) log(result['dirty-bitmaps'], indent=2)
log("\nbitmaps in backing image:")
log(result['inserted']['image']['backing-image']['format-specific'] \
['data']['bitmaps'], indent=2)
vm.shutdown() vm.shutdown()

82
tests/qemu-iotests/254.out
View File

@ -1,5 +1,9 @@
{"execute": "block-dirty-bitmap-add", "arguments": {"name": "bitmap0", "node": "drive0"}} {"execute": "block-dirty-bitmap-add", "arguments": {"name": "bitmap0", "node": "drive0"}}
{"return": {}} {"return": {}}
{"execute": "block-dirty-bitmap-add", "arguments": {"name": "bitmap1", "node": "drive0", "persistent": true}}
{"return": {}}
{"execute": "block-dirty-bitmap-add", "arguments": {"name": "bitmap2", "node": "drive0", "persistent": true}}
{"return": {}}
{ {
"execute": "transaction", "execute": "transaction",
"arguments": { "arguments": {
@ -31,6 +35,55 @@
"target": "bitmap0" "target": "bitmap0"
}, },
"type": "block-dirty-bitmap-merge" "type": "block-dirty-bitmap-merge"
},
{
"data": {
"name": "bitmap1",
"node": "snap",
"persistent": true
},
"type": "block-dirty-bitmap-add"
},
{
"data": {
"bitmaps": [
{
"name": "bitmap1",
"node": "base"
}
],
"node": "snap",
"target": "bitmap1"
},
"type": "block-dirty-bitmap-merge"
},
{
"data": {
"name": "bitmap2",
"node": "snap",
"persistent": true
},
"type": "block-dirty-bitmap-add"
},
{
"data": {
"bitmaps": [
{
"name": "bitmap2",
"node": "base"
}
],
"node": "snap",
"target": "bitmap2"
},
"type": "block-dirty-bitmap-merge"
},
{
"data": {
"name": "bitmap2",
"node": "base"
},
"type": "block-dirty-bitmap-remove"
} }
] ]
} }
@ -40,6 +93,24 @@
} }
query-block: device = drive0, node-name = snap, dirty-bitmaps: query-block: device = drive0, node-name = snap, dirty-bitmaps:
[ [
{
"busy": false,
"count": 524288,
"granularity": 65536,
"name": "bitmap2",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 524288,
"granularity": 65536,
"name": "bitmap1",
"persistent": true,
"recording": true,
"status": "active"
},
{ {
"busy": false, "busy": false,
"count": 524288, "count": 524288,
@ -50,3 +121,14 @@ query-block: device = drive0, node-name = snap, dirty-bitmaps:
"status": "active" "status": "active"
} }
] ]
bitmaps in backing image:
[
{
"flags": [
"auto"
],
"granularity": 65536,
"name": "bitmap1"
}
]

4
tests/qemu-iotests/256.out
View File

@ -113,7 +113,7 @@
{ {
"return": {} "return": {}
} }
{"data": {"device": "j2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"device": "j2", "len": 0, "offset": 0, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"data": {"device": "j3", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"device": "j3", "len": 0, "offset": 0, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
--- Done --- --- Done ---

560
tests/qemu-iotests/257 Executable file
View File

@ -0,0 +1,560 @@
#!/usr/bin/env python
#
# Test bitmap-sync backups (incremental, differential, and partials)
#
# Copyright (c) 2019 John Snow for Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# owner=jsnow@redhat.com
import math
import os
import iotests
from iotests import log, qemu_img
SIZE = 64 * 1024 * 1024
GRANULARITY = 64 * 1024
class Pattern:
def __init__(self, byte, offset, size=GRANULARITY):
self.byte = byte
self.offset = offset
self.size = size
def bits(self, granularity):
lower = self.offset // granularity
upper = (self.offset + self.size - 1) // granularity
return set(range(lower, upper + 1))
class PatternGroup:
"""Grouping of Pattern objects. Initialize with an iterable of Patterns."""
def __init__(self, patterns):
self.patterns = patterns
def bits(self, granularity):
"""Calculate the unique bits dirtied by this pattern grouping"""
res = set()
for pattern in self.patterns:
res |= pattern.bits(granularity)
return res
GROUPS = [
PatternGroup([
# Batch 0: 4 clusters
Pattern('0x49', 0x0000000),
Pattern('0x6c', 0x0100000), # 1M
Pattern('0x6f', 0x2000000), # 32M
Pattern('0x76', 0x3ff0000)]), # 64M - 64K
PatternGroup([
# Batch 1: 6 clusters (3 new)
Pattern('0x65', 0x0000000), # Full overwrite
Pattern('0x77', 0x00f8000), # Partial-left (1M-32K)
Pattern('0x72', 0x2008000), # Partial-right (32M+32K)
Pattern('0x69', 0x3fe0000)]), # Adjacent-left (64M - 128K)
PatternGroup([
# Batch 2: 7 clusters (3 new)
Pattern('0x74', 0x0010000), # Adjacent-right
Pattern('0x69', 0x00e8000), # Partial-left (1M-96K)
Pattern('0x6e', 0x2018000), # Partial-right (32M+96K)
Pattern('0x67', 0x3fe0000,
2*GRANULARITY)]), # Overwrite [(64M-128K)-64M)
PatternGroup([
# Batch 3: 8 clusters (5 new)
# Carefully chosen such that nothing re-dirties the one cluster
# that copies out successfully before failure in Group #1.
Pattern('0xaa', 0x0010000,
3*GRANULARITY), # Overwrite and 2x Adjacent-right
Pattern('0xbb', 0x00d8000), # Partial-left (1M-160K)
Pattern('0xcc', 0x2028000), # Partial-right (32M+160K)
Pattern('0xdd', 0x3fc0000)]), # New; leaving a gap to the right
]
class EmulatedBitmap:
def __init__(self, granularity=GRANULARITY):
self._bits = set()
self.granularity = granularity
def dirty_bits(self, bits):
self._bits |= set(bits)
def dirty_group(self, n):
self.dirty_bits(GROUPS[n].bits(self.granularity))
def clear(self):
self._bits = set()
def clear_bits(self, bits):
self._bits -= set(bits)
def clear_bit(self, bit):
self.clear_bits({bit})
def clear_group(self, n):
self.clear_bits(GROUPS[n].bits(self.granularity))
@property
def first_bit(self):
return sorted(self.bits)[0]
@property
def bits(self):
return self._bits
@property
def count(self):
return len(self.bits)
def compare(self, qmp_bitmap):
"""
Print a nice human-readable message checking that a bitmap as reported
by the QMP interface has as many bits set as we expect it to.
"""
name = qmp_bitmap.get('name', '(anonymous)')
log("= Checking Bitmap {:s} =".format(name))
want = self.count
have = qmp_bitmap['count'] // qmp_bitmap['granularity']
log("expecting {:d} dirty sectors; have {:d}. {:s}".format(
want, have, "OK!" if want == have else "ERROR!"))
log('')
class Drive:
"""Represents, vaguely, a drive attached to a VM.
Includes format, graph, and device information."""
def __init__(self, path, vm=None):
self.path = path
self.vm = vm
self.fmt = None
self.size = None
self.node = None
self.device = None
@property
def name(self):
return self.node or self.device
def img_create(self, fmt, size):
self.fmt = fmt
self.size = size
iotests.qemu_img_create('-f', self.fmt, self.path, str(self.size))
def create_target(self, name, fmt, size):
basename = os.path.basename(self.path)
file_node_name = "file_{}".format(basename)
vm = self.vm
log(vm.command('blockdev-create', job_id='bdc-file-job',
options={
'driver': 'file',
'filename': self.path,
'size': 0,
}))
vm.run_job('bdc-file-job')
log(vm.command('blockdev-add', driver='file',
node_name=file_node_name, filename=self.path))
log(vm.command('blockdev-create', job_id='bdc-fmt-job',
options={
'driver': fmt,
'file': file_node_name,
'size': size,
}))
vm.run_job('bdc-fmt-job')
log(vm.command('blockdev-add', driver=fmt,
node_name=name,
file=file_node_name))
self.fmt = fmt
self.size = size
self.node = name
def query_bitmaps(vm):
res = vm.qmp("query-block")
return {"bitmaps": {device['device'] or device['qdev']:
device.get('dirty-bitmaps', []) for
device in res['return']}}
def get_bitmap(bitmaps, drivename, name, recording=None):
"""
get a specific bitmap from the object returned by query_bitmaps.
:param recording: If specified, filter results by the specified value.
"""
for bitmap in bitmaps['bitmaps'][drivename]:
if bitmap.get('name', '') == name:
if recording is None:
return bitmap
elif bitmap.get('recording') == recording:
return bitmap
return None
def blockdev_backup(vm, device, target, sync, **kwargs):
# Strip any arguments explicitly nulled by the caller:
kwargs = {key: val for key, val in kwargs.items() if val is not None}
result = vm.qmp_log('blockdev-backup',
device=device,
target=target,
sync=sync,
**kwargs)
return result
def blockdev_backup_mktarget(drive, target_id, filepath, sync, **kwargs):
target_drive = Drive(filepath, vm=drive.vm)
target_drive.create_target(target_id, drive.fmt, drive.size)
blockdev_backup(drive.vm, drive.name, target_id, sync, **kwargs)
def reference_backup(drive, n, filepath):
log("--- Reference Backup #{:d} ---\n".format(n))
target_id = "ref_target_{:d}".format(n)
job_id = "ref_backup_{:d}".format(n)
blockdev_backup_mktarget(drive, target_id, filepath, "full",
job_id=job_id)
drive.vm.run_job(job_id, auto_dismiss=True)
log('')
def backup(drive, n, filepath, sync, **kwargs):
log("--- Test Backup #{:d} ---\n".format(n))
target_id = "backup_target_{:d}".format(n)
job_id = "backup_{:d}".format(n)
kwargs.setdefault('auto-finalize', False)
blockdev_backup_mktarget(drive, target_id, filepath, sync,
job_id=job_id, **kwargs)
return job_id
def perform_writes(drive, n):
log("--- Write #{:d} ---\n".format(n))
for pattern in GROUPS[n].patterns:
cmd = "write -P{:s} 0x{:07x} 0x{:x}".format(
pattern.byte,
pattern.offset,
pattern.size)
log(cmd)
log(drive.vm.hmp_qemu_io(drive.name, cmd))
bitmaps = query_bitmaps(drive.vm)
log(bitmaps, indent=2)
log('')
return bitmaps
def compare_images(image, reference, baseimg=None, expected_match=True):
"""
Print a nice human-readable message comparing these images.
"""
expected_ret = 0 if expected_match else 1
if baseimg:
assert qemu_img("rebase", "-u", "-b", baseimg, image) == 0
ret = qemu_img("compare", image, reference)
log('qemu_img compare "{:s}" "{:s}" ==> {:s}, {:s}'.format(
image, reference,
"Identical" if ret == 0 else "Mismatch",
"OK!" if ret == expected_ret else "ERROR!"),
filters=[iotests.filter_testfiles])
def test_bitmap_sync(bsync_mode, msync_mode='bitmap', failure=None):
"""
Test bitmap backup routines.
:param bsync_mode: Is the Bitmap Sync mode, and can be any of:
- on-success: This is the "incremental" style mode. Bitmaps are
synchronized to what was copied out only on success.
(Partial images must be discarded.)
- never: This is the "differential" style mode.
Bitmaps are never synchronized.
- always: This is a "best effort" style mode.
Bitmaps are always synchronized, regardless of failure.
(Partial images must be kept.)
:param msync_mode: The mirror sync mode to use for the first backup.
Can be any one of:
- bitmap: Backups based on bitmap manifest.
- full: Full backups.
- top: Full backups of the top layer only.
:param failure: Is the (optional) failure mode, and can be any of:
- None: No failure. Test the normative path. Default.
- simulated: Cancel the job right before it completes.
This also tests writes "during" the job.
- intermediate: This tests a job that fails mid-process and produces
an incomplete backup. Testing limitations prevent
testing competing writes.
"""
with iotests.FilePaths(['img', 'bsync1', 'bsync2',
'fbackup0', 'fbackup1', 'fbackup2']) as \
(img_path, bsync1, bsync2,
fbackup0, fbackup1, fbackup2), \
iotests.VM() as vm:
mode = "Mode {:s}; Bitmap Sync {:s}".format(msync_mode, bsync_mode)
preposition = "with" if failure else "without"
cond = "{:s} {:s}".format(preposition,
"{:s} failure".format(failure) if failure
else "failure")
log("\n=== {:s} {:s} ===\n".format(mode, cond))
log('--- Preparing image & VM ---\n')
drive0 = Drive(img_path, vm=vm)
drive0.img_create(iotests.imgfmt, SIZE)
vm.add_device("{},id=scsi0".format(iotests.get_virtio_scsi_device()))
vm.launch()
file_config = {
'driver': 'file',
'filename': drive0.path
}
if failure == 'intermediate':
file_config = {
'driver': 'blkdebug',
'image': file_config,
'set-state': [{
'event': 'flush_to_disk',
'state': 1,
'new_state': 2
}, {
'event': 'read_aio',
'state': 2,
'new_state': 3
}],
'inject-error': [{
'event': 'read_aio',
'errno': 5,
'state': 3,
'immediately': False,
'once': True
}]
}
vm.qmp_log('blockdev-add',
filters=[iotests.filter_qmp_testfiles],
node_name="drive0",
driver=drive0.fmt,
file=file_config)
drive0.node = 'drive0'
drive0.device = 'device0'
# Use share-rw to allow writes directly to the node;
# The anonymous block-backend for this configuration prevents us
# from using HMP's qemu-io commands to address the device.
vm.qmp_log("device_add", id=drive0.device,
drive=drive0.name, driver="scsi-hd",
share_rw=True)
log('')
# 0 - Writes and Reference Backup
perform_writes(drive0, 0)
reference_backup(drive0, 0, fbackup0)
log('--- Add Bitmap ---\n')
vm.qmp_log("block-dirty-bitmap-add", node=drive0.name,
name="bitmap0", granularity=GRANULARITY)
log('')
ebitmap = EmulatedBitmap()
# 1 - Writes and Reference Backup
bitmaps = perform_writes(drive0, 1)
ebitmap.dirty_group(1)
bitmap = get_bitmap(bitmaps, drive0.device, 'bitmap0')
ebitmap.compare(bitmap)
reference_backup(drive0, 1, fbackup1)
# 1 - Test Backup (w/ Optional induced failure)
if failure == 'intermediate':
# Activate blkdebug induced failure for second-to-next read
log(vm.hmp_qemu_io(drive0.name, 'flush'))
log('')
job = backup(drive0, 1, bsync1, msync_mode,
bitmap="bitmap0", bitmap_mode=bsync_mode)
def _callback():
"""Issue writes while the job is open to test bitmap divergence."""
# Note: when `failure` is 'intermediate', this isn't called.
log('')
bitmaps = perform_writes(drive0, 2)
# Named bitmap (static, should be unchanged)
ebitmap.compare(get_bitmap(bitmaps, drive0.device, 'bitmap0'))
# Anonymous bitmap (dynamic, shows new writes)
anonymous = EmulatedBitmap()
anonymous.dirty_group(2)
anonymous.compare(get_bitmap(bitmaps, drive0.device, '',
recording=True))
# Simulate the order in which this will happen:
# group 1 gets cleared first, then group two gets written.
if ((bsync_mode == 'on-success' and not failure) or
(bsync_mode == 'always')):
ebitmap.clear()
ebitmap.dirty_group(2)
vm.run_job(job, auto_dismiss=True, auto_finalize=False,
pre_finalize=_callback,
cancel=(failure == 'simulated'))
bitmaps = query_bitmaps(vm)
log(bitmaps, indent=2)
log('')
if bsync_mode == 'always' and failure == 'intermediate':
# TOP treats anything allocated as dirty, expect to see:
if msync_mode == 'top':
ebitmap.dirty_group(0)
# We manage to copy one sector (one bit) before the error.
ebitmap.clear_bit(ebitmap.first_bit)
# Full returns all bits set except what was copied/skipped
if msync_mode == 'full':
fail_bit = ebitmap.first_bit
ebitmap.clear()
ebitmap.dirty_bits(range(fail_bit, SIZE // GRANULARITY))
ebitmap.compare(get_bitmap(bitmaps, drive0.device, 'bitmap0'))
# 2 - Writes and Reference Backup
bitmaps = perform_writes(drive0, 3)
ebitmap.dirty_group(3)
ebitmap.compare(get_bitmap(bitmaps, drive0.device, 'bitmap0'))
reference_backup(drive0, 2, fbackup2)
# 2 - Bitmap Backup (In failure modes, this is a recovery.)
job = backup(drive0, 2, bsync2, "bitmap",
bitmap="bitmap0", bitmap_mode=bsync_mode)
vm.run_job(job, auto_dismiss=True, auto_finalize=False)
bitmaps = query_bitmaps(vm)
log(bitmaps, indent=2)
log('')
if bsync_mode != 'never':
ebitmap.clear()
ebitmap.compare(get_bitmap(bitmaps, drive0.device, 'bitmap0'))
log('--- Cleanup ---\n')
vm.qmp_log("block-dirty-bitmap-remove",
node=drive0.name, name="bitmap0")
log(query_bitmaps(vm), indent=2)
vm.shutdown()
log('')
log('--- Verification ---\n')
# 'simulated' failures will actually all pass here because we canceled
# while "pending". This is actually undefined behavior,
# don't rely on this to be true!
compare_images(bsync1, fbackup1, baseimg=fbackup0,
expected_match=failure != 'intermediate')
if not failure or bsync_mode == 'always':
# Always keep the last backup on success or when using 'always'
base = bsync1
else:
base = fbackup0
compare_images(bsync2, fbackup2, baseimg=base)
compare_images(img_path, fbackup2)
log('')
def test_backup_api():
"""
Test malformed and prohibited invocations of the backup API.
"""
with iotests.FilePaths(['img', 'bsync1']) as \
(img_path, backup_path), \
iotests.VM() as vm:
log("\n=== API failure tests ===\n")
log('--- Preparing image & VM ---\n')
drive0 = Drive(img_path, vm=vm)
drive0.img_create(iotests.imgfmt, SIZE)
vm.add_device("{},id=scsi0".format(iotests.get_virtio_scsi_device()))
vm.launch()
file_config = {
'driver': 'file',
'filename': drive0.path
}
vm.qmp_log('blockdev-add',
filters=[iotests.filter_qmp_testfiles],
node_name="drive0",
driver=drive0.fmt,
file=file_config)
drive0.node = 'drive0'
drive0.device = 'device0'
vm.qmp_log("device_add", id=drive0.device,
drive=drive0.name, driver="scsi-hd")
log('')
target0 = Drive(backup_path, vm=vm)
target0.create_target("backup_target", drive0.fmt, drive0.size)
log('')
vm.qmp_log("block-dirty-bitmap-add", node=drive0.name,
name="bitmap0", granularity=GRANULARITY)
log('')
log('-- Testing invalid QMP commands --\n')
error_cases = {
'incremental': {
None: ['on-success', 'always', 'never', None],
'bitmap404': ['on-success', 'always', 'never', None],
'bitmap0': ['always', 'never']
},
'bitmap': {
None: ['on-success', 'always', 'never', None],
'bitmap404': ['on-success', 'always', 'never', None],
'bitmap0': [None],
},
'full': {
None: ['on-success', 'always', 'never'],
'bitmap404': ['on-success', 'always', 'never', None],
'bitmap0': ['never', None],
},
'top': {
None: ['on-success', 'always', 'never'],
'bitmap404': ['on-success', 'always', 'never', None],
'bitmap0': ['never', None],
},
'none': {
None: ['on-success', 'always', 'never'],
'bitmap404': ['on-success', 'always', 'never', None],
'bitmap0': ['on-success', 'always', 'never', None],
}
}
# Dicts, as always, are not stably-ordered prior to 3.7, so use tuples:
for sync_mode in ('incremental', 'bitmap', 'full', 'top', 'none'):
log("-- Sync mode {:s} tests --\n".format(sync_mode))
for bitmap in (None, 'bitmap404', 'bitmap0'):
for policy in error_cases[sync_mode][bitmap]:
blockdev_backup(drive0.vm, drive0.name, "backup_target",
sync_mode, job_id='api_job',
bitmap=bitmap, bitmap_mode=policy)
log('')
def main():
for bsync_mode in ("never", "on-success", "always"):
for failure in ("simulated", "intermediate", None):
test_bitmap_sync(bsync_mode, "bitmap", failure)
for sync_mode in ('full', 'top'):
for bsync_mode in ('on-success', 'always'):
for failure in ('simulated', 'intermediate', None):
test_bitmap_sync(bsync_mode, sync_mode, failure)
test_backup_api()
if __name__ == '__main__':
iotests.script_main(main, supported_fmts=['qcow2'])

5421
tests/qemu-iotests/257.out Normal file
View File

File diff suppressed because it is too large Load Diff

1
tests/qemu-iotests/group
View File

@ -271,5 +271,6 @@
254 rw backing quick 254 rw backing quick
255 rw quick 255 rw quick
256 rw quick 256 rw quick
257 rw
258 rw quick 258 rw quick
262 rw quick migration 262 rw quick migration

102
tests/qemu-iotests/iotests.py
View File

@ -61,7 +61,6 @@ cachemode = os.environ.get('CACHEMODE')
qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE') qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE')
socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper') socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper')
debug = False
luks_default_secret_object = 'secret,id=keysec0,data=' + \ luks_default_secret_object = 'secret,id=keysec0,data=' + \
os.environ.get('IMGKEYSECRET', '') os.environ.get('IMGKEYSECRET', '')
@ -165,6 +164,10 @@ def qemu_io_silent(*args):
(-exitcode, ' '.join(args))) (-exitcode, ' '.join(args)))
return exitcode return exitcode
def get_virtio_scsi_device():
if qemu_default_machine == 's390-ccw-virtio':
return 'virtio-scsi-ccw'
return 'virtio-scsi-pci'
class QemuIoInteractive: class QemuIoInteractive:
def __init__(self, *args): def __init__(self, *args):
@ -359,31 +362,45 @@ class Timeout:
def timeout(self, signum, frame): def timeout(self, signum, frame):
raise Exception(self.errmsg) raise Exception(self.errmsg)
def file_pattern(name):
return "{0}-{1}".format(os.getpid(), name)
class FilePath(object): class FilePaths(object):
'''An auto-generated filename that cleans itself up. """
FilePaths is an auto-generated filename that cleans itself up.
Use this context manager to generate filenames and ensure that the file Use this context manager to generate filenames and ensure that the file
gets deleted:: gets deleted::
with TestFilePath('test.img') as img_path: with FilePaths(['test.img']) as img_path:
qemu_img('create', img_path, '1G') qemu_img('create', img_path, '1G')
# migration_sock_path is automatically deleted # migration_sock_path is automatically deleted
''' """
def __init__(self, name): def __init__(self, names):
filename = '{0}-{1}'.format(os.getpid(), name) self.paths = []
self.path = os.path.join(test_dir, filename) for name in names:
self.paths.append(os.path.join(test_dir, file_pattern(name)))
def __enter__(self): def __enter__(self):
return self.path return self.paths
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
try: try:
os.remove(self.path) for path in self.paths:
os.remove(path)
except OSError: except OSError:
pass pass
return False return False
class FilePath(FilePaths):
"""
FilePath is a specialization of FilePaths that takes a single filename.
"""
def __init__(self, name):
super(FilePath, self).__init__([name])
def __enter__(self):
return self.paths[0]
def file_path_remover(): def file_path_remover():
for path in reversed(file_path_remover.paths): for path in reversed(file_path_remover.paths):
@ -408,7 +425,7 @@ def file_path(*names):
paths = [] paths = []
for name in names: for name in names:
filename = '{0}-{1}'.format(os.getpid(), name) filename = file_pattern(name)
path = os.path.join(test_dir, filename) path = os.path.join(test_dir, filename)
file_path_remover.paths.append(path) file_path_remover.paths.append(path)
paths.append(path) paths.append(path)
@ -542,7 +559,23 @@ class VM(qtest.QEMUQtestMachine):
# Returns None on success, and an error string on failure # Returns None on success, and an error string on failure
def run_job(self, job, auto_finalize=True, auto_dismiss=False, def run_job(self, job, auto_finalize=True, auto_dismiss=False,
pre_finalize=None, use_log=True, wait=60.0): pre_finalize=None, cancel=False, use_log=True, wait=60.0):
"""
run_job moves a job from creation through to dismissal.
:param job: String. ID of recently-launched job
:param auto_finalize: Bool. True if the job was launched with
auto_finalize. Defaults to True.
:param auto_dismiss: Bool. True if the job was launched with
auto_dismiss=True. Defaults to False.
:param pre_finalize: Callback. A callable that takes no arguments to be
invoked prior to issuing job-finalize, if any.
:param cancel: Bool. When true, cancels the job after the pre_finalize
callback.
:param use_log: Bool. When false, does not log QMP messages.
:param wait: Float. Timeout value specifying how long to wait for any
event, in seconds. Defaults to 60.0.
"""
match_device = {'data': {'device': job}} match_device = {'data': {'device': job}}
match_id = {'data': {'id': job}} match_id = {'data': {'id': job}}
events = [ events = [
@ -571,7 +604,11 @@ class VM(qtest.QEMUQtestMachine):
elif status == 'pending' and not auto_finalize: elif status == 'pending' and not auto_finalize:
if pre_finalize: if pre_finalize:
pre_finalize() pre_finalize()
if use_log: if cancel and use_log:
self.qmp_log('job-cancel', id=job)
elif cancel:
self.qmp('job-cancel', id=job)
elif use_log:
self.qmp_log('job-finalize', id=job) self.qmp_log('job-finalize', id=job)
else: else:
self.qmp('job-finalize', id=job) self.qmp('job-finalize', id=job)
@ -858,11 +895,22 @@ def skip_if_unsupported(required_formats=[], read_only=False):
return func_wrapper return func_wrapper
return skip_test_decorator return skip_test_decorator
def main(supported_fmts=[], supported_oses=['linux'], supported_cache_modes=[], def execute_unittest(output, verbosity, debug):
unsupported_fmts=[]): runner = unittest.TextTestRunner(stream=output, descriptions=True,
'''Run tests''' verbosity=verbosity)
try:
# unittest.main() will use sys.exit(); so expect a SystemExit
# exception
unittest.main(testRunner=runner)
finally:
if not debug:
sys.stderr.write(re.sub(r'Ran (\d+) tests? in [\d.]+s',
r'Ran \1 tests', output.getvalue()))
global debug def execute_test(test_function=None,
supported_fmts=[], supported_oses=['linux'],
supported_cache_modes=[], unsupported_fmts=[]):
"""Run either unittest or script-style tests."""
# We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to
# indicate that we're not being run via "check". There may be # indicate that we're not being run via "check". There may be
@ -894,13 +942,15 @@ def main(supported_fmts=[], supported_oses=['linux'], supported_cache_modes=[],
logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN)) logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN))
class MyTestRunner(unittest.TextTestRunner): if not test_function:
def __init__(self, stream=output, descriptions=True, verbosity=verbosity): execute_unittest(output, verbosity, debug)
unittest.TextTestRunner.__init__(self, stream, descriptions, verbosity) else:
test_function()
# unittest.main() will use sys.exit() so expect a SystemExit exception def script_main(test_function, *args, **kwargs):
try: """Run script-style tests outside of the unittest framework"""
unittest.main(testRunner=MyTestRunner) execute_test(test_function, *args, **kwargs)
finally:
if not debug: def main(*args, **kwargs):
sys.stderr.write(re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', output.getvalue())) """Run tests using the unittest framework"""
execute_test(None, *args, **kwargs)

22
tests/test-hbitmap.c
View File

@ -1004,6 +1004,15 @@ static void test_hbitmap_next_zero_4(TestHBitmapData *data, const void *unused)
test_hbitmap_next_zero_do(data, 4); test_hbitmap_next_zero_do(data, 4);
} }
static void test_hbitmap_next_zero_after_truncate(TestHBitmapData *data,
const void *unused)
{
hbitmap_test_init(data, L1, 0);
hbitmap_test_truncate_impl(data, L1 * 2);
hbitmap_set(data->hb, 0, L1);
test_hbitmap_next_zero_check(data, 0);
}
static void test_hbitmap_next_dirty_area_check(TestHBitmapData *data, static void test_hbitmap_next_dirty_area_check(TestHBitmapData *data,
uint64_t offset, uint64_t offset,
uint64_t count) uint64_t count)
@ -1104,6 +1113,15 @@ static void test_hbitmap_next_dirty_area_4(TestHBitmapData *data,
test_hbitmap_next_dirty_area_do(data, 4); test_hbitmap_next_dirty_area_do(data, 4);
} }
static void test_hbitmap_next_dirty_area_after_truncate(TestHBitmapData *data,
const void *unused)
{
hbitmap_test_init(data, L1, 0);
hbitmap_test_truncate_impl(data, L1 * 2);
hbitmap_set(data->hb, L1 + 1, 1);
test_hbitmap_next_dirty_area_check(data, 0, UINT64_MAX);
}
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
g_test_init(&argc, &argv, NULL); g_test_init(&argc, &argv, NULL);
@ -1169,6 +1187,8 @@ int main(int argc, char **argv)
test_hbitmap_next_zero_0); test_hbitmap_next_zero_0);
hbitmap_test_add("/hbitmap/next_zero/next_zero_4", hbitmap_test_add("/hbitmap/next_zero/next_zero_4",
test_hbitmap_next_zero_4); test_hbitmap_next_zero_4);
hbitmap_test_add("/hbitmap/next_zero/next_zero_after_truncate",
test_hbitmap_next_zero_after_truncate);
hbitmap_test_add("/hbitmap/next_dirty_area/next_dirty_area_0", hbitmap_test_add("/hbitmap/next_dirty_area/next_dirty_area_0",
test_hbitmap_next_dirty_area_0); test_hbitmap_next_dirty_area_0);
@ -1176,6 +1196,8 @@ int main(int argc, char **argv)
test_hbitmap_next_dirty_area_1); test_hbitmap_next_dirty_area_1);
hbitmap_test_add("/hbitmap/next_dirty_area/next_dirty_area_4", hbitmap_test_add("/hbitmap/next_dirty_area/next_dirty_area_4",
test_hbitmap_next_dirty_area_4); test_hbitmap_next_dirty_area_4);
hbitmap_test_add("/hbitmap/next_dirty_area/next_dirty_area_after_truncate",
test_hbitmap_next_dirty_area_after_truncate);
g_test_run(); g_test_run();

49
util/hbitmap.c
View File

@ -781,12 +781,33 @@ void hbitmap_truncate(HBitmap *hb, uint64_t size)
bool hbitmap_can_merge(const HBitmap *a, const HBitmap *b) bool hbitmap_can_merge(const HBitmap *a, const HBitmap *b)
{ {
return (a->size == b->size) && (a->granularity == b->granularity); return (a->orig_size == b->orig_size);
} }
/** /**
* Given HBitmaps A and B, let A := A (BITOR) B. * hbitmap_sparse_merge: performs dst = dst | src
* Bitmap B will not be modified. * works with differing granularities.
* best used when src is sparsely populated.
*/
static void hbitmap_sparse_merge(HBitmap *dst, const HBitmap *src)
{
uint64_t offset = 0;
uint64_t count = src->orig_size;
while (hbitmap_next_dirty_area(src, &offset, &count)) {
hbitmap_set(dst, offset, count);
offset += count;
if (offset >= src->orig_size) {
break;
}
count = src->orig_size - offset;
}
}
/**
* Given HBitmaps A and B, let R := A (BITOR) B.
* Bitmaps A and B will not be modified,
* except when bitmap R is an alias of A or B.
* *
* @return true if the merge was successful, * @return true if the merge was successful,
* false if it was not attempted. * false if it was not attempted.
@ -801,7 +822,26 @@ bool hbitmap_merge(const HBitmap *a, const HBitmap *b, HBitmap *result)
} }
assert(hbitmap_can_merge(b, result)); assert(hbitmap_can_merge(b, result));
if (hbitmap_count(b) == 0) { if ((!hbitmap_count(a) && result == b) ||
(!hbitmap_count(b) && result == a)) {
return true;
}
if (!hbitmap_count(a) && !hbitmap_count(b)) {
hbitmap_reset_all(result);
return true;
}
if (a->granularity != b->granularity) {
if ((a != result) && (b != result)) {
hbitmap_reset_all(result);
}
if (a != result) {
hbitmap_sparse_merge(result, a);
}
if (b != result) {
hbitmap_sparse_merge(result, b);
}
return true; return true;
} }
@ -809,6 +849,7 @@ bool hbitmap_merge(const HBitmap *a, const HBitmap *b, HBitmap *result)
* It may be possible to improve running times for sparsely populated maps * It may be possible to improve running times for sparsely populated maps
* by using hbitmap_iter_next, but this is suboptimal for dense maps. * by using hbitmap_iter_next, but this is suboptimal for dense maps.
*/ */
assert(a->size == b->size);
for (i = HBITMAP_LEVELS - 1; i >= 0; i--) { for (i = HBITMAP_LEVELS - 1; i >= 0; i--) {
for (j = 0; j < a->sizes[i]; j++) { for (j = 0; j < a->sizes[i]; j++) {
result->levels[i][j] = a->levels[i][j] | b->levels[i][j]; result->levels[i][j] = a->levels[i][j] | b->levels[i][j];