417 lines
16 KiB
Plaintext
417 lines
16 KiB
Plaintext
|
#!/usr/bin/env python
|
||
|
|
#
|
||
|
|
# Test bitmap-sync backups (incremental, differential, and partials)
|
||
|
|
#
|
||
|
|
# Copyright (c) 2019 John Snow for Red Hat, Inc.
|
||
|
|
#
|
||
|
|
# This program is free software; you can redistribute it and/or modify
|
||
|
|
# it under the terms of the GNU General Public License as published by
|
||
|
|
# the Free Software Foundation; either version 2 of the License, or
|
||
|
|
# (at your option) any later version.
|
||
|
|
#
|
||
|
|
# This program is distributed in the hope that it will be useful,
|
||
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
|
|
# GNU General Public License for more details.
|
||
|
|
#
|
||
|
|
# You should have received a copy of the GNU General Public License
|
||
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||
|
|
#
|
||
|
|
# owner=jsnow@redhat.com
|
||
|
|
|
||
|
|
from collections import namedtuple
|
||
|
|
import math
|
||
|
|
import os
|
||
|
|
|
||
|
|
import iotests
|
||
|
|
from iotests import log, qemu_img
|
||
|
|
|
||
|
|
SIZE = 64 * 1024 * 1024
|
||
|
|
GRANULARITY = 64 * 1024
|
||
|
|
|
||
|
|
Pattern = namedtuple('Pattern', ['byte', 'offset', 'size'])
|
||
|
|
def mkpattern(byte, offset, size=GRANULARITY):
|
||
|
|
"""Constructor for Pattern() with default size"""
|
||
|
|
return Pattern(byte, offset, size)
|
||
|
|
|
||
|
|
class PatternGroup:
|
||
|
|
"""Grouping of Pattern objects. Initialize with an iterable of Patterns."""
|
||
|
|
def __init__(self, patterns):
|
||
|
|
self.patterns = patterns
|
||
|
|
|
||
|
|
def bits(self, granularity):
|
||
|
|
"""Calculate the unique bits dirtied by this pattern grouping"""
|
||
|
|
res = set()
|
||
|
|
for pattern in self.patterns:
|
||
|
|
lower = pattern.offset // granularity
|
||
|
|
upper = (pattern.offset + pattern.size - 1) // granularity
|
||
|
|
res = res | set(range(lower, upper + 1))
|
||
|
|
return res
|
||
|
|
|
||
|
|
GROUPS = [
|
||
|
|
PatternGroup([
|
||
|
|
# Batch 0: 4 clusters
|
||
|
|
mkpattern('0x49', 0x0000000),
|
||
|
|
mkpattern('0x6c', 0x0100000), # 1M
|
||
|
|
mkpattern('0x6f', 0x2000000), # 32M
|
||
|
|
mkpattern('0x76', 0x3ff0000)]), # 64M - 64K
|
||
|
|
PatternGroup([
|
||
|
|
# Batch 1: 6 clusters (3 new)
|
||
|
|
mkpattern('0x65', 0x0000000), # Full overwrite
|
||
|
|
mkpattern('0x77', 0x00f8000), # Partial-left (1M-32K)
|
||
|
|
mkpattern('0x72', 0x2008000), # Partial-right (32M+32K)
|
||
|
|
mkpattern('0x69', 0x3fe0000)]), # Adjacent-left (64M - 128K)
|
||
|
|
PatternGroup([
|
||
|
|
# Batch 2: 7 clusters (3 new)
|
||
|
|
mkpattern('0x74', 0x0010000), # Adjacent-right
|
||
|
|
mkpattern('0x69', 0x00e8000), # Partial-left (1M-96K)
|
||
|
|
mkpattern('0x6e', 0x2018000), # Partial-right (32M+96K)
|
||
|
|
mkpattern('0x67', 0x3fe0000,
|
||
|
|
2*GRANULARITY)]), # Overwrite [(64M-128K)-64M)
|
||
|
|
PatternGroup([
|
||
|
|
# Batch 3: 8 clusters (5 new)
|
||
|
|
# Carefully chosen such that nothing re-dirties the one cluster
|
||
|
|
# that copies out successfully before failure in Group #1.
|
||
|
|
mkpattern('0xaa', 0x0010000,
|
||
|
|
3*GRANULARITY), # Overwrite and 2x Adjacent-right
|
||
|
|
mkpattern('0xbb', 0x00d8000), # Partial-left (1M-160K)
|
||
|
|
mkpattern('0xcc', 0x2028000), # Partial-right (32M+160K)
|
||
|
|
mkpattern('0xdd', 0x3fc0000)]), # New; leaving a gap to the right
|
||
|
|
]
|
||
|
|
|
||
|
|
class Drive:
|
||
|
|
"""Represents, vaguely, a drive attached to a VM.
|
||
|
|
Includes format, graph, and device information."""
|
||
|
|
|
||
|
|
def __init__(self, path, vm=None):
|
||
|
|
self.path = path
|
||
|
|
self.vm = vm
|
||
|
|
self.fmt = None
|
||
|
|
self.size = None
|
||
|
|
self.node = None
|
||
|
|
self.device = None
|
||
|
|
|
||
|
|
@property
|
||
|
|
def name(self):
|
||
|
|
return self.node or self.device
|
||
|
|
|
||
|
|
def img_create(self, fmt, size):
|
||
|
|
self.fmt = fmt
|
||
|
|
self.size = size
|
||
|
|
iotests.qemu_img_create('-f', self.fmt, self.path, str(self.size))
|
||
|
|
|
||
|
|
def create_target(self, name, fmt, size):
|
||
|
|
basename = os.path.basename(self.path)
|
||
|
|
file_node_name = "file_{}".format(basename)
|
||
|
|
vm = self.vm
|
||
|
|
|
||
|
|
log(vm.command('blockdev-create', job_id='bdc-file-job',
|
||
|
|
options={
|
||
|
|
'driver': 'file',
|
||
|
|
'filename': self.path,
|
||
|
|
'size': 0,
|
||
|
|
}))
|
||
|
|
vm.run_job('bdc-file-job')
|
||
|
|
log(vm.command('blockdev-add', driver='file',
|
||
|
|
node_name=file_node_name, filename=self.path))
|
||
|
|
|
||
|
|
log(vm.command('blockdev-create', job_id='bdc-fmt-job',
|
||
|
|
options={
|
||
|
|
'driver': fmt,
|
||
|
|
'file': file_node_name,
|
||
|
|
'size': size,
|
||
|
|
}))
|
||
|
|
vm.run_job('bdc-fmt-job')
|
||
|
|
log(vm.command('blockdev-add', driver=fmt,
|
||
|
|
node_name=name,
|
||
|
|
file=file_node_name))
|
||
|
|
self.fmt = fmt
|
||
|
|
self.size = size
|
||
|
|
self.node = name
|
||
|
|
|
||
|
|
def query_bitmaps(vm):
|
||
|
|
res = vm.qmp("query-block")
|
||
|
|
return {"bitmaps": {device['device'] or device['qdev']:
|
||
|
|
device.get('dirty-bitmaps', []) for
|
||
|
|
device in res['return']}}
|
||
|
|
|
||
|
|
def get_bitmap(bitmaps, drivename, name, recording=None):
|
||
|
|
"""
|
||
|
|
get a specific bitmap from the object returned by query_bitmaps.
|
||
|
|
:param recording: If specified, filter results by the specified value.
|
||
|
|
"""
|
||
|
|
for bitmap in bitmaps['bitmaps'][drivename]:
|
||
|
|
if bitmap.get('name', '') == name:
|
||
|
|
if recording is None:
|
||
|
|
return bitmap
|
||
|
|
elif bitmap.get('recording') == recording:
|
||
|
|
return bitmap
|
||
|
|
return None
|
||
|
|
|
||
|
|
def reference_backup(drive, n, filepath):
|
||
|
|
log("--- Reference Backup #{:d} ---\n".format(n))
|
||
|
|
target_id = "ref_target_{:d}".format(n)
|
||
|
|
job_id = "ref_backup_{:d}".format(n)
|
||
|
|
target_drive = Drive(filepath, vm=drive.vm)
|
||
|
|
|
||
|
|
target_drive.create_target(target_id, drive.fmt, drive.size)
|
||
|
|
drive.vm.qmp_log("blockdev-backup",
|
||
|
|
job_id=job_id, device=drive.name,
|
||
|
|
target=target_id, sync="full")
|
||
|
|
drive.vm.run_job(job_id, auto_dismiss=True)
|
||
|
|
log('')
|
||
|
|
|
||
|
|
def bitmap_backup(drive, n, filepath, bitmap, bitmap_mode):
|
||
|
|
log("--- Bitmap Backup #{:d} ---\n".format(n))
|
||
|
|
target_id = "bitmap_target_{:d}".format(n)
|
||
|
|
job_id = "bitmap_backup_{:d}".format(n)
|
||
|
|
target_drive = Drive(filepath, vm=drive.vm)
|
||
|
|
|
||
|
|
target_drive.create_target(target_id, drive.fmt, drive.size)
|
||
|
|
drive.vm.qmp_log("blockdev-backup", job_id=job_id, device=drive.name,
|
||
|
|
target=target_id, sync="bitmap",
|
||
|
|
bitmap_mode=bitmap_mode,
|
||
|
|
bitmap=bitmap,
|
||
|
|
auto_finalize=False)
|
||
|
|
return job_id
|
||
|
|
|
||
|
|
def perform_writes(drive, n):
|
||
|
|
log("--- Write #{:d} ---\n".format(n))
|
||
|
|
for pattern in GROUPS[n].patterns:
|
||
|
|
cmd = "write -P{:s} 0x{:07x} 0x{:x}".format(
|
||
|
|
pattern.byte,
|
||
|
|
pattern.offset,
|
||
|
|
pattern.size)
|
||
|
|
log(cmd)
|
||
|
|
log(drive.vm.hmp_qemu_io(drive.name, cmd))
|
||
|
|
bitmaps = query_bitmaps(drive.vm)
|
||
|
|
log(bitmaps, indent=2)
|
||
|
|
log('')
|
||
|
|
return bitmaps
|
||
|
|
|
||
|
|
def calculate_bits(groups=None):
|
||
|
|
"""Calculate how many bits we expect to see dirtied."""
|
||
|
|
if groups:
|
||
|
|
bits = set.union(*(GROUPS[group].bits(GRANULARITY) for group in groups))
|
||
|
|
return len(bits)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
def bitmap_comparison(bitmap, groups=None, want=0):
|
||
|
|
"""
|
||
|
|
Print a nice human-readable message checking that this bitmap has as
|
||
|
|
many bits set as we expect it to.
|
||
|
|
"""
|
||
|
|
log("= Checking Bitmap {:s} =".format(bitmap.get('name', '(anonymous)')))
|
||
|
|
|
||
|
|
if groups:
|
||
|
|
want = calculate_bits(groups)
|
||
|
|
have = bitmap['count'] // bitmap['granularity']
|
||
|
|
|
||
|
|
log("expecting {:d} dirty sectors; have {:d}. {:s}".format(
|
||
|
|
want, have, "OK!" if want == have else "ERROR!"))
|
||
|
|
log('')
|
||
|
|
|
||
|
|
def compare_images(image, reference, baseimg=None, expected_match=True):
|
||
|
|
"""
|
||
|
|
Print a nice human-readable message comparing these images.
|
||
|
|
"""
|
||
|
|
expected_ret = 0 if expected_match else 1
|
||
|
|
if baseimg:
|
||
|
|
assert qemu_img("rebase", "-u", "-b", baseimg, image) == 0
|
||
|
|
ret = qemu_img("compare", image, reference)
|
||
|
|
log('qemu_img compare "{:s}" "{:s}" ==> {:s}, {:s}'.format(
|
||
|
|
image, reference,
|
||
|
|
"Identical" if ret == 0 else "Mismatch",
|
||
|
|
"OK!" if ret == expected_ret else "ERROR!"),
|
||
|
|
filters=[iotests.filter_testfiles])
|
||
|
|
|
||
|
|
def test_bitmap_sync(bsync_mode, failure=None):
|
||
|
|
"""
|
||
|
|
Test bitmap backup routines.
|
||
|
|
|
||
|
|
:param bsync_mode: Is the Bitmap Sync mode, and can be any of:
|
||
|
|
- on-success: This is the "incremental" style mode. Bitmaps are
|
||
|
|
synchronized to what was copied out only on success.
|
||
|
|
(Partial images must be discarded.)
|
||
|
|
- never: This is the "differential" style mode.
|
||
|
|
Bitmaps are never synchronized.
|
||
|
|
- always: This is a "best effort" style mode.
|
||
|
|
Bitmaps are always synchronized, regardless of failure.
|
||
|
|
(Partial images must be kept.)
|
||
|
|
|
||
|
|
:param failure: Is the (optional) failure mode, and can be any of:
|
||
|
|
- None: No failure. Test the normative path. Default.
|
||
|
|
- simulated: Cancel the job right before it completes.
|
||
|
|
This also tests writes "during" the job.
|
||
|
|
- intermediate: This tests a job that fails mid-process and produces
|
||
|
|
an incomplete backup. Testing limitations prevent
|
||
|
|
testing competing writes.
|
||
|
|
"""
|
||
|
|
with iotests.FilePaths(['img', 'bsync1', 'bsync2',
|
||
|
|
'fbackup0', 'fbackup1', 'fbackup2']) as \
|
||
|
|
(img_path, bsync1, bsync2,
|
||
|
|
fbackup0, fbackup1, fbackup2), \
|
||
|
|
iotests.VM() as vm:
|
||
|
|
|
||
|
|
mode = "Bitmap Sync Mode {:s}".format(bsync_mode)
|
||
|
|
preposition = "with" if failure else "without"
|
||
|
|
cond = "{:s} {:s}".format(preposition,
|
||
|
|
"{:s} failure".format(failure) if failure
|
||
|
|
else "failure")
|
||
|
|
log("\n=== {:s} {:s} ===\n".format(mode, cond))
|
||
|
|
|
||
|
|
log('--- Preparing image & VM ---\n')
|
||
|
|
drive0 = Drive(img_path, vm=vm)
|
||
|
|
drive0.img_create(iotests.imgfmt, SIZE)
|
||
|
|
vm.add_device("{},id=scsi0".format(iotests.get_virtio_scsi_device()))
|
||
|
|
vm.launch()
|
||
|
|
|
||
|
|
file_config = {
|
||
|
|
'driver': 'file',
|
||
|
|
'filename': drive0.path
|
||
|
|
}
|
||
|
|
|
||
|
|
if failure == 'intermediate':
|
||
|
|
file_config = {
|
||
|
|
'driver': 'blkdebug',
|
||
|
|
'image': file_config,
|
||
|
|
'set-state': [{
|
||
|
|
'event': 'flush_to_disk',
|
||
|
|
'state': 1,
|
||
|
|
'new_state': 2
|
||
|
|
}, {
|
||
|
|
'event': 'read_aio',
|
||
|
|
'state': 2,
|
||
|
|
'new_state': 3
|
||
|
|
}],
|
||
|
|
'inject-error': [{
|
||
|
|
'event': 'read_aio',
|
||
|
|
'errno': 5,
|
||
|
|
'state': 3,
|
||
|
|
'immediately': False,
|
||
|
|
'once': True
|
||
|
|
}]
|
||
|
|
}
|
||
|
|
|
||
|
|
vm.qmp_log('blockdev-add',
|
||
|
|
filters=[iotests.filter_qmp_testfiles],
|
||
|
|
node_name="drive0",
|
||
|
|
driver=drive0.fmt,
|
||
|
|
file=file_config)
|
||
|
|
drive0.node = 'drive0'
|
||
|
|
drive0.device = 'device0'
|
||
|
|
# Use share-rw to allow writes directly to the node;
|
||
|
|
# The anonymous block-backend for this configuration prevents us
|
||
|
|
# from using HMP's qemu-io commands to address the device.
|
||
|
|
vm.qmp_log("device_add", id=drive0.device,
|
||
|
|
drive=drive0.name, driver="scsi-hd",
|
||
|
|
share_rw=True)
|
||
|
|
log('')
|
||
|
|
|
||
|
|
# 0 - Writes and Reference Backup
|
||
|
|
perform_writes(drive0, 0)
|
||
|
|
reference_backup(drive0, 0, fbackup0)
|
||
|
|
log('--- Add Bitmap ---\n')
|
||
|
|
vm.qmp_log("block-dirty-bitmap-add", node=drive0.name,
|
||
|
|
name="bitmap0", granularity=GRANULARITY)
|
||
|
|
log('')
|
||
|
|
|
||
|
|
# 1 - Writes and Reference Backup
|
||
|
|
bitmaps = perform_writes(drive0, 1)
|
||
|
|
dirty_groups = {1}
|
||
|
|
bitmap = get_bitmap(bitmaps, drive0.device, 'bitmap0')
|
||
|
|
bitmap_comparison(bitmap, groups=dirty_groups)
|
||
|
|
reference_backup(drive0, 1, fbackup1)
|
||
|
|
|
||
|
|
# 1 - Bitmap Backup (Optional induced failure)
|
||
|
|
if failure == 'intermediate':
|
||
|
|
# Activate blkdebug induced failure for second-to-next read
|
||
|
|
log(vm.hmp_qemu_io(drive0.name, 'flush'))
|
||
|
|
log('')
|
||
|
|
job = bitmap_backup(drive0, 1, bsync1, "bitmap0", bsync_mode)
|
||
|
|
|
||
|
|
def _callback():
|
||
|
|
"""Issue writes while the job is open to test bitmap divergence."""
|
||
|
|
# Note: when `failure` is 'intermediate', this isn't called.
|
||
|
|
log('')
|
||
|
|
bitmaps = perform_writes(drive0, 2)
|
||
|
|
# Named bitmap (static, should be unchanged)
|
||
|
|
bitmap_comparison(get_bitmap(bitmaps, drive0.device, 'bitmap0'),
|
||
|
|
groups=dirty_groups)
|
||
|
|
# Anonymous bitmap (dynamic, shows new writes)
|
||
|
|
bitmap_comparison(get_bitmap(bitmaps, drive0.device, '',
|
||
|
|
recording=True), groups={2})
|
||
|
|
dirty_groups.add(2)
|
||
|
|
|
||
|
|
vm.run_job(job, auto_dismiss=True, auto_finalize=False,
|
||
|
|
pre_finalize=_callback,
|
||
|
|
cancel=(failure == 'simulated'))
|
||
|
|
bitmaps = query_bitmaps(vm)
|
||
|
|
bitmap = get_bitmap(bitmaps, drive0.device, 'bitmap0')
|
||
|
|
log(bitmaps, indent=2)
|
||
|
|
log('')
|
||
|
|
|
||
|
|
if ((bsync_mode == 'on-success' and not failure) or
|
||
|
|
(bsync_mode == 'always' and failure != 'intermediate')):
|
||
|
|
dirty_groups.remove(1)
|
||
|
|
|
||
|
|
if bsync_mode == 'always' and failure == 'intermediate':
|
||
|
|
# We manage to copy one sector (one bit) before the error.
|
||
|
|
bitmap_comparison(bitmap,
|
||
|
|
want=calculate_bits(groups=dirty_groups) - 1)
|
||
|
|
else:
|
||
|
|
bitmap_comparison(bitmap, groups=dirty_groups)
|
||
|
|
|
||
|
|
# 2 - Writes and Reference Backup
|
||
|
|
bitmaps = perform_writes(drive0, 3)
|
||
|
|
dirty_groups.add(3)
|
||
|
|
bitmap = get_bitmap(bitmaps, drive0.device, 'bitmap0')
|
||
|
|
if bsync_mode == 'always' and failure == 'intermediate':
|
||
|
|
# We're one bit short, still.
|
||
|
|
bitmap_comparison(bitmap,
|
||
|
|
want=calculate_bits(groups=dirty_groups) - 1)
|
||
|
|
else:
|
||
|
|
bitmap_comparison(bitmap, groups=dirty_groups)
|
||
|
|
reference_backup(drive0, 2, fbackup2)
|
||
|
|
|
||
|
|
# 2 - Bitmap Backup (In failure modes, this is a recovery.)
|
||
|
|
job = bitmap_backup(drive0, 2, bsync2, "bitmap0", bsync_mode)
|
||
|
|
vm.run_job(job, auto_dismiss=True, auto_finalize=False)
|
||
|
|
bitmaps = query_bitmaps(vm)
|
||
|
|
bitmap = get_bitmap(bitmaps, drive0.device, 'bitmap0')
|
||
|
|
log(bitmaps, indent=2)
|
||
|
|
log('')
|
||
|
|
bitmap_comparison(bitmap, groups={}
|
||
|
|
if bsync_mode != 'never'
|
||
|
|
else dirty_groups)
|
||
|
|
|
||
|
|
log('--- Cleanup ---\n')
|
||
|
|
vm.qmp_log("block-dirty-bitmap-remove",
|
||
|
|
node=drive0.name, name="bitmap0")
|
||
|
|
log(query_bitmaps(vm), indent=2)
|
||
|
|
vm.shutdown()
|
||
|
|
log('')
|
||
|
|
|
||
|
|
log('--- Verification ---\n')
|
||
|
|
# 'simulated' failures will actually all pass here because we canceled
|
||
|
|
# while "pending". This is actually undefined behavior,
|
||
|
|
# don't rely on this to be true!
|
||
|
|
compare_images(bsync1, fbackup1, baseimg=fbackup0,
|
||
|
|
expected_match=failure != 'intermediate')
|
||
|
|
if not failure or bsync_mode == 'always':
|
||
|
|
# Always keep the last backup on success or when using 'always'
|
||
|
|
base = bsync1
|
||
|
|
else:
|
||
|
|
base = fbackup0
|
||
|
|
compare_images(bsync2, fbackup2, baseimg=base)
|
||
|
|
compare_images(img_path, fbackup2)
|
||
|
|
log('')
|
||
|
|
|
||
|
|
def main():
|
||
|
|
for bsync_mode in ("never", "on-success", "always"):
|
||
|
|
for failure in ("simulated", "intermediate", None):
|
||
|
|
test_bitmap_sync(bsync_mode, failure)
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
iotests.script_main(main, supported_fmts=['qcow2'])
|