qemu/tests/qemu-iotests/199

#!/usr/bin/env python3
# group: rw migration
#
# Tests for dirty bitmaps postcopy migration.
#
# Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os
import iotests
from iotests import qemu_img

debug = False

disk_a = os.path.join(iotests.test_dir, 'disk_a')
disk_b = os.path.join(iotests.test_dir, 'disk_b')
size = '256G'
fifo = os.path.join(iotests.test_dir, 'mig_fifo')

granularity = 512
nb_bitmaps = 15

GiB = 1024 * 1024 * 1024

discards1 = (
    (0, GiB),
    (2 * GiB + 512 * 5, 512),
    (3 * GiB + 512 * 5, 512),
    (100 * GiB, GiB)
)

discards2 = (
    (3 * GiB + 512 * 8, 512),
    (4 * GiB + 512 * 8, 512),
    (50 * GiB, GiB),
    (100 * GiB + GiB // 2, GiB)
)


def apply_discards(vm, discards):
    for d in discards:
        vm.hmp_qemu_io('drive0', 'discard {} {}'.format(*d))


def event_seconds(event):
    return event['timestamp']['seconds'] + \
        event['timestamp']['microseconds'] / 1000000.0


def event_dist(e1, e2):
    return event_seconds(e2) - event_seconds(e1)


def check_bitmaps(vm, count):
    result = vm.qmp('query-block')

    if count == 0:
        assert 'dirty-bitmaps' not in result['return'][0]
    else:
        assert len(result['return'][0]['dirty-bitmaps']) == count


class TestDirtyBitmapPostcopyMigration(iotests.QMPTestCase):
    def tearDown(self):
        if debug:
            self.vm_a_events += self.vm_a.get_qmp_events()
            self.vm_b_events += self.vm_b.get_qmp_events()
            for e in self.vm_a_events:
                e['vm'] = 'SRC'
            for e in self.vm_b_events:
                e['vm'] = 'DST'
            events = (self.vm_a_events + self.vm_b_events)
            events = [(e['timestamp']['seconds'],
                       e['timestamp']['microseconds'],
                       e['vm'],
                       e['event'],
                       e.get('data', '')) for e in events]
            for e in sorted(events):
                print('{}.{:06} {} {} {}'.format(*e))

        self.vm_a.shutdown()
        self.vm_b.shutdown()
        os.remove(disk_a)
        os.remove(disk_b)
        os.remove(fifo)

    def setUp(self):
        os.mkfifo(fifo)
        qemu_img('create', '-f', iotests.imgfmt, disk_a, size)
        qemu_img('create', '-f', iotests.imgfmt, disk_b, size)
        self.vm_a = iotests.VM(path_suffix='a').add_drive(disk_a,
                                                          'discard=unmap')
        self.vm_b = iotests.VM(path_suffix='b').add_drive(disk_b,
                                                          'discard=unmap')
        self.vm_b.add_incoming("exec: cat '" + fifo + "'")
        self.vm_a.launch()
        self.vm_b.launch()

        # collect received events for debug
        self.vm_a_events = []
        self.vm_b_events = []

    def start_postcopy(self):
        """ Run migration until RESUME event on target. Return this event. """
        for i in range(nb_bitmaps):
            result = self.vm_a.qmp('block-dirty-bitmap-add', node='drive0',
                                   name='bitmap{}'.format(i),
                                   granularity=granularity,
                                   persistent=True)
            self.assert_qmp(result, 'return', {})

        result = self.vm_a.qmp('x-debug-block-dirty-bitmap-sha256',
                               node='drive0', name='bitmap0')
        empty_sha256 = result['return']['sha256']

        apply_discards(self.vm_a, discards1)

        result = self.vm_a.qmp('x-debug-block-dirty-bitmap-sha256',
                               node='drive0', name='bitmap0')
        self.discards1_sha256 = result['return']['sha256']

        # Check, that updating the bitmap by discards works
        assert self.discards1_sha256 != empty_sha256

        # We want to calculate resulting sha256. Do it in bitmap0, so, disable
        # other bitmaps
        for i in range(1, nb_bitmaps):
            result = self.vm_a.qmp('block-dirty-bitmap-disable', node='drive0',
                                   name='bitmap{}'.format(i))
            self.assert_qmp(result, 'return', {})

        apply_discards(self.vm_a, discards2)

        result = self.vm_a.qmp('x-debug-block-dirty-bitmap-sha256',
                               node='drive0', name='bitmap0')
        self.all_discards_sha256 = result['return']['sha256']

        # Now, enable some bitmaps, to be updated during migration
        for i in range(2, nb_bitmaps, 2):
            result = self.vm_a.qmp('block-dirty-bitmap-enable', node='drive0',
                                   name='bitmap{}'.format(i))
            self.assert_qmp(result, 'return', {})

        caps = [{'capability': 'dirty-bitmaps', 'state': True},
                {'capability': 'events', 'state': True}]

        result = self.vm_a.qmp('migrate-set-capabilities', capabilities=caps)
        self.assert_qmp(result, 'return', {})

        result = self.vm_b.qmp('migrate-set-capabilities', capabilities=caps)
        self.assert_qmp(result, 'return', {})

        result = self.vm_a.qmp('migrate', uri='exec:cat>' + fifo)
        self.assert_qmp(result, 'return', {})

        result = self.vm_a.qmp('migrate-start-postcopy')
        self.assert_qmp(result, 'return', {})

        event_resume = self.vm_b.event_wait('RESUME')
        self.vm_b_events.append(event_resume)
        return event_resume

    def test_postcopy_success(self):
        event_resume = self.start_postcopy()

        # enabled bitmaps should be updated
        apply_discards(self.vm_b, discards2)

        match = {'data': {'status': 'completed'}}
        event_complete = self.vm_b.event_wait('MIGRATION', match=match)
        self.vm_b_events.append(event_complete)

        # take queued event, should already been happened
        event_stop = self.vm_a.event_wait('STOP')
        self.vm_a_events.append(event_stop)

        downtime = event_dist(event_stop, event_resume)
        postcopy_time = event_dist(event_resume, event_complete)

        assert downtime * 10 < postcopy_time
        if debug:
            print('downtime:', downtime)
            print('postcopy_time:', postcopy_time)

        # check that there are no bitmaps stored on source
        self.vm_a_events += self.vm_a.get_qmp_events()
        self.vm_a.shutdown()
        self.vm_a.launch()
        check_bitmaps(self.vm_a, 0)

        # check that bitmaps are migrated and persistence works
        check_bitmaps(self.vm_b, nb_bitmaps)
        self.vm_b.shutdown()
        # recreate vm_b, so there is no incoming option, which prevents
        # loading bitmaps from disk
        self.vm_b = iotests.VM(path_suffix='b').add_drive(disk_b)
        self.vm_b.launch()
        check_bitmaps(self.vm_b, nb_bitmaps)

        # Check content of migrated bitmaps. Still, don't waste time checking
        # every bitmap
        for i in range(0, nb_bitmaps, 5):
            result = self.vm_b.qmp('x-debug-block-dirty-bitmap-sha256',
                                   node='drive0', name='bitmap{}'.format(i))
            sha = self.discards1_sha256 if i % 2 else self.all_discards_sha256
            self.assert_qmp(result, 'return/sha256', sha)

    def test_early_shutdown_destination(self):
        self.start_postcopy()

        self.vm_b_events += self.vm_b.get_qmp_events()
        self.vm_b.shutdown()
        # recreate vm_b, so there is no incoming option, which prevents
        # loading bitmaps from disk
        self.vm_b = iotests.VM(path_suffix='b').add_drive(disk_b)
        self.vm_b.launch()
        check_bitmaps(self.vm_b, 0)

        # Bitmaps will be lost if we just shutdown the vm, as they are marked
        # to skip storing to disk when prepared for migration. And that's
        # correct, as actual data may be modified in target vm, so we play
        # safe.
        # Still, this mark would be taken away if we do 'cont', and bitmaps
        # become persistent again. (see iotest 169 for such behavior case)
        result = self.vm_a.qmp('query-status')
        assert not result['return']['running']
        self.vm_a_events += self.vm_a.get_qmp_events()
        self.vm_a.shutdown()
        self.vm_a.launch()
        check_bitmaps(self.vm_a, 0)

    def test_early_kill_source(self):
        self.start_postcopy()

        self.vm_a_events = self.vm_a.get_qmp_events()
        self.vm_a.kill()

        self.vm_a.launch()

        match = {'data': {'status': 'completed'}}
        e_complete = self.vm_b.event_wait('MIGRATION', match=match)
        self.vm_b_events.append(e_complete)

        check_bitmaps(self.vm_a, 0)
        check_bitmaps(self.vm_b, 0)


if __name__ == '__main__':
    iotests.main(supported_fmts=['qcow2'])