2015-05-07 20:33:31 +03:00
|
|
|
/*
|
|
|
|
* QEMU System Emulator
|
|
|
|
*
|
|
|
|
* Copyright (c) 2003-2008 Fabrice Bellard
|
2015-05-08 14:20:21 +03:00
|
|
|
* Copyright (c) 2011-2015 Red Hat Inc
|
|
|
|
*
|
|
|
|
* Authors:
|
|
|
|
* Juan Quintela <quintela@redhat.com>
|
2015-05-07 20:33:31 +03:00
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
|
|
* in the Software without restriction, including without limitation the rights
|
|
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
|
|
* furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice shall be included in
|
|
|
|
* all copies or substantial portions of the Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
|
|
* THE SOFTWARE.
|
|
|
|
*/
|
2018-02-01 14:18:31 +03:00
|
|
|
|
2016-01-26 21:16:54 +03:00
|
|
|
#include "qemu/osdep.h"
|
2016-03-20 20:16:19 +03:00
|
|
|
#include "qemu/cutils.h"
|
2015-05-07 20:33:31 +03:00
|
|
|
#include "qemu/bitops.h"
|
|
|
|
#include "qemu/bitmap.h"
|
2022-02-08 23:08:52 +03:00
|
|
|
#include "qemu/madvise.h"
|
2015-05-08 14:54:36 +03:00
|
|
|
#include "qemu/main-loop.h"
|
2022-06-20 14:01:46 +03:00
|
|
|
#include "io/channel-null.h"
|
2017-04-05 22:47:50 +03:00
|
|
|
#include "xbzrle.h"
|
2017-04-17 21:26:27 +03:00
|
|
|
#include "ram.h"
|
2017-04-24 21:07:27 +03:00
|
|
|
#include "migration.h"
|
2017-04-24 14:42:55 +03:00
|
|
|
#include "migration/register.h"
|
2017-04-17 21:26:27 +03:00
|
|
|
#include "migration/misc.h"
|
2017-04-20 19:52:18 +03:00
|
|
|
#include "qemu-file.h"
|
2017-04-20 14:12:24 +03:00
|
|
|
#include "postcopy-ram.h"
|
2018-05-03 22:50:51 +03:00
|
|
|
#include "page_cache.h"
|
2015-05-07 20:33:31 +03:00
|
|
|
#include "qemu/error-report.h"
|
2018-02-01 14:18:31 +03:00
|
|
|
#include "qapi/error.h"
|
2019-05-15 14:37:46 +03:00
|
|
|
#include "qapi/qapi-types-migration.h"
|
2018-02-11 12:36:01 +03:00
|
|
|
#include "qapi/qapi-events-migration.h"
|
2017-10-05 23:00:31 +03:00
|
|
|
#include "qapi/qmp/qerror.h"
|
2015-05-07 20:33:31 +03:00
|
|
|
#include "trace.h"
|
|
|
|
#include "exec/ram_addr.h"
|
2017-10-05 14:13:20 +03:00
|
|
|
#include "exec/target_page.h"
|
2015-05-07 20:33:31 +03:00
|
|
|
#include "qemu/rcu_queue.h"
|
2016-10-27 09:42:59 +03:00
|
|
|
#include "migration/colo.h"
|
2018-05-03 22:50:51 +03:00
|
|
|
#include "block.h"
|
2020-06-29 12:35:03 +03:00
|
|
|
#include "sysemu/cpu-throttle.h"
|
migration: synchronize dirty bitmap for resume
This patch implements the first part of core RAM resume logic for
postcopy. ram_resume_prepare() is provided for the work.
When the migration is interrupted by network failure, the dirty bitmap
on the source side will be meaningless, because even the dirty bit is
cleared, it is still possible that the sent page was lost along the way
to destination. Here instead of continue the migration with the old
dirty bitmap on source, we ask the destination side to send back its
received bitmap, then invert it to be our initial dirty bitmap.
The source side send thread will issue the MIG_CMD_RECV_BITMAP requests,
once per ramblock, to ask for the received bitmap. On destination side,
MIG_RP_MSG_RECV_BITMAP will be issued, along with the requested bitmap.
Data will be received on the return-path thread of source, and the main
migration thread will be notified when all the ramblock bitmaps are
synchronized.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20180502104740.12123-17-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2018-05-02 13:47:32 +03:00
|
|
|
#include "savevm.h"
|
2016-01-15 13:40:13 +03:00
|
|
|
#include "qemu/iov.h"
|
2020-01-22 18:16:07 +03:00
|
|
|
#include "multifd.h"
|
2021-01-29 13:14:05 +03:00
|
|
|
#include "sysemu/runstate.h"
|
2023-03-01 23:18:45 +03:00
|
|
|
#include "options.h"
|
2021-01-29 13:14:05 +03:00
|
|
|
|
2021-07-04 19:14:44 +03:00
|
|
|
#include "hw/boards.h" /* for machine_dump_guest_core() */
|
|
|
|
|
2021-01-29 13:14:05 +03:00
|
|
|
#if defined(__linux__)
|
|
|
|
#include "qemu/userfaultfd.h"
|
|
|
|
#endif /* defined(__linux__) */
|
2015-05-07 20:33:31 +03:00
|
|
|
|
|
|
|
/***********************************************************/
|
|
|
|
/* ram save/restore */
|
|
|
|
|
2022-07-28 11:14:42 +03:00
|
|
|
/*
|
|
|
|
* RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
|
|
|
|
* worked for pages that were filled with the same char. We switched
|
2017-04-28 10:39:55 +03:00
|
|
|
* it to only search for the zero value. And to avoid confusion with
|
2022-07-28 11:14:42 +03:00
|
|
|
* RAM_SAVE_FLAG_COMPRESS_PAGE just rename it.
|
2017-04-28 10:39:55 +03:00
|
|
|
*/
|
2022-07-28 11:14:42 +03:00
|
|
|
/*
|
|
|
|
* RAM_SAVE_FLAG_FULL was obsoleted in 2009, it can be reused now
|
|
|
|
*/
|
|
|
|
#define RAM_SAVE_FLAG_FULL 0x01
|
2017-04-28 10:39:55 +03:00
|
|
|
#define RAM_SAVE_FLAG_ZERO 0x02
|
2015-05-07 20:33:31 +03:00
|
|
|
#define RAM_SAVE_FLAG_MEM_SIZE 0x04
|
|
|
|
#define RAM_SAVE_FLAG_PAGE 0x08
|
|
|
|
#define RAM_SAVE_FLAG_EOS 0x10
|
|
|
|
#define RAM_SAVE_FLAG_CONTINUE 0x20
|
|
|
|
#define RAM_SAVE_FLAG_XBZRLE 0x40
|
2022-07-28 11:14:42 +03:00
|
|
|
/* 0x80 is reserved in qemu-file.h for RAM_SAVE_FLAG_HOOK */
|
2015-05-07 20:33:31 +03:00
|
|
|
#define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
|
2022-07-28 11:14:42 +03:00
|
|
|
/* We can't use any flag that is bigger than 0x200 */
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2022-11-16 18:29:22 +03:00
|
|
|
int (*xbzrle_encode_buffer_func)(uint8_t *, uint8_t *, int,
|
|
|
|
uint8_t *, int) = xbzrle_encode_buffer;
|
|
|
|
#if defined(CONFIG_AVX512BW_OPT)
|
|
|
|
#include "qemu/cpuid.h"
|
|
|
|
static void __attribute__((constructor)) init_cpu_flag(void)
|
|
|
|
{
|
|
|
|
unsigned max = __get_cpuid_max(0, NULL);
|
|
|
|
int a, b, c, d;
|
|
|
|
if (max >= 1) {
|
|
|
|
__cpuid(1, a, b, c, d);
|
|
|
|
/* We must check that AVX is not just available, but usable. */
|
|
|
|
if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) {
|
|
|
|
int bv;
|
|
|
|
__asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0));
|
|
|
|
__cpuid_count(7, 0, a, b, c, d);
|
|
|
|
/* 0xe6:
|
|
|
|
* XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15
|
|
|
|
* and ZMM16-ZMM31 state are enabled by OS)
|
|
|
|
* XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS)
|
|
|
|
*/
|
|
|
|
if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512BW)) {
|
|
|
|
xbzrle_encode_buffer_func = xbzrle_encode_buffer_avx512;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2017-06-06 20:49:03 +03:00
|
|
|
XBZRLECacheStats xbzrle_counters;
|
|
|
|
|
2022-10-12 00:55:55 +03:00
|
|
|
/* used by the search for pages to send */
|
|
|
|
struct PageSearchStatus {
|
|
|
|
/* The migration channel used for a specific host page */
|
|
|
|
QEMUFile *pss_channel;
|
2022-10-12 00:55:56 +03:00
|
|
|
/* Last block from where we have sent data */
|
|
|
|
RAMBlock *last_sent_block;
|
2022-10-12 00:55:55 +03:00
|
|
|
/* Current block being searched */
|
|
|
|
RAMBlock *block;
|
|
|
|
/* Current page to search from */
|
|
|
|
unsigned long page;
|
|
|
|
/* Set once we wrap around */
|
|
|
|
bool complete_round;
|
|
|
|
/* Whether we're sending a host page */
|
|
|
|
bool host_page_sending;
|
|
|
|
/* The start/end of current host page. Invalid if host_page_sending==false */
|
|
|
|
unsigned long host_page_start;
|
|
|
|
unsigned long host_page_end;
|
|
|
|
};
|
|
|
|
typedef struct PageSearchStatus PageSearchStatus;
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
/* struct contains XBZRLE cache and a static page
|
|
|
|
used by the compression */
|
|
|
|
static struct {
|
|
|
|
/* buffer used for XBZRLE encoding */
|
|
|
|
uint8_t *encoded_buf;
|
|
|
|
/* buffer for storing page content */
|
|
|
|
uint8_t *current_buf;
|
|
|
|
/* Cache for XBZRLE, Protected by lock. */
|
|
|
|
PageCache *cache;
|
|
|
|
QemuMutex lock;
|
2017-05-09 17:22:01 +03:00
|
|
|
/* it will store a page full of zeros */
|
|
|
|
uint8_t *zero_target_page;
|
2017-06-28 12:52:27 +03:00
|
|
|
/* buffer used for XBZRLE decoding */
|
|
|
|
uint8_t *decoded_buf;
|
2015-05-07 20:33:31 +03:00
|
|
|
} XBZRLE;
|
|
|
|
|
|
|
|
static void XBZRLE_cache_lock(void)
|
|
|
|
{
|
2023-03-02 00:20:13 +03:00
|
|
|
if (migrate_xbzrle()) {
|
2015-05-07 20:33:31 +03:00
|
|
|
qemu_mutex_lock(&XBZRLE.lock);
|
2020-10-20 06:10:45 +03:00
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static void XBZRLE_cache_unlock(void)
|
|
|
|
{
|
2023-03-02 00:20:13 +03:00
|
|
|
if (migrate_xbzrle()) {
|
2015-05-07 20:33:31 +03:00
|
|
|
qemu_mutex_unlock(&XBZRLE.lock);
|
2020-10-20 06:10:45 +03:00
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* xbzrle_cache_resize: resize the xbzrle cache
|
|
|
|
*
|
2021-02-19 21:40:12 +03:00
|
|
|
* This function is called from migrate_params_apply in main
|
2017-03-23 17:06:39 +03:00
|
|
|
* thread, possibly while a migration is in progress. A running
|
|
|
|
* migration may be using the cache and might finish during this call,
|
|
|
|
* hence changes to the cache are protected by XBZRLE.lock().
|
|
|
|
*
|
2017-10-07 00:03:55 +03:00
|
|
|
* Returns 0 for success or -1 for error
|
2017-03-23 17:06:39 +03:00
|
|
|
*
|
|
|
|
* @new_size: new cache size
|
2017-10-05 23:00:31 +03:00
|
|
|
* @errp: set *errp if the check failed, with reason
|
2015-05-07 20:33:31 +03:00
|
|
|
*/
|
2021-02-02 17:17:32 +03:00
|
|
|
int xbzrle_cache_resize(uint64_t new_size, Error **errp)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
|
|
|
PageCache *new_cache;
|
2017-10-07 00:03:55 +03:00
|
|
|
int64_t ret = 0;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2017-10-05 23:00:31 +03:00
|
|
|
/* Check for truncation */
|
|
|
|
if (new_size != (size_t)new_size) {
|
|
|
|
error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
|
|
|
|
"exceeding address space");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2017-10-07 00:00:12 +03:00
|
|
|
if (new_size == migrate_xbzrle_cache_size()) {
|
|
|
|
/* nothing to do */
|
2017-10-07 00:03:55 +03:00
|
|
|
return 0;
|
2017-10-07 00:00:12 +03:00
|
|
|
}
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
XBZRLE_cache_lock();
|
|
|
|
|
|
|
|
if (XBZRLE.cache != NULL) {
|
2017-10-06 23:30:45 +03:00
|
|
|
new_cache = cache_init(new_size, TARGET_PAGE_SIZE, errp);
|
2015-05-07 20:33:31 +03:00
|
|
|
if (!new_cache) {
|
|
|
|
ret = -1;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
cache_fini(XBZRLE.cache);
|
|
|
|
XBZRLE.cache = new_cache;
|
|
|
|
}
|
|
|
|
out:
|
|
|
|
XBZRLE_cache_unlock();
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:46 +03:00
|
|
|
static bool postcopy_preempt_active(void)
|
|
|
|
{
|
|
|
|
return migrate_postcopy_preempt() && migration_in_postcopy();
|
|
|
|
}
|
|
|
|
|
2020-09-16 09:22:00 +03:00
|
|
|
bool ramblock_is_ignored(RAMBlock *block)
|
2019-02-15 20:45:46 +03:00
|
|
|
{
|
|
|
|
return !qemu_ram_is_migratable(block) ||
|
|
|
|
(migrate_ignore_shared() && qemu_ram_is_shared(block));
|
|
|
|
}
|
|
|
|
|
2018-06-05 19:25:45 +03:00
|
|
|
#undef RAMBLOCK_FOREACH
|
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque)
|
|
|
|
{
|
|
|
|
RAMBlock *block;
|
|
|
|
int ret = 0;
|
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
RCU_READ_LOCK_GUARD();
|
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
|
|
|
ret = func(block, opaque);
|
|
|
|
if (ret) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2017-10-05 14:13:20 +03:00
|
|
|
static void ramblock_recv_map_init(void)
|
|
|
|
{
|
|
|
|
RAMBlock *rb;
|
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
|
2017-10-05 14:13:20 +03:00
|
|
|
assert(!rb->receivedmap);
|
|
|
|
rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
int ramblock_recv_bitmap_test(RAMBlock *rb, void *host_addr)
|
|
|
|
{
|
|
|
|
return test_bit(ramblock_recv_bitmap_offset(host_addr, rb),
|
|
|
|
rb->receivedmap);
|
|
|
|
}
|
|
|
|
|
2018-03-12 20:21:08 +03:00
|
|
|
bool ramblock_recv_bitmap_test_byte_offset(RAMBlock *rb, uint64_t byte_offset)
|
|
|
|
{
|
|
|
|
return test_bit(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap);
|
|
|
|
}
|
|
|
|
|
2017-10-05 14:13:20 +03:00
|
|
|
void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr)
|
|
|
|
{
|
|
|
|
set_bit_atomic(ramblock_recv_bitmap_offset(host_addr, rb), rb->receivedmap);
|
|
|
|
}
|
|
|
|
|
|
|
|
void ramblock_recv_bitmap_set_range(RAMBlock *rb, void *host_addr,
|
|
|
|
size_t nr)
|
|
|
|
{
|
|
|
|
bitmap_set_atomic(rb->receivedmap,
|
|
|
|
ramblock_recv_bitmap_offset(host_addr, rb),
|
|
|
|
nr);
|
|
|
|
}
|
|
|
|
|
2018-05-02 13:47:28 +03:00
|
|
|
#define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
|
|
|
|
*
|
|
|
|
* Returns >0 if success with sent bytes, or <0 if error.
|
|
|
|
*/
|
|
|
|
int64_t ramblock_recv_bitmap_send(QEMUFile *file,
|
|
|
|
const char *block_name)
|
|
|
|
{
|
|
|
|
RAMBlock *block = qemu_ram_block_by_name(block_name);
|
|
|
|
unsigned long *le_bitmap, nbits;
|
|
|
|
uint64_t size;
|
|
|
|
|
|
|
|
if (!block) {
|
|
|
|
error_report("%s: invalid block name: %s", __func__, block_name);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2021-04-29 14:27:06 +03:00
|
|
|
nbits = block->postcopy_length >> TARGET_PAGE_BITS;
|
2018-05-02 13:47:28 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
|
|
|
|
* machines we may need 4 more bytes for padding (see below
|
|
|
|
* comment). So extend it a bit before hand.
|
|
|
|
*/
|
|
|
|
le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Always use little endian when sending the bitmap. This is
|
|
|
|
* required that when source and destination VMs are not using the
|
2020-09-17 10:50:21 +03:00
|
|
|
* same endianness. (Note: big endian won't work.)
|
2018-05-02 13:47:28 +03:00
|
|
|
*/
|
|
|
|
bitmap_to_le(le_bitmap, block->receivedmap, nbits);
|
|
|
|
|
|
|
|
/* Size of the bitmap, in bytes */
|
2018-07-10 12:18:55 +03:00
|
|
|
size = DIV_ROUND_UP(nbits, 8);
|
2018-05-02 13:47:28 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* size is always aligned to 8 bytes for 64bit machines, but it
|
|
|
|
* may not be true for 32bit machines. We need this padding to
|
|
|
|
* make sure the migration can survive even between 32bit and
|
|
|
|
* 64bit machines.
|
|
|
|
*/
|
|
|
|
size = ROUND_UP(size, 8);
|
|
|
|
|
|
|
|
qemu_put_be64(file, size);
|
|
|
|
qemu_put_buffer(file, (const uint8_t *)le_bitmap, size);
|
|
|
|
/*
|
|
|
|
* Mark as an end, in case the middle part is screwed up due to
|
2020-09-17 10:50:21 +03:00
|
|
|
* some "mysterious" reason.
|
2018-05-02 13:47:28 +03:00
|
|
|
*/
|
|
|
|
qemu_put_be64(file, RAMBLOCK_RECV_BITMAP_ENDING);
|
|
|
|
qemu_fflush(file);
|
|
|
|
|
2018-05-25 04:50:42 +03:00
|
|
|
g_free(le_bitmap);
|
2018-05-02 13:47:28 +03:00
|
|
|
|
|
|
|
if (qemu_file_get_error(file)) {
|
|
|
|
return qemu_file_get_error(file);
|
|
|
|
}
|
|
|
|
|
|
|
|
return size + sizeof(size);
|
|
|
|
}
|
|
|
|
|
2017-03-21 00:12:40 +03:00
|
|
|
/*
|
|
|
|
* An outstanding page request, on the source, having been received
|
|
|
|
* and queued
|
|
|
|
*/
|
|
|
|
struct RAMSrcPageRequest {
|
|
|
|
RAMBlock *rb;
|
|
|
|
hwaddr offset;
|
|
|
|
hwaddr len;
|
|
|
|
|
|
|
|
QSIMPLEQ_ENTRY(RAMSrcPageRequest) next_req;
|
|
|
|
};
|
|
|
|
|
2017-03-13 21:26:29 +03:00
|
|
|
/* State of RAM for migration */
|
|
|
|
struct RAMState {
|
2022-10-12 00:55:55 +03:00
|
|
|
/*
|
|
|
|
* PageSearchStatus structures for the channels when send pages.
|
|
|
|
* Protected by the bitmap_mutex.
|
|
|
|
*/
|
|
|
|
PageSearchStatus pss[RAM_CHANNEL_MAX];
|
2021-01-29 13:14:05 +03:00
|
|
|
/* UFFD file descriptor, used in 'write-tracking' migration */
|
|
|
|
int uffdio_fd;
|
2022-05-10 20:37:36 +03:00
|
|
|
/* total ram size in bytes */
|
|
|
|
uint64_t ram_bytes_total;
|
2017-03-13 21:26:29 +03:00
|
|
|
/* Last block that we have visited searching for dirty pages */
|
|
|
|
RAMBlock *last_seen_block;
|
2017-03-21 17:23:31 +03:00
|
|
|
/* Last dirty target page we have sent */
|
|
|
|
ram_addr_t last_page;
|
2017-03-13 21:26:29 +03:00
|
|
|
/* last ram version we have seen */
|
|
|
|
uint32_t last_version;
|
2017-03-13 21:35:50 +03:00
|
|
|
/* How many times we have dirty too many pages */
|
|
|
|
int dirty_rate_high_cnt;
|
2017-03-13 21:44:57 +03:00
|
|
|
/* these variables are used for bitmap sync */
|
|
|
|
/* last time we did a full bitmap_sync */
|
|
|
|
int64_t time_last_bitmap_sync;
|
2017-03-28 15:59:01 +03:00
|
|
|
/* bytes transferred at start_time */
|
2017-03-28 15:59:54 +03:00
|
|
|
uint64_t bytes_xfer_prev;
|
2017-03-28 16:02:43 +03:00
|
|
|
/* number of dirty pages since start_time */
|
2017-03-28 16:05:53 +03:00
|
|
|
uint64_t num_dirty_pages_period;
|
2017-03-13 21:49:19 +03:00
|
|
|
/* xbzrle misses since the beginning of the period */
|
|
|
|
uint64_t xbzrle_cache_miss_prev;
|
2020-04-30 03:59:35 +03:00
|
|
|
/* Amount of xbzrle pages since the beginning of the period */
|
|
|
|
uint64_t xbzrle_pages_prev;
|
|
|
|
/* Amount of xbzrle encoded bytes since the beginning of the period */
|
|
|
|
uint64_t xbzrle_bytes_prev;
|
migrate/ram: remove "ram_bulk_stage" and "fpo_enabled"
The bulk stage is kind of weird: migration_bitmap_find_dirty() will
indicate a dirty page, however, ram_save_host_page() will never save it, as
migration_bitmap_clear_dirty() detects that it is not dirty.
We already fill the bitmap in ram_list_init_bitmaps() with ones, marking
everything dirty - it didn't used to be that way, which is why we needed
an explicit first bulk stage.
Let's simplify: make the bitmap the single source of thuth. Explicitly
handle the "xbzrle_enabled after first round" case.
Regarding XBZRLE (implicitly handled via "ram_bulk_stage = false" right
now), there is now a slight change in behavior:
- Colo: When starting, it will be disabled (was implicitly enabled)
until the first round actually finishes.
- Free page hinting: When starting, XBZRLE will be disabled (was implicitly
enabled) until the first round actually finished.
- Snapshots: When starting, XBZRLE will be disabled. We essentially only
do a single run, so I guess it will never actually get disabled.
Postcopy seems to indirectly disable it in ram_save_page(), so there
shouldn't be really any change.
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Juan Quintela <quintela@redhat.com>
Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
Cc: Andrey Gruzdev <andrey.gruzdev@virtuozzo.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Message-Id: <20210216105039.40680-1-david@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2021-02-16 13:50:39 +03:00
|
|
|
/* Start using XBZRLE (e.g., after the first round). */
|
|
|
|
bool xbzrle_enabled;
|
2021-12-15 21:01:21 +03:00
|
|
|
/* Are we on the last stage of migration */
|
|
|
|
bool last_stage;
|
2018-09-06 10:01:00 +03:00
|
|
|
/* compression statistics since the beginning of the period */
|
|
|
|
/* amount of count that no free thread to compress data */
|
|
|
|
uint64_t compress_thread_busy_prev;
|
|
|
|
/* amount bytes after compression */
|
|
|
|
uint64_t compressed_size_prev;
|
|
|
|
/* amount of compressed pages */
|
|
|
|
uint64_t compress_pages_prev;
|
|
|
|
|
2018-09-03 12:26:42 +03:00
|
|
|
/* total handled target pages at the beginning of period */
|
|
|
|
uint64_t target_page_count_prev;
|
|
|
|
/* total handled target pages since start */
|
|
|
|
uint64_t target_page_count;
|
2017-06-06 20:49:03 +03:00
|
|
|
/* number of dirty bits in the bitmap */
|
2017-08-02 12:41:19 +03:00
|
|
|
uint64_t migration_dirty_pages;
|
2022-10-12 00:55:55 +03:00
|
|
|
/*
|
|
|
|
* Protects:
|
|
|
|
* - dirty/clear bitmap
|
|
|
|
* - migration_dirty_pages
|
|
|
|
* - pss structures
|
|
|
|
*/
|
2017-03-13 23:38:09 +03:00
|
|
|
QemuMutex bitmap_mutex;
|
2017-03-14 15:48:42 +03:00
|
|
|
/* The RAMBlock used in the last src_page_requests */
|
|
|
|
RAMBlock *last_req_rb;
|
2017-03-21 00:12:40 +03:00
|
|
|
/* Queue of outstanding page requests from the destination */
|
|
|
|
QemuMutex src_page_req_mutex;
|
2018-12-06 13:58:10 +03:00
|
|
|
QSIMPLEQ_HEAD(, RAMSrcPageRequest) src_page_requests;
|
2017-03-13 21:26:29 +03:00
|
|
|
};
|
|
|
|
typedef struct RAMState RAMState;
|
|
|
|
|
2017-05-04 12:46:24 +03:00
|
|
|
static RAMState *ram_state;
|
2017-03-13 21:26:29 +03:00
|
|
|
|
2018-12-11 11:24:51 +03:00
|
|
|
static NotifierWithReturnList precopy_notifier_list;
|
|
|
|
|
2022-01-19 11:09:18 +03:00
|
|
|
/* Whether postcopy has queued requests? */
|
|
|
|
static bool postcopy_has_request(RAMState *rs)
|
|
|
|
{
|
|
|
|
return !QSIMPLEQ_EMPTY_ATOMIC(&rs->src_page_requests);
|
|
|
|
}
|
|
|
|
|
2018-12-11 11:24:51 +03:00
|
|
|
void precopy_infrastructure_init(void)
|
|
|
|
{
|
|
|
|
notifier_with_return_list_init(&precopy_notifier_list);
|
|
|
|
}
|
|
|
|
|
|
|
|
void precopy_add_notifier(NotifierWithReturn *n)
|
|
|
|
{
|
|
|
|
notifier_with_return_list_add(&precopy_notifier_list, n);
|
|
|
|
}
|
|
|
|
|
|
|
|
void precopy_remove_notifier(NotifierWithReturn *n)
|
|
|
|
{
|
|
|
|
notifier_with_return_remove(n);
|
|
|
|
}
|
|
|
|
|
|
|
|
int precopy_notify(PrecopyNotifyReason reason, Error **errp)
|
|
|
|
{
|
|
|
|
PrecopyNotifyData pnd;
|
|
|
|
pnd.reason = reason;
|
|
|
|
pnd.errp = errp;
|
|
|
|
|
|
|
|
return notifier_with_return_list_notify(&precopy_notifier_list, &pnd);
|
|
|
|
}
|
|
|
|
|
2017-03-14 14:02:16 +03:00
|
|
|
uint64_t ram_bytes_remaining(void)
|
2017-03-13 23:58:11 +03:00
|
|
|
{
|
2017-12-15 14:51:23 +03:00
|
|
|
return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
|
|
|
|
0;
|
2017-03-13 23:58:11 +03:00
|
|
|
}
|
|
|
|
|
2023-02-22 17:56:45 +03:00
|
|
|
RAMStats ram_counters;
|
2017-03-14 20:41:03 +03:00
|
|
|
|
2022-02-22 23:02:03 +03:00
|
|
|
void ram_transferred_add(uint64_t bytes)
|
2021-12-21 12:34:40 +03:00
|
|
|
{
|
2021-12-21 12:34:41 +03:00
|
|
|
if (runstate_is_running()) {
|
2023-04-11 18:36:48 +03:00
|
|
|
stat64_add(&ram_counters.precopy_bytes, bytes);
|
2021-12-21 12:34:41 +03:00
|
|
|
} else if (migration_in_postcopy()) {
|
2023-02-22 17:56:45 +03:00
|
|
|
stat64_add(&ram_counters.postcopy_bytes, bytes);
|
2021-12-21 12:34:41 +03:00
|
|
|
} else {
|
2023-04-11 18:38:11 +03:00
|
|
|
stat64_add(&ram_counters.downtime_bytes, bytes);
|
2021-12-21 12:34:41 +03:00
|
|
|
}
|
2023-02-22 17:56:45 +03:00
|
|
|
stat64_add(&ram_counters.transferred, bytes);
|
2021-12-21 12:34:40 +03:00
|
|
|
}
|
|
|
|
|
2021-12-15 22:10:39 +03:00
|
|
|
struct MigrationOps {
|
|
|
|
int (*ram_save_target_page)(RAMState *rs, PageSearchStatus *pss);
|
|
|
|
};
|
|
|
|
typedef struct MigrationOps MigrationOps;
|
|
|
|
|
|
|
|
MigrationOps *migration_ops;
|
|
|
|
|
2018-09-06 10:01:00 +03:00
|
|
|
CompressionStats compression_counters;
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
struct CompressParam {
|
|
|
|
bool done;
|
2016-05-05 10:32:56 +03:00
|
|
|
bool quit;
|
2018-08-21 11:10:24 +03:00
|
|
|
bool zero_page;
|
2015-05-07 20:33:31 +03:00
|
|
|
QEMUFile *file;
|
|
|
|
QemuMutex mutex;
|
|
|
|
QemuCond cond;
|
|
|
|
RAMBlock *block;
|
|
|
|
ram_addr_t offset;
|
2018-03-30 10:51:22 +03:00
|
|
|
|
|
|
|
/* internally used fields */
|
2018-03-30 10:51:20 +03:00
|
|
|
z_stream stream;
|
2018-03-30 10:51:22 +03:00
|
|
|
uint8_t *originbuf;
|
2015-05-07 20:33:31 +03:00
|
|
|
};
|
|
|
|
typedef struct CompressParam CompressParam;
|
|
|
|
|
|
|
|
struct DecompressParam {
|
2016-05-05 10:32:51 +03:00
|
|
|
bool done;
|
2016-05-05 10:32:56 +03:00
|
|
|
bool quit;
|
2015-05-07 20:33:31 +03:00
|
|
|
QemuMutex mutex;
|
|
|
|
QemuCond cond;
|
|
|
|
void *des;
|
2016-01-22 18:09:21 +03:00
|
|
|
uint8_t *compbuf;
|
2015-05-07 20:33:31 +03:00
|
|
|
int len;
|
2018-03-30 10:51:21 +03:00
|
|
|
z_stream stream;
|
2015-05-07 20:33:31 +03:00
|
|
|
};
|
|
|
|
typedef struct DecompressParam DecompressParam;
|
|
|
|
|
|
|
|
static CompressParam *comp_param;
|
|
|
|
static QemuThread *compress_threads;
|
|
|
|
/* comp_done_cond is used to wake up the migration thread when
|
|
|
|
* one of the compression threads has finished the compression.
|
|
|
|
* comp_done_lock is used to co-work with comp_done_cond.
|
|
|
|
*/
|
2016-05-05 10:32:59 +03:00
|
|
|
static QemuMutex comp_done_lock;
|
|
|
|
static QemuCond comp_done_cond;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2018-03-30 10:51:22 +03:00
|
|
|
static QEMUFile *decomp_file;
|
2015-05-07 20:33:31 +03:00
|
|
|
static DecompressParam *decomp_param;
|
|
|
|
static QemuThread *decompress_threads;
|
2016-05-05 10:32:51 +03:00
|
|
|
static QemuMutex decomp_done_lock;
|
|
|
|
static QemuCond decomp_done_cond;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2022-10-12 00:55:57 +03:00
|
|
|
static int ram_save_host_page_urgent(PageSearchStatus *pss);
|
|
|
|
|
2018-08-21 11:10:24 +03:00
|
|
|
static bool do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
|
2018-08-21 11:10:23 +03:00
|
|
|
ram_addr_t offset, uint8_t *source_buf);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2022-10-12 00:55:54 +03:00
|
|
|
/* NOTE: page is the PFN not real ram_addr_t. */
|
|
|
|
static void pss_init(PageSearchStatus *pss, RAMBlock *rb, ram_addr_t page)
|
|
|
|
{
|
|
|
|
pss->block = rb;
|
|
|
|
pss->page = page;
|
|
|
|
pss->complete_round = false;
|
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:57 +03:00
|
|
|
/*
|
|
|
|
* Check whether two PSSs are actively sending the same page. Return true
|
|
|
|
* if it is, false otherwise.
|
|
|
|
*/
|
|
|
|
static bool pss_overlap(PageSearchStatus *pss1, PageSearchStatus *pss2)
|
|
|
|
{
|
|
|
|
return pss1->host_page_sending && pss2->host_page_sending &&
|
|
|
|
(pss1->host_page_start == pss2->host_page_start);
|
|
|
|
}
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
static void *do_data_compress(void *opaque)
|
|
|
|
{
|
|
|
|
CompressParam *param = opaque;
|
2016-05-05 10:32:57 +03:00
|
|
|
RAMBlock *block;
|
|
|
|
ram_addr_t offset;
|
2018-08-21 11:10:24 +03:00
|
|
|
bool zero_page;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2016-05-05 10:32:57 +03:00
|
|
|
qemu_mutex_lock(¶m->mutex);
|
2016-05-05 10:32:56 +03:00
|
|
|
while (!param->quit) {
|
2016-05-05 10:32:57 +03:00
|
|
|
if (param->block) {
|
|
|
|
block = param->block;
|
|
|
|
offset = param->offset;
|
|
|
|
param->block = NULL;
|
|
|
|
qemu_mutex_unlock(¶m->mutex);
|
|
|
|
|
2018-08-21 11:10:24 +03:00
|
|
|
zero_page = do_compress_ram_page(param->file, ¶m->stream,
|
|
|
|
block, offset, param->originbuf);
|
2016-05-05 10:32:57 +03:00
|
|
|
|
2016-05-05 10:32:59 +03:00
|
|
|
qemu_mutex_lock(&comp_done_lock);
|
2016-05-05 10:32:57 +03:00
|
|
|
param->done = true;
|
2018-08-21 11:10:24 +03:00
|
|
|
param->zero_page = zero_page;
|
2016-05-05 10:32:59 +03:00
|
|
|
qemu_cond_signal(&comp_done_cond);
|
|
|
|
qemu_mutex_unlock(&comp_done_lock);
|
2016-05-05 10:32:57 +03:00
|
|
|
|
|
|
|
qemu_mutex_lock(¶m->mutex);
|
|
|
|
} else {
|
2015-05-07 20:33:31 +03:00
|
|
|
qemu_cond_wait(¶m->cond, ¶m->mutex);
|
|
|
|
}
|
|
|
|
}
|
2016-05-05 10:32:57 +03:00
|
|
|
qemu_mutex_unlock(¶m->mutex);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2017-06-28 12:52:28 +03:00
|
|
|
static void compress_threads_save_cleanup(void)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
|
|
|
int i, thread_count;
|
|
|
|
|
2023-03-02 00:03:48 +03:00
|
|
|
if (!migrate_compress() || !comp_param) {
|
2015-05-07 20:33:31 +03:00
|
|
|
return;
|
|
|
|
}
|
2018-09-25 12:14:40 +03:00
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
thread_count = migrate_compress_threads();
|
|
|
|
for (i = 0; i < thread_count; i++) {
|
2018-03-30 10:51:20 +03:00
|
|
|
/*
|
|
|
|
* we use it as a indicator which shows if the thread is
|
|
|
|
* properly init'd or not
|
|
|
|
*/
|
|
|
|
if (!comp_param[i].file) {
|
|
|
|
break;
|
|
|
|
}
|
2018-09-25 12:14:40 +03:00
|
|
|
|
|
|
|
qemu_mutex_lock(&comp_param[i].mutex);
|
|
|
|
comp_param[i].quit = true;
|
|
|
|
qemu_cond_signal(&comp_param[i].cond);
|
|
|
|
qemu_mutex_unlock(&comp_param[i].mutex);
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
qemu_thread_join(compress_threads + i);
|
|
|
|
qemu_mutex_destroy(&comp_param[i].mutex);
|
|
|
|
qemu_cond_destroy(&comp_param[i].cond);
|
2018-03-30 10:51:20 +03:00
|
|
|
deflateEnd(&comp_param[i].stream);
|
2018-03-30 10:51:22 +03:00
|
|
|
g_free(comp_param[i].originbuf);
|
2018-03-30 10:51:20 +03:00
|
|
|
qemu_fclose(comp_param[i].file);
|
|
|
|
comp_param[i].file = NULL;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
2016-05-05 10:32:59 +03:00
|
|
|
qemu_mutex_destroy(&comp_done_lock);
|
|
|
|
qemu_cond_destroy(&comp_done_cond);
|
2015-05-07 20:33:31 +03:00
|
|
|
g_free(compress_threads);
|
|
|
|
g_free(comp_param);
|
|
|
|
compress_threads = NULL;
|
|
|
|
comp_param = NULL;
|
|
|
|
}
|
|
|
|
|
2018-03-30 10:51:20 +03:00
|
|
|
static int compress_threads_save_setup(void)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
|
|
|
int i, thread_count;
|
|
|
|
|
2023-03-02 00:03:48 +03:00
|
|
|
if (!migrate_compress()) {
|
2018-03-30 10:51:20 +03:00
|
|
|
return 0;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
thread_count = migrate_compress_threads();
|
|
|
|
compress_threads = g_new0(QemuThread, thread_count);
|
|
|
|
comp_param = g_new0(CompressParam, thread_count);
|
2016-05-05 10:32:59 +03:00
|
|
|
qemu_cond_init(&comp_done_cond);
|
|
|
|
qemu_mutex_init(&comp_done_lock);
|
2015-05-07 20:33:31 +03:00
|
|
|
for (i = 0; i < thread_count; i++) {
|
2018-03-30 10:51:22 +03:00
|
|
|
comp_param[i].originbuf = g_try_malloc(TARGET_PAGE_SIZE);
|
|
|
|
if (!comp_param[i].originbuf) {
|
|
|
|
goto exit;
|
|
|
|
}
|
|
|
|
|
2018-03-30 10:51:20 +03:00
|
|
|
if (deflateInit(&comp_param[i].stream,
|
|
|
|
migrate_compress_level()) != Z_OK) {
|
2018-03-30 10:51:22 +03:00
|
|
|
g_free(comp_param[i].originbuf);
|
2018-03-30 10:51:20 +03:00
|
|
|
goto exit;
|
|
|
|
}
|
|
|
|
|
2016-07-29 10:10:31 +03:00
|
|
|
/* comp_param[i].file is just used as a dummy buffer to save data,
|
|
|
|
* set its ops to empty.
|
2015-05-07 20:33:31 +03:00
|
|
|
*/
|
2022-06-20 14:02:05 +03:00
|
|
|
comp_param[i].file = qemu_file_new_output(
|
2022-06-20 14:01:46 +03:00
|
|
|
QIO_CHANNEL(qio_channel_null_new()));
|
2015-05-07 20:33:31 +03:00
|
|
|
comp_param[i].done = true;
|
2016-05-05 10:32:56 +03:00
|
|
|
comp_param[i].quit = false;
|
2015-05-07 20:33:31 +03:00
|
|
|
qemu_mutex_init(&comp_param[i].mutex);
|
|
|
|
qemu_cond_init(&comp_param[i].cond);
|
|
|
|
qemu_thread_create(compress_threads + i, "compress",
|
|
|
|
do_data_compress, comp_param + i,
|
|
|
|
QEMU_THREAD_JOINABLE);
|
|
|
|
}
|
2018-03-30 10:51:20 +03:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
exit:
|
|
|
|
compress_threads_save_cleanup();
|
|
|
|
return -1;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2017-03-23 17:06:39 +03:00
|
|
|
* save_page_header: write page header to wire
|
2015-05-07 20:33:31 +03:00
|
|
|
*
|
|
|
|
* If this is the 1st block, it also writes the block identification
|
|
|
|
*
|
2017-03-23 17:06:39 +03:00
|
|
|
* Returns the number of bytes written
|
2015-05-07 20:33:31 +03:00
|
|
|
*
|
2022-10-12 00:55:56 +03:00
|
|
|
* @pss: current PSS channel status
|
2015-05-07 20:33:31 +03:00
|
|
|
* @block: block that contains the page we want to send
|
|
|
|
* @offset: offset inside the block for the page
|
|
|
|
* in the lower bits, it contains flags
|
|
|
|
*/
|
2023-04-02 20:06:32 +03:00
|
|
|
static size_t save_page_header(PageSearchStatus *pss, QEMUFile *f,
|
|
|
|
RAMBlock *block, ram_addr_t offset)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2015-07-13 12:34:10 +03:00
|
|
|
size_t size, len;
|
2022-10-12 00:55:56 +03:00
|
|
|
bool same_block = (block == pss->last_sent_block);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2022-10-12 00:55:48 +03:00
|
|
|
if (same_block) {
|
2017-03-21 13:45:01 +03:00
|
|
|
offset |= RAM_SAVE_FLAG_CONTINUE;
|
|
|
|
}
|
2017-05-10 14:28:13 +03:00
|
|
|
qemu_put_be64(f, offset);
|
2015-05-07 20:33:31 +03:00
|
|
|
size = 8;
|
|
|
|
|
2022-10-12 00:55:48 +03:00
|
|
|
if (!same_block) {
|
2015-07-13 12:34:10 +03:00
|
|
|
len = strlen(block->idstr);
|
2017-05-10 14:28:13 +03:00
|
|
|
qemu_put_byte(f, len);
|
|
|
|
qemu_put_buffer(f, (uint8_t *)block->idstr, len);
|
2015-07-13 12:34:10 +03:00
|
|
|
size += 1 + len;
|
2022-10-12 00:55:56 +03:00
|
|
|
pss->last_sent_block = block;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
return size;
|
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
2021-07-08 19:21:59 +03:00
|
|
|
* mig_throttle_guest_down: throttle down the guest
|
2017-03-23 17:06:39 +03:00
|
|
|
*
|
|
|
|
* Reduce amount of guest cpu execution to hopefully slow down memory
|
|
|
|
* writes. If guest dirty memory rate is reduced below the rate at
|
|
|
|
* which we can transfer pages to the destination then we should be
|
|
|
|
* able to complete migration. Some workloads dirty memory way too
|
|
|
|
* fast and will not effectively converge, even with auto-converge.
|
2015-09-08 20:12:35 +03:00
|
|
|
*/
|
2020-04-13 13:15:08 +03:00
|
|
|
static void mig_throttle_guest_down(uint64_t bytes_dirty_period,
|
|
|
|
uint64_t bytes_dirty_threshold)
|
2015-09-08 20:12:35 +03:00
|
|
|
{
|
|
|
|
MigrationState *s = migrate_get_current();
|
2016-04-27 13:05:14 +03:00
|
|
|
uint64_t pct_initial = s->parameters.cpu_throttle_initial;
|
2020-04-13 13:15:08 +03:00
|
|
|
uint64_t pct_increment = s->parameters.cpu_throttle_increment;
|
|
|
|
bool pct_tailslow = s->parameters.cpu_throttle_tailslow;
|
2018-08-01 16:00:20 +03:00
|
|
|
int pct_max = s->parameters.max_cpu_throttle;
|
2015-09-08 20:12:35 +03:00
|
|
|
|
2020-04-13 13:15:08 +03:00
|
|
|
uint64_t throttle_now = cpu_throttle_get_percentage();
|
|
|
|
uint64_t cpu_now, cpu_ideal, throttle_inc;
|
|
|
|
|
2015-09-08 20:12:35 +03:00
|
|
|
/* We have not started throttling yet. Let's start it. */
|
|
|
|
if (!cpu_throttle_active()) {
|
|
|
|
cpu_throttle_set(pct_initial);
|
|
|
|
} else {
|
|
|
|
/* Throttling already on, just increase the rate */
|
2020-04-13 13:15:08 +03:00
|
|
|
if (!pct_tailslow) {
|
|
|
|
throttle_inc = pct_increment;
|
|
|
|
} else {
|
|
|
|
/* Compute the ideal CPU percentage used by Guest, which may
|
|
|
|
* make the dirty rate match the dirty rate threshold. */
|
|
|
|
cpu_now = 100 - throttle_now;
|
|
|
|
cpu_ideal = cpu_now * (bytes_dirty_threshold * 1.0 /
|
|
|
|
bytes_dirty_period);
|
|
|
|
throttle_inc = MIN(cpu_now - cpu_ideal, pct_increment);
|
|
|
|
}
|
|
|
|
cpu_throttle_set(MIN(throttle_now + throttle_inc, pct_max));
|
2015-09-08 20:12:35 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-11-09 06:04:54 +03:00
|
|
|
void mig_throttle_counter_reset(void)
|
|
|
|
{
|
|
|
|
RAMState *rs = ram_state;
|
|
|
|
|
|
|
|
rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
|
|
|
|
rs->num_dirty_pages_period = 0;
|
2023-02-22 17:56:45 +03:00
|
|
|
rs->bytes_xfer_prev = stat64_get(&ram_counters.transferred);
|
2021-11-09 06:04:54 +03:00
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
|
|
|
|
*
|
2017-03-13 21:26:29 +03:00
|
|
|
* @rs: current RAM state
|
2017-03-23 17:06:39 +03:00
|
|
|
* @current_addr: address for the zero page
|
|
|
|
*
|
|
|
|
* Update the xbzrle cache to reflect a page that's been sent as all 0.
|
2015-05-07 20:33:31 +03:00
|
|
|
* The important thing is that a stale (not-yet-0'd) page be replaced
|
|
|
|
* by the new data.
|
|
|
|
* As a bonus, if the page wasn't in the cache it gets added so that
|
2017-03-23 17:06:39 +03:00
|
|
|
* when a small write is made into the 0'd page it gets XBZRLE sent.
|
2015-05-07 20:33:31 +03:00
|
|
|
*/
|
2017-03-13 21:26:29 +03:00
|
|
|
static void xbzrle_cache_zero_page(RAMState *rs, ram_addr_t current_addr)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
|
|
|
/* We don't care if this fails to allocate a new cache page
|
|
|
|
* as long as it updated an old one */
|
2017-05-09 17:22:01 +03:00
|
|
|
cache_insert(XBZRLE.cache, current_addr, XBZRLE.zero_target_page,
|
2023-04-11 19:02:34 +03:00
|
|
|
stat64_get(&ram_counters.dirty_sync_count));
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
#define ENCODING_FLAG_XBZRLE 0x1
|
|
|
|
|
|
|
|
/**
|
|
|
|
* save_xbzrle_page: compress and send current page
|
|
|
|
*
|
|
|
|
* Returns: 1 means that we wrote the page
|
|
|
|
* 0 means that page is identical to the one already sent
|
|
|
|
* -1 means that xbzrle would be longer than normal
|
|
|
|
*
|
2017-03-13 21:39:02 +03:00
|
|
|
* @rs: current RAM state
|
2022-10-12 00:55:56 +03:00
|
|
|
* @pss: current PSS channel
|
2017-03-23 17:06:39 +03:00
|
|
|
* @current_data: pointer to the address of the page contents
|
|
|
|
* @current_addr: addr of the page
|
2015-05-07 20:33:31 +03:00
|
|
|
* @block: block that contains the page we want to send
|
|
|
|
* @offset: offset inside the block for the page
|
|
|
|
*/
|
2022-10-12 00:55:56 +03:00
|
|
|
static int save_xbzrle_page(RAMState *rs, PageSearchStatus *pss,
|
2022-10-12 00:55:53 +03:00
|
|
|
uint8_t **current_data, ram_addr_t current_addr,
|
|
|
|
RAMBlock *block, ram_addr_t offset)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
|
|
|
int encoded_len = 0, bytes_xbzrle;
|
|
|
|
uint8_t *prev_cached_page;
|
2022-10-12 00:55:56 +03:00
|
|
|
QEMUFile *file = pss->pss_channel;
|
2023-04-11 19:02:34 +03:00
|
|
|
uint64_t generation = stat64_get(&ram_counters.dirty_sync_count);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2023-04-11 19:02:34 +03:00
|
|
|
if (!cache_is_cached(XBZRLE.cache, current_addr, generation)) {
|
2017-06-06 20:49:03 +03:00
|
|
|
xbzrle_counters.cache_miss++;
|
2021-12-15 21:01:21 +03:00
|
|
|
if (!rs->last_stage) {
|
2015-05-07 20:33:31 +03:00
|
|
|
if (cache_insert(XBZRLE.cache, current_addr, *current_data,
|
2023-04-11 19:02:34 +03:00
|
|
|
generation) == -1) {
|
2015-05-07 20:33:31 +03:00
|
|
|
return -1;
|
|
|
|
} else {
|
|
|
|
/* update *current_data when the page has been
|
|
|
|
inserted into cache */
|
|
|
|
*current_data = get_cached_data(XBZRLE.cache, current_addr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2020-04-30 03:59:35 +03:00
|
|
|
/*
|
|
|
|
* Reaching here means the page has hit the xbzrle cache, no matter what
|
|
|
|
* encoding result it is (normal encoding, overflow or skipping the page),
|
2020-09-17 10:50:21 +03:00
|
|
|
* count the page as encoded. This is used to calculate the encoding rate.
|
2020-04-30 03:59:35 +03:00
|
|
|
*
|
|
|
|
* Example: 2 pages (8KB) being encoded, first page encoding generates 2KB,
|
|
|
|
* 2nd page turns out to be skipped (i.e. no new bytes written to the
|
|
|
|
* page), the overall encoding rate will be 8KB / 2KB = 4, which has the
|
|
|
|
* skipped page included. In this way, the encoding rate can tell if the
|
|
|
|
* guest page is good for xbzrle encoding.
|
|
|
|
*/
|
|
|
|
xbzrle_counters.pages++;
|
2015-05-07 20:33:31 +03:00
|
|
|
prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
|
|
|
|
|
|
|
|
/* save current buffer into memory */
|
|
|
|
memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
|
|
|
|
|
|
|
|
/* XBZRLE encoding (if there is no overflow) */
|
2022-11-16 18:29:22 +03:00
|
|
|
encoded_len = xbzrle_encode_buffer_func(prev_cached_page, XBZRLE.current_buf,
|
|
|
|
TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
|
|
|
|
TARGET_PAGE_SIZE);
|
2019-06-10 03:41:59 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Update the cache contents, so that it corresponds to the data
|
|
|
|
* sent, in all cases except where we skip the page.
|
|
|
|
*/
|
2021-12-15 21:01:21 +03:00
|
|
|
if (!rs->last_stage && encoded_len != 0) {
|
2019-06-10 03:41:59 +03:00
|
|
|
memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
|
|
|
|
/*
|
|
|
|
* In the case where we couldn't compress, ensure that the caller
|
|
|
|
* sends the data from the cache, since the guest might have
|
|
|
|
* changed the RAM since we copied it.
|
|
|
|
*/
|
|
|
|
*current_data = prev_cached_page;
|
|
|
|
}
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
if (encoded_len == 0) {
|
2017-01-24 00:32:05 +03:00
|
|
|
trace_save_xbzrle_page_skipping();
|
2015-05-07 20:33:31 +03:00
|
|
|
return 0;
|
|
|
|
} else if (encoded_len == -1) {
|
2017-01-24 00:32:05 +03:00
|
|
|
trace_save_xbzrle_page_overflow();
|
2017-06-06 20:49:03 +03:00
|
|
|
xbzrle_counters.overflow++;
|
2020-04-30 03:59:35 +03:00
|
|
|
xbzrle_counters.bytes += TARGET_PAGE_SIZE;
|
2015-05-07 20:33:31 +03:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Send XBZRLE based compressed page */
|
2023-04-02 20:06:32 +03:00
|
|
|
bytes_xbzrle = save_page_header(pss, pss->pss_channel, block,
|
2017-03-15 11:16:57 +03:00
|
|
|
offset | RAM_SAVE_FLAG_XBZRLE);
|
2022-10-12 00:55:53 +03:00
|
|
|
qemu_put_byte(file, ENCODING_FLAG_XBZRLE);
|
|
|
|
qemu_put_be16(file, encoded_len);
|
|
|
|
qemu_put_buffer(file, XBZRLE.encoded_buf, encoded_len);
|
2015-05-07 20:33:31 +03:00
|
|
|
bytes_xbzrle += encoded_len + 1 + 2;
|
2020-04-30 03:59:35 +03:00
|
|
|
/*
|
|
|
|
* Like compressed_size (please see update_compress_thread_counts),
|
|
|
|
* the xbzrle encoded bytes don't count the 8 byte header with
|
|
|
|
* RAM_SAVE_FLAG_CONTINUE.
|
|
|
|
*/
|
|
|
|
xbzrle_counters.bytes += bytes_xbzrle - 8;
|
2021-12-21 12:34:40 +03:00
|
|
|
ram_transferred_add(bytes_xbzrle);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
2022-10-12 00:55:52 +03:00
|
|
|
* pss_find_next_dirty: find the next dirty page of current ramblock
|
2015-11-05 21:11:01 +03:00
|
|
|
*
|
2022-10-12 00:55:52 +03:00
|
|
|
* This function updates pss->page to point to the next dirty page index
|
|
|
|
* within the ramblock to migrate, or the end of ramblock when nothing
|
|
|
|
* found. Note that when pss->host_page_sending==true it means we're
|
|
|
|
* during sending a host page, so we won't look for dirty page that is
|
|
|
|
* outside the host page boundary.
|
2017-03-23 17:06:39 +03:00
|
|
|
*
|
2022-10-12 00:55:52 +03:00
|
|
|
* @pss: the current page search status
|
2015-11-05 21:11:01 +03:00
|
|
|
*/
|
2022-10-12 00:55:52 +03:00
|
|
|
static void pss_find_next_dirty(PageSearchStatus *pss)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2022-10-12 00:55:52 +03:00
|
|
|
RAMBlock *rb = pss->block;
|
2017-03-22 17:18:04 +03:00
|
|
|
unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
|
|
|
|
unsigned long *bitmap = rb->bmap;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
if (ramblock_is_ignored(rb)) {
|
2022-10-12 00:55:52 +03:00
|
|
|
/* Points directly to the end, so we know no dirty page */
|
|
|
|
pss->page = size;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If during sending a host page, only look for dirty pages within the
|
|
|
|
* current host page being send.
|
|
|
|
*/
|
|
|
|
if (pss->host_page_sending) {
|
|
|
|
assert(pss->host_page_end);
|
|
|
|
size = MIN(size, pss->host_page_end);
|
2018-05-14 09:57:00 +03:00
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:52 +03:00
|
|
|
pss->page = find_next_bit(bitmap, size, pss->page);
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2021-09-04 19:09:07 +03:00
|
|
|
static void migration_clear_memory_region_dirty_bitmap(RAMBlock *rb,
|
2021-07-22 11:30:55 +03:00
|
|
|
unsigned long page)
|
|
|
|
{
|
|
|
|
uint8_t shift;
|
|
|
|
hwaddr size, start;
|
|
|
|
|
|
|
|
if (!rb->clear_bmap || !clear_bmap_test_and_clear(rb, page)) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
shift = rb->clear_bmap_shift;
|
|
|
|
/*
|
|
|
|
* CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
|
|
|
|
* can make things easier sometimes since then start address
|
|
|
|
* of the small chunk will always be 64 pages aligned so the
|
|
|
|
* bitmap will always be aligned to unsigned long. We should
|
|
|
|
* even be able to remove this restriction but I'm simply
|
|
|
|
* keeping it.
|
|
|
|
*/
|
|
|
|
assert(shift >= 6);
|
|
|
|
|
|
|
|
size = 1ULL << (TARGET_PAGE_BITS + shift);
|
2021-10-11 20:53:44 +03:00
|
|
|
start = QEMU_ALIGN_DOWN((ram_addr_t)page << TARGET_PAGE_BITS, size);
|
2021-07-22 11:30:55 +03:00
|
|
|
trace_migration_bitmap_clear_dirty(rb->idstr, start, size, page);
|
|
|
|
memory_region_clear_dirty_bitmap(rb->mr, start, size);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2021-09-04 19:09:07 +03:00
|
|
|
migration_clear_memory_region_dirty_bitmap_range(RAMBlock *rb,
|
2021-07-22 11:30:55 +03:00
|
|
|
unsigned long start,
|
|
|
|
unsigned long npages)
|
|
|
|
{
|
|
|
|
unsigned long i, chunk_pages = 1UL << rb->clear_bmap_shift;
|
|
|
|
unsigned long chunk_start = QEMU_ALIGN_DOWN(start, chunk_pages);
|
|
|
|
unsigned long chunk_end = QEMU_ALIGN_UP(start + npages, chunk_pages);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Clear pages from start to start + npages - 1, so the end boundary is
|
|
|
|
* exclusive.
|
|
|
|
*/
|
|
|
|
for (i = chunk_start; i < chunk_end; i += chunk_pages) {
|
2021-09-04 19:09:07 +03:00
|
|
|
migration_clear_memory_region_dirty_bitmap(rb, i);
|
2021-07-22 11:30:55 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-11-09 06:04:55 +03:00
|
|
|
/*
|
|
|
|
* colo_bitmap_find_diry:find contiguous dirty pages from start
|
|
|
|
*
|
|
|
|
* Returns the page offset within memory region of the start of the contiguout
|
|
|
|
* dirty page
|
|
|
|
*
|
|
|
|
* @rs: current RAM state
|
|
|
|
* @rb: RAMBlock where to search for dirty pages
|
|
|
|
* @start: page where we start the search
|
|
|
|
* @num: the number of contiguous dirty pages
|
|
|
|
*/
|
|
|
|
static inline
|
|
|
|
unsigned long colo_bitmap_find_dirty(RAMState *rs, RAMBlock *rb,
|
|
|
|
unsigned long start, unsigned long *num)
|
|
|
|
{
|
|
|
|
unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
|
|
|
|
unsigned long *bitmap = rb->bmap;
|
|
|
|
unsigned long first, next;
|
|
|
|
|
|
|
|
*num = 0;
|
|
|
|
|
|
|
|
if (ramblock_is_ignored(rb)) {
|
|
|
|
return size;
|
|
|
|
}
|
|
|
|
|
|
|
|
first = find_next_bit(bitmap, size, start);
|
|
|
|
if (first >= size) {
|
|
|
|
return first;
|
|
|
|
}
|
|
|
|
next = find_next_zero_bit(bitmap, size, first + 1);
|
|
|
|
assert(next >= first);
|
|
|
|
*num = next - first;
|
|
|
|
return first;
|
|
|
|
}
|
|
|
|
|
2017-03-21 17:18:05 +03:00
|
|
|
static inline bool migration_bitmap_clear_dirty(RAMState *rs,
|
2017-03-21 18:19:05 +03:00
|
|
|
RAMBlock *rb,
|
|
|
|
unsigned long page)
|
2015-11-05 21:11:09 +03:00
|
|
|
{
|
|
|
|
bool ret;
|
|
|
|
|
migration: Split log_clear() into smaller chunks
Currently we are doing log_clear() right after log_sync() which mostly
keeps the old behavior when log_clear() was still part of log_sync().
This patch tries to further optimize the migration log_clear() code
path to split huge log_clear()s into smaller chunks.
We do this by spliting the whole guest memory region into memory
chunks, whose size is decided by MigrationState.clear_bitmap_shift (an
example will be given below). With that, we don't do the dirty bitmap
clear operation on the remote node (e.g., KVM) when we fetch the dirty
bitmap, instead we explicitly clear the dirty bitmap for the memory
chunk for each of the first time we send a page in that chunk.
Here comes an example.
Assuming the guest has 64G memory, then before this patch the KVM
ioctl KVM_CLEAR_DIRTY_LOG will be a single one covering 64G memory.
If after the patch, let's assume when the clear bitmap shift is 18,
then the memory chunk size on x86_64 will be 1UL<<18 * 4K = 1GB. Then
instead of sending a big 64G ioctl, we'll send 64 small ioctls, each
of the ioctl will cover 1G of the guest memory. For each of the 64
small ioctls, we'll only send if any of the page in that small chunk
was going to be sent right away.
Signed-off-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Message-Id: <20190603065056.25211-12-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2019-06-03 09:50:56 +03:00
|
|
|
/*
|
|
|
|
* Clear dirty bitmap if needed. This _must_ be called before we
|
|
|
|
* send any of the page in the chunk because we need to make sure
|
|
|
|
* we can capture further page content changes when we sync dirty
|
|
|
|
* log the next time. So as long as we are going to send any of
|
|
|
|
* the page in the chunk we clear the remote dirty bitmap for all.
|
|
|
|
* Clearing it earlier won't be a problem, but too late will.
|
|
|
|
*/
|
2021-09-04 19:09:07 +03:00
|
|
|
migration_clear_memory_region_dirty_bitmap(rb, page);
|
migration: Split log_clear() into smaller chunks
Currently we are doing log_clear() right after log_sync() which mostly
keeps the old behavior when log_clear() was still part of log_sync().
This patch tries to further optimize the migration log_clear() code
path to split huge log_clear()s into smaller chunks.
We do this by spliting the whole guest memory region into memory
chunks, whose size is decided by MigrationState.clear_bitmap_shift (an
example will be given below). With that, we don't do the dirty bitmap
clear operation on the remote node (e.g., KVM) when we fetch the dirty
bitmap, instead we explicitly clear the dirty bitmap for the memory
chunk for each of the first time we send a page in that chunk.
Here comes an example.
Assuming the guest has 64G memory, then before this patch the KVM
ioctl KVM_CLEAR_DIRTY_LOG will be a single one covering 64G memory.
If after the patch, let's assume when the clear bitmap shift is 18,
then the memory chunk size on x86_64 will be 1UL<<18 * 4K = 1GB. Then
instead of sending a big 64G ioctl, we'll send 64 small ioctls, each
of the ioctl will cover 1G of the guest memory. For each of the 64
small ioctls, we'll only send if any of the page in that small chunk
was going to be sent right away.
Signed-off-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Message-Id: <20190603065056.25211-12-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2019-06-03 09:50:56 +03:00
|
|
|
|
2017-03-22 17:18:04 +03:00
|
|
|
ret = test_and_clear_bit(page, rb->bmap);
|
2015-11-05 21:11:09 +03:00
|
|
|
if (ret) {
|
2017-03-13 23:21:41 +03:00
|
|
|
rs->migration_dirty_pages--;
|
2015-11-05 21:11:09 +03:00
|
|
|
}
|
2018-12-11 11:24:49 +03:00
|
|
|
|
2015-11-05 21:11:09 +03:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
migration/ram: Handle RAMBlocks with a RamDiscardManager on the migration source
We don't want to migrate memory that corresponds to discarded ranges as
managed by a RamDiscardManager responsible for the mapped memory region of
the RAMBlock. The content of these pages is essentially stale and
without any guarantees for the VM ("logically unplugged").
Depending on the underlying memory type, even reading memory might populate
memory on the source, resulting in an undesired memory consumption. Of
course, on the destination, even writing a zeropage consumes memory,
which we also want to avoid (similar to free page hinting).
Currently, virtio-mem tries achieving that goal (not migrating "unplugged"
memory that was discarded) by going via qemu_guest_free_page_hint() - but
it's hackish and incomplete.
For example, background snapshots still end up reading all memory, as
they don't do bitmap syncs. Postcopy recovery code will re-add
previously cleared bits to the dirty bitmap and migrate them.
Let's consult the RamDiscardManager after setting up our dirty bitmap
initially and when postcopy recovery code reinitializes it: clear
corresponding bits in the dirty bitmaps (e.g., of the RAMBlock and inside
KVM). It's important to fixup the dirty bitmap *after* our initial bitmap
sync, such that the corresponding dirty bits in KVM are actually cleared.
As colo is incompatible with discarding of RAM and inhibits it, we don't
have to bother.
Note: if a misbehaving guest would use discarded ranges after migration
started we would still migrate that memory: however, then we already
populated that memory on the migration source.
Reviewed-by: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2021-10-11 20:53:41 +03:00
|
|
|
static void dirty_bitmap_clear_section(MemoryRegionSection *section,
|
|
|
|
void *opaque)
|
|
|
|
{
|
|
|
|
const hwaddr offset = section->offset_within_region;
|
|
|
|
const hwaddr size = int128_get64(section->size);
|
|
|
|
const unsigned long start = offset >> TARGET_PAGE_BITS;
|
|
|
|
const unsigned long npages = size >> TARGET_PAGE_BITS;
|
|
|
|
RAMBlock *rb = section->mr->ram_block;
|
|
|
|
uint64_t *cleared_bits = opaque;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We don't grab ram_state->bitmap_mutex because we expect to run
|
|
|
|
* only when starting migration or during postcopy recovery where
|
|
|
|
* we don't have concurrent access.
|
|
|
|
*/
|
|
|
|
if (!migration_in_postcopy() && !migrate_background_snapshot()) {
|
|
|
|
migration_clear_memory_region_dirty_bitmap_range(rb, start, npages);
|
|
|
|
}
|
|
|
|
*cleared_bits += bitmap_count_one_with_offset(rb->bmap, start, npages);
|
|
|
|
bitmap_clear(rb->bmap, start, npages);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Exclude all dirty pages from migration that fall into a discarded range as
|
|
|
|
* managed by a RamDiscardManager responsible for the mapped memory region of
|
|
|
|
* the RAMBlock. Clear the corresponding bits in the dirty bitmaps.
|
|
|
|
*
|
|
|
|
* Discarded pages ("logically unplugged") have undefined content and must
|
|
|
|
* not get migrated, because even reading these pages for migration might
|
|
|
|
* result in undesired behavior.
|
|
|
|
*
|
|
|
|
* Returns the number of cleared bits in the RAMBlock dirty bitmap.
|
|
|
|
*
|
|
|
|
* Note: The result is only stable while migrating (precopy/postcopy).
|
|
|
|
*/
|
|
|
|
static uint64_t ramblock_dirty_bitmap_clear_discarded_pages(RAMBlock *rb)
|
|
|
|
{
|
|
|
|
uint64_t cleared_bits = 0;
|
|
|
|
|
|
|
|
if (rb->mr && rb->bmap && memory_region_has_ram_discard_manager(rb->mr)) {
|
|
|
|
RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
|
|
|
|
MemoryRegionSection section = {
|
|
|
|
.mr = rb->mr,
|
|
|
|
.offset_within_region = 0,
|
|
|
|
.size = int128_make64(qemu_ram_get_used_length(rb)),
|
|
|
|
};
|
|
|
|
|
|
|
|
ram_discard_manager_replay_discarded(rdm, §ion,
|
|
|
|
dirty_bitmap_clear_section,
|
|
|
|
&cleared_bits);
|
|
|
|
}
|
|
|
|
return cleared_bits;
|
|
|
|
}
|
|
|
|
|
migration/postcopy: Handle RAMBlocks with a RamDiscardManager on the destination
Currently, when someone (i.e., the VM) accesses discarded parts inside a
RAMBlock with a RamDiscardManager managing the corresponding mapped memory
region, postcopy will request migration of the corresponding page from the
source. The source, however, will never answer, because it refuses to
migrate such pages with undefined content ("logically unplugged"): the
pages are never dirty, and get_queued_page() will consequently skip
processing these postcopy requests.
Especially reading discarded ("logically unplugged") ranges is supposed to
work in some setups (for example with current virtio-mem), although it
barely ever happens: still, not placing a page would currently stall the
VM, as it cannot make forward progress.
Let's check the state via the RamDiscardManager (the state e.g.,
of virtio-mem is migrated during precopy) and avoid sending a request
that will never get answered. Place a fresh zero page instead to keep
the VM working. This is the same behavior that would happen
automatically without userfaultfd being active, when accessing virtual
memory regions without populated pages -- "populate on demand".
For now, there are valid cases (as documented in the virtio-mem spec) where
a VM might read discarded memory; in the future, we will disallow that.
Then, we might want to handle that case differently, e.g., warning the
user that the VM seems to be mis-behaving.
Reviewed-by: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2021-10-11 20:53:43 +03:00
|
|
|
/*
|
|
|
|
* Check if a host-page aligned page falls into a discarded range as managed by
|
|
|
|
* a RamDiscardManager responsible for the mapped memory region of the RAMBlock.
|
|
|
|
*
|
|
|
|
* Note: The result is only stable while migrating (precopy/postcopy).
|
|
|
|
*/
|
|
|
|
bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start)
|
|
|
|
{
|
|
|
|
if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
|
|
|
|
RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
|
|
|
|
MemoryRegionSection section = {
|
|
|
|
.mr = rb->mr,
|
|
|
|
.offset_within_region = start,
|
|
|
|
.size = int128_make64(qemu_ram_pagesize(rb)),
|
|
|
|
};
|
|
|
|
|
|
|
|
return !ram_discard_manager_is_populated(rdm, §ion);
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2019-06-03 09:50:46 +03:00
|
|
|
/* Called with RCU critical section */
|
2019-08-08 06:31:55 +03:00
|
|
|
static void ramblock_sync_dirty_bitmap(RAMState *rs, RAMBlock *rb)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2020-06-22 06:20:37 +03:00
|
|
|
uint64_t new_dirty_pages =
|
|
|
|
cpu_physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length);
|
|
|
|
|
|
|
|
rs->migration_dirty_pages += new_dirty_pages;
|
|
|
|
rs->num_dirty_pages_period += new_dirty_pages;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* ram_pagesize_summary: calculate all the pagesizes of a VM
|
|
|
|
*
|
|
|
|
* Returns a summary bitmap of the page sizes of all RAMBlocks
|
|
|
|
*
|
|
|
|
* For VMs with just normal pages this is equivalent to the host page
|
|
|
|
* size. If it's got some huge pages then it's the OR of all the
|
|
|
|
* different page sizes.
|
2017-02-24 21:28:29 +03:00
|
|
|
*/
|
|
|
|
uint64_t ram_pagesize_summary(void)
|
|
|
|
{
|
|
|
|
RAMBlock *block;
|
|
|
|
uint64_t summary = 0;
|
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
2017-02-24 21:28:29 +03:00
|
|
|
summary |= block->page_size;
|
|
|
|
}
|
|
|
|
|
|
|
|
return summary;
|
|
|
|
}
|
|
|
|
|
2019-01-11 09:37:30 +03:00
|
|
|
uint64_t ram_get_total_transferred_pages(void)
|
|
|
|
{
|
2023-04-19 19:19:45 +03:00
|
|
|
return stat64_get(&ram_counters.normal_pages) +
|
2023-04-19 19:16:05 +03:00
|
|
|
stat64_get(&ram_counters.zero_pages) +
|
2022-10-12 00:55:51 +03:00
|
|
|
compression_counters.pages + xbzrle_counters.pages;
|
2019-01-11 09:37:30 +03:00
|
|
|
}
|
|
|
|
|
2018-06-04 12:55:12 +03:00
|
|
|
static void migration_update_rates(RAMState *rs, int64_t end_time)
|
|
|
|
{
|
2018-09-03 12:26:42 +03:00
|
|
|
uint64_t page_count = rs->target_page_count - rs->target_page_count_prev;
|
2018-09-06 10:01:00 +03:00
|
|
|
double compressed_size;
|
2018-06-04 12:55:12 +03:00
|
|
|
|
|
|
|
/* calculate period counters */
|
|
|
|
ram_counters.dirty_pages_rate = rs->num_dirty_pages_period * 1000
|
|
|
|
/ (end_time - rs->time_last_bitmap_sync);
|
|
|
|
|
2018-09-03 12:26:42 +03:00
|
|
|
if (!page_count) {
|
2018-06-04 12:55:12 +03:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2023-03-02 00:20:13 +03:00
|
|
|
if (migrate_xbzrle()) {
|
2020-04-30 03:59:35 +03:00
|
|
|
double encoded_size, unencoded_size;
|
|
|
|
|
2018-06-04 12:55:12 +03:00
|
|
|
xbzrle_counters.cache_miss_rate = (double)(xbzrle_counters.cache_miss -
|
2018-09-03 12:26:42 +03:00
|
|
|
rs->xbzrle_cache_miss_prev) / page_count;
|
2018-06-04 12:55:12 +03:00
|
|
|
rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss;
|
2020-04-30 03:59:35 +03:00
|
|
|
unencoded_size = (xbzrle_counters.pages - rs->xbzrle_pages_prev) *
|
|
|
|
TARGET_PAGE_SIZE;
|
|
|
|
encoded_size = xbzrle_counters.bytes - rs->xbzrle_bytes_prev;
|
2020-06-17 23:13:05 +03:00
|
|
|
if (xbzrle_counters.pages == rs->xbzrle_pages_prev || !encoded_size) {
|
2020-04-30 03:59:35 +03:00
|
|
|
xbzrle_counters.encoding_rate = 0;
|
|
|
|
} else {
|
|
|
|
xbzrle_counters.encoding_rate = unencoded_size / encoded_size;
|
|
|
|
}
|
|
|
|
rs->xbzrle_pages_prev = xbzrle_counters.pages;
|
|
|
|
rs->xbzrle_bytes_prev = xbzrle_counters.bytes;
|
2018-06-04 12:55:12 +03:00
|
|
|
}
|
2018-09-06 10:01:00 +03:00
|
|
|
|
2023-03-02 00:03:48 +03:00
|
|
|
if (migrate_compress()) {
|
2018-09-06 10:01:00 +03:00
|
|
|
compression_counters.busy_rate = (double)(compression_counters.busy -
|
|
|
|
rs->compress_thread_busy_prev) / page_count;
|
|
|
|
rs->compress_thread_busy_prev = compression_counters.busy;
|
|
|
|
|
|
|
|
compressed_size = compression_counters.compressed_size -
|
|
|
|
rs->compressed_size_prev;
|
|
|
|
if (compressed_size) {
|
|
|
|
double uncompressed_size = (compression_counters.pages -
|
|
|
|
rs->compress_pages_prev) * TARGET_PAGE_SIZE;
|
|
|
|
|
|
|
|
/* Compression-Ratio = Uncompressed-size / Compressed-size */
|
|
|
|
compression_counters.compression_rate =
|
|
|
|
uncompressed_size / compressed_size;
|
|
|
|
|
|
|
|
rs->compress_pages_prev = compression_counters.pages;
|
|
|
|
rs->compressed_size_prev = compression_counters.compressed_size;
|
|
|
|
}
|
|
|
|
}
|
2018-06-04 12:55:12 +03:00
|
|
|
}
|
|
|
|
|
2020-02-24 05:31:42 +03:00
|
|
|
static void migration_trigger_throttle(RAMState *rs)
|
|
|
|
{
|
|
|
|
MigrationState *s = migrate_get_current();
|
|
|
|
uint64_t threshold = s->parameters.throttle_trigger_threshold;
|
2022-10-12 00:55:51 +03:00
|
|
|
uint64_t bytes_xfer_period =
|
2023-02-22 17:56:45 +03:00
|
|
|
stat64_get(&ram_counters.transferred) - rs->bytes_xfer_prev;
|
2020-02-24 05:31:42 +03:00
|
|
|
uint64_t bytes_dirty_period = rs->num_dirty_pages_period * TARGET_PAGE_SIZE;
|
|
|
|
uint64_t bytes_dirty_threshold = bytes_xfer_period * threshold / 100;
|
|
|
|
|
|
|
|
/* During block migration the auto-converge logic incorrectly detects
|
|
|
|
* that ram migration makes no progress. Avoid this by disabling the
|
|
|
|
* throttling logic during the bulk phase of block migration. */
|
|
|
|
if (migrate_auto_converge() && !blk_mig_bulk_active()) {
|
|
|
|
/* The following detection logic can be refined later. For now:
|
|
|
|
Check to see if the ratio between dirtied bytes and the approx.
|
|
|
|
amount of bytes that just got transferred since the last time
|
|
|
|
we were in this routine reaches the threshold. If that happens
|
|
|
|
twice, start or increase throttling. */
|
|
|
|
|
|
|
|
if ((bytes_dirty_period > bytes_dirty_threshold) &&
|
|
|
|
(++rs->dirty_rate_high_cnt >= 2)) {
|
|
|
|
trace_migration_throttle();
|
|
|
|
rs->dirty_rate_high_cnt = 0;
|
2020-04-13 13:15:08 +03:00
|
|
|
mig_throttle_guest_down(bytes_dirty_period,
|
|
|
|
bytes_dirty_threshold);
|
2020-02-24 05:31:42 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-13 21:35:50 +03:00
|
|
|
static void migration_bitmap_sync(RAMState *rs)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
|
|
|
RAMBlock *block;
|
|
|
|
int64_t end_time;
|
|
|
|
|
2023-04-11 19:02:34 +03:00
|
|
|
stat64_add(&ram_counters.dirty_sync_count, 1);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2017-03-13 21:44:57 +03:00
|
|
|
if (!rs->time_last_bitmap_sync) {
|
|
|
|
rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
trace_migration_bitmap_sync_start();
|
2016-09-22 17:08:31 +03:00
|
|
|
memory_global_dirty_log_sync();
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2017-03-13 23:38:09 +03:00
|
|
|
qemu_mutex_lock(&rs->bitmap_mutex);
|
2019-10-07 17:36:39 +03:00
|
|
|
WITH_RCU_READ_LOCK_GUARD() {
|
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
|
|
|
ramblock_sync_dirty_bitmap(rs, block);
|
|
|
|
}
|
|
|
|
ram_counters.remaining = ram_bytes_remaining();
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
2017-03-13 23:38:09 +03:00
|
|
|
qemu_mutex_unlock(&rs->bitmap_mutex);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2018-02-06 20:37:39 +03:00
|
|
|
memory_global_after_dirty_log_sync();
|
2017-03-28 16:02:43 +03:00
|
|
|
trace_migration_bitmap_sync_end(rs->num_dirty_pages_period);
|
2017-03-14 04:55:07 +03:00
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
|
|
|
|
|
|
|
|
/* more than 1 second = 1000 millisecons */
|
2017-03-13 21:44:57 +03:00
|
|
|
if (end_time > rs->time_last_bitmap_sync + 1000) {
|
2020-02-24 05:31:42 +03:00
|
|
|
migration_trigger_throttle(rs);
|
2015-09-08 20:12:35 +03:00
|
|
|
|
2018-06-04 12:55:12 +03:00
|
|
|
migration_update_rates(rs, end_time);
|
|
|
|
|
2018-09-03 12:26:42 +03:00
|
|
|
rs->target_page_count_prev = rs->target_page_count;
|
2017-05-24 19:10:01 +03:00
|
|
|
|
|
|
|
/* reset period counters */
|
2017-03-13 21:44:57 +03:00
|
|
|
rs->time_last_bitmap_sync = end_time;
|
2017-03-28 16:02:43 +03:00
|
|
|
rs->num_dirty_pages_period = 0;
|
2023-02-22 17:56:45 +03:00
|
|
|
rs->bytes_xfer_prev = stat64_get(&ram_counters.transferred);
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
2023-03-02 00:08:09 +03:00
|
|
|
if (migrate_events()) {
|
2023-04-11 19:02:34 +03:00
|
|
|
uint64_t generation = stat64_get(&ram_counters.dirty_sync_count);
|
|
|
|
qapi_event_send_migration_pass(generation);
|
2015-12-16 14:47:36 +03:00
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2018-12-11 11:24:51 +03:00
|
|
|
static void migration_bitmap_sync_precopy(RAMState *rs)
|
|
|
|
{
|
|
|
|
Error *local_err = NULL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The current notifier usage is just an optimization to migration, so we
|
|
|
|
* don't stop the normal migration process in the error case.
|
|
|
|
*/
|
|
|
|
if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC, &local_err)) {
|
|
|
|
error_report_err(local_err);
|
2020-03-24 18:36:29 +03:00
|
|
|
local_err = NULL;
|
2018-12-11 11:24:51 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
migration_bitmap_sync(rs);
|
|
|
|
|
|
|
|
if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) {
|
|
|
|
error_report_err(local_err);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-16 12:19:38 +03:00
|
|
|
void ram_release_page(const char *rbname, uint64_t offset)
|
2021-12-16 11:58:49 +03:00
|
|
|
{
|
|
|
|
if (!migrate_release_ram() || !migration_in_postcopy()) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
ram_discard_range(rbname, offset, TARGET_PAGE_SIZE);
|
|
|
|
}
|
|
|
|
|
2018-08-21 11:10:22 +03:00
|
|
|
/**
|
|
|
|
* save_zero_page_to_file: send the zero page to the file
|
|
|
|
*
|
|
|
|
* Returns the size of data written to the file, 0 means the page is not
|
|
|
|
* a zero page
|
|
|
|
*
|
2022-10-12 00:55:56 +03:00
|
|
|
* @pss: current PSS channel
|
2018-08-21 11:10:22 +03:00
|
|
|
* @block: block that contains the page we want to send
|
|
|
|
* @offset: offset inside the block for the page
|
|
|
|
*/
|
2023-04-02 20:06:32 +03:00
|
|
|
static int save_zero_page_to_file(PageSearchStatus *pss, QEMUFile *file,
|
2018-08-21 11:10:22 +03:00
|
|
|
RAMBlock *block, ram_addr_t offset)
|
|
|
|
{
|
|
|
|
uint8_t *p = block->host + offset;
|
|
|
|
int len = 0;
|
|
|
|
|
2021-11-18 17:56:38 +03:00
|
|
|
if (buffer_is_zero(p, TARGET_PAGE_SIZE)) {
|
2023-04-02 20:06:32 +03:00
|
|
|
len += save_page_header(pss, file, block, offset | RAM_SAVE_FLAG_ZERO);
|
2018-08-21 11:10:22 +03:00
|
|
|
qemu_put_byte(file, 0);
|
|
|
|
len += 1;
|
2021-12-16 11:58:49 +03:00
|
|
|
ram_release_page(block->idstr, offset);
|
2018-08-21 11:10:22 +03:00
|
|
|
}
|
|
|
|
return len;
|
|
|
|
}
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
/**
|
2017-03-23 17:06:39 +03:00
|
|
|
* save_zero_page: send the zero page to the stream
|
2015-05-07 20:33:31 +03:00
|
|
|
*
|
2017-03-23 17:06:39 +03:00
|
|
|
* Returns the number of pages written.
|
2015-05-07 20:33:31 +03:00
|
|
|
*
|
2022-10-12 00:55:56 +03:00
|
|
|
* @pss: current PSS channel
|
2015-05-07 20:33:31 +03:00
|
|
|
* @block: block that contains the page we want to send
|
|
|
|
* @offset: offset inside the block for the page
|
|
|
|
*/
|
2023-04-02 20:06:32 +03:00
|
|
|
static int save_zero_page(PageSearchStatus *pss, QEMUFile *f, RAMBlock *block,
|
2022-10-12 00:55:53 +03:00
|
|
|
ram_addr_t offset)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2023-04-02 20:06:32 +03:00
|
|
|
int len = save_zero_page_to_file(pss, f, block, offset);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2018-08-21 11:10:22 +03:00
|
|
|
if (len) {
|
2023-04-19 19:16:05 +03:00
|
|
|
stat64_add(&ram_counters.zero_pages, 1);
|
2021-12-21 12:34:40 +03:00
|
|
|
ram_transferred_add(len);
|
2018-08-21 11:10:22 +03:00
|
|
|
return 1;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
2018-08-21 11:10:22 +03:00
|
|
|
return -1;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2018-03-30 10:51:23 +03:00
|
|
|
/*
|
|
|
|
* @pages: the number of pages written by the control path,
|
|
|
|
* < 0 - error
|
|
|
|
* > 0 - number of pages written
|
|
|
|
*
|
|
|
|
* Return true if the pages has been saved, otherwise false is returned.
|
|
|
|
*/
|
2022-10-12 00:55:53 +03:00
|
|
|
static bool control_save_page(PageSearchStatus *pss, RAMBlock *block,
|
|
|
|
ram_addr_t offset, int *pages)
|
2018-03-30 10:51:23 +03:00
|
|
|
{
|
|
|
|
uint64_t bytes_xmit = 0;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
*pages = -1;
|
2022-10-12 00:55:53 +03:00
|
|
|
ret = ram_control_save_page(pss->pss_channel, block->offset, offset,
|
|
|
|
TARGET_PAGE_SIZE, &bytes_xmit);
|
2018-03-30 10:51:23 +03:00
|
|
|
if (ret == RAM_SAVE_CONTROL_NOT_SUPP) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (bytes_xmit) {
|
2021-12-21 12:34:40 +03:00
|
|
|
ram_transferred_add(bytes_xmit);
|
2018-03-30 10:51:23 +03:00
|
|
|
*pages = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ret == RAM_SAVE_CONTROL_DELAYED) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (bytes_xmit > 0) {
|
2023-04-19 19:19:45 +03:00
|
|
|
stat64_add(&ram_counters.normal_pages, 1);
|
2018-03-30 10:51:23 +03:00
|
|
|
} else if (bytes_xmit == 0) {
|
2023-04-19 19:16:05 +03:00
|
|
|
stat64_add(&ram_counters.zero_pages, 1);
|
2018-03-30 10:51:23 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2018-03-30 10:51:27 +03:00
|
|
|
/*
|
|
|
|
* directly send the page to the stream
|
|
|
|
*
|
|
|
|
* Returns the number of pages written.
|
|
|
|
*
|
2022-10-12 00:55:56 +03:00
|
|
|
* @pss: current PSS channel
|
2018-03-30 10:51:27 +03:00
|
|
|
* @block: block that contains the page we want to send
|
|
|
|
* @offset: offset inside the block for the page
|
|
|
|
* @buf: the page to be sent
|
|
|
|
* @async: send to page asyncly
|
|
|
|
*/
|
2022-10-12 00:55:56 +03:00
|
|
|
static int save_normal_page(PageSearchStatus *pss, RAMBlock *block,
|
2022-10-12 00:55:53 +03:00
|
|
|
ram_addr_t offset, uint8_t *buf, bool async)
|
2018-03-30 10:51:27 +03:00
|
|
|
{
|
2022-10-12 00:55:56 +03:00
|
|
|
QEMUFile *file = pss->pss_channel;
|
|
|
|
|
2023-04-02 20:06:32 +03:00
|
|
|
ram_transferred_add(save_page_header(pss, pss->pss_channel, block,
|
2021-12-21 12:34:40 +03:00
|
|
|
offset | RAM_SAVE_FLAG_PAGE));
|
2018-03-30 10:51:27 +03:00
|
|
|
if (async) {
|
2022-10-12 00:55:53 +03:00
|
|
|
qemu_put_buffer_async(file, buf, TARGET_PAGE_SIZE,
|
2022-04-06 13:25:15 +03:00
|
|
|
migrate_release_ram() &&
|
2018-03-30 10:51:27 +03:00
|
|
|
migration_in_postcopy());
|
|
|
|
} else {
|
2022-10-12 00:55:53 +03:00
|
|
|
qemu_put_buffer(file, buf, TARGET_PAGE_SIZE);
|
2018-03-30 10:51:27 +03:00
|
|
|
}
|
2021-12-21 12:34:40 +03:00
|
|
|
ram_transferred_add(TARGET_PAGE_SIZE);
|
2023-04-19 19:19:45 +03:00
|
|
|
stat64_add(&ram_counters.normal_pages, 1);
|
2018-03-30 10:51:27 +03:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
/**
|
2017-03-23 17:06:39 +03:00
|
|
|
* ram_save_page: send the given page to the stream
|
2015-05-07 20:33:31 +03:00
|
|
|
*
|
2017-03-23 17:06:39 +03:00
|
|
|
* Returns the number of pages written.
|
2015-12-10 19:31:46 +03:00
|
|
|
* < 0 - error
|
|
|
|
* >=0 - Number of pages written - this might legally be 0
|
|
|
|
* if xbzrle noticed the page was the same.
|
2015-05-07 20:33:31 +03:00
|
|
|
*
|
2017-03-13 21:26:29 +03:00
|
|
|
* @rs: current RAM state
|
2015-05-07 20:33:31 +03:00
|
|
|
* @block: block that contains the page we want to send
|
|
|
|
* @offset: offset inside the block for the page
|
|
|
|
*/
|
2021-12-15 21:01:21 +03:00
|
|
|
static int ram_save_page(RAMState *rs, PageSearchStatus *pss)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
|
|
|
int pages = -1;
|
|
|
|
uint8_t *p;
|
|
|
|
bool send_async = true;
|
2016-01-15 06:37:44 +03:00
|
|
|
RAMBlock *block = pss->block;
|
2020-01-10 16:51:34 +03:00
|
|
|
ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
|
2018-03-30 10:51:23 +03:00
|
|
|
ram_addr_t current_addr = block->offset + offset;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2015-08-13 13:51:30 +03:00
|
|
|
p = block->host + offset;
|
2017-04-26 21:37:21 +03:00
|
|
|
trace_ram_save_page(block->idstr, (uint64_t)offset, p);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
|
|
|
XBZRLE_cache_lock();
|
migrate/ram: remove "ram_bulk_stage" and "fpo_enabled"
The bulk stage is kind of weird: migration_bitmap_find_dirty() will
indicate a dirty page, however, ram_save_host_page() will never save it, as
migration_bitmap_clear_dirty() detects that it is not dirty.
We already fill the bitmap in ram_list_init_bitmaps() with ones, marking
everything dirty - it didn't used to be that way, which is why we needed
an explicit first bulk stage.
Let's simplify: make the bitmap the single source of thuth. Explicitly
handle the "xbzrle_enabled after first round" case.
Regarding XBZRLE (implicitly handled via "ram_bulk_stage = false" right
now), there is now a slight change in behavior:
- Colo: When starting, it will be disabled (was implicitly enabled)
until the first round actually finishes.
- Free page hinting: When starting, XBZRLE will be disabled (was implicitly
enabled) until the first round actually finished.
- Snapshots: When starting, XBZRLE will be disabled. We essentially only
do a single run, so I guess it will never actually get disabled.
Postcopy seems to indirectly disable it in ram_save_page(), so there
shouldn't be really any change.
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Juan Quintela <quintela@redhat.com>
Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
Cc: Andrey Gruzdev <andrey.gruzdev@virtuozzo.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Message-Id: <20210216105039.40680-1-david@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2021-02-16 13:50:39 +03:00
|
|
|
if (rs->xbzrle_enabled && !migration_in_postcopy()) {
|
2022-10-12 00:55:56 +03:00
|
|
|
pages = save_xbzrle_page(rs, pss, &p, current_addr,
|
2022-10-12 00:55:53 +03:00
|
|
|
block, offset);
|
2021-12-15 21:01:21 +03:00
|
|
|
if (!rs->last_stage) {
|
2018-03-30 10:51:23 +03:00
|
|
|
/* Can't send this cached data async, since the cache page
|
|
|
|
* might get updated before it gets to the wire
|
2015-05-07 20:33:31 +03:00
|
|
|
*/
|
2018-03-30 10:51:23 +03:00
|
|
|
send_async = false;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* XBZRLE overflow or normal page */
|
|
|
|
if (pages == -1) {
|
2022-10-12 00:55:56 +03:00
|
|
|
pages = save_normal_page(pss, block, offset, p, send_async);
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
XBZRLE_cache_unlock();
|
|
|
|
|
|
|
|
return pages;
|
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:53 +03:00
|
|
|
static int ram_save_multifd_page(QEMUFile *file, RAMBlock *block,
|
2016-01-15 13:40:13 +03:00
|
|
|
ram_addr_t offset)
|
|
|
|
{
|
2022-10-12 00:55:53 +03:00
|
|
|
if (multifd_queue_page(file, block, offset) < 0) {
|
2019-06-25 16:18:17 +03:00
|
|
|
return -1;
|
|
|
|
}
|
2023-04-19 19:19:45 +03:00
|
|
|
stat64_add(&ram_counters.normal_pages, 1);
|
2016-01-15 13:40:13 +03:00
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2018-08-21 11:10:24 +03:00
|
|
|
static bool do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
|
2018-08-21 11:10:23 +03:00
|
|
|
ram_addr_t offset, uint8_t *source_buf)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2017-05-04 12:46:24 +03:00
|
|
|
RAMState *rs = ram_state;
|
2022-10-12 00:55:56 +03:00
|
|
|
PageSearchStatus *pss = &rs->pss[RAM_CHANNEL_PRECOPY];
|
2021-12-21 12:28:16 +03:00
|
|
|
uint8_t *p = block->host + offset;
|
2018-08-21 11:10:23 +03:00
|
|
|
int ret;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2023-04-02 20:06:32 +03:00
|
|
|
if (save_zero_page_to_file(pss, f, block, offset)) {
|
2021-12-16 11:39:49 +03:00
|
|
|
return true;
|
2018-08-21 11:10:24 +03:00
|
|
|
}
|
|
|
|
|
2023-04-02 20:06:32 +03:00
|
|
|
save_page_header(pss, f, block, offset | RAM_SAVE_FLAG_COMPRESS_PAGE);
|
2018-03-30 10:51:22 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* copy it to a internal buffer to avoid it being modified by VM
|
|
|
|
* so that we can catch up the error during compression and
|
|
|
|
* decompression
|
|
|
|
*/
|
|
|
|
memcpy(source_buf, p, TARGET_PAGE_SIZE);
|
2018-08-21 11:10:23 +03:00
|
|
|
ret = qemu_put_compression_data(f, stream, source_buf, TARGET_PAGE_SIZE);
|
|
|
|
if (ret < 0) {
|
|
|
|
qemu_file_set_error(migrate_get_current()->to_dst_file, ret);
|
2016-05-05 10:32:54 +03:00
|
|
|
error_report("compressed data failed!");
|
|
|
|
}
|
2021-12-16 11:39:49 +03:00
|
|
|
return false;
|
2018-08-21 11:10:24 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
update_compress_thread_counts(const CompressParam *param, int bytes_xmit)
|
|
|
|
{
|
2021-12-21 12:34:40 +03:00
|
|
|
ram_transferred_add(bytes_xmit);
|
2018-09-06 10:01:00 +03:00
|
|
|
|
2018-08-21 11:10:24 +03:00
|
|
|
if (param->zero_page) {
|
2023-04-19 19:16:05 +03:00
|
|
|
stat64_add(&ram_counters.zero_pages, 1);
|
2018-09-06 10:01:00 +03:00
|
|
|
return;
|
2018-08-21 11:10:24 +03:00
|
|
|
}
|
2018-09-06 10:01:00 +03:00
|
|
|
|
|
|
|
/* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */
|
|
|
|
compression_counters.compressed_size += bytes_xmit - 8;
|
|
|
|
compression_counters.pages++;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2018-09-06 10:01:01 +03:00
|
|
|
static bool save_page_use_compression(RAMState *rs);
|
|
|
|
|
2017-03-15 13:00:51 +03:00
|
|
|
static void flush_compressed_data(RAMState *rs)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2022-10-12 00:55:49 +03:00
|
|
|
MigrationState *ms = migrate_get_current();
|
2015-05-07 20:33:31 +03:00
|
|
|
int idx, len, thread_count;
|
|
|
|
|
2018-09-06 10:01:01 +03:00
|
|
|
if (!save_page_use_compression(rs)) {
|
2015-05-07 20:33:31 +03:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
thread_count = migrate_compress_threads();
|
2016-05-05 10:32:57 +03:00
|
|
|
|
2016-05-05 10:32:59 +03:00
|
|
|
qemu_mutex_lock(&comp_done_lock);
|
2015-05-07 20:33:31 +03:00
|
|
|
for (idx = 0; idx < thread_count; idx++) {
|
2016-05-05 10:32:57 +03:00
|
|
|
while (!comp_param[idx].done) {
|
2016-05-05 10:32:59 +03:00
|
|
|
qemu_cond_wait(&comp_done_cond, &comp_done_lock);
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
2016-05-05 10:32:57 +03:00
|
|
|
}
|
2016-05-05 10:32:59 +03:00
|
|
|
qemu_mutex_unlock(&comp_done_lock);
|
2016-05-05 10:32:57 +03:00
|
|
|
|
|
|
|
for (idx = 0; idx < thread_count; idx++) {
|
|
|
|
qemu_mutex_lock(&comp_param[idx].mutex);
|
2016-05-05 10:32:56 +03:00
|
|
|
if (!comp_param[idx].quit) {
|
2022-10-12 00:55:49 +03:00
|
|
|
len = qemu_put_qemu_file(ms->to_dst_file, comp_param[idx].file);
|
2018-08-21 11:10:24 +03:00
|
|
|
/*
|
|
|
|
* it's safe to fetch zero_page without holding comp_done_lock
|
|
|
|
* as there is no further request submitted to the thread,
|
|
|
|
* i.e, the thread should be waiting for a request at this point.
|
|
|
|
*/
|
|
|
|
update_compress_thread_counts(&comp_param[idx], len);
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
2016-05-05 10:32:57 +03:00
|
|
|
qemu_mutex_unlock(&comp_param[idx].mutex);
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void set_compress_params(CompressParam *param, RAMBlock *block,
|
|
|
|
ram_addr_t offset)
|
|
|
|
{
|
|
|
|
param->block = block;
|
|
|
|
param->offset = offset;
|
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:49 +03:00
|
|
|
static int compress_page_with_multi_thread(RAMBlock *block, ram_addr_t offset)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
|
|
|
int idx, thread_count, bytes_xmit = -1, pages = -1;
|
2018-08-21 11:10:20 +03:00
|
|
|
bool wait = migrate_compress_wait_thread();
|
2022-10-12 00:55:49 +03:00
|
|
|
MigrationState *ms = migrate_get_current();
|
2015-05-07 20:33:31 +03:00
|
|
|
|
|
|
|
thread_count = migrate_compress_threads();
|
2016-05-05 10:32:59 +03:00
|
|
|
qemu_mutex_lock(&comp_done_lock);
|
2018-08-21 11:10:20 +03:00
|
|
|
retry:
|
|
|
|
for (idx = 0; idx < thread_count; idx++) {
|
|
|
|
if (comp_param[idx].done) {
|
|
|
|
comp_param[idx].done = false;
|
2022-10-12 00:55:49 +03:00
|
|
|
bytes_xmit = qemu_put_qemu_file(ms->to_dst_file,
|
|
|
|
comp_param[idx].file);
|
2018-08-21 11:10:20 +03:00
|
|
|
qemu_mutex_lock(&comp_param[idx].mutex);
|
|
|
|
set_compress_params(&comp_param[idx], block, offset);
|
|
|
|
qemu_cond_signal(&comp_param[idx].cond);
|
|
|
|
qemu_mutex_unlock(&comp_param[idx].mutex);
|
|
|
|
pages = 1;
|
2018-08-21 11:10:24 +03:00
|
|
|
update_compress_thread_counts(&comp_param[idx], bytes_xmit);
|
2015-05-07 20:33:31 +03:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2018-08-21 11:10:20 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* wait for the free thread if the user specifies 'compress-wait-thread',
|
|
|
|
* otherwise we will post the page out in the main thread as normal page.
|
|
|
|
*/
|
|
|
|
if (pages < 0 && wait) {
|
|
|
|
qemu_cond_wait(&comp_done_cond, &comp_done_lock);
|
|
|
|
goto retry;
|
|
|
|
}
|
2016-05-05 10:32:59 +03:00
|
|
|
qemu_mutex_unlock(&comp_done_lock);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
|
|
|
return pages;
|
|
|
|
}
|
|
|
|
|
2022-06-21 14:29:36 +03:00
|
|
|
#define PAGE_ALL_CLEAN 0
|
|
|
|
#define PAGE_TRY_AGAIN 1
|
|
|
|
#define PAGE_DIRTY_FOUND 2
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* find_dirty_block: find the next dirty page and update any state
|
|
|
|
* associated with the search process.
|
2015-09-23 17:27:11 +03:00
|
|
|
*
|
2022-06-21 14:29:36 +03:00
|
|
|
* Returns:
|
|
|
|
* PAGE_ALL_CLEAN: no dirty page found, give up
|
|
|
|
* PAGE_TRY_AGAIN: no dirty page found, retry for next block
|
|
|
|
* PAGE_DIRTY_FOUND: dirty page found
|
2015-09-23 17:27:11 +03:00
|
|
|
*
|
2017-03-13 21:26:29 +03:00
|
|
|
* @rs: current RAM state
|
2017-03-23 17:06:39 +03:00
|
|
|
* @pss: data about the state of the current dirty page scan
|
|
|
|
* @again: set to false if the search has scanned the whole of RAM
|
2015-09-23 17:27:11 +03:00
|
|
|
*/
|
2022-06-21 14:29:36 +03:00
|
|
|
static int find_dirty_block(RAMState *rs, PageSearchStatus *pss)
|
2015-09-23 17:27:11 +03:00
|
|
|
{
|
2022-10-12 00:55:52 +03:00
|
|
|
/* Update pss->page for the next dirty bit in ramblock */
|
|
|
|
pss_find_next_dirty(pss);
|
|
|
|
|
2017-03-13 21:26:29 +03:00
|
|
|
if (pss->complete_round && pss->block == rs->last_seen_block &&
|
2017-03-21 17:36:51 +03:00
|
|
|
pss->page >= rs->last_page) {
|
2015-09-23 17:27:11 +03:00
|
|
|
/*
|
|
|
|
* We've been once around the RAM and haven't found anything.
|
|
|
|
* Give up.
|
|
|
|
*/
|
2022-06-21 14:29:36 +03:00
|
|
|
return PAGE_ALL_CLEAN;
|
2015-09-23 17:27:11 +03:00
|
|
|
}
|
2021-04-29 14:27:08 +03:00
|
|
|
if (!offset_in_ramblock(pss->block,
|
|
|
|
((ram_addr_t)pss->page) << TARGET_PAGE_BITS)) {
|
2015-09-23 17:27:11 +03:00
|
|
|
/* Didn't find anything in this RAM Block */
|
2017-03-21 17:36:51 +03:00
|
|
|
pss->page = 0;
|
2015-09-23 17:27:11 +03:00
|
|
|
pss->block = QLIST_NEXT_RCU(pss->block, next);
|
|
|
|
if (!pss->block) {
|
2018-09-06 10:00:59 +03:00
|
|
|
/*
|
|
|
|
* If memory migration starts over, we will meet a dirtied page
|
|
|
|
* which may still exists in compression threads's ring, so we
|
|
|
|
* should flush the compressed data to make sure the new page
|
|
|
|
* is not overwritten by the old one in the destination.
|
|
|
|
*
|
|
|
|
* Also If xbzrle is on, stop using the data compression at this
|
|
|
|
* point. In theory, xbzrle can do better than compression.
|
|
|
|
*/
|
|
|
|
flush_compressed_data(rs);
|
|
|
|
|
2015-09-23 17:27:11 +03:00
|
|
|
/* Hit the end of the list */
|
|
|
|
pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
|
|
|
|
/* Flag that we've looped */
|
|
|
|
pss->complete_round = true;
|
migrate/ram: remove "ram_bulk_stage" and "fpo_enabled"
The bulk stage is kind of weird: migration_bitmap_find_dirty() will
indicate a dirty page, however, ram_save_host_page() will never save it, as
migration_bitmap_clear_dirty() detects that it is not dirty.
We already fill the bitmap in ram_list_init_bitmaps() with ones, marking
everything dirty - it didn't used to be that way, which is why we needed
an explicit first bulk stage.
Let's simplify: make the bitmap the single source of thuth. Explicitly
handle the "xbzrle_enabled after first round" case.
Regarding XBZRLE (implicitly handled via "ram_bulk_stage = false" right
now), there is now a slight change in behavior:
- Colo: When starting, it will be disabled (was implicitly enabled)
until the first round actually finishes.
- Free page hinting: When starting, XBZRLE will be disabled (was implicitly
enabled) until the first round actually finished.
- Snapshots: When starting, XBZRLE will be disabled. We essentially only
do a single run, so I guess it will never actually get disabled.
Postcopy seems to indirectly disable it in ram_save_page(), so there
shouldn't be really any change.
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Juan Quintela <quintela@redhat.com>
Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
Cc: Andrey Gruzdev <andrey.gruzdev@virtuozzo.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Message-Id: <20210216105039.40680-1-david@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2021-02-16 13:50:39 +03:00
|
|
|
/* After the first round, enable XBZRLE. */
|
2023-03-02 00:20:13 +03:00
|
|
|
if (migrate_xbzrle()) {
|
migrate/ram: remove "ram_bulk_stage" and "fpo_enabled"
The bulk stage is kind of weird: migration_bitmap_find_dirty() will
indicate a dirty page, however, ram_save_host_page() will never save it, as
migration_bitmap_clear_dirty() detects that it is not dirty.
We already fill the bitmap in ram_list_init_bitmaps() with ones, marking
everything dirty - it didn't used to be that way, which is why we needed
an explicit first bulk stage.
Let's simplify: make the bitmap the single source of thuth. Explicitly
handle the "xbzrle_enabled after first round" case.
Regarding XBZRLE (implicitly handled via "ram_bulk_stage = false" right
now), there is now a slight change in behavior:
- Colo: When starting, it will be disabled (was implicitly enabled)
until the first round actually finishes.
- Free page hinting: When starting, XBZRLE will be disabled (was implicitly
enabled) until the first round actually finished.
- Snapshots: When starting, XBZRLE will be disabled. We essentially only
do a single run, so I guess it will never actually get disabled.
Postcopy seems to indirectly disable it in ram_save_page(), so there
shouldn't be really any change.
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Juan Quintela <quintela@redhat.com>
Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
Cc: Andrey Gruzdev <andrey.gruzdev@virtuozzo.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Message-Id: <20210216105039.40680-1-david@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2021-02-16 13:50:39 +03:00
|
|
|
rs->xbzrle_enabled = true;
|
|
|
|
}
|
2015-09-23 17:27:11 +03:00
|
|
|
}
|
|
|
|
/* Didn't find anything this time, but try again on the new block */
|
2022-06-21 14:29:36 +03:00
|
|
|
return PAGE_TRY_AGAIN;
|
2015-09-23 17:27:11 +03:00
|
|
|
} else {
|
2022-06-21 14:29:36 +03:00
|
|
|
/* We've found something */
|
|
|
|
return PAGE_DIRTY_FOUND;
|
2015-09-23 17:27:11 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* unqueue_page: gets a page of the queue
|
|
|
|
*
|
2015-11-05 21:11:09 +03:00
|
|
|
* Helper for 'get_queued_page' - gets a page off the queue
|
|
|
|
*
|
2017-03-23 17:06:39 +03:00
|
|
|
* Returns the block of the page (or NULL if none available)
|
|
|
|
*
|
2017-03-21 00:12:40 +03:00
|
|
|
* @rs: current RAM state
|
2017-03-23 17:06:39 +03:00
|
|
|
* @offset: used to return the offset within the RAMBlock
|
2015-11-05 21:11:09 +03:00
|
|
|
*/
|
2017-03-21 18:19:05 +03:00
|
|
|
static RAMBlock *unqueue_page(RAMState *rs, ram_addr_t *offset)
|
2015-11-05 21:11:09 +03:00
|
|
|
{
|
2022-01-19 11:09:18 +03:00
|
|
|
struct RAMSrcPageRequest *entry;
|
2015-11-05 21:11:09 +03:00
|
|
|
RAMBlock *block = NULL;
|
|
|
|
|
2022-01-19 11:09:18 +03:00
|
|
|
if (!postcopy_has_request(rs)) {
|
2018-08-21 11:10:25 +03:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2020-04-04 07:21:08 +03:00
|
|
|
QEMU_LOCK_GUARD(&rs->src_page_req_mutex);
|
2022-01-19 11:09:18 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* This should _never_ change even after we take the lock, because no one
|
|
|
|
* should be taking anything off the request list other than us.
|
|
|
|
*/
|
|
|
|
assert(postcopy_has_request(rs));
|
|
|
|
|
|
|
|
entry = QSIMPLEQ_FIRST(&rs->src_page_requests);
|
|
|
|
block = entry->rb;
|
|
|
|
*offset = entry->offset;
|
|
|
|
|
2022-08-02 09:19:49 +03:00
|
|
|
if (entry->len > TARGET_PAGE_SIZE) {
|
|
|
|
entry->len -= TARGET_PAGE_SIZE;
|
|
|
|
entry->offset += TARGET_PAGE_SIZE;
|
2022-01-19 11:09:18 +03:00
|
|
|
} else {
|
|
|
|
memory_region_unref(block->mr);
|
|
|
|
QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
|
|
|
|
g_free(entry);
|
|
|
|
migration_consume_urgent_request();
|
2015-11-05 21:11:09 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
return block;
|
|
|
|
}
|
|
|
|
|
2021-01-29 13:14:05 +03:00
|
|
|
#if defined(__linux__)
|
|
|
|
/**
|
|
|
|
* poll_fault_page: try to get next UFFD write fault page and, if pending fault
|
|
|
|
* is found, return RAM block pointer and page offset
|
|
|
|
*
|
|
|
|
* Returns pointer to the RAMBlock containing faulting page,
|
|
|
|
* NULL if no write faults are pending
|
|
|
|
*
|
|
|
|
* @rs: current RAM state
|
|
|
|
* @offset: page offset from the beginning of the block
|
|
|
|
*/
|
|
|
|
static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset)
|
|
|
|
{
|
|
|
|
struct uffd_msg uffd_msg;
|
|
|
|
void *page_address;
|
2021-04-01 12:22:26 +03:00
|
|
|
RAMBlock *block;
|
2021-01-29 13:14:05 +03:00
|
|
|
int res;
|
|
|
|
|
|
|
|
if (!migrate_background_snapshot()) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
res = uffd_read_events(rs->uffdio_fd, &uffd_msg, 1);
|
|
|
|
if (res <= 0) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
page_address = (void *)(uintptr_t) uffd_msg.arg.pagefault.address;
|
2021-04-01 12:22:26 +03:00
|
|
|
block = qemu_ram_block_from_host(page_address, false, offset);
|
|
|
|
assert(block && (block->flags & RAM_UF_WRITEPROTECT) != 0);
|
|
|
|
return block;
|
2021-01-29 13:14:05 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* ram_save_release_protection: release UFFD write protection after
|
|
|
|
* a range of pages has been saved
|
|
|
|
*
|
|
|
|
* @rs: current RAM state
|
|
|
|
* @pss: page-search-status structure
|
|
|
|
* @start_page: index of the first page in the range relative to pss->block
|
|
|
|
*
|
|
|
|
* Returns 0 on success, negative value in case of an error
|
|
|
|
*/
|
|
|
|
static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss,
|
|
|
|
unsigned long start_page)
|
|
|
|
{
|
|
|
|
int res = 0;
|
|
|
|
|
|
|
|
/* Check if page is from UFFD-managed region. */
|
|
|
|
if (pss->block->flags & RAM_UF_WRITEPROTECT) {
|
|
|
|
void *page_address = pss->block->host + (start_page << TARGET_PAGE_BITS);
|
2022-01-19 11:09:15 +03:00
|
|
|
uint64_t run_length = (pss->page - start_page) << TARGET_PAGE_BITS;
|
2021-01-29 13:14:05 +03:00
|
|
|
|
|
|
|
/* Flush async buffers before un-protect. */
|
2022-10-12 00:55:53 +03:00
|
|
|
qemu_fflush(pss->pss_channel);
|
2021-01-29 13:14:05 +03:00
|
|
|
/* Un-protect memory range. */
|
|
|
|
res = uffd_change_protection(rs->uffdio_fd, page_address, run_length,
|
|
|
|
false, false);
|
|
|
|
}
|
|
|
|
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* ram_write_tracking_available: check if kernel supports required UFFD features
|
|
|
|
*
|
|
|
|
* Returns true if supports, false otherwise
|
|
|
|
*/
|
|
|
|
bool ram_write_tracking_available(void)
|
|
|
|
{
|
|
|
|
uint64_t uffd_features;
|
|
|
|
int res;
|
|
|
|
|
|
|
|
res = uffd_query_features(&uffd_features);
|
|
|
|
return (res == 0 &&
|
|
|
|
(uffd_features & UFFD_FEATURE_PAGEFAULT_FLAG_WP) != 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* ram_write_tracking_compatible: check if guest configuration is
|
|
|
|
* compatible with 'write-tracking'
|
|
|
|
*
|
|
|
|
* Returns true if compatible, false otherwise
|
|
|
|
*/
|
|
|
|
bool ram_write_tracking_compatible(void)
|
|
|
|
{
|
|
|
|
const uint64_t uffd_ioctls_mask = BIT(_UFFDIO_WRITEPROTECT);
|
|
|
|
int uffd_fd;
|
2021-04-01 12:22:26 +03:00
|
|
|
RAMBlock *block;
|
2021-01-29 13:14:05 +03:00
|
|
|
bool ret = false;
|
|
|
|
|
|
|
|
/* Open UFFD file descriptor */
|
|
|
|
uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, false);
|
|
|
|
if (uffd_fd < 0) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
RCU_READ_LOCK_GUARD();
|
|
|
|
|
2021-04-01 12:22:26 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
2021-01-29 13:14:05 +03:00
|
|
|
uint64_t uffd_ioctls;
|
|
|
|
|
|
|
|
/* Nothing to do with read-only and MMIO-writable regions */
|
2021-04-01 12:22:26 +03:00
|
|
|
if (block->mr->readonly || block->mr->rom_device) {
|
2021-01-29 13:14:05 +03:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
/* Try to register block memory via UFFD-IO to track writes */
|
2021-04-01 12:22:26 +03:00
|
|
|
if (uffd_register_memory(uffd_fd, block->host, block->max_length,
|
2021-01-29 13:14:05 +03:00
|
|
|
UFFDIO_REGISTER_MODE_WP, &uffd_ioctls)) {
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
if ((uffd_ioctls & uffd_ioctls_mask) != uffd_ioctls_mask) {
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ret = true;
|
|
|
|
|
|
|
|
out:
|
|
|
|
uffd_close_fd(uffd_fd);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2021-10-11 20:53:45 +03:00
|
|
|
static inline void populate_read_range(RAMBlock *block, ram_addr_t offset,
|
|
|
|
ram_addr_t size)
|
|
|
|
{
|
2023-01-05 15:45:24 +03:00
|
|
|
const ram_addr_t end = offset + size;
|
|
|
|
|
2021-10-11 20:53:45 +03:00
|
|
|
/*
|
|
|
|
* We read one byte of each page; this will preallocate page tables if
|
|
|
|
* required and populate the shared zeropage on MAP_PRIVATE anonymous memory
|
|
|
|
* where no page was populated yet. This might require adaption when
|
|
|
|
* supporting other mappings, like shmem.
|
|
|
|
*/
|
2023-01-05 15:45:24 +03:00
|
|
|
for (; offset < end; offset += block->page_size) {
|
2021-10-11 20:53:45 +03:00
|
|
|
char tmp = *((char *)block->host + offset);
|
|
|
|
|
|
|
|
/* Don't optimize the read out */
|
|
|
|
asm volatile("" : "+r" (tmp));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-10-11 20:53:46 +03:00
|
|
|
static inline int populate_read_section(MemoryRegionSection *section,
|
|
|
|
void *opaque)
|
|
|
|
{
|
|
|
|
const hwaddr size = int128_get64(section->size);
|
|
|
|
hwaddr offset = section->offset_within_region;
|
|
|
|
RAMBlock *block = section->mr->ram_block;
|
|
|
|
|
|
|
|
populate_read_range(block, offset, size);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2021-04-01 12:22:25 +03:00
|
|
|
/*
|
2021-10-11 20:53:45 +03:00
|
|
|
* ram_block_populate_read: preallocate page tables and populate pages in the
|
|
|
|
* RAM block by reading a byte of each page.
|
2021-04-01 12:22:25 +03:00
|
|
|
*
|
|
|
|
* Since it's solely used for userfault_fd WP feature, here we just
|
|
|
|
* hardcode page size to qemu_real_host_page_size.
|
|
|
|
*
|
2021-04-01 12:22:26 +03:00
|
|
|
* @block: RAM block to populate
|
2021-04-01 12:22:25 +03:00
|
|
|
*/
|
2021-10-11 20:53:46 +03:00
|
|
|
static void ram_block_populate_read(RAMBlock *rb)
|
2021-04-01 12:22:25 +03:00
|
|
|
{
|
2021-10-11 20:53:46 +03:00
|
|
|
/*
|
|
|
|
* Skip populating all pages that fall into a discarded range as managed by
|
|
|
|
* a RamDiscardManager responsible for the mapped memory region of the
|
|
|
|
* RAMBlock. Such discarded ("logically unplugged") parts of a RAMBlock
|
|
|
|
* must not get populated automatically. We don't have to track
|
|
|
|
* modifications via userfaultfd WP reliably, because these pages will
|
|
|
|
* not be part of the migration stream either way -- see
|
|
|
|
* ramblock_dirty_bitmap_exclude_discarded_pages().
|
|
|
|
*
|
|
|
|
* Note: The result is only stable while migrating (precopy/postcopy).
|
|
|
|
*/
|
|
|
|
if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
|
|
|
|
RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
|
|
|
|
MemoryRegionSection section = {
|
|
|
|
.mr = rb->mr,
|
|
|
|
.offset_within_region = 0,
|
|
|
|
.size = rb->mr->size,
|
|
|
|
};
|
|
|
|
|
|
|
|
ram_discard_manager_replay_populated(rdm, §ion,
|
|
|
|
populate_read_section, NULL);
|
|
|
|
} else {
|
|
|
|
populate_read_range(rb, 0, rb->used_length);
|
|
|
|
}
|
2021-04-01 12:22:25 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* ram_write_tracking_prepare: prepare for UFFD-WP memory tracking
|
|
|
|
*/
|
|
|
|
void ram_write_tracking_prepare(void)
|
|
|
|
{
|
2021-04-01 12:22:26 +03:00
|
|
|
RAMBlock *block;
|
2021-04-01 12:22:25 +03:00
|
|
|
|
|
|
|
RCU_READ_LOCK_GUARD();
|
|
|
|
|
2021-04-01 12:22:26 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
2021-04-01 12:22:25 +03:00
|
|
|
/* Nothing to do with read-only and MMIO-writable regions */
|
2021-04-01 12:22:26 +03:00
|
|
|
if (block->mr->readonly || block->mr->rom_device) {
|
2021-04-01 12:22:25 +03:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Populate pages of the RAM block before enabling userfault_fd
|
|
|
|
* write protection.
|
|
|
|
*
|
|
|
|
* This stage is required since ioctl(UFFDIO_WRITEPROTECT) with
|
|
|
|
* UFFDIO_WRITEPROTECT_MODE_WP mode setting would silently skip
|
|
|
|
* pages with pte_none() entries in page table.
|
|
|
|
*/
|
2021-10-11 20:53:45 +03:00
|
|
|
ram_block_populate_read(block);
|
2021-04-01 12:22:25 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-01-05 15:45:28 +03:00
|
|
|
static inline int uffd_protect_section(MemoryRegionSection *section,
|
|
|
|
void *opaque)
|
|
|
|
{
|
|
|
|
const hwaddr size = int128_get64(section->size);
|
|
|
|
const hwaddr offset = section->offset_within_region;
|
|
|
|
RAMBlock *rb = section->mr->ram_block;
|
|
|
|
int uffd_fd = (uintptr_t)opaque;
|
|
|
|
|
|
|
|
return uffd_change_protection(uffd_fd, rb->host + offset, size, true,
|
|
|
|
false);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int ram_block_uffd_protect(RAMBlock *rb, int uffd_fd)
|
|
|
|
{
|
|
|
|
assert(rb->flags & RAM_UF_WRITEPROTECT);
|
|
|
|
|
|
|
|
/* See ram_block_populate_read() */
|
|
|
|
if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
|
|
|
|
RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
|
|
|
|
MemoryRegionSection section = {
|
|
|
|
.mr = rb->mr,
|
|
|
|
.offset_within_region = 0,
|
|
|
|
.size = rb->mr->size,
|
|
|
|
};
|
|
|
|
|
|
|
|
return ram_discard_manager_replay_populated(rdm, §ion,
|
|
|
|
uffd_protect_section,
|
|
|
|
(void *)(uintptr_t)uffd_fd);
|
|
|
|
}
|
|
|
|
return uffd_change_protection(uffd_fd, rb->host,
|
|
|
|
rb->used_length, true, false);
|
|
|
|
}
|
|
|
|
|
2021-01-29 13:14:05 +03:00
|
|
|
/*
|
|
|
|
* ram_write_tracking_start: start UFFD-WP memory tracking
|
|
|
|
*
|
|
|
|
* Returns 0 for success or negative value in case of error
|
|
|
|
*/
|
|
|
|
int ram_write_tracking_start(void)
|
|
|
|
{
|
|
|
|
int uffd_fd;
|
|
|
|
RAMState *rs = ram_state;
|
2021-04-01 12:22:26 +03:00
|
|
|
RAMBlock *block;
|
2021-01-29 13:14:05 +03:00
|
|
|
|
|
|
|
/* Open UFFD file descriptor */
|
|
|
|
uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, true);
|
|
|
|
if (uffd_fd < 0) {
|
|
|
|
return uffd_fd;
|
|
|
|
}
|
|
|
|
rs->uffdio_fd = uffd_fd;
|
|
|
|
|
|
|
|
RCU_READ_LOCK_GUARD();
|
|
|
|
|
2021-04-01 12:22:26 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
2021-01-29 13:14:05 +03:00
|
|
|
/* Nothing to do with read-only and MMIO-writable regions */
|
2021-04-01 12:22:26 +03:00
|
|
|
if (block->mr->readonly || block->mr->rom_device) {
|
2021-01-29 13:14:05 +03:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Register block memory with UFFD to track writes */
|
2021-04-01 12:22:26 +03:00
|
|
|
if (uffd_register_memory(rs->uffdio_fd, block->host,
|
|
|
|
block->max_length, UFFDIO_REGISTER_MODE_WP, NULL)) {
|
2021-01-29 13:14:05 +03:00
|
|
|
goto fail;
|
|
|
|
}
|
2023-01-05 15:45:25 +03:00
|
|
|
block->flags |= RAM_UF_WRITEPROTECT;
|
|
|
|
memory_region_ref(block->mr);
|
|
|
|
|
2021-01-29 13:14:05 +03:00
|
|
|
/* Apply UFFD write protection to the block memory range */
|
2023-01-05 15:45:28 +03:00
|
|
|
if (ram_block_uffd_protect(block, uffd_fd)) {
|
2021-01-29 13:14:05 +03:00
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
2021-04-01 12:22:26 +03:00
|
|
|
trace_ram_write_tracking_ramblock_start(block->idstr, block->page_size,
|
|
|
|
block->host, block->max_length);
|
2021-01-29 13:14:05 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
fail:
|
|
|
|
error_report("ram_write_tracking_start() failed: restoring initial memory state");
|
|
|
|
|
2021-04-01 12:22:26 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
|
|
|
if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
|
2021-01-29 13:14:05 +03:00
|
|
|
continue;
|
|
|
|
}
|
2021-04-01 12:22:26 +03:00
|
|
|
uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
|
2021-01-29 13:14:05 +03:00
|
|
|
/* Cleanup flags and remove reference */
|
2021-04-01 12:22:26 +03:00
|
|
|
block->flags &= ~RAM_UF_WRITEPROTECT;
|
|
|
|
memory_region_unref(block->mr);
|
2021-01-29 13:14:05 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
uffd_close_fd(uffd_fd);
|
|
|
|
rs->uffdio_fd = -1;
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* ram_write_tracking_stop: stop UFFD-WP memory tracking and remove protection
|
|
|
|
*/
|
|
|
|
void ram_write_tracking_stop(void)
|
|
|
|
{
|
|
|
|
RAMState *rs = ram_state;
|
2021-04-01 12:22:26 +03:00
|
|
|
RAMBlock *block;
|
2021-01-29 13:14:05 +03:00
|
|
|
|
|
|
|
RCU_READ_LOCK_GUARD();
|
|
|
|
|
2021-04-01 12:22:26 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
|
|
|
if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
|
2021-01-29 13:14:05 +03:00
|
|
|
continue;
|
|
|
|
}
|
2021-04-01 12:22:26 +03:00
|
|
|
uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
|
2021-01-29 13:14:05 +03:00
|
|
|
|
2021-04-01 12:22:26 +03:00
|
|
|
trace_ram_write_tracking_ramblock_stop(block->idstr, block->page_size,
|
|
|
|
block->host, block->max_length);
|
2021-01-29 13:14:05 +03:00
|
|
|
|
|
|
|
/* Cleanup flags and remove reference */
|
2021-04-01 12:22:26 +03:00
|
|
|
block->flags &= ~RAM_UF_WRITEPROTECT;
|
|
|
|
memory_region_unref(block->mr);
|
2021-01-29 13:14:05 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Finally close UFFD file descriptor */
|
|
|
|
uffd_close_fd(rs->uffdio_fd);
|
|
|
|
rs->uffdio_fd = -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
/* No target OS support, stubs just fail or ignore */
|
|
|
|
|
|
|
|
static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset)
|
|
|
|
{
|
|
|
|
(void) rs;
|
|
|
|
(void) offset;
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss,
|
|
|
|
unsigned long start_page)
|
|
|
|
{
|
|
|
|
(void) rs;
|
|
|
|
(void) pss;
|
|
|
|
(void) start_page;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool ram_write_tracking_available(void)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool ram_write_tracking_compatible(void)
|
|
|
|
{
|
|
|
|
assert(0);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
int ram_write_tracking_start(void)
|
|
|
|
{
|
|
|
|
assert(0);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
void ram_write_tracking_stop(void)
|
|
|
|
{
|
|
|
|
assert(0);
|
|
|
|
}
|
|
|
|
#endif /* defined(__linux__) */
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
2019-05-25 09:28:32 +03:00
|
|
|
* get_queued_page: unqueue a page from the postcopy requests
|
2017-03-23 17:06:39 +03:00
|
|
|
*
|
|
|
|
* Skips pages that are already sent (!dirty)
|
2015-11-05 21:11:09 +03:00
|
|
|
*
|
2019-05-11 02:37:29 +03:00
|
|
|
* Returns true if a queued page is found
|
2015-11-05 21:11:09 +03:00
|
|
|
*
|
2017-03-13 21:26:29 +03:00
|
|
|
* @rs: current RAM state
|
2017-03-23 17:06:39 +03:00
|
|
|
* @pss: data about the state of the current dirty page scan
|
2015-11-05 21:11:09 +03:00
|
|
|
*/
|
2017-03-21 18:19:05 +03:00
|
|
|
static bool get_queued_page(RAMState *rs, PageSearchStatus *pss)
|
2015-11-05 21:11:09 +03:00
|
|
|
{
|
|
|
|
RAMBlock *block;
|
|
|
|
ram_addr_t offset;
|
2022-08-02 09:19:49 +03:00
|
|
|
bool dirty;
|
|
|
|
|
|
|
|
do {
|
|
|
|
block = unqueue_page(rs, &offset);
|
|
|
|
/*
|
|
|
|
* We're sending this page, and since it's postcopy nothing else
|
|
|
|
* will dirty it, and we must make sure it doesn't get sent again
|
|
|
|
* even if this queue request was received after the background
|
|
|
|
* search already sent it.
|
|
|
|
*/
|
|
|
|
if (block) {
|
|
|
|
unsigned long page;
|
|
|
|
|
|
|
|
page = offset >> TARGET_PAGE_BITS;
|
|
|
|
dirty = test_bit(page, block->bmap);
|
|
|
|
if (!dirty) {
|
|
|
|
trace_get_queued_page_not_dirty(block->idstr, (uint64_t)offset,
|
|
|
|
page);
|
|
|
|
} else {
|
|
|
|
trace_get_queued_page(block->idstr, (uint64_t)offset, page);
|
|
|
|
}
|
|
|
|
}
|
2015-11-05 21:11:09 +03:00
|
|
|
|
2022-08-02 09:19:49 +03:00
|
|
|
} while (block && !dirty);
|
2015-11-05 21:11:09 +03:00
|
|
|
|
2022-10-12 00:55:58 +03:00
|
|
|
if (!block) {
|
2021-01-29 13:14:05 +03:00
|
|
|
/*
|
|
|
|
* Poll write faults too if background snapshot is enabled; that's
|
|
|
|
* when we have vcpus got blocked by the write protected pages.
|
|
|
|
*/
|
|
|
|
block = poll_fault_page(rs, &offset);
|
|
|
|
}
|
|
|
|
|
2015-11-05 21:11:09 +03:00
|
|
|
if (block) {
|
|
|
|
/*
|
|
|
|
* We want the background search to continue from the queued page
|
|
|
|
* since the guest is likely to want other pages near to the page
|
|
|
|
* it just requested.
|
|
|
|
*/
|
|
|
|
pss->block = block;
|
2017-03-21 17:36:51 +03:00
|
|
|
pss->page = offset >> TARGET_PAGE_BITS;
|
2019-06-05 04:08:28 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* This unqueued page would break the "one round" check, even is
|
|
|
|
* really rare.
|
|
|
|
*/
|
|
|
|
pss->complete_round = false;
|
2015-11-05 21:11:09 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
return !!block;
|
|
|
|
}
|
|
|
|
|
2015-11-05 21:11:08 +03:00
|
|
|
/**
|
2017-04-03 23:06:54 +03:00
|
|
|
* migration_page_queue_free: drop any remaining pages in the ram
|
|
|
|
* request queue
|
2015-11-05 21:11:08 +03:00
|
|
|
*
|
2017-03-23 17:06:39 +03:00
|
|
|
* It should be empty at the end anyway, but in error cases there may
|
|
|
|
* be some left. in case that there is any page left, we drop it.
|
|
|
|
*
|
2015-11-05 21:11:08 +03:00
|
|
|
*/
|
2017-05-04 12:45:01 +03:00
|
|
|
static void migration_page_queue_free(RAMState *rs)
|
2015-11-05 21:11:08 +03:00
|
|
|
{
|
2017-03-21 00:12:40 +03:00
|
|
|
struct RAMSrcPageRequest *mspr, *next_mspr;
|
2015-11-05 21:11:08 +03:00
|
|
|
/* This queue generally should be empty - but in the case of a failed
|
|
|
|
* migration might have some droppings in.
|
|
|
|
*/
|
2019-10-07 17:36:39 +03:00
|
|
|
RCU_READ_LOCK_GUARD();
|
2017-03-21 00:12:40 +03:00
|
|
|
QSIMPLEQ_FOREACH_SAFE(mspr, &rs->src_page_requests, next_req, next_mspr) {
|
2015-11-05 21:11:08 +03:00
|
|
|
memory_region_unref(mspr->rb->mr);
|
2017-03-21 00:12:40 +03:00
|
|
|
QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
|
2015-11-05 21:11:08 +03:00
|
|
|
g_free(mspr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2017-03-23 17:06:39 +03:00
|
|
|
* ram_save_queue_pages: queue the page for transmission
|
|
|
|
*
|
|
|
|
* A request from postcopy destination for example.
|
|
|
|
*
|
|
|
|
* Returns zero on success or negative on error
|
|
|
|
*
|
|
|
|
* @rbname: Name of the RAMBLock of the request. NULL means the
|
|
|
|
* same that last one.
|
|
|
|
* @start: starting address from the start of the RAMBlock
|
|
|
|
* @len: length (in bytes) to send
|
2015-11-05 21:11:08 +03:00
|
|
|
*/
|
2017-03-14 20:41:03 +03:00
|
|
|
int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len)
|
2015-11-05 21:11:08 +03:00
|
|
|
{
|
|
|
|
RAMBlock *ramblock;
|
2017-05-04 12:46:24 +03:00
|
|
|
RAMState *rs = ram_state;
|
2015-11-05 21:11:08 +03:00
|
|
|
|
2023-04-11 19:04:59 +03:00
|
|
|
stat64_add(&ram_counters.postcopy_requests, 1);
|
2019-10-07 17:36:39 +03:00
|
|
|
RCU_READ_LOCK_GUARD();
|
|
|
|
|
2015-11-05 21:11:08 +03:00
|
|
|
if (!rbname) {
|
|
|
|
/* Reuse last RAMBlock */
|
2017-03-14 15:48:42 +03:00
|
|
|
ramblock = rs->last_req_rb;
|
2015-11-05 21:11:08 +03:00
|
|
|
|
|
|
|
if (!ramblock) {
|
|
|
|
/*
|
|
|
|
* Shouldn't happen, we can't reuse the last RAMBlock if
|
|
|
|
* it's the 1st request.
|
|
|
|
*/
|
|
|
|
error_report("ram_save_queue_pages no previous block");
|
2020-01-06 21:23:31 +03:00
|
|
|
return -1;
|
2015-11-05 21:11:08 +03:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
ramblock = qemu_ram_block_by_name(rbname);
|
|
|
|
|
|
|
|
if (!ramblock) {
|
|
|
|
/* We shouldn't be asked for a non-existent RAMBlock */
|
|
|
|
error_report("ram_save_queue_pages no block '%s'", rbname);
|
2020-01-06 21:23:31 +03:00
|
|
|
return -1;
|
2015-11-05 21:11:08 +03:00
|
|
|
}
|
2017-03-14 15:48:42 +03:00
|
|
|
rs->last_req_rb = ramblock;
|
2015-11-05 21:11:08 +03:00
|
|
|
}
|
|
|
|
trace_ram_save_queue_pages(ramblock->idstr, start, len);
|
2021-04-29 14:27:08 +03:00
|
|
|
if (!offset_in_ramblock(ramblock, start + len - 1)) {
|
2015-11-10 19:42:05 +03:00
|
|
|
error_report("%s request overrun start=" RAM_ADDR_FMT " len="
|
|
|
|
RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
|
2015-11-05 21:11:08 +03:00
|
|
|
__func__, start, len, ramblock->used_length);
|
2020-01-06 21:23:31 +03:00
|
|
|
return -1;
|
2015-11-05 21:11:08 +03:00
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:57 +03:00
|
|
|
/*
|
|
|
|
* When with postcopy preempt, we send back the page directly in the
|
|
|
|
* rp-return thread.
|
|
|
|
*/
|
|
|
|
if (postcopy_preempt_active()) {
|
|
|
|
ram_addr_t page_start = start >> TARGET_PAGE_BITS;
|
|
|
|
size_t page_size = qemu_ram_pagesize(ramblock);
|
|
|
|
PageSearchStatus *pss = &ram_state->pss[RAM_CHANNEL_POSTCOPY];
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
qemu_mutex_lock(&rs->bitmap_mutex);
|
|
|
|
|
|
|
|
pss_init(pss, ramblock, page_start);
|
|
|
|
/*
|
|
|
|
* Always use the preempt channel, and make sure it's there. It's
|
|
|
|
* safe to access without lock, because when rp-thread is running
|
|
|
|
* we should be the only one who operates on the qemufile
|
|
|
|
*/
|
|
|
|
pss->pss_channel = migrate_get_current()->postcopy_qemufile_src;
|
|
|
|
assert(pss->pss_channel);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* It must be either one or multiple of host page size. Just
|
|
|
|
* assert; if something wrong we're mostly split brain anyway.
|
|
|
|
*/
|
|
|
|
assert(len % page_size == 0);
|
|
|
|
while (len) {
|
|
|
|
if (ram_save_host_page_urgent(pss)) {
|
|
|
|
error_report("%s: ram_save_host_page_urgent() failed: "
|
|
|
|
"ramblock=%s, start_addr=0x"RAM_ADDR_FMT,
|
|
|
|
__func__, ramblock->idstr, start);
|
|
|
|
ret = -1;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* NOTE: after ram_save_host_page_urgent() succeeded, pss->page
|
|
|
|
* will automatically be moved and point to the next host page
|
|
|
|
* we're going to send, so no need to update here.
|
|
|
|
*
|
|
|
|
* Normally QEMU never sends >1 host page in requests, so
|
|
|
|
* logically we don't even need that as the loop should only
|
|
|
|
* run once, but just to be consistent.
|
|
|
|
*/
|
|
|
|
len -= page_size;
|
|
|
|
};
|
|
|
|
qemu_mutex_unlock(&rs->bitmap_mutex);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2017-03-21 00:12:40 +03:00
|
|
|
struct RAMSrcPageRequest *new_entry =
|
2022-03-15 17:41:56 +03:00
|
|
|
g_new0(struct RAMSrcPageRequest, 1);
|
2015-11-05 21:11:08 +03:00
|
|
|
new_entry->rb = ramblock;
|
|
|
|
new_entry->offset = start;
|
|
|
|
new_entry->len = len;
|
|
|
|
|
|
|
|
memory_region_ref(ramblock->mr);
|
2017-03-21 00:12:40 +03:00
|
|
|
qemu_mutex_lock(&rs->src_page_req_mutex);
|
|
|
|
QSIMPLEQ_INSERT_TAIL(&rs->src_page_requests, new_entry, next_req);
|
2018-06-13 13:26:42 +03:00
|
|
|
migration_make_urgent_request();
|
2017-03-21 00:12:40 +03:00
|
|
|
qemu_mutex_unlock(&rs->src_page_req_mutex);
|
2015-11-05 21:11:08 +03:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-03-30 10:51:26 +03:00
|
|
|
static bool save_page_use_compression(RAMState *rs)
|
|
|
|
{
|
2023-03-02 00:03:48 +03:00
|
|
|
if (!migrate_compress()) {
|
2018-03-30 10:51:26 +03:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
migrate/ram: remove "ram_bulk_stage" and "fpo_enabled"
The bulk stage is kind of weird: migration_bitmap_find_dirty() will
indicate a dirty page, however, ram_save_host_page() will never save it, as
migration_bitmap_clear_dirty() detects that it is not dirty.
We already fill the bitmap in ram_list_init_bitmaps() with ones, marking
everything dirty - it didn't used to be that way, which is why we needed
an explicit first bulk stage.
Let's simplify: make the bitmap the single source of thuth. Explicitly
handle the "xbzrle_enabled after first round" case.
Regarding XBZRLE (implicitly handled via "ram_bulk_stage = false" right
now), there is now a slight change in behavior:
- Colo: When starting, it will be disabled (was implicitly enabled)
until the first round actually finishes.
- Free page hinting: When starting, XBZRLE will be disabled (was implicitly
enabled) until the first round actually finished.
- Snapshots: When starting, XBZRLE will be disabled. We essentially only
do a single run, so I guess it will never actually get disabled.
Postcopy seems to indirectly disable it in ram_save_page(), so there
shouldn't be really any change.
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Juan Quintela <quintela@redhat.com>
Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
Cc: Andrey Gruzdev <andrey.gruzdev@virtuozzo.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Message-Id: <20210216105039.40680-1-david@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2021-02-16 13:50:39 +03:00
|
|
|
* If xbzrle is enabled (e.g., after first round of migration), stop
|
|
|
|
* using the data compression. In theory, xbzrle can do better than
|
|
|
|
* compression.
|
2018-03-30 10:51:26 +03:00
|
|
|
*/
|
migrate/ram: remove "ram_bulk_stage" and "fpo_enabled"
The bulk stage is kind of weird: migration_bitmap_find_dirty() will
indicate a dirty page, however, ram_save_host_page() will never save it, as
migration_bitmap_clear_dirty() detects that it is not dirty.
We already fill the bitmap in ram_list_init_bitmaps() with ones, marking
everything dirty - it didn't used to be that way, which is why we needed
an explicit first bulk stage.
Let's simplify: make the bitmap the single source of thuth. Explicitly
handle the "xbzrle_enabled after first round" case.
Regarding XBZRLE (implicitly handled via "ram_bulk_stage = false" right
now), there is now a slight change in behavior:
- Colo: When starting, it will be disabled (was implicitly enabled)
until the first round actually finishes.
- Free page hinting: When starting, XBZRLE will be disabled (was implicitly
enabled) until the first round actually finished.
- Snapshots: When starting, XBZRLE will be disabled. We essentially only
do a single run, so I guess it will never actually get disabled.
Postcopy seems to indirectly disable it in ram_save_page(), so there
shouldn't be really any change.
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Juan Quintela <quintela@redhat.com>
Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
Cc: Andrey Gruzdev <andrey.gruzdev@virtuozzo.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Message-Id: <20210216105039.40680-1-david@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2021-02-16 13:50:39 +03:00
|
|
|
if (rs->xbzrle_enabled) {
|
|
|
|
return false;
|
2018-03-30 10:51:26 +03:00
|
|
|
}
|
|
|
|
|
migrate/ram: remove "ram_bulk_stage" and "fpo_enabled"
The bulk stage is kind of weird: migration_bitmap_find_dirty() will
indicate a dirty page, however, ram_save_host_page() will never save it, as
migration_bitmap_clear_dirty() detects that it is not dirty.
We already fill the bitmap in ram_list_init_bitmaps() with ones, marking
everything dirty - it didn't used to be that way, which is why we needed
an explicit first bulk stage.
Let's simplify: make the bitmap the single source of thuth. Explicitly
handle the "xbzrle_enabled after first round" case.
Regarding XBZRLE (implicitly handled via "ram_bulk_stage = false" right
now), there is now a slight change in behavior:
- Colo: When starting, it will be disabled (was implicitly enabled)
until the first round actually finishes.
- Free page hinting: When starting, XBZRLE will be disabled (was implicitly
enabled) until the first round actually finished.
- Snapshots: When starting, XBZRLE will be disabled. We essentially only
do a single run, so I guess it will never actually get disabled.
Postcopy seems to indirectly disable it in ram_save_page(), so there
shouldn't be really any change.
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Juan Quintela <quintela@redhat.com>
Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
Cc: Andrey Gruzdev <andrey.gruzdev@virtuozzo.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Message-Id: <20210216105039.40680-1-david@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2021-02-16 13:50:39 +03:00
|
|
|
return true;
|
2018-03-30 10:51:26 +03:00
|
|
|
}
|
|
|
|
|
2018-08-21 11:10:24 +03:00
|
|
|
/*
|
|
|
|
* try to compress the page before posting it out, return true if the page
|
|
|
|
* has been properly handled by compression, otherwise needs other
|
|
|
|
* paths to handle it
|
|
|
|
*/
|
2022-10-12 00:55:56 +03:00
|
|
|
static bool save_compress_page(RAMState *rs, PageSearchStatus *pss,
|
|
|
|
RAMBlock *block, ram_addr_t offset)
|
2018-08-21 11:10:24 +03:00
|
|
|
{
|
|
|
|
if (!save_page_use_compression(rs)) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* When starting the process of a new block, the first page of
|
|
|
|
* the block should be sent out before other pages in the same
|
|
|
|
* block, and all the pages in last block should have been sent
|
|
|
|
* out, keeping this order is important, because the 'cont' flag
|
|
|
|
* is used to avoid resending the block name.
|
|
|
|
*
|
|
|
|
* We post the fist page as normal page as compression will take
|
|
|
|
* much CPU resource.
|
|
|
|
*/
|
2022-10-12 00:55:56 +03:00
|
|
|
if (block != pss->last_sent_block) {
|
2018-08-21 11:10:24 +03:00
|
|
|
flush_compressed_data(rs);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:49 +03:00
|
|
|
if (compress_page_with_multi_thread(block, offset) > 0) {
|
2018-08-21 11:10:24 +03:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2018-09-06 10:01:00 +03:00
|
|
|
compression_counters.busy++;
|
2018-08-21 11:10:24 +03:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-11-05 21:11:09 +03:00
|
|
|
/**
|
2021-12-15 22:10:39 +03:00
|
|
|
* ram_save_target_page_legacy: save one target page
|
2015-11-05 21:11:09 +03:00
|
|
|
*
|
2017-03-23 17:06:39 +03:00
|
|
|
* Returns the number of pages written
|
2015-11-05 21:11:09 +03:00
|
|
|
*
|
2017-03-13 21:26:29 +03:00
|
|
|
* @rs: current RAM state
|
2017-03-23 17:06:39 +03:00
|
|
|
* @pss: data about the page we want to send
|
2015-11-05 21:11:09 +03:00
|
|
|
*/
|
2021-12-15 22:10:39 +03:00
|
|
|
static int ram_save_target_page_legacy(RAMState *rs, PageSearchStatus *pss)
|
2015-11-05 21:11:09 +03:00
|
|
|
{
|
2018-03-30 10:51:25 +03:00
|
|
|
RAMBlock *block = pss->block;
|
2020-01-10 16:51:34 +03:00
|
|
|
ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
|
2018-03-30 10:51:25 +03:00
|
|
|
int res;
|
|
|
|
|
2022-10-12 00:55:53 +03:00
|
|
|
if (control_save_page(pss, block, offset, &res)) {
|
2018-03-30 10:51:25 +03:00
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:56 +03:00
|
|
|
if (save_compress_page(rs, pss, block, offset)) {
|
2018-08-21 11:10:24 +03:00
|
|
|
return 1;
|
2018-03-30 10:51:26 +03:00
|
|
|
}
|
|
|
|
|
2023-04-02 20:06:32 +03:00
|
|
|
res = save_zero_page(pss, pss->pss_channel, block, offset);
|
2018-03-30 10:51:26 +03:00
|
|
|
if (res > 0) {
|
|
|
|
/* Must let xbzrle know, otherwise a previous (now 0'd) cached
|
|
|
|
* page would be stale
|
|
|
|
*/
|
2022-10-12 00:55:47 +03:00
|
|
|
if (rs->xbzrle_enabled) {
|
2018-03-30 10:51:26 +03:00
|
|
|
XBZRLE_cache_lock();
|
|
|
|
xbzrle_cache_zero_page(rs, block->offset + offset);
|
|
|
|
XBZRLE_cache_unlock();
|
|
|
|
}
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2018-03-30 10:51:28 +03:00
|
|
|
/*
|
2022-10-04 21:24:30 +03:00
|
|
|
* Do not use multifd in postcopy as one whole host page should be
|
|
|
|
* placed. Meanwhile postcopy requires atomic update of pages, so even
|
|
|
|
* if host page size == guest page size the dest guest during run may
|
|
|
|
* still see partially copied pages which is data corruption.
|
2018-03-30 10:51:28 +03:00
|
|
|
*/
|
2023-03-02 00:10:29 +03:00
|
|
|
if (migrate_multifd() && !migration_in_postcopy()) {
|
2022-10-12 00:55:53 +03:00
|
|
|
return ram_save_multifd_page(pss->pss_channel, block, offset);
|
2015-11-05 21:11:09 +03:00
|
|
|
}
|
|
|
|
|
2021-12-15 21:01:21 +03:00
|
|
|
return ram_save_page(rs, pss);
|
2015-11-05 21:11:09 +03:00
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:52 +03:00
|
|
|
/* Should be called before sending a host page */
|
|
|
|
static void pss_host_page_prepare(PageSearchStatus *pss)
|
|
|
|
{
|
|
|
|
/* How many guest pages are there in one host page? */
|
|
|
|
size_t guest_pfns = qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
|
|
|
|
|
|
|
|
pss->host_page_sending = true;
|
2023-01-20 19:31:47 +03:00
|
|
|
if (guest_pfns <= 1) {
|
|
|
|
/*
|
|
|
|
* This covers both when guest psize == host psize, or when guest
|
|
|
|
* has larger psize than the host (guest_pfns==0).
|
|
|
|
*
|
|
|
|
* For the latter, we always send one whole guest page per
|
|
|
|
* iteration of the host page (example: an Alpha VM on x86 host
|
|
|
|
* will have guest psize 8K while host psize 4K).
|
|
|
|
*/
|
|
|
|
pss->host_page_start = pss->page;
|
|
|
|
pss->host_page_end = pss->page + 1;
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* The host page spans over multiple guest pages, we send them
|
|
|
|
* within the same host page iteration.
|
|
|
|
*/
|
|
|
|
pss->host_page_start = ROUND_DOWN(pss->page, guest_pfns);
|
|
|
|
pss->host_page_end = ROUND_UP(pss->page + 1, guest_pfns);
|
|
|
|
}
|
2022-10-12 00:55:52 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Whether the page pointed by PSS is within the host page being sent.
|
|
|
|
* Must be called after a previous pss_host_page_prepare().
|
|
|
|
*/
|
|
|
|
static bool pss_within_range(PageSearchStatus *pss)
|
|
|
|
{
|
|
|
|
ram_addr_t ram_addr;
|
|
|
|
|
|
|
|
assert(pss->host_page_sending);
|
|
|
|
|
|
|
|
/* Over host-page boundary? */
|
|
|
|
if (pss->page >= pss->host_page_end) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
ram_addr = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
|
|
|
|
|
|
|
|
return offset_in_ramblock(pss->block, ram_addr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void pss_host_page_finish(PageSearchStatus *pss)
|
|
|
|
{
|
|
|
|
pss->host_page_sending = false;
|
|
|
|
/* This is not needed, but just to reset it */
|
|
|
|
pss->host_page_start = pss->host_page_end = 0;
|
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:57 +03:00
|
|
|
/*
|
|
|
|
* Send an urgent host page specified by `pss'. Need to be called with
|
|
|
|
* bitmap_mutex held.
|
|
|
|
*
|
|
|
|
* Returns 0 if save host page succeeded, false otherwise.
|
|
|
|
*/
|
|
|
|
static int ram_save_host_page_urgent(PageSearchStatus *pss)
|
|
|
|
{
|
|
|
|
bool page_dirty, sent = false;
|
|
|
|
RAMState *rs = ram_state;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
trace_postcopy_preempt_send_host_page(pss->block->idstr, pss->page);
|
|
|
|
pss_host_page_prepare(pss);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If precopy is sending the same page, let it be done in precopy, or
|
|
|
|
* we could send the same page in two channels and none of them will
|
|
|
|
* receive the whole page.
|
|
|
|
*/
|
|
|
|
if (pss_overlap(pss, &ram_state->pss[RAM_CHANNEL_PRECOPY])) {
|
|
|
|
trace_postcopy_preempt_hit(pss->block->idstr,
|
|
|
|
pss->page << TARGET_PAGE_BITS);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
do {
|
|
|
|
page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
|
|
|
|
|
|
|
|
if (page_dirty) {
|
|
|
|
/* Be strict to return code; it must be 1, or what else? */
|
2021-12-15 22:10:39 +03:00
|
|
|
if (migration_ops->ram_save_target_page(rs, pss) != 1) {
|
2022-10-12 00:55:57 +03:00
|
|
|
error_report_once("%s: ram_save_target_page failed", __func__);
|
|
|
|
ret = -1;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
sent = true;
|
|
|
|
}
|
|
|
|
pss_find_next_dirty(pss);
|
|
|
|
} while (pss_within_range(pss));
|
|
|
|
out:
|
|
|
|
pss_host_page_finish(pss);
|
|
|
|
/* For urgent requests, flush immediately if sent */
|
|
|
|
if (sent) {
|
|
|
|
qemu_fflush(pss->pss_channel);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2015-11-05 21:11:09 +03:00
|
|
|
/**
|
2017-03-23 17:06:39 +03:00
|
|
|
* ram_save_host_page: save a whole host page
|
2015-11-05 21:11:09 +03:00
|
|
|
*
|
2017-03-23 17:06:39 +03:00
|
|
|
* Starting at *offset send pages up to the end of the current host
|
|
|
|
* page. It's valid for the initial offset to point into the middle of
|
|
|
|
* a host page in which case the remainder of the hostpage is sent.
|
|
|
|
* Only dirty target pages are sent. Note that the host page size may
|
|
|
|
* be a huge page for this block.
|
2022-10-12 00:55:50 +03:00
|
|
|
*
|
2017-05-17 19:58:09 +03:00
|
|
|
* The saving stops at the boundary of the used_length of the block
|
|
|
|
* if the RAMBlock isn't a multiple of the host page size.
|
2015-11-05 21:11:09 +03:00
|
|
|
*
|
2022-10-12 00:55:50 +03:00
|
|
|
* The caller must be with ram_state.bitmap_mutex held to call this
|
|
|
|
* function. Note that this function can temporarily release the lock, but
|
|
|
|
* when the function is returned it'll make sure the lock is still held.
|
|
|
|
*
|
2017-03-23 17:06:39 +03:00
|
|
|
* Returns the number of pages written or negative on error
|
|
|
|
*
|
2017-03-13 21:26:29 +03:00
|
|
|
* @rs: current RAM state
|
2017-03-23 17:06:39 +03:00
|
|
|
* @pss: data about the page we want to send
|
2015-11-05 21:11:09 +03:00
|
|
|
*/
|
2021-12-15 21:01:21 +03:00
|
|
|
static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss)
|
2015-11-05 21:11:09 +03:00
|
|
|
{
|
2022-10-12 00:55:50 +03:00
|
|
|
bool page_dirty, preempt_active = postcopy_preempt_active();
|
2015-11-05 21:11:09 +03:00
|
|
|
int tmppages, pages = 0;
|
2017-03-21 17:36:51 +03:00
|
|
|
size_t pagesize_bits =
|
|
|
|
qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
|
2021-01-29 13:14:05 +03:00
|
|
|
unsigned long start_page = pss->page;
|
|
|
|
int res;
|
2017-02-24 21:28:39 +03:00
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
if (ramblock_is_ignored(pss->block)) {
|
2018-05-14 09:57:00 +03:00
|
|
|
error_report("block %s should not be migrated !", pss->block->idstr);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:52 +03:00
|
|
|
/* Update host page boundary information */
|
|
|
|
pss_host_page_prepare(pss);
|
|
|
|
|
2015-11-05 21:11:09 +03:00
|
|
|
do {
|
2022-10-12 00:55:50 +03:00
|
|
|
page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
|
2015-11-05 21:11:09 +03:00
|
|
|
|
2022-10-12 00:55:50 +03:00
|
|
|
/* Check the pages is dirty and if it is send it */
|
|
|
|
if (page_dirty) {
|
2021-03-16 15:57:16 +03:00
|
|
|
/*
|
2022-10-12 00:55:50 +03:00
|
|
|
* Properly yield the lock only in postcopy preempt mode
|
|
|
|
* because both migration thread and rp-return thread can
|
|
|
|
* operate on the bitmaps.
|
2021-03-16 15:57:16 +03:00
|
|
|
*/
|
2022-10-12 00:55:50 +03:00
|
|
|
if (preempt_active) {
|
|
|
|
qemu_mutex_unlock(&rs->bitmap_mutex);
|
2021-03-16 15:57:16 +03:00
|
|
|
}
|
2021-12-15 22:10:39 +03:00
|
|
|
tmppages = migration_ops->ram_save_target_page(rs, pss);
|
2022-10-12 00:55:50 +03:00
|
|
|
if (tmppages >= 0) {
|
|
|
|
pages += tmppages;
|
|
|
|
/*
|
|
|
|
* Allow rate limiting to happen in the middle of huge pages if
|
|
|
|
* something is sent in the current iteration.
|
|
|
|
*/
|
|
|
|
if (pagesize_bits > 1 && tmppages > 0) {
|
|
|
|
migration_rate_limit();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (preempt_active) {
|
|
|
|
qemu_mutex_lock(&rs->bitmap_mutex);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
tmppages = 0;
|
2021-03-16 15:57:15 +03:00
|
|
|
}
|
2022-10-12 00:55:50 +03:00
|
|
|
|
|
|
|
if (tmppages < 0) {
|
2022-10-12 00:55:52 +03:00
|
|
|
pss_host_page_finish(pss);
|
2022-10-12 00:55:50 +03:00
|
|
|
return tmppages;
|
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:52 +03:00
|
|
|
pss_find_next_dirty(pss);
|
|
|
|
} while (pss_within_range(pss));
|
|
|
|
|
|
|
|
pss_host_page_finish(pss);
|
2021-01-29 13:14:05 +03:00
|
|
|
|
|
|
|
res = ram_save_release_protection(rs, pss, start_page);
|
|
|
|
return (res < 0 ? res : pages);
|
2015-11-05 21:11:09 +03:00
|
|
|
}
|
2015-11-05 21:11:08 +03:00
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
/**
|
2017-03-23 17:06:39 +03:00
|
|
|
* ram_find_and_save_block: finds a dirty page and sends it to f
|
2015-05-07 20:33:31 +03:00
|
|
|
*
|
|
|
|
* Called within an RCU critical section.
|
|
|
|
*
|
2018-09-03 12:26:44 +03:00
|
|
|
* Returns the number of pages written where zero means no dirty pages,
|
|
|
|
* or negative on error
|
2015-05-07 20:33:31 +03:00
|
|
|
*
|
2017-03-13 21:26:29 +03:00
|
|
|
* @rs: current RAM state
|
2015-11-05 21:11:09 +03:00
|
|
|
*
|
|
|
|
* On systems where host-page-size > target-page-size it will send all the
|
|
|
|
* pages in a host page that are dirty.
|
2015-05-07 20:33:31 +03:00
|
|
|
*/
|
2021-12-15 21:01:21 +03:00
|
|
|
static int ram_find_and_save_block(RAMState *rs)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2022-10-12 00:55:55 +03:00
|
|
|
PageSearchStatus *pss = &rs->pss[RAM_CHANNEL_PRECOPY];
|
2015-05-07 20:33:31 +03:00
|
|
|
int pages = 0;
|
|
|
|
|
2017-02-08 17:28:45 +03:00
|
|
|
/* No dirty page as there is zero RAM */
|
2022-05-10 20:37:36 +03:00
|
|
|
if (!rs->ram_bytes_total) {
|
2017-02-08 17:28:45 +03:00
|
|
|
return pages;
|
|
|
|
}
|
|
|
|
|
2022-10-04 21:24:26 +03:00
|
|
|
/*
|
|
|
|
* Always keep last_seen_block/last_page valid during this procedure,
|
|
|
|
* because find_dirty_block() relies on these values (e.g., we compare
|
|
|
|
* last_seen_block with pss.block to see whether we searched all the
|
|
|
|
* ramblocks) to detect the completion of migration. Having NULL value
|
|
|
|
* of last_seen_block can conditionally cause below loop to run forever.
|
|
|
|
*/
|
|
|
|
if (!rs->last_seen_block) {
|
|
|
|
rs->last_seen_block = QLIST_FIRST_RCU(&ram_list.blocks);
|
|
|
|
rs->last_page = 0;
|
|
|
|
}
|
|
|
|
|
2022-10-12 00:55:55 +03:00
|
|
|
pss_init(pss, rs->last_seen_block, rs->last_page);
|
2015-09-23 17:27:10 +03:00
|
|
|
|
2022-06-21 14:29:36 +03:00
|
|
|
while (true){
|
2022-06-21 14:20:35 +03:00
|
|
|
if (!get_queued_page(rs, pss)) {
|
2022-10-12 00:55:58 +03:00
|
|
|
/* priority queue empty, so just search for something dirty */
|
2022-06-21 14:29:36 +03:00
|
|
|
int res = find_dirty_block(rs, pss);
|
|
|
|
if (res != PAGE_DIRTY_FOUND) {
|
|
|
|
if (res == PAGE_ALL_CLEAN) {
|
2022-06-21 14:20:35 +03:00
|
|
|
break;
|
2022-06-21 14:29:36 +03:00
|
|
|
} else if (res == PAGE_TRY_AGAIN) {
|
|
|
|
continue;
|
2022-06-21 14:20:35 +03:00
|
|
|
}
|
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
2022-06-21 14:20:35 +03:00
|
|
|
pages = ram_save_host_page(rs, pss);
|
2022-06-21 14:29:36 +03:00
|
|
|
if (pages) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2022-10-12 00:55:55 +03:00
|
|
|
rs->last_seen_block = pss->block;
|
|
|
|
rs->last_page = pss->page;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
|
|
|
return pages;
|
|
|
|
}
|
|
|
|
|
|
|
|
void acct_update_position(QEMUFile *f, size_t size, bool zero)
|
|
|
|
{
|
|
|
|
uint64_t pages = size / TARGET_PAGE_SIZE;
|
2017-03-13 22:30:21 +03:00
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
if (zero) {
|
2023-04-19 19:16:05 +03:00
|
|
|
stat64_add(&ram_counters.zero_pages, pages);
|
2015-05-07 20:33:31 +03:00
|
|
|
} else {
|
2023-04-19 19:19:45 +03:00
|
|
|
stat64_add(&ram_counters.normal_pages, pages);
|
2021-12-21 12:34:40 +03:00
|
|
|
ram_transferred_add(size);
|
2022-06-20 14:01:51 +03:00
|
|
|
qemu_file_credit_transfer(f, size);
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-10 20:18:19 +03:00
|
|
|
static uint64_t ram_bytes_total_with_ignored(void)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
|
|
|
RAMBlock *block;
|
|
|
|
uint64_t total = 0;
|
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
RCU_READ_LOCK_GUARD();
|
|
|
|
|
2022-05-10 20:18:19 +03:00
|
|
|
RAMBLOCK_FOREACH_MIGRATABLE(block) {
|
|
|
|
total += block->used_length;
|
2017-05-12 07:17:39 +03:00
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
return total;
|
|
|
|
}
|
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
uint64_t ram_bytes_total(void)
|
|
|
|
{
|
2022-05-10 20:18:19 +03:00
|
|
|
RAMBlock *block;
|
|
|
|
uint64_t total = 0;
|
|
|
|
|
|
|
|
RCU_READ_LOCK_GUARD();
|
|
|
|
|
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
|
|
|
total += block->used_length;
|
|
|
|
}
|
|
|
|
return total;
|
2019-02-15 20:45:46 +03:00
|
|
|
}
|
|
|
|
|
2017-06-28 12:52:27 +03:00
|
|
|
static void xbzrle_load_setup(void)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2017-06-28 12:52:27 +03:00
|
|
|
XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE);
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2017-06-28 12:52:27 +03:00
|
|
|
static void xbzrle_load_cleanup(void)
|
|
|
|
{
|
|
|
|
g_free(XBZRLE.decoded_buf);
|
|
|
|
XBZRLE.decoded_buf = NULL;
|
|
|
|
}
|
|
|
|
|
2017-10-19 09:31:58 +03:00
|
|
|
static void ram_state_cleanup(RAMState **rsp)
|
|
|
|
{
|
2018-02-12 19:03:39 +03:00
|
|
|
if (*rsp) {
|
|
|
|
migration_page_queue_free(*rsp);
|
|
|
|
qemu_mutex_destroy(&(*rsp)->bitmap_mutex);
|
|
|
|
qemu_mutex_destroy(&(*rsp)->src_page_req_mutex);
|
|
|
|
g_free(*rsp);
|
|
|
|
*rsp = NULL;
|
|
|
|
}
|
2017-10-19 09:31:58 +03:00
|
|
|
}
|
|
|
|
|
2017-10-19 09:31:59 +03:00
|
|
|
static void xbzrle_cleanup(void)
|
|
|
|
{
|
|
|
|
XBZRLE_cache_lock();
|
|
|
|
if (XBZRLE.cache) {
|
|
|
|
cache_fini(XBZRLE.cache);
|
|
|
|
g_free(XBZRLE.encoded_buf);
|
|
|
|
g_free(XBZRLE.current_buf);
|
|
|
|
g_free(XBZRLE.zero_target_page);
|
|
|
|
XBZRLE.cache = NULL;
|
|
|
|
XBZRLE.encoded_buf = NULL;
|
|
|
|
XBZRLE.current_buf = NULL;
|
|
|
|
XBZRLE.zero_target_page = NULL;
|
|
|
|
}
|
|
|
|
XBZRLE_cache_unlock();
|
|
|
|
}
|
|
|
|
|
2017-06-28 12:52:27 +03:00
|
|
|
static void ram_save_cleanup(void *opaque)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2017-05-04 12:46:24 +03:00
|
|
|
RAMState **rsp = opaque;
|
2017-03-22 17:18:04 +03:00
|
|
|
RAMBlock *block;
|
2017-03-13 23:51:55 +03:00
|
|
|
|
2021-01-29 13:14:05 +03:00
|
|
|
/* We don't use dirty log with background snapshots */
|
|
|
|
if (!migrate_background_snapshot()) {
|
|
|
|
/* caller have hold iothread lock or is in a bh, so there is
|
|
|
|
* no writing race against the migration bitmap
|
|
|
|
*/
|
2021-06-29 19:01:19 +03:00
|
|
|
if (global_dirty_tracking & GLOBAL_DIRTY_MIGRATION) {
|
|
|
|
/*
|
|
|
|
* do not stop dirty log without starting it, since
|
|
|
|
* memory_global_dirty_log_stop will assert that
|
|
|
|
* memory_global_dirty_log_start/stop used in pairs
|
|
|
|
*/
|
|
|
|
memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
|
|
|
|
}
|
2021-01-29 13:14:05 +03:00
|
|
|
}
|
2017-03-22 17:18:04 +03:00
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
migration: Split log_clear() into smaller chunks
Currently we are doing log_clear() right after log_sync() which mostly
keeps the old behavior when log_clear() was still part of log_sync().
This patch tries to further optimize the migration log_clear() code
path to split huge log_clear()s into smaller chunks.
We do this by spliting the whole guest memory region into memory
chunks, whose size is decided by MigrationState.clear_bitmap_shift (an
example will be given below). With that, we don't do the dirty bitmap
clear operation on the remote node (e.g., KVM) when we fetch the dirty
bitmap, instead we explicitly clear the dirty bitmap for the memory
chunk for each of the first time we send a page in that chunk.
Here comes an example.
Assuming the guest has 64G memory, then before this patch the KVM
ioctl KVM_CLEAR_DIRTY_LOG will be a single one covering 64G memory.
If after the patch, let's assume when the clear bitmap shift is 18,
then the memory chunk size on x86_64 will be 1UL<<18 * 4K = 1GB. Then
instead of sending a big 64G ioctl, we'll send 64 small ioctls, each
of the ioctl will cover 1G of the guest memory. For each of the 64
small ioctls, we'll only send if any of the page in that small chunk
was going to be sent right away.
Signed-off-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Message-Id: <20190603065056.25211-12-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2019-06-03 09:50:56 +03:00
|
|
|
g_free(block->clear_bmap);
|
|
|
|
block->clear_bmap = NULL;
|
2017-03-22 17:18:04 +03:00
|
|
|
g_free(block->bmap);
|
|
|
|
block->bmap = NULL;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2017-10-19 09:31:59 +03:00
|
|
|
xbzrle_cleanup();
|
2017-06-28 12:52:28 +03:00
|
|
|
compress_threads_save_cleanup();
|
2017-10-19 09:31:58 +03:00
|
|
|
ram_state_cleanup(rsp);
|
2021-12-15 22:10:39 +03:00
|
|
|
g_free(migration_ops);
|
|
|
|
migration_ops = NULL;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2017-03-13 21:26:29 +03:00
|
|
|
static void ram_state_reset(RAMState *rs)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2022-10-12 00:55:56 +03:00
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < RAM_CHANNEL_MAX; i++) {
|
|
|
|
rs->pss[i].last_sent_block = NULL;
|
|
|
|
}
|
|
|
|
|
2017-03-13 21:26:29 +03:00
|
|
|
rs->last_seen_block = NULL;
|
2017-03-21 17:23:31 +03:00
|
|
|
rs->last_page = 0;
|
2017-03-13 21:26:29 +03:00
|
|
|
rs->last_version = ram_list.version;
|
migrate/ram: remove "ram_bulk_stage" and "fpo_enabled"
The bulk stage is kind of weird: migration_bitmap_find_dirty() will
indicate a dirty page, however, ram_save_host_page() will never save it, as
migration_bitmap_clear_dirty() detects that it is not dirty.
We already fill the bitmap in ram_list_init_bitmaps() with ones, marking
everything dirty - it didn't used to be that way, which is why we needed
an explicit first bulk stage.
Let's simplify: make the bitmap the single source of thuth. Explicitly
handle the "xbzrle_enabled after first round" case.
Regarding XBZRLE (implicitly handled via "ram_bulk_stage = false" right
now), there is now a slight change in behavior:
- Colo: When starting, it will be disabled (was implicitly enabled)
until the first round actually finishes.
- Free page hinting: When starting, XBZRLE will be disabled (was implicitly
enabled) until the first round actually finished.
- Snapshots: When starting, XBZRLE will be disabled. We essentially only
do a single run, so I guess it will never actually get disabled.
Postcopy seems to indirectly disable it in ram_save_page(), so there
shouldn't be really any change.
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Juan Quintela <quintela@redhat.com>
Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
Cc: Andrey Gruzdev <andrey.gruzdev@virtuozzo.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Message-Id: <20210216105039.40680-1-david@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2021-02-16 13:50:39 +03:00
|
|
|
rs->xbzrle_enabled = false;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
#define MAX_WAIT 50 /* ms, half buffered_file limit */
|
|
|
|
|
2015-11-05 21:11:02 +03:00
|
|
|
/* **** functions for postcopy ***** */
|
|
|
|
|
2017-02-03 18:23:21 +03:00
|
|
|
void ram_postcopy_migrated_memory_release(MigrationState *ms)
|
|
|
|
{
|
|
|
|
struct RAMBlock *block;
|
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
2017-03-22 17:18:04 +03:00
|
|
|
unsigned long *bitmap = block->bmap;
|
|
|
|
unsigned long range = block->used_length >> TARGET_PAGE_BITS;
|
|
|
|
unsigned long run_start = find_next_zero_bit(bitmap, range, 0);
|
2017-02-03 18:23:21 +03:00
|
|
|
|
|
|
|
while (run_start < range) {
|
|
|
|
unsigned long run_end = find_next_bit(bitmap, range, run_start + 1);
|
2020-01-10 16:51:34 +03:00
|
|
|
ram_discard_range(block->idstr,
|
|
|
|
((ram_addr_t)run_start) << TARGET_PAGE_BITS,
|
|
|
|
((ram_addr_t)(run_end - run_start))
|
|
|
|
<< TARGET_PAGE_BITS);
|
2017-02-03 18:23:21 +03:00
|
|
|
run_start = find_next_zero_bit(bitmap, range, run_end + 1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* postcopy_send_discard_bm_ram: discard a RAMBlock
|
|
|
|
*
|
2015-11-05 21:11:02 +03:00
|
|
|
* Callback from postcopy_each_ram_send_discard for each RAMBlock
|
2017-03-23 17:06:39 +03:00
|
|
|
*
|
|
|
|
* @ms: current migration state
|
2019-07-15 05:05:49 +03:00
|
|
|
* @block: RAMBlock to discard
|
2015-11-05 21:11:02 +03:00
|
|
|
*/
|
2021-12-30 19:05:25 +03:00
|
|
|
static void postcopy_send_discard_bm_ram(MigrationState *ms, RAMBlock *block)
|
2015-11-05 21:11:02 +03:00
|
|
|
{
|
2017-03-22 17:18:04 +03:00
|
|
|
unsigned long end = block->used_length >> TARGET_PAGE_BITS;
|
2015-11-05 21:11:02 +03:00
|
|
|
unsigned long current;
|
2019-08-19 09:18:42 +03:00
|
|
|
unsigned long *bitmap = block->bmap;
|
2015-11-05 21:11:02 +03:00
|
|
|
|
2017-03-22 17:18:04 +03:00
|
|
|
for (current = 0; current < end; ) {
|
2019-08-19 09:18:42 +03:00
|
|
|
unsigned long one = find_next_bit(bitmap, end, current);
|
2019-06-27 05:08:21 +03:00
|
|
|
unsigned long zero, discard_length;
|
2015-11-05 21:11:02 +03:00
|
|
|
|
2019-06-27 05:08:21 +03:00
|
|
|
if (one >= end) {
|
|
|
|
break;
|
|
|
|
}
|
2015-11-05 21:11:02 +03:00
|
|
|
|
2019-08-19 09:18:42 +03:00
|
|
|
zero = find_next_zero_bit(bitmap, end, one + 1);
|
2019-06-27 05:08:21 +03:00
|
|
|
|
|
|
|
if (zero >= end) {
|
|
|
|
discard_length = end - one;
|
2015-11-05 21:11:02 +03:00
|
|
|
} else {
|
2019-06-27 05:08:21 +03:00
|
|
|
discard_length = zero - one;
|
|
|
|
}
|
2019-07-24 04:07:21 +03:00
|
|
|
postcopy_discard_send_range(ms, one, discard_length);
|
2019-06-27 05:08:21 +03:00
|
|
|
current = one + discard_length;
|
2015-11-05 21:11:02 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-07 14:50:13 +03:00
|
|
|
static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block);
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* postcopy_each_ram_send_discard: discard all RAMBlocks
|
|
|
|
*
|
2015-11-05 21:11:02 +03:00
|
|
|
* Utility for the outgoing postcopy code.
|
|
|
|
* Calls postcopy_send_discard_bm_ram for each RAMBlock
|
|
|
|
* passing it bitmap indexes and name.
|
|
|
|
* (qemu_ram_foreach_block ends up passing unscaled lengths
|
|
|
|
* which would mean postcopy code would have to deal with target page)
|
2017-03-23 17:06:39 +03:00
|
|
|
*
|
|
|
|
* @ms: current migration state
|
2015-11-05 21:11:02 +03:00
|
|
|
*/
|
2021-12-07 14:50:14 +03:00
|
|
|
static void postcopy_each_ram_send_discard(MigrationState *ms)
|
2015-11-05 21:11:02 +03:00
|
|
|
{
|
|
|
|
struct RAMBlock *block;
|
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
2019-07-24 04:07:21 +03:00
|
|
|
postcopy_discard_send_init(ms, block->idstr);
|
2015-11-05 21:11:02 +03:00
|
|
|
|
2021-12-07 14:50:13 +03:00
|
|
|
/*
|
|
|
|
* Deal with TPS != HPS and huge pages. It discard any partially sent
|
|
|
|
* host-page size chunks, mark any partially dirty host-page size
|
|
|
|
* chunks as all dirty. In this case the host-page is the host-page
|
|
|
|
* for the particular RAMBlock, i.e. it might be a huge page.
|
|
|
|
*/
|
|
|
|
postcopy_chunk_hostpages_pass(ms, block);
|
|
|
|
|
2015-11-05 21:11:02 +03:00
|
|
|
/*
|
|
|
|
* Postcopy sends chunks of bitmap over the wire, but it
|
|
|
|
* just needs indexes at this point, avoids it having
|
|
|
|
* target page specific code.
|
|
|
|
*/
|
2021-12-07 14:50:14 +03:00
|
|
|
postcopy_send_discard_bm_ram(ms, block);
|
2019-07-24 04:07:21 +03:00
|
|
|
postcopy_discard_send_finish(ms);
|
2015-11-05 21:11:02 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
2019-08-19 09:18:41 +03:00
|
|
|
* postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
|
2017-03-23 17:06:39 +03:00
|
|
|
*
|
|
|
|
* Helper for postcopy_chunk_hostpages; it's called twice to
|
|
|
|
* canonicalize the two bitmaps, that are similar, but one is
|
|
|
|
* inverted.
|
2015-11-05 21:11:15 +03:00
|
|
|
*
|
2017-03-23 17:06:39 +03:00
|
|
|
* Postcopy requires that all target pages in a hostpage are dirty or
|
|
|
|
* clean, not a mix. This function canonicalizes the bitmaps.
|
2015-11-05 21:11:15 +03:00
|
|
|
*
|
2017-03-23 17:06:39 +03:00
|
|
|
* @ms: current migration state
|
|
|
|
* @block: block that contains the page we want to canonicalize
|
2015-11-05 21:11:15 +03:00
|
|
|
*/
|
2019-08-19 09:18:42 +03:00
|
|
|
static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block)
|
2015-11-05 21:11:15 +03:00
|
|
|
{
|
2017-05-04 12:46:24 +03:00
|
|
|
RAMState *rs = ram_state;
|
2017-03-22 17:18:04 +03:00
|
|
|
unsigned long *bitmap = block->bmap;
|
2017-02-24 21:28:31 +03:00
|
|
|
unsigned int host_ratio = block->page_size / TARGET_PAGE_SIZE;
|
2017-03-22 17:18:04 +03:00
|
|
|
unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
|
2015-11-05 21:11:15 +03:00
|
|
|
unsigned long run_start;
|
|
|
|
|
2017-02-24 21:28:31 +03:00
|
|
|
if (block->page_size == TARGET_PAGE_SIZE) {
|
|
|
|
/* Easy case - TPS==HPS for a non-huge page RAMBlock */
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-08-19 09:18:42 +03:00
|
|
|
/* Find a dirty page */
|
|
|
|
run_start = find_next_bit(bitmap, pages, 0);
|
2015-11-05 21:11:15 +03:00
|
|
|
|
2017-03-22 17:18:04 +03:00
|
|
|
while (run_start < pages) {
|
2015-11-05 21:11:15 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If the start of this run of pages is in the middle of a host
|
|
|
|
* page, then we need to fixup this host page.
|
|
|
|
*/
|
2019-08-06 03:46:48 +03:00
|
|
|
if (QEMU_IS_ALIGNED(run_start, host_ratio)) {
|
2015-11-05 21:11:15 +03:00
|
|
|
/* Find the end of this run */
|
2019-08-19 09:18:42 +03:00
|
|
|
run_start = find_next_zero_bit(bitmap, pages, run_start + 1);
|
2015-11-05 21:11:15 +03:00
|
|
|
/*
|
|
|
|
* If the end isn't at the start of a host page, then the
|
|
|
|
* run doesn't finish at the end of a host page
|
|
|
|
* and we need to discard.
|
|
|
|
*/
|
|
|
|
}
|
|
|
|
|
2019-08-06 03:46:48 +03:00
|
|
|
if (!QEMU_IS_ALIGNED(run_start, host_ratio)) {
|
2015-11-05 21:11:15 +03:00
|
|
|
unsigned long page;
|
2019-08-06 03:46:47 +03:00
|
|
|
unsigned long fixup_start_addr = QEMU_ALIGN_DOWN(run_start,
|
|
|
|
host_ratio);
|
|
|
|
run_start = QEMU_ALIGN_UP(run_start, host_ratio);
|
2015-11-05 21:11:15 +03:00
|
|
|
|
|
|
|
/* Clean up the bitmap */
|
|
|
|
for (page = fixup_start_addr;
|
|
|
|
page < fixup_start_addr + host_ratio; page++) {
|
|
|
|
/*
|
|
|
|
* Remark them as dirty, updating the count for any pages
|
|
|
|
* that weren't previously dirty.
|
|
|
|
*/
|
2017-03-13 23:21:41 +03:00
|
|
|
rs->migration_dirty_pages += !test_and_set_bit(page, bitmap);
|
2015-11-05 21:11:15 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-08-19 09:18:42 +03:00
|
|
|
/* Find the next dirty page for the next iteration */
|
|
|
|
run_start = find_next_bit(bitmap, pages, run_start);
|
2015-11-05 21:11:15 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* ram_postcopy_send_discard_bitmap: transmit the discard bitmap
|
|
|
|
*
|
2015-11-05 21:11:02 +03:00
|
|
|
* Transmit the set of pages to be discarded after precopy to the target
|
|
|
|
* these are pages that:
|
|
|
|
* a) Have been previously transmitted but are now dirty again
|
|
|
|
* b) Pages that have never been transmitted, this ensures that
|
|
|
|
* any pages on the destination that have been mapped by background
|
|
|
|
* tasks get discarded (transparent huge pages is the specific concern)
|
|
|
|
* Hopefully this is pretty sparse
|
2017-03-23 17:06:39 +03:00
|
|
|
*
|
|
|
|
* @ms: current migration state
|
2015-11-05 21:11:02 +03:00
|
|
|
*/
|
2021-12-07 14:50:14 +03:00
|
|
|
void ram_postcopy_send_discard_bitmap(MigrationState *ms)
|
2015-11-05 21:11:02 +03:00
|
|
|
{
|
2017-05-04 12:46:24 +03:00
|
|
|
RAMState *rs = ram_state;
|
2015-11-05 21:11:02 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
RCU_READ_LOCK_GUARD();
|
2015-11-05 21:11:02 +03:00
|
|
|
|
|
|
|
/* This should be our last sync, the src is now paused */
|
2017-03-13 23:51:55 +03:00
|
|
|
migration_bitmap_sync(rs);
|
2015-11-05 21:11:02 +03:00
|
|
|
|
2017-03-22 17:18:04 +03:00
|
|
|
/* Easiest way to make sure we don't resume in the middle of a host-page */
|
2022-10-12 00:55:56 +03:00
|
|
|
rs->pss[RAM_CHANNEL_PRECOPY].last_sent_block = NULL;
|
2017-03-22 17:18:04 +03:00
|
|
|
rs->last_seen_block = NULL;
|
|
|
|
rs->last_page = 0;
|
2015-11-05 21:11:02 +03:00
|
|
|
|
2021-12-07 14:50:14 +03:00
|
|
|
postcopy_each_ram_send_discard(ms);
|
2015-11-05 21:11:02 +03:00
|
|
|
|
2021-12-07 14:50:14 +03:00
|
|
|
trace_ram_postcopy_send_discard_bitmap();
|
2015-11-05 21:11:02 +03:00
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* ram_discard_range: discard dirtied pages at the beginning of postcopy
|
2015-11-05 21:11:02 +03:00
|
|
|
*
|
2017-03-23 17:06:39 +03:00
|
|
|
* Returns zero on success
|
2015-11-05 21:11:02 +03:00
|
|
|
*
|
2017-03-23 17:11:59 +03:00
|
|
|
* @rbname: name of the RAMBlock of the request. NULL means the
|
|
|
|
* same that last one.
|
2017-03-23 17:06:39 +03:00
|
|
|
* @start: RAMBlock starting page
|
|
|
|
* @length: RAMBlock size
|
2015-11-05 21:11:02 +03:00
|
|
|
*/
|
2017-03-21 13:35:24 +03:00
|
|
|
int ram_discard_range(const char *rbname, uint64_t start, size_t length)
|
2015-11-05 21:11:02 +03:00
|
|
|
{
|
2017-03-23 17:11:59 +03:00
|
|
|
trace_ram_discard_range(rbname, start, length);
|
2017-02-24 21:28:32 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
RCU_READ_LOCK_GUARD();
|
2017-03-23 17:11:59 +03:00
|
|
|
RAMBlock *rb = qemu_ram_block_by_name(rbname);
|
2015-11-05 21:11:02 +03:00
|
|
|
|
|
|
|
if (!rb) {
|
2017-03-23 17:11:59 +03:00
|
|
|
error_report("ram_discard_range: Failed to find block '%s'", rbname);
|
2020-01-06 21:23:31 +03:00
|
|
|
return -1;
|
2015-11-05 21:11:02 +03:00
|
|
|
}
|
|
|
|
|
2018-07-23 15:33:02 +03:00
|
|
|
/*
|
|
|
|
* On source VM, we don't need to update the received bitmap since
|
|
|
|
* we don't even have one.
|
|
|
|
*/
|
|
|
|
if (rb->receivedmap) {
|
|
|
|
bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(),
|
|
|
|
length >> qemu_target_page_bits());
|
|
|
|
}
|
|
|
|
|
2020-01-06 21:23:31 +03:00
|
|
|
return ram_block_discard_range(rb, start, length);
|
2015-11-05 21:11:02 +03:00
|
|
|
}
|
|
|
|
|
2017-10-19 09:31:59 +03:00
|
|
|
/*
|
|
|
|
* For every allocation, we will try not to crash the VM if the
|
|
|
|
* allocation failed.
|
|
|
|
*/
|
|
|
|
static int xbzrle_init(void)
|
|
|
|
{
|
|
|
|
Error *local_err = NULL;
|
|
|
|
|
2023-03-02 00:20:13 +03:00
|
|
|
if (!migrate_xbzrle()) {
|
2017-10-19 09:31:59 +03:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
XBZRLE_cache_lock();
|
|
|
|
|
|
|
|
XBZRLE.zero_target_page = g_try_malloc0(TARGET_PAGE_SIZE);
|
|
|
|
if (!XBZRLE.zero_target_page) {
|
|
|
|
error_report("%s: Error allocating zero page", __func__);
|
|
|
|
goto err_out;
|
|
|
|
}
|
|
|
|
|
|
|
|
XBZRLE.cache = cache_init(migrate_xbzrle_cache_size(),
|
|
|
|
TARGET_PAGE_SIZE, &local_err);
|
|
|
|
if (!XBZRLE.cache) {
|
|
|
|
error_report_err(local_err);
|
|
|
|
goto free_zero_page;
|
|
|
|
}
|
|
|
|
|
|
|
|
XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
|
|
|
|
if (!XBZRLE.encoded_buf) {
|
|
|
|
error_report("%s: Error allocating encoded_buf", __func__);
|
|
|
|
goto free_cache;
|
|
|
|
}
|
|
|
|
|
|
|
|
XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
|
|
|
|
if (!XBZRLE.current_buf) {
|
|
|
|
error_report("%s: Error allocating current_buf", __func__);
|
|
|
|
goto free_encoded_buf;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* We are all good */
|
|
|
|
XBZRLE_cache_unlock();
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
free_encoded_buf:
|
|
|
|
g_free(XBZRLE.encoded_buf);
|
|
|
|
XBZRLE.encoded_buf = NULL;
|
|
|
|
free_cache:
|
|
|
|
cache_fini(XBZRLE.cache);
|
|
|
|
XBZRLE.cache = NULL;
|
|
|
|
free_zero_page:
|
|
|
|
g_free(XBZRLE.zero_target_page);
|
|
|
|
XBZRLE.zero_target_page = NULL;
|
|
|
|
err_out:
|
|
|
|
XBZRLE_cache_unlock();
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2017-05-04 12:46:24 +03:00
|
|
|
static int ram_state_init(RAMState **rsp)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2017-10-19 09:31:57 +03:00
|
|
|
*rsp = g_try_new0(RAMState, 1);
|
|
|
|
|
|
|
|
if (!*rsp) {
|
|
|
|
error_report("%s: Init ramstate fail", __func__);
|
|
|
|
return -1;
|
|
|
|
}
|
2017-05-04 12:46:24 +03:00
|
|
|
|
|
|
|
qemu_mutex_init(&(*rsp)->bitmap_mutex);
|
|
|
|
qemu_mutex_init(&(*rsp)->src_page_req_mutex);
|
|
|
|
QSIMPLEQ_INIT(&(*rsp)->src_page_requests);
|
2022-05-10 20:37:36 +03:00
|
|
|
(*rsp)->ram_bytes_total = ram_bytes_total();
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2017-10-19 09:31:57 +03:00
|
|
|
/*
|
2019-07-14 17:51:19 +03:00
|
|
|
* Count the total number of pages used by ram blocks not including any
|
|
|
|
* gaps due to alignment or unplugs.
|
2019-06-04 09:17:27 +03:00
|
|
|
* This must match with the initial values of dirty bitmap.
|
2017-10-19 09:31:57 +03:00
|
|
|
*/
|
2022-05-10 20:37:36 +03:00
|
|
|
(*rsp)->migration_dirty_pages = (*rsp)->ram_bytes_total >> TARGET_PAGE_BITS;
|
2017-10-19 09:31:57 +03:00
|
|
|
ram_state_reset(*rsp);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-10-19 09:32:00 +03:00
|
|
|
static void ram_list_init_bitmaps(void)
|
2017-10-19 09:31:57 +03:00
|
|
|
{
|
migration: Split log_clear() into smaller chunks
Currently we are doing log_clear() right after log_sync() which mostly
keeps the old behavior when log_clear() was still part of log_sync().
This patch tries to further optimize the migration log_clear() code
path to split huge log_clear()s into smaller chunks.
We do this by spliting the whole guest memory region into memory
chunks, whose size is decided by MigrationState.clear_bitmap_shift (an
example will be given below). With that, we don't do the dirty bitmap
clear operation on the remote node (e.g., KVM) when we fetch the dirty
bitmap, instead we explicitly clear the dirty bitmap for the memory
chunk for each of the first time we send a page in that chunk.
Here comes an example.
Assuming the guest has 64G memory, then before this patch the KVM
ioctl KVM_CLEAR_DIRTY_LOG will be a single one covering 64G memory.
If after the patch, let's assume when the clear bitmap shift is 18,
then the memory chunk size on x86_64 will be 1UL<<18 * 4K = 1GB. Then
instead of sending a big 64G ioctl, we'll send 64 small ioctls, each
of the ioctl will cover 1G of the guest memory. For each of the 64
small ioctls, we'll only send if any of the page in that small chunk
was going to be sent right away.
Signed-off-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Message-Id: <20190603065056.25211-12-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2019-06-03 09:50:56 +03:00
|
|
|
MigrationState *ms = migrate_get_current();
|
2017-10-19 09:32:00 +03:00
|
|
|
RAMBlock *block;
|
|
|
|
unsigned long pages;
|
migration: Split log_clear() into smaller chunks
Currently we are doing log_clear() right after log_sync() which mostly
keeps the old behavior when log_clear() was still part of log_sync().
This patch tries to further optimize the migration log_clear() code
path to split huge log_clear()s into smaller chunks.
We do this by spliting the whole guest memory region into memory
chunks, whose size is decided by MigrationState.clear_bitmap_shift (an
example will be given below). With that, we don't do the dirty bitmap
clear operation on the remote node (e.g., KVM) when we fetch the dirty
bitmap, instead we explicitly clear the dirty bitmap for the memory
chunk for each of the first time we send a page in that chunk.
Here comes an example.
Assuming the guest has 64G memory, then before this patch the KVM
ioctl KVM_CLEAR_DIRTY_LOG will be a single one covering 64G memory.
If after the patch, let's assume when the clear bitmap shift is 18,
then the memory chunk size on x86_64 will be 1UL<<18 * 4K = 1GB. Then
instead of sending a big 64G ioctl, we'll send 64 small ioctls, each
of the ioctl will cover 1G of the guest memory. For each of the 64
small ioctls, we'll only send if any of the page in that small chunk
was going to be sent right away.
Signed-off-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Message-Id: <20190603065056.25211-12-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2019-06-03 09:50:56 +03:00
|
|
|
uint8_t shift;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2017-02-08 17:28:45 +03:00
|
|
|
/* Skip setting bitmap if there is no RAM */
|
|
|
|
if (ram_bytes_total()) {
|
migration: Split log_clear() into smaller chunks
Currently we are doing log_clear() right after log_sync() which mostly
keeps the old behavior when log_clear() was still part of log_sync().
This patch tries to further optimize the migration log_clear() code
path to split huge log_clear()s into smaller chunks.
We do this by spliting the whole guest memory region into memory
chunks, whose size is decided by MigrationState.clear_bitmap_shift (an
example will be given below). With that, we don't do the dirty bitmap
clear operation on the remote node (e.g., KVM) when we fetch the dirty
bitmap, instead we explicitly clear the dirty bitmap for the memory
chunk for each of the first time we send a page in that chunk.
Here comes an example.
Assuming the guest has 64G memory, then before this patch the KVM
ioctl KVM_CLEAR_DIRTY_LOG will be a single one covering 64G memory.
If after the patch, let's assume when the clear bitmap shift is 18,
then the memory chunk size on x86_64 will be 1UL<<18 * 4K = 1GB. Then
instead of sending a big 64G ioctl, we'll send 64 small ioctls, each
of the ioctl will cover 1G of the guest memory. For each of the 64
small ioctls, we'll only send if any of the page in that small chunk
was going to be sent right away.
Signed-off-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Message-Id: <20190603065056.25211-12-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2019-06-03 09:50:56 +03:00
|
|
|
shift = ms->clear_bitmap_shift;
|
|
|
|
if (shift > CLEAR_BITMAP_SHIFT_MAX) {
|
|
|
|
error_report("clear_bitmap_shift (%u) too big, using "
|
|
|
|
"max value (%u)", shift, CLEAR_BITMAP_SHIFT_MAX);
|
|
|
|
shift = CLEAR_BITMAP_SHIFT_MAX;
|
|
|
|
} else if (shift < CLEAR_BITMAP_SHIFT_MIN) {
|
|
|
|
error_report("clear_bitmap_shift (%u) too small, using "
|
|
|
|
"min value (%u)", shift, CLEAR_BITMAP_SHIFT_MIN);
|
|
|
|
shift = CLEAR_BITMAP_SHIFT_MIN;
|
|
|
|
}
|
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
2017-10-19 09:32:00 +03:00
|
|
|
pages = block->max_length >> TARGET_PAGE_BITS;
|
2019-06-04 09:17:27 +03:00
|
|
|
/*
|
|
|
|
* The initial dirty bitmap for migration must be set with all
|
|
|
|
* ones to make sure we'll migrate every guest RAM page to
|
|
|
|
* destination.
|
2019-07-14 17:51:19 +03:00
|
|
|
* Here we set RAMBlock.bmap all to 1 because when rebegin a
|
|
|
|
* new migration after a failed migration, ram_list.
|
|
|
|
* dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
|
|
|
|
* guest memory.
|
2019-06-04 09:17:27 +03:00
|
|
|
*/
|
2017-03-22 17:18:04 +03:00
|
|
|
block->bmap = bitmap_new(pages);
|
2019-07-14 17:51:19 +03:00
|
|
|
bitmap_set(block->bmap, 0, pages);
|
migration: Split log_clear() into smaller chunks
Currently we are doing log_clear() right after log_sync() which mostly
keeps the old behavior when log_clear() was still part of log_sync().
This patch tries to further optimize the migration log_clear() code
path to split huge log_clear()s into smaller chunks.
We do this by spliting the whole guest memory region into memory
chunks, whose size is decided by MigrationState.clear_bitmap_shift (an
example will be given below). With that, we don't do the dirty bitmap
clear operation on the remote node (e.g., KVM) when we fetch the dirty
bitmap, instead we explicitly clear the dirty bitmap for the memory
chunk for each of the first time we send a page in that chunk.
Here comes an example.
Assuming the guest has 64G memory, then before this patch the KVM
ioctl KVM_CLEAR_DIRTY_LOG will be a single one covering 64G memory.
If after the patch, let's assume when the clear bitmap shift is 18,
then the memory chunk size on x86_64 will be 1UL<<18 * 4K = 1GB. Then
instead of sending a big 64G ioctl, we'll send 64 small ioctls, each
of the ioctl will cover 1G of the guest memory. For each of the 64
small ioctls, we'll only send if any of the page in that small chunk
was going to be sent right away.
Signed-off-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Message-Id: <20190603065056.25211-12-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2019-06-03 09:50:56 +03:00
|
|
|
block->clear_bmap_shift = shift;
|
|
|
|
block->clear_bmap = bitmap_new(clear_bmap_size(pages, shift));
|
2017-02-08 17:28:45 +03:00
|
|
|
}
|
2015-11-05 21:11:01 +03:00
|
|
|
}
|
2017-10-19 09:32:00 +03:00
|
|
|
}
|
|
|
|
|
migration/ram: Handle RAMBlocks with a RamDiscardManager on the migration source
We don't want to migrate memory that corresponds to discarded ranges as
managed by a RamDiscardManager responsible for the mapped memory region of
the RAMBlock. The content of these pages is essentially stale and
without any guarantees for the VM ("logically unplugged").
Depending on the underlying memory type, even reading memory might populate
memory on the source, resulting in an undesired memory consumption. Of
course, on the destination, even writing a zeropage consumes memory,
which we also want to avoid (similar to free page hinting).
Currently, virtio-mem tries achieving that goal (not migrating "unplugged"
memory that was discarded) by going via qemu_guest_free_page_hint() - but
it's hackish and incomplete.
For example, background snapshots still end up reading all memory, as
they don't do bitmap syncs. Postcopy recovery code will re-add
previously cleared bits to the dirty bitmap and migrate them.
Let's consult the RamDiscardManager after setting up our dirty bitmap
initially and when postcopy recovery code reinitializes it: clear
corresponding bits in the dirty bitmaps (e.g., of the RAMBlock and inside
KVM). It's important to fixup the dirty bitmap *after* our initial bitmap
sync, such that the corresponding dirty bits in KVM are actually cleared.
As colo is incompatible with discarding of RAM and inhibits it, we don't
have to bother.
Note: if a misbehaving guest would use discarded ranges after migration
started we would still migrate that memory: however, then we already
populated that memory on the migration source.
Reviewed-by: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2021-10-11 20:53:41 +03:00
|
|
|
static void migration_bitmap_clear_discarded_pages(RAMState *rs)
|
|
|
|
{
|
|
|
|
unsigned long pages;
|
|
|
|
RAMBlock *rb;
|
|
|
|
|
|
|
|
RCU_READ_LOCK_GUARD();
|
|
|
|
|
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
|
|
|
|
pages = ramblock_dirty_bitmap_clear_discarded_pages(rb);
|
|
|
|
rs->migration_dirty_pages -= pages;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-10-19 09:32:00 +03:00
|
|
|
static void ram_init_bitmaps(RAMState *rs)
|
|
|
|
{
|
|
|
|
/* For memory_global_dirty_log_start below. */
|
|
|
|
qemu_mutex_lock_iothread();
|
|
|
|
qemu_mutex_lock_ramlist();
|
2015-11-05 21:11:01 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
WITH_RCU_READ_LOCK_GUARD() {
|
|
|
|
ram_list_init_bitmaps();
|
2021-01-29 13:14:05 +03:00
|
|
|
/* We don't use dirty log with background snapshots */
|
|
|
|
if (!migrate_background_snapshot()) {
|
2021-06-29 19:01:19 +03:00
|
|
|
memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
|
2021-01-29 13:14:05 +03:00
|
|
|
migration_bitmap_sync_precopy(rs);
|
|
|
|
}
|
2019-10-07 17:36:39 +03:00
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
qemu_mutex_unlock_ramlist();
|
2016-02-15 21:57:57 +03:00
|
|
|
qemu_mutex_unlock_iothread();
|
migration/ram: Handle RAMBlocks with a RamDiscardManager on the migration source
We don't want to migrate memory that corresponds to discarded ranges as
managed by a RamDiscardManager responsible for the mapped memory region of
the RAMBlock. The content of these pages is essentially stale and
without any guarantees for the VM ("logically unplugged").
Depending on the underlying memory type, even reading memory might populate
memory on the source, resulting in an undesired memory consumption. Of
course, on the destination, even writing a zeropage consumes memory,
which we also want to avoid (similar to free page hinting).
Currently, virtio-mem tries achieving that goal (not migrating "unplugged"
memory that was discarded) by going via qemu_guest_free_page_hint() - but
it's hackish and incomplete.
For example, background snapshots still end up reading all memory, as
they don't do bitmap syncs. Postcopy recovery code will re-add
previously cleared bits to the dirty bitmap and migrate them.
Let's consult the RamDiscardManager after setting up our dirty bitmap
initially and when postcopy recovery code reinitializes it: clear
corresponding bits in the dirty bitmaps (e.g., of the RAMBlock and inside
KVM). It's important to fixup the dirty bitmap *after* our initial bitmap
sync, such that the corresponding dirty bits in KVM are actually cleared.
As colo is incompatible with discarding of RAM and inhibits it, we don't
have to bother.
Note: if a misbehaving guest would use discarded ranges after migration
started we would still migrate that memory: however, then we already
populated that memory on the migration source.
Reviewed-by: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2021-10-11 20:53:41 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* After an eventual first bitmap sync, fixup the initial bitmap
|
|
|
|
* containing all 1s to exclude any discarded pages from migration.
|
|
|
|
*/
|
|
|
|
migration_bitmap_clear_discarded_pages(rs);
|
2017-10-19 09:32:00 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static int ram_init_all(RAMState **rsp)
|
|
|
|
{
|
|
|
|
if (ram_state_init(rsp)) {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (xbzrle_init()) {
|
|
|
|
ram_state_cleanup(rsp);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
ram_init_bitmaps(*rsp);
|
2016-10-27 09:42:59 +03:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-05-02 13:47:33 +03:00
|
|
|
static void ram_state_resume_prepare(RAMState *rs, QEMUFile *out)
|
|
|
|
{
|
|
|
|
RAMBlock *block;
|
|
|
|
uint64_t pages = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Postcopy is not using xbzrle/compression, so no need for that.
|
|
|
|
* Also, since source are already halted, we don't need to care
|
|
|
|
* about dirty page logging as well.
|
|
|
|
*/
|
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
2018-05-02 13:47:33 +03:00
|
|
|
pages += bitmap_count_one(block->bmap,
|
|
|
|
block->used_length >> TARGET_PAGE_BITS);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* This may not be aligned with current bitmaps. Recalculate. */
|
|
|
|
rs->migration_dirty_pages = pages;
|
|
|
|
|
migrate/ram: remove "ram_bulk_stage" and "fpo_enabled"
The bulk stage is kind of weird: migration_bitmap_find_dirty() will
indicate a dirty page, however, ram_save_host_page() will never save it, as
migration_bitmap_clear_dirty() detects that it is not dirty.
We already fill the bitmap in ram_list_init_bitmaps() with ones, marking
everything dirty - it didn't used to be that way, which is why we needed
an explicit first bulk stage.
Let's simplify: make the bitmap the single source of thuth. Explicitly
handle the "xbzrle_enabled after first round" case.
Regarding XBZRLE (implicitly handled via "ram_bulk_stage = false" right
now), there is now a slight change in behavior:
- Colo: When starting, it will be disabled (was implicitly enabled)
until the first round actually finishes.
- Free page hinting: When starting, XBZRLE will be disabled (was implicitly
enabled) until the first round actually finished.
- Snapshots: When starting, XBZRLE will be disabled. We essentially only
do a single run, so I guess it will never actually get disabled.
Postcopy seems to indirectly disable it in ram_save_page(), so there
shouldn't be really any change.
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Juan Quintela <quintela@redhat.com>
Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
Cc: Andrey Gruzdev <andrey.gruzdev@virtuozzo.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Message-Id: <20210216105039.40680-1-david@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2021-02-16 13:50:39 +03:00
|
|
|
ram_state_reset(rs);
|
2018-05-02 13:47:33 +03:00
|
|
|
|
|
|
|
/* Update RAMState cache of output QEMUFile */
|
2022-10-12 00:55:59 +03:00
|
|
|
rs->pss[RAM_CHANNEL_PRECOPY].pss_channel = out;
|
2018-05-02 13:47:33 +03:00
|
|
|
|
|
|
|
trace_ram_state_resume_prepare(pages);
|
|
|
|
}
|
|
|
|
|
2018-12-11 11:24:50 +03:00
|
|
|
/*
|
|
|
|
* This function clears bits of the free pages reported by the caller from the
|
|
|
|
* migration dirty bitmap. @addr is the host address corresponding to the
|
|
|
|
* start of the continuous guest free pages, and @len is the total bytes of
|
|
|
|
* those pages.
|
|
|
|
*/
|
|
|
|
void qemu_guest_free_page_hint(void *addr, size_t len)
|
|
|
|
{
|
|
|
|
RAMBlock *block;
|
|
|
|
ram_addr_t offset;
|
|
|
|
size_t used_len, start, npages;
|
|
|
|
MigrationState *s = migrate_get_current();
|
|
|
|
|
|
|
|
/* This function is currently expected to be used during live migration */
|
|
|
|
if (!migration_is_setup_or_active(s->state)) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (; len > 0; len -= used_len, addr += used_len) {
|
|
|
|
block = qemu_ram_block_from_host(addr, false, &offset);
|
|
|
|
if (unlikely(!block || offset >= block->used_length)) {
|
|
|
|
/*
|
|
|
|
* The implementation might not support RAMBlock resize during
|
|
|
|
* live migration, but it could happen in theory with future
|
|
|
|
* updates. So we add a check here to capture that case.
|
|
|
|
*/
|
|
|
|
error_report_once("%s unexpected error", __func__);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (len <= block->used_length - offset) {
|
|
|
|
used_len = len;
|
|
|
|
} else {
|
|
|
|
used_len = block->used_length - offset;
|
|
|
|
}
|
|
|
|
|
|
|
|
start = offset >> TARGET_PAGE_BITS;
|
|
|
|
npages = used_len >> TARGET_PAGE_BITS;
|
|
|
|
|
|
|
|
qemu_mutex_lock(&ram_state->bitmap_mutex);
|
2021-07-22 11:30:55 +03:00
|
|
|
/*
|
|
|
|
* The skipped free pages are equavalent to be sent from clear_bmap's
|
|
|
|
* perspective, so clear the bits from the memory region bitmap which
|
|
|
|
* are initially set. Otherwise those skipped pages will be sent in
|
|
|
|
* the next round after syncing from the memory region bitmap.
|
|
|
|
*/
|
2021-09-04 19:09:07 +03:00
|
|
|
migration_clear_memory_region_dirty_bitmap_range(block, start, npages);
|
2018-12-11 11:24:50 +03:00
|
|
|
ram_state->migration_dirty_pages -=
|
|
|
|
bitmap_count_one_with_offset(block->bmap, start, npages);
|
|
|
|
bitmap_clear(block->bmap, start, npages);
|
|
|
|
qemu_mutex_unlock(&ram_state->bitmap_mutex);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/*
|
|
|
|
* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
|
2016-10-27 09:42:59 +03:00
|
|
|
* long-running RCU critical section. When rcu-reclaims in the code
|
|
|
|
* start to become numerous it will be necessary to reduce the
|
|
|
|
* granularity of these critical sections.
|
|
|
|
*/
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* ram_save_setup: Setup RAM for migration
|
|
|
|
*
|
|
|
|
* Returns zero to indicate success and negative for error
|
|
|
|
*
|
|
|
|
* @f: QEMUFile where to send the data
|
|
|
|
* @opaque: RAMState pointer
|
|
|
|
*/
|
2016-10-27 09:42:59 +03:00
|
|
|
static int ram_save_setup(QEMUFile *f, void *opaque)
|
|
|
|
{
|
2017-05-04 12:46:24 +03:00
|
|
|
RAMState **rsp = opaque;
|
2016-10-27 09:42:59 +03:00
|
|
|
RAMBlock *block;
|
2022-05-13 09:28:35 +03:00
|
|
|
int ret;
|
2016-10-27 09:42:59 +03:00
|
|
|
|
2018-03-30 10:51:20 +03:00
|
|
|
if (compress_threads_save_setup()) {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2016-10-27 09:42:59 +03:00
|
|
|
/* migration has already setup the bitmap, reuse it. */
|
|
|
|
if (!migration_in_colo_state()) {
|
2017-10-19 09:31:57 +03:00
|
|
|
if (ram_init_all(rsp) != 0) {
|
2018-03-30 10:51:20 +03:00
|
|
|
compress_threads_save_cleanup();
|
2016-10-27 09:42:59 +03:00
|
|
|
return -1;
|
2017-05-04 12:46:24 +03:00
|
|
|
}
|
2016-10-27 09:42:59 +03:00
|
|
|
}
|
2022-10-12 00:55:59 +03:00
|
|
|
(*rsp)->pss[RAM_CHANNEL_PRECOPY].pss_channel = f;
|
2016-10-27 09:42:59 +03:00
|
|
|
|
2019-10-07 17:36:38 +03:00
|
|
|
WITH_RCU_READ_LOCK_GUARD() {
|
2022-05-10 20:18:19 +03:00
|
|
|
qemu_put_be64(f, ram_bytes_total_with_ignored()
|
|
|
|
| RAM_SAVE_FLAG_MEM_SIZE);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2019-10-07 17:36:38 +03:00
|
|
|
RAMBLOCK_FOREACH_MIGRATABLE(block) {
|
|
|
|
qemu_put_byte(f, strlen(block->idstr));
|
|
|
|
qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
|
|
|
|
qemu_put_be64(f, block->used_length);
|
|
|
|
if (migrate_postcopy_ram() && block->page_size !=
|
|
|
|
qemu_host_page_size) {
|
|
|
|
qemu_put_be64(f, block->page_size);
|
|
|
|
}
|
|
|
|
if (migrate_ignore_shared()) {
|
|
|
|
qemu_put_be64(f, block->mr->addr);
|
|
|
|
}
|
2019-02-15 20:45:46 +03:00
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
ram_control_before_iterate(f, RAM_CONTROL_SETUP);
|
|
|
|
ram_control_after_iterate(f, RAM_CONTROL_SETUP);
|
|
|
|
|
2021-12-15 22:10:39 +03:00
|
|
|
migration_ops = g_malloc0(sizeof(MigrationOps));
|
|
|
|
migration_ops->ram_save_target_page = ram_save_target_page_legacy;
|
2023-03-17 12:57:13 +03:00
|
|
|
ret = multifd_send_sync_main(f);
|
2022-05-13 09:28:35 +03:00
|
|
|
if (ret < 0) {
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
|
2018-04-18 11:13:21 +03:00
|
|
|
qemu_fflush(f);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* ram_save_iterate: iterative stage for migration
|
|
|
|
*
|
|
|
|
* Returns zero to indicate success and negative for error
|
|
|
|
*
|
|
|
|
* @f: QEMUFile where to send the data
|
|
|
|
* @opaque: RAMState pointer
|
|
|
|
*/
|
2015-05-07 20:33:31 +03:00
|
|
|
static int ram_save_iterate(QEMUFile *f, void *opaque)
|
|
|
|
{
|
2017-05-04 12:46:24 +03:00
|
|
|
RAMState **temp = opaque;
|
|
|
|
RAMState *rs = *temp;
|
2019-12-18 07:12:36 +03:00
|
|
|
int ret = 0;
|
2015-05-07 20:33:31 +03:00
|
|
|
int i;
|
|
|
|
int64_t t0;
|
2016-11-04 16:10:17 +03:00
|
|
|
int done = 0;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2018-03-08 14:18:24 +03:00
|
|
|
if (blk_mig_bulk_active()) {
|
|
|
|
/* Avoid transferring ram during bulk phase of block migration as
|
|
|
|
* the bulk phase will usually take a long time and transferring
|
|
|
|
* ram updates during that time is pointless. */
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
migration: Move bitmap_mutex out of migration_bitmap_clear_dirty()
Taking the mutex every time for each dirty bit to clear is too slow, especially
we'll take/release even if the dirty bit is cleared. So far it's only used to
sync with special cases with qemu_guest_free_page_hint() against migration
thread, nothing really that serious yet. Let's move the lock to be upper.
There're two callers of migration_bitmap_clear_dirty().
For migration, move it into ram_save_iterate(). With the help of MAX_WAIT
logic, we'll only run ram_save_iterate() for no more than 50ms-ish time, so
taking the lock once there at the entry. It also means any call sites to
qemu_guest_free_page_hint() can be delayed; but it should be very rare, only
during migration, and I don't see a problem with it.
For COLO, move it up to colo_flush_ram_cache(). I think COLO forgot to take
that lock even when calling ramblock_sync_dirty_bitmap(), where another example
is migration_bitmap_sync() who took it right. So let the mutex cover both the
ramblock_sync_dirty_bitmap() and migration_bitmap_clear_dirty() calls.
It's even possible to drop the lock so we use atomic operations upon rb->bmap
and the variable migration_dirty_pages. I didn't do it just to still be safe,
also not predictable whether the frequent atomic ops could bring overhead too
e.g. on huge vms when it happens very often. When that really comes, we can
keep a local counter and periodically call atomic ops. Keep it simple for now.
Cc: Wei Wang <wei.w.wang@intel.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Hailiang Zhang <zhang.zhanghailiang@huawei.com>
Cc: Dr. David Alan Gilbert <dgilbert@redhat.com>
Cc: Juan Quintela <quintela@redhat.com>
Cc: Leonardo Bras Soares Passos <lsoaresp@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20210630200805.280905-1-peterx@redhat.com>
Reviewed-by: Wei Wang <wei.w.wang@intel.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2021-06-30 23:08:05 +03:00
|
|
|
/*
|
|
|
|
* We'll take this lock a little bit long, but it's okay for two reasons.
|
|
|
|
* Firstly, the only possible other thread to take it is who calls
|
|
|
|
* qemu_guest_free_page_hint(), which should be rare; secondly, see
|
|
|
|
* MAX_WAIT (if curious, further see commit 4508bd9ed8053ce) below, which
|
|
|
|
* guarantees that we'll at least released it in a regular basis.
|
|
|
|
*/
|
|
|
|
qemu_mutex_lock(&rs->bitmap_mutex);
|
2019-10-07 17:36:39 +03:00
|
|
|
WITH_RCU_READ_LOCK_GUARD() {
|
|
|
|
if (ram_list.version != rs->last_version) {
|
|
|
|
ram_state_reset(rs);
|
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
/* Read version before ram_list.blocks */
|
|
|
|
smp_rmb();
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
ram_control_before_iterate(f, RAM_CONTROL_ROUND);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
|
|
|
|
i = 0;
|
|
|
|
while ((ret = qemu_file_rate_limit(f)) == 0 ||
|
2022-01-19 11:09:18 +03:00
|
|
|
postcopy_has_request(rs)) {
|
2019-10-07 17:36:39 +03:00
|
|
|
int pages;
|
2018-06-13 13:26:42 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
if (qemu_file_get_error(f)) {
|
|
|
|
break;
|
|
|
|
}
|
2018-09-03 12:26:44 +03:00
|
|
|
|
2021-12-15 21:01:21 +03:00
|
|
|
pages = ram_find_and_save_block(rs);
|
2019-10-07 17:36:39 +03:00
|
|
|
/* no more pages to sent */
|
|
|
|
if (pages == 0) {
|
|
|
|
done = 1;
|
|
|
|
break;
|
|
|
|
}
|
2018-09-03 12:26:44 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
if (pages < 0) {
|
|
|
|
qemu_file_set_error(f, pages);
|
2015-05-07 20:33:31 +03:00
|
|
|
break;
|
|
|
|
}
|
2019-10-07 17:36:39 +03:00
|
|
|
|
|
|
|
rs->target_page_count += pages;
|
|
|
|
|
2019-11-07 15:39:07 +03:00
|
|
|
/*
|
|
|
|
* During postcopy, it is necessary to make sure one whole host
|
|
|
|
* page is sent in one chunk.
|
|
|
|
*/
|
|
|
|
if (migrate_postcopy_ram()) {
|
|
|
|
flush_compressed_data(rs);
|
|
|
|
}
|
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
/*
|
|
|
|
* we want to check in the 1st loop, just in case it was the 1st
|
|
|
|
* time and we had to sync the dirty bitmap.
|
|
|
|
* qemu_clock_get_ns() is a bit expensive, so we only check each
|
|
|
|
* some iterations
|
|
|
|
*/
|
|
|
|
if ((i & 63) == 0) {
|
|
|
|
uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) /
|
|
|
|
1000000;
|
|
|
|
if (t1 > MAX_WAIT) {
|
|
|
|
trace_ram_save_iterate_big_wait(t1, i);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
i++;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
}
|
migration: Move bitmap_mutex out of migration_bitmap_clear_dirty()
Taking the mutex every time for each dirty bit to clear is too slow, especially
we'll take/release even if the dirty bit is cleared. So far it's only used to
sync with special cases with qemu_guest_free_page_hint() against migration
thread, nothing really that serious yet. Let's move the lock to be upper.
There're two callers of migration_bitmap_clear_dirty().
For migration, move it into ram_save_iterate(). With the help of MAX_WAIT
logic, we'll only run ram_save_iterate() for no more than 50ms-ish time, so
taking the lock once there at the entry. It also means any call sites to
qemu_guest_free_page_hint() can be delayed; but it should be very rare, only
during migration, and I don't see a problem with it.
For COLO, move it up to colo_flush_ram_cache(). I think COLO forgot to take
that lock even when calling ramblock_sync_dirty_bitmap(), where another example
is migration_bitmap_sync() who took it right. So let the mutex cover both the
ramblock_sync_dirty_bitmap() and migration_bitmap_clear_dirty() calls.
It's even possible to drop the lock so we use atomic operations upon rb->bmap
and the variable migration_dirty_pages. I didn't do it just to still be safe,
also not predictable whether the frequent atomic ops could bring overhead too
e.g. on huge vms when it happens very often. When that really comes, we can
keep a local counter and periodically call atomic ops. Keep it simple for now.
Cc: Wei Wang <wei.w.wang@intel.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Hailiang Zhang <zhang.zhanghailiang@huawei.com>
Cc: Dr. David Alan Gilbert <dgilbert@redhat.com>
Cc: Juan Quintela <quintela@redhat.com>
Cc: Leonardo Bras Soares Passos <lsoaresp@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20210630200805.280905-1-peterx@redhat.com>
Reviewed-by: Wei Wang <wei.w.wang@intel.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2021-06-30 23:08:05 +03:00
|
|
|
qemu_mutex_unlock(&rs->bitmap_mutex);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Must occur before EOS (or any QEMUFile operation)
|
|
|
|
* because of RDMA protocol.
|
|
|
|
*/
|
|
|
|
ram_control_after_iterate(f, RAM_CONTROL_ROUND);
|
|
|
|
|
2018-03-08 14:18:24 +03:00
|
|
|
out:
|
2020-01-22 13:36:12 +03:00
|
|
|
if (ret >= 0
|
|
|
|
&& migration_is_setup_or_active(migrate_get_current()->state)) {
|
2022-10-12 00:55:59 +03:00
|
|
|
ret = multifd_send_sync_main(rs->pss[RAM_CHANNEL_PRECOPY].pss_channel);
|
2022-05-13 09:28:35 +03:00
|
|
|
if (ret < 0) {
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2019-12-18 07:12:36 +03:00
|
|
|
qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
|
|
|
|
qemu_fflush(f);
|
2021-12-21 12:34:40 +03:00
|
|
|
ram_transferred_add(8);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2019-12-18 07:12:36 +03:00
|
|
|
ret = qemu_file_get_error(f);
|
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
if (ret < 0) {
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2016-11-04 16:10:17 +03:00
|
|
|
return done;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* ram_save_complete: function called to send the remaining amount of ram
|
|
|
|
*
|
2018-09-03 12:26:44 +03:00
|
|
|
* Returns zero to indicate success or negative on error
|
2017-03-23 17:06:39 +03:00
|
|
|
*
|
|
|
|
* Called with iothread lock
|
|
|
|
*
|
|
|
|
* @f: QEMUFile where to send the data
|
|
|
|
* @opaque: RAMState pointer
|
|
|
|
*/
|
2015-05-07 20:33:31 +03:00
|
|
|
static int ram_save_complete(QEMUFile *f, void *opaque)
|
|
|
|
{
|
2017-05-04 12:46:24 +03:00
|
|
|
RAMState **temp = opaque;
|
|
|
|
RAMState *rs = *temp;
|
2018-09-03 12:26:44 +03:00
|
|
|
int ret = 0;
|
2017-03-13 21:26:29 +03:00
|
|
|
|
2021-12-15 21:01:21 +03:00
|
|
|
rs->last_stage = !migration_in_colo_state();
|
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
WITH_RCU_READ_LOCK_GUARD() {
|
|
|
|
if (!migration_in_postcopy()) {
|
|
|
|
migration_bitmap_sync_precopy(rs);
|
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
ram_control_before_iterate(f, RAM_CONTROL_FINISH);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
/* try transferring iterative blocks of memory */
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
/* flush all remaining blocks regardless of rate limiting */
|
2022-10-12 00:55:45 +03:00
|
|
|
qemu_mutex_lock(&rs->bitmap_mutex);
|
2019-10-07 17:36:39 +03:00
|
|
|
while (true) {
|
|
|
|
int pages;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2021-12-15 21:01:21 +03:00
|
|
|
pages = ram_find_and_save_block(rs);
|
2019-10-07 17:36:39 +03:00
|
|
|
/* no more blocks to sent */
|
|
|
|
if (pages == 0) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (pages < 0) {
|
|
|
|
ret = pages;
|
|
|
|
break;
|
|
|
|
}
|
2018-09-03 12:26:44 +03:00
|
|
|
}
|
2022-10-12 00:55:45 +03:00
|
|
|
qemu_mutex_unlock(&rs->bitmap_mutex);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
flush_compressed_data(rs);
|
|
|
|
ram_control_after_iterate(f, RAM_CONTROL_FINISH);
|
|
|
|
}
|
2015-07-09 09:47:58 +03:00
|
|
|
|
2022-05-13 09:28:35 +03:00
|
|
|
if (ret < 0) {
|
|
|
|
return ret;
|
2019-12-18 07:12:36 +03:00
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2022-10-12 00:55:59 +03:00
|
|
|
ret = multifd_send_sync_main(rs->pss[RAM_CHANNEL_PRECOPY].pss_channel);
|
2022-05-13 09:28:35 +03:00
|
|
|
if (ret < 0) {
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
|
|
|
|
qemu_fflush(f);
|
|
|
|
|
|
|
|
return 0;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2023-02-08 16:48:02 +03:00
|
|
|
static void ram_state_pending_estimate(void *opaque, uint64_t *must_precopy,
|
|
|
|
uint64_t *can_postcopy)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2017-05-04 12:46:24 +03:00
|
|
|
RAMState **temp = opaque;
|
|
|
|
RAMState *rs = *temp;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2022-10-03 03:00:03 +03:00
|
|
|
uint64_t remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2022-10-03 03:00:03 +03:00
|
|
|
if (migrate_postcopy_ram()) {
|
|
|
|
/* We can do postcopy, and all the data is postcopiable */
|
2023-02-08 16:48:02 +03:00
|
|
|
*can_postcopy += remaining_size;
|
2022-10-03 03:00:03 +03:00
|
|
|
} else {
|
2023-02-08 16:48:02 +03:00
|
|
|
*must_precopy += remaining_size;
|
2022-10-03 03:00:03 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-02-08 16:48:02 +03:00
|
|
|
static void ram_state_pending_exact(void *opaque, uint64_t *must_precopy,
|
|
|
|
uint64_t *can_postcopy)
|
2022-10-03 03:00:03 +03:00
|
|
|
{
|
2023-04-12 23:30:20 +03:00
|
|
|
MigrationState *s = migrate_get_current();
|
2022-10-03 03:00:03 +03:00
|
|
|
RAMState **temp = opaque;
|
|
|
|
RAMState *rs = *temp;
|
|
|
|
|
|
|
|
uint64_t remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
|
|
|
|
|
2023-04-12 23:30:20 +03:00
|
|
|
if (!migration_in_postcopy() && remaining_size < s->threshold_size) {
|
2015-05-07 20:33:31 +03:00
|
|
|
qemu_mutex_lock_iothread();
|
2019-10-07 17:36:39 +03:00
|
|
|
WITH_RCU_READ_LOCK_GUARD() {
|
|
|
|
migration_bitmap_sync_precopy(rs);
|
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
qemu_mutex_unlock_iothread();
|
2017-03-14 14:02:16 +03:00
|
|
|
remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
2015-11-05 21:10:54 +03:00
|
|
|
|
2017-07-10 19:30:15 +03:00
|
|
|
if (migrate_postcopy_ram()) {
|
|
|
|
/* We can do postcopy, and all the data is postcopiable */
|
2023-02-08 16:48:02 +03:00
|
|
|
*can_postcopy += remaining_size;
|
2017-07-10 19:30:15 +03:00
|
|
|
} else {
|
2023-02-08 16:48:02 +03:00
|
|
|
*must_precopy += remaining_size;
|
2017-07-10 19:30:15 +03:00
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
|
|
|
|
{
|
|
|
|
unsigned int xh_len;
|
|
|
|
int xh_flags;
|
2015-12-16 14:47:37 +03:00
|
|
|
uint8_t *loaded_data;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
|
|
|
/* extract RLE header */
|
|
|
|
xh_flags = qemu_get_byte(f);
|
|
|
|
xh_len = qemu_get_be16(f);
|
|
|
|
|
|
|
|
if (xh_flags != ENCODING_FLAG_XBZRLE) {
|
|
|
|
error_report("Failed to load XBZRLE page - wrong compression!");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (xh_len > TARGET_PAGE_SIZE) {
|
|
|
|
error_report("Failed to load XBZRLE page - len overflow!");
|
|
|
|
return -1;
|
|
|
|
}
|
2017-06-28 12:52:27 +03:00
|
|
|
loaded_data = XBZRLE.decoded_buf;
|
2015-05-07 20:33:31 +03:00
|
|
|
/* load data and decode */
|
2017-06-28 12:52:27 +03:00
|
|
|
/* it can change loaded_data to point to an internal buffer */
|
2015-12-16 14:47:37 +03:00
|
|
|
qemu_get_buffer_in_place(f, &loaded_data, xh_len);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
|
|
|
/* decode RLE */
|
2015-12-16 14:47:37 +03:00
|
|
|
if (xbzrle_decode_buffer(loaded_data, xh_len, host,
|
2015-05-07 20:33:31 +03:00
|
|
|
TARGET_PAGE_SIZE) == -1) {
|
|
|
|
error_report("Failed to load XBZRLE page - decode error!");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* ram_block_from_stream: read a RAMBlock id from the migration stream
|
|
|
|
*
|
|
|
|
* Must be called from within a rcu critical section.
|
|
|
|
*
|
2015-05-07 20:33:31 +03:00
|
|
|
* Returns a pointer from within the RCU-protected ram_list.
|
2015-11-05 21:11:11 +03:00
|
|
|
*
|
2022-03-01 11:39:07 +03:00
|
|
|
* @mis: the migration incoming state pointer
|
2017-03-23 17:06:39 +03:00
|
|
|
* @f: QEMUFile where to read the data from
|
|
|
|
* @flags: Page flags (mostly to see if it's a continuation of previous block)
|
migration: Postcopy preemption enablement
This patch enables postcopy-preempt feature.
It contains two major changes to the migration logic:
(1) Postcopy requests are now sent via a different socket from precopy
background migration stream, so as to be isolated from very high page
request delays.
(2) For huge page enabled hosts: when there's postcopy requests, they can now
intercept a partial sending of huge host pages on src QEMU.
After this patch, we'll live migrate a VM with two channels for postcopy: (1)
PRECOPY channel, which is the default channel that transfers background pages;
and (2) POSTCOPY channel, which only transfers requested pages.
There's no strict rule of which channel to use, e.g., if a requested page is
already being transferred on precopy channel, then we will keep using the same
precopy channel to transfer the page even if it's explicitly requested. In 99%
of the cases we'll prioritize the channels so we send requested page via the
postcopy channel as long as possible.
On the source QEMU, when we found a postcopy request, we'll interrupt the
PRECOPY channel sending process and quickly switch to the POSTCOPY channel.
After we serviced all the high priority postcopy pages, we'll switch back to
PRECOPY channel so that we'll continue to send the interrupted huge page again.
There's no new thread introduced on src QEMU.
On the destination QEMU, one new thread is introduced to receive page data from
the postcopy specific socket (done in the preparation patch).
This patch has a side effect: after sending postcopy pages, previously we'll
assume the guest will access follow up pages so we'll keep sending from there.
Now it's changed. Instead of going on with a postcopy requested page, we'll go
back and continue sending the precopy huge page (which can be intercepted by a
postcopy request so the huge page can be sent partially before).
Whether that's a problem is debatable, because "assuming the guest will
continue to access the next page" may not really suite when huge pages are
used, especially if the huge page is large (e.g. 1GB pages). So that locality
hint is much meaningless if huge pages are used.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220707185504.27203-1-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2022-07-07 21:55:04 +03:00
|
|
|
* @channel: the channel we're using
|
2015-11-05 21:11:11 +03:00
|
|
|
*/
|
2022-03-01 11:39:07 +03:00
|
|
|
static inline RAMBlock *ram_block_from_stream(MigrationIncomingState *mis,
|
migration: Postcopy preemption enablement
This patch enables postcopy-preempt feature.
It contains two major changes to the migration logic:
(1) Postcopy requests are now sent via a different socket from precopy
background migration stream, so as to be isolated from very high page
request delays.
(2) For huge page enabled hosts: when there's postcopy requests, they can now
intercept a partial sending of huge host pages on src QEMU.
After this patch, we'll live migrate a VM with two channels for postcopy: (1)
PRECOPY channel, which is the default channel that transfers background pages;
and (2) POSTCOPY channel, which only transfers requested pages.
There's no strict rule of which channel to use, e.g., if a requested page is
already being transferred on precopy channel, then we will keep using the same
precopy channel to transfer the page even if it's explicitly requested. In 99%
of the cases we'll prioritize the channels so we send requested page via the
postcopy channel as long as possible.
On the source QEMU, when we found a postcopy request, we'll interrupt the
PRECOPY channel sending process and quickly switch to the POSTCOPY channel.
After we serviced all the high priority postcopy pages, we'll switch back to
PRECOPY channel so that we'll continue to send the interrupted huge page again.
There's no new thread introduced on src QEMU.
On the destination QEMU, one new thread is introduced to receive page data from
the postcopy specific socket (done in the preparation patch).
This patch has a side effect: after sending postcopy pages, previously we'll
assume the guest will access follow up pages so we'll keep sending from there.
Now it's changed. Instead of going on with a postcopy requested page, we'll go
back and continue sending the precopy huge page (which can be intercepted by a
postcopy request so the huge page can be sent partially before).
Whether that's a problem is debatable, because "assuming the guest will
continue to access the next page" may not really suite when huge pages are
used, especially if the huge page is large (e.g. 1GB pages). So that locality
hint is much meaningless if huge pages are used.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220707185504.27203-1-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2022-07-07 21:55:04 +03:00
|
|
|
QEMUFile *f, int flags,
|
|
|
|
int channel)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
migration: Postcopy preemption enablement
This patch enables postcopy-preempt feature.
It contains two major changes to the migration logic:
(1) Postcopy requests are now sent via a different socket from precopy
background migration stream, so as to be isolated from very high page
request delays.
(2) For huge page enabled hosts: when there's postcopy requests, they can now
intercept a partial sending of huge host pages on src QEMU.
After this patch, we'll live migrate a VM with two channels for postcopy: (1)
PRECOPY channel, which is the default channel that transfers background pages;
and (2) POSTCOPY channel, which only transfers requested pages.
There's no strict rule of which channel to use, e.g., if a requested page is
already being transferred on precopy channel, then we will keep using the same
precopy channel to transfer the page even if it's explicitly requested. In 99%
of the cases we'll prioritize the channels so we send requested page via the
postcopy channel as long as possible.
On the source QEMU, when we found a postcopy request, we'll interrupt the
PRECOPY channel sending process and quickly switch to the POSTCOPY channel.
After we serviced all the high priority postcopy pages, we'll switch back to
PRECOPY channel so that we'll continue to send the interrupted huge page again.
There's no new thread introduced on src QEMU.
On the destination QEMU, one new thread is introduced to receive page data from
the postcopy specific socket (done in the preparation patch).
This patch has a side effect: after sending postcopy pages, previously we'll
assume the guest will access follow up pages so we'll keep sending from there.
Now it's changed. Instead of going on with a postcopy requested page, we'll go
back and continue sending the precopy huge page (which can be intercepted by a
postcopy request so the huge page can be sent partially before).
Whether that's a problem is debatable, because "assuming the guest will
continue to access the next page" may not really suite when huge pages are
used, especially if the huge page is large (e.g. 1GB pages). So that locality
hint is much meaningless if huge pages are used.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220707185504.27203-1-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2022-07-07 21:55:04 +03:00
|
|
|
RAMBlock *block = mis->last_recv_block[channel];
|
2015-05-07 20:33:31 +03:00
|
|
|
char id[256];
|
|
|
|
uint8_t len;
|
|
|
|
|
|
|
|
if (flags & RAM_SAVE_FLAG_CONTINUE) {
|
2016-01-15 06:37:41 +03:00
|
|
|
if (!block) {
|
2015-05-07 20:33:31 +03:00
|
|
|
error_report("Ack, bad migration stream!");
|
|
|
|
return NULL;
|
|
|
|
}
|
2016-01-15 06:37:41 +03:00
|
|
|
return block;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
len = qemu_get_byte(f);
|
|
|
|
qemu_get_buffer(f, (uint8_t *)id, len);
|
|
|
|
id[len] = 0;
|
|
|
|
|
2015-11-05 21:10:33 +03:00
|
|
|
block = qemu_ram_block_by_name(id);
|
2016-01-15 06:37:41 +03:00
|
|
|
if (!block) {
|
|
|
|
error_report("Can't find block %s", id);
|
|
|
|
return NULL;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
if (ramblock_is_ignored(block)) {
|
2018-05-14 09:57:00 +03:00
|
|
|
error_report("block %s should not be migrated !", id);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
migration: Postcopy preemption enablement
This patch enables postcopy-preempt feature.
It contains two major changes to the migration logic:
(1) Postcopy requests are now sent via a different socket from precopy
background migration stream, so as to be isolated from very high page
request delays.
(2) For huge page enabled hosts: when there's postcopy requests, they can now
intercept a partial sending of huge host pages on src QEMU.
After this patch, we'll live migrate a VM with two channels for postcopy: (1)
PRECOPY channel, which is the default channel that transfers background pages;
and (2) POSTCOPY channel, which only transfers requested pages.
There's no strict rule of which channel to use, e.g., if a requested page is
already being transferred on precopy channel, then we will keep using the same
precopy channel to transfer the page even if it's explicitly requested. In 99%
of the cases we'll prioritize the channels so we send requested page via the
postcopy channel as long as possible.
On the source QEMU, when we found a postcopy request, we'll interrupt the
PRECOPY channel sending process and quickly switch to the POSTCOPY channel.
After we serviced all the high priority postcopy pages, we'll switch back to
PRECOPY channel so that we'll continue to send the interrupted huge page again.
There's no new thread introduced on src QEMU.
On the destination QEMU, one new thread is introduced to receive page data from
the postcopy specific socket (done in the preparation patch).
This patch has a side effect: after sending postcopy pages, previously we'll
assume the guest will access follow up pages so we'll keep sending from there.
Now it's changed. Instead of going on with a postcopy requested page, we'll go
back and continue sending the precopy huge page (which can be intercepted by a
postcopy request so the huge page can be sent partially before).
Whether that's a problem is debatable, because "assuming the guest will
continue to access the next page" may not really suite when huge pages are
used, especially if the huge page is large (e.g. 1GB pages). So that locality
hint is much meaningless if huge pages are used.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220707185504.27203-1-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2022-07-07 21:55:04 +03:00
|
|
|
mis->last_recv_block[channel] = block;
|
2022-03-01 11:39:07 +03:00
|
|
|
|
2016-01-15 06:37:41 +03:00
|
|
|
return block;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void *host_from_ram_block_offset(RAMBlock *block,
|
|
|
|
ram_addr_t offset)
|
|
|
|
{
|
|
|
|
if (!offset_in_ramblock(block, offset)) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return block->host + offset;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2021-04-29 14:27:05 +03:00
|
|
|
static void *host_page_from_ram_block_offset(RAMBlock *block,
|
|
|
|
ram_addr_t offset)
|
|
|
|
{
|
|
|
|
/* Note: Explicitly no check against offset_in_ramblock(). */
|
|
|
|
return (void *)QEMU_ALIGN_DOWN((uintptr_t)(block->host + offset),
|
|
|
|
block->page_size);
|
|
|
|
}
|
|
|
|
|
|
|
|
static ram_addr_t host_page_offset_from_ram_block_offset(RAMBlock *block,
|
|
|
|
ram_addr_t offset)
|
|
|
|
{
|
|
|
|
return ((uintptr_t)block->host + offset) & (block->page_size - 1);
|
|
|
|
}
|
|
|
|
|
2018-09-03 07:38:48 +03:00
|
|
|
static inline void *colo_cache_from_block_offset(RAMBlock *block,
|
2020-02-24 09:54:11 +03:00
|
|
|
ram_addr_t offset, bool record_bitmap)
|
2018-09-03 07:38:48 +03:00
|
|
|
{
|
|
|
|
if (!offset_in_ramblock(block, offset)) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
if (!block->colo_cache) {
|
|
|
|
error_report("%s: colo_cache is NULL in block :%s",
|
|
|
|
__func__, block->idstr);
|
|
|
|
return NULL;
|
|
|
|
}
|
2018-09-03 07:38:49 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* During colo checkpoint, we need bitmap of these migrated pages.
|
|
|
|
* It help us to decide which pages in ram cache should be flushed
|
|
|
|
* into VM's RAM later.
|
|
|
|
*/
|
2020-02-24 09:54:11 +03:00
|
|
|
if (record_bitmap &&
|
|
|
|
!test_and_set_bit(offset >> TARGET_PAGE_BITS, block->bmap)) {
|
2018-09-03 07:38:49 +03:00
|
|
|
ram_state->migration_dirty_pages++;
|
|
|
|
}
|
2018-09-03 07:38:48 +03:00
|
|
|
return block->colo_cache + offset;
|
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* ram_handle_compressed: handle the zero page case
|
|
|
|
*
|
2015-05-07 20:33:31 +03:00
|
|
|
* If a page (or a whole RDMA chunk) has been
|
|
|
|
* determined to be zero, then zap it.
|
2017-03-23 17:06:39 +03:00
|
|
|
*
|
|
|
|
* @host: host address for the zero page
|
|
|
|
* @ch: what the page is filled from. We only support zero
|
|
|
|
* @size: size of the zero page
|
2015-05-07 20:33:31 +03:00
|
|
|
*/
|
|
|
|
void ram_handle_compressed(void *host, uint8_t ch, uint64_t size)
|
|
|
|
{
|
2021-11-18 17:56:38 +03:00
|
|
|
if (ch != 0 || !buffer_is_zero(host, size)) {
|
2015-05-07 20:33:31 +03:00
|
|
|
memset(host, ch, size);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-30 10:51:21 +03:00
|
|
|
/* return the size after decompression, or negative value on error */
|
|
|
|
static int
|
|
|
|
qemu_uncompress_data(z_stream *stream, uint8_t *dest, size_t dest_len,
|
|
|
|
const uint8_t *source, size_t source_len)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = inflateReset(stream);
|
|
|
|
if (err != Z_OK) {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
stream->avail_in = source_len;
|
|
|
|
stream->next_in = (uint8_t *)source;
|
|
|
|
stream->avail_out = dest_len;
|
|
|
|
stream->next_out = dest;
|
|
|
|
|
|
|
|
err = inflate(stream, Z_NO_FLUSH);
|
|
|
|
if (err != Z_STREAM_END) {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return stream->total_out;
|
|
|
|
}
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
static void *do_data_decompress(void *opaque)
|
|
|
|
{
|
|
|
|
DecompressParam *param = opaque;
|
|
|
|
unsigned long pagesize;
|
2016-05-05 10:32:58 +03:00
|
|
|
uint8_t *des;
|
2018-03-30 10:51:22 +03:00
|
|
|
int len, ret;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2016-05-05 10:32:58 +03:00
|
|
|
qemu_mutex_lock(¶m->mutex);
|
2016-05-05 10:32:56 +03:00
|
|
|
while (!param->quit) {
|
2016-05-05 10:32:58 +03:00
|
|
|
if (param->des) {
|
|
|
|
des = param->des;
|
|
|
|
len = param->len;
|
|
|
|
param->des = 0;
|
|
|
|
qemu_mutex_unlock(¶m->mutex);
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
pagesize = TARGET_PAGE_SIZE;
|
2018-03-30 10:51:22 +03:00
|
|
|
|
|
|
|
ret = qemu_uncompress_data(¶m->stream, des, pagesize,
|
|
|
|
param->compbuf, len);
|
2018-05-03 11:06:11 +03:00
|
|
|
if (ret < 0 && migrate_get_current()->decompress_error_check) {
|
2018-03-30 10:51:22 +03:00
|
|
|
error_report("decompress data failed");
|
|
|
|
qemu_file_set_error(decomp_file, ret);
|
|
|
|
}
|
2016-05-05 10:32:51 +03:00
|
|
|
|
2016-05-05 10:32:58 +03:00
|
|
|
qemu_mutex_lock(&decomp_done_lock);
|
|
|
|
param->done = true;
|
|
|
|
qemu_cond_signal(&decomp_done_cond);
|
|
|
|
qemu_mutex_unlock(&decomp_done_lock);
|
|
|
|
|
|
|
|
qemu_mutex_lock(¶m->mutex);
|
|
|
|
} else {
|
|
|
|
qemu_cond_wait(¶m->cond, ¶m->mutex);
|
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
2016-05-05 10:32:58 +03:00
|
|
|
qemu_mutex_unlock(¶m->mutex);
|
2015-05-07 20:33:31 +03:00
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2018-03-30 10:51:22 +03:00
|
|
|
static int wait_for_decompress_done(void)
|
2016-05-05 10:32:52 +03:00
|
|
|
{
|
|
|
|
int idx, thread_count;
|
|
|
|
|
2023-03-02 00:03:48 +03:00
|
|
|
if (!migrate_compress()) {
|
2018-03-30 10:51:22 +03:00
|
|
|
return 0;
|
2016-05-05 10:32:52 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
thread_count = migrate_decompress_threads();
|
|
|
|
qemu_mutex_lock(&decomp_done_lock);
|
|
|
|
for (idx = 0; idx < thread_count; idx++) {
|
|
|
|
while (!decomp_param[idx].done) {
|
|
|
|
qemu_cond_wait(&decomp_done_cond, &decomp_done_lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
qemu_mutex_unlock(&decomp_done_lock);
|
2018-03-30 10:51:22 +03:00
|
|
|
return qemu_file_get_error(decomp_file);
|
2016-05-05 10:32:52 +03:00
|
|
|
}
|
|
|
|
|
2017-06-28 12:52:28 +03:00
|
|
|
static void compress_threads_load_cleanup(void)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
|
|
|
int i, thread_count;
|
|
|
|
|
2023-03-02 00:03:48 +03:00
|
|
|
if (!migrate_compress()) {
|
2016-04-20 12:56:01 +03:00
|
|
|
return;
|
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
thread_count = migrate_decompress_threads();
|
|
|
|
for (i = 0; i < thread_count; i++) {
|
2018-03-30 10:51:21 +03:00
|
|
|
/*
|
|
|
|
* we use it as a indicator which shows if the thread is
|
|
|
|
* properly init'd or not
|
|
|
|
*/
|
|
|
|
if (!decomp_param[i].compbuf) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
qemu_mutex_lock(&decomp_param[i].mutex);
|
2016-05-05 10:32:56 +03:00
|
|
|
decomp_param[i].quit = true;
|
2015-05-07 20:33:31 +03:00
|
|
|
qemu_cond_signal(&decomp_param[i].cond);
|
|
|
|
qemu_mutex_unlock(&decomp_param[i].mutex);
|
|
|
|
}
|
|
|
|
for (i = 0; i < thread_count; i++) {
|
2018-03-30 10:51:21 +03:00
|
|
|
if (!decomp_param[i].compbuf) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
qemu_thread_join(decompress_threads + i);
|
|
|
|
qemu_mutex_destroy(&decomp_param[i].mutex);
|
|
|
|
qemu_cond_destroy(&decomp_param[i].cond);
|
2018-03-30 10:51:21 +03:00
|
|
|
inflateEnd(&decomp_param[i].stream);
|
2015-05-07 20:33:31 +03:00
|
|
|
g_free(decomp_param[i].compbuf);
|
2018-03-30 10:51:21 +03:00
|
|
|
decomp_param[i].compbuf = NULL;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
g_free(decompress_threads);
|
|
|
|
g_free(decomp_param);
|
|
|
|
decompress_threads = NULL;
|
|
|
|
decomp_param = NULL;
|
2018-03-30 10:51:22 +03:00
|
|
|
decomp_file = NULL;
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2018-03-30 10:51:22 +03:00
|
|
|
static int compress_threads_load_setup(QEMUFile *f)
|
2018-03-30 10:51:21 +03:00
|
|
|
{
|
|
|
|
int i, thread_count;
|
|
|
|
|
2023-03-02 00:03:48 +03:00
|
|
|
if (!migrate_compress()) {
|
2018-03-30 10:51:21 +03:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
thread_count = migrate_decompress_threads();
|
|
|
|
decompress_threads = g_new0(QemuThread, thread_count);
|
|
|
|
decomp_param = g_new0(DecompressParam, thread_count);
|
|
|
|
qemu_mutex_init(&decomp_done_lock);
|
|
|
|
qemu_cond_init(&decomp_done_cond);
|
2018-03-30 10:51:22 +03:00
|
|
|
decomp_file = f;
|
2018-03-30 10:51:21 +03:00
|
|
|
for (i = 0; i < thread_count; i++) {
|
|
|
|
if (inflateInit(&decomp_param[i].stream) != Z_OK) {
|
|
|
|
goto exit;
|
|
|
|
}
|
|
|
|
|
|
|
|
decomp_param[i].compbuf = g_malloc0(compressBound(TARGET_PAGE_SIZE));
|
|
|
|
qemu_mutex_init(&decomp_param[i].mutex);
|
|
|
|
qemu_cond_init(&decomp_param[i].cond);
|
|
|
|
decomp_param[i].done = true;
|
|
|
|
decomp_param[i].quit = false;
|
|
|
|
qemu_thread_create(decompress_threads + i, "decompress",
|
|
|
|
do_data_decompress, decomp_param + i,
|
|
|
|
QEMU_THREAD_JOINABLE);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
exit:
|
|
|
|
compress_threads_load_cleanup();
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2015-12-16 14:47:38 +03:00
|
|
|
static void decompress_data_with_multi_threads(QEMUFile *f,
|
2015-05-07 20:33:31 +03:00
|
|
|
void *host, int len)
|
|
|
|
{
|
|
|
|
int idx, thread_count;
|
|
|
|
|
|
|
|
thread_count = migrate_decompress_threads();
|
2021-03-11 06:15:35 +03:00
|
|
|
QEMU_LOCK_GUARD(&decomp_done_lock);
|
2015-05-07 20:33:31 +03:00
|
|
|
while (true) {
|
|
|
|
for (idx = 0; idx < thread_count; idx++) {
|
2016-05-05 10:32:51 +03:00
|
|
|
if (decomp_param[idx].done) {
|
2016-05-05 10:32:58 +03:00
|
|
|
decomp_param[idx].done = false;
|
|
|
|
qemu_mutex_lock(&decomp_param[idx].mutex);
|
2015-12-16 14:47:38 +03:00
|
|
|
qemu_get_buffer(f, decomp_param[idx].compbuf, len);
|
2015-05-07 20:33:31 +03:00
|
|
|
decomp_param[idx].des = host;
|
|
|
|
decomp_param[idx].len = len;
|
2016-05-05 10:32:58 +03:00
|
|
|
qemu_cond_signal(&decomp_param[idx].cond);
|
|
|
|
qemu_mutex_unlock(&decomp_param[idx].mutex);
|
2015-05-07 20:33:31 +03:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (idx < thread_count) {
|
|
|
|
break;
|
2016-05-05 10:32:51 +03:00
|
|
|
} else {
|
|
|
|
qemu_cond_wait(&decomp_done_cond, &decomp_done_lock);
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-10-16 08:52:01 +03:00
|
|
|
static void colo_init_ram_state(void)
|
|
|
|
{
|
|
|
|
ram_state_init(&ram_state);
|
|
|
|
}
|
|
|
|
|
2018-09-03 07:38:48 +03:00
|
|
|
/*
|
|
|
|
* colo cache: this is for secondary VM, we cache the whole
|
|
|
|
* memory of the secondary VM, it is need to hold the global lock
|
|
|
|
* to call this helper.
|
|
|
|
*/
|
|
|
|
int colo_init_ram_cache(void)
|
|
|
|
{
|
|
|
|
RAMBlock *block;
|
|
|
|
|
2019-12-13 17:07:22 +03:00
|
|
|
WITH_RCU_READ_LOCK_GUARD() {
|
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
|
|
|
block->colo_cache = qemu_anon_ram_alloc(block->used_length,
|
2021-05-10 14:43:21 +03:00
|
|
|
NULL, false, false);
|
2019-12-13 17:07:22 +03:00
|
|
|
if (!block->colo_cache) {
|
|
|
|
error_report("%s: Can't alloc memory for COLO cache of block %s,"
|
|
|
|
"size 0x" RAM_ADDR_FMT, __func__, block->idstr,
|
|
|
|
block->used_length);
|
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
|
|
|
if (block->colo_cache) {
|
|
|
|
qemu_anon_ram_free(block->colo_cache, block->used_length);
|
|
|
|
block->colo_cache = NULL;
|
|
|
|
}
|
2019-10-07 17:36:39 +03:00
|
|
|
}
|
2019-12-13 17:07:22 +03:00
|
|
|
return -errno;
|
2019-10-07 17:36:39 +03:00
|
|
|
}
|
2021-07-04 19:14:44 +03:00
|
|
|
if (!machine_dump_guest_core(current_machine)) {
|
|
|
|
qemu_madvise(block->colo_cache, block->used_length,
|
|
|
|
QEMU_MADV_DONTDUMP);
|
|
|
|
}
|
2018-09-03 07:38:48 +03:00
|
|
|
}
|
|
|
|
}
|
2019-12-13 17:07:22 +03:00
|
|
|
|
2018-09-03 07:38:49 +03:00
|
|
|
/*
|
|
|
|
* Record the dirty pages that sent by PVM, we use this dirty bitmap together
|
|
|
|
* with to decide which page in cache should be flushed into SVM's RAM. Here
|
|
|
|
* we use the same name 'ram_bitmap' as for migration.
|
|
|
|
*/
|
|
|
|
if (ram_bytes_total()) {
|
|
|
|
RAMBlock *block;
|
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
2018-09-03 07:38:49 +03:00
|
|
|
unsigned long pages = block->max_length >> TARGET_PAGE_BITS;
|
|
|
|
block->bmap = bitmap_new(pages);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-10-16 08:52:01 +03:00
|
|
|
colo_init_ram_state();
|
2018-09-03 07:38:48 +03:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-02-24 09:54:10 +03:00
|
|
|
/* TODO: duplicated with ram_init_bitmaps */
|
|
|
|
void colo_incoming_start_dirty_log(void)
|
|
|
|
{
|
|
|
|
RAMBlock *block = NULL;
|
|
|
|
/* For memory_global_dirty_log_start below. */
|
|
|
|
qemu_mutex_lock_iothread();
|
|
|
|
qemu_mutex_lock_ramlist();
|
|
|
|
|
|
|
|
memory_global_dirty_log_sync();
|
|
|
|
WITH_RCU_READ_LOCK_GUARD() {
|
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
|
|
|
ramblock_sync_dirty_bitmap(ram_state, block);
|
|
|
|
/* Discard this dirty bitmap record */
|
|
|
|
bitmap_zero(block->bmap, block->max_length >> TARGET_PAGE_BITS);
|
|
|
|
}
|
2021-06-29 19:01:19 +03:00
|
|
|
memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
|
2020-02-24 09:54:10 +03:00
|
|
|
}
|
|
|
|
ram_state->migration_dirty_pages = 0;
|
|
|
|
qemu_mutex_unlock_ramlist();
|
|
|
|
qemu_mutex_unlock_iothread();
|
|
|
|
}
|
|
|
|
|
2018-09-03 07:38:48 +03:00
|
|
|
/* It is need to hold the global lock to call this helper */
|
|
|
|
void colo_release_ram_cache(void)
|
|
|
|
{
|
|
|
|
RAMBlock *block;
|
|
|
|
|
2021-06-29 19:01:19 +03:00
|
|
|
memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
2018-09-03 07:38:49 +03:00
|
|
|
g_free(block->bmap);
|
|
|
|
block->bmap = NULL;
|
|
|
|
}
|
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
WITH_RCU_READ_LOCK_GUARD() {
|
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
|
|
|
if (block->colo_cache) {
|
|
|
|
qemu_anon_ram_free(block->colo_cache, block->used_length);
|
|
|
|
block->colo_cache = NULL;
|
|
|
|
}
|
2018-09-03 07:38:48 +03:00
|
|
|
}
|
|
|
|
}
|
2020-02-24 09:54:10 +03:00
|
|
|
ram_state_cleanup(&ram_state);
|
2018-09-03 07:38:48 +03:00
|
|
|
}
|
|
|
|
|
2017-06-28 12:52:27 +03:00
|
|
|
/**
|
|
|
|
* ram_load_setup: Setup RAM for migration incoming side
|
|
|
|
*
|
|
|
|
* Returns zero to indicate success and negative for error
|
|
|
|
*
|
|
|
|
* @f: QEMUFile where to receive the data
|
|
|
|
* @opaque: RAMState pointer
|
|
|
|
*/
|
|
|
|
static int ram_load_setup(QEMUFile *f, void *opaque)
|
|
|
|
{
|
2018-03-30 10:51:22 +03:00
|
|
|
if (compress_threads_load_setup(f)) {
|
2018-03-30 10:51:21 +03:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2017-06-28 12:52:27 +03:00
|
|
|
xbzrle_load_setup();
|
2017-10-05 14:13:20 +03:00
|
|
|
ramblock_recv_map_init();
|
2018-09-03 07:38:48 +03:00
|
|
|
|
2017-06-28 12:52:27 +03:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int ram_load_cleanup(void *opaque)
|
|
|
|
{
|
2017-10-05 14:13:20 +03:00
|
|
|
RAMBlock *rb;
|
2018-07-18 10:48:03 +03:00
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
|
2019-11-21 03:08:42 +03:00
|
|
|
qemu_ram_block_writeback(rb);
|
2018-07-18 10:48:03 +03:00
|
|
|
}
|
|
|
|
|
2017-06-28 12:52:27 +03:00
|
|
|
xbzrle_load_cleanup();
|
2017-06-28 12:52:28 +03:00
|
|
|
compress_threads_load_cleanup();
|
2017-10-05 14:13:20 +03:00
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
|
2017-10-05 14:13:20 +03:00
|
|
|
g_free(rb->receivedmap);
|
|
|
|
rb->receivedmap = NULL;
|
|
|
|
}
|
2018-09-03 07:38:48 +03:00
|
|
|
|
2017-06-28 12:52:27 +03:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* ram_postcopy_incoming_init: allocate postcopy data structures
|
|
|
|
*
|
|
|
|
* Returns 0 for success and negative if there was one error
|
|
|
|
*
|
|
|
|
* @mis: current migration incoming state
|
|
|
|
*
|
|
|
|
* Allocate data structures etc needed by incoming migration with
|
|
|
|
* postcopy-ram. postcopy-ram's similarly names
|
|
|
|
* postcopy_ram_incoming_init does the work.
|
2015-11-05 21:11:03 +03:00
|
|
|
*/
|
|
|
|
int ram_postcopy_incoming_init(MigrationIncomingState *mis)
|
|
|
|
{
|
2018-06-20 23:27:36 +03:00
|
|
|
return postcopy_ram_incoming_init(mis);
|
2015-11-05 21:11:03 +03:00
|
|
|
}
|
|
|
|
|
2017-03-23 17:06:39 +03:00
|
|
|
/**
|
|
|
|
* ram_load_postcopy: load a page in postcopy case
|
|
|
|
*
|
|
|
|
* Returns 0 for success or -errno in case of error
|
|
|
|
*
|
2015-11-05 21:11:11 +03:00
|
|
|
* Called in postcopy mode by ram_load().
|
|
|
|
* rcu_read_lock is taken prior to this being called.
|
2017-03-23 17:06:39 +03:00
|
|
|
*
|
|
|
|
* @f: QEMUFile where to send the data
|
2022-07-07 21:55:02 +03:00
|
|
|
* @channel: the channel to use for loading
|
2015-11-05 21:11:11 +03:00
|
|
|
*/
|
2022-07-07 21:55:02 +03:00
|
|
|
int ram_load_postcopy(QEMUFile *f, int channel)
|
2015-11-05 21:11:11 +03:00
|
|
|
{
|
|
|
|
int flags = 0, ret = 0;
|
|
|
|
bool place_needed = false;
|
2018-07-10 12:18:53 +03:00
|
|
|
bool matches_target_page_size = false;
|
2015-11-05 21:11:11 +03:00
|
|
|
MigrationIncomingState *mis = migration_incoming_get_current();
|
2022-07-07 21:55:02 +03:00
|
|
|
PostcopyTmpPage *tmp_page = &mis->postcopy_tmp_pages[channel];
|
2015-11-05 21:11:11 +03:00
|
|
|
|
|
|
|
while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
|
|
|
|
ram_addr_t addr;
|
|
|
|
void *page_buffer = NULL;
|
|
|
|
void *place_source = NULL;
|
2017-02-24 21:28:35 +03:00
|
|
|
RAMBlock *block = NULL;
|
2015-11-05 21:11:11 +03:00
|
|
|
uint8_t ch;
|
2019-11-07 15:39:07 +03:00
|
|
|
int len;
|
2015-11-05 21:11:11 +03:00
|
|
|
|
|
|
|
addr = qemu_get_be64(f);
|
2018-02-08 13:31:05 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If qemu file error, we should stop here, and then "addr"
|
|
|
|
* may be invalid
|
|
|
|
*/
|
|
|
|
ret = qemu_file_get_error(f);
|
|
|
|
if (ret) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2015-11-05 21:11:11 +03:00
|
|
|
flags = addr & ~TARGET_PAGE_MASK;
|
|
|
|
addr &= TARGET_PAGE_MASK;
|
|
|
|
|
2022-07-07 21:55:02 +03:00
|
|
|
trace_ram_load_postcopy_loop(channel, (uint64_t)addr, flags);
|
2019-11-07 15:39:07 +03:00
|
|
|
if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE |
|
|
|
|
RAM_SAVE_FLAG_COMPRESS_PAGE)) {
|
migration: Postcopy preemption enablement
This patch enables postcopy-preempt feature.
It contains two major changes to the migration logic:
(1) Postcopy requests are now sent via a different socket from precopy
background migration stream, so as to be isolated from very high page
request delays.
(2) For huge page enabled hosts: when there's postcopy requests, they can now
intercept a partial sending of huge host pages on src QEMU.
After this patch, we'll live migrate a VM with two channels for postcopy: (1)
PRECOPY channel, which is the default channel that transfers background pages;
and (2) POSTCOPY channel, which only transfers requested pages.
There's no strict rule of which channel to use, e.g., if a requested page is
already being transferred on precopy channel, then we will keep using the same
precopy channel to transfer the page even if it's explicitly requested. In 99%
of the cases we'll prioritize the channels so we send requested page via the
postcopy channel as long as possible.
On the source QEMU, when we found a postcopy request, we'll interrupt the
PRECOPY channel sending process and quickly switch to the POSTCOPY channel.
After we serviced all the high priority postcopy pages, we'll switch back to
PRECOPY channel so that we'll continue to send the interrupted huge page again.
There's no new thread introduced on src QEMU.
On the destination QEMU, one new thread is introduced to receive page data from
the postcopy specific socket (done in the preparation patch).
This patch has a side effect: after sending postcopy pages, previously we'll
assume the guest will access follow up pages so we'll keep sending from there.
Now it's changed. Instead of going on with a postcopy requested page, we'll go
back and continue sending the precopy huge page (which can be intercepted by a
postcopy request so the huge page can be sent partially before).
Whether that's a problem is debatable, because "assuming the guest will
continue to access the next page" may not really suite when huge pages are
used, especially if the huge page is large (e.g. 1GB pages). So that locality
hint is much meaningless if huge pages are used.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220707185504.27203-1-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2022-07-07 21:55:04 +03:00
|
|
|
block = ram_block_from_stream(mis, f, flags, channel);
|
2021-04-29 14:27:05 +03:00
|
|
|
if (!block) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
break;
|
|
|
|
}
|
2016-01-15 06:37:41 +03:00
|
|
|
|
2021-04-29 14:27:06 +03:00
|
|
|
/*
|
|
|
|
* Relying on used_length is racy and can result in false positives.
|
|
|
|
* We might place pages beyond used_length in case RAM was shrunk
|
|
|
|
* while in postcopy, which is fine - trying to place via
|
|
|
|
* UFFDIO_COPY/UFFDIO_ZEROPAGE will never segfault.
|
|
|
|
*/
|
|
|
|
if (!block->host || addr >= block->postcopy_length) {
|
2015-11-05 21:11:11 +03:00
|
|
|
error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
|
|
|
|
ret = -EINVAL;
|
|
|
|
break;
|
|
|
|
}
|
migration: Introduce postcopy channels on dest node
Postcopy handles huge pages in a special way that currently we can only have
one "channel" to transfer the page.
It's because when we install pages using UFFDIO_COPY, we need to have the whole
huge page ready, it also means we need to have a temp huge page when trying to
receive the whole content of the page.
Currently all maintainance around this tmp page is global: firstly we'll
allocate a temp huge page, then we maintain its status mostly within
ram_load_postcopy().
To enable multiple channels for postcopy, the first thing we need to do is to
prepare N temp huge pages as caching, one for each channel.
Meanwhile we need to maintain the tmp huge page status per-channel too.
To give some example, some local variables maintained in ram_load_postcopy()
are listed; they are responsible for maintaining temp huge page status:
- all_zero: this keeps whether this huge page contains all zeros
- target_pages: this counts how many target pages have been copied
- host_page: this keeps the host ptr for the page to install
Move all these fields to be together with the temp huge pages to form a new
structure called PostcopyTmpPage. Then for each (future) postcopy channel, we
need one structure to keep the state around.
For vanilla postcopy, obviously there's only one channel. It contains both
precopy and postcopy pages.
This patch teaches the dest migration node to start realize the possible number
of postcopy channels by introducing the "postcopy_channels" variable. Its
value is calculated when setup postcopy on dest node (during POSTCOPY_LISTEN
phase).
Vanilla postcopy will have channels=1, but when postcopy-preempt capability is
enabled (in the future), we will boost it to 2 because even during partial
sending of a precopy huge page we still want to preempt it and start sending
the postcopy requested page right away (so we start to keep two temp huge
pages; more if we want to enable multifd). In this patch there's a TODO marked
for that; so far the channels is always set to 1.
We need to send one "host huge page" on one channel only and we cannot split
them, because otherwise the data upon the same huge page can locate on more
than one channel so we need more complicated logic to manage. One temp host
huge page for each channel will be enough for us for now.
Postcopy will still always use the index=0 huge page even after this patch.
However it prepares for the latter patches where it can start to use multiple
channels (which needs src intervention, because only src knows which channel we
should use).
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220301083925.33483-5-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
dgilbert: Fixed up long line
2022-03-01 11:39:04 +03:00
|
|
|
tmp_page->target_pages++;
|
2018-07-10 12:18:53 +03:00
|
|
|
matches_target_page_size = block->page_size == TARGET_PAGE_SIZE;
|
2015-11-05 21:11:11 +03:00
|
|
|
/*
|
2017-02-24 21:28:37 +03:00
|
|
|
* Postcopy requires that we place whole host pages atomically;
|
|
|
|
* these may be huge pages for RAMBlocks that are backed by
|
|
|
|
* hugetlbfs.
|
2015-11-05 21:11:11 +03:00
|
|
|
* To make it atomic, the data is read into a temporary page
|
|
|
|
* that's moved into place later.
|
|
|
|
* The migration protocol uses, possibly smaller, target-pages
|
|
|
|
* however the source ensures it always sends all the components
|
2019-11-07 15:39:06 +03:00
|
|
|
* of a host page in one chunk.
|
2015-11-05 21:11:11 +03:00
|
|
|
*/
|
migration: Introduce postcopy channels on dest node
Postcopy handles huge pages in a special way that currently we can only have
one "channel" to transfer the page.
It's because when we install pages using UFFDIO_COPY, we need to have the whole
huge page ready, it also means we need to have a temp huge page when trying to
receive the whole content of the page.
Currently all maintainance around this tmp page is global: firstly we'll
allocate a temp huge page, then we maintain its status mostly within
ram_load_postcopy().
To enable multiple channels for postcopy, the first thing we need to do is to
prepare N temp huge pages as caching, one for each channel.
Meanwhile we need to maintain the tmp huge page status per-channel too.
To give some example, some local variables maintained in ram_load_postcopy()
are listed; they are responsible for maintaining temp huge page status:
- all_zero: this keeps whether this huge page contains all zeros
- target_pages: this counts how many target pages have been copied
- host_page: this keeps the host ptr for the page to install
Move all these fields to be together with the temp huge pages to form a new
structure called PostcopyTmpPage. Then for each (future) postcopy channel, we
need one structure to keep the state around.
For vanilla postcopy, obviously there's only one channel. It contains both
precopy and postcopy pages.
This patch teaches the dest migration node to start realize the possible number
of postcopy channels by introducing the "postcopy_channels" variable. Its
value is calculated when setup postcopy on dest node (during POSTCOPY_LISTEN
phase).
Vanilla postcopy will have channels=1, but when postcopy-preempt capability is
enabled (in the future), we will boost it to 2 because even during partial
sending of a precopy huge page we still want to preempt it and start sending
the postcopy requested page right away (so we start to keep two temp huge
pages; more if we want to enable multifd). In this patch there's a TODO marked
for that; so far the channels is always set to 1.
We need to send one "host huge page" on one channel only and we cannot split
them, because otherwise the data upon the same huge page can locate on more
than one channel so we need more complicated logic to manage. One temp host
huge page for each channel will be enough for us for now.
Postcopy will still always use the index=0 huge page even after this patch.
However it prepares for the latter patches where it can start to use multiple
channels (which needs src intervention, because only src knows which channel we
should use).
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220301083925.33483-5-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
dgilbert: Fixed up long line
2022-03-01 11:39:04 +03:00
|
|
|
page_buffer = tmp_page->tmp_huge_page +
|
2021-04-29 14:27:05 +03:00
|
|
|
host_page_offset_from_ram_block_offset(block, addr);
|
|
|
|
/* If all TP are zero then we can optimise the place */
|
migration: Introduce postcopy channels on dest node
Postcopy handles huge pages in a special way that currently we can only have
one "channel" to transfer the page.
It's because when we install pages using UFFDIO_COPY, we need to have the whole
huge page ready, it also means we need to have a temp huge page when trying to
receive the whole content of the page.
Currently all maintainance around this tmp page is global: firstly we'll
allocate a temp huge page, then we maintain its status mostly within
ram_load_postcopy().
To enable multiple channels for postcopy, the first thing we need to do is to
prepare N temp huge pages as caching, one for each channel.
Meanwhile we need to maintain the tmp huge page status per-channel too.
To give some example, some local variables maintained in ram_load_postcopy()
are listed; they are responsible for maintaining temp huge page status:
- all_zero: this keeps whether this huge page contains all zeros
- target_pages: this counts how many target pages have been copied
- host_page: this keeps the host ptr for the page to install
Move all these fields to be together with the temp huge pages to form a new
structure called PostcopyTmpPage. Then for each (future) postcopy channel, we
need one structure to keep the state around.
For vanilla postcopy, obviously there's only one channel. It contains both
precopy and postcopy pages.
This patch teaches the dest migration node to start realize the possible number
of postcopy channels by introducing the "postcopy_channels" variable. Its
value is calculated when setup postcopy on dest node (during POSTCOPY_LISTEN
phase).
Vanilla postcopy will have channels=1, but when postcopy-preempt capability is
enabled (in the future), we will boost it to 2 because even during partial
sending of a precopy huge page we still want to preempt it and start sending
the postcopy requested page right away (so we start to keep two temp huge
pages; more if we want to enable multifd). In this patch there's a TODO marked
for that; so far the channels is always set to 1.
We need to send one "host huge page" on one channel only and we cannot split
them, because otherwise the data upon the same huge page can locate on more
than one channel so we need more complicated logic to manage. One temp host
huge page for each channel will be enough for us for now.
Postcopy will still always use the index=0 huge page even after this patch.
However it prepares for the latter patches where it can start to use multiple
channels (which needs src intervention, because only src knows which channel we
should use).
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220301083925.33483-5-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
dgilbert: Fixed up long line
2022-03-01 11:39:04 +03:00
|
|
|
if (tmp_page->target_pages == 1) {
|
|
|
|
tmp_page->host_addr =
|
|
|
|
host_page_from_ram_block_offset(block, addr);
|
|
|
|
} else if (tmp_page->host_addr !=
|
|
|
|
host_page_from_ram_block_offset(block, addr)) {
|
2015-11-05 21:11:12 +03:00
|
|
|
/* not the 1st TP within the HP */
|
2022-07-07 21:55:02 +03:00
|
|
|
error_report("Non-same host page detected on channel %d: "
|
2022-03-01 11:39:05 +03:00
|
|
|
"Target host page %p, received host page %p "
|
|
|
|
"(rb %s offset 0x"RAM_ADDR_FMT" target_pages %d)",
|
2022-07-07 21:55:02 +03:00
|
|
|
channel, tmp_page->host_addr,
|
2022-03-01 11:39:05 +03:00
|
|
|
host_page_from_ram_block_offset(block, addr),
|
|
|
|
block->idstr, addr, tmp_page->target_pages);
|
2021-04-29 14:27:05 +03:00
|
|
|
ret = -EINVAL;
|
|
|
|
break;
|
2015-11-05 21:11:11 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If it's the last part of a host page then we place the host
|
|
|
|
* page
|
|
|
|
*/
|
migration: Introduce postcopy channels on dest node
Postcopy handles huge pages in a special way that currently we can only have
one "channel" to transfer the page.
It's because when we install pages using UFFDIO_COPY, we need to have the whole
huge page ready, it also means we need to have a temp huge page when trying to
receive the whole content of the page.
Currently all maintainance around this tmp page is global: firstly we'll
allocate a temp huge page, then we maintain its status mostly within
ram_load_postcopy().
To enable multiple channels for postcopy, the first thing we need to do is to
prepare N temp huge pages as caching, one for each channel.
Meanwhile we need to maintain the tmp huge page status per-channel too.
To give some example, some local variables maintained in ram_load_postcopy()
are listed; they are responsible for maintaining temp huge page status:
- all_zero: this keeps whether this huge page contains all zeros
- target_pages: this counts how many target pages have been copied
- host_page: this keeps the host ptr for the page to install
Move all these fields to be together with the temp huge pages to form a new
structure called PostcopyTmpPage. Then for each (future) postcopy channel, we
need one structure to keep the state around.
For vanilla postcopy, obviously there's only one channel. It contains both
precopy and postcopy pages.
This patch teaches the dest migration node to start realize the possible number
of postcopy channels by introducing the "postcopy_channels" variable. Its
value is calculated when setup postcopy on dest node (during POSTCOPY_LISTEN
phase).
Vanilla postcopy will have channels=1, but when postcopy-preempt capability is
enabled (in the future), we will boost it to 2 because even during partial
sending of a precopy huge page we still want to preempt it and start sending
the postcopy requested page right away (so we start to keep two temp huge
pages; more if we want to enable multifd). In this patch there's a TODO marked
for that; so far the channels is always set to 1.
We need to send one "host huge page" on one channel only and we cannot split
them, because otherwise the data upon the same huge page can locate on more
than one channel so we need more complicated logic to manage. One temp host
huge page for each channel will be enough for us for now.
Postcopy will still always use the index=0 huge page even after this patch.
However it prepares for the latter patches where it can start to use multiple
channels (which needs src intervention, because only src knows which channel we
should use).
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220301083925.33483-5-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
dgilbert: Fixed up long line
2022-03-01 11:39:04 +03:00
|
|
|
if (tmp_page->target_pages ==
|
|
|
|
(block->page_size / TARGET_PAGE_SIZE)) {
|
2019-11-07 15:39:04 +03:00
|
|
|
place_needed = true;
|
|
|
|
}
|
migration: Introduce postcopy channels on dest node
Postcopy handles huge pages in a special way that currently we can only have
one "channel" to transfer the page.
It's because when we install pages using UFFDIO_COPY, we need to have the whole
huge page ready, it also means we need to have a temp huge page when trying to
receive the whole content of the page.
Currently all maintainance around this tmp page is global: firstly we'll
allocate a temp huge page, then we maintain its status mostly within
ram_load_postcopy().
To enable multiple channels for postcopy, the first thing we need to do is to
prepare N temp huge pages as caching, one for each channel.
Meanwhile we need to maintain the tmp huge page status per-channel too.
To give some example, some local variables maintained in ram_load_postcopy()
are listed; they are responsible for maintaining temp huge page status:
- all_zero: this keeps whether this huge page contains all zeros
- target_pages: this counts how many target pages have been copied
- host_page: this keeps the host ptr for the page to install
Move all these fields to be together with the temp huge pages to form a new
structure called PostcopyTmpPage. Then for each (future) postcopy channel, we
need one structure to keep the state around.
For vanilla postcopy, obviously there's only one channel. It contains both
precopy and postcopy pages.
This patch teaches the dest migration node to start realize the possible number
of postcopy channels by introducing the "postcopy_channels" variable. Its
value is calculated when setup postcopy on dest node (during POSTCOPY_LISTEN
phase).
Vanilla postcopy will have channels=1, but when postcopy-preempt capability is
enabled (in the future), we will boost it to 2 because even during partial
sending of a precopy huge page we still want to preempt it and start sending
the postcopy requested page right away (so we start to keep two temp huge
pages; more if we want to enable multifd). In this patch there's a TODO marked
for that; so far the channels is always set to 1.
We need to send one "host huge page" on one channel only and we cannot split
them, because otherwise the data upon the same huge page can locate on more
than one channel so we need more complicated logic to manage. One temp host
huge page for each channel will be enough for us for now.
Postcopy will still always use the index=0 huge page even after this patch.
However it prepares for the latter patches where it can start to use multiple
channels (which needs src intervention, because only src knows which channel we
should use).
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220301083925.33483-5-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
dgilbert: Fixed up long line
2022-03-01 11:39:04 +03:00
|
|
|
place_source = tmp_page->tmp_huge_page;
|
2015-11-05 21:11:11 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
|
2017-04-28 10:39:55 +03:00
|
|
|
case RAM_SAVE_FLAG_ZERO:
|
2015-11-05 21:11:11 +03:00
|
|
|
ch = qemu_get_byte(f);
|
2019-11-07 15:39:02 +03:00
|
|
|
/*
|
|
|
|
* Can skip to set page_buffer when
|
|
|
|
* this is a zero page and (block->page_size == TARGET_PAGE_SIZE).
|
|
|
|
*/
|
|
|
|
if (ch || !matches_target_page_size) {
|
|
|
|
memset(page_buffer, ch, TARGET_PAGE_SIZE);
|
|
|
|
}
|
2015-11-05 21:11:11 +03:00
|
|
|
if (ch) {
|
migration: Introduce postcopy channels on dest node
Postcopy handles huge pages in a special way that currently we can only have
one "channel" to transfer the page.
It's because when we install pages using UFFDIO_COPY, we need to have the whole
huge page ready, it also means we need to have a temp huge page when trying to
receive the whole content of the page.
Currently all maintainance around this tmp page is global: firstly we'll
allocate a temp huge page, then we maintain its status mostly within
ram_load_postcopy().
To enable multiple channels for postcopy, the first thing we need to do is to
prepare N temp huge pages as caching, one for each channel.
Meanwhile we need to maintain the tmp huge page status per-channel too.
To give some example, some local variables maintained in ram_load_postcopy()
are listed; they are responsible for maintaining temp huge page status:
- all_zero: this keeps whether this huge page contains all zeros
- target_pages: this counts how many target pages have been copied
- host_page: this keeps the host ptr for the page to install
Move all these fields to be together with the temp huge pages to form a new
structure called PostcopyTmpPage. Then for each (future) postcopy channel, we
need one structure to keep the state around.
For vanilla postcopy, obviously there's only one channel. It contains both
precopy and postcopy pages.
This patch teaches the dest migration node to start realize the possible number
of postcopy channels by introducing the "postcopy_channels" variable. Its
value is calculated when setup postcopy on dest node (during POSTCOPY_LISTEN
phase).
Vanilla postcopy will have channels=1, but when postcopy-preempt capability is
enabled (in the future), we will boost it to 2 because even during partial
sending of a precopy huge page we still want to preempt it and start sending
the postcopy requested page right away (so we start to keep two temp huge
pages; more if we want to enable multifd). In this patch there's a TODO marked
for that; so far the channels is always set to 1.
We need to send one "host huge page" on one channel only and we cannot split
them, because otherwise the data upon the same huge page can locate on more
than one channel so we need more complicated logic to manage. One temp host
huge page for each channel will be enough for us for now.
Postcopy will still always use the index=0 huge page even after this patch.
However it prepares for the latter patches where it can start to use multiple
channels (which needs src intervention, because only src knows which channel we
should use).
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220301083925.33483-5-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
dgilbert: Fixed up long line
2022-03-01 11:39:04 +03:00
|
|
|
tmp_page->all_zero = false;
|
2015-11-05 21:11:11 +03:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
case RAM_SAVE_FLAG_PAGE:
|
migration: Introduce postcopy channels on dest node
Postcopy handles huge pages in a special way that currently we can only have
one "channel" to transfer the page.
It's because when we install pages using UFFDIO_COPY, we need to have the whole
huge page ready, it also means we need to have a temp huge page when trying to
receive the whole content of the page.
Currently all maintainance around this tmp page is global: firstly we'll
allocate a temp huge page, then we maintain its status mostly within
ram_load_postcopy().
To enable multiple channels for postcopy, the first thing we need to do is to
prepare N temp huge pages as caching, one for each channel.
Meanwhile we need to maintain the tmp huge page status per-channel too.
To give some example, some local variables maintained in ram_load_postcopy()
are listed; they are responsible for maintaining temp huge page status:
- all_zero: this keeps whether this huge page contains all zeros
- target_pages: this counts how many target pages have been copied
- host_page: this keeps the host ptr for the page to install
Move all these fields to be together with the temp huge pages to form a new
structure called PostcopyTmpPage. Then for each (future) postcopy channel, we
need one structure to keep the state around.
For vanilla postcopy, obviously there's only one channel. It contains both
precopy and postcopy pages.
This patch teaches the dest migration node to start realize the possible number
of postcopy channels by introducing the "postcopy_channels" variable. Its
value is calculated when setup postcopy on dest node (during POSTCOPY_LISTEN
phase).
Vanilla postcopy will have channels=1, but when postcopy-preempt capability is
enabled (in the future), we will boost it to 2 because even during partial
sending of a precopy huge page we still want to preempt it and start sending
the postcopy requested page right away (so we start to keep two temp huge
pages; more if we want to enable multifd). In this patch there's a TODO marked
for that; so far the channels is always set to 1.
We need to send one "host huge page" on one channel only and we cannot split
them, because otherwise the data upon the same huge page can locate on more
than one channel so we need more complicated logic to manage. One temp host
huge page for each channel will be enough for us for now.
Postcopy will still always use the index=0 huge page even after this patch.
However it prepares for the latter patches where it can start to use multiple
channels (which needs src intervention, because only src knows which channel we
should use).
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220301083925.33483-5-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
dgilbert: Fixed up long line
2022-03-01 11:39:04 +03:00
|
|
|
tmp_page->all_zero = false;
|
2018-07-10 12:18:53 +03:00
|
|
|
if (!matches_target_page_size) {
|
|
|
|
/* For huge pages, we always use temporary buffer */
|
2015-11-05 21:11:11 +03:00
|
|
|
qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
|
|
|
|
} else {
|
2018-07-10 12:18:53 +03:00
|
|
|
/*
|
|
|
|
* For small pages that matches target page size, we
|
|
|
|
* avoid the qemu_file copy. Instead we directly use
|
|
|
|
* the buffer of QEMUFile to place the page. Note: we
|
|
|
|
* cannot do any QEMUFile operation before using that
|
|
|
|
* buffer to make sure the buffer is valid when
|
|
|
|
* placing the page.
|
2015-11-05 21:11:11 +03:00
|
|
|
*/
|
|
|
|
qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
|
|
|
|
TARGET_PAGE_SIZE);
|
|
|
|
}
|
|
|
|
break;
|
2019-11-07 15:39:07 +03:00
|
|
|
case RAM_SAVE_FLAG_COMPRESS_PAGE:
|
migration: Introduce postcopy channels on dest node
Postcopy handles huge pages in a special way that currently we can only have
one "channel" to transfer the page.
It's because when we install pages using UFFDIO_COPY, we need to have the whole
huge page ready, it also means we need to have a temp huge page when trying to
receive the whole content of the page.
Currently all maintainance around this tmp page is global: firstly we'll
allocate a temp huge page, then we maintain its status mostly within
ram_load_postcopy().
To enable multiple channels for postcopy, the first thing we need to do is to
prepare N temp huge pages as caching, one for each channel.
Meanwhile we need to maintain the tmp huge page status per-channel too.
To give some example, some local variables maintained in ram_load_postcopy()
are listed; they are responsible for maintaining temp huge page status:
- all_zero: this keeps whether this huge page contains all zeros
- target_pages: this counts how many target pages have been copied
- host_page: this keeps the host ptr for the page to install
Move all these fields to be together with the temp huge pages to form a new
structure called PostcopyTmpPage. Then for each (future) postcopy channel, we
need one structure to keep the state around.
For vanilla postcopy, obviously there's only one channel. It contains both
precopy and postcopy pages.
This patch teaches the dest migration node to start realize the possible number
of postcopy channels by introducing the "postcopy_channels" variable. Its
value is calculated when setup postcopy on dest node (during POSTCOPY_LISTEN
phase).
Vanilla postcopy will have channels=1, but when postcopy-preempt capability is
enabled (in the future), we will boost it to 2 because even during partial
sending of a precopy huge page we still want to preempt it and start sending
the postcopy requested page right away (so we start to keep two temp huge
pages; more if we want to enable multifd). In this patch there's a TODO marked
for that; so far the channels is always set to 1.
We need to send one "host huge page" on one channel only and we cannot split
them, because otherwise the data upon the same huge page can locate on more
than one channel so we need more complicated logic to manage. One temp host
huge page for each channel will be enough for us for now.
Postcopy will still always use the index=0 huge page even after this patch.
However it prepares for the latter patches where it can start to use multiple
channels (which needs src intervention, because only src knows which channel we
should use).
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220301083925.33483-5-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
dgilbert: Fixed up long line
2022-03-01 11:39:04 +03:00
|
|
|
tmp_page->all_zero = false;
|
2019-11-07 15:39:07 +03:00
|
|
|
len = qemu_get_be32(f);
|
|
|
|
if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
|
|
|
|
error_report("Invalid compressed data length: %d", len);
|
|
|
|
ret = -EINVAL;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
decompress_data_with_multi_threads(f, page_buffer, len);
|
|
|
|
break;
|
|
|
|
|
2015-11-05 21:11:11 +03:00
|
|
|
case RAM_SAVE_FLAG_EOS:
|
|
|
|
/* normal exit */
|
2018-02-28 11:10:07 +03:00
|
|
|
multifd_recv_sync_main();
|
2015-11-05 21:11:11 +03:00
|
|
|
break;
|
|
|
|
default:
|
2020-10-20 06:10:42 +03:00
|
|
|
error_report("Unknown combination of migration flags: 0x%x"
|
2015-11-05 21:11:11 +03:00
|
|
|
" (postcopy mode)", flags);
|
|
|
|
ret = -EINVAL;
|
2018-02-08 13:31:05 +03:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2019-11-07 15:39:07 +03:00
|
|
|
/* Got the whole host page, wait for decompress before placing. */
|
|
|
|
if (place_needed) {
|
|
|
|
ret |= wait_for_decompress_done();
|
|
|
|
}
|
|
|
|
|
2018-02-08 13:31:05 +03:00
|
|
|
/* Detect for any possible file errors */
|
|
|
|
if (!ret && qemu_file_get_error(f)) {
|
|
|
|
ret = qemu_file_get_error(f);
|
2015-11-05 21:11:11 +03:00
|
|
|
}
|
|
|
|
|
2018-02-08 13:31:05 +03:00
|
|
|
if (!ret && place_needed) {
|
migration: Introduce postcopy channels on dest node
Postcopy handles huge pages in a special way that currently we can only have
one "channel" to transfer the page.
It's because when we install pages using UFFDIO_COPY, we need to have the whole
huge page ready, it also means we need to have a temp huge page when trying to
receive the whole content of the page.
Currently all maintainance around this tmp page is global: firstly we'll
allocate a temp huge page, then we maintain its status mostly within
ram_load_postcopy().
To enable multiple channels for postcopy, the first thing we need to do is to
prepare N temp huge pages as caching, one for each channel.
Meanwhile we need to maintain the tmp huge page status per-channel too.
To give some example, some local variables maintained in ram_load_postcopy()
are listed; they are responsible for maintaining temp huge page status:
- all_zero: this keeps whether this huge page contains all zeros
- target_pages: this counts how many target pages have been copied
- host_page: this keeps the host ptr for the page to install
Move all these fields to be together with the temp huge pages to form a new
structure called PostcopyTmpPage. Then for each (future) postcopy channel, we
need one structure to keep the state around.
For vanilla postcopy, obviously there's only one channel. It contains both
precopy and postcopy pages.
This patch teaches the dest migration node to start realize the possible number
of postcopy channels by introducing the "postcopy_channels" variable. Its
value is calculated when setup postcopy on dest node (during POSTCOPY_LISTEN
phase).
Vanilla postcopy will have channels=1, but when postcopy-preempt capability is
enabled (in the future), we will boost it to 2 because even during partial
sending of a precopy huge page we still want to preempt it and start sending
the postcopy requested page right away (so we start to keep two temp huge
pages; more if we want to enable multifd). In this patch there's a TODO marked
for that; so far the channels is always set to 1.
We need to send one "host huge page" on one channel only and we cannot split
them, because otherwise the data upon the same huge page can locate on more
than one channel so we need more complicated logic to manage. One temp host
huge page for each channel will be enough for us for now.
Postcopy will still always use the index=0 huge page even after this patch.
However it prepares for the latter patches where it can start to use multiple
channels (which needs src intervention, because only src knows which channel we
should use).
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220301083925.33483-5-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
dgilbert: Fixed up long line
2022-03-01 11:39:04 +03:00
|
|
|
if (tmp_page->all_zero) {
|
|
|
|
ret = postcopy_place_page_zero(mis, tmp_page->host_addr, block);
|
2015-11-05 21:11:11 +03:00
|
|
|
} else {
|
migration: Introduce postcopy channels on dest node
Postcopy handles huge pages in a special way that currently we can only have
one "channel" to transfer the page.
It's because when we install pages using UFFDIO_COPY, we need to have the whole
huge page ready, it also means we need to have a temp huge page when trying to
receive the whole content of the page.
Currently all maintainance around this tmp page is global: firstly we'll
allocate a temp huge page, then we maintain its status mostly within
ram_load_postcopy().
To enable multiple channels for postcopy, the first thing we need to do is to
prepare N temp huge pages as caching, one for each channel.
Meanwhile we need to maintain the tmp huge page status per-channel too.
To give some example, some local variables maintained in ram_load_postcopy()
are listed; they are responsible for maintaining temp huge page status:
- all_zero: this keeps whether this huge page contains all zeros
- target_pages: this counts how many target pages have been copied
- host_page: this keeps the host ptr for the page to install
Move all these fields to be together with the temp huge pages to form a new
structure called PostcopyTmpPage. Then for each (future) postcopy channel, we
need one structure to keep the state around.
For vanilla postcopy, obviously there's only one channel. It contains both
precopy and postcopy pages.
This patch teaches the dest migration node to start realize the possible number
of postcopy channels by introducing the "postcopy_channels" variable. Its
value is calculated when setup postcopy on dest node (during POSTCOPY_LISTEN
phase).
Vanilla postcopy will have channels=1, but when postcopy-preempt capability is
enabled (in the future), we will boost it to 2 because even during partial
sending of a precopy huge page we still want to preempt it and start sending
the postcopy requested page right away (so we start to keep two temp huge
pages; more if we want to enable multifd). In this patch there's a TODO marked
for that; so far the channels is always set to 1.
We need to send one "host huge page" on one channel only and we cannot split
them, because otherwise the data upon the same huge page can locate on more
than one channel so we need more complicated logic to manage. One temp host
huge page for each channel will be enough for us for now.
Postcopy will still always use the index=0 huge page even after this patch.
However it prepares for the latter patches where it can start to use multiple
channels (which needs src intervention, because only src knows which channel we
should use).
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220301083925.33483-5-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
dgilbert: Fixed up long line
2022-03-01 11:39:04 +03:00
|
|
|
ret = postcopy_place_page(mis, tmp_page->host_addr,
|
|
|
|
place_source, block);
|
2015-11-05 21:11:11 +03:00
|
|
|
}
|
2020-04-21 11:52:56 +03:00
|
|
|
place_needed = false;
|
migration: Introduce postcopy channels on dest node
Postcopy handles huge pages in a special way that currently we can only have
one "channel" to transfer the page.
It's because when we install pages using UFFDIO_COPY, we need to have the whole
huge page ready, it also means we need to have a temp huge page when trying to
receive the whole content of the page.
Currently all maintainance around this tmp page is global: firstly we'll
allocate a temp huge page, then we maintain its status mostly within
ram_load_postcopy().
To enable multiple channels for postcopy, the first thing we need to do is to
prepare N temp huge pages as caching, one for each channel.
Meanwhile we need to maintain the tmp huge page status per-channel too.
To give some example, some local variables maintained in ram_load_postcopy()
are listed; they are responsible for maintaining temp huge page status:
- all_zero: this keeps whether this huge page contains all zeros
- target_pages: this counts how many target pages have been copied
- host_page: this keeps the host ptr for the page to install
Move all these fields to be together with the temp huge pages to form a new
structure called PostcopyTmpPage. Then for each (future) postcopy channel, we
need one structure to keep the state around.
For vanilla postcopy, obviously there's only one channel. It contains both
precopy and postcopy pages.
This patch teaches the dest migration node to start realize the possible number
of postcopy channels by introducing the "postcopy_channels" variable. Its
value is calculated when setup postcopy on dest node (during POSTCOPY_LISTEN
phase).
Vanilla postcopy will have channels=1, but when postcopy-preempt capability is
enabled (in the future), we will boost it to 2 because even during partial
sending of a precopy huge page we still want to preempt it and start sending
the postcopy requested page right away (so we start to keep two temp huge
pages; more if we want to enable multifd). In this patch there's a TODO marked
for that; so far the channels is always set to 1.
We need to send one "host huge page" on one channel only and we cannot split
them, because otherwise the data upon the same huge page can locate on more
than one channel so we need more complicated logic to manage. One temp host
huge page for each channel will be enough for us for now.
Postcopy will still always use the index=0 huge page even after this patch.
However it prepares for the latter patches where it can start to use multiple
channels (which needs src intervention, because only src knows which channel we
should use).
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220301083925.33483-5-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
dgilbert: Fixed up long line
2022-03-01 11:39:04 +03:00
|
|
|
postcopy_temp_page_reset(tmp_page);
|
2015-11-05 21:11:11 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
migration/ram.c: do not set 'postcopy_running' in POSTCOPY_INCOMING_END
When migrating a VM with 'migrate_set_capability postcopy-ram on'
a postcopy_state is set during the process, ending up with the
state POSTCOPY_INCOMING_END when the migration is over. This
postcopy_state is taken into account inside ram_load to check
how it will load the memory pages. This same ram_load is called when
in a loadvm command.
Inside ram_load, the logic to see if we're at postcopy_running state
is:
postcopy_running = postcopy_state_get() >= POSTCOPY_INCOMING_LISTENING
postcopy_state_get() returns this enum type:
typedef enum {
POSTCOPY_INCOMING_NONE = 0,
POSTCOPY_INCOMING_ADVISE,
POSTCOPY_INCOMING_DISCARD,
POSTCOPY_INCOMING_LISTENING,
POSTCOPY_INCOMING_RUNNING,
POSTCOPY_INCOMING_END
} PostcopyState;
In the case where ram_load is executed and postcopy_state is
POSTCOPY_INCOMING_END, postcopy_running will be set to 'true' and
ram_load will behave like a postcopy is in progress. This scenario isn't
achievable in a migration but it is reproducible when executing
savevm/loadvm after migrating with 'postcopy-ram on', causing loadvm
to fail with Error -22:
Source:
(qemu) migrate_set_capability postcopy-ram on
(qemu) migrate tcp:127.0.0.1:4444
Dest:
(qemu) migrate_set_capability postcopy-ram on
(qemu)
ubuntu1704-intel login:
Ubuntu 17.04 ubuntu1704-intel ttyS0
ubuntu1704-intel login: (qemu)
(qemu) savevm test1
(qemu) loadvm test1
Unknown combination of migration flags: 0x4 (postcopy mode)
error while loading state for instance 0x0 of device 'ram'
Error -22 while loading VM state
(qemu)
This patch fixes this problem by changing the existing logic for
postcopy_advised and postcopy_running in ram_load, making them
'false' if we're at POSTCOPY_INCOMING_END state.
Signed-off-by: Daniel Henrique Barboza <danielhb@linux.vnet.ibm.com>
CC: Juan Quintela <quintela@redhat.com>
CC: Dr. David Alan Gilbert <dgilbert@redhat.com>
Reviewed-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Reported-by: Balamuruhan S <bala24@linux.vnet.ibm.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2017-11-17 01:35:26 +03:00
|
|
|
static bool postcopy_is_running(void)
|
|
|
|
{
|
|
|
|
PostcopyState ps = postcopy_state_get();
|
|
|
|
return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
|
|
|
|
}
|
|
|
|
|
2018-09-03 07:38:50 +03:00
|
|
|
/*
|
|
|
|
* Flush content of RAM cache into SVM's memory.
|
|
|
|
* Only flush the pages that be dirtied by PVM or SVM or both.
|
|
|
|
*/
|
2020-05-11 14:10:51 +03:00
|
|
|
void colo_flush_ram_cache(void)
|
2018-09-03 07:38:50 +03:00
|
|
|
{
|
|
|
|
RAMBlock *block = NULL;
|
|
|
|
void *dst_host;
|
|
|
|
void *src_host;
|
|
|
|
unsigned long offset = 0;
|
|
|
|
|
2018-09-03 07:38:55 +03:00
|
|
|
memory_global_dirty_log_sync();
|
2019-10-07 17:36:39 +03:00
|
|
|
WITH_RCU_READ_LOCK_GUARD() {
|
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
|
|
|
ramblock_sync_dirty_bitmap(ram_state, block);
|
|
|
|
}
|
2018-09-03 07:38:55 +03:00
|
|
|
}
|
|
|
|
|
2018-09-03 07:38:50 +03:00
|
|
|
trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
|
2019-10-07 17:36:39 +03:00
|
|
|
WITH_RCU_READ_LOCK_GUARD() {
|
|
|
|
block = QLIST_FIRST_RCU(&ram_list.blocks);
|
2018-09-03 07:38:50 +03:00
|
|
|
|
2019-10-07 17:36:39 +03:00
|
|
|
while (block) {
|
2021-11-09 06:04:55 +03:00
|
|
|
unsigned long num = 0;
|
2018-09-03 07:38:50 +03:00
|
|
|
|
2021-11-09 06:04:55 +03:00
|
|
|
offset = colo_bitmap_find_dirty(ram_state, block, offset, &num);
|
2021-04-29 14:27:08 +03:00
|
|
|
if (!offset_in_ramblock(block,
|
|
|
|
((ram_addr_t)offset) << TARGET_PAGE_BITS)) {
|
2019-10-07 17:36:39 +03:00
|
|
|
offset = 0;
|
2021-11-09 06:04:55 +03:00
|
|
|
num = 0;
|
2019-10-07 17:36:39 +03:00
|
|
|
block = QLIST_NEXT_RCU(block, next);
|
|
|
|
} else {
|
2021-11-09 06:04:55 +03:00
|
|
|
unsigned long i = 0;
|
|
|
|
|
|
|
|
for (i = 0; i < num; i++) {
|
|
|
|
migration_bitmap_clear_dirty(ram_state, block, offset + i);
|
|
|
|
}
|
2020-01-10 16:51:34 +03:00
|
|
|
dst_host = block->host
|
|
|
|
+ (((ram_addr_t)offset) << TARGET_PAGE_BITS);
|
|
|
|
src_host = block->colo_cache
|
|
|
|
+ (((ram_addr_t)offset) << TARGET_PAGE_BITS);
|
2021-11-09 06:04:55 +03:00
|
|
|
memcpy(dst_host, src_host, TARGET_PAGE_SIZE * num);
|
|
|
|
offset += num;
|
2019-10-07 17:36:39 +03:00
|
|
|
}
|
2018-09-03 07:38:50 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
trace_colo_flush_ram_cache_end();
|
|
|
|
}
|
|
|
|
|
2019-07-25 03:20:23 +03:00
|
|
|
/**
|
|
|
|
* ram_load_precopy: load pages in precopy case
|
|
|
|
*
|
|
|
|
* Returns 0 for success or -errno in case of error
|
|
|
|
*
|
|
|
|
* Called in precopy mode by ram_load().
|
|
|
|
* rcu_read_lock is taken prior to this being called.
|
|
|
|
*
|
|
|
|
* @f: QEMUFile where to send the data
|
|
|
|
*/
|
|
|
|
static int ram_load_precopy(QEMUFile *f)
|
2015-05-07 20:33:31 +03:00
|
|
|
{
|
2022-03-01 11:39:07 +03:00
|
|
|
MigrationIncomingState *mis = migration_incoming_get_current();
|
2019-11-25 16:36:32 +03:00
|
|
|
int flags = 0, ret = 0, invalid_flags = 0, len = 0, i = 0;
|
2017-02-24 21:28:30 +03:00
|
|
|
/* ADVISE is earlier, it shows the source has the postcopy capability on */
|
2023-01-17 14:22:46 +03:00
|
|
|
bool postcopy_advised = migration_incoming_postcopy_advised();
|
2023-03-02 00:03:48 +03:00
|
|
|
if (!migrate_compress()) {
|
2016-11-02 14:40:46 +03:00
|
|
|
invalid_flags |= RAM_SAVE_FLAG_COMPRESS_PAGE;
|
|
|
|
}
|
2015-11-05 21:11:11 +03:00
|
|
|
|
2019-07-25 03:20:23 +03:00
|
|
|
while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
|
2015-05-07 20:33:31 +03:00
|
|
|
ram_addr_t addr, total_ram_bytes;
|
2020-02-24 09:54:10 +03:00
|
|
|
void *host = NULL, *host_bak = NULL;
|
2015-05-07 20:33:31 +03:00
|
|
|
uint8_t ch;
|
|
|
|
|
2019-11-25 16:36:32 +03:00
|
|
|
/*
|
|
|
|
* Yield periodically to let main loop run, but an iteration of
|
|
|
|
* the main loop is expensive, so do it each some iterations
|
|
|
|
*/
|
|
|
|
if ((i & 32767) == 0 && qemu_in_coroutine()) {
|
|
|
|
aio_co_schedule(qemu_get_current_aio_context(),
|
|
|
|
qemu_coroutine_self());
|
|
|
|
qemu_coroutine_yield();
|
|
|
|
}
|
|
|
|
i++;
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
addr = qemu_get_be64(f);
|
|
|
|
flags = addr & ~TARGET_PAGE_MASK;
|
|
|
|
addr &= TARGET_PAGE_MASK;
|
|
|
|
|
2016-11-02 14:40:46 +03:00
|
|
|
if (flags & invalid_flags) {
|
|
|
|
if (flags & invalid_flags & RAM_SAVE_FLAG_COMPRESS_PAGE) {
|
|
|
|
error_report("Received an unexpected compressed page");
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = -EINVAL;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2017-04-28 10:39:55 +03:00
|
|
|
if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE |
|
2015-11-05 21:10:39 +03:00
|
|
|
RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
|
migration: Postcopy preemption enablement
This patch enables postcopy-preempt feature.
It contains two major changes to the migration logic:
(1) Postcopy requests are now sent via a different socket from precopy
background migration stream, so as to be isolated from very high page
request delays.
(2) For huge page enabled hosts: when there's postcopy requests, they can now
intercept a partial sending of huge host pages on src QEMU.
After this patch, we'll live migrate a VM with two channels for postcopy: (1)
PRECOPY channel, which is the default channel that transfers background pages;
and (2) POSTCOPY channel, which only transfers requested pages.
There's no strict rule of which channel to use, e.g., if a requested page is
already being transferred on precopy channel, then we will keep using the same
precopy channel to transfer the page even if it's explicitly requested. In 99%
of the cases we'll prioritize the channels so we send requested page via the
postcopy channel as long as possible.
On the source QEMU, when we found a postcopy request, we'll interrupt the
PRECOPY channel sending process and quickly switch to the POSTCOPY channel.
After we serviced all the high priority postcopy pages, we'll switch back to
PRECOPY channel so that we'll continue to send the interrupted huge page again.
There's no new thread introduced on src QEMU.
On the destination QEMU, one new thread is introduced to receive page data from
the postcopy specific socket (done in the preparation patch).
This patch has a side effect: after sending postcopy pages, previously we'll
assume the guest will access follow up pages so we'll keep sending from there.
Now it's changed. Instead of going on with a postcopy requested page, we'll go
back and continue sending the precopy huge page (which can be intercepted by a
postcopy request so the huge page can be sent partially before).
Whether that's a problem is debatable, because "assuming the guest will
continue to access the next page" may not really suite when huge pages are
used, especially if the huge page is large (e.g. 1GB pages). So that locality
hint is much meaningless if huge pages are used.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20220707185504.27203-1-peterx@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
2022-07-07 21:55:04 +03:00
|
|
|
RAMBlock *block = ram_block_from_stream(mis, f, flags,
|
|
|
|
RAM_CHANNEL_PRECOPY);
|
2016-01-15 06:37:41 +03:00
|
|
|
|
2020-02-24 09:54:10 +03:00
|
|
|
host = host_from_ram_block_offset(block, addr);
|
2018-09-03 07:38:48 +03:00
|
|
|
/*
|
2020-02-24 09:54:10 +03:00
|
|
|
* After going into COLO stage, we should not load the page
|
|
|
|
* into SVM's memory directly, we put them into colo_cache firstly.
|
|
|
|
* NOTE: We need to keep a copy of SVM's ram in colo_cache.
|
|
|
|
* Previously, we copied all these memory in preparing stage of COLO
|
|
|
|
* while we need to stop VM, which is a time-consuming process.
|
|
|
|
* Here we optimize it by a trick, back-up every page while in
|
|
|
|
* migration process while COLO is enabled, though it affects the
|
|
|
|
* speed of the migration, but it obviously reduce the downtime of
|
|
|
|
* back-up all SVM'S memory in COLO preparing stage.
|
2018-09-03 07:38:48 +03:00
|
|
|
*/
|
2020-02-24 09:54:10 +03:00
|
|
|
if (migration_incoming_colo_enabled()) {
|
|
|
|
if (migration_incoming_in_colo_state()) {
|
|
|
|
/* In COLO stage, put all pages into cache temporarily */
|
2020-02-24 09:54:11 +03:00
|
|
|
host = colo_cache_from_block_offset(block, addr, true);
|
2020-02-24 09:54:10 +03:00
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* In migration stage but before COLO stage,
|
|
|
|
* Put all pages into both cache and SVM's memory.
|
|
|
|
*/
|
2020-02-24 09:54:11 +03:00
|
|
|
host_bak = colo_cache_from_block_offset(block, addr, false);
|
2020-02-24 09:54:10 +03:00
|
|
|
}
|
2018-09-03 07:38:48 +03:00
|
|
|
}
|
2015-11-05 21:10:39 +03:00
|
|
|
if (!host) {
|
|
|
|
error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
|
|
|
|
ret = -EINVAL;
|
|
|
|
break;
|
|
|
|
}
|
2018-09-03 07:38:48 +03:00
|
|
|
if (!migration_incoming_in_colo_state()) {
|
|
|
|
ramblock_recv_bitmap_set(block, host);
|
|
|
|
}
|
|
|
|
|
2017-04-26 21:37:21 +03:00
|
|
|
trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
|
2015-11-05 21:10:39 +03:00
|
|
|
}
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
|
|
|
|
case RAM_SAVE_FLAG_MEM_SIZE:
|
|
|
|
/* Synchronize RAM block list */
|
|
|
|
total_ram_bytes = addr;
|
|
|
|
while (!ret && total_ram_bytes) {
|
|
|
|
RAMBlock *block;
|
|
|
|
char id[256];
|
|
|
|
ram_addr_t length;
|
|
|
|
|
|
|
|
len = qemu_get_byte(f);
|
|
|
|
qemu_get_buffer(f, (uint8_t *)id, len);
|
|
|
|
id[len] = 0;
|
|
|
|
length = qemu_get_be64(f);
|
|
|
|
|
2015-11-05 21:10:33 +03:00
|
|
|
block = qemu_ram_block_by_name(id);
|
2018-05-14 09:57:00 +03:00
|
|
|
if (block && !qemu_ram_is_migratable(block)) {
|
|
|
|
error_report("block %s should not be migrated !", id);
|
|
|
|
ret = -EINVAL;
|
|
|
|
} else if (block) {
|
2015-11-05 21:10:33 +03:00
|
|
|
if (length != block->used_length) {
|
|
|
|
Error *local_err = NULL;
|
2015-05-07 20:33:31 +03:00
|
|
|
|
2016-05-10 05:04:59 +03:00
|
|
|
ret = qemu_ram_resize(block, length,
|
2015-11-05 21:10:33 +03:00
|
|
|
&local_err);
|
|
|
|
if (local_err) {
|
|
|
|
error_report_err(local_err);
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
}
|
2017-02-24 21:28:30 +03:00
|
|
|
/* For postcopy we need to check hugepage sizes match */
|
2021-02-04 19:35:22 +03:00
|
|
|
if (postcopy_advised && migrate_postcopy_ram() &&
|
2017-02-24 21:28:30 +03:00
|
|
|
block->page_size != qemu_host_page_size) {
|
|
|
|
uint64_t remote_page_size = qemu_get_be64(f);
|
|
|
|
if (remote_page_size != block->page_size) {
|
|
|
|
error_report("Mismatched RAM page size %s "
|
|
|
|
"(local) %zd != %" PRId64,
|
|
|
|
id, block->page_size,
|
|
|
|
remote_page_size);
|
|
|
|
ret = -EINVAL;
|
|
|
|
}
|
|
|
|
}
|
2019-02-15 20:45:46 +03:00
|
|
|
if (migrate_ignore_shared()) {
|
|
|
|
hwaddr addr = qemu_get_be64(f);
|
|
|
|
if (ramblock_is_ignored(block) &&
|
|
|
|
block->mr->addr != addr) {
|
|
|
|
error_report("Mismatched GPAs for block %s "
|
|
|
|
"%" PRId64 "!= %" PRId64,
|
|
|
|
id, (uint64_t)addr,
|
|
|
|
(uint64_t)block->mr->addr);
|
|
|
|
ret = -EINVAL;
|
|
|
|
}
|
|
|
|
}
|
2015-11-05 21:10:33 +03:00
|
|
|
ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
|
|
|
|
block->idstr);
|
|
|
|
} else {
|
2015-05-07 20:33:31 +03:00
|
|
|
error_report("Unknown ramblock \"%s\", cannot "
|
|
|
|
"accept migration", id);
|
|
|
|
ret = -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
total_ram_bytes -= length;
|
|
|
|
}
|
|
|
|
break;
|
2015-11-05 21:10:39 +03:00
|
|
|
|
2017-04-28 10:39:55 +03:00
|
|
|
case RAM_SAVE_FLAG_ZERO:
|
2015-05-07 20:33:31 +03:00
|
|
|
ch = qemu_get_byte(f);
|
|
|
|
ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
|
|
|
|
break;
|
2015-11-05 21:10:39 +03:00
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
case RAM_SAVE_FLAG_PAGE:
|
|
|
|
qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
|
|
|
|
break;
|
|
|
|
|
2015-11-05 21:10:39 +03:00
|
|
|
case RAM_SAVE_FLAG_COMPRESS_PAGE:
|
2015-05-07 20:33:31 +03:00
|
|
|
len = qemu_get_be32(f);
|
|
|
|
if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
|
|
|
|
error_report("Invalid compressed data length: %d", len);
|
|
|
|
ret = -EINVAL;
|
|
|
|
break;
|
|
|
|
}
|
2015-12-16 14:47:38 +03:00
|
|
|
decompress_data_with_multi_threads(f, host, len);
|
2015-05-07 20:33:31 +03:00
|
|
|
break;
|
2015-11-05 21:10:39 +03:00
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
case RAM_SAVE_FLAG_XBZRLE:
|
|
|
|
if (load_xbzrle(f, addr, host) < 0) {
|
|
|
|
error_report("Failed to decompress XBZRLE page at "
|
|
|
|
RAM_ADDR_FMT, addr);
|
|
|
|
ret = -EINVAL;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case RAM_SAVE_FLAG_EOS:
|
|
|
|
/* normal exit */
|
2018-02-28 11:10:07 +03:00
|
|
|
multifd_recv_sync_main();
|
2015-05-07 20:33:31 +03:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
if (flags & RAM_SAVE_FLAG_HOOK) {
|
2015-06-11 20:17:23 +03:00
|
|
|
ram_control_load_hook(f, RAM_CONTROL_HOOK, NULL);
|
2015-05-07 20:33:31 +03:00
|
|
|
} else {
|
2020-10-20 06:10:42 +03:00
|
|
|
error_report("Unknown combination of migration flags: 0x%x",
|
2015-05-07 20:33:31 +03:00
|
|
|
flags);
|
|
|
|
ret = -EINVAL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!ret) {
|
|
|
|
ret = qemu_file_get_error(f);
|
|
|
|
}
|
2020-02-24 09:54:10 +03:00
|
|
|
if (!ret && host_bak) {
|
|
|
|
memcpy(host_bak, host, TARGET_PAGE_SIZE);
|
|
|
|
}
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|
|
|
|
|
2019-11-07 15:39:03 +03:00
|
|
|
ret |= wait_for_decompress_done();
|
2019-07-25 03:20:23 +03:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int ram_load(QEMUFile *f, void *opaque, int version_id)
|
|
|
|
{
|
|
|
|
int ret = 0;
|
|
|
|
static uint64_t seq_iter;
|
|
|
|
/*
|
|
|
|
* If system is running in postcopy mode, page inserts to host memory must
|
|
|
|
* be atomic
|
|
|
|
*/
|
|
|
|
bool postcopy_running = postcopy_is_running();
|
|
|
|
|
|
|
|
seq_iter++;
|
|
|
|
|
|
|
|
if (version_id != 4) {
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This RCU critical section can be very long running.
|
|
|
|
* When RCU reclaims in the code start to become numerous,
|
|
|
|
* it will be necessary to reduce the granularity of this
|
|
|
|
* critical section.
|
|
|
|
*/
|
2019-10-07 17:36:39 +03:00
|
|
|
WITH_RCU_READ_LOCK_GUARD() {
|
|
|
|
if (postcopy_running) {
|
2022-07-07 21:55:02 +03:00
|
|
|
/*
|
|
|
|
* Note! Here RAM_CHANNEL_PRECOPY is the precopy channel of
|
|
|
|
* postcopy migration, we have another RAM_CHANNEL_POSTCOPY to
|
|
|
|
* service fast page faults.
|
|
|
|
*/
|
|
|
|
ret = ram_load_postcopy(f, RAM_CHANNEL_PRECOPY);
|
2019-10-07 17:36:39 +03:00
|
|
|
} else {
|
|
|
|
ret = ram_load_precopy(f);
|
|
|
|
}
|
2019-07-25 03:20:23 +03:00
|
|
|
}
|
2017-01-24 00:32:05 +03:00
|
|
|
trace_ram_load_complete(ret, seq_iter);
|
2018-09-03 07:38:50 +03:00
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2017-07-10 19:30:14 +03:00
|
|
|
static bool ram_has_postcopy(void *opaque)
|
|
|
|
{
|
2018-07-18 10:48:02 +03:00
|
|
|
RAMBlock *rb;
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
|
2018-07-18 10:48:02 +03:00
|
|
|
if (ramblock_is_pmem(rb)) {
|
|
|
|
info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
|
|
|
|
"is not supported now!", rb->idstr, rb->host);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-10 19:30:14 +03:00
|
|
|
return migrate_postcopy_ram();
|
|
|
|
}
|
|
|
|
|
migration: synchronize dirty bitmap for resume
This patch implements the first part of core RAM resume logic for
postcopy. ram_resume_prepare() is provided for the work.
When the migration is interrupted by network failure, the dirty bitmap
on the source side will be meaningless, because even the dirty bit is
cleared, it is still possible that the sent page was lost along the way
to destination. Here instead of continue the migration with the old
dirty bitmap on source, we ask the destination side to send back its
received bitmap, then invert it to be our initial dirty bitmap.
The source side send thread will issue the MIG_CMD_RECV_BITMAP requests,
once per ramblock, to ask for the received bitmap. On destination side,
MIG_RP_MSG_RECV_BITMAP will be issued, along with the requested bitmap.
Data will be received on the return-path thread of source, and the main
migration thread will be notified when all the ramblock bitmaps are
synchronized.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20180502104740.12123-17-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2018-05-02 13:47:32 +03:00
|
|
|
/* Sync all the dirty bitmap with destination VM. */
|
|
|
|
static int ram_dirty_bitmap_sync_all(MigrationState *s, RAMState *rs)
|
|
|
|
{
|
|
|
|
RAMBlock *block;
|
|
|
|
QEMUFile *file = s->to_dst_file;
|
|
|
|
int ramblock_count = 0;
|
|
|
|
|
|
|
|
trace_ram_dirty_bitmap_sync_start();
|
|
|
|
|
2019-02-15 20:45:46 +03:00
|
|
|
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
|
migration: synchronize dirty bitmap for resume
This patch implements the first part of core RAM resume logic for
postcopy. ram_resume_prepare() is provided for the work.
When the migration is interrupted by network failure, the dirty bitmap
on the source side will be meaningless, because even the dirty bit is
cleared, it is still possible that the sent page was lost along the way
to destination. Here instead of continue the migration with the old
dirty bitmap on source, we ask the destination side to send back its
received bitmap, then invert it to be our initial dirty bitmap.
The source side send thread will issue the MIG_CMD_RECV_BITMAP requests,
once per ramblock, to ask for the received bitmap. On destination side,
MIG_RP_MSG_RECV_BITMAP will be issued, along with the requested bitmap.
Data will be received on the return-path thread of source, and the main
migration thread will be notified when all the ramblock bitmaps are
synchronized.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20180502104740.12123-17-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2018-05-02 13:47:32 +03:00
|
|
|
qemu_savevm_send_recv_bitmap(file, block->idstr);
|
|
|
|
trace_ram_dirty_bitmap_request(block->idstr);
|
|
|
|
ramblock_count++;
|
|
|
|
}
|
|
|
|
|
|
|
|
trace_ram_dirty_bitmap_sync_wait();
|
|
|
|
|
|
|
|
/* Wait until all the ramblocks' dirty bitmap synced */
|
|
|
|
while (ramblock_count--) {
|
|
|
|
qemu_sem_wait(&s->rp_state.rp_sem);
|
|
|
|
}
|
|
|
|
|
|
|
|
trace_ram_dirty_bitmap_sync_complete();
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void ram_dirty_bitmap_reload_notify(MigrationState *s)
|
|
|
|
{
|
|
|
|
qemu_sem_post(&s->rp_state.rp_sem);
|
|
|
|
}
|
|
|
|
|
2018-05-02 13:47:28 +03:00
|
|
|
/*
|
|
|
|
* Read the received bitmap, revert it as the initial dirty bitmap.
|
|
|
|
* This is only used when the postcopy migration is paused but wants
|
|
|
|
* to resume from a middle point.
|
|
|
|
*/
|
|
|
|
int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *block)
|
|
|
|
{
|
|
|
|
int ret = -EINVAL;
|
2021-07-22 20:58:38 +03:00
|
|
|
/* from_dst_file is always valid because we're within rp_thread */
|
2018-05-02 13:47:28 +03:00
|
|
|
QEMUFile *file = s->rp_state.from_dst_file;
|
|
|
|
unsigned long *le_bitmap, nbits = block->used_length >> TARGET_PAGE_BITS;
|
2018-07-10 12:18:55 +03:00
|
|
|
uint64_t local_size = DIV_ROUND_UP(nbits, 8);
|
2018-05-02 13:47:28 +03:00
|
|
|
uint64_t size, end_mark;
|
|
|
|
|
|
|
|
trace_ram_dirty_bitmap_reload_begin(block->idstr);
|
|
|
|
|
|
|
|
if (s->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
|
|
|
|
error_report("%s: incorrect state %s", __func__,
|
|
|
|
MigrationStatus_str(s->state));
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Note: see comments in ramblock_recv_bitmap_send() on why we
|
2020-09-17 10:50:21 +03:00
|
|
|
* need the endianness conversion, and the paddings.
|
2018-05-02 13:47:28 +03:00
|
|
|
*/
|
|
|
|
local_size = ROUND_UP(local_size, 8);
|
|
|
|
|
|
|
|
/* Add paddings */
|
|
|
|
le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
|
|
|
|
|
|
|
|
size = qemu_get_be64(file);
|
|
|
|
|
|
|
|
/* The size of the bitmap should match with our ramblock */
|
|
|
|
if (size != local_size) {
|
|
|
|
error_report("%s: ramblock '%s' bitmap size mismatch "
|
|
|
|
"(0x%"PRIx64" != 0x%"PRIx64")", __func__,
|
|
|
|
block->idstr, size, local_size);
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
size = qemu_get_buffer(file, (uint8_t *)le_bitmap, local_size);
|
|
|
|
end_mark = qemu_get_be64(file);
|
|
|
|
|
|
|
|
ret = qemu_file_get_error(file);
|
|
|
|
if (ret || size != local_size) {
|
|
|
|
error_report("%s: read bitmap failed for ramblock '%s': %d"
|
|
|
|
" (size 0x%"PRIx64", got: 0x%"PRIx64")",
|
|
|
|
__func__, block->idstr, ret, local_size, size);
|
|
|
|
ret = -EIO;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (end_mark != RAMBLOCK_RECV_BITMAP_ENDING) {
|
2020-11-03 14:25:58 +03:00
|
|
|
error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIx64,
|
2018-05-02 13:47:28 +03:00
|
|
|
__func__, block->idstr, end_mark);
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2020-09-17 10:50:21 +03:00
|
|
|
* Endianness conversion. We are during postcopy (though paused).
|
2018-05-02 13:47:28 +03:00
|
|
|
* The dirty bitmap won't change. We can directly modify it.
|
|
|
|
*/
|
|
|
|
bitmap_from_le(block->bmap, le_bitmap, nbits);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* What we received is "received bitmap". Revert it as the initial
|
|
|
|
* dirty bitmap for this ramblock.
|
|
|
|
*/
|
|
|
|
bitmap_complement(block->bmap, block->bmap, nbits);
|
|
|
|
|
migration/ram: Handle RAMBlocks with a RamDiscardManager on the migration source
We don't want to migrate memory that corresponds to discarded ranges as
managed by a RamDiscardManager responsible for the mapped memory region of
the RAMBlock. The content of these pages is essentially stale and
without any guarantees for the VM ("logically unplugged").
Depending on the underlying memory type, even reading memory might populate
memory on the source, resulting in an undesired memory consumption. Of
course, on the destination, even writing a zeropage consumes memory,
which we also want to avoid (similar to free page hinting).
Currently, virtio-mem tries achieving that goal (not migrating "unplugged"
memory that was discarded) by going via qemu_guest_free_page_hint() - but
it's hackish and incomplete.
For example, background snapshots still end up reading all memory, as
they don't do bitmap syncs. Postcopy recovery code will re-add
previously cleared bits to the dirty bitmap and migrate them.
Let's consult the RamDiscardManager after setting up our dirty bitmap
initially and when postcopy recovery code reinitializes it: clear
corresponding bits in the dirty bitmaps (e.g., of the RAMBlock and inside
KVM). It's important to fixup the dirty bitmap *after* our initial bitmap
sync, such that the corresponding dirty bits in KVM are actually cleared.
As colo is incompatible with discarding of RAM and inhibits it, we don't
have to bother.
Note: if a misbehaving guest would use discarded ranges after migration
started we would still migrate that memory: however, then we already
populated that memory on the migration source.
Reviewed-by: Peter Xu <peterx@redhat.com>
Signed-off-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2021-10-11 20:53:41 +03:00
|
|
|
/* Clear dirty bits of discarded ranges that we don't want to migrate. */
|
|
|
|
ramblock_dirty_bitmap_clear_discarded_pages(block);
|
|
|
|
|
|
|
|
/* We'll recalculate migration_dirty_pages in ram_state_resume_prepare(). */
|
2018-05-02 13:47:28 +03:00
|
|
|
trace_ram_dirty_bitmap_reload_complete(block->idstr);
|
|
|
|
|
migration: synchronize dirty bitmap for resume
This patch implements the first part of core RAM resume logic for
postcopy. ram_resume_prepare() is provided for the work.
When the migration is interrupted by network failure, the dirty bitmap
on the source side will be meaningless, because even the dirty bit is
cleared, it is still possible that the sent page was lost along the way
to destination. Here instead of continue the migration with the old
dirty bitmap on source, we ask the destination side to send back its
received bitmap, then invert it to be our initial dirty bitmap.
The source side send thread will issue the MIG_CMD_RECV_BITMAP requests,
once per ramblock, to ask for the received bitmap. On destination side,
MIG_RP_MSG_RECV_BITMAP will be issued, along with the requested bitmap.
Data will be received on the return-path thread of source, and the main
migration thread will be notified when all the ramblock bitmaps are
synchronized.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20180502104740.12123-17-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2018-05-02 13:47:32 +03:00
|
|
|
/*
|
|
|
|
* We succeeded to sync bitmap for current ramblock. If this is
|
|
|
|
* the last one to sync, we need to notify the main send thread.
|
|
|
|
*/
|
|
|
|
ram_dirty_bitmap_reload_notify(s);
|
|
|
|
|
2018-05-02 13:47:28 +03:00
|
|
|
ret = 0;
|
|
|
|
out:
|
2018-05-25 04:50:42 +03:00
|
|
|
g_free(le_bitmap);
|
2018-05-02 13:47:28 +03:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
migration: synchronize dirty bitmap for resume
This patch implements the first part of core RAM resume logic for
postcopy. ram_resume_prepare() is provided for the work.
When the migration is interrupted by network failure, the dirty bitmap
on the source side will be meaningless, because even the dirty bit is
cleared, it is still possible that the sent page was lost along the way
to destination. Here instead of continue the migration with the old
dirty bitmap on source, we ask the destination side to send back its
received bitmap, then invert it to be our initial dirty bitmap.
The source side send thread will issue the MIG_CMD_RECV_BITMAP requests,
once per ramblock, to ask for the received bitmap. On destination side,
MIG_RP_MSG_RECV_BITMAP will be issued, along with the requested bitmap.
Data will be received on the return-path thread of source, and the main
migration thread will be notified when all the ramblock bitmaps are
synchronized.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20180502104740.12123-17-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2018-05-02 13:47:32 +03:00
|
|
|
static int ram_resume_prepare(MigrationState *s, void *opaque)
|
|
|
|
{
|
|
|
|
RAMState *rs = *(RAMState **)opaque;
|
2018-05-02 13:47:33 +03:00
|
|
|
int ret;
|
migration: synchronize dirty bitmap for resume
This patch implements the first part of core RAM resume logic for
postcopy. ram_resume_prepare() is provided for the work.
When the migration is interrupted by network failure, the dirty bitmap
on the source side will be meaningless, because even the dirty bit is
cleared, it is still possible that the sent page was lost along the way
to destination. Here instead of continue the migration with the old
dirty bitmap on source, we ask the destination side to send back its
received bitmap, then invert it to be our initial dirty bitmap.
The source side send thread will issue the MIG_CMD_RECV_BITMAP requests,
once per ramblock, to ask for the received bitmap. On destination side,
MIG_RP_MSG_RECV_BITMAP will be issued, along with the requested bitmap.
Data will be received on the return-path thread of source, and the main
migration thread will be notified when all the ramblock bitmaps are
synchronized.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20180502104740.12123-17-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2018-05-02 13:47:32 +03:00
|
|
|
|
2018-05-02 13:47:33 +03:00
|
|
|
ret = ram_dirty_bitmap_sync_all(s, rs);
|
|
|
|
if (ret) {
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
ram_state_resume_prepare(rs, s->to_dst_file);
|
|
|
|
|
|
|
|
return 0;
|
migration: synchronize dirty bitmap for resume
This patch implements the first part of core RAM resume logic for
postcopy. ram_resume_prepare() is provided for the work.
When the migration is interrupted by network failure, the dirty bitmap
on the source side will be meaningless, because even the dirty bit is
cleared, it is still possible that the sent page was lost along the way
to destination. Here instead of continue the migration with the old
dirty bitmap on source, we ask the destination side to send back its
received bitmap, then invert it to be our initial dirty bitmap.
The source side send thread will issue the MIG_CMD_RECV_BITMAP requests,
once per ramblock, to ask for the received bitmap. On destination side,
MIG_RP_MSG_RECV_BITMAP will be issued, along with the requested bitmap.
Data will be received on the return-path thread of source, and the main
migration thread will be notified when all the ramblock bitmaps are
synchronized.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20180502104740.12123-17-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2018-05-02 13:47:32 +03:00
|
|
|
}
|
|
|
|
|
2022-07-07 21:55:02 +03:00
|
|
|
void postcopy_preempt_shutdown_file(MigrationState *s)
|
|
|
|
{
|
|
|
|
qemu_put_be64(s->postcopy_qemufile_src, RAM_SAVE_FLAG_EOS);
|
|
|
|
qemu_fflush(s->postcopy_qemufile_src);
|
|
|
|
}
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
static SaveVMHandlers savevm_ram_handlers = {
|
2017-06-28 12:52:24 +03:00
|
|
|
.save_setup = ram_save_setup,
|
2015-05-07 20:33:31 +03:00
|
|
|
.save_live_iterate = ram_save_iterate,
|
2015-11-05 21:11:00 +03:00
|
|
|
.save_live_complete_postcopy = ram_save_complete,
|
2015-11-05 21:10:41 +03:00
|
|
|
.save_live_complete_precopy = ram_save_complete,
|
2017-07-10 19:30:14 +03:00
|
|
|
.has_postcopy = ram_has_postcopy,
|
2022-10-03 03:00:03 +03:00
|
|
|
.state_pending_exact = ram_state_pending_exact,
|
|
|
|
.state_pending_estimate = ram_state_pending_estimate,
|
2015-05-07 20:33:31 +03:00
|
|
|
.load_state = ram_load,
|
2017-06-28 12:52:27 +03:00
|
|
|
.save_cleanup = ram_save_cleanup,
|
|
|
|
.load_setup = ram_load_setup,
|
|
|
|
.load_cleanup = ram_load_cleanup,
|
migration: synchronize dirty bitmap for resume
This patch implements the first part of core RAM resume logic for
postcopy. ram_resume_prepare() is provided for the work.
When the migration is interrupted by network failure, the dirty bitmap
on the source side will be meaningless, because even the dirty bit is
cleared, it is still possible that the sent page was lost along the way
to destination. Here instead of continue the migration with the old
dirty bitmap on source, we ask the destination side to send back its
received bitmap, then invert it to be our initial dirty bitmap.
The source side send thread will issue the MIG_CMD_RECV_BITMAP requests,
once per ramblock, to ask for the received bitmap. On destination side,
MIG_RP_MSG_RECV_BITMAP will be issued, along with the requested bitmap.
Data will be received on the return-path thread of source, and the main
migration thread will be notified when all the ramblock bitmaps are
synchronized.
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
Message-Id: <20180502104740.12123-17-peterx@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
2018-05-02 13:47:32 +03:00
|
|
|
.resume_prepare = ram_resume_prepare,
|
2015-05-07 20:33:31 +03:00
|
|
|
};
|
|
|
|
|
2021-04-29 14:27:02 +03:00
|
|
|
static void ram_mig_ram_block_resized(RAMBlockNotifier *n, void *host,
|
|
|
|
size_t old_size, size_t new_size)
|
|
|
|
{
|
2021-04-29 14:27:04 +03:00
|
|
|
PostcopyState ps = postcopy_state_get();
|
2021-04-29 14:27:02 +03:00
|
|
|
ram_addr_t offset;
|
|
|
|
RAMBlock *rb = qemu_ram_block_from_host(host, false, &offset);
|
|
|
|
Error *err = NULL;
|
|
|
|
|
|
|
|
if (ramblock_is_ignored(rb)) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!migration_is_idle()) {
|
|
|
|
/*
|
|
|
|
* Precopy code on the source cannot deal with the size of RAM blocks
|
|
|
|
* changing at random points in time - especially after sending the
|
|
|
|
* RAM block sizes in the migration stream, they must no longer change.
|
|
|
|
* Abort and indicate a proper reason.
|
|
|
|
*/
|
|
|
|
error_setg(&err, "RAM block '%s' resized during precopy.", rb->idstr);
|
2021-09-29 17:43:10 +03:00
|
|
|
migration_cancel(err);
|
2021-04-29 14:27:02 +03:00
|
|
|
error_free(err);
|
|
|
|
}
|
2021-04-29 14:27:04 +03:00
|
|
|
|
|
|
|
switch (ps) {
|
|
|
|
case POSTCOPY_INCOMING_ADVISE:
|
|
|
|
/*
|
|
|
|
* Update what ram_postcopy_incoming_init()->init_range() does at the
|
|
|
|
* time postcopy was advised. Syncing RAM blocks with the source will
|
|
|
|
* result in RAM resizes.
|
|
|
|
*/
|
|
|
|
if (old_size < new_size) {
|
|
|
|
if (ram_discard_range(rb->idstr, old_size, new_size - old_size)) {
|
|
|
|
error_report("RAM block '%s' discard of resized RAM failed",
|
|
|
|
rb->idstr);
|
|
|
|
}
|
|
|
|
}
|
2021-04-29 14:27:06 +03:00
|
|
|
rb->postcopy_length = new_size;
|
2021-04-29 14:27:04 +03:00
|
|
|
break;
|
|
|
|
case POSTCOPY_INCOMING_NONE:
|
|
|
|
case POSTCOPY_INCOMING_RUNNING:
|
|
|
|
case POSTCOPY_INCOMING_END:
|
|
|
|
/*
|
|
|
|
* Once our guest is running, postcopy does no longer care about
|
|
|
|
* resizes. When growing, the new memory was not available on the
|
|
|
|
* source, no handler needed.
|
|
|
|
*/
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
error_report("RAM block '%s' resized during postcopy state: %d",
|
|
|
|
rb->idstr, ps);
|
|
|
|
exit(-1);
|
|
|
|
}
|
2021-04-29 14:27:02 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static RAMBlockNotifier ram_mig_ram_notifier = {
|
|
|
|
.ram_block_resized = ram_mig_ram_block_resized,
|
|
|
|
};
|
|
|
|
|
2015-05-07 20:33:31 +03:00
|
|
|
void ram_mig_init(void)
|
|
|
|
{
|
|
|
|
qemu_mutex_init(&XBZRLE.lock);
|
2019-08-22 14:54:33 +03:00
|
|
|
register_savevm_live("ram", 0, 4, &savevm_ram_handlers, &ram_state);
|
2021-04-29 14:27:02 +03:00
|
|
|
ram_block_notifier_add(&ram_mig_ram_notifier);
|
2015-05-07 20:33:31 +03:00
|
|
|
}
|