hw/cxl: Make the CXL fixed memory window setup a machine parameter.

Paolo Bonzini requested this change to simplify the ongoing
effort to allow machine setup entirely via RPC.

Includes shortening the command line form cxl-fixed-memory-window
to cxl-fmw as the command lines are extremely long even with this
change.

The json change is needed to ensure that there is
a CXLFixedMemoryWindowOptionsList even though the actual
element in the json is never used. Similar to existing
SgxEpcProperties.

Update qemu-options.hx to reflect that this is now a -machine
parameter.  The bulk of -M / -machine parameters are documented
under machine, so use that in preference to M.

Update cxl-test and bios-tables-test to reflect new parameters.

Signed-off-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Reviewed-by: Ben Widawsky <ben@bwidawsk.net>
Reviewed-by: Davidlohr Bueso <dave@stgolabs.net>
Message-Id: <20220608145440.26106-2-Jonathan.Cameron@huawei.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
This commit is contained in:
Jonathan Cameron 2022-06-08 15:54:33 +01:00 committed by Michael S. Tsirkin
parent 92344e76b8
commit 03b39fcf64
14 changed files with 150 additions and 126 deletions

View File

@ -251,7 +251,7 @@ A very simple setup with just one directly attached CXL Type 3 device::
-device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
-device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
-device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,id=cxl-pmem0 \
-cxl-fixed-memory-window targets.0=cxl.1,size=4G
-M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G
A setup suitable for 4 way interleave. Only one fixed window provided, to enable 2 way
interleave across 2 CXL host bridges. Each host bridge has 2 CXL Root Ports, with
@ -277,7 +277,7 @@ the CXL Type3 device directly attached (no switches).::
-device cxl-type3,bus=root_port15,memdev=cxl-mem3,lsa=cxl-lsa3,id=cxl-pmem2 \
-device cxl-rp,port=1,bus=cxl.2,id=root_port16,chassis=0,slot=6 \
-device cxl-type3,bus=root_port16,memdev=cxl-mem4,lsa=cxl-lsa4,id=cxl-pmem3 \
-cxl-fixed-memory-window targets.0=cxl.1,targets.1=cxl.2,size=4G,interleave-granularity=8k
-M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.targets.1=cxl.2,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k
Kernel Configuration Options
----------------------------

View File

@ -629,20 +629,6 @@ static void machine_set_nvdimm_persistence(Object *obj, const char *value,
nvdimms_state->persistence_string = g_strdup(value);
}
static bool machine_get_cxl(Object *obj, Error **errp)
{
MachineState *ms = MACHINE(obj);
return ms->cxl_devices_state->is_enabled;
}
static void machine_set_cxl(Object *obj, bool value, Error **errp)
{
MachineState *ms = MACHINE(obj);
ms->cxl_devices_state->is_enabled = value;
}
void machine_class_allow_dynamic_sysbus_dev(MachineClass *mc, const char *type)
{
QAPI_LIST_PREPEND(mc->allowed_dynamic_sysbus_devices, g_strdup(type));
@ -929,8 +915,6 @@ static void machine_class_init(ObjectClass *oc, void *data)
mc->default_ram_size = 128 * MiB;
mc->rom_file_has_mr = true;
/* Few machines support CXL, so default to off */
mc->cxl_supported = false;
/* numa node memory size aligned on 8MB by default.
* On Linux, each node's border has to be 8MB aligned
*/
@ -1092,13 +1076,7 @@ static void machine_initfn(Object *obj)
}
if (mc->cxl_supported) {
Object *obj = OBJECT(ms);
ms->cxl_devices_state = g_new0(CXLState, 1);
object_property_add_bool(obj, "cxl", machine_get_cxl, machine_set_cxl);
object_property_set_description(obj, "cxl",
"Set on/off to enable/disable "
"CXL instantiation");
}
if (mc->cpu_index_to_instance_props && mc->get_default_cpu_node_id) {

View File

@ -6,11 +6,9 @@
#include "qemu/osdep.h"
#include "qapi/error.h"
#include "hw/cxl/cxl.h"
void cxl_fixed_memory_window_config(MachineState *ms,
CXLFixedMemoryWindowOptions *object,
Error **errp) {};
#include "hw/cxl/cxl_host.h"
void cxl_fixed_memory_window_link_targets(Error **errp) {};
void cxl_machine_init(Object *obj, CXLState *state) {};
const MemoryRegionOps cfmws_ops;

View File

@ -15,14 +15,15 @@
#include "qapi/qapi-visit-machine.h"
#include "hw/cxl/cxl.h"
#include "hw/cxl/cxl_host.h"
#include "hw/pci/pci_bus.h"
#include "hw/pci/pci_bridge.h"
#include "hw/pci/pci_host.h"
#include "hw/pci/pcie_port.h"
void cxl_fixed_memory_window_config(MachineState *ms,
CXLFixedMemoryWindowOptions *object,
Error **errp)
static void cxl_fixed_memory_window_config(CXLState *cxl_state,
CXLFixedMemoryWindowOptions *object,
Error **errp)
{
CXLFixedWindow *fw = g_malloc0(sizeof(*fw));
strList *target;
@ -62,8 +63,7 @@ void cxl_fixed_memory_window_config(MachineState *ms,
fw->enc_int_gran = 0;
}
ms->cxl_devices_state->fixed_windows =
g_list_append(ms->cxl_devices_state->fixed_windows, fw);
cxl_state->fixed_windows = g_list_append(cxl_state->fixed_windows, fw);
return;
}
@ -220,3 +220,65 @@ const MemoryRegionOps cfmws_ops = {
.unaligned = true,
},
};
static void machine_get_cxl(Object *obj, Visitor *v, const char *name,
void *opaque, Error **errp)
{
CXLState *cxl_state = opaque;
bool value = cxl_state->is_enabled;
visit_type_bool(v, name, &value, errp);
}
static void machine_set_cxl(Object *obj, Visitor *v, const char *name,
void *opaque, Error **errp)
{
CXLState *cxl_state = opaque;
bool value;
if (!visit_type_bool(v, name, &value, errp)) {
return;
}
cxl_state->is_enabled = value;
}
static void machine_get_cfmw(Object *obj, Visitor *v, const char *name,
void *opaque, Error **errp)
{
CXLFixedMemoryWindowOptionsList **list = opaque;
visit_type_CXLFixedMemoryWindowOptionsList(v, name, list, errp);
}
static void machine_set_cfmw(Object *obj, Visitor *v, const char *name,
void *opaque, Error **errp)
{
CXLState *state = opaque;
CXLFixedMemoryWindowOptionsList *cfmw_list = NULL;
CXLFixedMemoryWindowOptionsList *it;
visit_type_CXLFixedMemoryWindowOptionsList(v, name, &cfmw_list, errp);
if (!cfmw_list) {
return;
}
for (it = cfmw_list; it; it = it->next) {
cxl_fixed_memory_window_config(state, it->value, errp);
}
state->cfmw_list = cfmw_list;
}
void cxl_machine_init(Object *obj, CXLState *state)
{
object_property_add(obj, "cxl", "bool", machine_get_cxl,
machine_set_cxl, NULL, state);
object_property_set_description(obj, "cxl",
"Set on/off to enable/disable "
"CXL instantiation");
object_property_add(obj, "cxl-fmw", "CXLFixedMemoryWindow",
machine_get_cfmw, machine_set_cfmw,
NULL, state);
object_property_set_description(obj, "cxl-fmw",
"CXL Fixed Memory Windows (array)");
}

View File

@ -76,6 +76,7 @@
#include "hw/mem/pc-dimm.h"
#include "hw/mem/nvdimm.h"
#include "hw/cxl/cxl.h"
#include "hw/cxl/cxl_host.h"
#include "qapi/error.h"
#include "qapi/qapi-visit-common.h"
#include "qapi/qapi-visit-machine.h"
@ -1682,6 +1683,7 @@ static void pc_machine_set_max_fw_size(Object *obj, Visitor *v,
static void pc_machine_initfn(Object *obj)
{
PCMachineState *pcms = PC_MACHINE(obj);
MachineState *ms = MACHINE(obj);
#ifdef CONFIG_VMPORT
pcms->vmport = ON_OFF_AUTO_AUTO;
@ -1706,6 +1708,7 @@ static void pc_machine_initfn(Object *obj)
pcms->pcspk = isa_new(TYPE_PC_SPEAKER);
object_property_add_alias(OBJECT(pcms), "pcspk-audiodev",
OBJECT(pcms->pcspk), "audiodev");
cxl_machine_init(obj, ms->cxl_devices_state);
}
static void pc_machine_reset(MachineState *machine)

View File

@ -468,7 +468,7 @@ static void pxb_cxl_dev_realize(PCIDevice *dev, Error **errp)
error_setg(errp, "pxb-cxl devices cannot reside on a PCI bus");
return;
}
if (!ms->cxl_devices_state->is_enabled) {
if (!ms->cxl_devices_state || !ms->cxl_devices_state->is_enabled) {
error_setg(errp, "Machine does not have cxl=on");
return;
}

View File

@ -362,6 +362,7 @@ struct MachineState {
struct NVDIMMState *nvdimms_state;
struct CXLState *cxl_devices_state;
struct NumaState *numa_state;
CXLFixedMemoryWindowOptionsList *cfmws_list;
};
#define DEFINE_MACHINE(namestr, machine_initfn) \

View File

@ -12,6 +12,7 @@
#include "qapi/qapi-types-machine.h"
#include "qapi/qapi-visit-machine.h"
#include "hw/pci/pci_bridge.h"
#include "hw/pci/pci_host.h"
#include "cxl_pci.h"
@ -40,6 +41,7 @@ typedef struct CXLState {
MemoryRegion host_mr;
unsigned int next_mr_idx;
GList *fixed_windows;
CXLFixedMemoryWindowOptionsList *cfmw_list;
} CXLState;
struct CXLHost {
@ -51,11 +53,6 @@ struct CXLHost {
#define TYPE_PXB_CXL_HOST "pxb-cxl-host"
OBJECT_DECLARE_SIMPLE_TYPE(CXLHost, PXB_CXL_HOST)
void cxl_fixed_memory_window_config(MachineState *ms,
CXLFixedMemoryWindowOptions *object,
Error **errp);
void cxl_fixed_memory_window_link_targets(Error **errp);
extern const MemoryRegionOps cfmws_ops;
#endif

21
include/hw/cxl/cxl_host.h Normal file
View File

@ -0,0 +1,21 @@
/*
* QEMU CXL Host Setup
*
* Copyright (c) 2022 Huawei
*
* This work is licensed under the terms of the GNU GPL, version 2. See the
* COPYING file in the top-level directory.
*/
#include "qemu/osdep.h"
#include "hw/cxl/cxl.h"
#include "hw/boards.h"
#ifndef CXL_HOST_H
#define CXL_HOST_H
void cxl_machine_init(Object *obj, CXLState *state);
extern const MemoryRegionOps cfmws_ops;
#endif

View File

@ -523,6 +523,19 @@
'*interleave-granularity': 'size',
'targets': ['str'] }}
##
# @CXLFMWProperties:
#
# List of CXL Fixed Memory Windows.
#
# @cxl-fmw: List of CXLFixedMemoryWindowOptions
#
# Since 7.1
##
{ 'struct' : 'CXLFMWProperties',
'data': { 'cxl-fmw': ['CXLFixedMemoryWindowOptions'] }
}
##
# @X86CPURegister32:
#

View File

@ -36,7 +36,8 @@ DEF("machine", HAS_ARG, QEMU_OPTION_machine, \
" nvdimm=on|off controls NVDIMM support (default=off)\n"
" memory-encryption=@var{} memory encryption object to use (default=none)\n"
" hmat=on|off controls ACPI HMAT support (default=off)\n"
" memory-backend='backend-id' specifies explicitly provided backend for main RAM (default=none)\n",
" memory-backend='backend-id' specifies explicitly provided backend for main RAM (default=none)\n"
" cxl-fmw.0.targets.0=firsttarget,cxl-fmw.0.targets.1=secondtarget,cxl-fmw.0.size=size[,cxl-fmw.0.interleave-granularity=granularity]\n",
QEMU_ARCH_ALL)
SRST
``-machine [type=]name[,prop=value[,...]]``
@ -124,6 +125,38 @@ SRST
-object memory-backend-ram,id=pc.ram,size=512M,x-use-canonical-path-for-ramblock-id=off
-machine memory-backend=pc.ram
-m 512M
``cxl-fmw.0.targets.0=firsttarget,cxl-fmw.0.targets.1=secondtarget,cxl-fmw.0.size=size[,cxl-fmw.0.interleave-granularity=granularity]``
Define a CXL Fixed Memory Window (CFMW).
Described in the CXL 2.0 ECN: CEDT CFMWS & QTG _DSM.
They are regions of Host Physical Addresses (HPA) on a system which
may be interleaved across one or more CXL host bridges. The system
software will assign particular devices into these windows and
configure the downstream Host-managed Device Memory (HDM) decoders
in root ports, switch ports and devices appropriately to meet the
interleave requirements before enabling the memory devices.
``targets.X=target`` provides the mapping to CXL host bridges
which may be identified by the id provied in the -device entry.
Multiple entries are needed to specify all the targets when
the fixed memory window represents interleaved memory. X is the
target index from 0.
``size=size`` sets the size of the CFMW. This must be a multiple of
256MiB. The region will be aligned to 256MiB but the location is
platform and configuration dependent.
``interleave-granularity=granularity`` sets the granularity of
interleave. Default 256KiB. Only 256KiB, 512KiB, 1024KiB, 2048KiB
4096KiB, 8192KiB and 16384KiB granularities supported.
Example:
::
-machine cxl-fmw.0.targets.0=cxl.0,cxl-fmw.0.targets.1=cxl.1,cxl-fmw.0.size=128G,cxl-fmw.0.interleave-granularity=512k
ERST
DEF("M", HAS_ARG, QEMU_OPTION_M,
@ -467,44 +500,6 @@ SRST
-numa hmat-cache,node-id=1,size=10K,level=1,associativity=direct,policy=write-back,line=8
ERST
DEF("cxl-fixed-memory-window", HAS_ARG, QEMU_OPTION_cxl_fixed_memory_window,
"-cxl-fixed-memory-window targets.0=firsttarget,targets.1=secondtarget,size=size[,interleave-granularity=granularity]\n",
QEMU_ARCH_ALL)
SRST
``-cxl-fixed-memory-window targets.0=firsttarget,targets.1=secondtarget,size=size[,interleave-granularity=granularity]``
Define a CXL Fixed Memory Window (CFMW).
Described in the CXL 2.0 ECN: CEDT CFMWS & QTG _DSM.
They are regions of Host Physical Addresses (HPA) on a system which
may be interleaved across one or more CXL host bridges. The system
software will assign particular devices into these windows and
configure the downstream Host-managed Device Memory (HDM) decoders
in root ports, switch ports and devices appropriately to meet the
interleave requirements before enabling the memory devices.
``targets.X=firsttarget`` provides the mapping to CXL host bridges
which may be identified by the id provied in the -device entry.
Multiple entries are needed to specify all the targets when
the fixed memory window represents interleaved memory. X is the
target index from 0.
``size=size`` sets the size of the CFMW. This must be a multiple of
256MiB. The region will be aligned to 256MiB but the location is
platform and configuration dependent.
``interleave-granularity=granularity`` sets the granularity of
interleave. Default 256KiB. Only 256KiB, 512KiB, 1024KiB, 2048KiB
4096KiB, 8192KiB and 16384KiB granularities supported.
Example:
::
-cxl-fixed-memory-window targets.0=cxl.0,targets.1=cxl.1,size=128G,interleave-granularity=512k
ERST
DEF("add-fd", HAS_ARG, QEMU_OPTION_add_fd,
"-add-fd fd=fd,set=set[,opaque=opaque]\n"
" Add 'fd' to fd 'set'\n", QEMU_ARCH_ALL)

View File

@ -147,12 +147,6 @@ typedef struct BlockdevOptionsQueueEntry {
typedef QSIMPLEQ_HEAD(, BlockdevOptionsQueueEntry) BlockdevOptionsQueue;
typedef struct CXLFMWOptionQueueEntry {
CXLFixedMemoryWindowOptions *opts;
Location loc;
QSIMPLEQ_ENTRY(CXLFMWOptionQueueEntry) entry;
} CXLFMWOptionQueueEntry;
typedef struct ObjectOption {
ObjectOptions *opts;
QTAILQ_ENTRY(ObjectOption) next;
@ -179,8 +173,6 @@ static int snapshot;
static bool preconfig_requested;
static QemuPluginList plugin_list = QTAILQ_HEAD_INITIALIZER(plugin_list);
static BlockdevOptionsQueue bdo_queue = QSIMPLEQ_HEAD_INITIALIZER(bdo_queue);
static QSIMPLEQ_HEAD(, CXLFMWOptionQueueEntry) CXLFMW_opts =
QSIMPLEQ_HEAD_INITIALIZER(CXLFMW_opts);
static bool nographic = false;
static int mem_prealloc; /* force preallocation of physical target memory */
static const char *vga_model = NULL;
@ -1072,24 +1064,6 @@ static void parse_display(const char *p)
}
}
static void parse_cxl_fixed_memory_window(const char *optarg)
{
CXLFMWOptionQueueEntry *cfmws_entry;
Visitor *v;
v = qobject_input_visitor_new_str(optarg, "cxl-fixed-memory-window",
&error_fatal);
cfmws_entry = g_new(CXLFMWOptionQueueEntry, 1);
visit_type_CXLFixedMemoryWindowOptions(v, NULL, &cfmws_entry->opts,
&error_fatal);
if (!cfmws_entry->opts) {
exit(1);
}
visit_free(v);
loc_save(&cfmws_entry->loc);
QSIMPLEQ_INSERT_TAIL(&CXLFMW_opts, cfmws_entry, entry);
}
static inline bool nonempty_str(const char *str)
{
return str && *str;
@ -1948,20 +1922,6 @@ static void qemu_create_late_backends(void)
qemu_semihosting_console_init();
}
static void cxl_set_opts(void)
{
while (!QSIMPLEQ_EMPTY(&CXLFMW_opts)) {
CXLFMWOptionQueueEntry *cfmws_entry = QSIMPLEQ_FIRST(&CXLFMW_opts);
loc_restore(&cfmws_entry->loc);
QSIMPLEQ_REMOVE_HEAD(&CXLFMW_opts, entry);
cxl_fixed_memory_window_config(current_machine, cfmws_entry->opts,
&error_fatal);
qapi_free_CXLFixedMemoryWindowOptions(cfmws_entry->opts);
g_free(cfmws_entry);
}
}
static void qemu_resolve_machine_memdev(void)
{
if (ram_memdev_id) {
@ -2789,9 +2749,6 @@ void qemu_init(int argc, char **argv, char **envp)
exit(1);
}
break;
case QEMU_OPTION_cxl_fixed_memory_window:
parse_cxl_fixed_memory_window(optarg);
break;
case QEMU_OPTION_display:
parse_display(optarg);
break;
@ -3598,7 +3555,6 @@ void qemu_init(int argc, char **argv, char **envp)
qemu_resolve_machine_memdev();
parse_numa_opts(current_machine);
cxl_set_opts();
if (vmstate_dump_file) {
/* dump and exit */

View File

@ -1582,8 +1582,8 @@ static void test_acpi_q35_cxl(void)
" -device cxl-type3,bus=rp3,memdev=cxl-mem3,lsa=lsa3"
" -device cxl-rp,port=1,bus=cxl.2,id=rp4,chassis=0,slot=6"
" -device cxl-type3,bus=rp4,memdev=cxl-mem4,lsa=lsa4"
" -cxl-fixed-memory-window targets.0=cxl.1,size=4G,interleave-granularity=8k"
" -cxl-fixed-memory-window targets.0=cxl.1,targets.1=cxl.2,size=4G,interleave-granularity=8k",
" -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k,"
"cxl-fmw.1.targets.0=cxl.1,cxl-fmw.1.targets.1=cxl.2,cxl-fmw.1.size=4G,cxl-fmw.1.interleave-granularity=8k",
tmp_path, tmp_path, tmp_path, tmp_path,
tmp_path, tmp_path, tmp_path, tmp_path);
test_acpi_one(params, &data);

View File

@ -10,12 +10,12 @@
#define QEMU_PXB_CMD "-machine q35,cxl=on " \
"-device pxb-cxl,id=cxl.0,bus=pcie.0,bus_nr=52 " \
"-cxl-fixed-memory-window targets.0=cxl.0,size=4G "
"-M cxl-fmw.0.targets.0=cxl.0,cxl-fmw.0.size=4G "
#define QEMU_2PXB_CMD "-machine q35,cxl=on " \
"-device pxb-cxl,id=cxl.0,bus=pcie.0,bus_nr=52 " \
"-device pxb-cxl,id=cxl.1,bus=pcie.0,bus_nr=53 " \
"-cxl-fixed-memory-window targets.0=cxl.0,targets.1=cxl.1,size=4G "
"-M cxl-fmw.0.targets.0=cxl.0,cxl-fmw.0.targets.1=cxl.1,cxl-fmw.0.size=4G "
#define QEMU_RP "-device cxl-rp,id=rp0,bus=cxl.0,chassis=0,slot=0 "