mirrors/nvidia-open-gpu-kernel-modules
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

550.78

Browse Source
This commit is contained in:
Bernhard Stoeckner 2024-04-25 16:24:58 +02:00
parent ea4c27fad6
commit 083cd9cf17
No known key found for this signature in database
GPG Key ID: 7D23DC2750FAC2E1
36 changed files with 36185 additions and 35632 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -2,6 +2,8 @@
## Release 550 Entries
### [550.78] 2024-04-25
### [550.76] 2024-04-17
### [550.67] 2024-03-19

6
README.md
View File

@ -1,7 +1,7 @@
# NVIDIA Linux Open GPU Kernel Module Source
This is the source release of the NVIDIA Linux open GPU kernel modules,
version 550.76.
version 550.78.
## How to Build
@ -17,7 +17,7 @@ as root:
Note that the kernel modules built here must be used with GSP
firmware and user-space NVIDIA GPU driver components from a corresponding
550.76 driver release. This can be achieved by installing
550.78 driver release. This can be achieved by installing
the NVIDIA GPU driver from the .run file using the `--no-kernel-modules`
option. E.g.,
@ -188,7 +188,7 @@ encountered specific to them.
For details on feature support and limitations, see the NVIDIA GPU driver
end user README here:
https://us.download.nvidia.com/XFree86/Linux-x86_64/550.76/README/kernel_open.html
https://us.download.nvidia.com/XFree86/Linux-x86_64/550.78/README/kernel_open.html
For vGPU support, please refer to the README.vgpu packaged in the vGPU Host
Package for more details.

2
kernel-open/Kbuild
View File

@ -72,7 +72,7 @@ EXTRA_CFLAGS += -I$(src)/common/inc
EXTRA_CFLAGS += -I$(src)
EXTRA_CFLAGS += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-format-extra-args
EXTRA_CFLAGS += -D__KERNEL__ -DMODULE -DNVRM
EXTRA_CFLAGS += -DNV_VERSION_STRING=\"550.76\"
EXTRA_CFLAGS += -DNV_VERSION_STRING=\"550.78\"
ifneq ($(SYSSRCHOST1X),)
EXTRA_CFLAGS += -I$(SYSSRCHOST1X)

16
src/common/inc/nvBldVer.h
View File

@ -43,18 +43,18 @@
#endif
#if defined(NV_LINUX) || defined(NV_BSD) || defined(NV_SUNOS)
#define NV_BUILD_BRANCH_VERSION "rel/gpu_drv/r550/r550_00-237"
#define NV_BUILD_CHANGELIST_NUM (34145289)
#define NV_BUILD_BRANCH_VERSION "rel/gpu_drv/r550/r550_00-242"
#define NV_BUILD_CHANGELIST_NUM (34157620)
#define NV_BUILD_TYPE "Official"
#define NV_BUILD_NAME "rel/gpu_drv/r550/r550_00-237"
#define NV_LAST_OFFICIAL_CHANGELIST_NUM (34145289)
#define NV_BUILD_NAME "rel/gpu_drv/r550/r550_00-242"
#define NV_LAST_OFFICIAL_CHANGELIST_NUM (34157620)
#else /* Windows builds */
#define NV_BUILD_BRANCH_VERSION "r550_00-227"
#define NV_BUILD_CHANGELIST_NUM (34145289)
#define NV_BUILD_BRANCH_VERSION "r550_00-233"
#define NV_BUILD_CHANGELIST_NUM (34158633)
#define NV_BUILD_TYPE "Official"
#define NV_BUILD_NAME "552.19"
#define NV_LAST_OFFICIAL_CHANGELIST_NUM (34145289)
#define NV_BUILD_NAME "552.25"
#define NV_LAST_OFFICIAL_CHANGELIST_NUM (34158633)
#define NV_BUILD_BRANCH_BASE_VERSION R550
#endif
// End buildmeister python edited section

2
src/common/inc/nvUnixVersion.h
View File

@ -4,7 +4,7 @@
#if defined(NV_LINUX) || defined(NV_BSD) || defined(NV_SUNOS) || defined(NV_VMWARE) || defined(NV_QNX) || defined(NV_INTEGRITY) || \
(defined(RMCFG_FEATURE_PLATFORM_GSP) && RMCFG_FEATURE_PLATFORM_GSP == 1)
#define NV_VERSION_STRING "550.76"
#define NV_VERSION_STRING "550.78"
#else

16
src/common/sdk/nvidia/inc/class/cl0050.h
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -30,6 +30,8 @@
// Source file: class/cl0050.finn
//
#include "nvcfg_sdk.h"
#define NV_CE_UTILS (0x50U) /* finn: Evaluated from "NV0050_ALLOCATION_PARAMETERS_MESSAGE_ID" */
#define NV0050_ALLOCATION_PARAMETERS_MESSAGE_ID (0x0050U)
@ -37,6 +39,7 @@
typedef struct NV0050_ALLOCATION_PARAMETERS {
NvHandle hVaspace;
NV_DECLARE_ALIGNED(NvU64 flags, 8);
NvU32 forceCeId;
} NV0050_ALLOCATION_PARAMETERS;
@ -54,3 +57,14 @@ typedef struct NV0050_ALLOCATION_PARAMETERS {
#define NV0050_CEUTILS_FLAGS_FIFO_LITE 2:2
#define NV0050_CEUTILS_FLAGS_FIFO_LITE_FALSE (0x00000000)
#define NV0050_CEUTILS_FLAGS_FIFO_LITE_TRUE (0x00000001)
// Force a specific CE engine to be used be setting forceCeId
#define NV0050_CEUTILS_FLAGS_FORCE_CE_ID 4:4
#define NV0050_CEUTILS_FLAGS_FORCE_CE_ID_FALSE (0x00000000)
#define NV0050_CEUTILS_FLAGS_FORCE_CE_ID_TRUE (0x00000001)
// Use a CC secure channel
#define NV0050_CEUTILS_FLAGS_CC_SECURE 5:5
#define NV0050_CEUTILS_FLAGS_CC_SECURE_FALSE (0x00000000)
#define NV0050_CEUTILS_FLAGS_CC_SECURE_TRUE (0x00000001)

20
src/common/sdk/nvidia/inc/class/clc8b5.h
View File

@ -52,6 +52,10 @@
#define NVC8B5_LAUNCH_DMA_SRC_TYPE_PHYSICAL (0x00000001)
#define NVC8B5_LAUNCH_DMA_DST_TYPE 13:13
#define NVC8B5_LAUNCH_DMA_DST_TYPE_PHYSICAL (0x00000001)
#define NVC8B5_LAUNCH_DMA_COPY_TYPE 21:20
#define NVC8B5_LAUNCH_DMA_COPY_TYPE_PROT2PROT (0x00000000)
#define NVC8B5_LAUNCH_DMA_COPY_TYPE_DEFAULT (0x00000000)
#define NVC8B5_LAUNCH_DMA_COPY_TYPE_SECURE (0x00000001)
#define NVC8B5_LAUNCH_DMA_MEMORY_SCRUB_ENABLE 23:23
#define NVC8B5_LAUNCH_DMA_MEMORY_SCRUB_ENABLE_TRUE (0x00000001)
#define NVC8B5_LAUNCH_DMA_DISABLE_PLC 26:26
@ -61,6 +65,22 @@
#define NVC8B5_OFFSET_OUT_LOWER (0x0000040C)
#define NVC8B5_OFFSET_OUT_LOWER_VALUE 31:0
#define NVC8B5_LINE_LENGTH_IN (0x00000418)
#define NVC8B5_SET_SECURE_COPY_MODE (0x00000500)
#define NVC8B5_SET_SECURE_COPY_MODE_MODE 0:0
#define NVC8B5_SET_SECURE_COPY_MODE_MODE_ENCRYPT (0x00000000)
#define NVC8B5_SET_SECURE_COPY_MODE_MODE_DECRYPT (0x00000001)
#define NVC8B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_UPPER (0x00000514)
#define NVC8B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_UPPER_UPPER 24:0
#define NVC8B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_LOWER (0x00000518)
#define NVC8B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_LOWER_LOWER 31:0
#define NVC8B5_SET_ENCRYPT_AUTH_TAG_ADDR_UPPER (0x00000530)
#define NVC8B5_SET_ENCRYPT_AUTH_TAG_ADDR_UPPER_UPPER 24:0
#define NVC8B5_SET_ENCRYPT_AUTH_TAG_ADDR_LOWER (0x00000534)
#define NVC8B5_SET_ENCRYPT_AUTH_TAG_ADDR_LOWER_LOWER 31:0
#define NVC8B5_SET_ENCRYPT_IV_ADDR_UPPER (0x00000538)
#define NVC8B5_SET_ENCRYPT_IV_ADDR_UPPER_UPPER 24:0
#define NVC8B5_SET_ENCRYPT_IV_ADDR_LOWER (0x0000053C)
#define NVC8B5_SET_ENCRYPT_IV_ADDR_LOWER_LOWER 31:0
#define NVC8B5_SET_MEMORY_SCRUB_PARAMETERS (0x000006FC)
#define NVC8B5_SET_MEMORY_SCRUB_PARAMETERS_DISCARDABLE 0:0
#define NVC8B5_SET_MEMORY_SCRUB_PARAMETERS_DISCARDABLE_FALSE (0x00000000)

28
src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h
View File

@ -3839,6 +3839,34 @@ typedef struct NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_GPU_STATE_PARAMS {
NvBool bAcceptClientRequest;
} NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_GPU_STATE_PARAMS;
/*!
* NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY
*
* This control call can be used to set CC security policy on GSP.
* This is a internal command sent from Kernel RM to Physical RM.
*
* attackerAdvantage [IN]
* The minimum and maximum values for attackerAdvantage.
* The probability of an attacker successfully guessing the contents of
* an encrypted packet go up ("attacker advantage").
*
* Possible status values returned are:
* NV_OK
* NV_ERR_INVALID_OBJECT_HANDLE
* NV_ERR_INVALID_STATE
* NV_ERR_INVALID_ARGUMENT
* NV_ERR_NOT_SUPPORTED
*/
#define NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY (0x20800ae8) /* finn: Evaluated from "(FINN_NV20_SUBDEVICE_0_INTERNAL_INTERFACE_ID << 8) | NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS_MESSAGE_ID" */
#define NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS_MESSAGE_ID (0xE8U)
typedef struct NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS {
NV_DECLARE_ALIGNED(NvU64 attackerAdvantage, 8);
} NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS;
/*
* NV2080_CTRL_CMD_INTERNAL_MEMMGR_MEMORY_TRANSFER_WITH_GSP

16
src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080spdm.h
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -168,20 +168,20 @@ typedef union RM_GSP_SPDM_CMD *PRM_GSP_SPDM_CMD;
* SPDM message structure.
*/
typedef struct RM_GSP_SPDM_MSG {
NvU8 msgType;
NvU8 msgType;
// status returned from GSP message infrastructure.
NvU32 status;
NvU32 status;
NvU32 rsvd1;
NvU32 rsvd1;
NvU32 rsvd2;
NvU32 rsvd2;
NvU32 rsvd3;
NvU32 rsvd3;
NvU32 rsvd4;
NvU32 rsvd4;
NvBool rsvd5;
NvU32 rsvd5;
} RM_GSP_SPDM_MSG;
typedef struct RM_GSP_SPDM_MSG *PRM_GSP_SPDM_MSG;

3
src/common/sdk/nvidia/inc/ctrl/ctrlc56f.h
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2018-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -243,6 +243,7 @@ typedef struct NVC56F_CTRL_ROTATE_SECURE_CHANNEL_IV_PARAMS {
* The minimum and maximum values for attackerAdvantage.
* The probability of an attacker successfully guessing the contents of an encrypted packet go up ("attacker advantage").
*/
#define SECURITY_POLICY_ATTACKER_ADVANTAGE_DEFAULT (60)
#define SET_SECURITY_POLICY_ATTACKER_ADVANTAGE_MIN (50)
#define SET_SECURITY_POLICY_ATTACKER_ADVANTAGE_MAX (75)

48946
src/nvidia/generated/g_bindata_kgspGetBinArchiveConcatenatedFMC_GH100.c
View File

File diff suppressed because it is too large Load Diff

20846
src/nvidia/generated/g_bindata_kgspGetBinArchiveGspRmCcFmcGfwProdSigned_GH100.c
View File

File diff suppressed because it is too large Load Diff

10
src/nvidia/generated/g_ce_utils_nvoc.h
View File

@ -7,7 +7,7 @@ extern "C" {
#endif
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -59,6 +59,12 @@ typedef struct
NvU64 length;
NvU64 flags;
NvU64 submittedWorkId; // Payload to poll for async completion
NvBool bSecureCopy; // The copy encrypts/decrypts when copying to/from unprotected memory
NvBool bEncrypt; // Toggle encrypt/decrypt
NvU64 authTagAddr; // encryption authTag address. Same aperture as unencrypted operand assumed. 16 byte aligned
NvU64 encryptIvAddr; // IV value that was used for ecryption, requirements are the same as for authTagAddr. Required
} CEUTILS_MEMCOPY_PARAMS;
struct KernelChannel;
@ -90,8 +96,6 @@ struct CeUtils {
struct Object *__nvoc_pbase_Object;
struct CeUtils *__nvoc_pbase_CeUtils;
NvHandle hClient;
NvHandle hDevice;
NvHandle hSubdevice;
OBJCHANNEL *pChannel;
struct OBJGPU *pGpu;
struct KernelCE *pKCe;

50
src/nvidia/generated/g_conf_compute_nvoc.h
View File

@ -7,7 +7,7 @@ extern "C" {
#endif
/*
* SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -39,6 +39,7 @@ extern "C" {
#include "gpu/spdm/spdm.h"
#include "ctrl/ctrl2080/ctrl2080spdm.h"
#include "ctrl/ctrl2080/ctrl2080internal.h"
#include "ctrl/ctrlc56f.h"
#include "cc_drv.h"
#include "conf_compute/cc_keystore.h"
#include "kernel/gpu/fifo/kernel_channel.h"
@ -154,12 +155,16 @@ struct ConfidentialCompute {
NvU32 keyRotationEnableMask;
KEY_ROTATION_STATS_INFO lowerThreshold;
KEY_ROTATION_STATS_INFO upperThreshold;
NvU64 attackerAdvantage;
NvU8 PRIVATE_FIELD(m_exportMasterKey)[32];
void *PRIVATE_FIELD(m_keySlot);
KEY_ROTATION_STATUS PRIVATE_FIELD(keyRotationState)[62];
KEY_ROTATION_STATS_INFO PRIVATE_FIELD(aggregateStats)[62];
KEY_ROTATION_STATS_INFO PRIVATE_FIELD(freedChannelAggregateStats)[62];
PTMR_EVENT PRIVATE_FIELD(ppKeyRotationTimer)[62];
NvU64 PRIVATE_FIELD(keyRotationLimitDelta);
NvU64 PRIVATE_FIELD(keyRotationUpperLimit);
NvU64 PRIVATE_FIELD(keyRotationLowerLimit);
};
#ifndef __NVOC_CLASS_ConfidentialCompute_TYPEDEF__
@ -711,6 +716,39 @@ static inline NV_STATUS confComputeUpdateFreedChannelStats(struct OBJGPU *pGpu,
#define confComputeUpdateFreedChannelStats(pGpu, pConfCompute, pKernelChannel) confComputeUpdateFreedChannelStats_IMPL(pGpu, pConfCompute, pKernelChannel)
#endif //__nvoc_conf_compute_h_disabled
NV_STATUS confComputeSetKeyRotationThreshold_IMPL(struct ConfidentialCompute *pConfCompute, NvU64 attackerAdvantage);
#ifdef __nvoc_conf_compute_h_disabled
static inline NV_STATUS confComputeSetKeyRotationThreshold(struct ConfidentialCompute *pConfCompute, NvU64 attackerAdvantage) {
NV_ASSERT_FAILED_PRECOMP("ConfidentialCompute was disabled!");
return NV_ERR_NOT_SUPPORTED;
}
#else //__nvoc_conf_compute_h_disabled
#define confComputeSetKeyRotationThreshold(pConfCompute, attackerAdvantage) confComputeSetKeyRotationThreshold_IMPL(pConfCompute, attackerAdvantage)
#endif //__nvoc_conf_compute_h_disabled
NvBool confComputeIsUpperThresholdCrossed_IMPL(struct ConfidentialCompute *pConfCompute, const KEY_ROTATION_STATS_INFO *pStatsInfo);
#ifdef __nvoc_conf_compute_h_disabled
static inline NvBool confComputeIsUpperThresholdCrossed(struct ConfidentialCompute *pConfCompute, const KEY_ROTATION_STATS_INFO *pStatsInfo) {
NV_ASSERT_FAILED_PRECOMP("ConfidentialCompute was disabled!");
return NV_FALSE;
}
#else //__nvoc_conf_compute_h_disabled
#define confComputeIsUpperThresholdCrossed(pConfCompute, pStatsInfo) confComputeIsUpperThresholdCrossed_IMPL(pConfCompute, pStatsInfo)
#endif //__nvoc_conf_compute_h_disabled
NvBool confComputeIsLowerThresholdCrossed_IMPL(struct ConfidentialCompute *pConfCompute, const KEY_ROTATION_STATS_INFO *pStatsInfo);
#ifdef __nvoc_conf_compute_h_disabled
static inline NvBool confComputeIsLowerThresholdCrossed(struct ConfidentialCompute *pConfCompute, const KEY_ROTATION_STATS_INFO *pStatsInfo) {
NV_ASSERT_FAILED_PRECOMP("ConfidentialCompute was disabled!");
return NV_FALSE;
}
#else //__nvoc_conf_compute_h_disabled
#define confComputeIsLowerThresholdCrossed(pConfCompute, pStatsInfo) confComputeIsLowerThresholdCrossed_IMPL(pConfCompute, pStatsInfo)
#endif //__nvoc_conf_compute_h_disabled
#undef PRIVATE_FIELD
#ifndef NVOC_CONF_COMPUTE_H_PRIVATE_ACCESS_ALLOWED
@ -750,6 +788,16 @@ NV_STATUS NVOC_PRIVATE_FUNCTION(confComputeKeyStoreUpdateKey)(struct Confidentia
#undef confComputeKeyStoreUpdateKey_HAL
NV_STATUS NVOC_PRIVATE_FUNCTION(confComputeKeyStoreUpdateKey_HAL)(struct ConfidentialCompute *pConfCompute, NvU32 globalKeyId);
#ifndef __nvoc_conf_compute_h_disabled
#undef confComputeIsUpperThresholdCrossed
NvBool NVOC_PRIVATE_FUNCTION(confComputeIsUpperThresholdCrossed)(struct ConfidentialCompute *pConfCompute, const KEY_ROTATION_STATS_INFO *pStatsInfo);
#endif //__nvoc_conf_compute_h_disabled
#ifndef __nvoc_conf_compute_h_disabled
#undef confComputeIsLowerThresholdCrossed
NvBool NVOC_PRIVATE_FUNCTION(confComputeIsLowerThresholdCrossed)(struct ConfidentialCompute *pConfCompute, const KEY_ROTATION_STATS_INFO *pStatsInfo);
#endif //__nvoc_conf_compute_h_disabled
#endif // NVOC_CONF_COMPUTE_H_PRIVATE_ACCESS_ALLOWED

1
src/nvidia/generated/g_gpu_mgr_nvoc.h
View File

@ -303,6 +303,7 @@ struct OBJGPUMGR {
GPU_HANDLE_ID gpuHandleIDList[32];
NvU32 numGpuHandles;
CONF_COMPUTE_CAPS ccCaps;
NvU64 ccAttackerAdvantage;
pcieP2PCapsInfoList pcieP2PCapsInfoCache;
void *pcieP2PCapsInfoLock;
};

49
src/nvidia/generated/g_kernel_ce_nvoc.c
View File

@ -83,14 +83,22 @@ static NvBool __nvoc_thunk_KernelCE_engstateIsPresent(OBJGPU *pGpu, struct OBJEN
return kceIsPresent(pGpu, (struct KernelCE *)(((unsigned char *)pKCe) - __nvoc_rtti_KernelCE_OBJENGSTATE.offset));
}
static NV_STATUS __nvoc_thunk_KernelCE_engstateStateLoad(OBJGPU *arg0, struct OBJENGSTATE *arg1, NvU32 arg2) {
return kceStateLoad(arg0, (struct KernelCE *)(((unsigned char *)arg1) - __nvoc_rtti_KernelCE_OBJENGSTATE.offset), arg2);
static NV_STATUS __nvoc_thunk_KernelCE_engstateStateInitLocked(OBJGPU *arg0, struct OBJENGSTATE *arg1) {
return kceStateInitLocked(arg0, (struct KernelCE *)(((unsigned char *)arg1) - __nvoc_rtti_KernelCE_OBJENGSTATE.offset));
}
static NV_STATUS __nvoc_thunk_KernelCE_engstateStateUnload(OBJGPU *pGpu, struct OBJENGSTATE *pKCe, NvU32 flags) {
return kceStateUnload(pGpu, (struct KernelCE *)(((unsigned char *)pKCe) - __nvoc_rtti_KernelCE_OBJENGSTATE.offset), flags);
}
static NV_STATUS __nvoc_thunk_KernelCE_engstateStateLoad(OBJGPU *arg0, struct OBJENGSTATE *arg1, NvU32 arg2) {
return kceStateLoad(arg0, (struct KernelCE *)(((unsigned char *)arg1) - __nvoc_rtti_KernelCE_OBJENGSTATE.offset), arg2);
}
static void __nvoc_thunk_KernelCE_engstateStateDestroy(OBJGPU *arg0, struct OBJENGSTATE *arg1) {
kceStateDestroy(arg0, (struct KernelCE *)(((unsigned char *)arg1) - __nvoc_rtti_KernelCE_OBJENGSTATE.offset));
}
static void __nvoc_thunk_KernelCE_intrservRegisterIntrService(OBJGPU *arg0, struct IntrService *arg1, IntrServiceRecord arg2[171]) {
kceRegisterIntrService(arg0, (struct KernelCE *)(((unsigned char *)arg1) - __nvoc_rtti_KernelCE_IntrService.offset), arg2);
}
@ -99,10 +107,6 @@ static NV_STATUS __nvoc_thunk_KernelCE_intrservServiceNotificationInterrupt(OBJG
return kceServiceNotificationInterrupt(arg0, (struct KernelCE *)(((unsigned char *)arg1) - __nvoc_rtti_KernelCE_IntrService.offset), arg2);
}
static NV_STATUS __nvoc_thunk_OBJENGSTATE_kceStateInitLocked(POBJGPU pGpu, struct KernelCE *pEngstate) {
return engstateStateInitLocked(pGpu, (struct OBJENGSTATE *)(((unsigned char *)pEngstate) + __nvoc_rtti_KernelCE_OBJENGSTATE.offset));
}
static NV_STATUS __nvoc_thunk_OBJENGSTATE_kceStatePreLoad(POBJGPU pGpu, struct KernelCE *pEngstate, NvU32 arg0) {
return engstateStatePreLoad(pGpu, (struct OBJENGSTATE *)(((unsigned char *)pEngstate) + __nvoc_rtti_KernelCE_OBJENGSTATE.offset), arg0);
}
@ -111,10 +115,6 @@ static NV_STATUS __nvoc_thunk_OBJENGSTATE_kceStatePostUnload(POBJGPU pGpu, struc
return engstateStatePostUnload(pGpu, (struct OBJENGSTATE *)(((unsigned char *)pEngstate) + __nvoc_rtti_KernelCE_OBJENGSTATE.offset), arg0);
}
static void __nvoc_thunk_OBJENGSTATE_kceStateDestroy(POBJGPU pGpu, struct KernelCE *pEngstate) {
engstateStateDestroy(pGpu, (struct OBJENGSTATE *)(((unsigned char *)pEngstate) + __nvoc_rtti_KernelCE_OBJENGSTATE.offset));
}
static NV_STATUS __nvoc_thunk_OBJENGSTATE_kceStatePreUnload(POBJGPU pGpu, struct KernelCE *pEngstate, NvU32 arg0) {
return engstateStatePreUnload(pGpu, (struct OBJENGSTATE *)(((unsigned char *)pEngstate) + __nvoc_rtti_KernelCE_OBJENGSTATE.offset), arg0);
}
@ -172,6 +172,17 @@ void __nvoc_init_dataField_KernelCE(KernelCE *pThis, RmHalspecOwner *pRmhalspeco
PORT_UNREFERENCED_VARIABLE(chipHal_HalVarIdx);
PORT_UNREFERENCED_VARIABLE(rmVariantHal);
PORT_UNREFERENCED_VARIABLE(rmVariantHal_HalVarIdx);
// Hal field -- bCcFipsSelfTestRequired
if (( ((chipHal_HalVarIdx >> 5) == 1UL) && ((1UL << (chipHal_HalVarIdx & 0x1f)) & 0x10000000UL) )) /* ChipHal: GH100 */
{
pThis->bCcFipsSelfTestRequired = ((NvBool)(0 == 0));
}
// default
else
{
pThis->bCcFipsSelfTestRequired = ((NvBool)(0 != 0));
}
}
NV_STATUS __nvoc_ctor_OBJENGSTATE(OBJENGSTATE* );
@ -210,13 +221,17 @@ static void __nvoc_init_funcTable_KernelCE_1(KernelCE *pThis, RmHalspecOwner *pR
// Hal function -- kceIsPresent
pThis->__kceIsPresent__ = &kceIsPresent_IMPL;
// Hal function -- kceStateLoad
pThis->__kceStateLoad__ = &kceStateLoad_GP100;
pThis->__kceStateInitLocked__ = &kceStateInitLocked_IMPL;
// Hal function -- kceStateUnload
// default
pThis->__kceStateUnload__ = &kceStateUnload_56cd7a;
// Hal function -- kceStateLoad
pThis->__kceStateLoad__ = &kceStateLoad_GP100;
pThis->__kceStateDestroy__ = &kceStateDestroy_IMPL;
pThis->__kceRegisterIntrService__ = &kceRegisterIntrService_IMPL;
pThis->__kceServiceNotificationInterrupt__ = &kceServiceNotificationInterrupt_IMPL;
@ -436,22 +451,22 @@ static void __nvoc_init_funcTable_KernelCE_1(KernelCE *pThis, RmHalspecOwner *pR
pThis->__nvoc_base_OBJENGSTATE.__engstateIsPresent__ = &__nvoc_thunk_KernelCE_engstateIsPresent;
pThis->__nvoc_base_OBJENGSTATE.__engstateStateLoad__ = &__nvoc_thunk_KernelCE_engstateStateLoad;
pThis->__nvoc_base_OBJENGSTATE.__engstateStateInitLocked__ = &__nvoc_thunk_KernelCE_engstateStateInitLocked;
pThis->__nvoc_base_OBJENGSTATE.__engstateStateUnload__ = &__nvoc_thunk_KernelCE_engstateStateUnload;
pThis->__nvoc_base_OBJENGSTATE.__engstateStateLoad__ = &__nvoc_thunk_KernelCE_engstateStateLoad;
pThis->__nvoc_base_OBJENGSTATE.__engstateStateDestroy__ = &__nvoc_thunk_KernelCE_engstateStateDestroy;
pThis->__nvoc_base_IntrService.__intrservRegisterIntrService__ = &__nvoc_thunk_KernelCE_intrservRegisterIntrService;
pThis->__nvoc_base_IntrService.__intrservServiceNotificationInterrupt__ = &__nvoc_thunk_KernelCE_intrservServiceNotificationInterrupt;
pThis->__kceStateInitLocked__ = &__nvoc_thunk_OBJENGSTATE_kceStateInitLocked;
pThis->__kceStatePreLoad__ = &__nvoc_thunk_OBJENGSTATE_kceStatePreLoad;
pThis->__kceStatePostUnload__ = &__nvoc_thunk_OBJENGSTATE_kceStatePostUnload;
pThis->__kceStateDestroy__ = &__nvoc_thunk_OBJENGSTATE_kceStateDestroy;
pThis->__kceStatePreUnload__ = &__nvoc_thunk_OBJENGSTATE_kceStatePreUnload;
pThis->__kceStateInitUnlocked__ = &__nvoc_thunk_OBJENGSTATE_kceStateInitUnlocked;

41
src/nvidia/generated/g_kernel_ce_nvoc.h
View File

@ -115,8 +115,10 @@ struct KernelCE {
struct KernelCE *__nvoc_pbase_KernelCE;
NV_STATUS (*__kceConstructEngine__)(OBJGPU *, struct KernelCE *, ENGDESCRIPTOR);
NvBool (*__kceIsPresent__)(OBJGPU *, struct KernelCE *);
NV_STATUS (*__kceStateLoad__)(OBJGPU *, struct KernelCE *, NvU32);
NV_STATUS (*__kceStateInitLocked__)(OBJGPU *, struct KernelCE *);
NV_STATUS (*__kceStateUnload__)(OBJGPU *, struct KernelCE *, NvU32);
NV_STATUS (*__kceStateLoad__)(OBJGPU *, struct KernelCE *, NvU32);
void (*__kceStateDestroy__)(OBJGPU *, struct KernelCE *);
void (*__kceRegisterIntrService__)(OBJGPU *, struct KernelCE *, IntrServiceRecord *);
NV_STATUS (*__kceServiceNotificationInterrupt__)(OBJGPU *, struct KernelCE *, IntrServiceServiceNotificationInterruptArguments *);
NV_STATUS (*__kceGetP2PCes__)(struct KernelCE *, OBJGPU *, NvU32, NvU32 *);
@ -136,10 +138,8 @@ struct KernelCE {
NvU32 (*__kceGetGrceSupportedLceMask__)(OBJGPU *, struct KernelCE *);
NvBool (*__kceIsGenXorHigherSupported__)(OBJGPU *, struct KernelCE *, NvU32);
void (*__kceApplyGen4orHigherMapping__)(OBJGPU *, struct KernelCE *, NvU32 *, NvU32 *, NvU32, NvU32);
NV_STATUS (*__kceStateInitLocked__)(POBJGPU, struct KernelCE *);
NV_STATUS (*__kceStatePreLoad__)(POBJGPU, struct KernelCE *, NvU32);
NV_STATUS (*__kceStatePostUnload__)(POBJGPU, struct KernelCE *, NvU32);
void (*__kceStateDestroy__)(POBJGPU, struct KernelCE *);
NV_STATUS (*__kceStatePreUnload__)(POBJGPU, struct KernelCE *, NvU32);
NV_STATUS (*__kceStateInitUnlocked__)(POBJGPU, struct KernelCE *);
void (*__kceInitMissing__)(POBJGPU, struct KernelCE *);
@ -156,6 +156,7 @@ struct KernelCE {
NvBool bIsAutoConfigEnabled;
NvBool bUseGen4Mapping;
struct IoAperture aperture;
NvBool bCcFipsSelfTestRequired;
};
#ifndef __NVOC_CLASS_KernelCE_TYPEDEF__
@ -191,10 +192,12 @@ NV_STATUS __nvoc_objCreate_KernelCE(KernelCE**, Dynamic*, NvU32);
#define kceConstructEngine(pGpu, pKCe, arg0) kceConstructEngine_DISPATCH(pGpu, pKCe, arg0)
#define kceIsPresent(pGpu, pKCe) kceIsPresent_DISPATCH(pGpu, pKCe)
#define kceIsPresent_HAL(pGpu, pKCe) kceIsPresent_DISPATCH(pGpu, pKCe)
#define kceStateLoad(arg0, arg1, arg2) kceStateLoad_DISPATCH(arg0, arg1, arg2)
#define kceStateLoad_HAL(arg0, arg1, arg2) kceStateLoad_DISPATCH(arg0, arg1, arg2)
#define kceStateInitLocked(arg0, arg1) kceStateInitLocked_DISPATCH(arg0, arg1)
#define kceStateUnload(pGpu, pKCe, flags) kceStateUnload_DISPATCH(pGpu, pKCe, flags)
#define kceStateUnload_HAL(pGpu, pKCe, flags) kceStateUnload_DISPATCH(pGpu, pKCe, flags)
#define kceStateLoad(arg0, arg1, arg2) kceStateLoad_DISPATCH(arg0, arg1, arg2)
#define kceStateLoad_HAL(arg0, arg1, arg2) kceStateLoad_DISPATCH(arg0, arg1, arg2)
#define kceStateDestroy(arg0, arg1) kceStateDestroy_DISPATCH(arg0, arg1)
#define kceRegisterIntrService(arg0, arg1, arg2) kceRegisterIntrService_DISPATCH(arg0, arg1, arg2)
#define kceServiceNotificationInterrupt(arg0, arg1, arg2) kceServiceNotificationInterrupt_DISPATCH(arg0, arg1, arg2)
#define kceGetP2PCes(arg0, pGpu, gpuMask, nvlinkP2PCeMask) kceGetP2PCes_DISPATCH(arg0, pGpu, gpuMask, nvlinkP2PCeMask)
@ -231,10 +234,8 @@ NV_STATUS __nvoc_objCreate_KernelCE(KernelCE**, Dynamic*, NvU32);
#define kceIsGenXorHigherSupported_HAL(pGpu, pCe, checkGen) kceIsGenXorHigherSupported_DISPATCH(pGpu, pCe, checkGen)
#define kceApplyGen4orHigherMapping(pGpu, pCe, arg0, arg1, arg2, arg3) kceApplyGen4orHigherMapping_DISPATCH(pGpu, pCe, arg0, arg1, arg2, arg3)
#define kceApplyGen4orHigherMapping_HAL(pGpu, pCe, arg0, arg1, arg2, arg3) kceApplyGen4orHigherMapping_DISPATCH(pGpu, pCe, arg0, arg1, arg2, arg3)
#define kceStateInitLocked(pGpu, pEngstate) kceStateInitLocked_DISPATCH(pGpu, pEngstate)
#define kceStatePreLoad(pGpu, pEngstate, arg0) kceStatePreLoad_DISPATCH(pGpu, pEngstate, arg0)
#define kceStatePostUnload(pGpu, pEngstate, arg0) kceStatePostUnload_DISPATCH(pGpu, pEngstate, arg0)
#define kceStateDestroy(pGpu, pEngstate) kceStateDestroy_DISPATCH(pGpu, pEngstate)
#define kceStatePreUnload(pGpu, pEngstate, arg0) kceStatePreUnload_DISPATCH(pGpu, pEngstate, arg0)
#define kceStateInitUnlocked(pGpu, pEngstate) kceStateInitUnlocked_DISPATCH(pGpu, pEngstate)
#define kceInitMissing(pGpu, pEngstate) kceInitMissing_DISPATCH(pGpu, pEngstate)
@ -366,10 +367,10 @@ static inline NvBool kceIsPresent_DISPATCH(OBJGPU *pGpu, struct KernelCE *pKCe)
return pKCe->__kceIsPresent__(pGpu, pKCe);
}
NV_STATUS kceStateLoad_GP100(OBJGPU *arg0, struct KernelCE *arg1, NvU32 arg2);
NV_STATUS kceStateInitLocked_IMPL(OBJGPU *arg0, struct KernelCE *arg1);
static inline NV_STATUS kceStateLoad_DISPATCH(OBJGPU *arg0, struct KernelCE *arg1, NvU32 arg2) {
return arg1->__kceStateLoad__(arg0, arg1, arg2);
static inline NV_STATUS kceStateInitLocked_DISPATCH(OBJGPU *arg0, struct KernelCE *arg1) {
return arg1->__kceStateInitLocked__(arg0, arg1);
}
static inline NV_STATUS kceStateUnload_56cd7a(OBJGPU *pGpu, struct KernelCE *pKCe, NvU32 flags) {
@ -380,6 +381,18 @@ static inline NV_STATUS kceStateUnload_DISPATCH(OBJGPU *pGpu, struct KernelCE *p
return pKCe->__kceStateUnload__(pGpu, pKCe, flags);
}
NV_STATUS kceStateLoad_GP100(OBJGPU *arg0, struct KernelCE *arg1, NvU32 arg2);
static inline NV_STATUS kceStateLoad_DISPATCH(OBJGPU *arg0, struct KernelCE *arg1, NvU32 arg2) {
return arg1->__kceStateLoad__(arg0, arg1, arg2);
}
void kceStateDestroy_IMPL(OBJGPU *arg0, struct KernelCE *arg1);
static inline void kceStateDestroy_DISPATCH(OBJGPU *arg0, struct KernelCE *arg1) {
arg1->__kceStateDestroy__(arg0, arg1);
}
void kceRegisterIntrService_IMPL(OBJGPU *arg0, struct KernelCE *arg1, IntrServiceRecord arg2[171]);
static inline void kceRegisterIntrService_DISPATCH(OBJGPU *arg0, struct KernelCE *arg1, IntrServiceRecord arg2[171]) {
@ -574,10 +587,6 @@ static inline void kceApplyGen4orHigherMapping_DISPATCH(OBJGPU *pGpu, struct Ker
pCe->__kceApplyGen4orHigherMapping__(pGpu, pCe, arg0, arg1, arg2, arg3);
}
static inline NV_STATUS kceStateInitLocked_DISPATCH(POBJGPU pGpu, struct KernelCE *pEngstate) {
return pEngstate->__kceStateInitLocked__(pGpu, pEngstate);
}
static inline NV_STATUS kceStatePreLoad_DISPATCH(POBJGPU pGpu, struct KernelCE *pEngstate, NvU32 arg0) {
return pEngstate->__kceStatePreLoad__(pGpu, pEngstate, arg0);
}
@ -586,10 +595,6 @@ static inline NV_STATUS kceStatePostUnload_DISPATCH(POBJGPU pGpu, struct KernelC
return pEngstate->__kceStatePostUnload__(pGpu, pEngstate, arg0);
}
static inline void kceStateDestroy_DISPATCH(POBJGPU pGpu, struct KernelCE *pEngstate) {
pEngstate->__kceStateDestroy__(pGpu, pEngstate);
}
static inline NV_STATUS kceStatePreUnload_DISPATCH(POBJGPU pGpu, struct KernelCE *pEngstate, NvU32 arg0) {
return pEngstate->__kceStatePreUnload__(pGpu, pEngstate, arg0);
}

2
src/nvidia/generated/g_kernel_sec2_nvoc.h
View File

@ -7,7 +7,7 @@ extern "C" {
#endif
/*
* SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a

3
src/nvidia/generated/g_mem_mgr_nvoc.h
View File

@ -337,6 +337,9 @@ typedef struct OBJCHANNEL
// Used by Partition Scrubber
KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance;
NvHandle hPartitionRef;
NvBool bSecure;
} OBJCHANNEL, *POBJCHANNEL;
#define NV_METHOD(SubCh, Method, Num) \

593
src/nvidia/generated/g_subdevice_nvoc.c
View File

File diff suppressed because it is too large Load Diff

8
src/nvidia/generated/g_subdevice_nvoc.h
View File

@ -641,6 +641,7 @@ struct Subdevice {
NV_STATUS (*__subdeviceCtrlCmdInternalConfComputeRotateKeys__)(struct Subdevice *, NV2080_CTRL_INTERNAL_CONF_COMPUTE_ROTATE_KEYS_PARAMS *);
NV_STATUS (*__subdeviceCtrlCmdInternalConfComputeRCChannelsForKeyRotation__)(struct Subdevice *, NV2080_CTRL_INTERNAL_CONF_COMPUTE_RC_CHANNELS_FOR_KEY_ROTATION_PARAMS *);
NV_STATUS (*__subdeviceCtrlCmdInternalConfComputeSetGpuState__)(struct Subdevice *, NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_GPU_STATE_PARAMS *);
NV_STATUS (*__subdeviceCtrlCmdInternalConfComputeSetSecurityPolicy__)(struct Subdevice *, NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS *);
NV_STATUS (*__subdeviceCtrlCmdInternalInitUserSharedData__)(struct Subdevice *, NV2080_CTRL_INTERNAL_INIT_USER_SHARED_DATA_PARAMS *);
NV_STATUS (*__subdeviceCtrlCmdInternalUserSharedDataSetDataPoll__)(struct Subdevice *, NV2080_CTRL_INTERNAL_USER_SHARED_DATA_SET_DATA_POLL_PARAMS *);
NV_STATUS (*__subdeviceCtrlCmdInternalGspStartTrace__)(struct Subdevice *, NV2080_CTRL_CMD_INTERNAL_GSP_START_TRACE_INFO_PARAMS *);
@ -1296,6 +1297,7 @@ NV_STATUS __nvoc_objCreate_Subdevice(Subdevice**, Dynamic*, NvU32, struct CALL_C
#define subdeviceCtrlCmdInternalConfComputeRotateKeys(pSubdevice, pParams) subdeviceCtrlCmdInternalConfComputeRotateKeys_DISPATCH(pSubdevice, pParams)
#define subdeviceCtrlCmdInternalConfComputeRCChannelsForKeyRotation(pSubdevice, pParams) subdeviceCtrlCmdInternalConfComputeRCChannelsForKeyRotation_DISPATCH(pSubdevice, pParams)
#define subdeviceCtrlCmdInternalConfComputeSetGpuState(pSubdevice, pParams) subdeviceCtrlCmdInternalConfComputeSetGpuState_DISPATCH(pSubdevice, pParams)
#define subdeviceCtrlCmdInternalConfComputeSetSecurityPolicy(pSubdevice, pParams) subdeviceCtrlCmdInternalConfComputeSetSecurityPolicy_DISPATCH(pSubdevice, pParams)
#define subdeviceCtrlCmdInternalInitUserSharedData(pSubdevice, pParams) subdeviceCtrlCmdInternalInitUserSharedData_DISPATCH(pSubdevice, pParams)
#define subdeviceCtrlCmdInternalUserSharedDataSetDataPoll(pSubdevice, pParams) subdeviceCtrlCmdInternalUserSharedDataSetDataPoll_DISPATCH(pSubdevice, pParams)
#define subdeviceCtrlCmdInternalGspStartTrace(pSubdevice, pParams) subdeviceCtrlCmdInternalGspStartTrace_DISPATCH(pSubdevice, pParams)
@ -4587,6 +4589,12 @@ static inline NV_STATUS subdeviceCtrlCmdInternalConfComputeSetGpuState_DISPATCH(
return pSubdevice->__subdeviceCtrlCmdInternalConfComputeSetGpuState__(pSubdevice, pParams);
}
NV_STATUS subdeviceCtrlCmdInternalConfComputeSetSecurityPolicy_IMPL(struct Subdevice *pSubdevice, NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS *pParams);
static inline NV_STATUS subdeviceCtrlCmdInternalConfComputeSetSecurityPolicy_DISPATCH(struct Subdevice *pSubdevice, NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS *pParams) {
return pSubdevice->__subdeviceCtrlCmdInternalConfComputeSetSecurityPolicy__(pSubdevice, pParams);
}
NV_STATUS subdeviceCtrlCmdInternalInitUserSharedData_IMPL(struct Subdevice *pSubdevice, NV2080_CTRL_INTERNAL_INIT_USER_SHARED_DATA_PARAMS *pParams);
static inline NV_STATUS subdeviceCtrlCmdInternalInitUserSharedData_DISPATCH(struct Subdevice *pSubdevice, NV2080_CTRL_INTERNAL_INIT_USER_SHARED_DATA_PARAMS *pParams) {

4
src/nvidia/inc/kernel/gpu/mem_mgr/ce_utils_sizes.h
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -33,7 +33,7 @@
#define CE_MAX_BYTES_PER_LINE 0xffffffffULL
#define CE_METHOD_SIZE_PER_BLOCK 0x64
#define FAST_SCRUBBER_METHOD_SIZE_PER_BLOCK 0x78
#define FAST_SCRUBBER_METHOD_SIZE_PER_BLOCK 0x94
// number of bytes per sec2 method-stream (including host methods)
#define SEC2_METHOD_SIZE_PER_BLOCK 0x94

9
src/nvidia/inc/kernel/gpu/mem_mgr/channel_utils.h
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -130,9 +130,16 @@ typedef struct
NV_ADDRESS_SPACE srcAddressSpace;
NvU32 dstCpuCacheAttrib;
NvU32 srcCpuCacheAttrib;
NvBool bSecureCopy; // The copy encrypts/decrypts protected memory
NvBool bEncrypt; // encrypt/decrypt
NvU64 authTagAddr;
NvU64 encryptIvAddr;
} CHANNEL_PB_INFO;
NV_STATUS channelSetupIDs(OBJCHANNEL *pChannel, OBJGPU *pGpu, NvBool bUseVasForCeCopy, NvBool bMIGInUse);
NV_STATUS channelAllocSubdevice(OBJGPU *pGpu, OBJCHANNEL *pChannel);
void channelSetupChannelBufferSizes(OBJCHANNEL *pChannel);
NvU32 channelReadChannelMemdesc(OBJCHANNEL *pChannel, NvU32 offset);

194
src/nvidia/src/kernel/gpu/ce/kernel_ce.c
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -25,6 +25,7 @@
#include "gpu/ce/kernel_ce.h"
#include "gpu/ce/kernel_ce_private.h"
#include "gpu/eng_desc.h"
#include "gpu/mem_mgr/ce_utils.h"
#include "gpu_mgr/gpu_mgr.h"
#include "kernel/gpu/intr/intr_service.h"
#include "kernel/gpu/nvlink/kernel_nvlink.h"
@ -32,6 +33,8 @@
#include "vgpu/sdk-structures.h"
#include "nvRmReg.h"
#include "gpu/conf_compute/ccsl.h"
NV_STATUS kceConstructEngine_IMPL(OBJGPU *pGpu, KernelCE *pKCe, ENGDESCRIPTOR engDesc)
{
NV_ASSERT_OR_RETURN(!RMCFG_FEATURE_PLATFORM_GSP, NV_ERR_NOT_SUPPORTED);
@ -90,6 +93,195 @@ NvBool kceIsNewMissingEngineRemovalSequenceEnabled_IMPL(OBJGPU *pGpu, KernelCE *
return NV_TRUE;
}
#define CE_FIPS_SELF_TEST_DATA_SIZE 16
#define CE_FIPS_SELF_TEST_AUTH_TAG_SIZE 16
#define CE_FIPS_SELF_TEST_IV_SIZE 12
NV_STATUS
kceRunFipsSelfTest
(
OBJGPU *pGpu,
void *pArg
)
{
KernelCE *pKCe = pArg;
MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
MEMORY_DESCRIPTOR *pSrcMemDesc = NULL;
MEMORY_DESCRIPTOR *pDstMemDesc = NULL;
MEMORY_DESCRIPTOR *pAuthMemDesc = NULL;
MEMORY_DESCRIPTOR *pIvMemDesc = NULL;
CeUtils *pCeUtils = NULL;
pCcslContext pCcslCtx = NULL;
NV_STATUS status;
NV0050_ALLOCATION_PARAMETERS ceUtilsParams = {0};
CEUTILS_MEMCOPY_PARAMS params = {0};
NvU8 ceTestPlaintext[CE_FIPS_SELF_TEST_DATA_SIZE] = {
0x2d, 0x71, 0xbc, 0xfa, 0x91, 0x4e, 0x4a, 0xc0,
0x45, 0xb2, 0xaa, 0x60, 0x95, 0x5f, 0xad, 0x24
};
NvU8 decryptedData[CE_FIPS_SELF_TEST_DATA_SIZE] = { 0 };
NvU8 encryptedData[CE_FIPS_SELF_TEST_DATA_SIZE] = { 0 };
NvU8 dataAuth[CE_FIPS_SELF_TEST_AUTH_TAG_SIZE] = { 0 };
NV_ASSERT_OR_RETURN(gpuIsCCFeatureEnabled(pGpu), NV_ERR_NOT_SUPPORTED);
if (!gpuCheckEngineTable(pGpu, RM_ENGINE_TYPE_COPY(pKCe->publicID)) ||
ceIsCeGrce(pGpu, RM_ENGINE_TYPE_COPY(pKCe->publicID)))
{
// CE doesn't support encryption
return NV_OK;
}
if (kmigmgrIsMIGSupported(pGpu, pKernelMIGManager) &&
kmigmgrGetStaticInfo(pGpu, pKernelMIGManager) == NULL)
{
// Wait for KernelMigManager, as it might remap CEs
return NV_WARN_MORE_PROCESSING_REQUIRED;
}
NV_PRINTF(LEVEL_INFO, "Running FIPS test for CE%u\n", pKCe->publicID);
ceUtilsParams.flags |= DRF_DEF(0050_CEUTILS, _FLAGS, _FORCE_CE_ID, _TRUE);
ceUtilsParams.flags |= DRF_DEF(0050_CEUTILS, _FLAGS, _CC_SECURE, _TRUE);
ceUtilsParams.forceCeId = pKCe->publicID;
NV_ASSERT_OK_OR_GOTO(status,
objCreate(&pCeUtils, pMemoryManager, CeUtils, ENG_GET_GPU(pMemoryManager), NULL, &ceUtilsParams), failed);
NV_ASSERT_OK_OR_GOTO(status,
ccslContextInitViaChannel_HAL(&pCcslCtx, pCeUtils->pChannel->hClient,
pCeUtils->pChannel->channelId),
failed);
NV_ASSERT_OK_OR_GOTO(status, memdescCreate(&pSrcMemDesc, pGpu, sizeof ceTestPlaintext, 0, NV_TRUE, ADDR_FBMEM,
NV_MEMORY_UNCACHED, MEMDESC_ALLOC_FLAGS_PROTECTED), failed);
NV_ASSERT_OK_OR_GOTO(status, memdescAlloc(pSrcMemDesc), failed);
NV_ASSERT_OK_OR_GOTO(status, memdescCreate(&pDstMemDesc, pGpu, sizeof encryptedData, 0, NV_TRUE, ADDR_SYSMEM,
NV_MEMORY_UNCACHED, MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY), failed);
NV_ASSERT_OK_OR_GOTO(status, memdescAlloc(pDstMemDesc), failed);
NV_ASSERT_OK_OR_GOTO(status, memdescCreate(&pAuthMemDesc, pGpu, sizeof dataAuth, 0, NV_TRUE, ADDR_SYSMEM,
NV_MEMORY_UNCACHED, MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY), failed);
NV_ASSERT_OK_OR_GOTO(status, memdescAlloc(pAuthMemDesc), failed);
NV_ASSERT_OK_OR_GOTO(status, memdescCreate(&pIvMemDesc, pGpu, CE_FIPS_SELF_TEST_IV_SIZE, 0, NV_TRUE, ADDR_SYSMEM,
NV_MEMORY_UNCACHED, MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY), failed);
NV_ASSERT_OK_OR_GOTO(status, memdescAlloc(pIvMemDesc), failed);
TRANSFER_SURFACE srcSurface = { .pMemDesc = pSrcMemDesc, .offset = 0 };
TRANSFER_SURFACE dstSurface = { .pMemDesc = pDstMemDesc, .offset = 0 };
TRANSFER_SURFACE authSurface = { .pMemDesc = pAuthMemDesc, .offset = 0 };
// Write data to allocations, encrypt using CE, and read back the results
NV_ASSERT_OK_OR_GOTO(status, memmgrMemDescMemSet(pMemoryManager, pDstMemDesc, 0, 0), failed);
NV_ASSERT_OK_OR_GOTO(status, memmgrMemDescMemSet(pMemoryManager, pAuthMemDesc, 0, 0), failed);
NV_ASSERT_OK_OR_GOTO(status,
memmgrMemWrite(pMemoryManager, &srcSurface, ceTestPlaintext, sizeof ceTestPlaintext, TRANSFER_FLAGS_NONE),
failed);
params.bSecureCopy = NV_TRUE;
params.authTagAddr = memdescGetPhysAddr(pAuthMemDesc, AT_GPU, 0);
params.encryptIvAddr = memdescGetPhysAddr(pIvMemDesc, AT_GPU, 0);
params.pDstMemDesc = pDstMemDesc;
params.dstOffset = 0;
params.pSrcMemDesc = pSrcMemDesc;
params.srcOffset = 0;
params.length = sizeof ceTestPlaintext;
params.bEncrypt = NV_TRUE;
NV_ASSERT_OK_OR_GOTO(status, ceutilsMemcopy(pCeUtils, &params), failed);
NV_ASSERT_OK_OR_GOTO(status,
memmgrMemRead(pMemoryManager, &dstSurface, encryptedData, sizeof encryptedData, TRANSFER_FLAGS_NONE), failed);
NV_ASSERT_OK_OR_GOTO(status,
memmgrMemRead(pMemoryManager, &authSurface, dataAuth, sizeof dataAuth, TRANSFER_FLAGS_NONE), failed);
// Decrypt using CPU and validate
NV_ASSERT_OK_OR_GOTO(status,
ccslDecrypt_HAL(pCcslCtx, sizeof decryptedData, encryptedData, NULL, NULL, 0, decryptedData, dataAuth),
failed);
NV_ASSERT_TRUE_OR_GOTO(status, portMemCmp(decryptedData, ceTestPlaintext, sizeof ceTestPlaintext) == 0,
NV_ERR_INVALID_STATE, failed);
// Encrypt using CPU
NV_ASSERT_OK_OR_GOTO(status,
ccslEncrypt_HAL(pCcslCtx, sizeof ceTestPlaintext, ceTestPlaintext, NULL, 0, encryptedData, dataAuth), failed);
// Write data to allocations, decrypt using CE, read back, and validate
NV_ASSERT_OK_OR_GOTO(status,
memmgrMemWrite(pMemoryManager, &dstSurface, encryptedData, sizeof encryptedData, TRANSFER_FLAGS_NONE), failed);
NV_ASSERT_OK_OR_GOTO(status,
memmgrMemWrite(pMemoryManager, &authSurface, dataAuth, sizeof dataAuth, TRANSFER_FLAGS_NONE), failed);
NV_ASSERT_OK_OR_GOTO(status, memmgrMemDescMemSet(pMemoryManager, pSrcMemDesc, 0, 0), failed);
params.pDstMemDesc = pSrcMemDesc;
params.dstOffset = 0;
params.pSrcMemDesc = pDstMemDesc;
params.srcOffset = 0;
params.length = sizeof ceTestPlaintext;
params.bEncrypt = NV_FALSE;
NV_ASSERT_OK_OR_GOTO(status, ceutilsMemcopy(pCeUtils, &params), failed);
NV_ASSERT_OK_OR_GOTO(status,
memmgrMemRead(pMemoryManager, &srcSurface, decryptedData, sizeof decryptedData, TRANSFER_FLAGS_NONE), failed);
NV_ASSERT_TRUE_OR_GOTO(status, portMemCmp(decryptedData, ceTestPlaintext, sizeof ceTestPlaintext) == 0,
NV_ERR_INVALID_STATE, failed);
failed:
ccslContextClear(pCcslCtx);
objDelete(pCeUtils);
memdescFree(pSrcMemDesc);
memdescDestroy(pSrcMemDesc);
memdescFree(pDstMemDesc);
memdescDestroy(pDstMemDesc);
memdescFree(pAuthMemDesc);
memdescDestroy(pAuthMemDesc);
memdescFree(pIvMemDesc);
memdescDestroy(pIvMemDesc);
NV_PRINTF(LEVEL_INFO, "Test finished with status 0x%x\n", status);
return status;
}
NV_STATUS
kceStateInitLocked_IMPL
(
OBJGPU *pGpu,
KernelCE *pKCe
)
{
if (!gpuIsCCFeatureEnabled(pGpu) || !IS_SILICON(pGpu))
{
pKCe->bCcFipsSelfTestRequired = NV_FALSE;
}
if (pKCe->bCcFipsSelfTestRequired)
{
NV_ASSERT_OK_OR_RETURN(
kfifoAddSchedulingHandler(pGpu, GPU_GET_KERNEL_FIFO(pGpu), kceRunFipsSelfTest, pKCe, NULL, NULL));
}
return NV_OK;
}
void
kceStateDestroy_IMPL
(
OBJGPU *pGpu,
KernelCE *pKCe
)
{
if (pKCe->bCcFipsSelfTestRequired)
{
kfifoRemoveSchedulingHandler(pGpu, GPU_GET_KERNEL_FIFO(pGpu), kceRunFipsSelfTest, pKCe, NULL, NULL);
}
}
static void printCaps(OBJGPU *pGpu, KernelCE *pKCe, RM_ENGINE_TYPE rmEngineType, const NvU8 *capsTbl)
{
NV_PRINTF(LEVEL_INFO, "LCE%d caps (engineType = %d (%d))\n", pKCe->publicID,

48
src/nvidia/src/kernel/gpu/conf_compute/arch/hopper/conf_compute_key_rotation_gh100.c
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -36,10 +36,10 @@ static void getKeyPairForKeySpace(NvU32 keySpace, NvBool bKernel, NvU32 *pGlobal
static NV_STATUS triggerKeyRotationByKeyPair(OBJGPU *pGpu, ConfidentialCompute *pConfCompute, NvU32 h2dKey, NvU32 d2hKey);
static NV_STATUS calculateEncryptionStatsByKeyPair(OBJGPU *pGpu, ConfidentialCompute *pConfCompute, NvU32 h2dKey, NvU32 d2hKey);
static NV_STATUS notifyKeyRotationByKeyPair(OBJGPU *pGpu, ConfidentialCompute *pConfCompute, NvU32 h2dKey);
static NvBool confComputeIsLowerThresholdCrossed(ConfidentialCompute *pConfCompute, KEY_ROTATION_STATS_INFO *pH2DInfo,
KEY_ROTATION_STATS_INFO *pD2HInfo);
static NvBool confComputeIsUpperThresholdCrossed(ConfidentialCompute *pConfCompute, KEY_ROTATION_STATS_INFO *pH2DInfo,
KEY_ROTATION_STATS_INFO *pD2HInfo);
static NvBool isLowerThresholdCrossed(ConfidentialCompute *pConfCompute, KEY_ROTATION_STATS_INFO *pH2DInfo,
KEY_ROTATION_STATS_INFO *pD2HInfo);
static NvBool isUpperThresholdCrossed(ConfidentialCompute *pConfCompute, KEY_ROTATION_STATS_INFO *pH2DInfo,
KEY_ROTATION_STATS_INFO *pD2HInfo);
static NV_STATUS keyRotationTimeoutCallback(OBJGPU *pGpu, OBJTMR *pTmr, TMR_EVENT *pTmrEvent);
/*!
@ -227,15 +227,15 @@ triggerKeyRotationByKeyPair
NV_ASSERT_OK_OR_RETURN(confComputeGetKeySlotFromGlobalKeyId(pConfCompute, h2dKey, &h2dIndex));
NV_ASSERT_OK_OR_RETURN(confComputeGetKeySlotFromGlobalKeyId(pConfCompute, d2hKey, &d2hIndex));
if (confComputeIsUpperThresholdCrossed(pConfCompute, &pConfCompute->aggregateStats[h2dIndex],
&pConfCompute->aggregateStats[d2hIndex]))
if (isUpperThresholdCrossed(pConfCompute, &pConfCompute->aggregateStats[h2dIndex],
&pConfCompute->aggregateStats[d2hIndex]))
{
NV_PRINTF(LEVEL_ERROR, "Crossed UPPER threshold for key = 0x%x\n", h2dKey);
NV_ASSERT_OK_OR_RETURN(confComputeSetKeyRotationStatus(pConfCompute, h2dKey, KEY_ROTATION_STATUS_FAILED_THRESHOLD));
NV_ASSERT_OK_OR_RETURN(confComputeScheduleKeyRotationWorkItem(pGpu, pConfCompute, h2dKey, d2hKey));
}
else if (confComputeIsLowerThresholdCrossed(pConfCompute, &pConfCompute->aggregateStats[h2dIndex],
&pConfCompute->aggregateStats[d2hIndex]))
else if (isLowerThresholdCrossed(pConfCompute, &pConfCompute->aggregateStats[h2dIndex],
&pConfCompute->aggregateStats[d2hIndex]))
{
NV_PRINTF(LEVEL_INFO, "Crossed LOWER threshold for key = 0x%x\n", h2dKey);
if (state == KEY_ROTATION_STATUS_IDLE)
@ -363,7 +363,7 @@ calculateEncryptionStatsByKeyPair
}
static NvBool
confComputeIsUpperThresholdCrossed
isUpperThresholdCrossed
(
ConfidentialCompute *pConfCompute,
KEY_ROTATION_STATS_INFO *pH2DInfo,
@ -376,22 +376,13 @@ confComputeIsUpperThresholdCrossed
}
else
{
if ((pH2DInfo->totalBytesEncrypted > pConfCompute->upperThreshold.totalBytesEncrypted) ||
(pH2DInfo->totalEncryptOps > pConfCompute->upperThreshold.totalEncryptOps))
{
return NV_TRUE;
}
else if ((pD2HInfo->totalBytesEncrypted > pConfCompute->upperThreshold.totalBytesEncrypted) ||
(pD2HInfo->totalEncryptOps > pConfCompute->upperThreshold.totalEncryptOps))
{
return NV_TRUE;
}
return (confComputeIsUpperThresholdCrossed(pConfCompute, pH2DInfo) ||
confComputeIsUpperThresholdCrossed(pConfCompute, pD2HInfo));
}
return NV_FALSE;
}
static NvBool
confComputeIsLowerThresholdCrossed
isLowerThresholdCrossed
(
ConfidentialCompute *pConfCompute,
KEY_ROTATION_STATS_INFO *pH2DInfo,
@ -404,18 +395,9 @@ confComputeIsLowerThresholdCrossed
}
else
{
if ((pH2DInfo->totalBytesEncrypted > pConfCompute->lowerThreshold.totalBytesEncrypted) ||
(pH2DInfo->totalEncryptOps > pConfCompute->lowerThreshold.totalEncryptOps))
{
return NV_TRUE;
}
else if ((pD2HInfo->totalBytesEncrypted > pConfCompute->lowerThreshold.totalBytesEncrypted) ||
(pD2HInfo->totalEncryptOps > pConfCompute->lowerThreshold.totalEncryptOps))
{
return NV_TRUE;
}
return (confComputeIsLowerThresholdCrossed(pConfCompute, pH2DInfo) ||
confComputeIsLowerThresholdCrossed(pConfCompute, pD2HInfo));
}
return NV_FALSE;
}
static void

14
src/nvidia/src/kernel/gpu/conf_compute/conf_compute.c
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -50,6 +50,8 @@
static NV_STATUS _confComputeInitRegistryOverrides(OBJGPU *, ConfidentialCompute*);
static NvU32 _confComputeGetKeyspaceSize(NvU16 keyspace);
#define KEY_ROTATION_THRESHOLD_DELTA 20000000ull
NV_STATUS
confComputeConstructEngine_IMPL(OBJGPU *pGpu,
ConfidentialCompute *pConfCompute,
@ -154,6 +156,11 @@ confComputeConstructEngine_IMPL(OBJGPU *pGpu,
}
}
// init key rotation state
pConfCompute->attackerAdvantage = SECURITY_POLICY_ATTACKER_ADVANTAGE_DEFAULT;
pConfCompute->keyRotationLimitDelta = KEY_ROTATION_THRESHOLD_DELTA;
NV_ASSERT_OK_OR_RETURN(confComputeSetKeyRotationThreshold(pConfCompute,
pConfCompute->attackerAdvantage));
for (NvU32 i = 0; i < CC_KEYSPACE_TOTAL_SIZE; i++)
{
pConfCompute->keyRotationState[i] = KEY_ROTATION_STATUS_IDLE;
@ -165,6 +172,7 @@ confComputeConstructEngine_IMPL(OBJGPU *pGpu,
pConfCompute->keyRotationChannelRefCount = 0;
pConfCompute->keyRotationEnableMask = 0;
NV_ASSERT_OK_OR_RETURN(confComputeEnableKeyRotationSupport_HAL(pGpu, pConfCompute));
return NV_OK;
}
@ -427,8 +435,8 @@ confComputeStatePostLoad_IMPL
NvU32 flags
)
{
NV_STATUS status = NV_OK;
RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
NV_STATUS status = NV_OK;
RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
pGpu->hInternalClient,

57
src/nvidia/src/kernel/gpu/conf_compute/conf_compute_api.c
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -384,7 +384,14 @@ confComputeApiCtrlCmdSystemGetSecurityPolicy_IMPL
NV_CONF_COMPUTE_CTRL_GET_SECURITY_POLICY_PARAMS *pParams
)
{
return NV_ERR_NOT_SUPPORTED;
OBJSYS *pSys = SYS_GET_INSTANCE();
OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
pParams->attackerAdvantage = pGpuMgr->ccAttackerAdvantage;
return NV_OK;
}
NV_STATUS
@ -394,6 +401,48 @@ confComputeApiCtrlCmdSystemSetSecurityPolicy_IMPL
NV_CONF_COMPUTE_CTRL_SET_SECURITY_POLICY_PARAMS *pParams
)
{
return NV_ERR_NOT_SUPPORTED;
}
OBJSYS *pSys = SYS_GET_INSTANCE();
OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
OBJGPU *pGpu;
NvU32 gpuMask;
NvU32 gpuInstance = 0;
RM_API *pRmApi = NULL;
NV_STATUS status = NV_OK;
NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS params = {0};
LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
// CC security policy can only be set before GpuReadyState is set.
NV_ASSERT_OR_RETURN(pConfComputeApi->pCcCaps->bAcceptClientRequest == NV_FALSE, NV_ERR_INVALID_STATE);
if ((pParams->attackerAdvantage < SET_SECURITY_POLICY_ATTACKER_ADVANTAGE_MIN) ||
(pParams->attackerAdvantage > SET_SECURITY_POLICY_ATTACKER_ADVANTAGE_MAX))
{
return NV_ERR_INVALID_ARGUMENT;
}
params.attackerAdvantage = pParams->attackerAdvantage;
(void)gpumgrGetGpuAttachInfo(NULL, &gpuMask);
while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
{
pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
ConfidentialCompute* pConfCompute = GPU_GET_CONF_COMPUTE(pGpu);
status = pRmApi->Control(pRmApi,
pGpu->hInternalClient,
pGpu->hInternalSubdevice,
NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY,
&params,
sizeof(params));
if (status != NV_OK)
return status;
NV_ASSERT_OK_OR_RETURN(confComputeSetKeyRotationThreshold(pConfCompute,
pParams->attackerAdvantage));
}
pGpuMgr->ccAttackerAdvantage = pParams->attackerAdvantage;
return status;
}

74
src/nvidia/src/kernel/gpu/conf_compute/conf_compute_key_rotation.c
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -430,3 +430,75 @@ confComputeUpdateFreedChannelStats_IMPL
pConfCompute->freedChannelAggregateStats[d2hIndex].totalEncryptOps += pEncStats->numEncryptionsD2H;
return NV_OK;
}
NV_STATUS
confComputeSetKeyRotationThreshold_IMPL(ConfidentialCompute *pConfCompute,
NvU64 attackerAdvantage)
{
//
// Limit beyond which an encryption key cannot be used.
// The index is the attacker advantage as described in
// https://datatracker.ietf.org/doc/draft-irtf-cfrg-aead-limits/
// The limit is expressed in units of total amount of data encrypted
// (in units of 16 B) plus the number of encryption invocations.
//
const NvU32 offset = 50;
static const NvU64 keyRotationUpperThreshold[] = {
777472127993ull,
549755813887ull,
388736063996ull,
274877906943ull,
194368031997ull,
137438953471ull,
97184015998ull,
68719476735ull,
48592007999ull,
34359738367ull,
24296003999ull,
17179869183ull,
12148001999ull,
8589934591ull,
6074000999ull,
4294967295ull,
3037000499ull,
2147483647ull,
1518500249ull,
1073741823ull,
759250124ull,
536870911ull,
379625061ull,
268435455ull,
189812530ull,
134217727ull};
NV_ASSERT_OR_RETURN((attackerAdvantage >= offset) &&
(attackerAdvantage <= (offset + NV_ARRAY_ELEMENTS(keyRotationUpperThreshold) - 1)),
NV_ERR_INVALID_ARGUMENT);
pConfCompute->keyRotationUpperLimit = keyRotationUpperThreshold[attackerAdvantage - offset];
pConfCompute->keyRotationLowerLimit = pConfCompute->keyRotationUpperLimit -
pConfCompute->keyRotationLimitDelta;
NV_PRINTF(LEVEL_INFO, "Setting key rotation attacker advantage to %llu.\n", attackerAdvantage);
NV_PRINTF(LEVEL_INFO, "Key rotation lower limit is %llu and upper limit is %llu.\n",
pConfCompute->keyRotationLowerLimit, pConfCompute->keyRotationUpperLimit);
return NV_OK;
}
NvBool confComputeIsUpperThresholdCrossed_IMPL(ConfidentialCompute *pConfCompute,
const KEY_ROTATION_STATS_INFO *pStatsInfo)
{
const NvU64 totalEncryptWork = (pStatsInfo->totalBytesEncrypted / 16) + pStatsInfo->totalEncryptOps;
return (totalEncryptWork > pConfCompute->keyRotationUpperLimit);
}
NvBool confComputeIsLowerThresholdCrossed_IMPL(ConfidentialCompute *pConfCompute,
const KEY_ROTATION_STATS_INFO *pStatsInfo)
{
const NvU64 totalEncryptWork = (pStatsInfo->totalBytesEncrypted / 16) + pStatsInfo->totalEncryptOps;
return (totalEncryptWork > pConfCompute->keyRotationLowerLimit);
}

166
src/nvidia/src/kernel/gpu/fifo/kernel_fifo.c
View File

@ -3021,57 +3021,64 @@ kfifoTriggerPostSchedulingEnableCallback_IMPL
{
NV_STATUS status = NV_OK;
FifoSchedulingHandlerEntry *pEntry;
NvBool bRetry = NV_FALSE;
NvBool bFirstPass = NV_TRUE;
NvBool bRetry;
for (pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList);
pEntry != NULL;
pEntry = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry))
do
{
NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL,
status = NV_ERR_INVALID_STATE; break;);
NvBool bMadeProgress = NV_FALSE;
pEntry->bHandled = NV_FALSE;
status = pEntry->pCallback(pGpu, pEntry->pCallbackParam);
bRetry = NV_FALSE;
// Retry mechanism: Some callbacks depend on other callbacks in this list.
bRetry = bRetry || (status == NV_WARN_MORE_PROCESSING_REQUIRED);
for (pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList);
pEntry != NULL;
pEntry = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry))
{
NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL,
status = NV_ERR_INVALID_STATE; break;);
if (status == NV_WARN_MORE_PROCESSING_REQUIRED)
// Quash retry status
status = NV_OK;
else if (status == NV_OK)
// Successfully handled, no need to retry
pEntry->bHandled = NV_TRUE;
else
// Actual error, abort
break;
}
if (bFirstPass)
{
// Reset bHandled set by previous call (fore example, for dor suspend-resume)
pEntry->bHandled = NV_FALSE;
}
else if (pEntry->bHandled)
{
continue;
}
// If we hit an actual error or completed everything successfully, return early.
if ((status != NV_OK) || !bRetry)
return status;
status = pEntry->pCallback(pGpu, pEntry->pCallbackParam);
// Second pass, retry anything that asked nicely to be deferred
for (pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList);
pEntry != NULL;
pEntry = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry))
{
NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL,
status = NV_ERR_INVALID_STATE; break;);
if (status == NV_WARN_MORE_PROCESSING_REQUIRED)
{
// Retry mechanism: Some callbacks depend on other callbacks in this list.
bRetry = NV_TRUE;
// Quash retry status
status = NV_OK;
}
else if (status == NV_OK)
{
// Successfully handled, no need to retry
pEntry->bHandled = NV_TRUE;
bMadeProgress = NV_TRUE;
}
else
{
// Actual error, abort
NV_ASSERT(0);
break;
}
}
// Skip anything that was completed successfully
if (pEntry->bHandled)
continue;
// We are stuck in a loop, and all remaining callbacks are returning NV_WARN_MORE_PROCESSING_REQUIRED
NV_ASSERT_OR_RETURN(bMadeProgress || status != NV_OK, NV_ERR_INVALID_STATE);
NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR,
pEntry->pCallback(pGpu, pEntry->pCallbackParam),
break; );
}
bFirstPass = NV_FALSE;
} while (bRetry && status == NV_OK);
return status;
}
/*!
* @brief Notify handlers that scheduling will soon be disabled.
*
@ -3089,53 +3096,60 @@ kfifoTriggerPreSchedulingDisableCallback_IMPL
{
NV_STATUS status = NV_OK;
FifoSchedulingHandlerEntry *pEntry;
NvBool bRetry = NV_FALSE;
NvBool bFirstPass = NV_TRUE;
NvBool bRetry;
// First pass
for (pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList);
pEntry != NULL;
pEntry = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry))
do
{
NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL,
status = NV_ERR_INVALID_STATE; break;);
NvBool bMadeProgress = NV_FALSE;
pEntry->bHandled = NV_FALSE;
status = pEntry->pCallback(pGpu, pEntry->pCallbackParam);
bRetry = NV_FALSE;
// Retry mechanism: Some callbacks depend on other callbacks in this list.
bRetry = bRetry || (status == NV_WARN_MORE_PROCESSING_REQUIRED);
for (pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList);
pEntry != NULL;
pEntry = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry))
{
NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL,
status = NV_ERR_INVALID_STATE; break;);
if (status == NV_WARN_MORE_PROCESSING_REQUIRED)
// Quash retry status
status = NV_OK;
else if (status == NV_OK)
// Successfully handled, no need to retry
pEntry->bHandled = NV_TRUE;
else
// Actual error, abort
break;
}
if (bFirstPass)
{
// Reset bHandled set by previous call (fore example, for dor suspend-resume)
pEntry->bHandled = NV_FALSE;
}
else if (pEntry->bHandled)
{
continue;
}
// If we hit an actual error or completed everything successfully, return early.
if ((status != NV_OK) || !bRetry)
return status;
status = pEntry->pCallback(pGpu, pEntry->pCallbackParam);
// Second pass, retry anything that asked nicely to be deferred
for (pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList);
pEntry != NULL;
pEntry = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry))
{
NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL,
status = NV_ERR_INVALID_STATE; break;);
if (status == NV_WARN_MORE_PROCESSING_REQUIRED)
{
// Retry mechanism: Some callbacks depend on other callbacks in this list.
bRetry = NV_TRUE;
// Quash retry status
status = NV_OK;
}
else if (status == NV_OK)
{
// Successfully handled, no need to retry
pEntry->bHandled = NV_TRUE;
bMadeProgress = NV_TRUE;
}
else
{
// Actual error, abort
NV_ASSERT(0);
break;
}
}
// Skip anything that was completed successfully
if (pEntry->bHandled)
continue;
// We are stuck in a loop, and all remaining callbacks are returning NV_WARN_MORE_PROCESSING_REQUIRED
NV_ASSERT_OR_RETURN(bMadeProgress || status != NV_OK, NV_ERR_INVALID_STATE);
NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR,
pEntry->pCallback(pGpu, pEntry->pCallbackParam),
break; );
}
bFirstPass = NV_FALSE;
} while (bRetry && status == NV_OK);
return status;
}

241
src/nvidia/src/kernel/gpu/mem_mgr/arch/maxwell/mem_utils_gm107.c
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2012-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2012-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -47,9 +47,7 @@
#include "class/cla06fsubch.h" // NVA06F_SUBCHANNEL_COPY_ENGINE
#include "class/cl003e.h" // NV01_MEMORY_SYSTEM
#include "class/cl0040.h" // NV01_MEMORY_LOCAL_USER
#include "class/cl0080.h" // NV01_DEVICE_0
#include "class/cl50a0.h" // NV50_MEMORY_VIRTUAL
#include "class/clc637.h" // AMPERE_SMC_PARTITION_REF
#include "class/cl00c2.h" // NV01_MEMORY_LOCAL_PHYSICAL
#include "class/clb0b5.h" // MAXWELL_DMA_COPY_A
#include "class/clc8b5.h" // HOPPER_DMA_COPY_A
@ -395,17 +393,16 @@ memmgrMemUtilsChannelInitialize_GM107
{
NV_STATUS rmStatus;
NV_STATUS lockStatus;
RsClient *pRsClient;
NvHandle hClient;
NvHandle hDevice;
NvHandle hPhysMem;
NvU64 size;
NvHandle hChannel;
NvHandle hErrNotifierVirt;
NvHandle hErrNotifierPhys;
NvHandle hPushBuffer;
RsClient *pRsClient = pChannel->pRsClient;
NvHandle hClient = pChannel->hClient;
NvHandle hDevice = pChannel->deviceId;
NvHandle hPhysMem = pChannel->physMemId;
NvU64 size = pChannel->channelSize;
NvHandle hChannel = pChannel->channelId;
NvHandle hErrNotifierVirt = pChannel->errNotifierIdVirt;
NvHandle hErrNotifierPhys = pChannel->errNotifierIdPhys;
NvHandle hPushBuffer = pChannel->pushBufferId;
RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
Heap *pHeap = GPU_GET_HEAP(pGpu);
NvBool bMIGInUse = IS_MIG_IN_USE(pGpu);
NvU8 *pErrNotifierCpuVA = NULL;
NV_ADDRESS_SPACE userdAddrSpace;
@ -422,61 +419,15 @@ memmgrMemUtilsChannelInitialize_GM107
//
union
{
NV0080_ALLOC_PARAMETERS nv0080;
NV2080_ALLOC_PARAMETERS nv2080;
NVC637_ALLOCATION_PARAMETERS nvC637;
NV_VASPACE_ALLOCATION_PARAMETERS va;
NV_MEMORY_ALLOCATION_PARAMS mem;
} *pParams = NULL;
size = pChannel->channelSize;
hPhysMem = pChannel->physMemId;
hChannel = pChannel->channelId;
hErrNotifierVirt = pChannel->errNotifierIdVirt;
hErrNotifierPhys = pChannel->errNotifierIdPhys;
hPushBuffer = pChannel->pushBufferId;
if (pCl->getProperty(pCl, PDB_PROP_CL_IS_CHIPSET_IO_COHERENT))
{
cacheSnoopFlag = DRF_DEF(OS46, _FLAGS, _CACHE_SNOOP, _ENABLE);
}
if (!pChannel->bClientAllocated)
{
NV_CHECK_OK_OR_RETURN(
LEVEL_ERROR,
pRmApi->AllocWithHandle(pRmApi, NV01_NULL_OBJECT, NV01_NULL_OBJECT,
NV01_NULL_OBJECT, NV01_ROOT,
&pChannel->hClient, sizeof(pChannel->hClient)));
NV_ASSERT_OK_OR_GOTO(
rmStatus,
serverGetClientUnderLock(&g_resServ, pChannel->hClient, &pRsClient),
exit_free_client);
pChannel->pRsClient = pRsClient;
if (IS_VIRTUAL(pGpu))
{
NV_ASSERT_OK_OR_GOTO(
rmStatus,
clientSetHandleGenerator(pRsClient, RS_UNIQUE_HANDLE_BASE,
RS_UNIQUE_HANDLE_RANGE/2 - VGPU_RESERVED_HANDLE_RANGE),
exit_free_client);
}
else
{
NV_ASSERT_OK_OR_GOTO(
rmStatus,
clientSetHandleGenerator(pRsClient, 1U, ~0U - 1U),
exit_free_client);
}
}
else
pRsClient = pChannel->pRsClient;
hClient = pRsClient->hClient;
pParams = portMemAllocNonPaged(sizeof(*pParams));
if (pParams == NULL)
{
@ -484,102 +435,11 @@ memmgrMemUtilsChannelInitialize_GM107
goto exit_free_client;
}
if (pChannel->deviceId == NV01_NULL_OBJECT)
{
NV_ASSERT_OK_OR_GOTO(
rmStatus,
clientGenResourceHandle(pRsClient, &pChannel->deviceId),
exit_free_client);
NV0080_ALLOC_PARAMETERS *pNv0080 = &pParams->nv0080;
portMemSet(pNv0080, 0, sizeof(*pNv0080));
// Which device are we?
pNv0080->deviceId = gpuGetDeviceInstance(pGpu);
pNv0080->hClientShare = hClient;
NV_CHECK_OK_OR_GOTO(
rmStatus,
LEVEL_ERROR,
pRmApi->AllocWithHandle(pRmApi, hClient, hClient, pChannel->deviceId,
NV01_DEVICE_0, pNv0080, sizeof(*pNv0080)),
exit_free_client);
}
hDevice = pChannel->deviceId;
// allocate a subdevice
if (pChannel->subdeviceId == NV01_NULL_OBJECT)
{
NV_ASSERT_OK_OR_GOTO(
rmStatus,
clientGenResourceHandle(pRsClient, &pChannel->subdeviceId),
exit_free_client);
NV2080_ALLOC_PARAMETERS *pNv2080 = &pParams->nv2080;
portMemSet(pNv2080, 0, sizeof(*pNv2080));
pNv2080->subDeviceId = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
NV_CHECK_OK_OR_GOTO(
rmStatus,
LEVEL_ERROR,
pRmApi->AllocWithHandle(pRmApi, hClient, hDevice, pChannel->subdeviceId,
NV20_SUBDEVICE_0,
pNv2080,
sizeof(*pNv2080)),
exit_free_client);
}
// MIG support is only added for PMA scrubber
if (bMIGInUse && (pChannel->pKernelMIGGpuInstance != NULL))
{
NVC637_ALLOCATION_PARAMETERS *pNvC637 = &pParams->nvC637;
NV_ASSERT_OK_OR_GOTO(
rmStatus,
clientGenResourceHandle(pRsClient, &pChannel->hPartitionRef),
exit_free_client);
portMemSet(pNvC637, 0, sizeof(*pNvC637));
pNvC637->swizzId = pChannel->pKernelMIGGpuInstance->swizzId;
NV_ASSERT_OK_OR_GOTO(
rmStatus,
pRmApi->AllocWithHandle(pRmApi, hClient,
pChannel->subdeviceId,
pChannel->hPartitionRef,
AMPERE_SMC_PARTITION_REF,
pNvC637,
sizeof(*pNvC637)),
exit_free_client);
pHeap = pChannel->pKernelMIGGpuInstance->pMemoryPartitionHeap;
}
//
// client allocated userd only supported on volta+
// TODO: Use property to check if client allocated userd is supported
//
pChannel->bClientUserd = NV_FALSE;
if (IsVOLTAorBetter(pGpu))
{
NvU32 pmaConfig = 0;
pmaConfig = PMA_QUERY_NUMA_ENABLED | PMA_QUERY_NUMA_ONLINED;
NV_ASSERT_OK_OR_GOTO(
rmStatus,
pmaQueryConfigs(&pHeap->pmaObject, &pmaConfig),
exit_free_client);
if (pmaConfig & PMA_QUERY_NUMA_ENABLED)
{
if (pmaConfig & PMA_QUERY_NUMA_ONLINED)
pChannel->bClientUserd = NV_TRUE;
else
pChannel->bClientUserd = NV_FALSE;
}
else
{
pChannel->bClientUserd = NV_TRUE;
}
}
pChannel->bClientUserd = IsVOLTAorBetter(pGpu);
//
// We need to allocate a VAS to use for CE copies, but also for
@ -1044,56 +904,6 @@ memmgrMemUtilsCopyEngineInitialize_GM107
return rmStatus;
}
static NV_STATUS _memUtilsGetCe_GM107
(
OBJGPU *pGpu,
NvHandle hClient,
NvHandle hDevice,
KernelCE **ppKCe
)
{
KernelCE *pKCe = NULL;
KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
NV_STATUS status = NV_OK;
NvU32 ceInst = 0;
*ppKCe = NULL;
if (IS_MIG_IN_USE(pGpu))
{
RsClient *pClient;
Device *pDevice;
NV_ASSERT_OK_OR_RETURN(
serverGetClientUnderLock(&g_resServ, hClient, &pClient));
NV_ASSERT_OK_OR_RETURN(
deviceGetByHandle(pClient, hDevice, &pDevice));
status = kmigmgrGetGPUInstanceScrubberCe(pGpu, GPU_GET_KERNEL_MIG_MANAGER(pGpu), pDevice, &ceInst);
}
else
{
NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, gpuUpdateEngineTable(pGpu));
KCE_ITER_ALL_BEGIN(pGpu, pKCe, 0)
if (kbusCheckEngine_HAL(pGpu, pKernelBus, ENG_CE(pKCe->publicID)) &&
!ceIsCeGrce(pGpu, RM_ENGINE_TYPE_COPY(pKCe->publicID)) &&
gpuCheckEngineTable(pGpu, RM_ENGINE_TYPE_COPY(pKCe->publicID)))
{
ceInst = kceInst;
break;
}
KCE_ITER_END_OR_RETURN_ERROR
}
NV_ASSERT_OK_OR_RETURN(status);
*ppKCe = GPU_GET_KCE(pGpu, ceInst);
return status;
}
static NV_STATUS _memUtilsAllocCe_GM107
(
OBJGPU *pGpu,
@ -1106,16 +916,11 @@ static NV_STATUS _memUtilsAllocCe_GM107
)
{
KernelCE *pKCe = NULL;
NVC0B5_ALLOCATION_PARAMETERS createParams;
NVC0B5_ALLOCATION_PARAMETERS createParams = {0};
RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
createParams.version = NVC0B5_ALLOCATION_PARAMETERS_VERSION_1;
NV_ASSERT_OK_OR_RETURN(_memUtilsGetCe_GM107(pGpu, hClientId, hDeviceId, &pKCe));
NV_ASSERT_OR_RETURN((pKCe != NULL), NV_ERR_INVALID_STATE);
createParams.engineType = NV2080_ENGINE_TYPE_COPY(pKCe->publicID);
createParams.engineType = NV2080_ENGINE_TYPE_COPY(pChannel->ceId);
memmgrMemUtilsGetCopyEngineClass_HAL(pGpu, pMemoryManager, &pChannel->hTdCopyClass);
pChannel->engineType = gpuGetRmEngineType(createParams.engineType);
@ -1135,7 +940,6 @@ static NV_STATUS _memUtilsAllocCe_GM107
&createParams,
sizeof(createParams)));
pChannel->ceId = pKCe->publicID;
return NV_OK;
}
@ -1280,21 +1084,15 @@ _memUtilsAllocateChannel
NvU32 hClass;
RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
NvBool bMIGInUse = IS_MIG_IN_USE(pGpu);
RM_ENGINE_TYPE engineType;
NvU32 flags = DRF_DEF(OS04, _FLAGS, _CHANNEL_SKIP_SCRUBBER, _TRUE);
RM_ENGINE_TYPE engineType = (pChannel->type == SWL_SCRUBBER_CHANNEL) ?
RM_ENGINE_TYPE_SEC2 : RM_ENGINE_TYPE_COPY(pChannel->ceId);
if (pChannel->type == SWL_SCRUBBER_CHANNEL)
if (pChannel->bSecure)
{
engineType = RM_ENGINE_TYPE_SEC2;
flags |= DRF_DEF(OS04, _FLAGS, _CC_SECURE, _TRUE);
}
else
{
KernelCE *pKCe = NULL;
NV_ASSERT_OK_OR_RETURN(_memUtilsGetCe_GM107(pGpu, hClientId, hDeviceId, &pKCe));
NV_ASSERT_OR_RETURN((pKCe != NULL), NV_ERR_INVALID_STATE);
engineType = RM_ENGINE_TYPE_COPY(pKCe->publicID);
}
portMemSet(&channelGPFIFOAllocParams, 0, sizeof(NV_CHANNEL_ALLOC_PARAMS));
channelGPFIFOAllocParams.hObjectError = hObjectError;
channelGPFIFOAllocParams.hObjectBuffer = hObjectBuffer;
@ -1363,7 +1161,6 @@ _memUtilsAllocateChannel
SLI_LOOP_END
}
NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(
rmStatus,
pRmApi->AllocWithHandle(pRmApi,
@ -1534,6 +1331,8 @@ memmgrMemUtilsAllocateEccScrubber_GM107
OBJCHANNEL *pChannel
)
{
NV_ASSERT_OK_OR_RETURN(channelAllocSubdevice(pGpu, pChannel));
memmgrMemUtilsChannelInitialize_HAL(pGpu, pMemoryManager, pChannel);
memmgrMemUtilsCopyEngineInitialize_HAL(pGpu, pMemoryManager, pChannel);
@ -1565,6 +1364,8 @@ memmgrMemUtilsAllocateEccAllocScrubber_GM107
NV_STATUS lockStatus;
RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
NV_ASSERT_OK_OR_RETURN(channelAllocSubdevice(pGpu, pChannel));
memmgrMemUtilsChannelInitialize_HAL(pGpu, pMemoryManager, pEccSyncChannel);
memmgrMemUtilsCopyEngineInitialize_HAL(pGpu, pMemoryManager, pEccSyncChannel);

72
src/nvidia/src/kernel/gpu/mem_mgr/ce_utils.c
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -36,6 +36,7 @@
#include "gpu/mem_mgr/ce_utils.h"
#include "kernel/gpu/mem_mgr/ce_utils_sizes.h"
#include "vgpu/rpc_headers.h"
#include "gpu/device/device.h"
#include "class/clb0b5.h" // MAXWELL_DMA_COPY_A
#include "class/clc0b5.h" // PASCAL_DMA_COPY_A
@ -47,6 +48,50 @@
#include "class/cl0080.h"
static NV_STATUS _memUtilsGetCe
(
OBJGPU *pGpu,
NvHandle hClient,
NvHandle hDevice,
NvU32 *pCeInstance
)
{
if (IS_MIG_IN_USE(pGpu))
{
RsClient *pClient;
Device *pDevice;
NV_ASSERT_OK_OR_RETURN(
serverGetClientUnderLock(&g_resServ, hClient, &pClient));
NV_ASSERT_OK_OR_RETURN(
deviceGetByHandle(pClient, hDevice, &pDevice));
NV_ASSERT_OK_OR_RETURN(kmigmgrGetGPUInstanceScrubberCe(pGpu, GPU_GET_KERNEL_MIG_MANAGER(pGpu), pDevice, pCeInstance));
return NV_OK;
}
else
{
KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, gpuUpdateEngineTable(pGpu));
KernelCE *pKCe = NULL;
KCE_ITER_ALL_BEGIN(pGpu, pKCe, 0)
if (kbusCheckEngine_HAL(pGpu, pKernelBus, ENG_CE(pKCe->publicID)) &&
!ceIsCeGrce(pGpu, RM_ENGINE_TYPE_COPY(pKCe->publicID)) &&
gpuCheckEngineTable(pGpu, RM_ENGINE_TYPE_COPY(pKCe->publicID)))
{
*pCeInstance = pKCe->publicID;
return NV_OK;
}
KCE_ITER_END
}
return NV_ERR_INSUFFICIENT_RESOURCES;
}
NV_STATUS
ceutilsConstruct_IMPL
(
@ -58,6 +103,7 @@ ceutilsConstruct_IMPL
{
NV_STATUS status = NV_OK;
NvU64 allocFlags = pAllocParams->flags;
NvBool bForceCeId = FLD_TEST_DRF(0050_CEUTILS, _FLAGS, _FORCE_CE_ID, _TRUE, allocFlags);
NV_ASSERT_OR_RETURN(pGpu, NV_ERR_INVALID_STATE);
NvBool bMIGInUse = IS_MIG_IN_USE(pGpu);
@ -116,16 +162,14 @@ ceutilsConstruct_IMPL
pChannel->bClientAllocated = NV_TRUE;
pChannel->pGpu = pGpu;
pChannel->deviceId = pCeUtils->hDevice;
pChannel->subdeviceId = pCeUtils->hSubdevice;
pChannel->pKernelMIGGpuInstance = pKernelMIGGPUInstance;
// We'll allocate new VAS for now. Sharing client VAS will be added later
pChannel->hVASpaceId = NV01_NULL_OBJECT;
pChannel->bUseVasForCeCopy = FLD_TEST_DRF(0050_CEUTILS, _FLAGS, _VIRTUAL_MODE, _TRUE, allocFlags);
pChannel->bSecure = FLD_TEST_DRF(0050_CEUTILS, _FLAGS, _CC_SECURE, _TRUE, allocFlags);
// Detect if we can enable fast scrub on this channel
status = memmgrMemUtilsGetCopyEngineClass_HAL(pGpu, pMemoryManager, &pCeUtils->hTdCopyClass);
NV_ASSERT_OR_GOTO(status == NV_OK, free_channel);
@ -158,6 +202,19 @@ ceutilsConstruct_IMPL
channelSetupChannelBufferSizes(pChannel);
NV_ASSERT_OK_OR_GOTO(status, channelAllocSubdevice(pGpu, pChannel), free_client);
if (bForceCeId)
{
pChannel->ceId = pAllocParams->forceCeId;
}
else
{
NV_ASSERT_OK_OR_GOTO(status,
_memUtilsGetCe(pGpu, pChannel->hClient, pChannel->deviceId, &pChannel->ceId),
free_client);
}
status = memmgrMemUtilsChannelInitialize_HAL(pGpu, pMemoryManager, pChannel);
NV_ASSERT_OR_GOTO(status == NV_OK, free_channel);
@ -563,6 +620,11 @@ ceutilsMemcopy_IMPL
channelPbInfo.srcCpuCacheAttrib = pSrcMemDesc->_cpuCacheAttrib;
channelPbInfo.dstCpuCacheAttrib = pDstMemDesc->_cpuCacheAttrib;
channelPbInfo.bSecureCopy = pParams->bSecureCopy;
channelPbInfo.bEncrypt = pParams->bEncrypt;
channelPbInfo.authTagAddr = pParams->authTagAddr;
channelPbInfo.encryptIvAddr = pParams->encryptIvAddr;
srcPageGranularity = pSrcMemDesc->pageArrayGranularity;
dstPageGranularity = pDstMemDesc->pageArrayGranularity;
bSrcContig = memdescGetContiguity(pSrcMemDesc, AT_GPU);

227
src/nvidia/src/kernel/gpu/mem_mgr/channel_utils.c
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -28,10 +28,14 @@
#include "utils/nvassert.h"
#include "core/locks.h"
#include "gpu/mem_mgr/mem_mgr.h"
#include "vgpu/rpc.h"
#include "kernel/gpu/mem_mgr/ce_utils_sizes.h"
#include "kernel/gpu/mem_mgr/channel_utils.h"
#include "class/clcba2.h"
#include "class/cl0080.h" // NV01_DEVICE_0
#include "class/clc637.h" // AMPERE_SMC_PARTITION_REF
#define SEC2_WL_METHOD_ARRAY_SIZE 16
#define SHA_256_HASH_SIZE_BYTE 32
@ -40,7 +44,7 @@
static NvU32 channelPushMemoryProperties(OBJCHANNEL *pChannel, CHANNEL_PB_INFO *pChannelPbInfo, NvU32 **ppPtr);
static void channelPushMethod(OBJCHANNEL *pChannel, CHANNEL_PB_INFO *pChannelPbInfo,
NvBool bPipelined, NvBool bInsertFinishPayload,
NvU32 launchType, NvU32 semaValue, NvU32 **ppPtr);
NvU32 launchType, NvU32 semaValue, NvU32 copyType, NvU32 **ppPtr);
/* Public APIs */
NV_STATUS
@ -91,6 +95,125 @@ channelSetupIDs
return NV_OK;
}
NV_STATUS
channelAllocSubdevice
(
OBJGPU *pGpu,
OBJCHANNEL *pChannel
)
{
RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
NV_STATUS rmStatus = NV_OK;
RsClient *pRsClient;
if (!pChannel->bClientAllocated)
{
NV_CHECK_OK_OR_RETURN(
LEVEL_ERROR,
pRmApi->AllocWithHandle(pRmApi, NV01_NULL_OBJECT, NV01_NULL_OBJECT,
NV01_NULL_OBJECT, NV01_ROOT,
&pChannel->hClient, sizeof(pChannel->hClient)));
NV_ASSERT_OK_OR_GOTO(
rmStatus,
serverGetClientUnderLock(&g_resServ, pChannel->hClient, &pRsClient),
exit_free_client);
pChannel->pRsClient = pRsClient;
if (IS_VIRTUAL(pGpu))
{
NV_ASSERT_OK_OR_GOTO(
rmStatus,
clientSetHandleGenerator(pRsClient, RS_UNIQUE_HANDLE_BASE,
RS_UNIQUE_HANDLE_RANGE/2 - VGPU_RESERVED_HANDLE_RANGE),
exit_free_client);
}
else
{
NV_ASSERT_OK_OR_GOTO(
rmStatus,
clientSetHandleGenerator(pRsClient, 1U, ~0U - 1U),
exit_free_client);
}
}
else
pRsClient = pChannel->pRsClient;
if (pChannel->deviceId == NV01_NULL_OBJECT)
{
NV_ASSERT_OK_OR_GOTO(
rmStatus,
clientGenResourceHandle(pRsClient, &pChannel->deviceId),
exit_free_client);
NV0080_ALLOC_PARAMETERS params = {0};
// Which device are we?
params.deviceId = gpuGetDeviceInstance(pGpu);
params.hClientShare = pChannel->hClient;
NV_CHECK_OK_OR_GOTO(
rmStatus,
LEVEL_ERROR,
pRmApi->AllocWithHandle(pRmApi, pChannel->hClient, pChannel->hClient, pChannel->deviceId,
NV01_DEVICE_0, &params, sizeof(params)),
exit_free_client);
}
// allocate a subdevice
if (pChannel->subdeviceId == NV01_NULL_OBJECT)
{
NV_ASSERT_OK_OR_GOTO(
rmStatus,
clientGenResourceHandle(pRsClient, &pChannel->subdeviceId),
exit_free_client);
NV2080_ALLOC_PARAMETERS params = {0};
params.subDeviceId = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
NV_CHECK_OK_OR_GOTO(
rmStatus,
LEVEL_ERROR,
pRmApi->AllocWithHandle(pRmApi, pChannel->hClient, pChannel->deviceId, pChannel->subdeviceId,
NV20_SUBDEVICE_0,
&params,
sizeof(params)),
exit_free_client);
}
// MIG support is only added for PMA scrubber
if (IS_MIG_IN_USE(pGpu) && (pChannel->pKernelMIGGpuInstance != NULL))
{
NV_ASSERT_OK_OR_GOTO(
rmStatus,
clientGenResourceHandle(pRsClient, &pChannel->hPartitionRef),
exit_free_client);
NVC637_ALLOCATION_PARAMETERS params = {0};
params.swizzId = pChannel->pKernelMIGGpuInstance->swizzId;
NV_ASSERT_OK_OR_GOTO(
rmStatus,
pRmApi->AllocWithHandle(pRmApi, pChannel->hClient,
pChannel->subdeviceId,
pChannel->hPartitionRef,
AMPERE_SMC_PARTITION_REF,
&params,
sizeof(params)),
exit_free_client);
}
exit_free_client:
if(rmStatus != NV_OK && !pChannel->bClientAllocated)
{
pRmApi->Free(pRmApi, pChannel->hClient, pChannel->hClient);
}
return rmStatus;
}
void
channelSetupChannelBufferSizes
(
@ -335,15 +458,15 @@ channelFillGpFifo
NvU32 methodsLength
)
{
OBJGPU *pGpu = pChannel->pGpu;
KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
NvBool bReleaseMapping = NV_FALSE;
NvU32 *pGpEntry;
NvU32 GpEntry0;
NvU32 GpEntry1;
NvU64 pbPutOffset;
OBJGPU *pGpu;
KernelBus *pKernelBus;
MemoryManager *pMemoryManager;
NvBool bReleaseMapping = NV_FALSE;
//
// Use BAR1 if CPU access is allowed, otherwise allocate and init shadow
// buffer for DMA access
@ -353,13 +476,6 @@ channelFillGpFifo
TRANSFER_FLAGS_SHADOW_INIT_MEM);
NV_ASSERT_OR_RETURN(putIndex < pChannel->channelNumGpFifioEntries, NV_ERR_INVALID_STATE);
NV_ASSERT_OR_RETURN(pChannel != NULL, NV_ERR_INVALID_STATE);
pGpu = pChannel->pGpu;
NV_ASSERT_OR_RETURN(pGpu != NULL, NV_ERR_INVALID_STATE);
pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
if (pChannel->pbCpuVA == NULL)
{
@ -424,8 +540,27 @@ channelFillGpFifo
return NV_ERR_GENERIC;
}
// Update doorbell with work submission token
if (pChannel->bUseDoorbellRegister)
if (RMCFG_FEATURE_PLATFORM_GSP ||
kfifoIsLiteModeEnabled_HAL(pGpu, pKernelFifo))
{
KernelChannel *pKernelChannel;
NvU32 workSubmitToken;
{
RsClient *pClient;
NV_ASSERT_OK(serverGetClientUnderLock(&g_resServ, pChannel->hClient, &pClient));
NV_ASSERT_OK(CliGetKernelChannel(pClient, pChannel->channelId, &pKernelChannel));
}
NV_ASSERT_OK_OR_RETURN(
kfifoGenerateWorkSubmitToken_HAL(pGpu,
pKernelFifo,
pKernelChannel,
&workSubmitToken, NV_TRUE));
kfifoUpdateUsermodeDoorbell_HAL(pGpu, pKernelFifo, workSubmitToken, kchannelGetRunlistId(pKernelChannel));
}
else if (pChannel->bUseDoorbellRegister)
{
if (pChannel->pTokenFromNotifier == NULL)
{
@ -592,6 +727,53 @@ channelAddHostSema
*ppPtr = pPtr;
}
static NvU32
channelPushSecureCopyProperties
(
OBJCHANNEL *pChannel,
CHANNEL_PB_INFO *pChannelPbInfo,
NvU32 *pCopyType,
NvU32 **ppPtr
)
{
NvU32 *pPtr = *ppPtr;
if (!pChannelPbInfo->bSecureCopy)
{
*pCopyType = FLD_SET_DRF(C8B5, _LAUNCH_DMA, _COPY_TYPE, _DEFAULT, *pCopyType);
return NV_OK;
}
NV_ASSERT_OR_RETURN(gpuIsCCFeatureEnabled(pChannel->pGpu), NV_ERR_NOT_SUPPORTED);
NV_ASSERT_OR_RETURN(pChannel->bSecure, NV_ERR_NOT_SUPPORTED);
NV_ASSERT_OR_RETURN(pChannel->hTdCopyClass >= HOPPER_DMA_COPY_A, NV_ERR_NOT_SUPPORTED);
if (pChannelPbInfo->bEncrypt)
{
NV_PUSH_INC_1U(RM_SUBCHANNEL,
NVC8B5_SET_SECURE_COPY_MODE, DRF_DEF(C8B5, _SET_SECURE_COPY_MODE, _MODE, _ENCRYPT));
NV_PUSH_INC_4U(RM_SUBCHANNEL,
NVC8B5_SET_ENCRYPT_AUTH_TAG_ADDR_UPPER, NvU64_HI32(pChannelPbInfo->authTagAddr),
NVC8B5_SET_ENCRYPT_AUTH_TAG_ADDR_LOWER, NvU64_LO32(pChannelPbInfo->authTagAddr),
NVC8B5_SET_ENCRYPT_IV_ADDR_UPPER, NvU64_HI32(pChannelPbInfo->encryptIvAddr),
NVC8B5_SET_ENCRYPT_IV_ADDR_LOWER, NvU64_LO32(pChannelPbInfo->encryptIvAddr));
}
else
{
NV_PUSH_INC_1U(RM_SUBCHANNEL,
NVC8B5_SET_SECURE_COPY_MODE, DRF_DEF(C8B5, _SET_SECURE_COPY_MODE, _MODE, _DECRYPT));
NV_PUSH_INC_2U(RM_SUBCHANNEL,
NVC8B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_UPPER, NvU64_HI32(pChannelPbInfo->authTagAddr),
NVC8B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_LOWER, NvU64_LO32(pChannelPbInfo->authTagAddr));
}
*ppPtr = pPtr;
*pCopyType = FLD_SET_DRF(C8B5, _LAUNCH_DMA, _COPY_TYPE, _SECURE, *pCopyType);
return NV_OK;
}
/** single helper function to fill the push buffer with the methods needed for
* memsetting using CE. This function is much more efficient in the sense it
* decouples the mem(set/copy) operation from managing channel resources.
@ -607,6 +789,7 @@ channelFillCePb
CHANNEL_PB_INFO *pChannelPbInfo
)
{
NvU32 copyType = 0;
NvU32 launchType = 0;
NvU32 *pPtr = (NvU32 *)((NvU8 *)pChannel->pbCpuVA + (putIndex * pChannel->methodSizePerBlock));
NvU32 *pStartPtr = pPtr;
@ -616,6 +799,9 @@ channelFillCePb
NV_PUSH_INC_1U(RM_SUBCHANNEL, NV906F_SET_OBJECT, pChannel->classEngineID);
if (channelPushSecureCopyProperties(pChannel, pChannelPbInfo, &copyType, &pPtr) != NV_OK)
return 0;
// Side effect - pushed target addresses, aperture and REMAP method for memset
launchType = channelPushMemoryProperties(pChannel, pChannelPbInfo, &pPtr);
@ -635,7 +821,10 @@ channelFillCePb
}
// Side effect - pushed LAUNCH_DMA methods
channelPushMethod(pChannel, pChannelPbInfo, bPipelined, bInsertFinishPayload, launchType, semaValue, &pPtr);
channelPushMethod(pChannel, pChannelPbInfo, bPipelined, bInsertFinishPayload,
launchType, semaValue,
copyType,
&pPtr);
channelAddHostSema(pChannel, putIndex, &pPtr);
@ -899,6 +1088,7 @@ channelPushMethod
NvBool bInsertFinishPayload,
NvU32 launchType,
NvU32 semaValue,
NvU32 copyType,
NvU32 **ppPtr
)
{
@ -952,6 +1142,7 @@ channelPushMethod
launchType |
pipelinedValue |
flushValue |
semaValue);
semaValue |
copyType);
*ppPtr = pPtr;
}

6
src/nvidia/src/kernel/gpu/mem_mgr/sec2_utils.c
View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@ -221,6 +221,8 @@ sec2utilsConstruct_IMPL
pChannel->type = SWL_SCRUBBER_CHANNEL;
pChannel->engineType = RM_ENGINE_TYPE_SEC2;
pChannel->bSecure = NV_TRUE;
// Detect if we can enable fast scrub on this channel
NV_ASSERT_OK_OR_GOTO(status, _sec2GetClass(pGpu, &pSec2Utils->sec2Class), free_client);
pChannel->sec2Class = pSec2Utils->sec2Class;
@ -240,6 +242,8 @@ sec2utilsConstruct_IMPL
pChannel->engineType = NV2080_ENGINE_TYPE_SEC2;
NV_ASSERT_OK_OR_GOTO(status, channelAllocSubdevice(pGpu, pChannel), free_client);
pMemoryManager->bScrubChannelSetupInProgress = NV_TRUE;
NV_ASSERT_OK_OR_GOTO(status, memmgrMemUtilsChannelInitialize_HAL(pGpu, pMemoryManager, pChannel), free_channel);
pMemoryManager->bScrubChannelSetupInProgress = NV_FALSE;

2
src/nvidia/src/kernel/gpu/mig_mgr/kernel_mig_manager.c
View File

@ -765,7 +765,7 @@ _kmigmgrHandlePostSchedulingEnableCallback
if ((pKernelMIGManager == NULL) || !kmigmgrIsMIGSupported(pGpu, pKernelMIGManager))
{
NV_PRINTF(LEVEL_INFO, "MIG not supported on this GPU.\n");
return NV_ERR_NOT_SUPPORTED;
return NV_OK;
}
if (!IS_MIG_ENABLED(pGpu) && !IS_VIRTUAL(pGpu) &&

3
src/nvidia/src/kernel/gpu_mgr/gpu_mgr.c
View File

@ -49,6 +49,7 @@
#include "gpu/conf_compute/conf_compute.h"
#include "gpu/gpu_fabric_probe.h"
#include "gpu/mig_mgr/gpu_instance_subscription.h"
#include "ctrl/ctrlc56f.h"
// local static funcs
static void gpumgrSetAttachInfo(OBJGPU *, GPUATTACHARG *);
@ -255,6 +256,8 @@ gpumgrConstruct_IMPL(OBJGPUMGR *pGpuMgr)
portMemSet(pGpuMgr->cachedMIGInfo, 0, sizeof(pGpuMgr->cachedMIGInfo));
pGpuMgr->ccAttackerAdvantage = SECURITY_POLICY_ATTACKER_ADVANTAGE_DEFAULT;
return NV_OK;
}

2
version.mk
View File

@ -1,4 +1,4 @@
NVIDIA_VERSION = 550.76
NVIDIA_VERSION = 550.78
# This file.
VERSION_MK_FILE := $(lastword $(MAKEFILE_LIST))