/* * QEMU NVM Express End-to-End Data Protection support * * Copyright (c) 2021 Samsung Electronics Co., Ltd. * * Authors: * Klaus Jensen <k.jensen@samsung.com> * Gollu Appalanaidu <anaidu.gollu@samsung.com> */ #include "qemu/osdep.h" #include "qapi/error.h" #include "sysemu/block-backend.h" #include "nvme.h" #include "dif.h" #include "trace.h" uint16_t nvme_check_prinfo(NvmeNamespace *ns, uint8_t prinfo, uint64_t slba, uint64_t reftag) { uint64_t mask = ns->pif ? 0xffffffffffff : 0xffffffff; if ((NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) == NVME_ID_NS_DPS_TYPE_1) && (prinfo & NVME_PRINFO_PRCHK_REF) && (slba & mask) != reftag) { return NVME_INVALID_PROT_INFO | NVME_DNR; } if ((NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) == NVME_ID_NS_DPS_TYPE_3) && (prinfo & NVME_PRINFO_PRCHK_REF)) { return NVME_INVALID_PROT_INFO; } return NVME_SUCCESS; } /* from Linux kernel (crypto/crct10dif_common.c) */ static uint16_t crc16_t10dif(uint16_t crc, const unsigned char *buffer, size_t len) { unsigned int i; for (i = 0; i < len; i++) { crc = (crc << 8) ^ crc16_t10dif_table[((crc >> 8) ^ buffer[i]) & 0xff]; } return crc; } /* from Linux kernel (lib/crc64.c) */ static uint64_t crc64_nvme(uint64_t crc, const unsigned char *buffer, size_t len) { size_t i; for (i = 0; i < len; i++) { crc = (crc >> 8) ^ crc64_nvme_table[(crc & 0xff) ^ buffer[i]]; } return crc ^ (uint64_t)~0; } static void nvme_dif_pract_generate_dif_crc16(NvmeNamespace *ns, uint8_t *buf, size_t len, uint8_t *mbuf, size_t mlen, uint16_t apptag, uint64_t *reftag) { uint8_t *end = buf + len; int16_t pil = 0; if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) { pil = ns->lbaf.ms - nvme_pi_tuple_size(ns); } trace_pci_nvme_dif_pract_generate_dif_crc16(len, ns->lbasz, ns->lbasz + pil, apptag, *reftag); for (; buf < end; buf += ns->lbasz, mbuf += ns->lbaf.ms) { NvmeDifTuple *dif = (NvmeDifTuple *)(mbuf + pil); uint16_t crc = crc16_t10dif(0x0, buf, ns->lbasz); if (pil) { crc = crc16_t10dif(crc, mbuf, pil); } dif->g16.guard = cpu_to_be16(crc); dif->g16.apptag = cpu_to_be16(apptag); dif->g16.reftag = cpu_to_be32(*reftag); if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) != NVME_ID_NS_DPS_TYPE_3) { (*reftag)++; } } } static void nvme_dif_pract_generate_dif_crc64(NvmeNamespace *ns, uint8_t *buf, size_t len, uint8_t *mbuf, size_t mlen, uint16_t apptag, uint64_t *reftag) { uint8_t *end = buf + len; int16_t pil = 0; if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) { pil = ns->lbaf.ms - 16; } trace_pci_nvme_dif_pract_generate_dif_crc64(len, ns->lbasz, ns->lbasz + pil, apptag, *reftag); for (; buf < end; buf += ns->lbasz, mbuf += ns->lbaf.ms) { NvmeDifTuple *dif = (NvmeDifTuple *)(mbuf + pil); uint64_t crc = crc64_nvme(~0ULL, buf, ns->lbasz); if (pil) { crc = crc64_nvme(~crc, mbuf, pil); } dif->g64.guard = cpu_to_be64(crc); dif->g64.apptag = cpu_to_be16(apptag); dif->g64.sr[0] = *reftag >> 40; dif->g64.sr[1] = *reftag >> 32; dif->g64.sr[2] = *reftag >> 24; dif->g64.sr[3] = *reftag >> 16; dif->g64.sr[4] = *reftag >> 8; dif->g64.sr[5] = *reftag; if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) != NVME_ID_NS_DPS_TYPE_3) { (*reftag)++; } } } void nvme_dif_pract_generate_dif(NvmeNamespace *ns, uint8_t *buf, size_t len, uint8_t *mbuf, size_t mlen, uint16_t apptag, uint64_t *reftag) { switch (ns->pif) { case NVME_PI_GUARD_16: return nvme_dif_pract_generate_dif_crc16(ns, buf, len, mbuf, mlen, apptag, reftag); case NVME_PI_GUARD_64: return nvme_dif_pract_generate_dif_crc64(ns, buf, len, mbuf, mlen, apptag, reftag); } abort(); } static uint16_t nvme_dif_prchk_crc16(NvmeNamespace *ns, NvmeDifTuple *dif, uint8_t *buf, uint8_t *mbuf, size_t pil, uint8_t prinfo, uint16_t apptag, uint16_t appmask, uint64_t reftag) { switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { case NVME_ID_NS_DPS_TYPE_3: if (be32_to_cpu(dif->g16.reftag) != 0xffffffff) { break; } /* fallthrough */ case NVME_ID_NS_DPS_TYPE_1: case NVME_ID_NS_DPS_TYPE_2: if (be16_to_cpu(dif->g16.apptag) != 0xffff) { break; } trace_pci_nvme_dif_prchk_disabled_crc16(be16_to_cpu(dif->g16.apptag), be32_to_cpu(dif->g16.reftag)); return NVME_SUCCESS; } if (prinfo & NVME_PRINFO_PRCHK_GUARD) { uint16_t crc = crc16_t10dif(0x0, buf, ns->lbasz); if (pil) { crc = crc16_t10dif(crc, mbuf, pil); } trace_pci_nvme_dif_prchk_guard_crc16(be16_to_cpu(dif->g16.guard), crc); if (be16_to_cpu(dif->g16.guard) != crc) { return NVME_E2E_GUARD_ERROR; } } if (prinfo & NVME_PRINFO_PRCHK_APP) { trace_pci_nvme_dif_prchk_apptag(be16_to_cpu(dif->g16.apptag), apptag, appmask); if ((be16_to_cpu(dif->g16.apptag) & appmask) != (apptag & appmask)) { return NVME_E2E_APP_ERROR; } } if (prinfo & NVME_PRINFO_PRCHK_REF) { trace_pci_nvme_dif_prchk_reftag_crc16(be32_to_cpu(dif->g16.reftag), reftag); if (be32_to_cpu(dif->g16.reftag) != reftag) { return NVME_E2E_REF_ERROR; } } return NVME_SUCCESS; } static uint16_t nvme_dif_prchk_crc64(NvmeNamespace *ns, NvmeDifTuple *dif, uint8_t *buf, uint8_t *mbuf, size_t pil, uint8_t prinfo, uint16_t apptag, uint16_t appmask, uint64_t reftag) { uint64_t r = 0; r |= (uint64_t)dif->g64.sr[0] << 40; r |= (uint64_t)dif->g64.sr[1] << 32; r |= (uint64_t)dif->g64.sr[2] << 24; r |= (uint64_t)dif->g64.sr[3] << 16; r |= (uint64_t)dif->g64.sr[4] << 8; r |= (uint64_t)dif->g64.sr[5]; switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { case NVME_ID_NS_DPS_TYPE_3: if (r != 0xffffffffffff) { break; } /* fallthrough */ case NVME_ID_NS_DPS_TYPE_1: case NVME_ID_NS_DPS_TYPE_2: if (be16_to_cpu(dif->g64.apptag) != 0xffff) { break; } trace_pci_nvme_dif_prchk_disabled_crc64(be16_to_cpu(dif->g16.apptag), r); return NVME_SUCCESS; } if (prinfo & NVME_PRINFO_PRCHK_GUARD) { uint64_t crc = crc64_nvme(~0ULL, buf, ns->lbasz); if (pil) { crc = crc64_nvme(~crc, mbuf, pil); } trace_pci_nvme_dif_prchk_guard_crc64(be64_to_cpu(dif->g64.guard), crc); if (be64_to_cpu(dif->g64.guard) != crc) { return NVME_E2E_GUARD_ERROR; } } if (prinfo & NVME_PRINFO_PRCHK_APP) { trace_pci_nvme_dif_prchk_apptag(be16_to_cpu(dif->g64.apptag), apptag, appmask); if ((be16_to_cpu(dif->g64.apptag) & appmask) != (apptag & appmask)) { return NVME_E2E_APP_ERROR; } } if (prinfo & NVME_PRINFO_PRCHK_REF) { trace_pci_nvme_dif_prchk_reftag_crc64(r, reftag); if (r != reftag) { return NVME_E2E_REF_ERROR; } } return NVME_SUCCESS; } static uint16_t nvme_dif_prchk(NvmeNamespace *ns, NvmeDifTuple *dif, uint8_t *buf, uint8_t *mbuf, size_t pil, uint8_t prinfo, uint16_t apptag, uint16_t appmask, uint64_t reftag) { switch (ns->pif) { case NVME_PI_GUARD_16: return nvme_dif_prchk_crc16(ns, dif, buf, mbuf, pil, prinfo, apptag, appmask, reftag); case NVME_PI_GUARD_64: return nvme_dif_prchk_crc64(ns, dif, buf, mbuf, pil, prinfo, apptag, appmask, reftag); } abort(); } uint16_t nvme_dif_check(NvmeNamespace *ns, uint8_t *buf, size_t len, uint8_t *mbuf, size_t mlen, uint8_t prinfo, uint64_t slba, uint16_t apptag, uint16_t appmask, uint64_t *reftag) { uint8_t *bufp, *end = buf + len; int16_t pil = 0; uint16_t status; status = nvme_check_prinfo(ns, prinfo, slba, *reftag); if (status) { return status; } if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) { pil = ns->lbaf.ms - nvme_pi_tuple_size(ns); } trace_pci_nvme_dif_check(prinfo, ns->lbasz + pil); for (bufp = buf; bufp < end; bufp += ns->lbasz, mbuf += ns->lbaf.ms) { NvmeDifTuple *dif = (NvmeDifTuple *)(mbuf + pil); status = nvme_dif_prchk(ns, dif, bufp, mbuf, pil, prinfo, apptag, appmask, *reftag); if (status) { /* * The first block of a 'raw' image is always allocated, so we * cannot reliably know if the block is all zeroes or not. For * CRC16 this works fine because the T10 CRC16 is 0x0 for all * zeroes, but the Rocksoft CRC64 is not. Thus, if a guard error is * detected for the first block, check if it is zeroed and manually * set the protection information to all ones to disable protection * information checking. */ if (status == NVME_E2E_GUARD_ERROR && slba == 0x0 && bufp == buf) { g_autofree uint8_t *zeroes = g_malloc0(ns->lbasz); if (memcmp(bufp, zeroes, ns->lbasz) == 0) { memset(mbuf + pil, 0xff, nvme_pi_tuple_size(ns)); } } else { return status; } } if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) != NVME_ID_NS_DPS_TYPE_3) { (*reftag)++; } } return NVME_SUCCESS; } uint16_t nvme_dif_mangle_mdata(NvmeNamespace *ns, uint8_t *mbuf, size_t mlen, uint64_t slba) { BlockBackend *blk = ns->blkconf.blk; BlockDriverState *bs = blk_bs(blk); int64_t moffset = 0, offset = nvme_l2b(ns, slba); uint8_t *mbufp, *end; bool zeroed; int16_t pil = 0; int64_t bytes = (mlen / ns->lbaf.ms) << ns->lbaf.ds; int64_t pnum = 0; Error *err = NULL; if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) { pil = ns->lbaf.ms - nvme_pi_tuple_size(ns); } do { int ret; bytes -= pnum; ret = bdrv_block_status(bs, offset, bytes, &pnum, NULL, NULL); if (ret < 0) { error_setg_errno(&err, -ret, "unable to get block status"); error_report_err(err); return NVME_INTERNAL_DEV_ERROR; } zeroed = !!(ret & BDRV_BLOCK_ZERO); trace_pci_nvme_block_status(offset, bytes, pnum, ret, zeroed); if (zeroed) { mbufp = mbuf + moffset; mlen = (pnum >> ns->lbaf.ds) * ns->lbaf.ms; end = mbufp + mlen; for (; mbufp < end; mbufp += ns->lbaf.ms) { memset(mbufp + pil, 0xff, nvme_pi_tuple_size(ns)); } } moffset += (pnum >> ns->lbaf.ds) * ns->lbaf.ms; offset += pnum; } while (pnum != bytes); return NVME_SUCCESS; } static void nvme_dif_rw_cb(void *opaque, int ret) { NvmeBounceContext *ctx = opaque; NvmeRequest *req = ctx->req; NvmeNamespace *ns = req->ns; BlockBackend *blk = ns->blkconf.blk; trace_pci_nvme_dif_rw_cb(nvme_cid(req), blk_name(blk)); qemu_iovec_destroy(&ctx->data.iov); g_free(ctx->data.bounce); qemu_iovec_destroy(&ctx->mdata.iov); g_free(ctx->mdata.bounce); g_free(ctx); nvme_rw_complete_cb(req, ret); } static void nvme_dif_rw_check_cb(void *opaque, int ret) { NvmeBounceContext *ctx = opaque; NvmeRequest *req = ctx->req; NvmeNamespace *ns = req->ns; NvmeCtrl *n = nvme_ctrl(req); NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; uint64_t slba = le64_to_cpu(rw->slba); uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control)); uint16_t apptag = le16_to_cpu(rw->apptag); uint16_t appmask = le16_to_cpu(rw->appmask); uint64_t reftag = le32_to_cpu(rw->reftag); uint64_t cdw3 = le32_to_cpu(rw->cdw3); uint16_t status; reftag |= cdw3 << 32; trace_pci_nvme_dif_rw_check_cb(nvme_cid(req), prinfo, apptag, appmask, reftag); if (ret) { goto out; } status = nvme_dif_mangle_mdata(ns, ctx->mdata.bounce, ctx->mdata.iov.size, slba); if (status) { req->status = status; goto out; } status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size, ctx->mdata.bounce, ctx->mdata.iov.size, prinfo, slba, apptag, appmask, &reftag); if (status) { req->status = status; goto out; } status = nvme_bounce_data(n, ctx->data.bounce, ctx->data.iov.size, NVME_TX_DIRECTION_FROM_DEVICE, req); if (status) { req->status = status; goto out; } if (prinfo & NVME_PRINFO_PRACT && ns->lbaf.ms == nvme_pi_tuple_size(ns)) { goto out; } status = nvme_bounce_mdata(n, ctx->mdata.bounce, ctx->mdata.iov.size, NVME_TX_DIRECTION_FROM_DEVICE, req); if (status) { req->status = status; } out: nvme_dif_rw_cb(ctx, ret); } static void nvme_dif_rw_mdata_in_cb(void *opaque, int ret) { NvmeBounceContext *ctx = opaque; NvmeRequest *req = ctx->req; NvmeNamespace *ns = req->ns; NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; uint64_t slba = le64_to_cpu(rw->slba); uint32_t nlb = le16_to_cpu(rw->nlb) + 1; size_t mlen = nvme_m2b(ns, nlb); uint64_t offset = nvme_moff(ns, slba); BlockBackend *blk = ns->blkconf.blk; trace_pci_nvme_dif_rw_mdata_in_cb(nvme_cid(req), blk_name(blk)); if (ret) { goto out; } ctx->mdata.bounce = g_malloc(mlen); qemu_iovec_reset(&ctx->mdata.iov); qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen); req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0, nvme_dif_rw_check_cb, ctx); return; out: nvme_dif_rw_cb(ctx, ret); } static void nvme_dif_rw_mdata_out_cb(void *opaque, int ret) { NvmeBounceContext *ctx = opaque; NvmeRequest *req = ctx->req; NvmeNamespace *ns = req->ns; NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; uint64_t slba = le64_to_cpu(rw->slba); uint64_t offset = nvme_moff(ns, slba); BlockBackend *blk = ns->blkconf.blk; trace_pci_nvme_dif_rw_mdata_out_cb(nvme_cid(req), blk_name(blk)); if (ret) { goto out; } req->aiocb = blk_aio_pwritev(blk, offset, &ctx->mdata.iov, 0, nvme_dif_rw_cb, ctx); return; out: nvme_dif_rw_cb(ctx, ret); } uint16_t nvme_dif_rw(NvmeCtrl *n, NvmeRequest *req) { NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd; NvmeNamespace *ns = req->ns; BlockBackend *blk = ns->blkconf.blk; bool wrz = rw->opcode == NVME_CMD_WRITE_ZEROES; uint32_t nlb = le16_to_cpu(rw->nlb) + 1; uint64_t slba = le64_to_cpu(rw->slba); size_t len = nvme_l2b(ns, nlb); size_t mlen = nvme_m2b(ns, nlb); size_t mapped_len = len; int64_t offset = nvme_l2b(ns, slba); uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control)); uint16_t apptag = le16_to_cpu(rw->apptag); uint16_t appmask = le16_to_cpu(rw->appmask); uint64_t reftag = le32_to_cpu(rw->reftag); uint64_t cdw3 = le32_to_cpu(rw->cdw3); bool pract = !!(prinfo & NVME_PRINFO_PRACT); NvmeBounceContext *ctx; uint16_t status; reftag |= cdw3 << 32; trace_pci_nvme_dif_rw(pract, prinfo); ctx = g_new0(NvmeBounceContext, 1); ctx->req = req; if (wrz) { BdrvRequestFlags flags = BDRV_REQ_MAY_UNMAP; if (prinfo & NVME_PRINFO_PRCHK_MASK) { status = NVME_INVALID_PROT_INFO | NVME_DNR; goto err; } if (pract) { uint8_t *mbuf, *end; int16_t pil = ns->lbaf.ms - nvme_pi_tuple_size(ns); status = nvme_check_prinfo(ns, prinfo, slba, reftag); if (status) { goto err; } flags = 0; ctx->mdata.bounce = g_malloc0(mlen); qemu_iovec_init(&ctx->mdata.iov, 1); qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen); mbuf = ctx->mdata.bounce; end = mbuf + mlen; if (ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT) { pil = 0; } for (; mbuf < end; mbuf += ns->lbaf.ms) { NvmeDifTuple *dif = (NvmeDifTuple *)(mbuf + pil); switch (ns->pif) { case NVME_PI_GUARD_16: dif->g16.apptag = cpu_to_be16(apptag); dif->g16.reftag = cpu_to_be32(reftag); break; case NVME_PI_GUARD_64: dif->g64.guard = cpu_to_be64(0x6482d367eb22b64e); dif->g64.apptag = cpu_to_be16(apptag); dif->g64.sr[0] = reftag >> 40; dif->g64.sr[1] = reftag >> 32; dif->g64.sr[2] = reftag >> 24; dif->g64.sr[3] = reftag >> 16; dif->g64.sr[4] = reftag >> 8; dif->g64.sr[5] = reftag; break; default: abort(); } switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { case NVME_ID_NS_DPS_TYPE_1: case NVME_ID_NS_DPS_TYPE_2: reftag++; } } } req->aiocb = blk_aio_pwrite_zeroes(blk, offset, len, flags, nvme_dif_rw_mdata_out_cb, ctx); return NVME_NO_COMPLETE; } if (nvme_ns_ext(ns) && !(pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) { mapped_len += mlen; } status = nvme_map_dptr(n, &req->sg, mapped_len, &req->cmd); if (status) { goto err; } ctx->data.bounce = g_malloc(len); qemu_iovec_init(&ctx->data.iov, 1); qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, len); if (req->cmd.opcode == NVME_CMD_READ) { block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size, BLOCK_ACCT_READ); req->aiocb = blk_aio_preadv(ns->blkconf.blk, offset, &ctx->data.iov, 0, nvme_dif_rw_mdata_in_cb, ctx); return NVME_NO_COMPLETE; } status = nvme_bounce_data(n, ctx->data.bounce, ctx->data.iov.size, NVME_TX_DIRECTION_TO_DEVICE, req); if (status) { goto err; } ctx->mdata.bounce = g_malloc(mlen); qemu_iovec_init(&ctx->mdata.iov, 1); qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen); if (!(pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) { status = nvme_bounce_mdata(n, ctx->mdata.bounce, ctx->mdata.iov.size, NVME_TX_DIRECTION_TO_DEVICE, req); if (status) { goto err; } } status = nvme_check_prinfo(ns, prinfo, slba, reftag); if (status) { goto err; } if (pract) { /* splice generated protection information into the buffer */ nvme_dif_pract_generate_dif(ns, ctx->data.bounce, ctx->data.iov.size, ctx->mdata.bounce, ctx->mdata.iov.size, apptag, &reftag); } else { status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size, ctx->mdata.bounce, ctx->mdata.iov.size, prinfo, slba, apptag, appmask, &reftag); if (status) { goto err; } } block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size, BLOCK_ACCT_WRITE); req->aiocb = blk_aio_pwritev(ns->blkconf.blk, offset, &ctx->data.iov, 0, nvme_dif_rw_mdata_out_cb, ctx); return NVME_NO_COMPLETE; err: qemu_iovec_destroy(&ctx->data.iov); g_free(ctx->data.bounce); qemu_iovec_destroy(&ctx->mdata.iov); g_free(ctx->mdata.bounce); g_free(ctx); return status; }