2013-03-09 13:21:05 +04:00
|
|
|
/*
|
2016-06-01 11:23:39 +03:00
|
|
|
* QEMU TX packets abstractions
|
2013-03-09 13:21:05 +04:00
|
|
|
*
|
|
|
|
* Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com)
|
|
|
|
*
|
|
|
|
* Developed by Daynix Computing LTD (http://www.daynix.com)
|
|
|
|
*
|
|
|
|
* Authors:
|
|
|
|
* Dmitry Fleytman <dmitry@daynix.com>
|
|
|
|
* Tamir Shomer <tamirs@daynix.com>
|
|
|
|
* Yan Vugenfirer <yan@daynix.com>
|
|
|
|
*
|
|
|
|
* This work is licensed under the terms of the GNU GPL, version 2 or later.
|
|
|
|
* See the COPYING file in the top-level directory.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
2016-06-06 19:56:37 +03:00
|
|
|
#include "qemu/osdep.h"
|
2016-06-01 11:23:39 +03:00
|
|
|
#include "net_tx_pkt.h"
|
2013-03-09 13:21:05 +04:00
|
|
|
#include "net/eth.h"
|
|
|
|
#include "net/checksum.h"
|
|
|
|
#include "net/tap.h"
|
|
|
|
#include "net/net.h"
|
2022-12-22 13:03:28 +03:00
|
|
|
#include "hw/pci/pci_device.h"
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
enum {
|
2016-06-01 11:23:39 +03:00
|
|
|
NET_TX_PKT_VHDR_FRAG = 0,
|
|
|
|
NET_TX_PKT_L2HDR_FRAG,
|
|
|
|
NET_TX_PKT_L3HDR_FRAG,
|
|
|
|
NET_TX_PKT_PL_START_FRAG
|
2013-03-09 13:21:05 +04:00
|
|
|
};
|
|
|
|
|
|
|
|
/* TX packet private context */
|
2016-06-01 11:23:39 +03:00
|
|
|
struct NetTxPkt {
|
2016-06-01 11:23:42 +03:00
|
|
|
PCIDevice *pci_dev;
|
|
|
|
|
2013-03-09 13:21:05 +04:00
|
|
|
struct virtio_net_hdr virt_hdr;
|
|
|
|
bool has_virt_hdr;
|
|
|
|
|
|
|
|
struct iovec *raw;
|
|
|
|
uint32_t raw_frags;
|
|
|
|
uint32_t max_raw_frags;
|
|
|
|
|
|
|
|
struct iovec *vec;
|
|
|
|
|
|
|
|
uint8_t l2_hdr[ETH_MAX_L2_HDR_LEN];
|
2016-06-01 11:23:41 +03:00
|
|
|
uint8_t l3_hdr[ETH_MAX_IP_DGRAM_LEN];
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
uint32_t payload_len;
|
|
|
|
|
|
|
|
uint32_t payload_frags;
|
|
|
|
uint32_t max_payload_frags;
|
|
|
|
|
|
|
|
uint16_t hdr_len;
|
|
|
|
eth_pkt_types_e packet_type;
|
|
|
|
uint8_t l4proto;
|
2016-06-01 11:23:41 +03:00
|
|
|
|
|
|
|
bool is_loopback;
|
2013-03-09 13:21:05 +04:00
|
|
|
};
|
|
|
|
|
2016-06-01 11:23:42 +03:00
|
|
|
void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev,
|
|
|
|
uint32_t max_frags, bool has_virt_hdr)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
2016-06-01 11:23:39 +03:00
|
|
|
struct NetTxPkt *p = g_malloc0(sizeof *p);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
2016-06-01 11:23:42 +03:00
|
|
|
p->pci_dev = pci_dev;
|
|
|
|
|
2016-08-16 14:28:01 +03:00
|
|
|
p->vec = g_new(struct iovec, max_frags + NET_TX_PKT_PL_START_FRAG);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
2016-08-16 14:28:01 +03:00
|
|
|
p->raw = g_new(struct iovec, max_frags);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
p->max_payload_frags = max_frags;
|
|
|
|
p->max_raw_frags = max_frags;
|
|
|
|
p->has_virt_hdr = has_virt_hdr;
|
2016-06-01 11:23:39 +03:00
|
|
|
p->vec[NET_TX_PKT_VHDR_FRAG].iov_base = &p->virt_hdr;
|
|
|
|
p->vec[NET_TX_PKT_VHDR_FRAG].iov_len =
|
2013-03-09 13:21:05 +04:00
|
|
|
p->has_virt_hdr ? sizeof p->virt_hdr : 0;
|
2016-06-01 11:23:39 +03:00
|
|
|
p->vec[NET_TX_PKT_L2HDR_FRAG].iov_base = &p->l2_hdr;
|
2016-06-01 11:23:41 +03:00
|
|
|
p->vec[NET_TX_PKT_L3HDR_FRAG].iov_base = &p->l3_hdr;
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
*pkt = p;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
void net_tx_pkt_uninit(struct NetTxPkt *pkt)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
|
|
|
if (pkt) {
|
|
|
|
g_free(pkt->vec);
|
|
|
|
g_free(pkt->raw);
|
|
|
|
g_free(pkt);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
void net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt *pkt)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
|
|
|
uint16_t csum;
|
|
|
|
assert(pkt);
|
|
|
|
struct ip_header *ip_hdr;
|
2016-06-01 11:23:39 +03:00
|
|
|
ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base;
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
ip_hdr->ip_len = cpu_to_be16(pkt->payload_len +
|
2016-06-01 11:23:39 +03:00
|
|
|
pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
ip_hdr->ip_sum = 0;
|
|
|
|
csum = net_raw_checksum((uint8_t *)ip_hdr,
|
2016-06-01 11:23:39 +03:00
|
|
|
pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len);
|
2013-03-09 13:21:05 +04:00
|
|
|
ip_hdr->ip_sum = cpu_to_be16(csum);
|
2016-06-01 11:23:41 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
void net_tx_pkt_update_ip_checksums(struct NetTxPkt *pkt)
|
|
|
|
{
|
|
|
|
uint16_t csum;
|
|
|
|
uint32_t cntr, cso;
|
|
|
|
assert(pkt);
|
|
|
|
uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN;
|
|
|
|
void *ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base;
|
|
|
|
|
|
|
|
if (pkt->payload_len + pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len >
|
|
|
|
ETH_MAX_IP_DGRAM_LEN) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 ||
|
|
|
|
gso_type == VIRTIO_NET_HDR_GSO_UDP) {
|
|
|
|
/* Calculate IP header checksum */
|
|
|
|
net_tx_pkt_update_ip_hdr_checksum(pkt);
|
|
|
|
|
|
|
|
/* Calculate IP pseudo header checksum */
|
|
|
|
cntr = eth_calc_ip4_pseudo_hdr_csum(ip_hdr, pkt->payload_len, &cso);
|
|
|
|
csum = cpu_to_be16(~net_checksum_finish(cntr));
|
|
|
|
} else if (gso_type == VIRTIO_NET_HDR_GSO_TCPV6) {
|
|
|
|
/* Calculate IP pseudo header checksum */
|
|
|
|
cntr = eth_calc_ip6_pseudo_hdr_csum(ip_hdr, pkt->payload_len,
|
|
|
|
IP_PROTO_TCP, &cso);
|
|
|
|
csum = cpu_to_be16(~net_checksum_finish(cntr));
|
|
|
|
} else {
|
|
|
|
return;
|
|
|
|
}
|
2013-03-09 13:21:05 +04:00
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
iov_from_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags,
|
2013-03-09 13:21:05 +04:00
|
|
|
pkt->virt_hdr.csum_offset, &csum, sizeof(csum));
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
static void net_tx_pkt_calculate_hdr_len(struct NetTxPkt *pkt)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
2016-06-01 11:23:39 +03:00
|
|
|
pkt->hdr_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len +
|
|
|
|
pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len;
|
2013-03-09 13:21:05 +04:00
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
static bool net_tx_pkt_parse_headers(struct NetTxPkt *pkt)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
|
|
|
struct iovec *l2_hdr, *l3_hdr;
|
|
|
|
size_t bytes_read;
|
|
|
|
size_t full_ip6hdr_len;
|
|
|
|
uint16_t l3_proto;
|
|
|
|
|
|
|
|
assert(pkt);
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
l2_hdr = &pkt->vec[NET_TX_PKT_L2HDR_FRAG];
|
|
|
|
l3_hdr = &pkt->vec[NET_TX_PKT_L3HDR_FRAG];
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 0, l2_hdr->iov_base,
|
|
|
|
ETH_MAX_L2_HDR_LEN);
|
2015-08-18 12:45:55 +03:00
|
|
|
if (bytes_read < sizeof(struct eth_header)) {
|
|
|
|
l2_hdr->iov_len = 0;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
l2_hdr->iov_len = sizeof(struct eth_header);
|
|
|
|
switch (be16_to_cpu(PKT_GET_ETH_HDR(l2_hdr->iov_base)->h_proto)) {
|
|
|
|
case ETH_P_VLAN:
|
|
|
|
l2_hdr->iov_len += sizeof(struct vlan_header);
|
|
|
|
break;
|
|
|
|
case ETH_P_DVLAN:
|
|
|
|
l2_hdr->iov_len += 2 * sizeof(struct vlan_header);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (bytes_read < l2_hdr->iov_len) {
|
2013-03-09 13:21:05 +04:00
|
|
|
l2_hdr->iov_len = 0;
|
2016-06-01 11:23:41 +03:00
|
|
|
l3_hdr->iov_len = 0;
|
|
|
|
pkt->packet_type = ETH_PKT_UCAST;
|
2013-03-09 13:21:05 +04:00
|
|
|
return false;
|
2016-06-01 11:23:41 +03:00
|
|
|
} else {
|
|
|
|
l2_hdr->iov_len = ETH_MAX_L2_HDR_LEN;
|
|
|
|
l2_hdr->iov_len = eth_get_l2_hdr_length(l2_hdr->iov_base);
|
|
|
|
pkt->packet_type = get_eth_packet_type(l2_hdr->iov_base);
|
2013-03-09 13:21:05 +04:00
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
l3_proto = eth_get_l3_proto(l2_hdr, 1, l2_hdr->iov_len);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
switch (l3_proto) {
|
|
|
|
case ETH_P_IP:
|
|
|
|
bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
|
|
|
|
l3_hdr->iov_base, sizeof(struct ip_header));
|
|
|
|
|
|
|
|
if (bytes_read < sizeof(struct ip_header)) {
|
|
|
|
l3_hdr->iov_len = 0;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
l3_hdr->iov_len = IP_HDR_GET_LEN(l3_hdr->iov_base);
|
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
if (l3_hdr->iov_len < sizeof(struct ip_header)) {
|
2013-03-09 13:21:05 +04:00
|
|
|
l3_hdr->iov_len = 0;
|
|
|
|
return false;
|
|
|
|
}
|
2016-06-01 11:23:41 +03:00
|
|
|
|
2018-02-09 22:03:40 +03:00
|
|
|
pkt->l4proto = IP_HDR_GET_P(l3_hdr->iov_base);
|
2016-06-01 11:23:41 +03:00
|
|
|
|
|
|
|
if (IP_HDR_GET_LEN(l3_hdr->iov_base) != sizeof(struct ip_header)) {
|
|
|
|
/* copy optional IPv4 header data if any*/
|
|
|
|
bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags,
|
|
|
|
l2_hdr->iov_len + sizeof(struct ip_header),
|
|
|
|
l3_hdr->iov_base + sizeof(struct ip_header),
|
|
|
|
l3_hdr->iov_len - sizeof(struct ip_header));
|
|
|
|
if (bytes_read < l3_hdr->iov_len - sizeof(struct ip_header)) {
|
|
|
|
l3_hdr->iov_len = 0;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-03-09 13:21:05 +04:00
|
|
|
break;
|
|
|
|
|
|
|
|
case ETH_P_IPV6:
|
2016-06-01 11:23:41 +03:00
|
|
|
{
|
|
|
|
eth_ip6_hdr_info hdrinfo;
|
|
|
|
|
2013-03-09 13:21:05 +04:00
|
|
|
if (!eth_parse_ipv6_hdr(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
|
2016-06-01 11:23:41 +03:00
|
|
|
&hdrinfo)) {
|
2013-03-09 13:21:05 +04:00
|
|
|
l3_hdr->iov_len = 0;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
pkt->l4proto = hdrinfo.l4proto;
|
|
|
|
full_ip6hdr_len = hdrinfo.full_hdr_len;
|
|
|
|
|
|
|
|
if (full_ip6hdr_len > ETH_MAX_IP_DGRAM_LEN) {
|
|
|
|
l3_hdr->iov_len = 0;
|
|
|
|
return false;
|
|
|
|
}
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
|
|
|
|
l3_hdr->iov_base, full_ip6hdr_len);
|
|
|
|
|
|
|
|
if (bytes_read < full_ip6hdr_len) {
|
|
|
|
l3_hdr->iov_len = 0;
|
|
|
|
return false;
|
|
|
|
} else {
|
|
|
|
l3_hdr->iov_len = full_ip6hdr_len;
|
|
|
|
}
|
|
|
|
break;
|
2016-06-01 11:23:41 +03:00
|
|
|
}
|
2013-03-09 13:21:05 +04:00
|
|
|
default:
|
|
|
|
l3_hdr->iov_len = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
net_tx_pkt_calculate_hdr_len(pkt);
|
2013-03-09 13:21:05 +04:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
static void net_tx_pkt_rebuild_payload(struct NetTxPkt *pkt)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
2016-06-01 11:23:41 +03:00
|
|
|
pkt->payload_len = iov_size(pkt->raw, pkt->raw_frags) - pkt->hdr_len;
|
2016-06-01 11:23:39 +03:00
|
|
|
pkt->payload_frags = iov_copy(&pkt->vec[NET_TX_PKT_PL_START_FRAG],
|
2013-03-09 13:21:05 +04:00
|
|
|
pkt->max_payload_frags,
|
|
|
|
pkt->raw, pkt->raw_frags,
|
2016-06-01 11:23:41 +03:00
|
|
|
pkt->hdr_len, pkt->payload_len);
|
|
|
|
}
|
2013-03-09 13:21:05 +04:00
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
bool net_tx_pkt_parse(struct NetTxPkt *pkt)
|
|
|
|
{
|
|
|
|
if (net_tx_pkt_parse_headers(pkt)) {
|
|
|
|
net_tx_pkt_rebuild_payload(pkt);
|
2013-03-09 13:21:05 +04:00
|
|
|
return true;
|
|
|
|
} else {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
struct virtio_net_hdr *net_tx_pkt_get_vhdr(struct NetTxPkt *pkt)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
|
|
|
assert(pkt);
|
|
|
|
return &pkt->virt_hdr;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
static uint8_t net_tx_pkt_get_gso_type(struct NetTxPkt *pkt,
|
2013-03-09 13:21:05 +04:00
|
|
|
bool tso_enable)
|
|
|
|
{
|
|
|
|
uint8_t rc = VIRTIO_NET_HDR_GSO_NONE;
|
|
|
|
uint16_t l3_proto;
|
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
l3_proto = eth_get_l3_proto(&pkt->vec[NET_TX_PKT_L2HDR_FRAG], 1,
|
2016-06-01 11:23:39 +03:00
|
|
|
pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
if (!tso_enable) {
|
|
|
|
goto func_exit;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
rc = eth_get_gso_type(l3_proto, pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
|
2013-03-09 13:21:05 +04:00
|
|
|
pkt->l4proto);
|
|
|
|
|
|
|
|
func_exit:
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
void net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable,
|
2013-03-09 13:21:05 +04:00
|
|
|
bool csum_enable, uint32_t gso_size)
|
|
|
|
{
|
|
|
|
struct tcp_hdr l4hdr;
|
|
|
|
assert(pkt);
|
|
|
|
|
|
|
|
/* csum has to be enabled if tso is. */
|
|
|
|
assert(csum_enable || !tso_enable);
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
pkt->virt_hdr.gso_type = net_tx_pkt_get_gso_type(pkt, tso_enable);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
switch (pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
|
|
|
|
case VIRTIO_NET_HDR_GSO_NONE:
|
|
|
|
pkt->virt_hdr.hdr_len = 0;
|
|
|
|
pkt->virt_hdr.gso_size = 0;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case VIRTIO_NET_HDR_GSO_UDP:
|
2016-06-01 11:23:41 +03:00
|
|
|
pkt->virt_hdr.gso_size = gso_size;
|
2013-03-09 13:21:05 +04:00
|
|
|
pkt->virt_hdr.hdr_len = pkt->hdr_len + sizeof(struct udp_header);
|
|
|
|
break;
|
|
|
|
|
|
|
|
case VIRTIO_NET_HDR_GSO_TCPV4:
|
|
|
|
case VIRTIO_NET_HDR_GSO_TCPV6:
|
2016-06-01 11:23:39 +03:00
|
|
|
iov_to_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags,
|
2013-03-09 13:21:05 +04:00
|
|
|
0, &l4hdr, sizeof(l4hdr));
|
|
|
|
pkt->virt_hdr.hdr_len = pkt->hdr_len + l4hdr.th_off * sizeof(uint32_t);
|
2016-06-01 11:23:41 +03:00
|
|
|
pkt->virt_hdr.gso_size = gso_size;
|
2013-03-09 13:21:05 +04:00
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
2013-07-25 20:21:28 +04:00
|
|
|
g_assert_not_reached();
|
2013-03-09 13:21:05 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
if (csum_enable) {
|
|
|
|
switch (pkt->l4proto) {
|
|
|
|
case IP_PROTO_TCP:
|
|
|
|
pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
|
|
|
|
pkt->virt_hdr.csum_start = pkt->hdr_len;
|
|
|
|
pkt->virt_hdr.csum_offset = offsetof(struct tcp_hdr, th_sum);
|
|
|
|
break;
|
|
|
|
case IP_PROTO_UDP:
|
|
|
|
pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
|
|
|
|
pkt->virt_hdr.csum_start = pkt->hdr_len;
|
|
|
|
pkt->virt_hdr.csum_offset = offsetof(struct udp_hdr, uh_sum);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt,
|
|
|
|
uint16_t vlan, uint16_t vlan_ethtype)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
|
|
|
bool is_new;
|
|
|
|
assert(pkt);
|
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
eth_setup_vlan_headers_ex(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base,
|
|
|
|
vlan, vlan_ethtype, &is_new);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
/* update l2hdrlen */
|
|
|
|
if (is_new) {
|
|
|
|
pkt->hdr_len += sizeof(struct vlan_header);
|
2016-06-01 11:23:39 +03:00
|
|
|
pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len +=
|
2013-03-09 13:21:05 +04:00
|
|
|
sizeof(struct vlan_header);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, hwaddr pa,
|
2013-03-09 13:21:05 +04:00
|
|
|
size_t len)
|
|
|
|
{
|
|
|
|
hwaddr mapped_len = 0;
|
|
|
|
struct iovec *ventry;
|
|
|
|
assert(pkt);
|
2020-08-01 19:42:38 +03:00
|
|
|
|
|
|
|
if (pkt->raw_frags >= pkt->max_raw_frags) {
|
|
|
|
return false;
|
|
|
|
}
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
if (!len) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
ventry = &pkt->raw[pkt->raw_frags];
|
|
|
|
mapped_len = len;
|
|
|
|
|
2016-06-01 11:23:42 +03:00
|
|
|
ventry->iov_base = pci_dma_map(pkt->pci_dev, pa,
|
|
|
|
&mapped_len, DMA_DIRECTION_TO_DEVICE);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
if ((ventry->iov_base != NULL) && (len == mapped_len)) {
|
|
|
|
ventry->iov_len = mapped_len;
|
|
|
|
pkt->raw_frags++;
|
|
|
|
return true;
|
|
|
|
} else {
|
2013-03-09 13:21:05 +04:00
|
|
|
return false;
|
|
|
|
}
|
2016-06-01 11:23:41 +03:00
|
|
|
}
|
2013-03-09 13:21:05 +04:00
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
bool net_tx_pkt_has_fragments(struct NetTxPkt *pkt)
|
|
|
|
{
|
|
|
|
return pkt->raw_frags > 0;
|
2013-03-09 13:21:05 +04:00
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
eth_pkt_types_e net_tx_pkt_get_packet_type(struct NetTxPkt *pkt)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
|
|
|
assert(pkt);
|
|
|
|
|
|
|
|
return pkt->packet_type;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
size_t net_tx_pkt_get_total_len(struct NetTxPkt *pkt)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
|
|
|
assert(pkt);
|
|
|
|
|
|
|
|
return pkt->hdr_len + pkt->payload_len;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
void net_tx_pkt_dump(struct NetTxPkt *pkt)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
2016-06-01 11:23:39 +03:00
|
|
|
#ifdef NET_TX_PKT_DEBUG
|
2013-03-09 13:21:05 +04:00
|
|
|
assert(pkt);
|
|
|
|
|
|
|
|
printf("TX PKT: hdr_len: %d, pkt_type: 0x%X, l2hdr_len: %lu, "
|
|
|
|
"l3hdr_len: %lu, payload_len: %u\n", pkt->hdr_len, pkt->packet_type,
|
2016-06-01 11:23:39 +03:00
|
|
|
pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len,
|
|
|
|
pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len, pkt->payload_len);
|
2013-03-09 13:21:05 +04:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
void net_tx_pkt_reset(struct NetTxPkt *pkt)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/* no assert, as reset can be called before tx_pkt_init */
|
|
|
|
if (!pkt) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
memset(&pkt->virt_hdr, 0, sizeof(pkt->virt_hdr));
|
|
|
|
|
|
|
|
assert(pkt->vec);
|
2016-06-01 11:23:41 +03:00
|
|
|
|
2013-03-09 13:21:05 +04:00
|
|
|
pkt->payload_len = 0;
|
|
|
|
pkt->payload_frags = 0;
|
|
|
|
|
2021-07-15 22:32:19 +03:00
|
|
|
if (pkt->max_raw_frags > 0) {
|
|
|
|
assert(pkt->raw);
|
|
|
|
for (i = 0; i < pkt->raw_frags; i++) {
|
|
|
|
assert(pkt->raw[i].iov_base);
|
|
|
|
pci_dma_unmap(pkt->pci_dev, pkt->raw[i].iov_base,
|
|
|
|
pkt->raw[i].iov_len, DMA_DIRECTION_TO_DEVICE, 0);
|
|
|
|
}
|
2013-03-09 13:21:05 +04:00
|
|
|
}
|
|
|
|
pkt->raw_frags = 0;
|
|
|
|
|
|
|
|
pkt->hdr_len = 0;
|
|
|
|
pkt->l4proto = 0;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
2016-06-01 11:23:39 +03:00
|
|
|
struct iovec *iov = &pkt->vec[NET_TX_PKT_L2HDR_FRAG];
|
2013-03-09 13:21:05 +04:00
|
|
|
uint32_t csum_cntr;
|
|
|
|
uint16_t csum = 0;
|
2016-06-01 11:23:41 +03:00
|
|
|
uint32_t cso;
|
2013-03-09 13:21:05 +04:00
|
|
|
/* num of iovec without vhdr */
|
2016-06-01 11:23:39 +03:00
|
|
|
uint32_t iov_len = pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1;
|
2013-03-09 13:21:05 +04:00
|
|
|
uint16_t csl;
|
|
|
|
size_t csum_offset = pkt->virt_hdr.csum_start + pkt->virt_hdr.csum_offset;
|
2020-06-29 04:17:59 +03:00
|
|
|
uint16_t l3_proto = eth_get_l3_proto(iov, 1, iov->iov_len);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
/* Put zero to checksum field */
|
|
|
|
iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum);
|
|
|
|
|
|
|
|
/* Calculate L4 TCP/UDP checksum */
|
|
|
|
csl = pkt->payload_len;
|
|
|
|
|
2020-06-29 04:17:59 +03:00
|
|
|
csum_cntr = 0;
|
|
|
|
cso = 0;
|
2013-03-09 13:21:05 +04:00
|
|
|
/* add pseudo header to csum */
|
2020-06-29 04:17:59 +03:00
|
|
|
if (l3_proto == ETH_P_IP) {
|
|
|
|
csum_cntr = eth_calc_ip4_pseudo_hdr_csum(
|
|
|
|
pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
|
|
|
|
csl, &cso);
|
|
|
|
} else if (l3_proto == ETH_P_IPV6) {
|
|
|
|
csum_cntr = eth_calc_ip6_pseudo_hdr_csum(
|
|
|
|
pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
|
|
|
|
csl, pkt->l4proto, &cso);
|
|
|
|
}
|
2016-06-01 11:23:41 +03:00
|
|
|
|
|
|
|
/* data checksum */
|
|
|
|
csum_cntr +=
|
|
|
|
net_checksum_add_iov(iov, iov_len, pkt->virt_hdr.csum_start, csl, cso);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
/* Put the checksum obtained into the packet */
|
2017-11-16 17:06:06 +03:00
|
|
|
csum = cpu_to_be16(net_checksum_finish_nozero(csum_cntr));
|
2013-03-09 13:21:05 +04:00
|
|
|
iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum);
|
|
|
|
}
|
|
|
|
|
|
|
|
enum {
|
2016-06-01 11:23:39 +03:00
|
|
|
NET_TX_PKT_FRAGMENT_L2_HDR_POS = 0,
|
|
|
|
NET_TX_PKT_FRAGMENT_L3_HDR_POS,
|
|
|
|
NET_TX_PKT_FRAGMENT_HEADER_NUM
|
2013-03-09 13:21:05 +04:00
|
|
|
};
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
#define NET_MAX_FRAG_SG_LIST (64)
|
2013-03-09 13:21:05 +04:00
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
static size_t net_tx_pkt_fetch_fragment(struct NetTxPkt *pkt,
|
2013-03-09 13:21:05 +04:00
|
|
|
int *src_idx, size_t *src_offset, struct iovec *dst, int *dst_idx)
|
|
|
|
{
|
|
|
|
size_t fetched = 0;
|
|
|
|
struct iovec *src = pkt->vec;
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
*dst_idx = NET_TX_PKT_FRAGMENT_HEADER_NUM;
|
2013-03-09 13:21:05 +04:00
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
while (fetched < IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size)) {
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
/* no more place in fragment iov */
|
2016-06-01 11:23:39 +03:00
|
|
|
if (*dst_idx == NET_MAX_FRAG_SG_LIST) {
|
2013-03-09 13:21:05 +04:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* no more data in iovec */
|
2016-06-01 11:23:39 +03:00
|
|
|
if (*src_idx == (pkt->payload_frags + NET_TX_PKT_PL_START_FRAG)) {
|
2013-03-09 13:21:05 +04:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
dst[*dst_idx].iov_base = src[*src_idx].iov_base + *src_offset;
|
|
|
|
dst[*dst_idx].iov_len = MIN(src[*src_idx].iov_len - *src_offset,
|
2016-06-01 11:23:41 +03:00
|
|
|
IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size) - fetched);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
*src_offset += dst[*dst_idx].iov_len;
|
|
|
|
fetched += dst[*dst_idx].iov_len;
|
|
|
|
|
|
|
|
if (*src_offset == src[*src_idx].iov_len) {
|
|
|
|
*src_offset = 0;
|
|
|
|
(*src_idx)++;
|
|
|
|
}
|
|
|
|
|
|
|
|
(*dst_idx)++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return fetched;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
static inline void net_tx_pkt_sendv(struct NetTxPkt *pkt,
|
|
|
|
NetClientState *nc, const struct iovec *iov, int iov_cnt)
|
|
|
|
{
|
|
|
|
if (pkt->is_loopback) {
|
2021-02-24 08:27:52 +03:00
|
|
|
qemu_receive_packet_iov(nc, iov, iov_cnt);
|
2016-06-01 11:23:41 +03:00
|
|
|
} else {
|
|
|
|
qemu_sendv_packet(nc, iov, iov_cnt);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
static bool net_tx_pkt_do_sw_fragmentation(struct NetTxPkt *pkt,
|
2013-03-09 13:21:05 +04:00
|
|
|
NetClientState *nc)
|
|
|
|
{
|
2016-06-01 11:23:39 +03:00
|
|
|
struct iovec fragment[NET_MAX_FRAG_SG_LIST];
|
2013-03-09 13:21:05 +04:00
|
|
|
size_t fragment_len = 0;
|
|
|
|
bool more_frags = false;
|
|
|
|
|
|
|
|
/* some pointers for shorter code */
|
|
|
|
void *l2_iov_base, *l3_iov_base;
|
|
|
|
size_t l2_iov_len, l3_iov_len;
|
2016-06-01 11:23:39 +03:00
|
|
|
int src_idx = NET_TX_PKT_PL_START_FRAG, dst_idx;
|
2013-03-09 13:21:05 +04:00
|
|
|
size_t src_offset = 0;
|
|
|
|
size_t fragment_offset = 0;
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
l2_iov_base = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base;
|
|
|
|
l2_iov_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len;
|
|
|
|
l3_iov_base = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base;
|
|
|
|
l3_iov_len = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len;
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
/* Copy headers */
|
2016-06-01 11:23:39 +03:00
|
|
|
fragment[NET_TX_PKT_FRAGMENT_L2_HDR_POS].iov_base = l2_iov_base;
|
|
|
|
fragment[NET_TX_PKT_FRAGMENT_L2_HDR_POS].iov_len = l2_iov_len;
|
|
|
|
fragment[NET_TX_PKT_FRAGMENT_L3_HDR_POS].iov_base = l3_iov_base;
|
|
|
|
fragment[NET_TX_PKT_FRAGMENT_L3_HDR_POS].iov_len = l3_iov_len;
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
|
|
|
|
/* Put as much data as possible and send */
|
|
|
|
do {
|
2016-06-01 11:23:39 +03:00
|
|
|
fragment_len = net_tx_pkt_fetch_fragment(pkt, &src_idx, &src_offset,
|
2013-03-09 13:21:05 +04:00
|
|
|
fragment, &dst_idx);
|
|
|
|
|
|
|
|
more_frags = (fragment_offset + fragment_len < pkt->payload_len);
|
|
|
|
|
|
|
|
eth_setup_ip4_fragmentation(l2_iov_base, l2_iov_len, l3_iov_base,
|
|
|
|
l3_iov_len, fragment_len, fragment_offset, more_frags);
|
|
|
|
|
|
|
|
eth_fix_ip4_checksum(l3_iov_base, l3_iov_len);
|
|
|
|
|
2016-06-01 11:23:41 +03:00
|
|
|
net_tx_pkt_sendv(pkt, nc, fragment, dst_idx);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
fragment_offset += fragment_len;
|
|
|
|
|
2016-08-04 10:30:14 +03:00
|
|
|
} while (fragment_len && more_frags);
|
2013-03-09 13:21:05 +04:00
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc)
|
2013-03-09 13:21:05 +04:00
|
|
|
{
|
|
|
|
assert(pkt);
|
|
|
|
|
|
|
|
if (!pkt->has_virt_hdr &&
|
|
|
|
pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
|
2016-06-01 11:23:39 +03:00
|
|
|
net_tx_pkt_do_sw_csum(pkt);
|
2013-03-09 13:21:05 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Since underlying infrastructure does not support IP datagrams longer
|
|
|
|
* than 64K we should drop such packets and don't even try to send
|
|
|
|
*/
|
|
|
|
if (VIRTIO_NET_HDR_GSO_NONE != pkt->virt_hdr.gso_type) {
|
|
|
|
if (pkt->payload_len >
|
|
|
|
ETH_MAX_IP_DGRAM_LEN -
|
2016-06-01 11:23:39 +03:00
|
|
|
pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len) {
|
2013-03-09 13:21:05 +04:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (pkt->has_virt_hdr ||
|
|
|
|
pkt->virt_hdr.gso_type == VIRTIO_NET_HDR_GSO_NONE) {
|
2020-07-16 06:53:24 +03:00
|
|
|
net_tx_pkt_fix_ip6_payload_len(pkt);
|
2016-06-01 11:23:41 +03:00
|
|
|
net_tx_pkt_sendv(pkt, nc, pkt->vec,
|
2016-06-01 11:23:39 +03:00
|
|
|
pkt->payload_frags + NET_TX_PKT_PL_START_FRAG);
|
2013-03-09 13:21:05 +04:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-06-01 11:23:39 +03:00
|
|
|
return net_tx_pkt_do_sw_fragmentation(pkt, nc);
|
2013-03-09 13:21:05 +04:00
|
|
|
}
|
2016-06-01 11:23:41 +03:00
|
|
|
|
|
|
|
bool net_tx_pkt_send_loopback(struct NetTxPkt *pkt, NetClientState *nc)
|
|
|
|
{
|
|
|
|
bool res;
|
|
|
|
|
|
|
|
pkt->is_loopback = true;
|
|
|
|
res = net_tx_pkt_send(pkt, nc);
|
|
|
|
pkt->is_loopback = false;
|
|
|
|
|
|
|
|
return res;
|
|
|
|
}
|
2020-07-16 06:53:24 +03:00
|
|
|
|
|
|
|
void net_tx_pkt_fix_ip6_payload_len(struct NetTxPkt *pkt)
|
|
|
|
{
|
|
|
|
struct iovec *l2 = &pkt->vec[NET_TX_PKT_L2HDR_FRAG];
|
|
|
|
if (eth_get_l3_proto(l2, 1, l2->iov_len) == ETH_P_IPV6) {
|
|
|
|
struct ip6_header *ip6 = (struct ip6_header *) pkt->l3_hdr;
|
|
|
|
/*
|
|
|
|
* TODO: if qemu would support >64K packets - add jumbo option check
|
|
|
|
* something like that:
|
|
|
|
* 'if (ip6->ip6_plen == 0 && !has_jumbo_option(ip6)) {'
|
|
|
|
*/
|
|
|
|
if (ip6->ip6_plen == 0) {
|
|
|
|
if (pkt->payload_len <= ETH_MAX_IP_DGRAM_LEN) {
|
|
|
|
ip6->ip6_plen = htons(pkt->payload_len);
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* TODO: if qemu would support >64K packets
|
|
|
|
* add jumbo option for packets greater then 65,535 bytes
|
|
|
|
*/
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|