qemu/tools/ebpf/rss.bpf.c

/*
 * eBPF RSS program
 *
 * Developed by Daynix Computing LTD (http://www.daynix.com)
 *
 * Authors:
 *  Andrew Melnychenko <andrew@daynix.com>
 *  Yuri Benditovich <yuri.benditovich@daynix.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * Prepare:
 * Requires llvm, clang, bpftool, linux kernel tree
 *
 * Build rss.bpf.skeleton.h:
 * make -f Makefile.ebpf clean all
 */

#include <stddef.h>
#include <stdbool.h>
#include <linux/bpf.h>

#include <linux/in.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/ipv6.h>

#include <linux/udp.h>
#include <linux/tcp.h>

#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>
#include <linux/virtio_net.h>

#define INDIRECTION_TABLE_SIZE 128
#define HASH_CALCULATION_BUFFER_SIZE 36

struct rss_config_t {
    __u8 redirect;
    __u8 populate_hash;
    __u32 hash_types;
    __u16 indirections_len;
    __u16 default_queue;
} __attribute__((packed));

struct toeplitz_key_data_t {
    __u32 leftmost_32_bits;
    __u8 next_byte[HASH_CALCULATION_BUFFER_SIZE];
};

struct packet_hash_info_t {
    __u8 is_ipv4;
    __u8 is_ipv6;
    __u8 is_udp;
    __u8 is_tcp;
    __u8 is_ipv6_ext_src;
    __u8 is_ipv6_ext_dst;
    __u8 is_fragmented;

    __u16 src_port;
    __u16 dst_port;

    union {
        struct {
            __be32 in_src;
            __be32 in_dst;
        };

        struct {
            struct in6_addr in6_src;
            struct in6_addr in6_dst;
            struct in6_addr in6_ext_src;
            struct in6_addr in6_ext_dst;
        };
    };
};

struct {
    __uint(type, BPF_MAP_TYPE_ARRAY);
    __uint(key_size, sizeof(__u32));
    __uint(value_size, sizeof(struct rss_config_t));
    __uint(max_entries, 1);
    __uint(map_flags, BPF_F_MMAPABLE);
} tap_rss_map_configurations SEC(".maps");

struct {
    __uint(type, BPF_MAP_TYPE_ARRAY);
    __uint(key_size, sizeof(__u32));
    __uint(value_size, sizeof(struct toeplitz_key_data_t));
    __uint(max_entries, 1);
    __uint(map_flags, BPF_F_MMAPABLE);
} tap_rss_map_toeplitz_key SEC(".maps");

struct {
    __uint(type, BPF_MAP_TYPE_ARRAY);
    __uint(key_size, sizeof(__u32));
    __uint(value_size, sizeof(__u16));
    __uint(max_entries, INDIRECTION_TABLE_SIZE);
    __uint(map_flags, BPF_F_MMAPABLE);
} tap_rss_map_indirection_table SEC(".maps");

static inline void net_rx_rss_add_chunk(__u8 *rss_input, size_t *bytes_written,
                                        const void *ptr, size_t size) {
    __builtin_memcpy(&rss_input[*bytes_written], ptr, size);
    *bytes_written += size;
}

static inline
void net_toeplitz_add(__u32 *result,
                      __u8 *input,
                      __u32 len
        , struct toeplitz_key_data_t *key) {

    __u32 accumulator = *result;
    __u32 leftmost_32_bits = key->leftmost_32_bits;
    __u32 byte;

    for (byte = 0; byte < HASH_CALCULATION_BUFFER_SIZE; byte++) {
        __u8 input_byte = input[byte];
        __u8 key_byte = key->next_byte[byte];
        __u8 bit;

        for (bit = 0; bit < 8; bit++) {
            if (input_byte & (1 << 7)) {
                accumulator ^= leftmost_32_bits;
            }

            leftmost_32_bits =
                    (leftmost_32_bits << 1) | ((key_byte & (1 << 7)) >> 7);

            input_byte <<= 1;
            key_byte <<= 1;
        }
    }

    *result = accumulator;
}


static inline int ip6_extension_header_type(__u8 hdr_type)
{
    switch (hdr_type) {
    case IPPROTO_HOPOPTS:
    case IPPROTO_ROUTING:
    case IPPROTO_FRAGMENT:
    case IPPROTO_ICMPV6:
    case IPPROTO_NONE:
    case IPPROTO_DSTOPTS:
    case IPPROTO_MH:
        return 1;
    default:
        return 0;
    }
}
/*
 * According to
 * https://www.iana.org/assignments/ipv6-parameters/ipv6-parameters.xhtml
 * we expect that there are would be no more than 11 extensions in IPv6 header,
 * also there is 27 TLV options for Destination and Hop-by-hop extensions.
 * Need to choose reasonable amount of maximum extensions/options we may
 * check to find ext src/dst.
 */
#define IP6_EXTENSIONS_COUNT 11
#define IP6_OPTIONS_COUNT 30

static inline int parse_ipv6_ext(struct __sk_buff *skb,
        struct packet_hash_info_t *info,
        __u8 *l4_protocol, size_t *l4_offset)
{
    int err = 0;

    if (!ip6_extension_header_type(*l4_protocol)) {
        return 0;
    }

    struct ipv6_opt_hdr ext_hdr = {};

    for (unsigned int i = 0; i < IP6_EXTENSIONS_COUNT; ++i) {

        err = bpf_skb_load_bytes_relative(skb, *l4_offset, &ext_hdr,
                                    sizeof(ext_hdr), BPF_HDR_START_NET);
        if (err) {
            goto error;
        }

        if (*l4_protocol == IPPROTO_ROUTING) {
            struct ipv6_rt_hdr ext_rt = {};

            err = bpf_skb_load_bytes_relative(skb, *l4_offset, &ext_rt,
                                        sizeof(ext_rt), BPF_HDR_START_NET);
            if (err) {
                goto error;
            }

            if ((ext_rt.type == IPV6_SRCRT_TYPE_2) &&
                    (ext_rt.hdrlen == sizeof(struct in6_addr) / 8) &&
                    (ext_rt.segments_left == 1)) {

                err = bpf_skb_load_bytes_relative(skb,
                    *l4_offset + offsetof(struct rt2_hdr, addr),
                    &info->in6_ext_dst, sizeof(info->in6_ext_dst),
                    BPF_HDR_START_NET);
                if (err) {
                    goto error;
                }

                info->is_ipv6_ext_dst = 1;
            }

        } else if (*l4_protocol == IPPROTO_DSTOPTS) {
            struct ipv6_opt_t {
                __u8 type;
                __u8 length;
            } __attribute__((packed)) opt = {};

            size_t opt_offset = sizeof(ext_hdr);

            for (unsigned int j = 0; j < IP6_OPTIONS_COUNT; ++j) {
                err = bpf_skb_load_bytes_relative(skb, *l4_offset + opt_offset,
                                        &opt, sizeof(opt), BPF_HDR_START_NET);
                if (err) {
                    goto error;
                }

                if (opt.type == IPV6_TLV_HAO) {
                    err = bpf_skb_load_bytes_relative(skb,
                        *l4_offset + opt_offset
                        + offsetof(struct ipv6_destopt_hao, addr),
                        &info->in6_ext_src, sizeof(info->in6_ext_src),
                        BPF_HDR_START_NET);
                    if (err) {
                        goto error;
                    }

                    info->is_ipv6_ext_src = 1;
                    break;
                }

                opt_offset += (opt.type == IPV6_TLV_PAD1) ?
                              1 : opt.length + sizeof(opt);

                if (opt_offset + 1 >= ext_hdr.hdrlen * 8) {
                    break;
                }
            }
        } else if (*l4_protocol == IPPROTO_FRAGMENT) {
            info->is_fragmented = true;
        }

        *l4_protocol = ext_hdr.nexthdr;
        *l4_offset += (ext_hdr.hdrlen + 1) * 8;

        if (!ip6_extension_header_type(ext_hdr.nexthdr)) {
            return 0;
        }
    }

    return 0;
error:
    return err;
}

static __be16 parse_eth_type(struct __sk_buff *skb)
{
    unsigned int offset = 12;
    __be16 ret = 0;
    int err = 0;

    err = bpf_skb_load_bytes_relative(skb, offset, &ret, sizeof(ret),
                                BPF_HDR_START_MAC);
    if (err) {
        return 0;
    }

    switch (bpf_ntohs(ret)) {
    case ETH_P_8021AD:
        offset += 4;
    case ETH_P_8021Q:
        offset += 4;
        err = bpf_skb_load_bytes_relative(skb, offset, &ret, sizeof(ret),
                                    BPF_HDR_START_MAC);
    default:
        break;
    }

    if (err) {
        return 0;
    }

    return ret;
}

static inline int parse_packet(struct __sk_buff *skb,
        struct packet_hash_info_t *info)
{
    int err = 0;

    if (!info || !skb) {
        return -1;
    }

    size_t l4_offset = 0;
    __u8 l4_protocol = 0;
    __u16 l3_protocol = bpf_ntohs(parse_eth_type(skb));
    if (l3_protocol == 0) {
        err = -1;
        goto error;
    }

    if (l3_protocol == ETH_P_IP) {
        info->is_ipv4 = 1;

        struct iphdr ip = {};
        err = bpf_skb_load_bytes_relative(skb, 0, &ip, sizeof(ip),
                                    BPF_HDR_START_NET);
        if (err) {
            goto error;
        }

        info->in_src = ip.saddr;
        info->in_dst = ip.daddr;
        info->is_fragmented = !!(bpf_ntohs(ip.frag_off) & (0x2000 | 0x1fff));

        l4_protocol = ip.protocol;
        l4_offset = ip.ihl * 4;
    } else if (l3_protocol == ETH_P_IPV6) {
        info->is_ipv6 = 1;

        struct ipv6hdr ip6 = {};
        err = bpf_skb_load_bytes_relative(skb, 0, &ip6, sizeof(ip6),
                                    BPF_HDR_START_NET);
        if (err) {
            goto error;
        }

        info->in6_src = ip6.saddr;
        info->in6_dst = ip6.daddr;

        l4_protocol = ip6.nexthdr;
        l4_offset = sizeof(ip6);

        err = parse_ipv6_ext(skb, info, &l4_protocol, &l4_offset);
        if (err) {
            goto error;
        }
    }

    if (l4_protocol != 0 && !info->is_fragmented) {
        if (l4_protocol == IPPROTO_TCP) {
            info->is_tcp = 1;

            struct tcphdr tcp = {};
            err = bpf_skb_load_bytes_relative(skb, l4_offset, &tcp, sizeof(tcp),
                                        BPF_HDR_START_NET);
            if (err) {
                goto error;
            }

            info->src_port = tcp.source;
            info->dst_port = tcp.dest;
        } else if (l4_protocol == IPPROTO_UDP) { /* TODO: add udplite? */
            info->is_udp = 1;

            struct udphdr udp = {};
            err = bpf_skb_load_bytes_relative(skb, l4_offset, &udp, sizeof(udp),
                                        BPF_HDR_START_NET);
            if (err) {
                goto error;
            }

            info->src_port = udp.source;
            info->dst_port = udp.dest;
        }
    }

    return 0;

error:
    return err;
}

static inline bool calculate_rss_hash(struct __sk_buff *skb,
                                      struct rss_config_t *config,
                                      struct toeplitz_key_data_t *toe,
                                      __u32 *result)
{
    __u8 rss_input[HASH_CALCULATION_BUFFER_SIZE] = {};
    size_t bytes_written = 0;
    int err = 0;
    struct packet_hash_info_t packet_info = {};

    err = parse_packet(skb, &packet_info);
    if (err) {
        return false;
    }

    if (packet_info.is_ipv4) {
        if (packet_info.is_tcp &&
            config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) {

            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.in_src,
                                 sizeof(packet_info.in_src));
            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.in_dst,
                                 sizeof(packet_info.in_dst));
            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.src_port,
                                 sizeof(packet_info.src_port));
            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.dst_port,
                                 sizeof(packet_info.dst_port));
        } else if (packet_info.is_udp &&
                   config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) {

            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.in_src,
                                 sizeof(packet_info.in_src));
            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.in_dst,
                                 sizeof(packet_info.in_dst));
            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.src_port,
                                 sizeof(packet_info.src_port));
            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.dst_port,
                                 sizeof(packet_info.dst_port));
        } else if (config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.in_src,
                                 sizeof(packet_info.in_src));
            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.in_dst,
                                 sizeof(packet_info.in_dst));
        }
    } else if (packet_info.is_ipv6) {
        if (packet_info.is_tcp &&
            config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) {

            if (packet_info.is_ipv6_ext_src &&
                config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) {

                net_rx_rss_add_chunk(rss_input, &bytes_written,
                                     &packet_info.in6_ext_src,
                                     sizeof(packet_info.in6_ext_src));
            } else {
                net_rx_rss_add_chunk(rss_input, &bytes_written,
                                     &packet_info.in6_src,
                                     sizeof(packet_info.in6_src));
            }
            if (packet_info.is_ipv6_ext_dst &&
                config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) {

                net_rx_rss_add_chunk(rss_input, &bytes_written,
                                     &packet_info.in6_ext_dst,
                                     sizeof(packet_info.in6_ext_dst));
            } else {
                net_rx_rss_add_chunk(rss_input, &bytes_written,
                                     &packet_info.in6_dst,
                                     sizeof(packet_info.in6_dst));
            }
            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.src_port,
                                 sizeof(packet_info.src_port));
            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.dst_port,
                                 sizeof(packet_info.dst_port));
        } else if (packet_info.is_udp &&
                   config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) {

            if (packet_info.is_ipv6_ext_src &&
               config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) {

                net_rx_rss_add_chunk(rss_input, &bytes_written,
                                     &packet_info.in6_ext_src,
                                     sizeof(packet_info.in6_ext_src));
            } else {
                net_rx_rss_add_chunk(rss_input, &bytes_written,
                                     &packet_info.in6_src,
                                     sizeof(packet_info.in6_src));
            }
            if (packet_info.is_ipv6_ext_dst &&
               config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) {

                net_rx_rss_add_chunk(rss_input, &bytes_written,
                                     &packet_info.in6_ext_dst,
                                     sizeof(packet_info.in6_ext_dst));
            } else {
                net_rx_rss_add_chunk(rss_input, &bytes_written,
                                     &packet_info.in6_dst,
                                     sizeof(packet_info.in6_dst));
            }

            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.src_port,
                                 sizeof(packet_info.src_port));
            net_rx_rss_add_chunk(rss_input, &bytes_written,
                                 &packet_info.dst_port,
                                 sizeof(packet_info.dst_port));

        } else if (config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
            if (packet_info.is_ipv6_ext_src &&
               config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) {

                net_rx_rss_add_chunk(rss_input, &bytes_written,
                                     &packet_info.in6_ext_src,
                                     sizeof(packet_info.in6_ext_src));
            } else {
                net_rx_rss_add_chunk(rss_input, &bytes_written,
                                     &packet_info.in6_src,
                                     sizeof(packet_info.in6_src));
            }
            if (packet_info.is_ipv6_ext_dst &&
                config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) {

                net_rx_rss_add_chunk(rss_input, &bytes_written,
                                     &packet_info.in6_ext_dst,
                                     sizeof(packet_info.in6_ext_dst));
            } else {
                net_rx_rss_add_chunk(rss_input, &bytes_written,
                                     &packet_info.in6_dst,
                                     sizeof(packet_info.in6_dst));
            }
        }
    }

    if (!bytes_written) {
        return false;
    }

    net_toeplitz_add(result, rss_input, bytes_written, toe);

    return true;
}

SEC("socket")
int tun_rss_steering_prog(struct __sk_buff *skb)
{

    struct rss_config_t *config;
    struct toeplitz_key_data_t *toe;

    __u32 key = 0;
    __u32 hash = 0;

    config = bpf_map_lookup_elem(&tap_rss_map_configurations, &key);
    toe = bpf_map_lookup_elem(&tap_rss_map_toeplitz_key, &key);

    if (!config || !toe) {
        return 0;
    }

    if (config->redirect && calculate_rss_hash(skb, config, toe, &hash)) {
        __u32 table_idx = hash % config->indirections_len;
        __u16 *queue = 0;

        queue = bpf_map_lookup_elem(&tap_rss_map_indirection_table,
                                    &table_idx);

        if (queue) {
            return *queue;
        }
    }

    return config->default_queue;
}

char _license[] SEC("license") = "GPL v2";