/* $NetBSD: bpfjit.c,v 1.8 2014/05/22 13:35:45 alnsn Exp $ */ /*- * Copyright (c) 2011-2014 Alexander Nasonov. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #ifdef _KERNEL __KERNEL_RCSID(0, "$NetBSD: bpfjit.c,v 1.8 2014/05/22 13:35:45 alnsn Exp $"); #else __RCSID("$NetBSD: bpfjit.c,v 1.8 2014/05/22 13:35:45 alnsn Exp $"); #endif #include #include #ifndef _KERNEL #include #define BJ_ASSERT(c) assert(c) #else #define BJ_ASSERT(c) KASSERT(c) #endif #ifndef _KERNEL #include #define BJ_ALLOC(sz) malloc(sz) #define BJ_FREE(p, sz) free(p) #else #include #define BJ_ALLOC(sz) kmem_alloc(sz, KM_SLEEP) #define BJ_FREE(p, sz) kmem_free(p, sz) #endif #ifndef _KERNEL #include #include #include #include #else #include #include #endif #define __BPF_PRIVATE #include #include #include #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE #include /* for stderr */ #endif /* * Permanent register assignments. */ #define BJ_BUF SLJIT_SAVED_REG1 #define BJ_WIRELEN SLJIT_SAVED_REG2 #define BJ_BUFLEN SLJIT_SAVED_REG3 #define BJ_AREG SLJIT_TEMPORARY_REG1 #define BJ_TMP1REG SLJIT_TEMPORARY_REG2 #define BJ_TMP2REG SLJIT_TEMPORARY_REG3 #define BJ_XREG SLJIT_TEMPORARY_EREG1 #define BJ_TMP3REG SLJIT_TEMPORARY_EREG2 typedef unsigned int bpfjit_init_mask_t; #define BJ_INIT_NOBITS 0u #define BJ_INIT_MBIT(k) (1u << (k)) #define BJ_INIT_MMASK (BJ_INIT_MBIT(BPF_MEMWORDS) - 1u) #define BJ_INIT_ABIT BJ_INIT_MBIT(BPF_MEMWORDS) #define BJ_INIT_XBIT BJ_INIT_MBIT(BPF_MEMWORDS + 1) typedef uint32_t bpfjit_abc_length_t; #define MAX_ABC_LENGTH UINT32_MAX struct bpfjit_stack { uint32_t mem[BPF_MEMWORDS]; #ifdef _KERNEL void *tmp; #endif }; /* * Data for BPF_JMP instruction. * Forward declaration for struct bpfjit_jump. */ struct bpfjit_jump_data; /* * Node of bjumps list. */ struct bpfjit_jump { struct sljit_jump *sjump; SLIST_ENTRY(bpfjit_jump) entries; struct bpfjit_jump_data *jdata; }; /* * Data for BPF_JMP instruction. */ struct bpfjit_jump_data { /* * These entries make up bjumps list: * jtf[0] - when coming from jt path, * jtf[1] - when coming from jf path. */ struct bpfjit_jump jtf[2]; /* * Length calculated by Array Bounds Check Elimination (ABC) pass. */ bpfjit_abc_length_t abc_length; /* * Length checked by the last out-of-bounds check. */ bpfjit_abc_length_t checked_length; }; /* * Data for "read from packet" instructions. * See also read_pkt_insn() function below. */ struct bpfjit_read_pkt_data { /* * Length calculated by Array Bounds Check Elimination (ABC) pass. */ bpfjit_abc_length_t abc_length; /* * If positive, emit "if (buflen < check_length) return 0" * out-of-bounds check. * We assume that buflen is never equal to UINT32_MAX (otherwise, * we'd need a special bool variable to emit unconditional "return 0"). */ bpfjit_abc_length_t check_length; }; /* * Additional (optimization-related) data for bpf_insn. */ struct bpfjit_insn_data { /* List of jumps to this insn. */ SLIST_HEAD(, bpfjit_jump) bjumps; union { struct bpfjit_jump_data jdata; struct bpfjit_read_pkt_data rdata; } u; bpfjit_init_mask_t invalid; bool unreachable; }; #ifdef _KERNEL uint32_t m_xword(const struct mbuf *, uint32_t, int *); uint32_t m_xhalf(const struct mbuf *, uint32_t, int *); uint32_t m_xbyte(const struct mbuf *, uint32_t, int *); MODULE(MODULE_CLASS_MISC, bpfjit, "sljit") static int bpfjit_modcmd(modcmd_t cmd, void *arg) { switch (cmd) { case MODULE_CMD_INIT: bpfjit_module_ops.bj_free_code = &bpfjit_free_code; membar_producer(); bpfjit_module_ops.bj_generate_code = &bpfjit_generate_code; membar_producer(); return 0; case MODULE_CMD_FINI: return EOPNOTSUPP; default: return ENOTTY; } } #endif static uint32_t read_width(const struct bpf_insn *pc) { switch (BPF_SIZE(pc->code)) { case BPF_W: return 4; case BPF_H: return 2; case BPF_B: return 1; default: BJ_ASSERT(false); return 0; } } static bool grow_jumps(struct sljit_jump ***jumps, size_t *size) { struct sljit_jump **newptr; const size_t elemsz = sizeof(struct sljit_jump *); size_t old_size = *size; size_t new_size = 2 * old_size; if (new_size < old_size || new_size > SIZE_MAX / elemsz) return false; newptr = BJ_ALLOC(new_size * elemsz); if (newptr == NULL) return false; memcpy(newptr, *jumps, old_size * elemsz); BJ_FREE(*jumps, old_size * elemsz); *jumps = newptr; *size = new_size; return true; } static bool append_jump(struct sljit_jump *jump, struct sljit_jump ***jumps, size_t *size, size_t *max_size) { if (*size == *max_size && !grow_jumps(jumps, max_size)) return false; (*jumps)[(*size)++] = jump; return true; } /* * Generate code for BPF_LD+BPF_B+BPF_ABS A <- P[k:1]. */ static int emit_read8(struct sljit_compiler* compiler, uint32_t k) { return sljit_emit_op1(compiler, SLJIT_MOV_UB, BJ_AREG, 0, SLJIT_MEM1(BJ_BUF), k); } /* * Generate code for BPF_LD+BPF_H+BPF_ABS A <- P[k:2]. */ static int emit_read16(struct sljit_compiler* compiler, uint32_t k) { int status; /* tmp1 = buf[k]; */ status = sljit_emit_op1(compiler, SLJIT_MOV_UB, BJ_TMP1REG, 0, SLJIT_MEM1(BJ_BUF), k); if (status != SLJIT_SUCCESS) return status; /* A = buf[k+1]; */ status = sljit_emit_op1(compiler, SLJIT_MOV_UB, BJ_AREG, 0, SLJIT_MEM1(BJ_BUF), k+1); if (status != SLJIT_SUCCESS) return status; /* tmp1 = tmp1 << 8; */ status = sljit_emit_op2(compiler, SLJIT_SHL, BJ_TMP1REG, 0, BJ_TMP1REG, 0, SLJIT_IMM, 8); if (status != SLJIT_SUCCESS) return status; /* A = A + tmp1; */ status = sljit_emit_op2(compiler, SLJIT_ADD, BJ_AREG, 0, BJ_AREG, 0, BJ_TMP1REG, 0); return status; } /* * Generate code for BPF_LD+BPF_W+BPF_ABS A <- P[k:4]. */ static int emit_read32(struct sljit_compiler* compiler, uint32_t k) { int status; /* tmp1 = buf[k]; */ status = sljit_emit_op1(compiler, SLJIT_MOV_UB, BJ_TMP1REG, 0, SLJIT_MEM1(BJ_BUF), k); if (status != SLJIT_SUCCESS) return status; /* tmp2 = buf[k+1]; */ status = sljit_emit_op1(compiler, SLJIT_MOV_UB, BJ_TMP2REG, 0, SLJIT_MEM1(BJ_BUF), k+1); if (status != SLJIT_SUCCESS) return status; /* A = buf[k+3]; */ status = sljit_emit_op1(compiler, SLJIT_MOV_UB, BJ_AREG, 0, SLJIT_MEM1(BJ_BUF), k+3); if (status != SLJIT_SUCCESS) return status; /* tmp1 = tmp1 << 24; */ status = sljit_emit_op2(compiler, SLJIT_SHL, BJ_TMP1REG, 0, BJ_TMP1REG, 0, SLJIT_IMM, 24); if (status != SLJIT_SUCCESS) return status; /* A = A + tmp1; */ status = sljit_emit_op2(compiler, SLJIT_ADD, BJ_AREG, 0, BJ_AREG, 0, BJ_TMP1REG, 0); if (status != SLJIT_SUCCESS) return status; /* tmp1 = buf[k+2]; */ status = sljit_emit_op1(compiler, SLJIT_MOV_UB, BJ_TMP1REG, 0, SLJIT_MEM1(BJ_BUF), k+2); if (status != SLJIT_SUCCESS) return status; /* tmp2 = tmp2 << 16; */ status = sljit_emit_op2(compiler, SLJIT_SHL, BJ_TMP2REG, 0, BJ_TMP2REG, 0, SLJIT_IMM, 16); if (status != SLJIT_SUCCESS) return status; /* A = A + tmp2; */ status = sljit_emit_op2(compiler, SLJIT_ADD, BJ_AREG, 0, BJ_AREG, 0, BJ_TMP2REG, 0); if (status != SLJIT_SUCCESS) return status; /* tmp1 = tmp1 << 8; */ status = sljit_emit_op2(compiler, SLJIT_SHL, BJ_TMP1REG, 0, BJ_TMP1REG, 0, SLJIT_IMM, 8); if (status != SLJIT_SUCCESS) return status; /* A = A + tmp1; */ status = sljit_emit_op2(compiler, SLJIT_ADD, BJ_AREG, 0, BJ_AREG, 0, BJ_TMP1REG, 0); return status; } #ifdef _KERNEL /* * Generate m_xword/m_xhalf/m_xbyte call. * * pc is one of: * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] * BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) * * The dst variable should be * - BJ_AREG when emitting code for BPF_LD instructions, * - BJ_XREG or any of BJ_TMP[1-3]REG registers when emitting * code for BPF_MSH instruction. */ static int emit_xcall(struct sljit_compiler* compiler, const struct bpf_insn *pc, int dst, sljit_w dstw, struct sljit_jump **ret0_jump, uint32_t (*fn)(const struct mbuf *, uint32_t, int *)) { #if BJ_XREG == SLJIT_RETURN_REG || \ BJ_XREG == SLJIT_TEMPORARY_REG1 || \ BJ_XREG == SLJIT_TEMPORARY_REG2 || \ BJ_XREG == SLJIT_TEMPORARY_REG3 #error "Not supported assignment of registers." #endif int status; /* * The third argument of fn is an address on stack. */ const int arg3_offset = offsetof(struct bpfjit_stack, tmp); if (BPF_CLASS(pc->code) == BPF_LDX) { /* save A */ status = sljit_emit_op1(compiler, SLJIT_MOV, BJ_TMP3REG, 0, BJ_AREG, 0); if (status != SLJIT_SUCCESS) return status; } /* * Prepare registers for fn(buf, k, &err) call. */ status = sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_TEMPORARY_REG1, 0, BJ_BUF, 0); if (status != SLJIT_SUCCESS) return status; if (BPF_CLASS(pc->code) == BPF_LD && BPF_MODE(pc->code) == BPF_IND) { status = sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_TEMPORARY_REG2, 0, BJ_XREG, 0, SLJIT_IMM, (uint32_t)pc->k); } else { status = sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_TEMPORARY_REG2, 0, SLJIT_IMM, (uint32_t)pc->k); } if (status != SLJIT_SUCCESS) return status; status = sljit_get_local_base(compiler, SLJIT_TEMPORARY_REG3, 0, arg3_offset); if (status != SLJIT_SUCCESS) return status; /* fn(buf, k, &err); */ status = sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(fn)); if (dst != SLJIT_RETURN_REG) { /* move return value to dst */ status = sljit_emit_op1(compiler, SLJIT_MOV, dst, dstw, SLJIT_RETURN_REG, 0); if (status != SLJIT_SUCCESS) return status; } if (BPF_CLASS(pc->code) == BPF_LDX) { /* restore A */ status = sljit_emit_op1(compiler, SLJIT_MOV, BJ_AREG, 0, BJ_TMP3REG, 0); if (status != SLJIT_SUCCESS) return status; } /* tmp3 = *err; */ status = sljit_emit_op1(compiler, SLJIT_MOV_UI, SLJIT_TEMPORARY_REG3, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), arg3_offset); if (status != SLJIT_SUCCESS) return status; /* if (tmp3 != 0) return 0; */ *ret0_jump = sljit_emit_cmp(compiler, SLJIT_C_NOT_EQUAL, SLJIT_TEMPORARY_REG3, 0, SLJIT_IMM, 0); if (*ret0_jump == NULL) return SLJIT_ERR_ALLOC_FAILED; return status; } #endif /* * Generate code for * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] */ static int emit_pkt_read(struct sljit_compiler* compiler, const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) { int status = 0; /* XXX gcc 4.1 */ uint32_t width; struct sljit_jump *jump; #ifdef _KERNEL struct sljit_label *label; struct sljit_jump *over_mchain_jump; const bool check_zero_buflen = (to_mchain_jump != NULL); #endif const uint32_t k = pc->k; #ifdef _KERNEL if (to_mchain_jump == NULL) { to_mchain_jump = sljit_emit_cmp(compiler, SLJIT_C_EQUAL, BJ_BUFLEN, 0, SLJIT_IMM, 0); if (to_mchain_jump == NULL) return SLJIT_ERR_ALLOC_FAILED; } #endif width = read_width(pc); if (BPF_MODE(pc->code) == BPF_IND) { /* tmp1 = buflen - (pc->k + width); */ status = sljit_emit_op2(compiler, SLJIT_SUB, BJ_TMP1REG, 0, BJ_BUFLEN, 0, SLJIT_IMM, k + width); if (status != SLJIT_SUCCESS) return status; /* buf += X; */ status = sljit_emit_op2(compiler, SLJIT_ADD, BJ_BUF, 0, BJ_BUF, 0, BJ_XREG, 0); if (status != SLJIT_SUCCESS) return status; /* if (tmp1 < X) return 0; */ jump = sljit_emit_cmp(compiler, SLJIT_C_LESS, BJ_TMP1REG, 0, BJ_XREG, 0); if (jump == NULL) return SLJIT_ERR_ALLOC_FAILED; if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) return SLJIT_ERR_ALLOC_FAILED; } switch (width) { case 4: status = emit_read32(compiler, k); break; case 2: status = emit_read16(compiler, k); break; case 1: status = emit_read8(compiler, k); break; } if (status != SLJIT_SUCCESS) return status; if (BPF_MODE(pc->code) == BPF_IND) { /* buf -= X; */ status = sljit_emit_op2(compiler, SLJIT_SUB, BJ_BUF, 0, BJ_BUF, 0, BJ_XREG, 0); if (status != SLJIT_SUCCESS) return status; } #ifdef _KERNEL over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); if (over_mchain_jump == NULL) return SLJIT_ERR_ALLOC_FAILED; /* entry point to mchain handler */ label = sljit_emit_label(compiler); if (label == NULL) return SLJIT_ERR_ALLOC_FAILED; sljit_set_label(to_mchain_jump, label); if (check_zero_buflen) { /* if (buflen != 0) return 0; */ jump = sljit_emit_cmp(compiler, SLJIT_C_NOT_EQUAL, BJ_BUFLEN, 0, SLJIT_IMM, 0); if (jump == NULL) return SLJIT_ERR_ALLOC_FAILED; if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) return SLJIT_ERR_ALLOC_FAILED; } switch (width) { case 4: status = emit_xcall(compiler, pc, BJ_AREG, 0, &jump, &m_xword); break; case 2: status = emit_xcall(compiler, pc, BJ_AREG, 0, &jump, &m_xhalf); break; case 1: status = emit_xcall(compiler, pc, BJ_AREG, 0, &jump, &m_xbyte); break; } if (status != SLJIT_SUCCESS) return status; if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) return SLJIT_ERR_ALLOC_FAILED; label = sljit_emit_label(compiler); if (label == NULL) return SLJIT_ERR_ALLOC_FAILED; sljit_set_label(over_mchain_jump, label); #endif return status; } /* * Generate code for BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf). */ static int emit_msh(struct sljit_compiler* compiler, const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) { int status; #ifdef _KERNEL struct sljit_label *label; struct sljit_jump *jump, *over_mchain_jump; const bool check_zero_buflen = (to_mchain_jump != NULL); #endif const uint32_t k = pc->k; #ifdef _KERNEL if (to_mchain_jump == NULL) { to_mchain_jump = sljit_emit_cmp(compiler, SLJIT_C_EQUAL, BJ_BUFLEN, 0, SLJIT_IMM, 0); if (to_mchain_jump == NULL) return SLJIT_ERR_ALLOC_FAILED; } #endif /* tmp1 = buf[k] */ status = sljit_emit_op1(compiler, SLJIT_MOV_UB, BJ_TMP1REG, 0, SLJIT_MEM1(BJ_BUF), k); if (status != SLJIT_SUCCESS) return status; /* tmp1 &= 0xf */ status = sljit_emit_op2(compiler, SLJIT_AND, BJ_TMP1REG, 0, BJ_TMP1REG, 0, SLJIT_IMM, 0xf); if (status != SLJIT_SUCCESS) return status; /* tmp1 = tmp1 << 2 */ status = sljit_emit_op2(compiler, SLJIT_SHL, BJ_XREG, 0, BJ_TMP1REG, 0, SLJIT_IMM, 2); if (status != SLJIT_SUCCESS) return status; #ifdef _KERNEL over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); if (over_mchain_jump == NULL) return SLJIT_ERR_ALLOC_FAILED; /* entry point to mchain handler */ label = sljit_emit_label(compiler); if (label == NULL) return SLJIT_ERR_ALLOC_FAILED; sljit_set_label(to_mchain_jump, label); if (check_zero_buflen) { /* if (buflen != 0) return 0; */ jump = sljit_emit_cmp(compiler, SLJIT_C_NOT_EQUAL, BJ_BUFLEN, 0, SLJIT_IMM, 0); if (jump == NULL) return SLJIT_ERR_ALLOC_FAILED; if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) return SLJIT_ERR_ALLOC_FAILED; } status = emit_xcall(compiler, pc, BJ_TMP1REG, 0, &jump, &m_xbyte); if (status != SLJIT_SUCCESS) return status; if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) return SLJIT_ERR_ALLOC_FAILED; /* tmp1 &= 0xf */ status = sljit_emit_op2(compiler, SLJIT_AND, BJ_TMP1REG, 0, BJ_TMP1REG, 0, SLJIT_IMM, 0xf); if (status != SLJIT_SUCCESS) return status; /* tmp1 = tmp1 << 2 */ status = sljit_emit_op2(compiler, SLJIT_SHL, BJ_XREG, 0, BJ_TMP1REG, 0, SLJIT_IMM, 2); if (status != SLJIT_SUCCESS) return status; label = sljit_emit_label(compiler); if (label == NULL) return SLJIT_ERR_ALLOC_FAILED; sljit_set_label(over_mchain_jump, label); #endif return status; } static int emit_pow2_division(struct sljit_compiler* compiler, uint32_t k) { int shift = 0; int status = SLJIT_SUCCESS; while (k > 1) { k >>= 1; shift++; } BJ_ASSERT(k == 1 && shift < 32); if (shift != 0) { status = sljit_emit_op2(compiler, SLJIT_LSHR|SLJIT_INT_OP, BJ_AREG, 0, BJ_AREG, 0, SLJIT_IMM, shift); } return status; } #if !defined(BPFJIT_USE_UDIV) static sljit_uw divide(sljit_uw x, sljit_uw y) { return (uint32_t)x / (uint32_t)y; } #endif /* * Generate A = A / div. * divt,divw are either SLJIT_IMM,pc->k or BJ_XREG,0. */ static int emit_division(struct sljit_compiler* compiler, int divt, sljit_w divw) { int status; #if BJ_XREG == SLJIT_RETURN_REG || \ BJ_XREG == SLJIT_TEMPORARY_REG1 || \ BJ_XREG == SLJIT_TEMPORARY_REG2 || \ BJ_AREG == SLJIT_TEMPORARY_REG2 #error "Not supported assignment of registers." #endif #if BJ_AREG != SLJIT_TEMPORARY_REG1 status = sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_TEMPORARY_REG1, 0, BJ_AREG, 0); if (status != SLJIT_SUCCESS) return status; #endif status = sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_TEMPORARY_REG2, 0, divt, divw); if (status != SLJIT_SUCCESS) return status; #if defined(BPFJIT_USE_UDIV) status = sljit_emit_op0(compiler, SLJIT_UDIV|SLJIT_INT_OP); #if BJ_AREG != SLJIT_TEMPORARY_REG1 status = sljit_emit_op1(compiler, SLJIT_MOV, BJ_AREG, 0, SLJIT_TEMPORARY_REG1, 0); if (status != SLJIT_SUCCESS) return status; #endif #else status = sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(divide)); #if BJ_AREG != SLJIT_RETURN_REG status = sljit_emit_op1(compiler, SLJIT_MOV, BJ_AREG, 0, SLJIT_RETURN_REG, 0); if (status != SLJIT_SUCCESS) return status; #endif #endif return status; } /* * Return true if pc is a "read from packet" instruction. * If length is not NULL and return value is true, *length will * be set to a safe length required to read a packet. */ static bool read_pkt_insn(const struct bpf_insn *pc, bpfjit_abc_length_t *length) { bool rv; bpfjit_abc_length_t width; switch (BPF_CLASS(pc->code)) { default: rv = false; break; case BPF_LD: rv = BPF_MODE(pc->code) == BPF_ABS || BPF_MODE(pc->code) == BPF_IND; if (rv) width = read_width(pc); break; case BPF_LDX: rv = pc->code == (BPF_LDX|BPF_B|BPF_MSH); width = 1; break; } if (rv && length != NULL) { *length = (pc->k > UINT32_MAX - width) ? UINT32_MAX : pc->k + width; } return rv; } static void optimize_init(struct bpfjit_insn_data *insn_dat, size_t insn_count) { size_t i; for (i = 0; i < insn_count; i++) { SLIST_INIT(&insn_dat[i].bjumps); insn_dat[i].invalid = BJ_INIT_NOBITS; } } /* * The function divides instructions into blocks. Destination of a jump * instruction starts a new block. BPF_RET and BPF_JMP instructions * terminate a block. Blocks are linear, that is, there are no jumps out * from the middle of a block and there are no jumps in to the middle of * a block. * * The function also sets bits in *initmask for memwords that * need to be initialized to zero. Note that this set should be empty * for any valid kernel filter program. */ static bool optimize_pass1(const struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, size_t insn_count, bpfjit_init_mask_t *initmask, int *nscratches) { struct bpfjit_jump *jtf; size_t i; uint32_t jt, jf; bpfjit_init_mask_t invalid; /* borrowed from bpf_filter() */ bool unreachable; *nscratches = 2; *initmask = BJ_INIT_NOBITS; unreachable = false; invalid = ~BJ_INIT_NOBITS; for (i = 0; i < insn_count; i++) { if (!SLIST_EMPTY(&insn_dat[i].bjumps)) unreachable = false; insn_dat[i].unreachable = unreachable; if (unreachable) continue; invalid |= insn_dat[i].invalid; switch (BPF_CLASS(insns[i].code)) { case BPF_RET: if (BPF_RVAL(insns[i].code) == BPF_A) *initmask |= invalid & BJ_INIT_ABIT; unreachable = true; continue; case BPF_LD: if (BPF_MODE(insns[i].code) == BPF_IND || BPF_MODE(insns[i].code) == BPF_ABS) { if (BPF_MODE(insns[i].code) == BPF_IND && *nscratches < 4) { /* uses BJ_XREG */ *nscratches = 4; } if (*nscratches < 3 && read_width(&insns[i]) == 4) { /* uses BJ_TMP2REG */ *nscratches = 3; } } if (BPF_MODE(insns[i].code) == BPF_IND) *initmask |= invalid & BJ_INIT_XBIT; if (BPF_MODE(insns[i].code) == BPF_MEM && (uint32_t)insns[i].k < BPF_MEMWORDS) { *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); } invalid &= ~BJ_INIT_ABIT; continue; case BPF_LDX: #if defined(_KERNEL) /* uses BJ_TMP3REG */ *nscratches = 5; #endif /* uses BJ_XREG */ if (*nscratches < 4) *nscratches = 4; if (BPF_MODE(insns[i].code) == BPF_MEM && (uint32_t)insns[i].k < BPF_MEMWORDS) { *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); } invalid &= ~BJ_INIT_XBIT; continue; case BPF_ST: *initmask |= invalid & BJ_INIT_ABIT; if ((uint32_t)insns[i].k < BPF_MEMWORDS) invalid &= ~BJ_INIT_MBIT(insns[i].k); continue; case BPF_STX: /* uses BJ_XREG */ if (*nscratches < 4) *nscratches = 4; *initmask |= invalid & BJ_INIT_XBIT; if ((uint32_t)insns[i].k < BPF_MEMWORDS) invalid &= ~BJ_INIT_MBIT(insns[i].k); continue; case BPF_ALU: *initmask |= invalid & BJ_INIT_ABIT; if (insns[i].code != (BPF_ALU|BPF_NEG) && BPF_SRC(insns[i].code) == BPF_X) { *initmask |= invalid & BJ_INIT_XBIT; /* uses BJ_XREG */ if (*nscratches < 4) *nscratches = 4; } invalid &= ~BJ_INIT_ABIT; continue; case BPF_MISC: switch (BPF_MISCOP(insns[i].code)) { case BPF_TAX: // X <- A /* uses BJ_XREG */ if (*nscratches < 4) *nscratches = 4; *initmask |= invalid & BJ_INIT_ABIT; invalid &= ~BJ_INIT_XBIT; continue; case BPF_TXA: // A <- X /* uses BJ_XREG */ if (*nscratches < 4) *nscratches = 4; *initmask |= invalid & BJ_INIT_XBIT; invalid &= ~BJ_INIT_ABIT; continue; } continue; case BPF_JMP: /* Initialize abc_length for ABC pass. */ insn_dat[i].u.jdata.abc_length = MAX_ABC_LENGTH; if (BPF_OP(insns[i].code) == BPF_JA) { jt = jf = insns[i].k; } else { jt = insns[i].jt; jf = insns[i].jf; } if (jt >= insn_count - (i + 1) || jf >= insn_count - (i + 1)) { return false; } if (jt > 0 && jf > 0) unreachable = true; jt += i + 1; jf += i + 1; jtf = insn_dat[i].u.jdata.jtf; jtf[0].sjump = NULL; jtf[0].jdata = &insn_dat[i].u.jdata; SLIST_INSERT_HEAD(&insn_dat[jt].bjumps, &jtf[0], entries); if (jf != jt) { jtf[1].sjump = NULL; jtf[1].jdata = &insn_dat[i].u.jdata; SLIST_INSERT_HEAD(&insn_dat[jf].bjumps, &jtf[1], entries); } insn_dat[jf].invalid |= invalid; insn_dat[jt].invalid |= invalid; invalid = 0; continue; } } return true; } /* * Array Bounds Check Elimination (ABC) pass. */ static void optimize_pass2(const struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, size_t insn_count) { struct bpfjit_jump *jmp; const struct bpf_insn *pc; struct bpfjit_insn_data *pd; size_t i; bpfjit_abc_length_t length, abc_length = 0; for (i = insn_count; i != 0; i--) { pc = &insns[i-1]; pd = &insn_dat[i-1]; if (pd->unreachable) continue; switch (BPF_CLASS(pc->code)) { case BPF_RET: abc_length = 0; break; case BPF_JMP: abc_length = pd->u.jdata.abc_length; break; default: if (read_pkt_insn(pc, &length)) { if (abc_length < length) abc_length = length; pd->u.rdata.abc_length = abc_length; } break; } SLIST_FOREACH(jmp, &pd->bjumps, entries) { if (jmp->jdata->abc_length > abc_length) jmp->jdata->abc_length = abc_length; } } } static void optimize_pass3(const struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, size_t insn_count) { struct bpfjit_jump *jmp; size_t i; bpfjit_abc_length_t checked_length = 0; for (i = 0; i < insn_count; i++) { if (insn_dat[i].unreachable) continue; SLIST_FOREACH(jmp, &insn_dat[i].bjumps, entries) { if (jmp->jdata->checked_length < checked_length) checked_length = jmp->jdata->checked_length; } if (BPF_CLASS(insns[i].code) == BPF_JMP) { insn_dat[i].u.jdata.checked_length = checked_length; } else if (read_pkt_insn(&insns[i], NULL)) { struct bpfjit_read_pkt_data *rdata = &insn_dat[i].u.rdata; rdata->check_length = 0; if (checked_length < rdata->abc_length) { checked_length = rdata->abc_length; rdata->check_length = checked_length; } } } } static bool optimize(const struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, size_t insn_count, bpfjit_init_mask_t *initmask, int *nscratches) { optimize_init(insn_dat, insn_count); if (!optimize_pass1(insns, insn_dat, insn_count, initmask, nscratches)) { return false; } optimize_pass2(insns, insn_dat, insn_count); optimize_pass3(insns, insn_dat, insn_count); return true; } /* * Convert BPF_ALU operations except BPF_NEG and BPF_DIV to sljit operation. */ static int bpf_alu_to_sljit_op(const struct bpf_insn *pc) { /* * Note: all supported 64bit arches have 32bit multiply * instruction so SLJIT_INT_OP doesn't have any overhead. */ switch (BPF_OP(pc->code)) { case BPF_ADD: return SLJIT_ADD; case BPF_SUB: return SLJIT_SUB; case BPF_MUL: return SLJIT_MUL|SLJIT_INT_OP; case BPF_OR: return SLJIT_OR; case BPF_AND: return SLJIT_AND; case BPF_LSH: return SLJIT_SHL; case BPF_RSH: return SLJIT_LSHR|SLJIT_INT_OP; default: BJ_ASSERT(false); return 0; } } /* * Convert BPF_JMP operations except BPF_JA to sljit condition. */ static int bpf_jmp_to_sljit_cond(const struct bpf_insn *pc, bool negate) { /* * Note: all supported 64bit arches have 32bit comparison * instructions so SLJIT_INT_OP doesn't have any overhead. */ int rv = SLJIT_INT_OP; switch (BPF_OP(pc->code)) { case BPF_JGT: rv |= negate ? SLJIT_C_LESS_EQUAL : SLJIT_C_GREATER; break; case BPF_JGE: rv |= negate ? SLJIT_C_LESS : SLJIT_C_GREATER_EQUAL; break; case BPF_JEQ: rv |= negate ? SLJIT_C_NOT_EQUAL : SLJIT_C_EQUAL; break; case BPF_JSET: rv |= negate ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL; break; default: BJ_ASSERT(false); } return rv; } /* * Convert BPF_K and BPF_X to sljit register. */ static int kx_to_reg(const struct bpf_insn *pc) { switch (BPF_SRC(pc->code)) { case BPF_K: return SLJIT_IMM; case BPF_X: return BJ_XREG; default: BJ_ASSERT(false); return 0; } } static sljit_w kx_to_reg_arg(const struct bpf_insn *pc) { switch (BPF_SRC(pc->code)) { case BPF_K: return (uint32_t)pc->k; /* SLJIT_IMM, pc->k, */ case BPF_X: return 0; /* BJ_XREG, 0, */ default: BJ_ASSERT(false); return 0; } } bpfjit_func_t bpfjit_generate_code(bpf_ctx_t *bc, struct bpf_insn *insns, size_t insn_count) { void *rv; struct sljit_compiler *compiler; size_t i; int status; int branching, negate; unsigned int rval, mode, src; /* optimization related */ bpfjit_init_mask_t initmask; int nscratches; /* a list of jumps to out-of-bound return from a generated function */ struct sljit_jump **ret0; size_t ret0_size, ret0_maxsize; const struct bpf_insn *pc; struct bpfjit_insn_data *insn_dat; /* for local use */ struct sljit_label *label; struct sljit_jump *jump; struct bpfjit_jump *bjump, *jtf; struct sljit_jump *to_mchain_jump; uint32_t jt, jf; rv = NULL; compiler = NULL; insn_dat = NULL; ret0 = NULL; if (insn_count == 0 || insn_count > SIZE_MAX / sizeof(insn_dat[0])) goto fail; insn_dat = BJ_ALLOC(insn_count * sizeof(insn_dat[0])); if (insn_dat == NULL) goto fail; if (!optimize(insns, insn_dat, insn_count, &initmask, &nscratches)) { goto fail; } #if defined(_KERNEL) /* bpf_filter() checks initialization of memwords. */ BJ_ASSERT((initmask & BJ_INIT_MMASK) == 0); #endif ret0_size = 0; ret0_maxsize = 64; ret0 = BJ_ALLOC(ret0_maxsize * sizeof(ret0[0])); if (ret0 == NULL) goto fail; compiler = sljit_create_compiler(); if (compiler == NULL) goto fail; #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE sljit_compiler_verbose(compiler, stderr); #endif status = sljit_emit_enter(compiler, 3, nscratches, 3, sizeof(struct bpfjit_stack)); if (status != SLJIT_SUCCESS) goto fail; for (i = 0; i < BPF_MEMWORDS; i++) { if (initmask & BJ_INIT_MBIT(i)) { status = sljit_emit_op1(compiler, SLJIT_MOV_UI, SLJIT_MEM1(SLJIT_LOCALS_REG), offsetof(struct bpfjit_stack, mem) + i * sizeof(uint32_t), SLJIT_IMM, 0); if (status != SLJIT_SUCCESS) goto fail; } } if (initmask & BJ_INIT_ABIT) { /* A = 0; */ status = sljit_emit_op1(compiler, SLJIT_MOV, BJ_AREG, 0, SLJIT_IMM, 0); if (status != SLJIT_SUCCESS) goto fail; } if (initmask & BJ_INIT_XBIT) { /* X = 0; */ status = sljit_emit_op1(compiler, SLJIT_MOV, BJ_XREG, 0, SLJIT_IMM, 0); if (status != SLJIT_SUCCESS) goto fail; } for (i = 0; i < insn_count; i++) { if (insn_dat[i].unreachable) continue; to_mchain_jump = NULL; /* * Resolve jumps to the current insn. */ label = NULL; SLIST_FOREACH(bjump, &insn_dat[i].bjumps, entries) { if (bjump->sjump != NULL) { if (label == NULL) label = sljit_emit_label(compiler); if (label == NULL) goto fail; sljit_set_label(bjump->sjump, label); } } if (read_pkt_insn(&insns[i], NULL) && insn_dat[i].u.rdata.check_length > 0) { /* if (buflen < check_length) return 0; */ jump = sljit_emit_cmp(compiler, SLJIT_C_LESS, BJ_BUFLEN, 0, SLJIT_IMM, insn_dat[i].u.rdata.check_length); if (jump == NULL) goto fail; #ifdef _KERNEL to_mchain_jump = jump; #else if (!append_jump(jump, &ret0, &ret0_size, &ret0_maxsize)) goto fail; #endif } pc = &insns[i]; switch (BPF_CLASS(pc->code)) { default: goto fail; case BPF_LD: /* BPF_LD+BPF_IMM A <- k */ if (pc->code == (BPF_LD|BPF_IMM)) { status = sljit_emit_op1(compiler, SLJIT_MOV, BJ_AREG, 0, SLJIT_IMM, (uint32_t)pc->k); if (status != SLJIT_SUCCESS) goto fail; continue; } /* BPF_LD+BPF_MEM A <- M[k] */ if (pc->code == (BPF_LD|BPF_MEM)) { if ((uint32_t)pc->k >= BPF_MEMWORDS) goto fail; status = sljit_emit_op1(compiler, SLJIT_MOV_UI, BJ_AREG, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), offsetof(struct bpfjit_stack, mem) + pc->k * sizeof(uint32_t)); if (status != SLJIT_SUCCESS) goto fail; continue; } /* BPF_LD+BPF_W+BPF_LEN A <- len */ if (pc->code == (BPF_LD|BPF_W|BPF_LEN)) { status = sljit_emit_op1(compiler, SLJIT_MOV, BJ_AREG, 0, BJ_WIRELEN, 0); if (status != SLJIT_SUCCESS) goto fail; continue; } mode = BPF_MODE(pc->code); if (mode != BPF_ABS && mode != BPF_IND) goto fail; status = emit_pkt_read(compiler, pc, to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); if (status != SLJIT_SUCCESS) goto fail; continue; case BPF_LDX: mode = BPF_MODE(pc->code); /* BPF_LDX+BPF_W+BPF_IMM X <- k */ if (mode == BPF_IMM) { if (BPF_SIZE(pc->code) != BPF_W) goto fail; status = sljit_emit_op1(compiler, SLJIT_MOV, BJ_XREG, 0, SLJIT_IMM, (uint32_t)pc->k); if (status != SLJIT_SUCCESS) goto fail; continue; } /* BPF_LDX+BPF_W+BPF_LEN X <- len */ if (mode == BPF_LEN) { if (BPF_SIZE(pc->code) != BPF_W) goto fail; status = sljit_emit_op1(compiler, SLJIT_MOV, BJ_XREG, 0, BJ_WIRELEN, 0); if (status != SLJIT_SUCCESS) goto fail; continue; } /* BPF_LDX+BPF_W+BPF_MEM X <- M[k] */ if (mode == BPF_MEM) { if (BPF_SIZE(pc->code) != BPF_W) goto fail; if ((uint32_t)pc->k >= BPF_MEMWORDS) goto fail; status = sljit_emit_op1(compiler, SLJIT_MOV_UI, BJ_XREG, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), offsetof(struct bpfjit_stack, mem) + pc->k * sizeof(uint32_t)); if (status != SLJIT_SUCCESS) goto fail; continue; } /* BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) */ if (mode != BPF_MSH || BPF_SIZE(pc->code) != BPF_B) goto fail; status = emit_msh(compiler, pc, to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); if (status != SLJIT_SUCCESS) goto fail; continue; case BPF_ST: if (pc->code != BPF_ST || (uint32_t)pc->k >= BPF_MEMWORDS) { goto fail; } status = sljit_emit_op1(compiler, SLJIT_MOV_UI, SLJIT_MEM1(SLJIT_LOCALS_REG), offsetof(struct bpfjit_stack, mem) + pc->k * sizeof(uint32_t), BJ_AREG, 0); if (status != SLJIT_SUCCESS) goto fail; continue; case BPF_STX: if (pc->code != BPF_STX || (uint32_t)pc->k >= BPF_MEMWORDS) { goto fail; } status = sljit_emit_op1(compiler, SLJIT_MOV_UI, SLJIT_MEM1(SLJIT_LOCALS_REG), offsetof(struct bpfjit_stack, mem) + pc->k * sizeof(uint32_t), BJ_XREG, 0); if (status != SLJIT_SUCCESS) goto fail; continue; case BPF_ALU: if (pc->code == (BPF_ALU|BPF_NEG)) { status = sljit_emit_op1(compiler, SLJIT_NEG, BJ_AREG, 0, BJ_AREG, 0); if (status != SLJIT_SUCCESS) goto fail; continue; } if (BPF_OP(pc->code) != BPF_DIV) { status = sljit_emit_op2(compiler, bpf_alu_to_sljit_op(pc), BJ_AREG, 0, BJ_AREG, 0, kx_to_reg(pc), kx_to_reg_arg(pc)); if (status != SLJIT_SUCCESS) goto fail; continue; } /* BPF_DIV */ src = BPF_SRC(pc->code); if (src != BPF_X && src != BPF_K) goto fail; /* division by zero? */ if (src == BPF_X) { jump = sljit_emit_cmp(compiler, SLJIT_C_EQUAL|SLJIT_INT_OP, BJ_XREG, 0, SLJIT_IMM, 0); if (jump == NULL) goto fail; if (!append_jump(jump, &ret0, &ret0_size, &ret0_maxsize)) goto fail; } else if (pc->k == 0) { jump = sljit_emit_jump(compiler, SLJIT_JUMP); if (jump == NULL) goto fail; if (!append_jump(jump, &ret0, &ret0_size, &ret0_maxsize)) goto fail; } if (src == BPF_X) { status = emit_division(compiler, BJ_XREG, 0); if (status != SLJIT_SUCCESS) goto fail; } else if (pc->k != 0) { if (pc->k & (pc->k - 1)) { status = emit_division(compiler, SLJIT_IMM, (uint32_t)pc->k); } else { status = emit_pow2_division(compiler, (uint32_t)pc->k); } if (status != SLJIT_SUCCESS) goto fail; } continue; case BPF_JMP: if (BPF_OP(pc->code) == BPF_JA) { jt = jf = pc->k; } else { jt = pc->jt; jf = pc->jf; } negate = (jt == 0) ? 1 : 0; branching = (jt == jf) ? 0 : 1; jtf = insn_dat[i].u.jdata.jtf; if (branching) { if (BPF_OP(pc->code) != BPF_JSET) { jump = sljit_emit_cmp(compiler, bpf_jmp_to_sljit_cond(pc, negate), BJ_AREG, 0, kx_to_reg(pc), kx_to_reg_arg(pc)); } else { status = sljit_emit_op2(compiler, SLJIT_AND, BJ_TMP1REG, 0, BJ_AREG, 0, kx_to_reg(pc), kx_to_reg_arg(pc)); if (status != SLJIT_SUCCESS) goto fail; jump = sljit_emit_cmp(compiler, bpf_jmp_to_sljit_cond(pc, negate), BJ_TMP1REG, 0, SLJIT_IMM, 0); } if (jump == NULL) goto fail; BJ_ASSERT(jtf[negate].sjump == NULL); jtf[negate].sjump = jump; } if (!branching || (jt != 0 && jf != 0)) { jump = sljit_emit_jump(compiler, SLJIT_JUMP); if (jump == NULL) goto fail; BJ_ASSERT(jtf[branching].sjump == NULL); jtf[branching].sjump = jump; } continue; case BPF_RET: rval = BPF_RVAL(pc->code); if (rval == BPF_X) goto fail; /* BPF_RET+BPF_K accept k bytes */ if (rval == BPF_K) { status = sljit_emit_return(compiler, SLJIT_MOV_UI, SLJIT_IMM, (uint32_t)pc->k); if (status != SLJIT_SUCCESS) goto fail; } /* BPF_RET+BPF_A accept A bytes */ if (rval == BPF_A) { status = sljit_emit_return(compiler, SLJIT_MOV_UI, BJ_AREG, 0); if (status != SLJIT_SUCCESS) goto fail; } continue; case BPF_MISC: switch (BPF_MISCOP(pc->code)) { case BPF_TAX: status = sljit_emit_op1(compiler, SLJIT_MOV_UI, BJ_XREG, 0, BJ_AREG, 0); if (status != SLJIT_SUCCESS) goto fail; continue; case BPF_TXA: status = sljit_emit_op1(compiler, SLJIT_MOV, BJ_AREG, 0, BJ_XREG, 0); if (status != SLJIT_SUCCESS) goto fail; continue; } goto fail; } /* switch */ } /* main loop */ BJ_ASSERT(ret0_size <= ret0_maxsize); if (ret0_size > 0) { label = sljit_emit_label(compiler); if (label == NULL) goto fail; for (i = 0; i < ret0_size; i++) sljit_set_label(ret0[i], label); } status = sljit_emit_return(compiler, SLJIT_MOV_UI, SLJIT_IMM, 0); if (status != SLJIT_SUCCESS) goto fail; rv = sljit_generate_code(compiler); fail: if (compiler != NULL) sljit_free_compiler(compiler); if (insn_dat != NULL) BJ_FREE(insn_dat, insn_count * sizeof(insn_dat[0])); if (ret0 != NULL) BJ_FREE(ret0, ret0_maxsize * sizeof(ret0[0])); return (bpfjit_func_t)rv; } void bpfjit_free_code(bpfjit_func_t code) { sljit_free_code((void *)code); }