Make ipforward_rt and ip6_forward_rt percpu
Sharing one rtcache between CPUs is just a bad idea. Reviewed by knakahara@
This commit is contained in:
ozaki-r
parent
ed4e6cd45a
commit
543e39c0d3
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: ip_input.c,v 1.339 2016/08/01 03:15:30 ozaki-r Exp $ */
|
||||
/* $NetBSD: ip_input.c,v 1.340 2016/08/31 09:14:47 ozaki-r Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
|
||||
@ -91,7 +91,7 @@
|
||||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.339 2016/08/01 03:15:30 ozaki-r Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.340 2016/08/31 09:14:47 ozaki-r Exp $");
|
||||
|
||||
#ifdef _KERNEL_OPT
|
||||
#include "opt_inet.h"
|
||||
@ -225,7 +225,7 @@ pfil_head_t * inet_pfil_hook __read_mostly;
|
||||
ipid_state_t * ip_ids __read_mostly;
|
||||
percpu_t * ipstat_percpu __read_mostly;
|
||||
|
||||
static struct route ipforward_rt __cacheline_aligned;
|
||||
static percpu_t *ipforward_rt_percpu __cacheline_aligned;
|
||||
|
||||
uint16_t ip_id;
|
||||
|
||||
@ -345,6 +345,10 @@ ip_init(void)
|
||||
#endif /* MBUFTRACE */
|
||||
|
||||
ipstat_percpu = percpu_alloc(sizeof(uint64_t) * IP_NSTATS);
|
||||
|
||||
ipforward_rt_percpu = percpu_alloc(sizeof(struct route));
|
||||
if (ipforward_rt_percpu == NULL)
|
||||
panic("failed to allocate ipforward_rt_percpu");
|
||||
}
|
||||
|
||||
static struct in_ifaddr *
|
||||
@ -1163,11 +1167,14 @@ ip_rtaddr(struct in_addr dst)
|
||||
struct sockaddr dst;
|
||||
struct sockaddr_in dst4;
|
||||
} u;
|
||||
struct route *ro;
|
||||
|
||||
sockaddr_in_init(&u.dst4, &dst, 0);
|
||||
|
||||
SOFTNET_LOCK();
|
||||
rt = rtcache_lookup(&ipforward_rt, &u.dst);
|
||||
ro = percpu_getref(ipforward_rt_percpu);
|
||||
rt = rtcache_lookup(ro, &u.dst);
|
||||
percpu_putref(ipforward_rt_percpu);
|
||||
SOFTNET_UNLOCK();
|
||||
if (rt == NULL)
|
||||
return NULL;
|
||||
@ -1300,6 +1307,7 @@ ip_forward(struct mbuf *m, int srcrt, struct ifnet *rcvif)
|
||||
struct sockaddr_in dst4;
|
||||
} u;
|
||||
uint64_t *ips;
|
||||
struct route *ro;
|
||||
|
||||
KASSERTMSG(cpu_softintr_p(), "ip_forward: not in the software "
|
||||
"interrupt handler; synchronization assumptions violated");
|
||||
@ -1331,7 +1339,9 @@ ip_forward(struct mbuf *m, int srcrt, struct ifnet *rcvif)
|
||||
|
||||
sockaddr_in_init(&u.dst4, &ip->ip_dst, 0);
|
||||
|
||||
if ((rt = rtcache_lookup(&ipforward_rt, &u.dst)) == NULL) {
|
||||
ro = percpu_getref(ipforward_rt_percpu);
|
||||
if ((rt = rtcache_lookup(ro, &u.dst)) == NULL) {
|
||||
percpu_putref(ipforward_rt_percpu);
|
||||
icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, dest, 0);
|
||||
SOFTNET_UNLOCK();
|
||||
return;
|
||||
@ -1376,7 +1386,7 @@ ip_forward(struct mbuf *m, int srcrt, struct ifnet *rcvif)
|
||||
}
|
||||
}
|
||||
|
||||
error = ip_output(m, NULL, &ipforward_rt,
|
||||
error = ip_output(m, NULL, ro,
|
||||
(IP_FORWARDING | (ip_directedbcast ? IP_ALLOWBROADCAST : 0)),
|
||||
NULL, NULL);
|
||||
|
||||
@ -1398,17 +1408,19 @@ ip_forward(struct mbuf *m, int srcrt, struct ifnet *rcvif)
|
||||
if (mcopy) {
|
||||
#ifdef GATEWAY
|
||||
if (mcopy->m_flags & M_CANFASTFWD)
|
||||
ipflow_create(&ipforward_rt, mcopy);
|
||||
ipflow_create(ro, mcopy);
|
||||
#endif
|
||||
m_freem(mcopy);
|
||||
}
|
||||
|
||||
percpu_putref(ipforward_rt_percpu);
|
||||
SOFTNET_UNLOCK();
|
||||
return;
|
||||
|
||||
redirect:
|
||||
error:
|
||||
if (mcopy == NULL) {
|
||||
percpu_putref(ipforward_rt_percpu);
|
||||
SOFTNET_UNLOCK();
|
||||
return;
|
||||
}
|
||||
@ -1432,7 +1444,7 @@ error:
|
||||
type = ICMP_UNREACH;
|
||||
code = ICMP_UNREACH_NEEDFRAG;
|
||||
|
||||
if ((rt = rtcache_validate(&ipforward_rt)) != NULL)
|
||||
if ((rt = rtcache_validate(ro)) != NULL)
|
||||
destmtu = rt->rt_ifp->if_mtu;
|
||||
#ifdef IPSEC
|
||||
if (ipsec_used)
|
||||
@ -1450,10 +1462,12 @@ error:
|
||||
*/
|
||||
if (mcopy)
|
||||
m_freem(mcopy);
|
||||
percpu_putref(ipforward_rt_percpu);
|
||||
SOFTNET_UNLOCK();
|
||||
return;
|
||||
}
|
||||
icmp_error(mcopy, type, code, dest, destmtu);
|
||||
percpu_putref(ipforward_rt_percpu);
|
||||
SOFTNET_UNLOCK();
|
||||
}
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: ip6_forward.c,v 1.80 2016/06/28 02:02:56 ozaki-r Exp $ */
|
||||
/* $NetBSD: ip6_forward.c,v 1.81 2016/08/31 09:14:47 ozaki-r Exp $ */
|
||||
/* $KAME: ip6_forward.c,v 1.109 2002/09/11 08:10:17 sakane Exp $ */
|
||||
|
||||
/*
|
||||
@ -31,7 +31,7 @@
|
||||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.80 2016/06/28 02:02:56 ozaki-r Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.81 2016/08/31 09:14:47 ozaki-r Exp $");
|
||||
|
||||
#ifdef _KERNEL_OPT
|
||||
#include "opt_gateway.h"
|
||||
@ -48,6 +48,7 @@ __KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.80 2016/06/28 02:02:56 ozaki-r Exp
|
||||
#include <sys/time.h>
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/syslog.h>
|
||||
#include <sys/percpu.h>
|
||||
|
||||
#include <net/if.h>
|
||||
#include <net/route.h>
|
||||
@ -72,7 +73,7 @@ __KERNEL_RCSID(0, "$NetBSD: ip6_forward.c,v 1.80 2016/06/28 02:02:56 ozaki-r Exp
|
||||
|
||||
#include <net/net_osdep.h>
|
||||
|
||||
struct route ip6_forward_rt;
|
||||
extern percpu_t *ip6_forward_rt_percpu;
|
||||
|
||||
extern pfil_head_t *inet6_pfil_hook; /* XXX */
|
||||
|
||||
@ -135,6 +136,7 @@ ip6_forward(struct mbuf *m, int srcrt)
|
||||
struct in6_addr src_in6, dst_in6;
|
||||
struct ifnet *rcvif = NULL;
|
||||
struct psref psref;
|
||||
struct route *ro;
|
||||
#ifdef IPSEC
|
||||
int needipsec = 0;
|
||||
struct secpolicy *sp = NULL;
|
||||
@ -203,6 +205,7 @@ ip6_forward(struct mbuf *m, int srcrt)
|
||||
}
|
||||
#endif /* IPSEC */
|
||||
|
||||
ro = percpu_getref(ip6_forward_rt_percpu);
|
||||
if (srcrt) {
|
||||
union {
|
||||
struct sockaddr dst;
|
||||
@ -210,17 +213,18 @@ ip6_forward(struct mbuf *m, int srcrt)
|
||||
} u;
|
||||
|
||||
sockaddr_in6_init(&u.dst6, &ip6->ip6_dst, 0, 0, 0);
|
||||
if ((rt = rtcache_lookup(&ip6_forward_rt, &u.dst)) == NULL) {
|
||||
if ((rt = rtcache_lookup(ro, &u.dst)) == NULL) {
|
||||
IP6_STATINC(IP6_STAT_NOROUTE);
|
||||
/* XXX in6_ifstat_inc(rt->rt_ifp, ifs6_in_noroute) */
|
||||
if (mcopy) {
|
||||
icmp6_error(mcopy, ICMP6_DST_UNREACH,
|
||||
ICMP6_DST_UNREACH_NOROUTE, 0);
|
||||
}
|
||||
percpu_putref(ip6_forward_rt_percpu);
|
||||
goto drop;
|
||||
}
|
||||
} else if ((rt = rtcache_validate(&ip6_forward_rt)) == NULL &&
|
||||
(rt = rtcache_update(&ip6_forward_rt, 1)) == NULL) {
|
||||
} else if ((rt = rtcache_validate(ro)) == NULL &&
|
||||
(rt = rtcache_update(ro, 1)) == NULL) {
|
||||
/*
|
||||
* rtcache_getdst(ip6_forward_rt)->sin6_addr was equal to
|
||||
* ip6->ip6_dst
|
||||
@ -231,9 +235,11 @@ ip6_forward(struct mbuf *m, int srcrt)
|
||||
icmp6_error(mcopy, ICMP6_DST_UNREACH,
|
||||
ICMP6_DST_UNREACH_NOROUTE, 0);
|
||||
}
|
||||
percpu_putref(ip6_forward_rt_percpu);
|
||||
goto drop;
|
||||
}
|
||||
dst = satocsin6(rtcache_getdst(&ip6_forward_rt));
|
||||
dst = satocsin6(rtcache_getdst(ro));
|
||||
percpu_putref(ip6_forward_rt_percpu);
|
||||
|
||||
/*
|
||||
* Source scope check: if a packet can't be delivered to its
|
||||
@ -318,10 +324,11 @@ ip6_forward(struct mbuf *m, int srcrt)
|
||||
*/
|
||||
if (rt->rt_ifp == rcvif && !srcrt && ip6_sendredirects &&
|
||||
(rt->rt_flags & (RTF_DYNAMIC|RTF_MODIFIED)) == 0) {
|
||||
ro = percpu_getref(ip6_forward_rt_percpu);
|
||||
if ((rt->rt_ifp->if_flags & IFF_POINTOPOINT) &&
|
||||
nd6_is_addr_neighbor(
|
||||
satocsin6(rtcache_getdst(&ip6_forward_rt)),
|
||||
nd6_is_addr_neighbor(satocsin6(rtcache_getdst(ro)),
|
||||
rt->rt_ifp)) {
|
||||
percpu_putref(ip6_forward_rt_percpu);
|
||||
/*
|
||||
* If the incoming interface is equal to the outgoing
|
||||
* one, the link attached to the interface is
|
||||
@ -341,6 +348,7 @@ ip6_forward(struct mbuf *m, int srcrt)
|
||||
ICMP6_DST_UNREACH_ADDR, 0);
|
||||
goto drop;
|
||||
}
|
||||
percpu_putref(ip6_forward_rt_percpu);
|
||||
type = ND_REDIRECT;
|
||||
}
|
||||
|
||||
@ -409,8 +417,10 @@ ip6_forward(struct mbuf *m, int srcrt)
|
||||
IP6_STATINC(IP6_STAT_REDIRECTSENT);
|
||||
else {
|
||||
#ifdef GATEWAY
|
||||
ro = percpu_getref(ip6_forward_rt_percpu);
|
||||
if (m->m_flags & M_CANFASTFWD)
|
||||
ip6flow_create(&ip6_forward_rt, m);
|
||||
ip6flow_create(ro, m);
|
||||
percpu_putref(ip6_forward_rt_percpu);
|
||||
#endif
|
||||
if (mcopy)
|
||||
goto freecopy;
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: ip6_input.c,v 1.166 2016/08/02 04:50:16 knakahara Exp $ */
|
||||
/* $NetBSD: ip6_input.c,v 1.167 2016/08/31 09:14:47 ozaki-r Exp $ */
|
||||
/* $KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $ */
|
||||
|
||||
/*
|
||||
@ -62,7 +62,7 @@
|
||||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.166 2016/08/02 04:50:16 knakahara Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.167 2016/08/31 09:14:47 ozaki-r Exp $");
|
||||
|
||||
#ifdef _KERNEL_OPT
|
||||
#include "opt_gateway.h"
|
||||
@ -87,6 +87,7 @@ __KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.166 2016/08/02 04:50:16 knakahara Ex
|
||||
#include <sys/proc.h>
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/cprng.h>
|
||||
#include <sys/percpu.h>
|
||||
|
||||
#include <net/if.h>
|
||||
#include <net/if_types.h>
|
||||
@ -143,6 +144,8 @@ pfil_head_t *inet6_pfil_hook;
|
||||
|
||||
percpu_t *ip6stat_percpu;
|
||||
|
||||
percpu_t *ip6_forward_rt_percpu __cacheline_aligned;
|
||||
|
||||
static void ip6_init2(void);
|
||||
static void ip6intr(void *);
|
||||
static struct m_tag *ip6_setdstifaddr(struct mbuf *, const struct in6_ifaddr *);
|
||||
@ -194,6 +197,10 @@ ip6_init(void)
|
||||
KASSERT(inet6_pfil_hook != NULL);
|
||||
|
||||
ip6stat_percpu = percpu_alloc(sizeof(uint64_t) * IP6_NSTATS);
|
||||
|
||||
ip6_forward_rt_percpu = percpu_alloc(sizeof(struct route));
|
||||
if (ip6_forward_rt_percpu == NULL)
|
||||
panic("failed to alllocate ip6_forward_rt_percpu");
|
||||
}
|
||||
|
||||
static void
|
||||
@ -239,8 +246,6 @@ ip6intr(void *arg __unused)
|
||||
mutex_exit(softnet_lock);
|
||||
}
|
||||
|
||||
extern struct route ip6_forward_rt;
|
||||
|
||||
void
|
||||
ip6_input(struct mbuf *m, struct ifnet *rcvif)
|
||||
{
|
||||
@ -256,6 +261,7 @@ ip6_input(struct mbuf *m, struct ifnet *rcvif)
|
||||
struct sockaddr dst;
|
||||
struct sockaddr_in6 dst6;
|
||||
} u;
|
||||
struct route *ro;
|
||||
|
||||
/*
|
||||
* make sure we don't have onion peering information into m_tag.
|
||||
@ -461,7 +467,9 @@ ip6_input(struct mbuf *m, struct ifnet *rcvif)
|
||||
/*
|
||||
* Unicast check
|
||||
*/
|
||||
rt = rtcache_lookup2(&ip6_forward_rt, &u.dst, 1, &hit);
|
||||
ro = percpu_getref(ip6_forward_rt_percpu);
|
||||
rt = rtcache_lookup2(ro, &u.dst, 1, &hit);
|
||||
percpu_putref(ip6_forward_rt_percpu);
|
||||
if (hit)
|
||||
IP6_STATINC(IP6_STAT_FORWARD_CACHEHIT);
|
||||
else
|
||||
|
||||
Loading…
Reference in New Issue
Block a user