Don't use rt_walktree to delete routes

Some functions use rt_walktree to scan the routing table and delete
matched routes. However, we shouldn't use rt_walktree to delete
routes because rt_walktree is recursive to the routing table (radix
tree) and isn't friendly to MP-ification. rt_walktree allows a caller
to pass a callback function to delete an matched entry. The callback
function is called from an API of the radix tree (rn_walktree) but
also calls an API of the radix tree to delete an entry.

This change adds a new API of the radix tree, rn_search_matched,
which returns a matched entry that is selected by a callback
function passed by a caller and the caller itself deletes the
entry. By using the API, we can avoid the recursive form.

sys/net/if.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: if.c,v 1.361 2016/11/05 23:30:22 pgoyette Exp $	*/
+/*	$NetBSD: if.c,v 1.362 2016/11/15 01:50:06 ozaki-r Exp $	*/
 
 /*-
  * Copyright (c) 1999, 2000, 2001, 2008 The NetBSD Foundation, Inc.
@@ -90,7 +90,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if.c,v 1.361 2016/11/05 23:30:22 pgoyette Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if.c,v 1.362 2016/11/15 01:50:06 ozaki-r Exp $");
 
 #if defined(_KERNEL_OPT)
 #include "opt_inet.h"
@@ -185,7 +185,7 @@ int	ifqmaxlen = IFQ_MAXLEN;
 
 struct psref_class		*ifa_psref_class __read_mostly;
 
-static int	if_rt_walktree(struct rtentry *, void *);
+static int	if_delroute_matcher(struct rtentry *, void *);
 
 static struct if_clone *if_clone_lookup(const char *, int *);
 
@@ -1279,11 +1279,9 @@ again:
 
 	if_free_sadl(ifp);
 
-	/* Walk the routing table looking for stragglers. */
+	/* Delete stray routes from the routing table. */
-	for (i = 0; i <= AF_MAX; i++) {
+	for (i = 0; i <= AF_MAX; i++)
-		while (rt_walktree(i, if_rt_walktree, ifp) == ERESTART)
+		rt_delete_matched_entries(i, if_delroute_matcher, ifp);
-			continue;
-	}
 
 	DOMAIN_FOREACH(dp) {
 		if (dp->dom_ifdetach != NULL && ifp->if_afdata[dp->dom_family])
@@ -1403,28 +1401,14 @@ if_detach_queues(struct ifnet *ifp, struct ifqueue *q)
  * ifnet.
  */
 static int
-if_rt_walktree(struct rtentry *rt, void *v)
+if_delroute_matcher(struct rtentry *rt, void *v)
 {
 	struct ifnet *ifp = (struct ifnet *)v;
-	int error;
-	struct rtentry *retrt;
 
-	if (rt->rt_ifp != ifp)
+	if (rt->rt_ifp == ifp)
+		return 1;
+	else
 		return 0;
-
-	/* Delete the entry. */
-	error = rtrequest(RTM_DELETE, rt_getkey(rt), rt->rt_gateway,
-	    rt_mask(rt), rt->rt_flags, &retrt);
-	if (error == 0) {
-		KASSERT(retrt == rt);
-		KASSERT((retrt->rt_flags & RTF_UP) == 0);
-		retrt->rt_ifp = NULL;
-		rtfree(retrt);
-	} else {
-		printf("%s: warning: unable to delete rtentry @ %p, "
-		    "error = %d\n", ifp->if_xname, rt, error);
-	}
-	return ERESTART;
 }
 
 /*

sys/net/radix.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: radix.c,v 1.45 2015/08/24 22:21:26 pooka Exp $	*/
+/*	$NetBSD: radix.c,v 1.46 2016/11/15 01:50:06 ozaki-r Exp $	*/
 
 /*
  * Copyright (c) 1988, 1989, 1993
@@ -36,7 +36,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: radix.c,v 1.45 2015/08/24 22:21:26 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: radix.c,v 1.46 2016/11/15 01:50:06 ozaki-r Exp $");
 
 #ifndef _NET_RADIX_H_
 #include <sys/param.h>
@@ -1005,6 +1005,37 @@ rn_walktree(
 	/* NOTREACHED */
 }
 
+struct radix_node *
+rn_search_matched(struct radix_node_head *h,
+    int (*matcher)(struct radix_node *, void *), void *w)
+{
+	bool matched;
+	struct radix_node *base, *next, *rn;
+	/*
+	 * This gets complicated because we may delete the node
+	 * while applying the function f to it, so we need to calculate
+	 * the successor node in advance.
+	 */
+	rn = rn_walkfirst(h->rnh_treetop, NULL, NULL);
+	for (;;) {
+		base = rn;
+		next = rn_walknext(rn, NULL, NULL);
+		/* Process leaves */
+		while ((rn = base) != NULL) {
+			base = rn->rn_dupedkey;
+			if (!(rn->rn_flags & RNF_ROOT)) {
+				matched = (*matcher)(rn, w);
+				if (matched)
+					return rn;
+			}
+		}
+		rn = next;
+		if (rn->rn_flags & RNF_ROOT)
+			return NULL;
+	}
+	/* NOTREACHED */
+}
+
 struct delayinit {
 	void **head;
 	int off;

sys/net/radix.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: radix.h,v 1.22 2009/05/27 17:46:50 pooka Exp $	*/
+/*	$NetBSD: radix.h,v 1.23 2016/11/15 01:50:06 ozaki-r Exp $	*/
 
 /*
  * Copyright (c) 1988, 1989, 1993
@@ -139,6 +139,10 @@ int	rn_refines(const void *, const void *);
 int	rn_walktree(struct radix_node_head *,
 	            int (*)(struct radix_node *, void *),
 		    void *);
+struct radix_node *
+	rn_search_matched(struct radix_node_head *,
+	                  int (*)(struct radix_node *, void *),
+		          void *);
 struct radix_node
 	 *rn_addmask(const void *, int, int),
 	 *rn_addroute(const void *, const void *, struct radix_node_head *,

sys/net/route.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: route.c,v 1.181 2016/10/25 02:45:09 ozaki-r Exp $	*/
+/*	$NetBSD: route.c,v 1.182 2016/11/15 01:50:06 ozaki-r Exp $	*/
 
 /*-
  * Copyright (c) 1998, 2008 The NetBSD Foundation, Inc.
@@ -97,7 +97,7 @@
 #endif
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: route.c,v 1.181 2016/10/25 02:45:09 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: route.c,v 1.182 2016/11/15 01:50:06 ozaki-r Exp $");
 
 #include <sys/param.h>
 #ifdef RTFLUSH_DEBUG
@@ -116,6 +116,7 @@ __KERNEL_RCSID(0, "$NetBSD: route.c,v 1.181 2016/10/25 02:45:09 ozaki-r Exp $");
 #include <sys/pool.h>
 #include <sys/kauth.h>
 #include <sys/workqueue.h>
+#include <sys/syslog.h>
 
 #include <net/if.h>
 #include <net/if_dl.h>
@@ -1634,6 +1635,44 @@ rt_check_reject_route(const struct rtentry *rt, const struct ifnet *ifp)
 	return 0;
 }
 
+void
+rt_delete_matched_entries(sa_family_t family, int (*f)(struct rtentry *, void *),
+    void *v)
+{
+
+	for (;;) {
+		int s;
+		int error;
+		struct rtentry *rt, *retrt = NULL;
+
+		s = splsoftnet();
+		rt = rtbl_search_matched_entry(family, f, v);
+		if (rt == NULL) {
+			splx(s);
+			return;
+		}
+		rt->rt_refcnt++;
+		splx(s);
+
+		error = rtrequest(RTM_DELETE, rt_getkey(rt), rt->rt_gateway,
+		    rt_mask(rt), rt->rt_flags, &retrt);
+		if (error == 0) {
+			KASSERT(retrt == rt);
+			KASSERT((retrt->rt_flags & RTF_UP) == 0);
+			retrt->rt_ifp = NULL;
+			rtfree(rt);
+			rtfree(retrt);
+		} else if (error == ESRCH) {
+			/* Someone deleted the entry already. */
+			rtfree(rt);
+		} else {
+			log(LOG_ERR, "%s: unable to delete rtentry @ %p, "
+			    "error = %d\n", rt->rt_ifp->if_xname, rt, error);
+			/* XXX how to treat this case? */
+		}
+	}
+}
+
 #ifdef DDB
 
 #include <machine/db_machdep.h>

sys/net/route.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: route.h,v 1.106 2016/10/25 02:45:09 ozaki-r Exp $	*/
+/*	$NetBSD: route.h,v 1.107 2016/11/15 01:50:06 ozaki-r Exp $	*/
 
 /*
  * Copyright (c) 1980, 1986, 1993
@@ -408,6 +408,8 @@ struct sockaddr *
 	rt_gettag(const struct rtentry *);
 
 int	rt_check_reject_route(const struct rtentry *, const struct ifnet *);
+void	rt_delete_matched_entries(sa_family_t,
+	    int (*)(struct rtentry *, void *), void *);
 
 static inline void
 rt_assert_referenced(const struct rtentry *rt)
@@ -504,6 +506,9 @@ struct rtentry *
 	rt_matchaddr(rtbl_t *, const struct sockaddr *);
 int	rt_refines(const struct sockaddr *, const struct sockaddr *);
 int	rt_walktree(sa_family_t, int (*)(struct rtentry *, void *), void *);
+struct rtentry *
+	rtbl_search_matched_entry(sa_family_t,
+	    int (*)(struct rtentry *, void *), void *);
 void	rtbl_init(void);
 
 #endif /* _KERNEL */

sys/net/rtbl.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: rtbl.c,v 1.3 2016/04/11 09:21:18 ozaki-r Exp $	*/
+/*	$NetBSD: rtbl.c,v 1.4 2016/11/15 01:50:06 ozaki-r Exp $	*/
 
 /*-
  * Copyright (c) 1998, 2008, 2011 The NetBSD Foundation, Inc.
@@ -95,7 +95,7 @@
 #endif /* _KERNEL && _KERNEL_OPT */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: rtbl.c,v 1.3 2016/04/11 09:21:18 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: rtbl.c,v 1.4 2016/11/15 01:50:06 ozaki-r Exp $");
 
 #include <sys/param.h>
 #include <sys/kmem.h>
@@ -204,6 +204,23 @@ rt_walktree(sa_family_t family, int (*f)(struct rtentry *, void *), void *v)
 	return rn_walktree(&t->t_rnh, rt_walktree_visitor, &rw);
 }
 
+struct rtentry *
+rtbl_search_matched_entry(sa_family_t family,
+    int (*f)(struct rtentry *, void *), void *v)
+{
+	rtbl_t *t = rt_tables[family];
+	struct rtwalk rw;
+
+	if (t == NULL)
+		return 0;
+
+	rw.rw_f = f;
+	rw.rw_v = v;
+
+	return (struct rtentry *)
+	    rn_search_matched(&t->t_rnh, rt_walktree_visitor, &rw);
+}
+
 rtbl_t *
 rt_gettable(sa_family_t af)
 {

sys/netinet6/nd6_rtr.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: nd6_rtr.c,v 1.119 2016/08/16 10:31:57 roy Exp $	*/
+/*	$NetBSD: nd6_rtr.c,v 1.120 2016/11/15 01:50:06 ozaki-r Exp $	*/
 /*	$KAME: nd6_rtr.c,v 1.95 2001/02/07 08:09:47 itojun Exp $	*/
 
 /*
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nd6_rtr.c,v 1.119 2016/08/16 10:31:57 roy Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nd6_rtr.c,v 1.120 2016/11/15 01:50:06 ozaki-r Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -80,7 +80,7 @@ static void in6_init_address_ltimes(struct nd_prefix *,
 	struct in6_addrlifetime *);
 static void purge_detached(struct ifnet *);
 
-static int rt6_deleteroute(struct rtentry *, void *);
+static int rt6_deleteroute_matcher(struct rtentry *, void *);
 
 extern int nd6_recalc_reachtm_interval;
 
@@ -2162,12 +2162,12 @@ rt6_flush(struct in6_addr *gateway, struct ifnet *ifp)
 		return;
 	}
 
-	rt_walktree(AF_INET6, rt6_deleteroute, (void *)gateway);
+	rt_delete_matched_entries(AF_INET6, rt6_deleteroute_matcher, gateway);
 	splx(s);
 }
 
 static int
-rt6_deleteroute(struct rtentry *rt, void *arg)
+rt6_deleteroute_matcher(struct rtentry *rt, void *arg)
 {
 	struct in6_addr *gate = (struct in6_addr *)arg;
 
@@ -2192,8 +2192,7 @@ rt6_deleteroute(struct rtentry *rt, void *arg)
 	if ((rt->rt_flags & RTF_HOST) == 0)
 		return (0);
 
-	return (rtrequest(RTM_DELETE, rt_getkey(rt), rt->rt_gateway,
+	return 1;
-	    rt_mask(rt), rt->rt_flags, NULL));
 }
 
 int

sys/nfs/nfs_boot.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: nfs_boot.c,v 1.86 2016/07/07 06:55:43 msaitoh Exp $	*/
+/*	$NetBSD: nfs_boot.c,v 1.87 2016/11/15 01:50:06 ozaki-r Exp $	*/
 
 /*-
  * Copyright (c) 1995, 1997 The NetBSD Foundation, Inc.
@@ -35,7 +35,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nfs_boot.c,v 1.86 2016/07/07 06:55:43 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nfs_boot.c,v 1.87 2016/11/15 01:50:06 ozaki-r Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_nfs.h"
@@ -97,7 +97,7 @@ int nfs_boot_bootstatic = 1; /* BOOTSTATIC enabled (default) */
 static int md_mount(struct sockaddr_in *mdsin, char *path,
 	struct nfs_args *argp, struct lwp *l);
 
-static int nfs_boot_delroute(struct rtentry *, void *);
+static int nfs_boot_delroute_matcher(struct rtentry *, void *);
 static void nfs_boot_defrt(struct in_addr *);
 static  int nfs_boot_getfh(struct nfs_dlmount *ndm, struct lwp *);
 
@@ -559,26 +559,20 @@ nfs_boot_defrt(struct in_addr *gw_ip)
 }
 
 static int
-nfs_boot_delroute(struct rtentry *rt, void *w)
+nfs_boot_delroute_matcher(struct rtentry *rt, void *w)
 {
-	int error;
 
 	if ((void *)rt->rt_ifp != w)
 		return 0;
 
-	error = rtrequest(RTM_DELETE, rt_getkey(rt), NULL, rt_mask(rt), 0,
+	return 1;
-	    NULL);
-	if (error != 0)
-		printf("%s: del route, error=%d\n", __func__, error);
-
-	return 0;
 }
 
 void
 nfs_boot_flushrt(struct ifnet *ifp)
 {
 
-	rt_walktree(AF_INET, nfs_boot_delroute, ifp);
+	rt_delete_matched_entries(AF_INET, nfs_boot_delroute_matcher, ifp);
 }
 
 /*