Aren
/
NetBSD
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Modularize the raidframe driver, including rework of the unit attach 

code to permit detaching (and possible module unloading).  Also,
convert tsleep()/wakeup() locking to use cv_wait_sig()/cv_broadcast().

Tested in non-modular, modular-builtin, and modular-loaded-at-runtime
environments.
This commit is contained in:
pgoyette 2015-12-26 00:58:45 +00:00
parent 0941badb50
commit e05b240459
3 changed files with 251 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
sys/dev/raidframe/rf_driver.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: rf_driver.c,v 1.131 2012/12/10 08:36:03 msaitoh Exp $ */ /* $NetBSD: rf_driver.c,v 1.132 2015/12/26 00:58:45 pgoyette Exp $ */
/*- /*-
* Copyright (c) 1999 The NetBSD Foundation, Inc. * Copyright (c) 1999 The NetBSD Foundation, Inc.
* All rights reserved. * All rights reserved.
@ -66,7 +66,7 @@
#include <sys/cdefs.h> #include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: rf_driver.c,v 1.131 2012/12/10 08:36:03 msaitoh Exp $"); __KERNEL_RCSID(0, "$NetBSD: rf_driver.c,v 1.132 2015/12/26 00:58:45 pgoyette Exp $");
#ifdef _KERNEL_OPT #ifdef _KERNEL_OPT
#include "opt_raid_diagnostic.h" #include "opt_raid_diagnostic.h"
@ -158,9 +158,10 @@ static void rf_alloc_mutex_cond(RF_Raid_t *);
/* called at system boot time */ /* called at system boot time */
int int
rf_BootRaidframe(void) rf_BootRaidframe(bool boot)
{ {
if (boot) {
if (raidframe_booted) if (raidframe_booted)
return (EBUSY); return (EBUSY);
raidframe_booted = 1; raidframe_booted = 1;
@ -168,6 +169,10 @@ rf_BootRaidframe(void)
configureCount = 0; configureCount = 0;
isconfigged = 0; isconfigged = 0;
globalShutdown = NULL; globalShutdown = NULL;
} else {
rf_destroy_mutex2(configureMutex);
raidframe_booted = 0;
}
return (0); return (0);
} }

4
sys/dev/raidframe/rf_driver.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: rf_driver.h,v 1.19 2011/04/30 01:44:36 mrg Exp $ */ /* $NetBSD: rf_driver.h,v 1.20 2015/12/26 00:58:45 pgoyette Exp $ */
/* /*
* rf_driver.h * rf_driver.h
*/ */
@ -42,7 +42,7 @@
#endif #endif
extern rf_declare_mutex2(rf_printf_mutex); extern rf_declare_mutex2(rf_printf_mutex);
int rf_BootRaidframe(void); int rf_BootRaidframe(bool);
int rf_UnbootRaidframe(void); int rf_UnbootRaidframe(void);
int rf_Shutdown(RF_Raid_t *); int rf_Shutdown(RF_Raid_t *);
int rf_Configure(RF_Raid_t *, RF_Config_t *, RF_AutoConfig_t *); int rf_Configure(RF_Raid_t *, RF_Config_t *, RF_AutoConfig_t *);

287
sys/dev/raidframe/rf_netbsdkintf.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: rf_netbsdkintf.c,v 1.326 2015/12/08 20:36:15 christos Exp $ */ /* $NetBSD: rf_netbsdkintf.c,v 1.327 2015/12/26 00:58:45 pgoyette Exp $ */
/*- /*-
* Copyright (c) 1996, 1997, 1998, 2008-2011 The NetBSD Foundation, Inc. * Copyright (c) 1996, 1997, 1998, 2008-2011 The NetBSD Foundation, Inc.
@ -101,7 +101,7 @@
***********************************************************/ ***********************************************************/
#include <sys/cdefs.h> #include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: rf_netbsdkintf.c,v 1.326 2015/12/08 20:36:15 christos Exp $"); __KERNEL_RCSID(0, "$NetBSD: rf_netbsdkintf.c,v 1.327 2015/12/26 00:58:45 pgoyette Exp $");
#ifdef _KERNEL_OPT #ifdef _KERNEL_OPT
#include "opt_compat_netbsd.h" #include "opt_compat_netbsd.h"
@ -126,6 +126,7 @@ __KERNEL_RCSID(0, "$NetBSD: rf_netbsdkintf.c,v 1.326 2015/12/08 20:36:15 christo
#include <sys/bufq.h> #include <sys/bufq.h>
#include <sys/reboot.h> #include <sys/reboot.h>
#include <sys/kauth.h> #include <sys/kauth.h>
#include <sys/module.h>
#include <prop/proplib.h> #include <prop/proplib.h>
@ -241,6 +242,8 @@ struct raid_softc {
int sc_unit; int sc_unit;
int sc_flags; /* flags */ int sc_flags; /* flags */
int sc_cflags; /* configuration flags */ int sc_cflags; /* configuration flags */
kmutex_t sc_mutex; /* interlock mutex */
kcondvar_t sc_cv; /* and the condvar */
uint64_t sc_size; /* size of the raid device */ uint64_t sc_size; /* size of the raid device */
char sc_xname[20]; /* XXX external name */ char sc_xname[20]; /* XXX external name */
struct disk sc_dkdev; /* generic disk device info */ struct disk sc_dkdev; /* generic disk device info */
@ -350,17 +353,21 @@ raidcreate(int unit) {
} }
sc->sc_unit = unit; sc->sc_unit = unit;
bufq_alloc(&sc->buf_queue, "fcfs", BUFQ_SORT_RAWBLOCK); bufq_alloc(&sc->buf_queue, "fcfs", BUFQ_SORT_RAWBLOCK);
cv_init(&sc->sc_cv, "raidunit");
mutex_init(&sc->sc_mutex, MUTEX_DEFAULT, IPL_NONE);
return sc; return sc;
} }
static void static void
raiddestroy(struct raid_softc *sc) { raiddestroy(struct raid_softc *sc) {
cv_destroy(&sc->sc_cv);
mutex_destroy(&sc->sc_mutex);
bufq_free(sc->buf_queue); bufq_free(sc->buf_queue);
kmem_free(sc, sizeof(*sc)); kmem_free(sc, sizeof(*sc));
} }
static struct raid_softc * static struct raid_softc *
raidget(int unit) { raidget(int unit, bool create) {
struct raid_softc *sc; struct raid_softc *sc;
if (unit < 0) { if (unit < 0) {
#ifdef DIAGNOSTIC #ifdef DIAGNOSTIC
@ -376,6 +383,8 @@ raidget(int unit) {
} }
} }
mutex_exit(&raid_lock); mutex_exit(&raid_lock);
if (!create)
return NULL;
if ((sc = raidcreate(unit)) == NULL) if ((sc = raidcreate(unit)) == NULL)
return NULL; return NULL;
mutex_enter(&raid_lock); mutex_enter(&raid_lock);
@ -395,34 +404,11 @@ raidput(struct raid_softc *sc) {
void void
raidattach(int num) raidattach(int num)
{ {
mutex_init(&raid_lock, MUTEX_DEFAULT, IPL_NONE);
/* This is where all the initialization stuff gets done. */
#if (RF_INCLUDE_PARITY_DECLUSTERING_DS > 0)
rf_init_mutex2(rf_sparet_wait_mutex, IPL_VM);
rf_init_cond2(rf_sparet_wait_cv, "sparetw");
rf_init_cond2(rf_sparet_resp_cv, "rfgst");
rf_sparet_wait_queue = rf_sparet_resp_queue = NULL;
#endif
if (rf_BootRaidframe() == 0)
aprint_verbose("Kernelized RAIDframe activated\n");
else
panic("Serious error booting RAID!!");
if (config_cfattach_attach(raid_cd.cd_name, &raid_ca)) {
aprint_error("raidattach: config_cfattach_attach failed?\n");
}
raidautoconfigdone = false;
/* /*
* Register a finalizer which will be used to auto-config RAID * Device attachment and associated initialization now occurs
* sets once all real hardware devices have been found. * as part of the module initialization.
*/ */
if (config_finalize_register(NULL, rf_autoconfig) != 0)
aprint_error("WARNING: unable to register RAIDframe finalizer\n");
} }
int int
@ -606,7 +592,7 @@ raidsize(dev_t dev)
int part, unit, omask, size; int part, unit, omask, size;
unit = raidunit(dev); unit = raidunit(dev);
if ((rs = raidget(unit)) == NULL) if ((rs = raidget(unit, false)) == NULL)
return -1; return -1;
if ((rs->sc_flags & RAIDF_INITED) == 0) if ((rs->sc_flags & RAIDF_INITED) == 0)
return (-1); return (-1);
@ -643,7 +629,7 @@ raiddump(dev_t dev, daddr_t blkno, void *va, size_t size)
int part, c, sparecol, j, scol, dumpto; int part, c, sparecol, j, scol, dumpto;
int error = 0; int error = 0;
if ((rs = raidget(unit)) == NULL) if ((rs = raidget(unit, false)) == NULL)
return ENXIO; return ENXIO;
raidPtr = &rs->sc_r; raidPtr = &rs->sc_r;
@ -656,7 +642,6 @@ raiddump(dev_t dev, daddr_t blkno, void *va, size_t size)
raidPtr->Layout.numParityCol != 1) raidPtr->Layout.numParityCol != 1)
return EINVAL; return EINVAL;
if ((error = raidlock(rs)) != 0) if ((error = raidlock(rs)) != 0)
return error; return error;
@ -779,7 +764,7 @@ raidopen(dev_t dev, int flags, int fmt,
int part, pmask; int part, pmask;
int error = 0; int error = 0;
if ((rs = raidget(unit)) == NULL) if ((rs = raidget(unit, true)) == NULL)
return ENXIO; return ENXIO;
if ((error = raidlock(rs)) != 0) if ((error = raidlock(rs)) != 0)
return (error); return (error);
@ -863,7 +848,7 @@ raidclose(dev_t dev, int flags, int fmt, struct lwp *l)
int error = 0; int error = 0;
int part; int part;
if ((rs = raidget(unit)) == NULL) if ((rs = raidget(unit, false)) == NULL)
return ENXIO; return ENXIO;
if ((error = raidlock(rs)) != 0) if ((error = raidlock(rs)) != 0)
@ -893,15 +878,31 @@ raidclose(dev_t dev, int flags, int fmt, struct lwp *l)
rf_update_component_labels(&rs->sc_r, rf_update_component_labels(&rs->sc_r,
RF_FINAL_COMPONENT_UPDATE); RF_FINAL_COMPONENT_UPDATE);
}
/* If the kernel is shutting down, it will detach if ((rs->sc_dkdev.dk_openmask == 0) &&
* this RAID set soon enough. ((rs->sc_flags & RAIDF_SHUTDOWN) != 0)) {
/*
* Detach this raid unit
*/ */
cfdata_t cf = NULL;
int retcode = 0;
if (rs->sc_dev != NULL) {
cf = device_cfdata(rs->sc_dev);
raidunlock(rs);
retcode = config_detach(rs->sc_dev, DETACH_QUIET);
if (retcode == 0)
/* free the pseudo device attach bits */
free(cf, M_RAIDFRAME);
} else {
raidput(rs);
}
return retcode;
} }
raidunlock(rs); raidunlock(rs);
return (0); return (0);
} }
static void static void
@ -912,7 +913,7 @@ raidstrategy(struct buf *bp)
int wlabel; int wlabel;
struct raid_softc *rs; struct raid_softc *rs;
if ((rs = raidget(unit)) == NULL) { if ((rs = raidget(unit, false)) == NULL) {
bp->b_error = ENXIO; bp->b_error = ENXIO;
goto done; goto done;
} }
@ -982,7 +983,7 @@ raidread(dev_t dev, struct uio *uio, int flags)
int unit = raidunit(dev); int unit = raidunit(dev);
struct raid_softc *rs; struct raid_softc *rs;
if ((rs = raidget(unit)) == NULL) if ((rs = raidget(unit, false)) == NULL)
return ENXIO; return ENXIO;
if ((rs->sc_flags & RAIDF_INITED) == 0) if ((rs->sc_flags & RAIDF_INITED) == 0)
@ -999,7 +1000,7 @@ raidwrite(dev_t dev, struct uio *uio, int flags)
int unit = raidunit(dev); int unit = raidunit(dev);
struct raid_softc *rs; struct raid_softc *rs;
if ((rs = raidget(unit)) == NULL) if ((rs = raidget(unit, false)) == NULL)
return ENXIO; return ENXIO;
if ((rs->sc_flags & RAIDF_INITED) == 0) if ((rs->sc_flags & RAIDF_INITED) == 0)
@ -1036,6 +1037,9 @@ raid_detach_unlocked(struct raid_softc *rs)
disk_detach(&rs->sc_dkdev); disk_detach(&rs->sc_dkdev);
disk_destroy(&rs->sc_dkdev); disk_destroy(&rs->sc_dkdev);
/* Free the softc */
raidput(rs);
aprint_normal_dev(rs->sc_dev, "detached\n"); aprint_normal_dev(rs->sc_dev, "detached\n");
return 0; return 0;
@ -1070,7 +1074,7 @@ raidioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
struct disklabel newlabel; struct disklabel newlabel;
#endif #endif
if ((rs = raidget(unit)) == NULL) if ((rs = raidget(unit, false)) == NULL)
return ENXIO; return ENXIO;
raidPtr = &rs->sc_r; raidPtr = &rs->sc_r;
@ -1190,23 +1194,27 @@ raidioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
RF_Free(k_cfg, sizeof(RF_Config_t)); RF_Free(k_cfg, sizeof(RF_Config_t));
db1_printf(("rf_ioctl: retcode=%d copyin.1\n", db1_printf(("rf_ioctl: retcode=%d copyin.1\n",
retcode)); retcode));
return (retcode); goto no_config;
} }
goto config; goto config;
config: config:
rs->sc_flags &= ~RAIDF_SHUTDOWN;
/* allocate a buffer for the layout-specific data, and copy it /* allocate a buffer for the layout-specific data, and copy it
* in */ * in */
if (k_cfg->layoutSpecificSize) { if (k_cfg->layoutSpecificSize) {
if (k_cfg->layoutSpecificSize > 10000) { if (k_cfg->layoutSpecificSize > 10000) {
/* sanity check */ /* sanity check */
RF_Free(k_cfg, sizeof(RF_Config_t)); RF_Free(k_cfg, sizeof(RF_Config_t));
return (EINVAL); retcode = EINVAL;
goto no_config;
} }
RF_Malloc(specific_buf, k_cfg->layoutSpecificSize, RF_Malloc(specific_buf, k_cfg->layoutSpecificSize,
(u_char *)); (u_char *));
if (specific_buf == NULL) { if (specific_buf == NULL) {
RF_Free(k_cfg, sizeof(RF_Config_t)); RF_Free(k_cfg, sizeof(RF_Config_t));
return (ENOMEM); retcode = ENOMEM;
goto no_config;
} }
retcode = copyin(k_cfg->layoutSpecific, specific_buf, retcode = copyin(k_cfg->layoutSpecific, specific_buf,
k_cfg->layoutSpecificSize); k_cfg->layoutSpecificSize);
@ -1216,7 +1224,7 @@ raidioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
k_cfg->layoutSpecificSize); k_cfg->layoutSpecificSize);
db1_printf(("rf_ioctl: retcode=%d copyin.2\n", db1_printf(("rf_ioctl: retcode=%d copyin.2\n",
retcode)); retcode));
return (retcode); goto no_config;
} }
} else } else
specific_buf = NULL; specific_buf = NULL;
@ -1253,6 +1261,13 @@ raidioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
} }
RF_Free(k_cfg, sizeof(RF_Config_t)); RF_Free(k_cfg, sizeof(RF_Config_t));
no_config:
/*
* If configuration failed, set sc_flags so that we
* will detach the device when we close it.
*/
if (retcode != 0)
rs->sc_flags |= RAIDF_SHUTDOWN;
return (retcode); return (retcode);
/* shutdown the system */ /* shutdown the system */
@ -2391,7 +2406,7 @@ raidgetdisklabel(dev_t dev)
struct cpu_disklabel *clp; struct cpu_disklabel *clp;
RF_Raid_t *raidPtr; RF_Raid_t *raidPtr;
if ((rs = raidget(unit)) == NULL) if ((rs = raidget(unit, false)) == NULL)
return; return;
lp = rs->sc_dkdev.dk_label; lp = rs->sc_dkdev.dk_label;
@ -2476,13 +2491,15 @@ raidlock(struct raid_softc *rs)
{ {
int error; int error;
mutex_enter(&rs->sc_mutex);
while ((rs->sc_flags & RAIDF_LOCKED) != 0) { while ((rs->sc_flags & RAIDF_LOCKED) != 0) {
rs->sc_flags |= RAIDF_WANTED; rs->sc_flags |= RAIDF_WANTED;
if ((error = error = cv_wait_sig(&rs->sc_cv, &rs->sc_mutex);
tsleep(rs, PRIBIO | PCATCH, "raidlck", 0)) != 0) if (error != 0)
return (error); return (error);
} }
rs->sc_flags |= RAIDF_LOCKED; rs->sc_flags |= RAIDF_LOCKED;
mutex_exit(&rs->sc_mutex);
return (0); return (0);
} }
/* /*
@ -2492,11 +2509,13 @@ static void
raidunlock(struct raid_softc *rs) raidunlock(struct raid_softc *rs)
{ {
mutex_enter(&rs->sc_mutex);
rs->sc_flags &= ~RAIDF_LOCKED; rs->sc_flags &= ~RAIDF_LOCKED;
if ((rs->sc_flags & RAIDF_WANTED) != 0) { if ((rs->sc_flags & RAIDF_WANTED) != 0) {
rs->sc_flags &= ~RAIDF_WANTED; rs->sc_flags &= ~RAIDF_WANTED;
wakeup(rs); cv_broadcast(&rs->sc_cv);
} }
mutex_exit(&rs->sc_mutex);
} }
@ -3765,7 +3784,7 @@ rf_auto_config_set(RF_ConfigSet_t *cset)
/* 1. Create a config structure */ /* 1. Create a config structure */
config = malloc(sizeof(*config), M_RAIDFRAME, M_NOWAIT|M_ZERO); config = malloc(sizeof(*config), M_RAIDFRAME, M_NOWAIT|M_ZERO);
if (config == NULL) { if (config == NULL) {
printf("Out of mem!?!?\n"); printf("%s: Out of mem - config!?!?\n", __func__);
/* XXX do something more intelligent here. */ /* XXX do something more intelligent here. */
return NULL; return NULL;
} }
@ -3777,12 +3796,22 @@ rf_auto_config_set(RF_ConfigSet_t *cset)
*/ */
raidID = cset->ac->clabel->last_unit; raidID = cset->ac->clabel->last_unit;
for (sc = raidget(raidID); sc->sc_r.valid != 0; sc = raidget(++raidID)) for (sc = raidget(raidID, false); sc && sc->sc_r.valid != 0;
sc = raidget(++raidID, false))
continue; continue;
#ifdef DEBUG #ifdef DEBUG
printf("Configuring raid%d:\n",raidID); printf("Configuring raid%d:\n",raidID);
#endif #endif
if (sc == NULL)
sc = raidget(raidID, true);
if (sc == NULL) {
printf("%s: Out of mem - softc!?!?\n", __func__);
/* XXX do something more intelligent here. */
free(config, M_RAIDFRAME);
return NULL;
}
raidPtr = &sc->sc_r; raidPtr = &sc->sc_r;
/* XXX all this stuff should be done SOMEWHERE ELSE! */ /* XXX all this stuff should be done SOMEWHERE ELSE! */
@ -3900,7 +3929,7 @@ static int
raid_detach(device_t self, int flags) raid_detach(device_t self, int flags)
{ {
int error; int error;
struct raid_softc *rs = raidget(device_unit(self)); struct raid_softc *rs = raidget(device_unit(self), false);
if (rs == NULL) if (rs == NULL)
return ENXIO; return ENXIO;
@ -3910,10 +3939,9 @@ raid_detach(device_t self, int flags)
error = raid_detach_unlocked(rs); error = raid_detach_unlocked(rs);
if (error != 0)
raidunlock(rs); raidunlock(rs);
/* XXXkd: raidput(rs) ??? */
return error; return error;
} }
@ -3979,3 +4007,154 @@ rf_sync_component_caches(RF_Raid_t *raidPtr)
} }
return error; return error;
} }
/*
* Module interface
*/
MODULE(MODULE_CLASS_DRIVER, raid, "dk_subr");
#ifdef _MODULE
CFDRIVER_DECL(raid, DV_DISK, NULL);
#endif
static int raid_modcmd(modcmd_t, void *);
static int raid_modcmd_init(void);
static int raid_modcmd_fini(void);
static int
raid_modcmd(modcmd_t cmd, void *data)
{
int error;
error = 0;
switch (cmd) {
case MODULE_CMD_INIT:
error = raid_modcmd_init();
break;
case MODULE_CMD_FINI:
error = raid_modcmd_fini();
break;
default:
error = ENOTTY;
break;
}
return error;
}
static int
raid_modcmd_init(void)
{
int error;
int bmajor, cmajor;
mutex_init(&raid_lock, MUTEX_DEFAULT, IPL_NONE);
mutex_enter(&raid_lock);
#if (RF_INCLUDE_PARITY_DECLUSTERING_DS > 0)
rf_init_mutex2(rf_sparet_wait_mutex, IPL_VM);
rf_init_cond2(rf_sparet_wait_cv, "sparetw");
rf_init_cond2(rf_sparet_resp_cv, "rfgst");
rf_sparet_wait_queue = rf_sparet_resp_queue = NULL;
#endif
bmajor = cmajor = -1;
error = devsw_attach("raid", &raid_bdevsw, &bmajor,
&raid_cdevsw, &cmajor);
if (error != 0 && error != EEXIST) {
aprint_error("%s: devsw_attach failed %d\n", __func__, error);
mutex_exit(&raid_lock);
return error;
}
#ifdef _MODULE
error = config_cfdriver_attach(&raid_cd);
if (error != 0) {
aprint_error("%s: config_cfdriver_attach failed %d\n",
__func__, error);
devsw_detach(&raid_bdevsw, &raid_cdevsw);
mutex_exit(&raid_lock);
return error;
}
#endif
error = config_cfattach_attach(raid_cd.cd_name, &raid_ca);
if (error != 0) {
aprint_error("%s: config_cfattach_attach failed %d\n",
__func__, error);
#ifdef _MODULE
config_cfdriver_detach(&raid_cd);
#endif
devsw_detach(&raid_bdevsw, &raid_cdevsw);
mutex_exit(&raid_lock);
return error;
}
raidautoconfigdone = false;
mutex_exit(&raid_lock);
if (error == 0) {
if (rf_BootRaidframe(true) == 0)
aprint_verbose("Kernelized RAIDframe activated\n");
else
panic("Serious error activating RAID!!");
}
/*
* Register a finalizer which will be used to auto-config RAID
* sets once all real hardware devices have been found.
*/
error = config_finalize_register(NULL, rf_autoconfig);
if (error != 0) {
aprint_error("WARNING: unable to register RAIDframe "
"finalizer\n");
}
return error;
}
static int
raid_modcmd_fini(void)
{
int error;
mutex_enter(&raid_lock);
/* Don't allow unload if raid device(s) exist. */
if (!LIST_EMPTY(&raids)) {
mutex_exit(&raid_lock);
return EBUSY;
}
error = config_cfattach_detach(raid_cd.cd_name, &raid_ca);
if (error != 0) {
mutex_exit(&raid_lock);
return error;
}
#ifdef _MODULE
error = config_cfdriver_detach(&raid_cd);
if (error != 0) {
config_cfattach_attach(raid_cd.cd_name, &raid_ca);
mutex_exit(&raid_lock);
return error;
}
#endif
error = devsw_detach(&raid_bdevsw, &raid_cdevsw);
if (error != 0) {
#ifdef _MODULE
config_cfdriver_attach(&raid_cd);
#endif
config_cfattach_attach(raid_cd.cd_name, &raid_ca);
mutex_exit(&raid_lock);
return error;
}
rf_BootRaidframe(false);
#if (RF_INCLUDE_PARITY_DECLUSTERING_DS > 0)
rf_destroy_mutex2(rf_sparet_wait_mutex);
rf_destroy_cond2(rf_sparet_wait_cv);
rf_destroy_cond2(rf_sparet_resp_cv);
#endif
mutex_exit(&raid_lock);
mutex_destroy(&raid_lock);
return error;
}