btreeTest: Revive test for BPlusTree implementation.

* Update pseudo cache with minimal set of block cache API.
* Update Inode, Journal and Volume stubs to current API.
* Add stubs for remaining VFS functions.
* Extend tests to also call BPlusTree::Validate() and bail on errors.
* Change duplicate tests to fill in the same value. While this isn't
  strictly allowed, the tree validation can't work unless the values
  match up.
* Lots of coding style cleanup, but much more should still be done...

The tests run through without errors.
This commit is contained in:
Michael Lotz 2014-10-21 00:11:23 +02:00
parent 9656eef895
commit 24e159e1ac
10 changed files with 315 additions and 101 deletions

View File

@ -14,6 +14,7 @@ Inode::Inode(const char *name,int32 mode)
:
fMode(mode)
{
rw_lock_init(&fLock, "inode lock");
fFile.SetTo(name,B_CREATE_FILE | B_READ_WRITE | B_ERASE_FILE);
fSize = 0;
fVolume = new Volume(&fFile);
@ -37,14 +38,14 @@ Inode::FindBlockRun(off_t pos, block_run &run, off_t &offset)
status_t
Inode::Append(Transaction *transaction, off_t bytes)
Inode::Append(Transaction& transaction, off_t bytes)
{
return SetFileSize(transaction,Size() + bytes);
return SetFileSize(transaction, Size() + bytes);
}
status_t
Inode::SetFileSize(Transaction *, off_t bytes)
Inode::SetFileSize(Transaction&, off_t bytes)
{
//printf("set size = %ld\n",bytes);
fSize = bytes;

View File

@ -11,6 +11,11 @@
#include <File.h>
#include "bfs.h"
#include "Utility.h"
#define ASSERT_READ_LOCKED_INODE(inode) inode->AssertReadLocked()
#define ASSERT_WRITE_LOCKED_INODE(inode) inode->AssertWriteLocked()
class Volume;
@ -19,32 +24,36 @@ class Transaction;
class Inode {
public:
Inode(const char *name,int32 mode = S_STR_INDEX | S_ALLOW_DUPS);
Inode(const char* name, int32 mode = S_STR_INDEX | S_ALLOW_DUPS);
~Inode();
rw_lock &Lock() { return fLock; }
rw_lock& Lock() { return fLock; }
status_t FindBlockRun(off_t pos,block_run &run,off_t &offset);
status_t Append(Transaction *,off_t bytes);
status_t SetFileSize(Transaction *,off_t bytes);
status_t FindBlockRun(off_t pos, block_run& run, off_t& offset);
status_t Append(Transaction&, off_t bytes);
status_t SetFileSize(Transaction&, off_t bytes);
Volume *GetVolume() const { return fVolume; }
Volume* GetVolume() const { return fVolume; }
off_t ID() const { return 0; }
int32 Mode() const { return fMode; }
char *Name() const { return "whatever"; }
block_run BlockRun() const { return block_run::Run(0,0,0); }
block_run Parent() const { return block_run::Run(0,0,0); }
const char* Name() const { return "whatever"; }
block_run BlockRun() const { return block_run::Run(0, 0, 0); }
block_run Parent() const { return block_run::Run(0, 0, 0); }
off_t BlockNumber() const { return 0; }
bfs_inode *Node() { return (bfs_inode *)1; }
bfs_inode* Node() { return (bfs_inode*)1; }
off_t Size() const { return fSize; }
bool IsContainer() const { return true; }
bool IsDirectory() const { return true; }
bool IsIndex() const { return is_index(Mode()); }
void AssertReadLocked() { ASSERT_READ_LOCKED_RW_LOCK(&fLock); }
void AssertWriteLocked() { ASSERT_WRITE_LOCKED_RW_LOCK(&fLock); }
private:
friend void dump_inode(Inode &inode);
friend void dump_inode(Inode& inode);
Volume *fVolume;
Volume* fVolume;
BFile fFile;
off_t fSize;
rw_lock fLock;
@ -54,8 +63,36 @@ class Inode {
off_t fOldSize;
off_t fOldLastModified;
off_t fBlockNumber;
void *fTree;
void *fAttributes;
void* fTree;
void* fAttributes;
};
class InodeReadLocker {
public:
InodeReadLocker(Inode* inode)
:
fLock(&inode->Lock())
{
rw_lock_read_lock(fLock);
}
~InodeReadLocker()
{
if (fLock != NULL)
rw_lock_read_unlock(fLock);
}
void Unlock()
{
if (fLock != NULL) {
rw_lock_read_unlock(fLock);
fLock = NULL;
}
}
private:
rw_lock* fLock;
};
#endif /* INODE_H */

View File

@ -9,20 +9,23 @@ rule FPreIncludes { return -include\ $(1:D=$(SUBDIR)) ; }
{
local defines = [ FDefines USER DEBUG ] ; # _NO_INLINE_ASM
local preIncludes = [ FPreIncludes Journal.h Inode.h ] ;
local preIncludes = [ FPreIncludes Inode.h Journal.h Volume.h ] ;
SubDirC++Flags $(defines) $(preIncludes) -fno-exceptions ; #-fcheck-memory-usage
}
SimpleTest btreeTest
: #test.cpp
: test.cpp
Volume.cpp
Inode.cpp
cache.cpp
BPlusTree.cpp
Utility.cpp
Debug.cpp
: be ;
QueryParserUtils.cpp
stubs.cpp
: be [ TargetLibstdc++ ] libkernelland_emu.so ;
# Tell Jam where to find these sources
SEARCH on [ FGristFiles BPlusTree.cpp Utility.cpp Debug.cpp ]
SEARCH on [ FGristFiles BPlusTree.cpp Debug.cpp ]
= [ FDirName $(HAIKU_TOP) src add-ons kernel file_systems bfs ] ;
SEARCH on [ FGristFiles QueryParserUtils.cpp ]
= [ FDirName $(HAIKU_TOP) src add-ons kernel file_systems shared ] ;

View File

@ -19,8 +19,13 @@
class TransactionListener
: public DoublyLinkedListLinkImpl<TransactionListener> {
public:
TransactionListener();
virtual ~TransactionListener();
TransactionListener()
{
}
virtual ~TransactionListener()
{
}
virtual void TransactionDone(bool success) = 0;
virtual void RemovedFromTransaction() = 0;
@ -31,9 +36,10 @@ typedef DoublyLinkedList<TransactionListener> TransactionListeners;
class Transaction {
public:
Transaction(Volume *volume,off_t refBlock)
Transaction(Volume *volume, off_t refBlock)
:
fVolume(volume)
fVolume(volume),
fID(volume->GenerateTransactionID())
{
}
@ -41,17 +47,62 @@ class Transaction {
{
}
status_t WriteBlocks(off_t blockNumber,const uint8 *buffer,size_t numBlocks = 1)
int32 ID() const
{
return cached_write(fVolume->Device(),blockNumber,buffer,numBlocks,fVolume->BlockSize());
return fID;
}
void Done()
Volume* GetVolume()
{
return fVolume;
}
status_t WriteBlocks(off_t blockNumber, const uint8* buffer,
size_t numBlocks = 1)
{
return cached_write(fVolume->Device(), blockNumber, buffer,
numBlocks);
}
status_t Start(Volume* volume, off_t refBlock)
{
return B_OK;
}
status_t Done()
{
NotifyListeners(true);
return B_OK;
}
void
AddListener(TransactionListener* listener)
{
fListeners.Add(listener);
}
void
RemoveListener(TransactionListener* listener)
{
fListeners.Remove(listener);
listener->RemovedFromTransaction();
}
void
NotifyListeners(bool success)
{
while (TransactionListener* listener = fListeners.RemoveHead()) {
listener->TransactionDone(success);
listener->RemovedFromTransaction();
}
}
protected:
Volume *fVolume;
Volume* fVolume;
int32 fID;
TransactionListeners fListeners;
};

View File

@ -16,3 +16,10 @@ Volume::Panic()
printf("PANIC!\n");
}
int32
Volume::GenerateTransactionID()
{
static int32 sTransactionID = 1;
return sTransactionID++;
}

View File

@ -19,14 +19,26 @@ class Volume {
public:
Volume(BFile *file) : fFile(file) {}
BFile *Device() { return fFile; }
bool IsInitializing() const { return false; }
bool IsValidInodeBlock(off_t) const { return true; }
BFile *Device() { return fFile; }
void *BlockCache() { return fFile; }
fs_volume *FSVolume() { return NULL; }
uint32 BlockSize() const { return 1024; }
uint32 BlockShift() const { return 10; /* 2^BlockShift == BlockSize */ }
int32 BlockSize() const { return 1024; }
off_t ToBlock(block_run run) const { return run.start; }
block_run ToBlockRun(off_t block) const { return block_run::Run(0,0,block); }
block_run ToBlockRun(off_t block) const
{
return block_run::Run(0, 0, block);
}
static void Panic();
int32 GenerateTransactionID();
private:
BFile *fFile;
};

View File

@ -21,86 +21,143 @@
*/
#define TRACE(x) /*printf x*/
static size_t sBlockSize;
BList gBlocks;
void
init_cache(BFile */*file*/,int32 /*blockSize*/)
init_cache(BFile* /*file*/, int32 blockSize)
{
sBlockSize = blockSize;
}
void
shutdown_cache(BFile *file,int32 blockSize)
shutdown_cache(BFile* file, int32 blockSize)
{
for (int32 i = 0;i < gBlocks.CountItems();i++) {
void *buffer = gBlocks.ItemAt(i);
for (int32 i = 0; i < gBlocks.CountItems(); i++) {
void* buffer = gBlocks.ItemAt(i);
if (buffer == NULL) {
printf("cache is corrupt!\n");
debugger("cache is corrupt!");
exit(-1);
}
file->WriteAt(i * blockSize,buffer,blockSize);
file->WriteAt(i * blockSize, buffer, blockSize);
free(buffer);
}
}
static status_t
readBlocks(BFile *file,uint32 num,uint32 size)
status_t
cached_write(void* cache, off_t num, const void* _data, off_t numBlocks)
{
if (num + numBlocks > gBlocks.CountItems()) {
debugger("cached write beyond loaded blocks");
exit(1);
}
for (off_t i = 0; i < numBlocks; i++) {
void* buffer = gBlocks.ItemAt(num + i);
const void* data = (uint8*)_data + i * sBlockSize;
if (buffer != data)
memcpy(buffer, data, sBlockSize);
}
return B_OK;
}
static status_t
read_blocks(void* cache, off_t num)
{
BFile* file = (BFile*)cache;
for (uint32 i = gBlocks.CountItems(); i <= num; i++) {
void *buffer = malloc(size);
void* buffer = malloc(sBlockSize);
if (buffer == NULL)
return B_NO_MEMORY;
gBlocks.AddItem(buffer);
if (file->ReadAt(i * size,buffer,size) < B_OK)
if (file->ReadAt(i * sBlockSize, buffer, sBlockSize) < 0)
return B_IO_ERROR;
}
return B_OK;
}
int
cached_write(BFile *file, off_t num,const void *data,off_t numBlocks, int blockSize)
{
//printf("cached_write(num = %Ld,data = %p,numBlocks = %Ld,blockSize = %ld)\n",num,data,numBlocks,blockSize);
if (file == NULL)
return B_BAD_VALUE;
if (num >= gBlocks.CountItems())
puts("Oh no!");
void *buffer = gBlocks.ItemAt(num);
if (buffer == NULL)
return B_BAD_VALUE;
if (buffer != data && numBlocks == 1)
memcpy(buffer,data,blockSize);
return B_OK;
}
void *
get_block(BFile *file, off_t num, int blockSize)
static void*
get_block(void* cache, off_t num)
{
//printf("get_block(num = %Ld,blockSize = %ld)\n",num,blockSize);
if (file == NULL)
return NULL;
//TRACE(("get_block(num = %" B_PRIdOFF ")\n", num);
if (num >= gBlocks.CountItems())
readBlocks(file,num,blockSize);
read_blocks(cache, num);
return gBlocks.ItemAt(num);
}
int
release_block(BFile *file, off_t num)
static void
release_block(void* cache, off_t num)
{
//printf("release_block(num = %Ld)\n",num);
return 0;
//TRACE(("release_block(num = %" B_PRIdOFF ")\n", num));
}
// #pragma mark - Block Cache API
const void*
block_cache_get(void* _cache, off_t blockNumber)
{
TRACE(("block_cache_get(block = %" B_PRIdOFF ")\n", blockNumber));
return get_block(_cache, blockNumber);
}
status_t
block_cache_make_writable(void* _cache, off_t blockNumber, int32 transaction)
{
TRACE(("block_cache_make_writable(block = %" B_PRIdOFF ", transaction = %"
B_PRId32 ")\n", blockNumber, transaction));
// We're always writable...
return B_OK;
}
void*
block_cache_get_writable(void* _cache, off_t blockNumber, int32 transaction)
{
TRACE(("block_cache_get_writable(block = %" B_PRIdOFF
", transaction = %" B_PRId32 ")\n", blockNumber, transaction));
return get_block(_cache, blockNumber);
}
status_t
block_cache_set_dirty(void* _cache, off_t blockNumber, bool dirty,
int32 transaction)
{
TRACE(("block_cache_set_dirty(block = %" B_PRIdOFF
", dirty = %s, transaction = %" B_PRId32 ")\n", blockNumber,
dirty ? "yes" : "no", transaction));
if (dirty)
debugger("setting to dirty not implemented\n");
return B_OK;
}
void
block_cache_put(void* _cache, off_t blockNumber)
{
TRACE(("block_cache_put(block = %" B_PRIdOFF ")\n", blockNumber));
release_block(_cache, blockNumber);
}

View File

@ -12,11 +12,24 @@
class BFile;
extern void init_cache(BFile *file, int32 blockSize);
extern void shutdown_cache(BFile *file, int32 blockSize);
extern void init_cache(BFile* file, int32 blockSize);
extern void shutdown_cache(BFile* file, int32 blockSize);
extern int cached_write(BFile *file, off_t bnum, const void *data,off_t num_blocks, int bsize);
extern void *get_block(BFile *file, off_t bnum, int bsize);
extern int release_block(BFile *file, off_t bnum);
extern status_t cached_write(void* cache, off_t num, const void* _data,
off_t numBlocks);
// Block Cache API
extern const void* block_cache_get(void* _cache, off_t blockNumber);
extern status_t block_cache_make_writable(void* _cache, off_t blockNumber,
int32 transaction);
extern void* block_cache_get_writable(void* _cache, off_t blockNumber,
int32 transaction);
extern status_t block_cache_set_dirty(void* _cache, off_t blockNumber,
bool dirty, int32 transaction);
extern void block_cache_put(void* _cache, off_t blockNumber);
#endif /* CACHE_H */

View File

@ -0,0 +1,15 @@
#include "fs_interface.h"
status_t
acquire_vnode(fs_volume* volume, ino_t vnodeID)
{
return B_OK;
}
status_t
put_vnode(fs_volume* volume, ino_t vnodeID)
{
return B_OK;
}

View File

@ -134,7 +134,7 @@ dumpKey(void *key, int32 length)
void
dumpKeys()
{
char *type;
const char *type;
switch (gType) {
case S_STR_INDEX:
type = "string";
@ -157,6 +157,9 @@ dumpKeys()
case S_DOUBLE_INDEX:
type = "double";
break;
default:
debugger("unknown type in gType");
return;
}
printf("Dumping %ld keys of type %s\n",gNum,type);
@ -373,6 +376,7 @@ checkTreeContents(BPlusTree *tree)
printf("Key ");
dumpKey(gKeys[i].data,gKeys[i].length);
printf(" found only %ld from %ld\n",gKeys[i].count,gKeys[i].in);
bailOut();
}
}
}
@ -381,16 +385,18 @@ checkTreeContents(BPlusTree *tree)
void
checkTreeIntegrity(BPlusTree *tree)
{
// simple test, just seeks down to every key - if it couldn't
// be found, something must be wrong
// simple test, just seeks down to every key - if it's in and couldn't
// be found or it's not in and can be found, something must be wrong
TreeIterator iterator(tree);
for (int32 i = 0;i < gNum;i++) {
if (gKeys[i].in == 0)
continue;
status_t status = iterator.Find((uint8 *)gKeys[i].data,gKeys[i].length);
if (status != B_OK) {
if (gKeys[i].in == 0) {
if (status == B_OK) {
printf("found key %" B_PRId32 " even though it's not in!\n", i);
bailOutWithKey(gKeys[i].data, gKeys[i].length);
}
} else if (status != B_OK) {
printf("TreeIterator::Find() returned: %s\n",strerror(status));
bailOutWithKey(gKeys[i].data,gKeys[i].length);
}
@ -406,6 +412,13 @@ checkTree(BPlusTree *tree)
checkTreeContents(tree);
checkTreeIntegrity(tree);
bool errorsFound = false;
tree->Validate(false, errorsFound);
if (errorsFound) {
printf("BPlusTree::Validate() found errors\n");
bailOut();
}
}
@ -416,7 +429,7 @@ checkTree(BPlusTree *tree)
void
addAllKeys(Transaction *transaction, BPlusTree *tree)
addAllKeys(Transaction &transaction, BPlusTree *tree)
{
printf("*** Adding all keys to the tree...\n");
for (int32 i = 0;i < gNum;i++) {
@ -437,7 +450,7 @@ addAllKeys(Transaction *transaction, BPlusTree *tree)
void
removeAllKeys(Transaction *transaction, BPlusTree *tree)
removeAllKeys(Transaction &transaction, BPlusTree *tree)
{
printf("*** Removing all keys from the tree...\n");
for (int32 i = 0;i < gNum;i++) {
@ -462,7 +475,7 @@ removeAllKeys(Transaction *transaction, BPlusTree *tree)
void
duplicateTest(Transaction *transaction,BPlusTree *tree)
duplicateTest(Transaction &transaction,BPlusTree *tree)
{
int32 index = int32(1.0 * gNum * rand() / RAND_MAX);
if (index == gNum)
@ -481,14 +494,15 @@ duplicateTest(Transaction *transaction,BPlusTree *tree)
printf("* insert %ld to %ld old entries...\n",insertCount,insertTotal + gKeys[index].in);
for (int32 j = 0;j < insertCount;j++) {
status = tree->Insert(transaction,(uint8 *)gKeys[index].data,gKeys[index].length,insertTotal);
status = tree->Insert(transaction,(uint8 *)gKeys[index].data,gKeys[index].length,gKeys[index].value);
if (status < B_OK) {
printf("BPlusTree::Insert() returned: %s\n",strerror(status));
bailOutWithKey(gKeys[index].data,gKeys[index].length);
}
insertTotal++;
gTreeCount++;
gKeys[index].in++;
if (gExcessive)
checkTree(tree);
}
@ -505,26 +519,27 @@ duplicateTest(Transaction *transaction,BPlusTree *tree)
printf("* remove %ld from %ld entries...\n",count,insertTotal + gKeys[index].in);
for (int32 j = 0;j < count;j++) {
status_t status = tree->Remove(transaction,(uint8 *)gKeys[index].data,gKeys[index].length,insertTotal - 1);
status_t status = tree->Remove(transaction,(uint8 *)gKeys[index].data,gKeys[index].length,gKeys[index].value);
if (status < B_OK) {
printf("BPlusTree::Remove() returned: %s\n",strerror(status));
bailOutWithKey(gKeys[index].data,gKeys[index].length);
}
insertTotal--;
gTreeCount--;
gKeys[index].in--;
if (gExcessive)
checkTree(tree);
}
}
if (!gExcessive)
if (gExcessive)
checkTree(tree);
}
void
addRandomSet(Transaction *transaction,BPlusTree *tree,int32 num)
addRandomSet(Transaction &transaction,BPlusTree *tree,int32 num)
{
printf("*** Add random set to tree (%ld to %ld old entries)...\n",num,gTreeCount);
@ -553,7 +568,7 @@ addRandomSet(Transaction *transaction,BPlusTree *tree,int32 num)
void
removeRandomSet(Transaction *transaction,BPlusTree *tree,int32 num)
removeRandomSet(Transaction &transaction,BPlusTree *tree,int32 num)
{
printf("*** Remove random set from tree (%ld from %ld entries)...\n",num,gTreeCount);
@ -708,6 +723,7 @@ main(int argc,char **argv)
srand(gSeed);
Inode inode("tree.data",gType | S_ALLOW_DUPS);
rw_lock_write_lock(&inode.Lock());
gVolume = inode.GetVolume();
Transaction transaction(gVolume,0);
@ -717,7 +733,7 @@ main(int argc,char **argv)
// Create the tree, the keys, and add all keys to the tree initially
//
BPlusTree tree(&transaction,&inode);
BPlusTree tree(transaction,&inode);
status_t status;
if ((status = tree.InitCheck()) < B_OK) {
fprintf(stderr,"creating tree failed: %s\n",strerror(status));
@ -733,7 +749,7 @@ main(int argc,char **argv)
dumpKeys();
for (int32 j = 0; j < gHard; j++ ) {
addAllKeys(&transaction, &tree);
addAllKeys(transaction, &tree);
//
// Run the tests (they will exit the app, if an error occurs)
@ -742,14 +758,16 @@ main(int argc,char **argv)
for (int32 i = 0;i < gIterations;i++) {
printf("---------- Test iteration %ld ---------------------------------\n",i+1);
addRandomSet(&transaction,&tree,int32(1.0 * gNum * rand() / RAND_MAX));
removeRandomSet(&transaction,&tree,int32(1.0 * gNum * rand() / RAND_MAX));
duplicateTest(&transaction,&tree);
addRandomSet(transaction,&tree,int32(1.0 * gNum * rand() / RAND_MAX));
removeRandomSet(transaction,&tree,int32(1.0 * gNum * rand() / RAND_MAX));
duplicateTest(transaction,&tree);
}
removeAllKeys(&transaction, &tree);
removeAllKeys(transaction, &tree);
}
transaction.Done();
// of course, we would have to free all our memory in a real application here...
// write the cache back to the tree